diff --git a/.coveragerc b/.coveragerc index 5d068fcb..9cda9250 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,9 +2,6 @@ [run] branch = True source = flexmeasures_client -omit = - src/flexmeasures_client/s2/* - */site-packages/flexmeasures_client/s2/* [paths] source = diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb624446..188fa1bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,7 +104,19 @@ jobs: - name: Install dependencies run: uv sync --frozen --extra s2 --group test - name: Run S2 tests - run: uv run pytest tests/s2 -rFEx --durations 10 --color yes + run: uv run pytest tests/s2 -rFEx --durations 10 --color yes --cov + - name: Show coverage report + run: uv run coverage report + - name: Generate coverage report + run: uv run coverage lcov -o coverage.lcov + - name: Upload partial coverage report + uses: coverallsapp/github-action@v2 + with: + fail-on-error: false + path-to-lcov: coverage.lcov + github-token: ${{ secrets.GITHUB_TOKEN }} + flag-name: s2 - py${{ matrix.python }} + parallel: true finalize: needs: [ test, test-s2] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 09dd0ef1..99b3eef6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -53,7 +53,8 @@ repos: name: mypy (static typing) pass_filenames: false language: system - entry: uv run poe type-check + # With the s2 extra, so that the S2 message types are there to check against. + entry: uv run --extra s2 poe type-check verbose: true - repo: local @@ -66,3 +67,14 @@ repos: Put screenshots in the FlexMeasures/screenshots repo, or leave them untracked and attach them to the PR/issue manually. files: '\.(png|jpe?g|gif|bmp|tiff?|webp|ico|psd)$' + - id: no-print-in-unattended-code + name: No print() in code that runs unattended + # A CEM's handlers run for days without anyone watching standard output, + # so what they have to say goes to the log, where a deployment keeps it. + # The example scripts print on purpose, and so do the client's interactive confirmations, which live outside s2/. + # Matched anywhere on a line, not only at its start, and with a space before the parenthesis too, + # so that `if x: print(...)`, `return print(...)`, `print (...)`, pprint and writes to stdout or stderr are caught. + language: pygrep + entry: '\b(print|pprint)\s*\(|sys\.std(out|err)\.write' + files: '^src/flexmeasures_client/s2/.*\.py$' + exclude: '^src/flexmeasures_client/s2/script/' diff --git a/CHANGELOG-S2.rst b/CHANGELOG-S2.rst index 502eea76..cce3da96 100644 --- a/CHANGELOG-S2.rst +++ b/CHANGELOG-S2.rst @@ -17,15 +17,26 @@ Added ----- * ``FRBCSimple`` takes an optional ``soc_minima_sensor_id``, ``soc_maxima_sensor_id``, ``usage_forecast_sensor_id`` and ``leakage_behaviour_sensor_id``, and records on them what a resource manager declares about its storage: the fill levels it asks to be kept between, the usage it expects, and what its storage leaks. Without a sensor, that declaration is not recorded, rather than being posted somewhere else [`#244 `__] -* ``FRBCSimple`` takes an ``energy_unit``, which a state of charge and the fill-level minima and maxima around it are all recorded in [`#244 `__] +* ``FRBCSimple`` takes an ``energy_unit``, which a state of charge and the fill-level minima and maxima around it are all recorded in. It is one of ``J``, ``kJ``, ``MJ``, ``Wh``, ``kWh`` and ``MWh``, and is refused when the handler is made otherwise. A state of charge goes into a flex model in ``kWh`` or ``MWh``, converted from any other of these, since FlexMeasures takes no other [`#244 `__, `#260 `__] * ``FRBCSimple`` takes a ``hard_bound_factor``, defaulting to 1.5, which sets how far above the declared fill-level range the hard upper bound of a schedule sits. The bound is widened further where the state of charge a schedule starts from is above it, and a factor below 1 is refused, since FlexMeasures clips the band into the hard bounds and a bound inside the declared range would narrow the band to something the resource manager never asked for [`#245 `__] -* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. Where the modes state different efficiencies, the lowest goes out, since FlexMeasures takes one for the whole device, and the choice is logged [`#253 `__] +* ``FRBCSimple`` takes an optional ``charging_efficiency_sensor_id``, records on it what the actuator's operation modes say about converting drawn power into stored energy, and asks for a schedule against it as ``charging-efficiency``. The ratio between a mode's fill rate and the power it draws is a heat pump's coefficient of performance, and without it the scheduler plans the storage as if an actuator converted power into stored energy one for one, so a plan reaches a fill-level target at a power the actuator cannot deliver it at. FlexMeasures takes one for the whole device, so where a mode converts differently at different fill levels, the element that applies where the schedule starts answers, and where the modes differ, the lowest goes out, and the choice is logged. A mode that draws power while the storage does not fill, such as a standby or circulation mode, states none, rather than an efficiency of zero that would tell the scheduler no power could fill the storage at all [`#253 `__], [`#256 `__] * ``fill_rate_in_watts()`` and ``operation_mode_conversion_efficiency()``, which state a fill rate in the watts S2 states a power range in, and take the ratio between the two. A fill rate is per second of whatever unit a fill level is recorded in, so a kilowatt hour per second is 3.6 million watts, and an energy unit neither knows is refused rather than converted by a factor that is wrong by thousands [`#253 `__] * ``FRBCSimple`` takes ``schedule_attempts``, defaulting to five, and ``schedule_retry_interval``, defaulting to two seconds, which bound how often a schedule is asked for again and how long between two attempts. One attempt is what the handler did before, where any failure cost the storage its instruction for that window [`#254 `__] +* ``FRBCSimple`` takes an optional ``production_price_sensor_id``, for a site where feeding back is priced differently from drawing, as under a feed-in tariff. Without it, both directions are priced by ``price_sensor_id``, as before [`#261 `__] +* ``FRBCSimple`` takes ``relax_site_capacity``: ``True`` lets a schedule breach the site's connection capacity at a price, for a site whose capacity is a target set by others, and ``False`` keeps it a hard limit. Unset, it is left to FlexMeasures, which since v1.0 relaxes it by default, so a home whose connection is a physical limit has to pass ``False`` [`#261 `__] +* An FRBC handler logs the instructions of a batch once, at debug level, by operation-mode name [`#261 `__] +* ``flexmeasures_client.s2.clock``, with a ``WallClock`` and a ``SimulatedClock``, and ``CEM(clock=...)``: the time a CEM and every handler it registers go by, timers and the moment a state of charge is known as of included [`#264 `__, `#268 `__] +* ``CEM(accept_simulation_metadata=True)``, which takes a co-simulation's messages wrapped with the simulated time (``S2Wrapper``) and sets its clock from them. Off by default, a wrapped message is refused as invalid. The example server turns it on with ``S2_ACCEPT_SIMULATION_METADATA`` [`#264 `__] +* ``flexmeasures_client.s2.site``: ``FRBC_SITE_SENSORS``, ``ensure_site`` and ``FRBCSiteConfigurator``, which gives each resource manager a FlexMeasures site of its own, named after it, and an FRBC handler wired to it [`#265 `__] +* ``CEM(site_configurator=...)``, which sets a resource manager's site up once its details arrive, answering its messages with a temporary error until then. The example server does so with ``S2_SITE_PER_RESOURCE_MANAGER`` [`#265 `__] +* ``examples/S2-co-simulation``, a FlexMeasures stack with one CEM per simulated home, for a co-simulation whose resource managers connect over websockets [`#265 `__, `#267 `__] +* ``flexmeasures_client.s2.measurements.MeasurementRecorder`` and ``CEM(measurement_recorder=...)``: where a resource manager's measured power is recorded, one per server so that a reconnecting resource manager is not recorded twice [`#266 `__] Changed ------- +* ``FRBCSimple`` plans one schedule at a time, for the newest request: a storage status arriving while a schedule is planned waits for it, and is dropped where a newer one arrives meanwhile, and a request out of date makes no further attempt after a failed one. Each status used to start a solve of its own beside the one in flight, so the newest is now planned after the attempt in flight rather than beside it [`#269 `__] +* An FRBC handler no longer turns a schedule into instructions where a newer storage status arrived while it was being planned, since sending would drop them anyway [`#261 `__] * ``FRBC.trigger_schedule()`` takes the number of the request it answers as a keyword argument, so a subclass that overrides it has to accept that argument [`#237 `__] * ``FRBC.send_storage_status()`` likewise takes the number of the planning request it answers, since a storage status is now what asks for a schedule [`#240 `__] * ``FRBCSimple`` takes a ``resolution``, defaulting to fifteen minutes, which a schedule's start is lined up with [`#240 `__] @@ -34,14 +45,40 @@ Changed * ``FRBCSimple`` asks for a schedule against what the resource manager declared about its storage: the fill levels it wants to be kept between as ``soc-minima`` and ``soc-maxima``, the usage it expects as ``soc-usage``, and what its storage leaks as ``storage-efficiency``, each only where a sensor holds it. Recording those declarations changed nothing about the schedule that came back until the flex model named them [`#245 `__] * ``fm_schedule_to_instructions()`` takes the fill levels its own running fill level is held between, so a caller that asked for a schedule against other bounds than the declared fill-level range is answered with the operation modes for the storage states the scheduler actually planned. Without them the declared range is used, as before [`#245 `__] * ``FRBCSimple`` asks for the declared fill-level range as a band whose breach is priced, with ``relax-soc-constraints`` in the flex context, rather than as the hard ``soc-min`` and ``soc-max``, which become wide bounds at zero and half again above the range, each widened to take in a state of charge beyond them. A hard bound at the declared range made states a storage really reaches infeasible, such as free-floating above the top of its range or being left below the bottom of it by a previous schedule, and an infeasible job is answered with no schedule at all, or, against a FlexMeasures older than v1.0.0, by a fallback scheduler that ignores the band [`#245 `__] -* ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A deployment whose state-of-charge sensor is in megawatt hours has to pass ``energy_unit="MWh"`` to keep its readings a thousand times smaller than they would otherwise become [`#244 `__] +* ``FRBCSimple`` posts a state of charge in ``kWh`` rather than ``MWh``, since S2 is a residential protocol and the example server's own state-of-charge sensor is in kilowatt hours. A state-of-charge sensor in another energy unit still holds the right values, since FlexMeasures converts what is posted into the sensor's own unit [`#244 `__] * ``FRBCSimple`` asks for a schedule again where an attempt failed in a way another attempt could answer differently. A scheduling job can fail on a transient of its own, from a worker crashing out of its solver to a pandas error over the windows two schedules overlap in, and FlexMeasures refuses a trigger outright where the state of charge it would plan from has not landed yet. Unanswered, such a failure leaves the resource manager polling for an instruction that never comes, since asking for a schedule runs as a task whose exception is logged and dropped. A cause that fails identically on every attempt is never asked about again: a scheduler colliding with its own belief, or an infeasible problem [`#254 `__] * ``FRBCSimple`` says so, as an error, where a schedule was planned by a fallback scheduler rather than the one it asked for. FlexMeasures answered an infeasible problem that way until v1.0.0 retired it, and an older server either serves that fallback job in place of the failed one or redirects to it, which this client follows, so either way the substitute arrives looking like the schedule that was asked for. It is a coarse charging policy rather than a plan, and it records no state of charge for a later window to be planned from [`#254 `__] +* The example resource manager is a heat pump keeping a hot-water buffer warm: it modulates from 400 to 1500 W at a COP of 4 while the buffer is cold and 3 once it is warm, declares its leakage, a day of hot-water use and a comfort profile, and simulates its buffer from S2's own definitions, following each instruction when it is due and reporting its fill level every ``S2_EXAMPLE_STATUS_INTERVAL`` seconds. It follows only the latest plan, honouring the revocations the example server now sends, and tells the CEM where each instruction stands, so that only what it has not yet carried out is revoked, and sends its leakage, usage forecast and comfort profile again every ``S2_EXAMPLE_FORECAST_INTERVAL`` seconds. The example server records each declaration on a sensor of its own, plans from now, seeds its demo prices again every 12 hours, and sets the site's graphs to show the buffer's fill level within its comfort band, and what fills and draws it [`#259 `__, `#262 `__] + +Removed +------- + +* ``flexmeasures_client.s2.cem.get_commodity_unit``, which stated power in kW where S2 states it in W [`#266 `__] +* ``op_mode_compute_factor``, which worked a factor back out of a fill rate and returned zero wherever an element's fill rate is one value; the factor now comes with the operation mode chosen, from its power range [`#258 `__] Fixed ----- -* ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind [`#252 `__] +* A CEM records measured power by the moment it was measured, as one average per step of its sensor's resolution (15 minutes on the example's sites, where it was five), once the step is over by its clock, with the first measurement after that, and in watts, as S2 states it. It binned measurements by its own now, which dropped every measurement a resource manager sent after the fact, as a co-simulation's does, labelled watts as kilowatts, and kept its samples in a buffer shared by every CEM in the process. Recording happens in the background, retried a few times where it fails [`#266 `__] +* A CEM without a sensor for a resource manager's measured power no longer records it on sensor 357, which was whichever sensor of whichever site on the server had that id, and answered it as recorded [`#265 `__] +* A message of the control type a CEM is selecting, arriving before the resource manager confirmed the selection, is taken as that confirmation, since a resource manager sends a control type's messages only once it works in it. A CEM used to stay without a control type, answering each such message with a temporary error that the resource manager retried, as a co-simulation's reconnecting resource managers showed. A message of any other control type activates nothing [`#263 `__] +* Closing a control-type handler cancels the work it still has in flight, recording included, and the example server's session ends with its connection. A resource manager that reconnected used to be served by two sessions planning for it at once [`#263 `__] +* A leakage behaviour is recorded as a storage efficiency each time a schedule is asked for, over the window the schedule covers and as a share of the fill level it starts from, or of the middle of the storage's range where it starts below that, and named in the flex model only where it was recorded. It was recorded once, on arrival, over the hours after the message and as a share of the top of the range, so planning went on without it once those hours had passed, and understated the loss at lower fill levels. FlexMeasures takes a storage efficiency as a share of the stock where S2 states a rate, so the loss is exact only at that one fill level [`#259 `__] +* A resolution boundary is found on the clock of the moment being floored, in the offset it carries, as FlexMeasures floors a series posted with it, and in the repeated hour of an autumn DST change it is the later boundary not after the moment, where it used to be taken for the first pass, an hour or more earlier, so a schedule could start in the past [`#259 `__] +* On a state-of-charge sensor that records instants, a state of charge is recorded as a reading at an instant, where it was recorded over the minute after it, which FlexMeasures took for a forecast. On a sensor with a resolution, which refuses an instant, it is still recorded over a minute [`#259 `__] +* ``FillRateBasedControlTUNES`` posts its usage forecast per hour, which is what a value labelled MW means to FlexMeasures, where it posted a quarter-hour amount that FlexMeasures planned as a quarter of the real draw. It reads it back in the resource manager's own units, per second, dividing out the fill-level scale it was posted with, and plans its instructions with its latest leakage behaviour [`#258 `__] +* ``FRBCSimple`` refuses a ``resolution`` or ``schedule_duration`` that is not positive, and a ``schedule_duration`` that is not a whole number of ``resolution`` steps, since FlexMeasures answers with whole steps only, and such a schedule would be read at the wrong step [`#258 `__] +* Translating a schedule into instructions takes every rate per second, as S2 states it. It advanced its running fill level by a fill rate per second times a step in hours, so the level moved 3600 times too little, and every step of a schedule was answered with the operation mode for the fill level the schedule started from. Leakage now lowers that level, from the element of the latest ``FRBC.LeakageBehaviour`` that applies, where it used to be read from two fields S2 does not have and so never counted, and so does usage, from the latest ``FRBC.UsageForecast``. Each step is followed exactly, as a resource manager follows an instruction: a fill level crossing into another element, or a change in usage within a step, is taken at the moment it happens, and a level that reaches a boundary its far side would push it back from stays at that boundary [`#258 `__] +* A schedule is read at the resolution FlexMeasures answers in, its duration over its number of values, rather than as quarter hours whatever it was, which made a schedule for an hourly or a five-minute power sensor raise, so that no instruction went out [`#258 `__] +* A new instruction goes out wherever the operation mode or its factor changes, including at a constant power where the fill level passes into another element of the mode, rather than only where the power changes. A difference in the factor of a millionth or less, which is the noise of converting a schedule from MW to W, is not a new instruction [`#258 `__] +* An instruction asks for the factor that draws the power the schedule asks for, taken from the operation mode's power range. It was worked back out of the fill rate, which asked for the bottom of the power range wherever the fill rate is one value and the power a range [`#258 `__] +* A leakage behaviour is recorded as a storage efficiency compounded per second, as S2 states the rate, where it was compounded per hour, which declared a buffer losing about 7% of its heat over twelve hours as holding it to within 0.002% [`#258 `__] +* An actuator's power and fill rate are recorded as covering one step of the handler's ``resolution``, rather than a quarter hour whatever it plans on [`#258 `__] +* ``FillRateBasedControlTUNES`` plans, and records its conversion efficiencies, when an operation mode has no diagnostic label, which S2 makes optional. It looked for ``THP`` or ``NES`` inside a label that may be absent, so one unlabelled mode stopped planning, and the recording of the efficiencies of every mode listed after it [`#257 `__] +* ``get_validation_error_summary`` is removed: it iterated a pydantic ``ValidationError``, which is not iterable, so it raised on any error it was given, and nothing called it [`#257 `__] +* A leakage behaviour is recorded as a storage efficiency at the rate of the element that applies at the latest fill level, or of its leakiest element where no fill level is known yet, where it took the last one listed, which S2 does not require to be the top one. Translating a schedule into instructions likewise no longer depends on the order a mode lists its elements in [`#256 `__] +* ``FillRateBasedControlTUNES`` records an ``FRBC.LeakageBehaviour`` instead of answering it with ``PERMANENT_ERROR``. It read its fill-level scale from a misspelt attribute, and the error that raised was taken for a failed post, so no leakage behaviour was ever recorded [`#255 `__] +* ``FRBCSimple`` records a second actuator status for the same moment as superseding the first, rather than colliding with it. FlexMeasures keys a belief on its sensor, its source, the event and the moment it was formed, and two statuses inside one resolution step milliseconds apart were refused with a duplicate key whose belief time had lost its sub-second part, losing the later one. A second apart is unambiguous, and the later status supersedes the earlier, which is what a resource manager reporting twice in a second means. A resource manager sends a status for each instruction it applies, and this handler sends several at once, so the power that survived was the one the actuator had already left behind. Statuses about different steps may interleave, so the spacing is kept per step [`#252 `__, `#259 `__] * A measurement that cannot be recorded is logged with the value, the unit, the sensor and the moment it was for, rather than disappearing, whether FlexMeasures refused it, the server could not be reached, or the unit it would have been posted in is one nothing converts into. The post runs in a task nobody awaits, so until now a refused measurement left the power sensor holding a schedule and nothing else, with the reason visible only as a dead task in the log [`#252 `__] * ``FRBCSimple`` converts an actuator's power into the unit it posts in, rather than labelling a value S2 states in watts with whatever ``power_unit`` says. A deployment recording its power sensor in ``kW`` had 883.7 W posted as 883.7 kW, which FlexMeasures stored as the thousandfold it claimed to be; the default of ``W`` needed no conversion and was unaffected [`#252 `__] * The example server creates its power sensor with ``consumption_is_positive``, as S2 states power, so that an actuator's measured power and the schedule planned for it point the same way on the sensor that holds both [`#252 `__] @@ -56,14 +93,14 @@ Fixed * An ``FRBCSimple`` handler accepts an ``FRBC.FillLevelTargetProfile``, an ``FRBC.UsageForecast`` and an ``FRBC.LeakageBehaviour`` instead of raising ``NotImplementedError`` for each. The CEM dispatches these as tasks nobody awaits, so every connection by a resource manager that sends all three logged three "Task exception was never retrieved" tracebacks, and recorded none of what they declared [`#244 `__] * A fill-level target profile is held inside the fill-level range the latest system description declared, so a target reaching past either end of it, such as a night setback below the range, is recorded as the range allows rather than as asked [`#244 `__] * The example CEM server answers an S2 handshake at once, instead of after configuring its site with FlexMeasures, which took longer than the timeout a resource manager waits for that answer [`#241 `__] -* The example CEM server seeds its demo prices from the current day, instead of from a date hardcoded when it was written, so that the window it schedules for has prices [`#241 `__] -* An FRBC handler asks for a schedule when a storage status reports a new state of charge, instead of when a system description arrives, so a resource manager that sends its description before its first status is planned for at all [`#240 `__] -* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window [`#240 `__] +* The example CEM server seeds its demo prices from the current day, instead of from a date hardcoded when it was written, and the example resource manager dates its system description now, so that the window it schedules for has prices. Before both, the two had agreed only by both being hardcoded to the same day [`#241 `__, `#259 `__] +* An FRBC handler asks for a schedule when a storage status reports a new state of charge, and when a system description arrives after the latest status and differs from the one before it, its validity included, so a resource manager is planned for whichever of the two it sends first, and whether or not its fill level moves [`#240 `__, `#259 `__] +* A schedule is asked for from a resolution boundary, instead of from whichever moment a resource manager happened to send its system description, which left FlexMeasures without prices for the window, and from the next resolution boundary rather than the start of the step under way, which a description days old made it slide into the past, and which, planned from the fill level reported mid-step, counted on the whole step where only part was left, so each replan within it asked for less power than the step needed [`#240 `__, `#259 `__] * An FRBC handler ignores a re-sent system description whose content is unchanged, instead of storing it again [`#240 `__] * An FRBC handler answers only the newest schedule request, instead of letting every request in a backlog complete its own round trip and send its own instruction batch [`#237 `__] * An FRBC handler sends an instruction only where it changes what the actuator does, which holds at the sending boundary for every producer of instructions [`#237 `__] * The instruction-status history is keyed by a string, as its type declares, instead of by the ``UUID`` that a no-op ``cast`` let through, which made every lookup miss [`#237 `__] -* ``FRBCSimple`` reports the power of the operation mode the actuator is actually in, taken from the mode's power range, and posts it to the power sensor [`#236 `__] +* ``FRBCSimple`` reports the power of the operation mode the actuator is actually in, taken from the power range of the element that applies at the latest fill level, and posts it to the power sensor [`#236 `__, `#256 `__] * A control-type handler can log before it is registered, and one without its own ``close()`` no longer breaks CEM teardown [`#236 `__] Added diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e030e46c..4d0d1f34 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -27,8 +27,14 @@ Added Changed ------- +* ``add_asset`` no longer requires a ``latitude`` and ``longitude``, which FlexMeasures does not require either, for an asset with no location, such as one standing for a community. ``generic_asset_type_id`` stays required, and half a location is refused [`#261 `__] * ``post_sensor_data``, and the deprecated ``post_measurements`` it was renamed from, wait for a server of FlexMeasures 0.33.0 or above that ingests the posted data asynchronously, answering ``202 Accepted`` with a job id, to finish doing so, so that a caller can read the data back or schedule over it; a failed job raises ``JobFailedError``, while a job still pending at the timeout, or one that cannot be looked up, is logged as an error and leaves the post successful. Pass ``await_ingestion=False`` to return as soon as the data is accepted [`#242 `__] +Removed +------- + +* ``response_handling.check_content_type``, which refused an answer in a JSON variant that aiohttp parses, such as ``application/problem+json``, with an error that raised an ``AttributeError`` of its own when printed. Such an answer is now read like any other. A content type that is not JSON at all is still reported as a ``ConnectionError``, by aiohttp's own check [`#257 `__] + Fixed ----- diff --git a/examples/S2-co-simulation/README.md b/examples/S2-co-simulation/README.md new file mode 100644 index 00000000..6e95fa06 --- /dev/null +++ b/examples/S2-co-simulation/README.md @@ -0,0 +1,71 @@ +# A FlexMeasures stack for an S2 co-simulation + +`docker-compose.cem.yml` runs FlexMeasures with one S2 CEM per simulated home, +for a co-simulation whose resource managers connect over websockets, such as a +community of homes steered at once. How to start it is at the top of the file. + +Each CEM is the example server (`src/flexmeasures_client/s2/script/websockets_server.py`) +with two switches on: + +- `S2_ACCEPT_SIMULATION_METADATA`: a resource manager wraps each message with the + simulated time, `{"message": ..., "metadata": {"dt": ...}}`, and the CEM, and every + handler it registers, go by that time (see `flexmeasures_client.s2.clock`). + `dt` has to name its time zone. Without the switch, a wrapped message is refused + as invalid, from the handshake on. +- `S2_SITE_PER_RESOURCE_MANAGER`: each resource manager gets a site of its own, + named after it, with the sensors in `flexmeasures_client.s2.site.FRBC_SITE_SENSORS`. + Messages are answered with a temporary error until its site is set up, which a + resource manager retries. Its schedules start from the simulated now and cover + 24 hours. The site's prices are for whoever runs the simulation to post. + +FlexMeasures reads three settings a co-simulation needs from its instance config +(`flexmeasures-instance/flexmeasures.cfg` in the flexmeasures checkout), not from the environment: + +```python +# A steering round replans a window at the same simulated moment, and that plan replaces the one before. +# Without this, the scheduler cannot save the second plan, and the home gets no instructions for its window. +FLEXMEASURES_ALLOW_DATA_OVERWRITE = True +# Every home triggers a schedule each round, all from one account, far above the default of 10 per 5 minutes, +# and every trigger past the limit waits minutes before it is retried. +FLEXMEASURES_API_TRIGGER_RATE_LIMIT = "1000 per minute" +# A bound on each solve, since a tail of degenerate problems otherwise proves optimality past the job timeout. +FLEXMEASURES_LP_SOLVER_OPTIONS = {"time_limit": "15"} +``` + +A co-simulation reads the sites by sensor name: `price`, `production price`, +`power` (the schedule), `measured-power` (what the home measured) and +`state of charge`. + +The stack uses FlexMeasures' own development database service, `dev-db`, +whose data FlexMeasures' compose file keeps in `docker-compose-data/dev-db` +inside the flexmeasures checkout it runs from. +A compose project name of its own does not separate that. +To keep a development database you care about out of it, +run it from a folder of its own holding checkouts named `flexmeasures` and `flexmeasures-client`: +the overlay mounts `../flexmeasures` and `../flexmeasures-client`, so those names are what it runs. +Git worktrees will do, with one thing to add: FlexMeasures' image reads its version from git, which it cannot through a worktree. +Give it as the build argument `FLEXMEASURES_VERSION` to each service built from FlexMeasures' Dockerfile, in one more overlay: + +```yaml +x-version: &version + build: + context: . + dockerfile: Dockerfile + args: + FLEXMEASURES_VERSION: "1.1.0" + +services: + server: + <<: *version + worker: + <<: *version + # and likewise worker-2, worker-3, automation-runner and each cem +``` + +Its host ports, 5000, 5433, 6380, 1025, 8025 and 8080 to 8084, have to be free too; +where they are not, an extra overlay can move them with compose's `!override`. + +It runs FlexMeasures from the flexmeasures checkout, which is mounted over the code the image installed, +with this client installed into the image's virtualenv, which is what FlexMeasures and its plugins run from. +The UI's third-party libraries are fetched into the installed package when the image is built, so a UI served from the mounted checkout may lack them; the API does not need them. +The server trusts the name the CEMs reach it by, `server`, alongside the loopback names a development server trusts by default. diff --git a/examples/S2-co-simulation/docker-compose.cem.yml b/examples/S2-co-simulation/docker-compose.cem.yml new file mode 100644 index 00000000..9304b049 --- /dev/null +++ b/examples/S2-co-simulation/docker-compose.cem.yml @@ -0,0 +1,175 @@ +# ------------------------------------------------------------------ +# A FlexMeasures stack with one S2 CEM per simulated home, +# for a co-simulation whose resource managers connect over websockets, +# such as the community simulations that steer several homes at once. +# +# It is an overlay on FlexMeasures' own docker-compose.yml, +# and runs FlexMeasures from the flexmeasures checkout and the CEMs from this one, both mounted. +# With checkouts named flexmeasures and flexmeasures-client side by side, +# run it from the flexmeasures folder, which holds the Dockerfile: +# +# docker compose \ +# -f docker-compose.yml \ +# -f ../flexmeasures-client/examples/S2-co-simulation/docker-compose.cem.yml \ +# up +# +# Resource managers connect to ws://localhost:8080/ws, ws://localhost:8081/ws and so on, one per home. +# Each gets a site of its own, named after it, +# and the CEM goes by the simulated time its messages carry (see README.md). +# ------------------------------------------------------------------ + +# Shared definition for every CEM. +# Each listens on port 8080 inside its own container; only the host port differs. +x-cem: &cem + build: + context: . + dockerfile: Dockerfile + image: flexmeasures-client-cem + depends_on: + - server + restart: always + environment: + FLEXMEASURES_BASE_URL: http://server:5000 + FLEXMEASURES_USER: toy-user@flexmeasures.io + FLEXMEASURES_PASSWORD: toy-password + LOGGING_LEVEL: INFO + # A resource manager in a co-simulation wraps each message with the simulated time. + S2_ACCEPT_SIMULATION_METADATA: "1" + # And each home is a site of its own, rather than all of them sharing one. + S2_SITE_PER_RESOURCE_MANAGER: "1" + volumes: + - ../flexmeasures-client:/app/flexmeasures-client:rw + entrypoint: ["/bin/sh", "-c"] + command: + - | + # Install and run with the same interpreter (`python3 -m pip`, not bare `pip`): + # in images built from FlexMeasures' uv-based Dockerfile, `python3` is the /app/.venv virtualenv, + # which may come without pip, and cannot see what the system pip installs. + python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade + # s2-python as this repo locks it, and as the co-simulation's resource managers use it. + # The checkout may be mounted without git history to read its version from, as a git worktree is, + # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install --break-system-packages -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1" + exec python3 /app/flexmeasures-client/src/flexmeasures_client/s2/script/websockets_server.py + +# Shared definition for every RQ worker: one per home simulated at once, +# like the CEMs and the server's gunicorn workers below, +# since every home's scheduling jobs otherwise queue behind one worker process. +x-worker: &worker + # Self-contained, since worker-2 and worker-3 do not exist in the base file to merge with. + build: + context: . + dockerfile: Dockerfile + depends_on: + - dev-db + - queue-db + - mailhog + restart: on-failure + volumes: + - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:rw + - ../flexmeasures/flexmeasures:/app/flexmeasures:rw + - ./flexmeasures-instance/:/app/instance/:rw + - ../flexmeasures-client:/app/flexmeasures-client:rw + entrypoint: ["/bin/sh", "-c"] + environment: + SQLALCHEMY_DATABASE_URI: "postgresql://fm-dev-db-user:fm-dev-db-pass@dev-db:5432/fm-dev-db" + FLEXMEASURES_REDIS_URL: queue-db + FLEXMEASURES_REDIS_PASSWORD: fm-redis-pass + SECRET_KEY: notsecret + SECURITY_TOTP_SECRETS: '{"1": "something-secret"}' + FLEXMEASURES_ENV: development + MAIL_SERVER: mailhog + MAIL_PORT: 1025 + LOGGING_LEVEL: INFO + command: + - | + # Into the image's virtualenv, which FlexMeasures and its plugins run from, with that interpreter's own pip. + python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade + python3 -m pip install -r /usr/var/flexmeasures-instance/requirements.txt + # The checkout may be mounted without git history to read its version from, as a git worktree is, + # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" + # Drop any lingering registration for the ingestion queue before this worker starts. + # FlexMeasures ingests posted sensor data asynchronously when Redis lists a worker for that queue, + # and a dead worker's entry counts too: posts then wait on a queue nobody serves, and a run hangs without an error. + # No worker here serves that queue, so removing those entries on every start is safe, and idempotent. + python3 - <<'PYGUARD' + import os, redis + r = redis.Redis(host=os.environ.get("FLEXMEASURES_REDIS_URL", "queue-db"), + port=int(os.environ.get("FLEXMEASURES_REDIS_PORT", 6379)), + password=os.environ.get("FLEXMEASURES_REDIS_PASSWORD")) + try: + for name in r.smembers("rq:workers"): + key = name.decode() + queues = (r.hget(key, "queues") or b"").decode() + if "ingestion" in queues or not queues: + r.srem("rq:workers", name); r.delete(key) + r.delete("rq:workers:ingestion") + except redis.RedisError as exc: + print(f"ingestion-registration guard skipped: {exc}") + PYGUARD + # Named after the container, which is unique per worker. + # Every queue the base file's worker serves but ingestion, which is left to the server to do as it receives the data. + flexmeasures jobs run-worker --name "flexmeasures-worker-$$(hostname)" --queue scheduling\|forecasting\|reporting + +services: + dev-db: + ports: + - "5433:5432" + queue-db: + ports: + - "6380:6379" + server: + environment: + # The CEMs reach the server by its service name, which a development server does not trust by default. + TRUSTED_HOSTS: "localhost,.localhost,127.0.0.1,[::1],server" + volumes: + - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:rw + - ./flexmeasures-instance/:/app/instance/:rw + - ../flexmeasures/flexmeasures:/app/flexmeasures:rw + - ../flexmeasures-client:/app/flexmeasures-client:rw + command: + - | + # Into the image's virtualenv, which FlexMeasures and its plugins run from, with that interpreter's own pip. + python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade + python3 -m pip install -r /usr/var/flexmeasures-instance/requirements.txt + # The checkout may be mounted without git history to read its version from, as a git worktree is, + # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one. + SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]" + flexmeasures db upgrade + if ! flexmeasures show accounts | grep -q "Docker Toy Account"; then + flexmeasures add toy-account --name 'Docker Toy Account' + fi + # Worker processes rather than threads: a slow, CPU-bound request held its worker's GIL, + # and under several threads that starved the other requests into client-side timeouts. + # One worker per CEM below, since every home's CEM can be mid-request at once; + # scale the two together. + gunicorn --bind 0.0.0.0:5000 --worker-tmp-dir /dev/shm --workers 5 --threads 1 --timeout 120 wsgi:application + # One worker per home simulated at once; add or remove them along with the CEMs. + worker: + <<: *worker + worker-2: + <<: *worker + worker-3: + <<: *worker + # One CEM per simulated home; add one by copying an entry and taking the next host port. + cem: + <<: *cem + ports: + - "8080:8080" + cem-2: + <<: *cem + ports: + - "8081:8080" + cem-3: + <<: *cem + ports: + - "8082:8080" + cem-4: + <<: *cem + ports: + - "8083:8080" + cem-5: + <<: *cem + ports: + - "8084:8080" diff --git a/pyproject.toml b/pyproject.toml index 3407ad3d..251b1d99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,15 +85,17 @@ help = "Run the test suite (excluding S2) and generate coverage report." cmd = "pytest --ignore=tests/s2 --cov --cov-report=" [tool.poe.tasks.test-s2] -help = "Run S2 tests." -cmd = "pytest tests/s2" +help = "Run S2 tests and generate coverage report." +cmd = "pytest tests/s2 --cov --cov-report=" [tool.poe.tasks.type-check] -help = "Run mypy on files with type hints." -shell = """ - files=$(find src -name \\*.py | xargs grep -l "from typing import") - mypy --follow-imports skip --ignore-missing-imports $files -""" +help = "Run mypy on every source file, including the bodies of functions without type hints, against the types of what they import." +cmd = "mypy --ignore-missing-imports --check-untyped-defs src" + +[tool.mypy] +# The S2 messages are pydantic models, so without the plugin mypy misreads their constructors and fields. +# It also makes a missing s2 extra fail loudly, rather than leaving every S2 type silently Any. +plugins = ["pydantic.mypy"] [tool.poe.tasks.lint] help = "Run pre-commit hooks on all files." diff --git a/src/flexmeasures_client/client.py b/src/flexmeasures_client/client.py index c344d900..9252c477 100644 --- a/src/flexmeasures_client/client.py +++ b/src/flexmeasures_client/client.py @@ -34,11 +34,7 @@ WrongAPIVersionError, WrongHostError, ) -from flexmeasures_client.response_handling import ( - check_content_type, - check_for_status, - check_response, -) +from flexmeasures_client.response_handling import check_for_status, check_response from flexmeasures_client.utils import apply_deprecated_parameter_aliases, merge_updates LOGGER = logging.getLogger(__name__) @@ -463,8 +459,11 @@ def report_rate_limit(message: str) -> None: "Request retry timeout while connecting to the API." ) from exception - check_content_type(response) - + # request_once has already parsed this answer with aiohttp's json(), + # which refuses a content type that is not JSON, and the loop above reports that as a ConnectionError. + # A check that used to follow here refused the JSON variants aiohttp accepts too, such as application/problem+json, + # with an error that could not even be printed, so it is gone. + # A body that is not JSON under a JSON content type is refused by json() itself, as a JSONDecodeError. return await response.json(), response.status async def request_once( @@ -563,9 +562,10 @@ async def get_headers(self, include_auth: bool) -> dict: headers = dict(CONTENT_TYPE_HEADERS) | {CLIENT_VERSION_HEADER: client_version} if include_auth: - if self.access_token is None: - await self.get_access_token() - headers |= {"Authorization": self.access_token} + access_token = self.access_token + if access_token is None: + access_token = await self.get_access_token() + headers |= {"Authorization": access_token} return headers def build_url(self, uri: str, path: str = path) -> URL: @@ -577,8 +577,8 @@ def build_url(self, uri: str, path: str = path) -> URL: ) return url - async def get_access_token(self): - """Get access token and store it on the FlexMeasuresClient.""" + async def get_access_token(self) -> str: + """Get access token, store it on the FlexMeasuresClient, and return it.""" response, _status = await self.request( uri="requestAuthToken", path="/api/", @@ -588,7 +588,13 @@ async def get_access_token(self): }, include_auth=False, ) - self.access_token = response["auth_token"] + if not isinstance(response, dict): + raise ValueError( + f"Expected a JSON object from requestAuthToken, but got a {type(response).__name__}." + ) + access_token: str = response["auth_token"] + self.access_token = access_token + return access_token async def get_versions(self): """Get the FlexMeasures version and the supported API versions of the FlexMeasures server. @@ -601,6 +607,10 @@ async def get_versions(self): method="GET", include_auth=False, ) + if not isinstance(response, dict): + raise ValueError( + f"Expected a JSON object from the API root, but got a {type(response).__name__}." + ) # Check whether the API version used by the client is supported by the server server_api_versions = response["versions"] @@ -1501,9 +1511,9 @@ async def add_asset( self, name: str, account_id: int, - latitude: float, - longitude: float, - generic_asset_type_id: int, + latitude: float | None = None, + longitude: float | None = None, + generic_asset_type_id: int | None = None, parent_asset_id: int | None = None, sensors_to_show: list | None = None, flex_context: dict | None = None, @@ -1527,15 +1537,27 @@ async def add_asset( 'status': 200 } - This function raises a ValueError when an unhandled status code is returned. + This function raises a ValueError when an unhandled status code is returned, + when no generic_asset_type_id is given, and when only one of latitude and longitude is. """ - asset = dict( + if generic_asset_type_id is None: + # Optional in the signature only so that latitude and longitude, before it, can be left out, + # while every argument keeps its position. + raise ValueError("add_asset needs a generic_asset_type_id.") + if (latitude is None) != (longitude is None): + raise ValueError( + "add_asset takes a location as both a latitude and a longitude, or neither." + ) + asset: dict = dict( name=name, account_id=account_id, - latitude=latitude, - longitude=longitude, generic_asset_type_id=generic_asset_type_id, ) + # A location is optional to FlexMeasures, which many assets have no use for, such as one standing for a community. + if latitude is not None: + asset["latitude"] = latitude + if longitude is not None: + asset["longitude"] = longitude if parent_asset_id: asset["parent_asset_id"] = parent_asset_id if sensors_to_show: diff --git a/src/flexmeasures_client/response_handling.py b/src/flexmeasures_client/response_handling.py index 70d1c4d5..1f22ae3f 100644 --- a/src/flexmeasures_client/response_handling.py +++ b/src/flexmeasures_client/response_handling.py @@ -8,11 +8,8 @@ from email.utils import parsedate_to_datetime from typing import TYPE_CHECKING, Callable -from aiohttp import ContentTypeError from yarl import URL -from flexmeasures_client.constants import CONTENT_TYPE - if TYPE_CHECKING: # Only imports the below statements during type checking from flexmeasures_client.client import FlexMeasuresClient @@ -163,17 +160,6 @@ async def check_response( return polling_step, reauth_once, URL(url) -def check_content_type(response): - """Check if response is in format application/json""" - content_type = response.headers.get("Content-Type", "") - if CONTENT_TYPE not in content_type: - text = response.text() - raise ContentTypeError( - "Unexpected content type response from the API", - {"Content-Type": content_type, "response": text}, - ) - - def check_for_status(status, expected_status: int | Iterable[int]): """Check if status is an acceptable 2xx status code. diff --git a/src/flexmeasures_client/s2/__init__.py b/src/flexmeasures_client/s2/__init__.py index 981269e1..04a7cbcb 100644 --- a/src/flexmeasures_client/s2/__init__.py +++ b/src/flexmeasures_client/s2/__init__.py @@ -6,7 +6,7 @@ from collections import deque from dataclasses import dataclass from datetime import datetime -from typing import Callable, Coroutine, Dict, Type +from typing import Callable, Dict, Type from zoneinfo import ZoneInfo import pydantic @@ -21,6 +21,7 @@ ) +from flexmeasures_client.s2.clock import Clock, WallClock from flexmeasures_client.s2.utils import SizeLimitOrderedDict, get_message_id @@ -75,7 +76,7 @@ async def wrap(*args, **kwargs): ) return outgoing_message - message_type = schema.__fields__.get("message_type").default + message_type = schema.model_fields["message_type"].default setattr(wrap, "_tag", Tag(message_type)) return wrap @@ -126,6 +127,8 @@ def __init__(self, max_size: int = 100, timezone: str = "UTC") -> None: self.outgoing_messages_status = SizeLimitOrderedDict(max_size=max_size) self._timezone = ZoneInfo(timezone) + #: The time this handler goes by; a CEM gives each handler it registers its own clock. + self._clock: Clock = WallClock() self.discover() @@ -176,7 +179,7 @@ def supports_message(self, message: Dict | pydantic.BaseModel | str) -> bool: message_type = "" if isinstance(message, pydantic.BaseModel): - message_type = message.message_type + message_type = message.model_dump().get("message_type", "") elif isinstance(message, dict): message_type = message.get("message_type", "") elif isinstance(message, str): @@ -186,7 +189,7 @@ def supports_message(self, message: Dict | pydantic.BaseModel | str) -> bool: async def handle_message( self, message: pydantic.BaseModel | str | Dict - ) -> Coroutine: + ) -> pydantic.BaseModel | None: """ Calls the handler linked to the message_type and converts the output to a serialized dict, i.e. it converts all the inner objects to Python @@ -243,11 +246,12 @@ def handle_revoke_object(self, message: RevokeObject) -> ReceptionStatus: Stores the revoked object ID into the objects_revoked list """ - self.objects_revoked.append(message.object_id) + self.objects_revoked.append(str(message.object_id)) return ReceptionStatus( subject_message_id=str(message.message_id), status=ReceptionStatusValues.OK ) - def now(self): - return datetime.now(self._timezone) + def now(self) -> datetime: + """The current moment by this handler's clock, in its time zone.""" + return self._clock.now().astimezone(self._timezone) diff --git a/src/flexmeasures_client/s2/cem.py b/src/flexmeasures_client/s2/cem.py index 7e83cd8b..6d1f70e4 100644 --- a/src/flexmeasures_client/s2/cem.py +++ b/src/flexmeasures_client/s2/cem.py @@ -1,13 +1,13 @@ from __future__ import annotations +import asyncio import json import logging import math from asyncio import Queue -from collections import defaultdict from datetime import datetime, timedelta from logging import Logger -from typing import Dict, Optional +from typing import Awaitable, Callable, Dict, Optional, Protocol import pandas as pd import pydantic @@ -32,17 +32,47 @@ from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.exceptions import ContentTypeError from flexmeasures_client.s2 import Handler, register +from flexmeasures_client.s2.clock import Clock, SimulatedClock from flexmeasures_client.s2.control_types import ControlTypeHandler +from flexmeasures_client.s2.measurements import MeasurementRecorder from flexmeasures_client.s2.utils import ( get_latest_compatible_version, get_reception_status, get_unique_id, ) +from flexmeasures_client.s2.wrapper import MetaData _LOGGER = logging.getLogger(__name__) +class ConfiguredHandler(Protocol): + """What a site configurator gives back: the handler for a resource manager, and its measured-power sensors.""" + + @property + def handler(self) -> ControlTypeHandler: ... + + @property + def power_sensor_ids(self) -> dict[str, int]: ... + + @property + def asset_id(self) -> int: ... + + +#: Gives a resource manager a site and a handler of its own, from its details (see flexmeasures_client.s2.site). +SiteConfigurator = Callable[[ResourceManagerDetails], Awaitable[ConfiguredHandler]] + +#: The control type each family of control-type messages belongs to, by the prefix of its message type. +CONTROL_TYPE_BY_MESSAGE_PREFIX: dict[str, ControlType] = { + "FRBC.": ControlType.FILL_RATE_BASED_CONTROL, + "PEBC.": ControlType.POWER_ENVELOPE_BASED_CONTROL, + "PPBC.": ControlType.POWER_PROFILE_BASED_CONTROL, + "OMBC.": ControlType.OPERATION_MODE_BASED_CONTROL, + "DDBC.": ControlType.DEMAND_DRIVEN_BASED_CONTROL, +} + + class CEM(Handler): __version__ = "0.0.2-beta" @@ -64,10 +94,6 @@ class CEM(Handler): _datastore: dict _minimum_measurement_period: pd.Timedelta = pd.Timedelta(minutes=5) - _power_buffer: defaultdict = defaultdict( - list - ) # {commodity_quantity: [(timestamp, value), ...]} - def __init__( self, fm_client: FlexMeasuresClient, @@ -76,18 +102,55 @@ def __init__( timers: dict[str, datetime] | None = None, datastore: dict | None = None, power_sensor_id: dict[str, int] | None = None, + site_configurator: SiteConfigurator | None = None, + measurement_recorder: MeasurementRecorder | None = None, + clock: Clock | None = None, + accept_simulation_metadata: bool = False, **kwargs, ) -> None: """ Customer Energy Manager (CEM) + + :param site_configurator: where a resource manager gets a site and a handler of its own on connecting; + called with its details, in the background (see FRBCSiteConfigurator). + Until the handler is registered, its messages are answered with TEMPORARY_ERROR. + Without one, the handlers registered up front serve every resource manager. + :param measurement_recorder: where the resource manager's measured power is recorded; one of its own by default. + A server that makes a CEM per connection hands each the same one, + so a resource manager that reconnects is not recorded twice. + :param clock: the time this CEM and every handler it registers go by; + the wall clock by default. + :param accept_simulation_metadata: whether to take messages wrapped with a simulated time, as a co-simulation sends them, + and set the clock to that time (see flexmeasures_client.s2.wrapper.S2Wrapper). + Off by default, since the wrapper is not part of S2, + and a live CEM should not let a resource manager set its time. + A resource manager that wraps its messages, from its handshake on, + is then refused with INVALID_MESSAGE and cannot connect. + On, the clock has to be a SimulatedClock, and one is made where none is given. """ super(CEM, self).__init__() + if accept_simulation_metadata: + clock = clock if clock is not None else SimulatedClock() + if not isinstance(clock, SimulatedClock): + raise ValueError( + "A CEM that accepts simulation metadata sets its clock from it, so it needs a SimulatedClock." + ) + if clock is not None: + self._clock = clock + self._accept_simulation_metadata = accept_simulation_metadata self._fm_client = fm_client self._sending_queue = Queue() self._power_sensors = dict() self.power_sensor_id = power_sensor_id self._control_types_handlers = dict() + self._site_configurator = site_configurator + self._measurement_recorder = measurement_recorder or MeasurementRecorder( + fm_client, logger=logger + ) + self._configuring: asyncio.Task | None = None + #: The control type this CEM asked the resource manager to select, and the id of that message, until confirmed. + self._selecting: tuple[ControlType, str] | None = None self._default_control_type = default_control_type if not logger: @@ -100,8 +163,12 @@ def __init__( self._datastore = datastore if datastore is not None else {} def _is_timer_due(self, name: str) -> bool: - now = datetime.now() + # By this handler's clock, so that a co-simulation's timers run on simulated time. + now = self.now() due_time = self._timers.get(name, now - self._minimum_measurement_period) + if due_time.tzinfo is None: + # A timer handed in without a zone was compared with the host's local time, so it is read as that, as before. + due_time = due_time.astimezone(self._timezone) if due_time <= now: # Get total seconds of the period period_seconds = self._minimum_measurement_period.total_seconds() @@ -118,6 +185,10 @@ def _is_timer_due(self, name: str) -> bool: next_due = now.replace(minute=0, second=0, microsecond=0) + timedelta( seconds=next_tick_seconds ) + # On a period boundary the ceiling is the moment itself, + # and a clock that stands still, as a simulated one does between messages, then found the timer due on every call. + if next_due <= now: + next_due += self._minimum_measurement_period self._timers[name] = next_due return True else: @@ -131,6 +202,8 @@ def supports_control_type(self, control_type: ControlType): async def close(self): self._is_closed = True + if self._configuring is not None: + self._configuring.cancel() for control_type, handler in self._control_types_handlers.items(): self._logger.debug(f"Closing handler for {control_type}") @@ -158,6 +231,9 @@ def register_control_type(self, control_type_handler: ControlTypeHandler): # add fm_client to control_type handler control_type_handler._fm_client = self._fm_client + # The handler goes by the CEM's time, which a co-simulation sets. + control_type_handler._clock = self._clock + # add sending queue control_type_handler._sending_queue = self._sending_queue @@ -176,26 +252,40 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): the control type handler and in the CEM, then the one defined in the control type prevails. """ - response = None + response: pydantic.BaseModel | None = None if isinstance(message, pydantic.BaseModel): message = json.loads(message.json()) if isinstance(message, str): message = json.loads(message) + assert isinstance( + message, dict + ) # what a message is from here on, whatever form it came in + + if "metadata" in message and "message" in message: + unwrapped = self._unwrap(message) + if unwrapped is None: + return + message = unwrapped self._logger.debug(f"Received: {message}") + self._activate_implicitly(message) # try to handle the message with the control_type handle + # A control type without a handler yet, as while a resource manager's site is being configured, + # leaves its messages unhandled, so they are answered with a TEMPORARY_ERROR for the resource manager to retry. + handler = self._control_types_handlers.get(self._control_type) + if handler is None and self._control_type is not None: + self._ensure_configuring() if ( self._control_type is not None and ( self._control_type not in [ControlType.NO_SELECTION, ControlType.NOT_CONTROLABLE] ) - and self._control_types_handlers[self._control_type].supports_message( - message - ) + and handler is not None + and handler.supports_message(message) ): response = await self._control_types_handlers[ self._control_type @@ -217,18 +307,94 @@ async def handle_message(self, message: Dict | pydantic.BaseModel | str): if response is not None: await self._sending_queue.put(response) + def _unwrap(self, wrapped: dict) -> dict | None: + """The S2 message inside a co-simulation's wrapper, with the clock set to the simulated time it carries. + + Where this CEM does not accept the wrapper, the time cannot be read, or there is no message inside, + the message is refused, the clock is left as it was, and None is returned so that nothing is handled. + """ + inner = wrapped["message"] + reason = None + if not isinstance(inner, dict) or not isinstance( + inner.get("message_type"), str + ): + reason = "There is no S2 message inside this wrapper." + inner = inner if isinstance(inner, dict) else {} + elif not self._accept_simulation_metadata: + reason = ( + "This CEM does not accept messages wrapped with simulation metadata." + ) + else: + try: + moment = MetaData.model_validate(wrapped["metadata"]).dt + assert isinstance(self._clock, SimulatedClock) + self._clock.set(moment) + except (pydantic.ValidationError, ValueError, TypeError) as exc: + reason = f"The simulated time in this message's metadata cannot be used: {exc}" + if reason is None: + return inner + self._logger.warning("%s Refusing %s.", reason, inner.get("message_type")) + if inner.get("message_id") is not None: + self._sending_queue.put_nowait( + ReceptionStatus( + subject_message_id=inner["message_id"], + status=ReceptionStatusValues.INVALID_MESSAGE, + diagnostic_label=reason, + ) + ) + return None + + def _activate_implicitly(self, message: dict) -> None: + """Take a message of the control type this CEM is selecting as the resource manager's acceptance of it. + + A resource manager sends a control type's messages only once that control type is active on its side. + So where one arrives for the control type this CEM asked for, while the CEM still waits for the confirmation, + the resource manager is working in it already. #188 saw the CEM stay in NO_SELECTION while a resource manager, + reconnecting during setup in a co-simulation, kept sending FRBC messages, each answered with a TEMPORARY_ERROR + and retried; whether the confirmation was lost or never sent was not established. + Only the control type being selected is activated, never another one a message happens to belong to, + and its pending confirmation is then no longer waited for. + """ + if self._control_type != ControlType.NO_SELECTION or self._selecting is None: + return + control_type, selection_message_id = self._selecting + message_type = message.get("message_type") + if not isinstance(message_type, str): + return + prefix = next( + ( + p + for p, ct in CONTROL_TYPE_BY_MESSAGE_PREFIX.items() + if ct == control_type + ), + None, + ) + if prefix is None or not message_type.startswith(prefix): + return + if control_type not in self._control_types_handlers: + return + self._logger.warning( + "Received %s before the resource manager confirmed the selection of %s; taking it as the confirmation.", + message_type, + control_type.value, + ) + self.success_callbacks.pop(selection_message_id, None) + self.failure_callbacks.pop(selection_message_id, None) + self.update_control_type(control_type) + def update_control_type(self, control_type: ControlType): """ Callback function that is triggered when we receive a confirmation that the message has been received. """ self._control_type = control_type + self._selecting = None - async def get_message(self) -> str: + async def get_message(self) -> dict: """Call this function to get the messages to be sent to the RM Returns: - str: message in JSON format + dict: the message, as JSON-serialisable Python objects """ message = await self._sending_queue.get() @@ -273,6 +439,7 @@ async def activate_control_type( message_id, self.update_control_type, control_type=control_type ) + self._selecting = (control_type, message_id) await self._sending_queue.put( SelectControlType(message_id=message_id, control_type=control_type) ) @@ -299,6 +466,7 @@ async def handle_handshake(self, message: Handshake): @register(ResourceManagerDetails) async def handle_resource_manager_details(self, message: ResourceManagerDetails): self._resource_manager_details = message + self._ensure_configuring() if ( not self._control_type @@ -312,86 +480,85 @@ async def handle_resource_manager_details(self, message: ResourceManagerDetails) return get_reception_status(message) - @register(PowerMeasurement) - async def handle_power_measurement(self, message: PowerMeasurement): - - for power_measurement in message.values: - commodity_quantity = power_measurement.commodity_quantity.value + def _ensure_configuring(self) -> None: + """Set the resource manager's site up in the background, unless that is done or under way. - if ( - self.power_sensor_id is None - and commodity_quantity == "ELECTRIC.POWER.L1" - ): - sensor_id = 357 - elif self.power_sensor_id: - s_id = self.power_sensor_id.get(commodity_quantity) - if s_id is None: - # TODO: create a new sensor or return ReceptionStatus - self._logger.debug( - f"No power sensor set up for {commodity_quantity}. Ignoring measurement {power_measurement.value} at {message.measurement_timestamp}." - ) - continue - sensor_id = s_id - else: - self._logger.warning( - f"No power sensor IDs set up. Ignoring measurement {power_measurement.value} at {message.measurement_timestamp}." - ) - continue + In the background, since a site takes a few requests to FlexMeasures to set up, + and the resource manager waits for a reception status meanwhile. + A setup that failed is tried again on the next message that needs it, + while the resource manager's messages are answered with TEMPORARY_ERROR, which it retries. + """ + if ( + self._site_configurator is None + or self._control_types_handlers + or (self._configuring is not None and not self._configuring.done()) + or not hasattr(self, "_resource_manager_details") + ): + return + self._configuring = asyncio.create_task( + self._configure_site(self._resource_manager_details) + ) + self._configuring.add_done_callback(self._report_configuring) + + def _report_configuring(self, task: asyncio.Task) -> None: + if task.cancelled() or task.exception() is None: + return + self._logger.error( + "Could not set a site up for resource manager %r; trying again on its next message.", + self._resource_manager_details.name, + exc_info=task.exception(), + ) - # Store the value in the buffer - self._power_buffer[commodity_quantity].append( - (message.measurement_timestamp, power_measurement.value) - ) + async def _configure_site(self, details: ResourceManagerDetails) -> None: + """Give the resource manager its site, and register the handler wired to it.""" + assert self._site_configurator is not None + site = await self._site_configurator(details) + self.power_sensor_id = {**(self.power_sensor_id or {}), **site.power_sensor_ids} + self.register_control_type(site.handler) + self._logger.info( + "Resource manager %r is served on site %s.", details.name, site.asset_id + ) - # Compute bin - now = datetime.now(self._timezone) - m = self._minimum_measurement_period // pd.Timedelta(minutes=1) - bin_end = now.replace( - second=0, microsecond=0, minute=(now.minute // m) * m - ) # e.g. 10:15:00 - bin_start = bin_end - self._minimum_measurement_period + @register(PowerMeasurement) + async def handle_power_measurement(self, message: PowerMeasurement): - # If timer not due, just collect values - if not self._is_timer_due(f"power_measurement_{commodity_quantity}"): - self._logger.debug( - f"Collecting 5-minute average for {commodity_quantity} ({bin_start.isoformat()} – {bin_end.isoformat()})" - ) - continue + if self._site_configurator is not None and not self._control_types_handlers: + # Its site is still being set up, so there is nowhere to record it yet: the resource manager is asked to resend it. + self._ensure_configuring() + return get_reception_status(message, ReceptionStatusValues.TEMPORARY_ERROR) - # Compute average of all buffered values in last 5 minutes - buffer = self._power_buffer[commodity_quantity] - period_values = [v for (t, v) in buffer if bin_start <= t < bin_end] + taken: list[tuple[int, float]] = [] + for power_measurement in message.values: + commodity_quantity = power_measurement.commodity_quantity.value - if not period_values: + # Without a sensor for it, a measurement is not recorded. + # It used to go to sensor 357 for ELECTRIC.POWER.L1, a sensor of whichever site on the server had that id. + sensor_id = (self.power_sensor_id or {}).get(commodity_quantity) + if sensor_id is None: self._logger.debug( - f"No samples found for {commodity_quantity} in {bin_start}–{bin_end}, skipping." + "No power sensor set up for %s. Ignoring measurement %s at %s.", + commodity_quantity, + power_measurement.value, + message.measurement_timestamp, ) continue + taken.append((sensor_id, power_measurement.value)) - avg_value = sum(period_values) / len(period_values) - self._logger.debug( - f"Posting 5-minute average for {commodity_quantity}: " - f"{avg_value} ({bin_start.isoformat()} – {bin_end.isoformat()})" + # Every sensor is made ready before any value is taken, so that a message asked for again is not taken in part twice. + try: + for sensor_id, _ in taken: + await self._measurement_recorder.resolution(sensor_id) + except (ValueError, ConnectionError, ContentTypeError) as exc: + self._logger.warning( + "Could not take the measured power at %s, so the resource manager is asked to send it again: %s", + message.measurement_timestamp, + exc, + ) + return get_reception_status(message, ReceptionStatusValues.TEMPORARY_ERROR) + for sensor_id, watts in taken: + await self._measurement_recorder.add( + sensor_id, message.measurement_timestamp, watts, now=self.now() ) - - # Send measurement - try: - await self._fm_client.post_sensor_data( - sensor_id, - start=bin_start.isoformat(), - duration=self._minimum_measurement_period.isoformat(), # TODO: not specified in S2 Protocol - values=[avg_value], - unit=get_commodity_unit(commodity_quantity), - ) - except Exception as e: # noqa: B902 - intentional safety net - self._logger.warning( - f"POSTing power measurement failed with error: {e}" - ) - - # Keep only samples newer than this bin (for next period) - self._power_buffer[commodity_quantity] = [ - (t, v) for (t, v) in buffer if t >= bin_end - ] return get_reception_status(message) @@ -412,7 +579,7 @@ def handle_revoke_object(self, message: RevokeObject): for initials, control_type in control_types.items(): if initials in str(message.object_type): self._control_types_handlers[control_type].revoke_message( - message.object_id + str(message.object_id) ) return get_reception_status(message, ReceptionStatusValues.OK) @@ -420,13 +587,3 @@ def handle_revoke_object(self, message: RevokeObject): async def send_message(self, message): self._logger.debug(f"Sent: {message}") await self._sending_queue.put(message) - - -def get_commodity_unit(commodity_quantity) -> str: - if "POWER" in commodity_quantity: - return "kW" - if "FLOW_RATE" in commodity_quantity: - return "m³/h" - if "TEMPERATURE" in commodity_quantity: - return "°C" - return "" diff --git a/src/flexmeasures_client/s2/clock.py b/src/flexmeasures_client/s2/clock.py new file mode 100644 index 00000000..55646905 --- /dev/null +++ b/src/flexmeasures_client/s2/clock.py @@ -0,0 +1,56 @@ +"""The time an S2 CEM and its handlers go by. + +A CEM that runs live goes by the wall clock. +One that steers a co-simulation goes by the time the simulation says it is, +which a resource manager sends along with its messages (see CEM's accept_simulation_metadata), +so that what the CEM records and plans is placed at simulated moments rather than at the moment it ran. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Protocol + + +class Clock(Protocol): + def now(self) -> datetime: + """The current moment, timezone-aware.""" + ... + + +class WallClock: + """The time it is.""" + + def now(self) -> datetime: + return datetime.now(timezone.utc) + + +class SimulatedClock: + """The time a simulation says it is, as last set. + + It stands still between settings, as a simulation's time does between the messages that carry it. + Until it is first set it goes by the wall clock, for a resource manager that does not wrap its first messages; + one that wraps every message from its handshake on, as the co-simulation's does, sets it with the first. + Background work, such as a schedule being planned, reads it as it stands, which may be a later message's time. + """ + + def __init__(self, start: datetime | None = None) -> None: + self._now: datetime | None = None + if start is not None: + self.set(start) + + def set(self, moment: datetime) -> None: + if moment.tzinfo is None or moment.utcoffset() is None: + raise ValueError( + f"A simulated time has to say which time zone it is in, and {moment.isoformat()} does not." + ) + self._now = moment + + @property + def is_set(self) -> bool: + return self._now is not None + + def now(self) -> datetime: + if self._now is None: + return datetime.now(timezone.utc) + return self._now diff --git a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py index 830f3b0d..39ef719e 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/__init__.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/__init__.py @@ -230,11 +230,23 @@ def handle_system_description( # store system_description message for later self._system_description_history[system_description_id] = message - # A description says what the device can do, not what it is doing, so it is stored and - # no schedule is asked for here. - # Planning happens on a storage status, which is what carries a new state of charge, - # and which is the only one of the two a resource manager is required to keep sending - # (see handle_storage_status). + # A description says what the device can do, and a storage status what it holds now, + # and a schedule needs both. Planning normally hangs off a storage status (see handle_storage_status), + # but a description that arrives after the latest status, and differs from the description before it, + # is answered with a schedule against that latest status. + # Anything but the message id counts as a difference, its validity included, + # so a resource manager that re-dates its description on every resend asks for a schedule every time. + # Otherwise a resource manager that sends its status first, or whose fill level holds steady + # so that no new status follows, is never planned for, or keeps being planned for with what it can no longer do. + if self._storage_status_history: + request_number = self._count_request(STORAGE_STATUS_REQUESTS) + task = asyncio.create_task( + self.trigger_schedule( + system_description_id, request_number=request_number + ) + ) + self.background_tasks.add(task) + task.add_done_callback(self.background_tasks.discard) return get_reception_status(message, status=ReceptionStatusValues.OK) @register(FRBCUsageForecast) diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py index 351f8f97..ffa491a6 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_simple.py @@ -4,6 +4,7 @@ """ import asyncio +import math from datetime import datetime, timedelta from zoneinfo import ZoneInfo @@ -26,12 +27,15 @@ import pandas as pd from flexmeasures_client.client import convert_units -from flexmeasures_client.exceptions import JobFailedError +from flexmeasures_client.exceptions import ContentTypeError, JobFailedError from flexmeasures_client.s2.control_types.FRBC import FRBC, STORAGE_STATUS_REQUESTS from flexmeasures_client.s2.control_types.FRBC.utils import ( + WATTS_PER_ENERGY_UNIT_PER_SECOND, clip_fill_level_target_profile, + element_for_fill_level, fm_schedule_to_instructions, get_soc_min_max, + om_label, operation_mode_conversion_efficiency, ) from flexmeasures_client.s2.control_types.translations import ( @@ -118,6 +122,8 @@ def _operation_mode_power_bands( class FRBCSimple(FRBC): _power_sensor_id: int _price_sensor_id: int + _production_price_sensor_id: int + _relax_site_capacity: bool | None _soc_sensor_id: int _rm_discharge_sensor_id: int _fill_rate_sensor_id: int | None @@ -141,7 +147,7 @@ class FRBCSimple(FRBC): _power_sensor_takes_consumption_as_positive: bool | None #: When a power value was last recorded for the actuator, and for which moment, #: so that a second status inside one event does not collide with the first. - _last_power_recorded: tuple[datetime, datetime] | None + _power_recorded_at: dict[datetime, datetime] #: The charging efficiency last recorded, and the window it was recorded for, #: so that the same value for the same window is not recorded again on every schedule. _charging_efficiency_recorded: tuple[datetime, float] | None @@ -155,6 +161,15 @@ def __init__( soc_sensor_id: int, rm_discharge_sensor_id: int, price_sensor_id: int, + # Where feeding power back is priced differently from drawing it, as a feed-in tariff is. + # Without one, both directions are priced by price_sensor_id. + production_price_sensor_id: int | None = None, + # Whether the site's connection capacity may be breached at a price (True) or never (False). + # Unset, it is left to FlexMeasures, which since v1.0 relaxes it at a high default breach price, + # under its relax-constraints umbrella, and before that did not. + # A home whose connection is a physical limit can pass False; + # a site whose capacity is a target set by others, as in a community steering its members, can pass True. + relax_site_capacity: bool | None = None, # Optional so existing callers keep working; when absent the actuator's fill rate is # simply not recorded, rather than being posted onto the power sensor. fill_rate_sensor_id: int | None = None, @@ -162,6 +177,9 @@ def __init__( soc_minima_sensor_id: int | None = None, soc_maxima_sensor_id: int | None = None, usage_forecast_sensor_id: int | None = None, + # Its resolution has to be the handler's resolution: a storage efficiency is what remains after one step, + # computed here for a step of the handler's resolution, + # and FlexMeasures reads it as one step of the sensor's own resolution. leakage_behaviour_sensor_id: int | None = None, # Likewise optional: without a sensor to record it on, no charging efficiency is declared, # and the scheduler plans the storage as if the actuator converted power into stored energy one for one. @@ -197,6 +215,12 @@ def __init__( super().__init__(max_size) self._power_sensor_id = power_sensor_id self._price_sensor_id = price_sensor_id + self._production_price_sensor_id = ( + production_price_sensor_id + if production_price_sensor_id is not None + else price_sensor_id + ) + self._relax_site_capacity = relax_site_capacity self._schedule_duration = schedule_duration self._resolution = resolution self._soc_sensor_id = soc_sensor_id @@ -208,6 +232,18 @@ def __init__( self._leakage_behaviour_sensor_id = leakage_behaviour_sensor_id self._charging_efficiency_sensor_id = charging_efficiency_sensor_id self._charging_efficiency_recorded = None + self._storage_efficiency_recorded: tuple[datetime, float] | None = None + if resolution <= timedelta(0) or schedule_duration <= timedelta(0): + raise ValueError( + f"resolution and schedule_duration have to be positive, but {resolution} and {schedule_duration} were given." + ) + if schedule_duration % resolution: + # FlexMeasures answers with whole steps of the power sensor's resolution only, + # so a duration between two whole numbers of steps comes back with fewer values than it spans, + # and the schedule's step, read as its duration over its number of values, would come out wrong. + raise ValueError( + f"schedule_duration has to be a whole number of resolution steps, but {schedule_duration} is not a multiple of {resolution}." + ) if schedule_attempts < 1: # A schedule is asked for at least once, or no schedule is asked for at all. raise ValueError( @@ -225,7 +261,9 @@ def __init__( ) self._fill_level_scale = fill_level_scale self._power_sensor_takes_consumption_as_positive = None - self._last_power_recorded = None + self._power_recorded_at = {} + self._soc_sensor_records_instants: bool | None = None + self._planning = asyncio.Lock() if hard_bound_factor < 1: # Below one, the hard upper bound sits inside the declared range, # and FlexMeasures clips the band into the hard bounds, @@ -235,15 +273,18 @@ def __init__( ) self._hard_bound_factor = hard_bound_factor self.power_unit = power_unit + if energy_unit not in WATTS_PER_ENERGY_UNIT_PER_SECOND: + # A fill rate is per second of this unit, and has to be compared with the watts S2 states a power in, + # so a unit that cannot be converted to watts cannot be planned with. + raise ValueError( + f"energy_unit has to be one of {', '.join(WATTS_PER_ENERGY_UNIT_PER_SECOND)}, but {energy_unit!r} was given." + ) self.energy_unit = energy_unit # delay the start of the schedule from the time `valid_from` # of the FRBC.SystemDescription. self._valid_from_shift = valid_from_shift - def now(self): - return datetime.now(self._timezone) - def fill_rate_unit(self) -> str: """The unit a rate of change of the fill level is recorded in. @@ -272,33 +313,100 @@ def align_to_resolution(self, moment: datetime) -> datetime: resolution_seconds = int(self._resolution.total_seconds()) if resolution_seconds <= 0: return moment - seconds_into_the_day = ( - moment.hour * 3600 + moment.minute * 60 + moment.second - ) % resolution_seconds - return moment.replace(microsecond=0) - timedelta(seconds=seconds_into_the_day) + # Floored on the moment's own clock, in the offset it carries, which is how FlexMeasures floors a series posted with it, + # so an hourly step in a zone half an hour off UTC, or a daily one, starts where FlexMeasures' steps do. + # A boundary in the repeated hour of an autumn DST change exists twice, and the one meant is the later of the two + # that is not after the moment: taking the first pass every time put a boundary an hour or more too early. + # A boundary in a spring gap, which only a resolution of more than an hour can land on, moves to the end of the gap. + # Compared as pandas timestamps, which compare instants: two datetimes on one local clock compare by wall time, + # whichever pass each is on. + at = pd.Timestamp(moment) + candidates = [ + at.floor( + self._resolution, ambiguous=first_pass, nonexistent="shift_forward" + ) + for first_pass in (True, False) + ] + boundary = max(c for c in candidates if c <= at) + return boundary.to_pydatetime() def schedule_start(self, valid_from: datetime) -> datetime: - """Where a schedule asked for at valid_from starts: shifted, then floored to a boundary.""" - return self.align_to_resolution(valid_from + self._valid_from_shift) + """Where a schedule asked for at valid_from starts: shifted and floored to a boundary, but never before now. + + Never before now, since planning happens on every storage status while a description may be days old: + anchored to that description alone, every status in a long session was answered with the same window, + which slid into the past, so its instructions were all due already and arrived as one burst. + Where now is past that boundary, the schedule starts at the next one, not at the start of the step now is in. + A schedule is planned from the fill level reported now, and FlexMeasures takes that to be the level at its start, + so a schedule starting at a step already under way was planned as if the whole step were left, + and every replan within it asked for less power over what remained of it, delivering less than planned. + The step under way keeps the instruction it is already following. + """ + now = self.now() + start = self.align_to_resolution(valid_from + self._valid_from_shift) + if start >= now: + return start + current = self.align_to_resolution(now) + return current if current == now else current + self._resolution + + async def _soc_duration(self) -> timedelta: + """What a reported fill level is posted as lasting: an instant, where the state-of-charge sensor records instants. + + Posted over the minute after it, a reading on such a sensor was taken for a forecast of that minute. + FlexMeasures refuses a single value of no duration on a sensor with a resolution, though, + so on any other sensor, or one that cannot be read, it is posted over a minute, as before. + The sensor is asked until it answers, and its answer kept, since its resolution is a property of how it was set up. + """ + if self._soc_sensor_records_instants is None: + try: + sensor = await self._fm_client.get_sensor(self._soc_sensor_id) + except (ValueError, ConnectionError, ContentTypeError) as exc: + self._logger.warning( + "Could not read the resolution of state-of-charge sensor %s, so a fill level is posted over a minute: %s", + self._soc_sensor_id, + exc, + ) + return timedelta(minutes=1) + resolution = ( + sensor.get("event_resolution") if isinstance(sensor, dict) else None + ) + try: + self._soc_sensor_records_instants = resolution is not None and ( + pd.Timedelta(resolution) == pd.Timedelta(0) + ) + except ValueError: + self._logger.warning( + "State-of-charge sensor %s states a resolution of %r, which is not a duration, so a fill level is posted over a minute.", + self._soc_sensor_id, + resolution, + ) + self._soc_sensor_records_instants = False + return ( + timedelta(0) if self._soc_sensor_records_instants else timedelta(minutes=1) + ) async def send_storage_status( self, status: FRBCStorageStatus, request_number: int | None = None ): + # Read by this handler's clock, and known as of then, which in a co-simulation is the simulated time: + # without a prior, FlexMeasures took the moment the request reached it, + # and a simulated reading was recorded as known years after it was taken. + now = self.now() await self._fm_client.post_measurements( self._soc_sensor_id, - start=self.now(), + start=now, # Scaled like every other fill level this handler records, or the level and the # band around it would be in different scales wherever fill_level_scale is not 1. values=[status.present_fill_level * self._fill_level_scale], unit=self.energy_unit, - duration=timedelta(minutes=1), + duration=await self._soc_duration(), + prior=now, ) # The state of charge this status reports is what a schedule starts from, so this is # where a schedule is asked for. - # A system description is needed to turn that schedule into instructions, and a resource - # manager sends one before its first status, so its absence here means the two have gone - # out of step rather than that nothing has arrived yet. + # A system description is needed to turn that schedule into instructions. + # Without one yet, nothing is planned here, and the description plans against this status when it arrives. if not self._system_description_history: self._logger.warning( "Cannot ask for a schedule: no FRBC.SystemDescription has arrived yet." @@ -393,6 +501,27 @@ def _in_the_power_unit(self, power: float) -> float: return power return convert_units([power], S2_POWER_UNIT, self.power_unit)[0] + @staticmethod + def _latest(history: dict): + """The message a history holds that arrived last, or None while it holds none.""" + return list(history.values())[-1] if history else None + + def _soc_unit(self) -> tuple[str, float]: + """The unit a state of charge goes to FlexMeasures in, and what a fill level is multiplied by to be in it.""" + if self.energy_unit in ("kWh", "MWh"): + return self.energy_unit, 1.0 + return "kWh", ( + WATTS_PER_ENERGY_UNIT_PER_SECOND[self.energy_unit] + / WATTS_PER_ENERGY_UNIT_PER_SECOND["kWh"] + ) + + def _latest_fill_level(self) -> float | None: + """The fill level the latest storage status reported, in the resource manager's own units, if it reported a number.""" + if not self._storage_status_history: + return None + fill_level = list(self._storage_status_history.values())[-1].present_fill_level + return None if math.isnan(fill_level) else fill_level + async def send_actuator_status(self, status: FRBCActuatorStatus): factor = status.operation_mode_factor sd: FRBCSystemDescription = list(self._system_description_history.values())[-1] @@ -426,7 +555,22 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): status.actuator_id, ) operation_mode = actuator.operation_modes[0] - element = operation_mode.elements[0] + + # The element the actuator runs on depends on the fill level, as a heat pump derates while its buffer warms, + # so it is the one that applies at the latest fill level reported, not whichever the mode happens to list first. + fill_level = self._latest_fill_level() + if fill_level is not None: + element = element_for_fill_level(operation_mode.elements, fill_level) + else: + # As if at the bottom of the range, which is as good a guess as any, and is said to be one below. + element = element_for_fill_level(operation_mode.elements, float("-inf")) + if len(operation_mode.elements) > 1: + self._logger.warning( + "FRBC.ActuatorStatus for operation mode %s arrived with no usable FRBC.StorageStatus yet, " + "so its power is recorded as the element at the bottom of the fill-level range states it, " + "which need not be the element the actuator runs on.", + om_label(operation_mode), + ) # Interpolate the element's POWER range, not its fill rate. A fill rate is the thermal # output a heat pump delivers into its buffer; the power range is what it draws @@ -456,15 +600,21 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): # and this handler sends several at once, so that is the ordinary case rather than an edge. # The comparison is on the event as FlexMeasures keys it, # since two statuses seconds apart still land on the same resolution step. + # Kept per event, since statuses for different steps can interleave within a second, + # and a comparison with the latest status alone then missed the one before it about the same step. event = self.align_to_resolution(start) recorded_at = self.now() - if self._last_power_recorded is not None: - last_event, last_recorded_at = self._last_power_recorded - if last_event == event and recorded_at - last_recorded_at < timedelta( - seconds=1 - ): - recorded_at = last_recorded_at + timedelta(seconds=1) - self._last_power_recorded = (event, recorded_at) + last_recorded_at = self._power_recorded_at.get(event) + if last_recorded_at is not None and recorded_at - last_recorded_at < timedelta( + seconds=1 + ): + recorded_at = last_recorded_at + timedelta(seconds=1) + self._power_recorded_at[event] = recorded_at + # Steps more than a day older than this one are forgotten, to keep this bounded: a status that late is not expected. + for old_event in [ + e for e in self._power_recorded_at if e < event - timedelta(days=1) + ]: + del self._power_recorded_at[old_event] # The value and the unit it is in travel together, # so that what is reported below is a pair that agrees even where the conversion is what failed. @@ -482,7 +632,7 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): prior=recorded_at, values=[value], unit=unit, - duration=timedelta(minutes=15), + duration=self._resolution, ) except (JobFailedError, ValueError, ConnectionError, NotImplementedError): # This runs in a task nobody awaits, @@ -511,11 +661,14 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): prior=self.now(), values=[rate], unit=self.fill_rate_unit(), - duration=timedelta(minutes=15), + duration=self._resolution, ) async def _record_charging_efficiency( - self, system_description: FRBCSystemDescription, start: datetime + self, + system_description: FRBCSystemDescription, + start: datetime, + fill_level: float, ) -> bool: """Record what the actuator converts drawn power into stored energy at, over the window a schedule covers. @@ -525,6 +678,8 @@ async def _record_charging_efficiency( a schedule planned against a sensor holding nothing for its window is planned as if the actuator converted power one for one. :param start: where the schedule starts, which is where the value is recorded from. + :param fill_level: the fill level the schedule starts from, in the resource manager's own units, + which picks the element of each mode that answers. :returns: whether that schedule may be planned against the sensor, which it may not where nothing is recorded on it for this window. """ @@ -534,19 +689,13 @@ async def _record_charging_efficiency( # One actuator, as everywhere else in this handler. actuator = system_description.actuators[0] efficiencies: dict[str, float] = {} - try: - for operation_mode in actuator.operation_modes: - efficiency = operation_mode_conversion_efficiency( - operation_mode, self._fill_level_scale, self.energy_unit - ) - if efficiency is not None: - efficiencies[operation_mode.id] = efficiency - except NotImplementedError: - self._logger.exception( - "Not asking for a schedule against a charging efficiency: a fill rate in %s cannot be compared against the watts S2 states a power range in.", - self.fill_rate_unit(), + # The energy unit is one fill_rate_in_watts knows, which the constructor saw to. + for operation_mode in actuator.operation_modes: + efficiency = operation_mode_conversion_efficiency( + operation_mode, self._fill_level_scale, self.energy_unit, fill_level ) - return False + if efficiency is not None: + efficiencies[str(operation_mode.id)] = efficiency if not efficiencies: # Every mode is idle, or every mode only feeds back, and neither converts drawn power into anything. @@ -621,8 +770,12 @@ def _is_worth_asking_again(self, failure: Exception) -> bool: return any(marker in message for marker in RETRYABLE_TRIGGER_REFUSALS) async def _ask_for_a_schedule( - self, start: datetime, flex_context: dict, flex_model: dict - ) -> dict | list[dict]: + self, + start: datetime, + flex_context: dict, + flex_model: dict, + request_number: int | None = None, + ) -> dict | list[dict] | None: """Ask FlexMeasures for a schedule, and ask again where the attempt is worth making again. One failed scheduling job otherwise leaves the storage without an instruction for its window. @@ -632,7 +785,9 @@ async def _ask_for_a_schedule( a worker crashed inside its solver, and the resource manager it belonged to polled until its round timed out, every round, for the rest of the run. - :param start: the window the schedule is asked for, which every attempt asks for again. + :param start: the window the schedule is asked for, which every attempt asks for again. + :param request_number: the planning request this answers; where a newer one arrives between attempts, + no further attempt is made and None is returned, so that the newer one is planned sooner. :raises JobFailedError: where the last attempt worth making failed inside the server. :raises ValueError: where the trigger itself was refused. """ @@ -674,6 +829,14 @@ async def _ask_for_a_schedule( failure, ) await asyncio.sleep(self._schedule_retry_interval.total_seconds()) + if request_number is not None and not self._is_latest_request( + STORAGE_STATUS_REQUESTS, request_number + ): + self._logger.info( + "Not asking again for planning request %s: a newer one arrived, and is planned instead.", + request_number, + ) + return None def _warn_where_a_fallback_scheduler_answered( self, schedule: dict | list[dict], start: datetime @@ -705,12 +868,39 @@ def _warn_where_a_fallback_scheduler_answered( async def trigger_schedule( self, system_description_id: str, request_number: int | None = None + ): + """Plan, one schedule at a time, and only for the newest request once it is this one's turn. + + Each storage status is a planning request. + One arriving while a schedule was being planned used to start a second solve beside it, whose answer replaced the first's, + so a resource manager reporting faster than FlexMeasures plans kept several solves running for nothing. + Now a request waits for the one in flight, and is dropped where a newer one has arrived meanwhile; + the one in flight, once out of date, makes no further attempt after a failed one. + So a burst of statuses costs at most the attempt in flight and one plan more, for the newest, + which starts once that attempt is over rather than beside it. + + :param system_description_id: system description to base the schedule on + :param request_number: which planning request this schedule answers, which is checked when its turn comes, + between attempts, and before the schedule is translated and sent + """ + async with self._planning: + if request_number is not None and not self._is_latest_request( + STORAGE_STATUS_REQUESTS, request_number + ): + self._logger.info( + "Not planning for request %s: a newer one arrived while the schedule before it was being planned.", + request_number, + ) + return + await self._plan(system_description_id, request_number) + + async def _plan( + self, system_description_id: str, request_number: int | None = None ): """Translates S2 System Description into FM API calls. :param system_description_id: system description to base the schedule on - :param request_number: which planning request this schedule answers, which is - checked once more before the instructions go out + :param request_number: which planning request this schedule answers """ system_description: FRBCSystemDescription = self._system_description_history[ @@ -726,7 +916,9 @@ async def trigger_schedule( -1 ].present_fill_level else: - print("Can't trigger schedule without knowing the status of the storage...") + self._logger.warning( + "Cannot ask for a schedule: no FRBC.StorageStatus has arrived yet, so there is no state of charge to plan from." + ) return # Scaled at the FlexMeasures edge, like every bound and level below it. @@ -741,7 +933,10 @@ async def trigger_schedule( # TODO: an element states one power range per commodity quantity, and no distinction is made between them here, # as the schedule translation does not make one either (see power_to_fill_rate_with_metrics). power_ranges = [ - tuple(sorted((power_range.start_of_range, power_range.end_of_range))) + ( + min(power_range.start_of_range, power_range.end_of_range), + max(power_range.start_of_range, power_range.end_of_range), + ) for operation_mode in actuator.operation_modes for element in operation_mode.elements for power_range in element.power_ranges @@ -772,12 +967,17 @@ async def trigger_schedule( # and the window a charging efficiency has to be recorded over to be planned against. start = self.schedule_start(system_description.valid_from) + # FlexMeasures takes a state of charge in kWh or MWh only. + # So a fill level recorded in another energy unit is converted here, at the edge, rather than refused, + # and the bounds read back below stay in the recorded unit. + # The band, the usage and the leakage are read from sensors, which FlexMeasures converts itself. + soc_unit, to_soc_unit = self._soc_unit() flex_model: dict = { - # The same unit the fill level is recorded in, or the scheduler would plan against bounds a thousandfold off the measurements it plans from. - "soc-unit": self.energy_unit, - "soc-at-start": soc_at_start, # TODO: use forecast of the SOC instead - "soc-min": soc_min, - "soc-max": soc_max, + "soc-unit": soc_unit, + # TODO: use forecast of the SOC instead + "soc-at-start": soc_at_start * to_soc_unit, + "soc-min": soc_min * to_soc_unit, + "soc-max": soc_max * to_soc_unit, } # What the resource manager declared about its storage, each entry only where a sensor holds it: # an entry naming a sensor that was never configured has the scheduler plan against nothing at all. @@ -801,14 +1001,16 @@ async def trigger_schedule( ) if self._usage_forecast_sensor_id is not None: flex_model["soc-usage"] = [{"sensor": self._usage_forecast_sensor_id}] - if self._leakage_behaviour_sensor_id is not None: + if await self._record_storage_efficiency(start, fill_level_at_start): flex_model["storage-efficiency"] = { "sensor": self._leakage_behaviour_sensor_id } # What the operation modes say about converting drawn power into stored energy. # Recorded first, since FlexMeasures reads it from the sensor named here and not from the flex model, # and named only where it was recorded. - if await self._record_charging_efficiency(system_description, start): + if await self._record_charging_efficiency( + system_description, start, fill_level_at_start + ): flex_model["charging-efficiency"] = { "sensor": self._charging_efficiency_sensor_id } @@ -850,7 +1052,7 @@ async def trigger_schedule( self._logger.debug("flex model: %s", flex_model) flex_context: dict = { - "production-price": {"sensor": self._price_sensor_id}, + "production-price": {"sensor": self._production_price_sensor_id}, "consumption-price": {"sensor": self._price_sensor_id}, "site-power-capacity": f"{3 * 25 * 230} VA", } @@ -860,10 +1062,27 @@ async def trigger_schedule( # The breach price is left to FlexMeasures' own default, # since what a breach of comfort is worth is the deployment's judgement, not the protocol's. flex_context["relax-soc-constraints"] = True + if self._relax_site_capacity is not None: + flex_context["relax-site-capacity-constraints"] = self._relax_site_capacity - schedule = await self._ask_for_a_schedule(start, flex_context, flex_model) + schedule = await self._ask_for_a_schedule( + start, flex_context, flex_model, request_number=request_number + ) + if schedule is None: + return self._warn_where_a_fallback_scheduler_answered(schedule, start) + # send_instruction_batch checks this once more, but translating a schedule follows every step of it, + # which is wasted on a request that is already out of date. + if request_number is not None and not self._is_latest_request( + STORAGE_STATUS_REQUESTS, request_number + ): + self._logger.info( + "Not turning the schedule for planning request %s into instructions: a newer storage status arrived while it was being planned.", + request_number, + ) + return + # translate FlexMeasures schedule into instructions. SOC -> Power -> PowerFactor instructions = fm_schedule_to_instructions( schedule, @@ -877,6 +1096,10 @@ async def trigger_schedule( soc_min / self._fill_level_scale, soc_max / self._fill_level_scale, ), + # What the storage loses by itself and what is drawn from it, as the resource manager last said, + # so that the fill level each step is answered at falls as well as rises. + leakage_behaviour=self._latest(self._leakage_behaviour_history), + usage_forecast=self._latest(self._usage_forecast_history), ) await self.send_instruction_batch( @@ -975,25 +1198,67 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): ) async def send_leakage_behaviour(self, leakage_behaviour: FRBCLeakageBehaviour): - """Record what a resource manager's storage loses on its own, as a storage efficiency.""" + """Keep a resource manager's leakage behaviour for planning, which records it when it asks for a schedule. + + Recorded on arrival instead, it covered the hours after the message rather than the window being planned, + at a fill level nobody had reported yet, and planning went on without it once those hours had passed. + """ if self._leakage_behaviour_sensor_id is None: self._logger.info( "Not recording an FRBC.LeakageBehaviour: no leakage-behaviour sensor is configured." ) - return - storage_efficiency = leakage_behaviour_to_storage_efficiency( + async def _record_storage_efficiency( + self, start: datetime, fill_level: float + ) -> bool: + """Record what the storage keeps of itself per step, over the window a schedule covers, from where it starts. + + :returns: whether the schedule may be planned against the sensor, which it may not where nothing is recorded on it. + """ + leakage_behaviour = self._latest(self._leakage_behaviour_history) + if ( + self._leakage_behaviour_sensor_id is None + or leakage_behaviour is None + or not self._system_description_history + ): + return False + # S2 states a leakage as a rate, and FlexMeasures takes a share of the stock, so one share stands for the rate + # only at one fill level, and the loss scales with the stock away from it. + # That fill level is where the schedule starts, as long as that is at least the middle of the storage's range: + # from a near-empty start, a share of the start level would let almost nothing remain per step, + # and the loss would be overstated many times over as soon as the plan filled the storage. + # Below the middle the loss is understated instead, by at most the ratio of the middle to the stock, which is the milder error. + storage_range = list(self._system_description_history.values())[ + -1 + ].storage.fill_level_range + middle = (storage_range.start_of_range + storage_range.end_of_range) / 2 + # What remains after one step of this handler's resolution, + # which FlexMeasures reads as one step of the leakage sensor's resolution, so the two have to agree. + efficiency = leakage_behaviour_to_storage_efficiency( message=leakage_behaviour, resolution=self._resolution, fill_level_scale=self._fill_level_scale, + fill_level=fill_level, + share_of=max(fill_level, middle), ) - - now = self.now() - await self._fm_client.post_sensor_data( - sensor_id=self._leakage_behaviour_sensor_id, - start=self.align_to_resolution(leakage_behaviour.valid_from or now), - prior=now, - values=[storage_efficiency], - unit="%", - duration=self._schedule_duration, - ) + if self._storage_efficiency_recorded == (start, efficiency): + return True + try: + await self._fm_client.post_sensor_data( + sensor_id=self._leakage_behaviour_sensor_id, + start=start, + prior=self.now(), + values=[efficiency], + unit="%", + duration=self._schedule_duration, + ) + except (JobFailedError, ValueError, ConnectionError): + self._logger.exception( + "Not asking for a schedule against a storage efficiency: could not record %s %% on sensor %s for %s.", + efficiency, + self._leakage_behaviour_sensor_id, + start, + ) + return False + self._storage_efficiency_recorded = (start, efficiency) + return True diff --git a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py index 3b6b956c..23babaf3 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/frbc_tunes.py @@ -169,8 +169,12 @@ def __init__( self._datastore = datastore if datastore is not None else {} def _is_timer_due(self, name: str) -> bool: - now = datetime.now() + # By this handler's clock, so that a co-simulation's timers run on simulated time. + now = self.now() due_time = self._timers.get(name, now - self._minimum_measurement_period) + if due_time.tzinfo is None: + # A timer handed in without a zone was compared with the host's local time, so it is read as that, as before. + due_time = due_time.astimezone(self._timezone) if due_time <= now: # Get total seconds of the period period_seconds = self._minimum_measurement_period.total_seconds() @@ -187,6 +191,10 @@ def _is_timer_due(self, name: str) -> bool: next_due = now.replace(minute=0, second=0, microsecond=0) + timedelta( seconds=next_tick_seconds ) + # On a period boundary the ceiling is the moment itself, + # and a clock that stands still, as a simulated one does between messages, then found the timer due on every call. + if next_due <= now: + next_due += self._minimum_measurement_period self._timers[name] = next_due return True else: @@ -195,9 +203,6 @@ def _is_timer_due(self, name: str) -> bool: ) return False - def now(self): - return datetime.now(self._timezone) - async def send_storage_status( self, status: FRBCStorageStatus, request_number: int | None = None ): @@ -239,7 +244,7 @@ async def send_leakage_behaviour(self, leakage: FRBCLeakageBehaviour): leakage_behaviour_to_storage_efficiency( message=leakage, resolution=timedelta(minutes=15), - fill_level_scale=self._fill_level_scalefill_level_scale, + fill_level_scale=self._fill_level_scale, ) ], unit=PERCENTAGE, @@ -278,10 +283,10 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): active_operation_mode_fill_rate_sensor_id = self._nes_fill_rate_sensor_id # Operation Mode Factor to fill rate - fill_rate = operation_mode.elements[0].fill_rate + fill_rate_range = operation_mode.elements[0].fill_rate fill_rate = ( - fill_rate.start_of_range - + (fill_rate.end_of_range - fill_rate.start_of_range) * factor + fill_rate_range.start_of_range + + (fill_rate_range.end_of_range - fill_rate_range.start_of_range) * factor ) * self._fill_level_scale # Send data to the sensor of the fill rate corresponding to the active operation mode @@ -310,7 +315,10 @@ async def send_actuator_status(self, status: FRBCActuatorStatus): duration=timedelta(minutes=15), ) - async def trigger_schedule(self): + # It takes neither argument the base class declares: TUNES plans from its latest description, + # and does not take part in dropping superseded requests. + # Accepting them and ignoring them would turn a caller's mistake into a silent one, so the mismatch stays. + async def trigger_schedule(self): # type: ignore[override] """ Ask FlexMeasures for a new schedule and create FRBC.Instructions to send back to the ResourceManager """ @@ -328,10 +336,10 @@ async def trigger_schedule(self): operation_mode = None efficiency_sensor_id = None for _operation_mode in actuator.operation_modes: - if "THP" in _operation_mode.diagnostic_label: + if "THP" in (_operation_mode.diagnostic_label or ""): operation_mode = _operation_mode efficiency_sensor_id = self._thp_efficiency_sensor_id - elif "NES" in _operation_mode.diagnostic_label: + elif "NES" in (_operation_mode.diagnostic_label or ""): operation_mode = _operation_mode efficiency_sensor_id = self._nes_efficiency_sensor_id else: @@ -477,9 +485,11 @@ async def trigger_schedule(self): except Exception as exc: self._logger.error(str(exc)) + # Posted per hour and scaled into FlexMeasures' units by send_usage_forecast, and read back as such, + # while the schedule translation takes a usage rate per second, in the resource manager's own units, as S2 states it. usage_forecast = pd.Series( usage_forecast["values"], index=idx, name="usage_forecast" - ) + ) / ((pd.Timedelta(hours=1) / pd.Timedelta(seconds=1)) * self._fill_level_scale) schedule_series = pd.Series(schedule["values"], index=idx, name="schedule") @@ -502,6 +512,12 @@ async def trigger_schedule(self): system_description, initial_fill_level=soc_at_start / self._fill_level_scale, logger=self._logger, + # What the storage loses by itself, so that the running fill level falls as well as rises. + leakage_behaviour=( + list(self._leakage_behaviour_history.values())[-1] + if self._leakage_behaviour_history + else None + ), ) # Revoke all previous instructions @@ -604,9 +620,9 @@ async def send_conversion_efficiencies( ) for operation_mode in actuator.operation_modes: - if "THP" in operation_mode.diagnostic_label: + if "THP" in (operation_mode.diagnostic_label or ""): sensor_id = self._thp_efficiency_sensor_id - elif "NES" in operation_mode.diagnostic_label: + elif "NES" in (operation_mode.diagnostic_label or ""): sensor_id = self._nes_efficiency_sensor_id else: continue @@ -653,12 +669,6 @@ async def update_flex_model(self, system_description: FRBCSystemDescription): parse_json_fields=True, ) - async def close(self): - """ - Can be used to stop recurring tasks. - """ - self._logger.debug(f"Closing {self.__class__.__name__} handler") - async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): """ Send FRBC.UsageForecast to FlexMeasures. @@ -676,23 +686,25 @@ async def send_usage_forecast(self, usage_forecast: FRBCUsageForecast): minute=(start_time.minute // 15) * 15, second=0, microsecond=0 ) - usage_forecast = translate_usage_forecast_to_fm( + usage_forecast_series = translate_usage_forecast_to_fm( usage_forecast, RESOLUTION, strategy="mean", fill_level_scale=self._fill_level_scale, ) - # Scale usage forecast e.g. [0, 100] %/s -> [0, 100] %/(15 min) - scale = timedelta(minutes=15) / timedelta(seconds=1) - scaled_usage_forecast = usage_forecast * scale + # From per second, as S2 states a usage rate, to per hour, which is what a value labelled MW means: + # FlexMeasures reads soc-usage as a power and multiplies it by the step's share of an hour. + # Posted per quarter hour instead, it was planned as a quarter of the real draw. + scale = timedelta(hours=1) / timedelta(seconds=1) + scaled_usage_forecast = usage_forecast_series * scale await self._fm_client.post_sensor_data( sensor_id=self._usage_forecast_sensor_id, start=start_time, values=scaled_usage_forecast.tolist(), - unit=POWER_UNIT, # e.g. [0, 100] MW/(15 min) - duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast)), + unit=POWER_UNIT, # a rate per hour, as MW is + duration=str(pd.Timedelta(RESOLUTION) * len(usage_forecast_series)), ) async def send_fill_level_target_profile( diff --git a/src/flexmeasures_client/s2/control_types/FRBC/utils.py b/src/flexmeasures_client/s2/control_types/FRBC/utils.py index 4acd8d8c..cec7fa08 100644 --- a/src/flexmeasures_client/s2/control_types/FRBC/utils.py +++ b/src/flexmeasures_client/s2/control_types/FRBC/utils.py @@ -1,9 +1,10 @@ +import bisect import logging +import math from datetime import timedelta from math import isclose -from typing import List +from typing import List, Sequence, TypeVar -import numpy as np import pandas as pd try: @@ -11,8 +12,10 @@ from s2python.frbc import ( FRBCInstruction, FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, FRBCOperationMode, FRBCSystemDescription, + FRBCUsageForecast, ) from s2python.frbc.frbc_operation_mode_element import FRBCOperationModeElement except ImportError: @@ -27,99 +30,179 @@ LOGGER = logging.getLogger(__name__) -def op_mode_compute_factor( - op_mode_elem: FRBCOperationModeElement, - fill_rate: float, - logger: logging.Logger = LOGGER, -) -> float: - """Compute the operation mode factor for a given fill rate.""" - start_fill_rate = op_mode_elem.fill_rate.start_of_range - end_fill_rate = op_mode_elem.fill_rate.end_of_range - delta_fill_rate = end_fill_rate - start_fill_rate - - fill_rate = max(fill_rate, start_fill_rate) - fill_rate = min(fill_rate, end_fill_rate) - - # Case that start_fill_rate == end_fill_rate - if np.isclose(delta_fill_rate, 0): - return 0 - - omf = (fill_rate - start_fill_rate) / delta_fill_rate - if omf < 0 or omf > 1: - logger.error( - f"Invalid operation mode factor {omf} computed from fill_rate {fill_rate}" - ) - return omf - - def compute_next_fill_level( fill_level: float, fill_rate: float, - leakage_behaviour: FRBCLeakageBehaviour | None, - deltaT: float, - usage: float = 0.0, + step: timedelta, + leakage_behaviour: FRBCLeakageBehaviour | None = None, + usage_rate: float = 0.0, ) -> float: + """The fill level one step later. + + This is the unit contract for every rate it takes, and it is the one S2 states: + each is a change in fill level per second, in the resource manager's own fill-level units. + A fill rate raises the fill level, while a usage rate and a leakage rate both lower it where they are positive. + The step is a duration, rather than a number of some unit, so that it cannot be taken in the wrong one. + + :param leakage_behaviour: what the storage loses by itself, from the element that applies at this fill level; + none where the resource manager has not said. + :param usage_rate: what is drawn from the storage, held over this step. """ - Compute the next fill level of a storage system over a time step. - - Parameters - ---------- - fill_level - Current fill level in system units (e.g., kWh, °C, height) - fill_rate - Fill rate chosen from the operation mode (units per hour) - leakage_behaviour - FRBC LeakageBehaviour; may be None - deltaT - Time step in hours - usage - Consumption/usage over the time step (in system units per hour) - - Returns - ------- - next_fill_level : float - Fill level after applying fill rate, leakage, and usage + leakage_rate = 0.0 + if leakage_behaviour is not None: + leakage_rate = element_for_fill_level( + leakage_behaviour.elements, fill_level + ).leakage_rate + return fill_level + (fill_rate - usage_rate - leakage_rate) * step.total_seconds() + + +#: How close two factors are to count as the same instruction. +#: It absorbs the float noise of converting a schedule from MW to W, which is not a new instruction. +FACTOR_TOLERANCE = 1e-6 + + +class UsageProfile: + """What a usage forecast expects, per second, at any moment: piecewise constant, looked up by bisection. + + A forecast is a run of elements, each lasting its own duration from where the one before it ended. + Outside that run, nothing is forecast, and nothing is assumed: the rate is zero. + Built once per schedule, since a forecast may hold hundreds of elements and a schedule asks about it often. """ - # --- Apply fill rate over deltaT --- - delta_fill = fill_rate * deltaT - - # --- Apply leakage over deltaT --- - total_leakage = 0.0 - if leakage_behaviour and leakage_behaviour.elements: - # Sum contributions from all relevant elements (or pick one based on fill level) - element = next( - ( - e - for e in leakage_behaviour.elements - if e.fill_level_range.start_of_range - <= fill_level - <= e.fill_level_range.end_of_range - ), - None, - ) - if element is None: - # Take closest element - element = min( - leakage_behaviour.elements, - key=lambda e: max( - 0.0, - e.fill_level_range.start_of_range - fill_level, - fill_level - e.fill_level_range.end_of_range, - ), - ) + def __init__( + self, usage_forecast: FRBCUsageForecast | None = None, rate: float | None = None + ): + #: The moments the rate changes at, and the rate from each of them on; one constant rate has none. + self._changes: list[pd.Timestamp] = [] + self._rates: list[float] = [] + self._before = 0.0 if rate is None else rate + if usage_forecast is None: + return + moment = pd.Timestamp(usage_forecast.start_time) + for element in usage_forecast.elements: + self._changes.append(moment) + self._rates.append(element.usage_rate_expected) + moment = moment + timedelta(milliseconds=element.duration.root) + self._changes.append(moment) + self._rates.append(0.0) + + def rate_at(self, moment: pd.Timestamp) -> float: + index = bisect.bisect_right(self._changes, moment) - 1 + return self._before if index < 0 else self._rates[index] + + def next_change_after(self, moment: pd.Timestamp) -> pd.Timestamp | None: + index = bisect.bisect_right(self._changes, moment) + return self._changes[index] if index < len(self._changes) else None + + +def usage_rate_at( + usage_forecast: FRBCUsageForecast | None, moment: pd.Timestamp +) -> float: + """The usage rate a forecast expects at a moment, per second, as S2 states it (see UsageProfile).""" + return UsageProfile(usage_forecast).rate_at(moment) + + +def _sides(elements, fill_level: float): + """The element that applies just below a fill level, and the one just above it. + + Inside an element both are that element; at a boundary two elements share, they are the two neighbours. + """ + above = element_for_fill_level(elements, fill_level) + below_candidates = [ + e + for e in elements + if e.fill_level_range.start_of_range + < fill_level + <= e.fill_level_range.end_of_range + ] + below = ( + max(below_candidates, key=lambda e: e.fill_level_range.start_of_range) + if below_candidates + else above + ) + return below, above - # leakage_rate_per_second is in system units lost per second - total_leakage = ( - getattr(element, "leakage_rate_per_second", 0.0) * deltaT * 3600 - ) # convert hours → seconds - # --- Apply usage --- - total_usage = usage * deltaT +def follow_operation_mode( + fill_level: float, + operation_mode: FRBCOperationMode, + factor: float, + start: pd.Timestamp, + step: timedelta, + leakage_behaviour: FRBCLeakageBehaviour | None = None, + usage: UsageProfile | None = None, + bounds: tuple[float, float] | None = None, +) -> float: + """The fill level after following one operation mode at one factor for a step, as a resource manager does. + + An instruction fixes the mode and the factor, not the element: + within the step the fill level can pass from one element into another, of the mode or of the leakage behaviour, + and the usage a forecast expects can change. + Every rate is constant between such moments, so the step is followed exactly, from one of them to the next. + A level that reaches a boundary its far side would push it back from, as where a heat pump cannot heat above it, + stays at that boundary, as a real storage does, rather than flipping across it. + So does a level that reaches one of the bounds. + + :param usage: what is drawn from the storage, per second; none where nothing is known. + :param bounds: the fill levels the storage is held between. + """ + usage = usage or UsageProfile() + leakage_elements = leakage_behaviour.elements if leakage_behaviour else [] + lowest, highest = bounds if bounds is not None else (-math.inf, math.inf) + boundaries = sorted( + { + edge + for e in list(operation_mode.elements) + list(leakage_elements) + for edge in ( + e.fill_level_range.start_of_range, + e.fill_level_range.end_of_range, + ) + } + | ({lowest, highest} - {-math.inf, math.inf}) + ) - # --- Next fill level --- - next_fill_level = fill_level + delta_fill - total_leakage - total_usage - return next_fill_level + def net_rate(mode_element, leakage_element, moment) -> float: + low, high = ( + mode_element.fill_rate.start_of_range, + mode_element.fill_rate.end_of_range, + ) + leakage = leakage_element.leakage_rate if leakage_element is not None else 0.0 + return low + (high - low) * factor - usage.rate_at(moment) - leakage + + moment, end = pd.Timestamp(start), pd.Timestamp(start) + step + fill_level = min(max(fill_level, lowest), highest) + while moment < end: + mode_below, mode_above = _sides(operation_mode.elements, fill_level) + leak_below, leak_above = ( + _sides(leakage_elements, fill_level) if leakage_elements else (None, None) + ) + rising = net_rate(mode_above, leak_above, moment) + falling = net_rate(mode_below, leak_below, moment) + if rising > 0 and fill_level < highest: + rate = rising + elif falling < 0 and fill_level > lowest: + rate = falling + else: + rate = 0.0 # held where it is, at a boundary or a bound, until something changes + horizon = end + change = usage.next_change_after(moment) + if change is not None and change < horizon: + horizon = change + target = None + if rate > 0: + target = next((b for b in boundaries if b > fill_level), None) + elif rate < 0: + target = next((b for b in reversed(boundaries) if b < fill_level), None) + if target is not None: + # Compared in seconds first: a slow rate can put a boundary further off than a timedelta can say. + seconds_to_target = (target - fill_level) / rate + if seconds_to_target < (horizon - moment).total_seconds(): + fill_level = target + moment = moment + pd.Timedelta(seconds=seconds_to_target) + continue + fill_level += rate * (horizon - moment).total_seconds() + moment = horizon + return min(max(fill_level, lowest), highest) def fm_schedule_to_instructions( @@ -128,15 +211,30 @@ def fm_schedule_to_instructions( initial_fill_level: float, logger: logging.Logger = LOGGER, fill_level_bounds: tuple[float, float] | None = None, + leakage_behaviour: FRBCLeakageBehaviour | None = None, + usage_forecast: FRBCUsageForecast | None = None, ) -> List[FRBCInstruction]: """Turn a FlexMeasures schedule into the instructions that follow it. + Each step is answered with the operation mode, and the factor within it, that draws the power the schedule asks for + at the fill level the storage is then at, so a running fill level is kept alongside, under the contract of compute_next_fill_level. + An instruction is sent where that answer changes, which can happen at a constant power too, + where the fill level passes from one element of a mode into another. + + :param schedule: a schedule as FlexMeasures returns it, or a frame indexed by time with a "schedule" column. + A frame may also carry a "usage_forecast" column, per second as S2 states it, + which then takes the place of usage_forecast. :param fill_level_bounds: the fill levels this conversion holds its own running fill level between, in the resource manager's units. These have to be the hard bounds the schedule was planned against, or a step is answered with the operation mode for a fill level the scheduler never planned. Without them the declared fill-level range is used, which is right only for a caller that asked for a schedule against that same range. + :param leakage_behaviour: what the storage loses by itself, as the resource manager last said. + :param usage_forecast: what is drawn from the storage, as the resource manager last forecast it. + + The step is the schedule's own: a schedule from FlexMeasures states it as its duration over its number of values, + which is the resolution of the sensor it was asked for, whatever resolution the caller plans on. """ if fill_level_bounds is not None and fill_level_bounds[0] > fill_level_bounds[1]: @@ -151,85 +249,102 @@ def fm_schedule_to_instructions( return [] if isinstance(schedule, dict): + values = schedule["values"] + if len(values) == 0: + return [] start = pd.Timestamp(schedule["start"]) - schedule_duration = pd.Timedelta(schedule["duration"]) - idx = pd.DatetimeIndex( - pd.date_range( - start=start, - end=start + schedule_duration - timedelta(minutes=15), - freq="15min", - ) + step = pd.Timedelta(schedule["duration"]) / len(values) + idx = pd.date_range(start=start, periods=len(values), freq=step) + schedule = pd.Series(values, index=idx, name="schedule").to_frame() + else: + # A frame of one row has no later step for its fill level to matter to. + step = ( + pd.Timedelta(schedule.index[1] - schedule.index[0]) + if len(schedule) > 1 + else pd.Timedelta(0) ) - schedule = pd.Series(schedule["values"], index=idx, name="schedule").to_frame() - - previous_power = None - instructions = [] - - actuator = system_description.actuators[0] if len(system_description.actuators) != 1: raise NotImplementedError( f"This CEM only supports 1 actuator but" f"{len(system_description.actuators)} were provided" ) - + actuator = system_description.actuators[0] operation_modes: list[FRBCOperationMode] = actuator.operation_modes + modes_by_id = {str(mode.id): mode for mode in operation_modes} + + # Held between the hard bounds the schedule was planned against, + # so that each step is answered with the operation mode for the fill level the scheduler planned. + # Falling back to the declared range keeps every other caller as it was. + if fill_level_bounds is None: + fill_level_range: NumberRange = system_description.storage.fill_level_range + fill_level_min = fill_level_range.start_of_range + fill_level_max = fill_level_range.end_of_range + else: + fill_level_min, fill_level_max = fill_level_bounds + instructions: list[FRBCInstruction] = [] + last_instructed: tuple[str, float] | None = None + forecast_usage = UsageProfile(usage_forecast) fill_level = initial_fill_level - - deltaT = timedelta(minutes=15) / timedelta(hours=1) + has_usage_column = "usage_forecast" in schedule.columns for timestamp, row in schedule.iterrows(): power = row["schedule"] - usage = row.get("usage_forecast", 0) - - if previous_power is None or not isclose(previous_power, power): - # Convert from power to fill rate - results = [ - (om, *power_to_fill_rate_with_metrics(om, power, fill_level)) - for om in operation_modes - ] - - # Step 1: minimize fill-level penalty (primary) - min_fill_penalty = min(r[2] for r in results) - fill_level_candidates = [r for r in results if r[2] == min_fill_penalty] - - # Step 2: among those, minimize power penalty (secondary) - min_power_penalty = min(r[3] for r in fill_level_candidates) - power_candidates = [ - r for r in fill_level_candidates if r[3] == min_power_penalty - ] - - # Step 3: among those, pick the highest efficiency - best = max(power_candidates, key=lambda r: r[4]) - - ( - best_operation_mode, - best_fill_rate, - best_fill_penalty, - best_power_penalty, - best_efficiency, - best_element, - ) = best - - logger.debug( - "Selected FRBC operation mode '%s' (element='%s') for power=%.3f, fill_level=%.3f: " - "fill_rate=%.6f, fill_penalty=%.6f, power_penalty=%.6f, efficiency=%.6f. Details: %s", - om_label(best_operation_mode), - getattr(best_element, "id", "unknown"), - power, - fill_level, - best_fill_rate, - best_fill_penalty, - best_power_penalty, - best_efficiency, - explain_choice(results, best), - ) - operation_mode_factor = op_mode_compute_factor( - best_element, fill_rate=best_fill_rate, logger=logger - ) + # The mode and factor for this power at this fill level, every step, + # since the element that applies can change while the power does not. + results = [ + (om, *power_to_fill_rate_with_metrics(om, power, fill_level)) + for om in operation_modes + ] + + # Step 1: minimize fill-level penalty (primary) + min_fill_penalty = min(r[2] for r in results) + fill_level_candidates = [r for r in results if r[2] == min_fill_penalty] + + # Step 2: among those, minimize power penalty (secondary) + min_power_penalty = min(r[3] for r in fill_level_candidates) + power_candidates = [ + r for r in fill_level_candidates if r[3] == min_power_penalty + ] + # Step 3: among those, pick the highest efficiency + best = max(power_candidates, key=lambda r: r[4]) + + ( + best_operation_mode, + best_fill_rate, + best_fill_penalty, + best_power_penalty, + best_efficiency, + best_element, + operation_mode_factor, + ) = best + + logger.debug( + "Selected FRBC operation mode '%s' for power=%.3f, fill_level=%.3f: " + "factor=%.6f, fill_rate=%.6f, fill_penalty=%.6f, power_penalty=%.6f, efficiency=%.6f. Details: %s", + om_label(best_operation_mode), + power, + fill_level, + operation_mode_factor, + best_fill_rate, + best_fill_penalty, + best_power_penalty, + best_efficiency, + explain_choice(results, best), + ) + + # The factor is the one that draws the power asked for, taken from the power range itself. + # It used to be worked back out of the fill rate instead, + # which fails wherever the fill-rate range is a single value and the power range is not: + # asked for 1200 W from a mode running 800 to 1200 W at one fill rate, that asked for 800 W. + answer = (str(best_operation_mode.id), operation_mode_factor) + if last_instructed is None or not ( + answer[0] == last_instructed[0] + and isclose(answer[1], last_instructed[1], abs_tol=FACTOR_TOLERANCE) + ): instruction = FRBCInstruction( message_id=get_unique_id(), id=get_unique_id(), @@ -243,29 +358,43 @@ def fm_schedule_to_instructions( f"Instruction created: at {timestamp} set {actuator.diagnostic_label if isinstance(actuator.diagnostic_label, str) else actuator} to {best_operation_mode.diagnostic_label if isinstance(best_operation_mode.diagnostic_label, str) else best_operation_mode} with factor {operation_mode_factor}" ) instructions.append(instruction) - - # Update fill level - fill_level = compute_next_fill_level( - fill_level=fill_level, - fill_rate=best_fill_rate, # S2-derived fill rate - leakage_behaviour=getattr(actuator, "leakage_behaviour", None), - deltaT=deltaT, + last_instructed = answer + + # A gap in a usage column, as FlexMeasures returns for a moment it holds nothing for, is no usage known, not NaN: + # a NaN fill level would be held at the top of the bounds, and answered there from then on. + usage = forecast_usage + if has_usage_column: + usage = UsageProfile( + rate=( + 0.0 + if pd.isna(row["usage_forecast"]) + else float(row["usage_forecast"]) + ) + ) + fill_level = follow_operation_mode( + fill_level, + best_operation_mode, + operation_mode_factor, + pd.Timestamp(timestamp), + step, + leakage_behaviour=leakage_behaviour, usage=usage, + bounds=(fill_level_min, fill_level_max), ) - # Hold the running fill level between the hard bounds the schedule was planned against, - # so that each step is answered with the operation mode for the fill level the scheduler planned. - # Falling back to the declared range keeps every other caller as it was. - if fill_level_bounds is None: - fill_level_range: NumberRange = system_description.storage.fill_level_range - fill_level_min = fill_level_range.start_of_range - fill_level_max = fill_level_range.end_of_range - else: - fill_level_min, fill_level_max = fill_level_bounds - fill_level = max(fill_level_min, min(fill_level_max, fill_level)) - - previous_power = power - + if logger.isEnabledFor(logging.DEBUG): + # The whole batch in one line, by mode name, which the per-step lines above do not give. + logger.debug( + "Instructions: %s", + [ + ( + instruction.execution_time.isoformat(), + om_label(modes_by_id[str(instruction.operation_mode)]), + round(instruction.operation_mode_factor, 6), + ) + for instruction in instructions + ], + ) return instructions @@ -317,9 +446,12 @@ def fill_rate_in_watts(fill_rate: float, energy_unit: str) -> float: def operation_mode_conversion_efficiency( - operation_mode: FRBCOperationMode, fill_level_scale: float, energy_unit: str + operation_mode: FRBCOperationMode, + fill_level_scale: float, + energy_unit: str, + fill_level: float, ) -> float | None: - """How much energy an operation mode puts into the storage for each unit of power it draws. + """How much energy an operation mode puts into the storage for each unit of power it draws, at a fill level. An operation mode states a fill rate and a power range, so the ratio between the two is what the mode converts drawn power into stored energy at: @@ -327,31 +459,37 @@ def operation_mode_conversion_efficiency( Both sides are taken in watts, since FlexMeasures asks for a charging efficiency as a dimensionless ratio, and for a thermal actuator that ratio is well above one. - The ratio is taken at the top of the last element's range, rather than as the slope over that range. + A mode may convert differently at different fill levels, as a heat pump does when its buffer warms, + and FlexMeasures takes one charging efficiency for a whole schedule. + The element that applies at the fill level the schedule starts from answers, + since the start of a schedule is the part that runs before the next storage status asks for a new one. + Not the lowest element over the whole range: + that overstates what the actuator needs to fill the storage, so the plan overshoots its upper band and buys more than it uses, + and against a usage the plan cannot relax it can turn a problem the actuator can meet into one the scheduler finds infeasible. + + The ratio is taken at the top of that element's range, rather than as the slope over it. An actuator may well declare degenerate ranges, where a mode runs at one power and one fill rate only (the heaters in our own smoke tests state on as [883.7, 883.7] W), and a slope is zero over such a range while the ratio is the efficiency the mode really runs at. - Where an efficiency varies over a mode's range, this is the one it reaches at full tilt. :param fill_level_scale: converts a fill level, and so a fill rate, from the resource manager's own units into the unit it is recorded in. :param energy_unit: the unit a fill level is recorded in, which the fill rate is per second of. - :returns: the efficiency, or None where the mode states no power to convert. - An idle mode draws nothing, and a mode that only feeds power back is discharging, - which FlexMeasures takes as a separate discharging efficiency that this does not state. + :param fill_level: where the schedule starts, in the resource manager's own units, as the elements' ranges are. + :returns: the efficiency, or None where the element that applies converts no drawn power into stored energy. + It converts nothing where it draws no power, as an idle mode does, + where it only feeds power back, which FlexMeasures takes as a separate discharging efficiency, + or where it draws power while the storage does not fill, as a standby or circulation mode does: + no single ratio states that, and a ratio of zero would tell the scheduler the storage cannot be filled at all. """ - if not operation_mode.elements: - return None - element = operation_mode.elements[-1] - if not element.power_ranges: - return None + element = element_for_fill_level(operation_mode.elements, fill_level) # todo: an element states one power range per commodity quantity, # and no distinction is made between them here, # as the rest of this handler does not make one either (see power_to_fill_rate_with_metrics). power = element.power_ranges[0].end_of_range - if power <= 0: - return None fill_rate = element.fill_rate.end_of_range * fill_level_scale + if power <= 0 or fill_rate <= 0: + return None return fill_rate_in_watts(fill_rate, energy_unit) / power @@ -397,7 +535,7 @@ def power_to_fill_rate_with_metrics( operation_mode: FRBCOperationMode, power: float, fill_level: float, -) -> tuple[float, float, float, float, FRBCOperationModeElement]: +) -> tuple[float, float, float, float, FRBCOperationModeElement, float]: """ Evaluate an operation mode for given power and fill level. @@ -407,23 +545,14 @@ def power_to_fill_rate_with_metrics( power_penalty: float efficiency: float element: FRBCOperationModeElement that produced fill_rate + factor: the operation mode factor that draws this power, which is what an instruction asks for """ - elements = operation_mode.elements - - # --- 1. Select element with minimal fill-level penalty --- - best_element, fill_penalty = min( - ( - ( - e, - clamp_distance( - fill_level, - e.fill_level_range.start_of_range, - e.fill_level_range.end_of_range, - ), - ) - for e in elements - ), - key=lambda x: x[1], + # --- 1. Select the element that applies at this fill level --- + best_element = element_for_fill_level(operation_mode.elements, fill_level) + fill_penalty = clamp_distance( + fill_level, + best_element.fill_level_range.start_of_range, + best_element.fill_level_range.end_of_range, ) # --- 2. Select power range with minimal power penalty --- @@ -440,7 +569,13 @@ def power_to_fill_rate_with_metrics( pr_end = power_range.end_of_range if pr_end == pr_start: - factor = 0.0 + # Every factor draws this one power, so the power cannot say which factor is meant. + # The top of the fill-rate range is asked for, which is where operation_mode_conversion_efficiency + # declares the efficiency FlexMeasures plans with, so the plan and the instruction agree. + # Where the fill-rate range is a single value too, the factor makes no difference and stays at zero. + fr_start = best_element.fill_rate.start_of_range + fr_end = best_element.fill_rate.end_of_range + factor = 0.0 if fr_end == fr_start else 1.0 efficiency = 0.0 else: factor = (power - pr_start) / (pr_end - pr_start) @@ -454,7 +589,7 @@ def power_to_fill_rate_with_metrics( fr_end = best_element.fill_rate.end_of_range fill_rate = fr_start + factor * (fr_end - fr_start) - return fill_rate, fill_penalty, power_penalty, efficiency, best_element + return fill_rate, fill_penalty, power_penalty, efficiency, best_element, factor def clamp_distance(value: float, start: float, end: float) -> float: @@ -465,17 +600,68 @@ def clamp_distance(value: float, start: float, end: float) -> float: return 0.0 +ElementWithFillLevelRange = TypeVar( + "ElementWithFillLevelRange", FRBCOperationModeElement, FRBCLeakageBehaviourElement +) + + +def element_for_fill_level( + elements: Sequence[ElementWithFillLevelRange], fill_level: float +) -> ElementWithFillLevelRange: + """The element that applies at a fill level, whatever order the elements were listed in. + + S2 lets the elements of an operation mode, or of a leakage behaviour, cover the fill-level range in pieces, in any order. + s2python checks that an operation mode's elements are contiguous, on a sorted copy, and leaves the list as it was sent; + it checks nothing of the kind for a leakage behaviour, whose elements may leave gaps or overlap. + + Two neighbouring elements share a boundary, which belongs to the upper of the two: + of every element whose range holds the fill level, the one starting highest applies, and of two starting together, the one reaching higher. + A range that is a single point therefore applies only at the very top. + The standard says a range starts below where it ends, so such a range is outside it, though s2python lets it through. + Two elements with the same range are ambiguous, which overlapping elements always are, and the first listed answers. + + A fill level outside the range the elements cover is answered as the nearest end of that range is, + since a storage may be outside its declared range, and the answer should not jump as it crosses that end. + A fill level in a gap between elements is answered by the nearer, and on a tie the lower. + + :raises ValueError: for a fill level that is not a number, which no element applies at. + """ + if math.isnan(fill_level): + raise ValueError("No element applies at a fill level that is not a number.") + + def bounds(e: ElementWithFillLevelRange) -> tuple[float, float]: + return e.fill_level_range.start_of_range, e.fill_level_range.end_of_range + + covered_from = min(bounds(e)[0] for e in elements) + covered_to = max(bounds(e)[1] for e in elements) + clamped = min(max(fill_level, covered_from), covered_to) + holding = [e for e in elements if bounds(e)[0] <= clamped <= bounds(e)[1]] + if holding: + return max(holding, key=bounds) + return min( + elements, key=lambda e: (clamp_distance(clamped, *bounds(e)), *bounds(e)) + ) + + def om_label(operation_mode: FRBCOperationMode) -> str: label = getattr(operation_mode, "diagnostic_label", None) - return label if isinstance(label, str) else operation_mode.id + return label if isinstance(label, str) else str(operation_mode.id) def explain_choice( results: list[ - tuple[FRBCOperationMode, float, float, float, float, FRBCOperationModeElement] + tuple[ + FRBCOperationMode, + float, + float, + float, + float, + FRBCOperationModeElement, + float, + ] ], best: tuple[ - FRBCOperationMode, float, float, float, float, FRBCOperationModeElement + FRBCOperationMode, float, float, float, float, FRBCOperationModeElement, float ], ) -> str: """ @@ -488,11 +674,12 @@ def explain_choice( best_power_penalty, best_efficiency, best_element, + _best_factor, ) = best lines: list[str] = [] - for om, fr, fill_pen, power_pen, eff, element in results: + for om, fr, fill_pen, power_pen, eff, element, _factor in results: label = om_label(om) element_label = getattr(element, "id", "unknown") diff --git a/src/flexmeasures_client/s2/control_types/__init__.py b/src/flexmeasures_client/s2/control_types/__init__.py index 09f526f3..c708c2b2 100644 --- a/src/flexmeasures_client/s2/control_types/__init__.py +++ b/src/flexmeasures_client/s2/control_types/__init__.py @@ -44,15 +44,20 @@ async def send_message(self, message: BaseModel) -> None: await self._sending_queue.put(message) async def close(self): - """Release any resources / stop recurring tasks for this handler. + """Stop this handler's work in flight, when the CEM closes as its websocket tears down. - Default no-op so CEM.close(), which calls close() on every registered handler when a - websocket tears down, works for any control-type handler. - Subclasses that own recurring tasks override this. - Without it, a handler lacking close() raised AttributeError inside CEM.close(), + Defined here so that CEM.close(), which calls close() on every registered handler, works for any of them: + a handler lacking close() raised AttributeError inside CEM.close(), killing the CEM's request handler and hanging the resource manager. """ self._logger.debug(f"Closing {self.__class__.__name__} handler") + # Work still in flight would otherwise go on after the session has ended, + # planning for a resource manager that is no longer there and racing the session that replaces it. + # That includes recording what the resource manager last reported, which is stopped too: + # a reading cut off mid-post is reported again by a resource manager that reconnects. + # Not every handler keeps background tasks, so a handler without them has nothing to stop. + for task in list(getattr(self, "background_tasks", ())): + task.cancel() @register(InstructionStatusUpdate) def handle_instruction_status_update(self, message: InstructionStatusUpdate): diff --git a/src/flexmeasures_client/s2/control_types/translations.py b/src/flexmeasures_client/s2/control_types/translations.py index a51a14be..e61744db 100644 --- a/src/flexmeasures_client/s2/control_types/translations.py +++ b/src/flexmeasures_client/s2/control_types/translations.py @@ -1,5 +1,3 @@ -# flake8: noqa - from datetime import timedelta import numpy as np @@ -17,11 +15,15 @@ "Install it using `pip install flexmeasures-client[s2]`." ) +from flexmeasures_client.s2.control_types.FRBC.utils import element_for_fill_level + def leakage_behaviour_to_storage_efficiency( message: FRBCLeakageBehaviour, resolution=timedelta(minutes=15), fill_level_scale: float = 1, + fill_level: float | None = None, + share_of: float | None = None, ) -> float: """ Convert a FRBC.LeakageBehaviour message into a FlexMeasures compatible storage efficiency. @@ -32,7 +34,7 @@ def leakage_behaviour_to_storage_efficiency( due to leakage within the given range of the fill level. This is defined as a function of the fill level. - Storage Efficiency: percentage of the storage that remains after one time period. + Storage Efficiency: percentage of the storage that remains after one time step of the given resolution. Example: @@ -56,17 +58,32 @@ def leakage_behaviour_to_storage_efficiency( """ - last_element = message.elements[-1] + # FlexMeasures takes one storage efficiency here, for the whole window it covers. + # Where the fill level is known, the element that applies at it answers, as for a charging efficiency. + # Where it is not, the leakiest answers, looked for rather than assumed to come last, + # since S2 does not fix the order the elements are listed in, and the leakiest is not always the top one. + if fill_level is not None: + element = element_for_fill_level(message.elements, fill_level) + else: + element = max(message.elements, key=lambda e: e.leakage_rate) max_fill_level = max( e.fill_level_range.end_of_range * fill_level_scale for e in message.elements ) - leakage = last_element.leakage_rate * fill_level_scale - base_resolution = timedelta(hours=1) # TODO: use timedelta(seconds=1) - - # Discuss conversions - storage_efficiency = 100 * (1 - (leakage / max_fill_level)) ** ( - resolution / base_resolution + leakage = element.leakage_rate * fill_level_scale + # S2 states a leakage rate per second, so what remains after one second is compounded over the seconds in a step. + # This used to compound it over the hours in a step instead, + # which declared a buffer losing 1.5 kWh of 10 a day as holding its heat to within 0.002% over twelve hours, + # where it loses about 7%, and had the scheduler pre-charge in a cheap hour on the strength of it. + # FlexMeasures applies a storage efficiency to the stock as it stands, so the loss is a share of a fill level: + # of share_of where given, as the fill level a schedule starts from is, + # and otherwise of the top of the range, which understates it at lower fill levels. + reference = ( + share_of * fill_level_scale + if share_of is not None and share_of > 0 + else max_fill_level ) + seconds_per_step = pd.Timedelta(resolution) / pd.Timedelta(seconds=1) + storage_efficiency = 100 * max(0.0, 1 - leakage / reference) ** seconds_per_step return storage_efficiency @@ -75,7 +92,7 @@ def unevenly_ts_to_evenly( start: pd.Timestamp, values: list[float], durations: list[pd.Timedelta], - target_resolution: str, + target_resolution: str | timedelta, strategy="mean", ) -> pd.Series: """ @@ -132,7 +149,7 @@ def unevenly_ts_to_evenly( def translate_usage_forecast_to_fm( usage_forecast: FRBCUsageForecast, - resolution: str = "1h", + resolution: str | timedelta = "1h", strategy: str = "mean", fill_level_scale: float = 1, ) -> pd.Series: @@ -166,7 +183,7 @@ def translate_usage_forecast_to_fm( def translate_fill_level_target_profile( fill_level_target_profile: FRBCFillLevelTargetProfile, - resolution: str = "1h", + resolution: str | timedelta = "1h", fill_level_scale: float = 1, ) -> tuple[pd.Series, pd.Series]: """ diff --git a/src/flexmeasures_client/s2/measurements.py b/src/flexmeasures_client/s2/measurements.py new file mode 100644 index 00000000..5f154514 --- /dev/null +++ b/src/flexmeasures_client/s2/measurements.py @@ -0,0 +1,201 @@ +"""Recording what a resource manager measures on FlexMeasures sensors, one value per step of each sensor's resolution.""" + +from __future__ import annotations + +import asyncio +from collections import defaultdict +from datetime import datetime, timedelta +from logging import Logger, getLogger + +import pandas as pd + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.exceptions import ( + ContentTypeError, + InsufficientServerVersionError, + JobFailedError, + WrongAPIVersionError, +) + +_LOGGER = getLogger(__name__) + +#: The unit S2 states every power in, measured ones included. +S2_POWER_UNIT = "W" + +#: How long a recorded step is remembered, so that a resource manager sending it again is not recorded twice. +REMEMBERED_FOR = timedelta(days=7) + +#: How often a step is tried before it is given up on, and how long to wait between tries. +ATTEMPTS = 3 +RETRY_AFTER = timedelta(seconds=30) + +#: What a post can fail with, short of a defect in this code. +POST_FAILURES = ( + ValueError, + ConnectionError, + ContentTypeError, + JobFailedError, + InsufficientServerVersionError, + WrongAPIVersionError, +) + + +class MeasurementRecorder: + """Records measured power on a sensor as the average of what was measured in each step of its resolution. + + A step is recorded once it is over by the CEM's clock, as it stands when the step's last measurement arrives, + with the moment it ended as the moment it became known, + so a measurement is a belief formed at the end of what it measures, rather than a forecast of it. + Steps are taken by the time each measurement is stamped with, not by when it arrives, + since a co-simulation's resource manager sends a window's measurements together once the window has passed, + each stamped at the start of its quarter hour, while a live one sends them as they are taken, + in which case a step is recorded with the first measurement after it is over. + + Recording happens in the background, so that a resource manager is not kept waiting for FlexMeasures + before its measurement is acknowledged, one post at a time per sensor. + A step that could not be recorded is tried again a while later, and given up on after a few tries. + + Made once per server and handed to each CEM, it outlives the reconnections of a resource manager, + which a co-simulation makes every round, and so remembers what was recorded: + a step sent again is not recorded again, and the first value recorded for a step stands. + A sensor is meant to be one resource manager's: one that several share has their measurements averaged together. + """ + + def __init__(self, fm_client: FlexMeasuresClient, logger: Logger | None = None): + self._fm_client = fm_client + self._logger = logger or _LOGGER + self._resolutions: dict[int, timedelta] = {} + self._pending: dict[int, dict[datetime, list[float]]] = defaultdict(dict) + self._attempts: dict[int, dict[datetime, int]] = defaultdict(dict) + #: By sensor, the steps recorded, in the order they were, which is roughly that of time. + self._recorded: dict[int, dict[datetime, None]] = defaultdict(dict) + self._locks: dict[int, asyncio.Lock] = defaultdict(asyncio.Lock) + self._tasks: set[asyncio.Task] = set() + + async def resolution(self, sensor_id: int) -> timedelta: + """The sensor's resolution, asked once, since it is a property of how the sensor was set up. + + A caller taking several measurements at once asks this for each sensor first, + so that none is taken where one of them cannot be. + """ + if sensor_id not in self._resolutions: + sensor = await self._fm_client.get_sensor(sensor_id) + resolution = ( + sensor.get("event_resolution") if isinstance(sensor, dict) else None + ) + if resolution is None: + raise ValueError( + f"Sensor {sensor_id} states no resolution, so its steps are unknown." + ) + self._resolutions[sensor_id] = pd.Timedelta(resolution).to_pytimedelta() + return self._resolutions[sensor_id] + + async def add( + self, sensor_id: int, measured_at: datetime, watts: float, now: datetime + ) -> None: + """Take one measurement, and record in the background every step of this sensor that is over by now.""" + resolution = await self.resolution(sensor_id) + step = step_of(measured_at, resolution) + if step in self._recorded[sensor_id]: + self._logger.debug( + "Not recording %s W on sensor %s for %s again.", watts, sensor_id, step + ) + else: + self._pending[sensor_id].setdefault(step, []).append(watts) + self._in_background(self.record(sensor_id, now)) + + async def record(self, sensor_id: int, now: datetime) -> None: + """Record every pending step of this sensor that is over by now, each in a request of its own. + + Each with its own end as the moment it became known, which one request for several steps cannot say. + One at a time per sensor, so a step is not posted twice, nor taken from under a post in flight. + """ + async with self._locks[sensor_id]: + resolution = await self.resolution(sensor_id) + over = sorted( + step for step in self._pending[sensor_id] if step + resolution <= now + ) + failed = False + for step in over: + samples = self._pending[sensor_id][step] + try: + await self._fm_client.post_sensor_data( + sensor_id, + start=step, + duration=resolution, + values=[sum(samples) / len(samples)], + unit=S2_POWER_UNIT, + prior=step + resolution, + ) + except POST_FAILURES as exc: + failed = True + attempts = self._attempts[sensor_id].get(step, 0) + 1 + if attempts < ATTEMPTS: + self._attempts[sensor_id][step] = attempts + self._logger.warning( + "Could not record measured power on sensor %s for %s; trying again later: %s", + sensor_id, + step, + exc, + ) + continue + self._logger.error( + "Giving up recording %s W of measured power on sensor %s for %s after %s tries: %s", + samples, + sensor_id, + step, + attempts, + exc, + ) + del self._pending[sensor_id][step] + self._attempts[sensor_id].pop(step, None) + self._recorded[sensor_id][step] = None + self._forget_before(sensor_id, now - REMEMBERED_FOR) + if failed: + self._in_background(self._retry(sensor_id, now)) + + async def _retry(self, sensor_id: int, now: datetime) -> None: + await asyncio.sleep(RETRY_AFTER.total_seconds()) + await self.record(sensor_id, now) + + async def settle(self) -> None: + """Wait for every recording in the background to finish, retries included.""" + while self._tasks: + await asyncio.gather(*list(self._tasks), return_exceptions=True) + + def _in_background(self, recording) -> None: + task = asyncio.create_task(recording) + self._tasks.add(task) + task.add_done_callback(self._done) + + def _done(self, task: asyncio.Task) -> None: + self._tasks.discard(task) + if not task.cancelled() and task.exception() is not None: + self._logger.error( + "Recording measured power failed.", exc_info=task.exception() + ) + + def _forget_before(self, sensor_id: int, cutoff: datetime) -> None: + """Forget recorded steps from before the cutoff, oldest first, so that this costs little when nothing is old.""" + recorded = self._recorded[sensor_id] + while recorded: + oldest = next(iter(recorded)) + if oldest >= cutoff: + break + del recorded[oldest] + + +def step_of(moment: datetime, resolution: timedelta) -> datetime: + """The start of the step a moment falls in, on the moment's own clock, as FlexMeasures floors it. + + In the repeated hour of an autumn DST change a boundary exists twice, and the step is the later of the two + that is not after the moment; taken from the moment's pass alone, flooring cannot always be done at all. + """ + if not resolution: + return moment + at = pd.Timestamp(moment) + candidates = [ + at.floor(resolution, ambiguous=first_pass, nonexistent="shift_forward") + for first_pass in (True, False) + ] + return max(c for c in candidates if c <= at).to_pydatetime() diff --git a/src/flexmeasures_client/s2/script/__init__.py b/src/flexmeasures_client/s2/script/__init__.py new file mode 100644 index 00000000..c777fdcf --- /dev/null +++ b/src/flexmeasures_client/s2/script/__init__.py @@ -0,0 +1 @@ +"""Example scripts: a CEM server and a resource manager that talk S2 to each other, run by path as the CEM documentation shows.""" diff --git a/src/flexmeasures_client/s2/script/websockets_client.py b/src/flexmeasures_client/s2/script/websockets_client.py index 510062fe..b73ce4c9 100644 --- a/src/flexmeasures_client/s2/script/websockets_client.py +++ b/src/flexmeasures_client/s2/script/websockets_client.py @@ -1,6 +1,21 @@ +"""An example resource manager: a heat pump keeping a hot-water buffer warm, speaking S2's FRBC to a CEM. + +Run it by path, next to the example CEM server, as the CEM documentation shows. +It declares what the heat pump can do and what its buffer needs, then simulates that buffer: +it follows each instruction from the moment it is due, and reports its fill level every so often, +which is what asks the CEM for a new schedule. + +The buffer is simulated from S2's own definitions, and not with the CEM's code, +so that it is a second opinion on where the instructions take the storage rather than a copy of the first. +""" + +from __future__ import annotations + import asyncio -from datetime import datetime, timezone -from zoneinfo import ZoneInfo +import os +from collections.abc import Callable +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone import aiohttp from s2python.common import ( @@ -10,30 +25,288 @@ Duration, EnergyManagementRole, Handshake, + InstructionStatus, + InstructionStatusUpdate, NumberRange, PowerRange, ReceptionStatus, ReceptionStatusValues, ResourceManagerDetails, + RevokeObject, Role, RoleType, + Transition, ) from s2python.frbc import ( FRBCActuatorDescription, FRBCActuatorStatus, + FRBCFillLevelTargetProfile, + FRBCFillLevelTargetProfileElement, FRBCInstruction, + FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, FRBCOperationMode, FRBCOperationModeElement, FRBCStorageDescription, FRBCStorageStatus, FRBCSystemDescription, + FRBCUsageForecast, + FRBCUsageForecastElement, ) from flexmeasures_client.s2.utils import get_unique_id +#: The range the buffer's fill level should stay in, in kilowatt hours of stored heat. +FILL_LEVEL_RANGE = (2.0, 10.0) +#: Below this fill level the water is cold enough for the heat pump to run at a COP of 4; above it, 3. +COLD_BELOW = 6.0 +#: What the buffer holds when the simulation starts, and what it can physically hold at most. +STARTING_FILL_LEVEL = 5.0 +PHYSICAL_CAPACITY = 12.0 +#: How often the simulated buffer reports its fill level, in seconds of wall-clock time. +STATUS_INTERVAL = float(os.getenv("S2_EXAMPLE_STATUS_INTERVAL", "300")) +#: How often the forecasts are sent again, each covering the day from then on, so that planning never runs out of them. +FORECAST_INTERVAL = float(os.getenv("S2_EXAMPLE_FORECAST_INTERVAL", "3600")) +ELECTRIC_POWER = CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC + + +def heat(watts: float) -> float: + """A rate of heat in watts as a fill rate in kilowatt hours per second, which is how S2 states a fill rate.""" + return watts / 3.6e6 + + +def per_second(kwh_per_hour: float) -> float: + return kwh_per_hour / 3600 + + +def _element( + lowest: float, + highest: float, + power: tuple[float, float], + heat_in_watts: tuple[float, float], +) -> FRBCOperationModeElement: + return FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=lowest, end_of_range=highest), + fill_rate=NumberRange( + start_of_range=heat(heat_in_watts[0]), end_of_range=heat(heat_in_watts[1]) + ), + power_ranges=[ + PowerRange( + start_of_range=power[0], + end_of_range=power[1], + commodity_quantity=ELECTRIC_POWER, + ) + ], + ) + + +def operation_modes() -> list[FRBCOperationMode]: + """Idle, or heating at anything from 400 to 1500 W, at a COP of 4 while the buffer is cold and 3 once it is warm.""" + idle = FRBCOperationMode( + id=get_unique_id(), + diagnostic_label="idle", + elements=[_element(0, PHYSICAL_CAPACITY, (0, 0), (0, 0))], + abnormal_condition_only=False, + ) + heating = FRBCOperationMode( + id=get_unique_id(), + diagnostic_label="heating", + elements=[ + _element(0, COLD_BELOW, (400, 1500), (4 * 400, 4 * 1500)), + _element(COLD_BELOW, PHYSICAL_CAPACITY, (400, 1500), (3 * 400, 3 * 1500)), + ], + abnormal_condition_only=False, + ) + return [idle, heating] + + +def system_description( + modes: list[FRBCOperationMode], valid_from: datetime +) -> FRBCSystemDescription: + # S2 lists the transitions an actuator can make, so the heat pump says it can go from either mode to the other, + # at once and at no cost. + transitions = [ + Transition( + id=get_unique_id(), + from_=a.id, + to=b.id, + start_timers=[], + blocking_timers=[], + abnormal_condition_only=False, + ) + for a in modes + for b in modes + if a.id != b.id + ] + return FRBCSystemDescription( + message_id=get_unique_id(), + valid_from=valid_from, + actuators=[ + FRBCActuatorDescription( + id=get_unique_id(), + diagnostic_label="heat pump", + supported_commodities=[Commodity.ELECTRICITY], + operation_modes=modes, + transitions=transitions, + timers=[], + ) + ], + storage=FRBCStorageDescription( + diagnostic_label="hot-water buffer", + fill_level_label="kWh of stored heat", + provides_leakage_behaviour=True, + provides_fill_level_target_profile=True, + provides_usage_forecast=True, + fill_level_range=NumberRange( + start_of_range=FILL_LEVEL_RANGE[0], end_of_range=FILL_LEVEL_RANGE[1] + ), + ), + ) + + +def leakage_behaviour(valid_from: datetime) -> FRBCLeakageBehaviour: + """A warmer buffer loses more: 0.8 kWh a day while cold, 1.5 kWh a day once warm.""" + return FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=valid_from, + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=COLD_BELOW), + leakage_rate=per_second(0.8 / 24), + ), + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange( + start_of_range=COLD_BELOW, end_of_range=PHYSICAL_CAPACITY + ), + leakage_rate=per_second(1.5 / 24), + ), + ], + ) + + +def expected_usage(hour: int) -> float: + """Hot water drawn in a given hour of the day, in kWh: showers in the morning, washing up in the evening.""" + return {7: 3.0, 19: 1.0, 20: 1.0}.get(hour, 0.1) + + +def usage_forecast(start: datetime) -> FRBCUsageForecast: + """A day of hot-water use, hour by hour, from the hour start falls in.""" + start = start.replace(minute=0, second=0, microsecond=0) + return FRBCUsageForecast( + message_id=get_unique_id(), + start_time=start, + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=3600 * 1000), + usage_rate_expected=per_second(expected_usage((start.hour + h) % 24)), + ) + for h in range(24) + ], + ) + + +def fill_level_target_profile(start: datetime) -> FRBCFillLevelTargetProfile: + """Kept within the declared range, and at least 6 kWh in the hour before the morning showers.""" + start = start.replace(minute=0, second=0, microsecond=0) + return FRBCFillLevelTargetProfile( + message_id=get_unique_id(), + start_time=start, + elements=[ + FRBCFillLevelTargetProfileElement( + duration=Duration(root=3600 * 1000), + fill_level_range=NumberRange( + start_of_range=6.0 if (start.hour + h) % 24 == 6 else 3.0, + end_of_range=FILL_LEVEL_RANGE[1], + ), + ) + for h in range(24) + ], + ) + + +@dataclass +class Buffer: + """The simulated buffer: where its fill level goes, following S2's definitions. + + S2 states every rate per second, in the fill level's own units: + a fill rate raises the fill level, and usage and leakage lower it. + An operation mode's fill rate is interpolated between the ends of its element's range by the factor, + from the element whose fill-level range holds the fill level. + """ + + modes: dict[str, FRBCOperationMode] + leakage: FRBCLeakageBehaviour + fill_level: float = STARTING_FILL_LEVEL + mode_id: str | None = None + factor: float = 0.0 + at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + + @staticmethod + def _holding(elements, fill_level: float): + """The element whose range holds the fill level, the upper one at a shared boundary, the nearest outside.""" + lowest = min(e.fill_level_range.start_of_range for e in elements) + highest = max(e.fill_level_range.end_of_range for e in elements) + level = min(max(fill_level, lowest), highest) + return max( + ( + e + for e in elements + if e.fill_level_range.start_of_range + <= level + <= e.fill_level_range.end_of_range + ), + key=lambda e: ( + e.fill_level_range.start_of_range, + e.fill_level_range.end_of_range, + ), + ) + + def rates(self, moment: datetime) -> tuple[float, float, float]: + """Fill, usage and leakage, each per second, at this fill level and moment.""" + fill = 0.0 + if self.mode_id is not None: + element = self._holding(self.modes[self.mode_id].elements, self.fill_level) + low, high = element.fill_rate.start_of_range, element.fill_rate.end_of_range + fill = low + (high - low) * self.factor + usage = per_second(expected_usage(moment.hour)) + leakage = self._holding(self.leakage.elements, self.fill_level).leakage_rate + return fill, usage, leakage + + def advance(self, until: datetime, step: timedelta = timedelta(minutes=1)) -> None: + """Move the fill level forward to a moment, a minute at a time, so that an element boundary is crossed in time.""" + while self.at < until: + dt = min(step, until - self.at) + fill, usage, leakage = self.rates(self.at) + self.fill_level += (fill - usage - leakage) * dt.total_seconds() + self.fill_level = min(max(self.fill_level, 0.0), PHYSICAL_CAPACITY) + self.at += dt + + def follow(self, instruction: FRBCInstruction, moment: datetime) -> None: + self.advance(moment) + self.mode_id = str(instruction.operation_mode) + self.factor = instruction.operation_mode_factor + + +async def send_instruction_status( + ws, instruction_id, status: InstructionStatus +) -> None: + """Tell the CEM where an instruction stands, which is what it revokes by: only one not yet carried out.""" + await ws.send_str( + InstructionStatusUpdate( + message_id=get_unique_id(), + instruction_id=instruction_id, + status_type=status, + timestamp=datetime.now(timezone.utc), + ).model_dump_json() + ) + async def apply_at_execution_time( - ws, instruction: FRBCInstruction, previous_operation_mode_id=None + ws, + instruction: FRBCInstruction, + applied: dict | None = None, + buffer: Buffer | None = None, + on_due: Callable[[], None] | None = None, ) -> None: """Enter the operation mode an instruction asks for, at the moment it asks for it. @@ -42,6 +315,9 @@ async def apply_at_execution_time( A resource manager that answers on arrival reports a mode it is not in yet, and a customer energy manager recording that status has a measurement of a power the device is not drawing, timestamped now. + + The mode it reports leaving is the one the last instruction carried out put it in, kept in applied, + rather than whatever the instruction before it asked for, which may have been revoked and never carried out. """ delay = (instruction.execution_time - datetime.now(timezone.utc)).total_seconds() @@ -49,28 +325,58 @@ async def apply_at_execution_time( print(f"WAITING: {delay:.0f}s until this instruction is due") await asyncio.sleep(delay) + # From here on it is being carried out, so a revocation arriving now is too late to stop it. + if on_due is not None: + on_due() + now = datetime.now(timezone.utc) + if buffer is not None: + buffer.follow(instruction, now) status = FRBCActuatorStatus( message_id=get_unique_id(), actuator_id=instruction.actuator_id, active_operation_mode_id=instruction.operation_mode, operation_mode_factor=instruction.operation_mode_factor, - previous_operation_mode_id=previous_operation_mode_id, + previous_operation_mode_id=(applied or {}).get("mode"), # The moment the transition happened, so that what is recorded about it is placed there, # rather than wherever the message is read. - transition_timestamp=datetime.now(timezone.utc), + transition_timestamp=now, ) + if applied is not None: + applied["mode"] = instruction.operation_mode print("SENDING: FRBC.ActuatorStatus") await ws.send_str(status.model_dump_json()) + await send_instruction_status(ws, instruction.id, InstructionStatus.SUCCEEDED) -async def answer_instructions(ws) -> None: - """Acknowledge every instruction, and report the transition each one asks for when it is due.""" +async def answer_instructions(ws, buffer: Buffer | None = None) -> None: + """Acknowledge every instruction, and report the transition each one asks for when it is due. - active_operation_mode_id = None + An instruction the CEM revokes, because a newer plan replaces the one it belonged to, is not followed: + without that, the buffer would end up in whatever the last instruction to fire across every plan ever sent said. + """ + + applied: dict = {"mode": None} + pending: dict[str, asyncio.Task] = {} async for msg in ws: payload = msg.json() print("RECEIVED: ", payload) - if payload.get("message_type") != "FRBC.Instruction": + message_type = payload.get("message_type") + if message_type == "RevokeObject": + revocation = RevokeObject.model_validate(payload) + task = pending.pop(str(revocation.object_id), None) + await ws.send_str( + ReceptionStatus( + subject_message_id=revocation.message_id, + status=ReceptionStatusValues.OK, + ).model_dump_json() + ) + if task is not None: + task.cancel() + await send_instruction_status( + ws, revocation.object_id, InstructionStatus.REVOKED + ) + continue + if message_type != "FRBC.Instruction": continue instruction = FRBCInstruction.model_validate(payload) await ws.send_str( @@ -79,140 +385,144 @@ async def answer_instructions(ws) -> None: status=ReceptionStatusValues.OK, ).model_dump_json() ) + await send_instruction_status(ws, instruction.id, InstructionStatus.ACCEPTED) # In a task of its own, so that an instruction due later does not hold up the ones behind it. - asyncio.create_task( - apply_at_execution_time(ws, instruction, active_operation_mode_id) - ) - active_operation_mode_id = instruction.operation_mode + key = str(instruction.id) + def due(key: str = key) -> None: + pending.pop(key, None) -async def main_s2(): - async with aiohttp.ClientSession() as session: - async with session.ws_connect("http://localhost:8080/ws") as ws: - message = Handshake( - message_id=get_unique_id(), - role=EnergyManagementRole.RM, - supported_protocol_versions=["0.1.0"], - ) + def forget(done: asyncio.Task, key: str = key) -> None: + pending.pop(key, None) - print("SENDING: HANDSHAKE") + task = asyncio.create_task( + apply_at_execution_time(ws, instruction, applied, buffer, on_due=due) + ) + pending[key] = task + task.add_done_callback(forget) - await ws.send_str(message.model_dump_json()) - response = await ws.receive() - - print("RECEIVING: ", response.json()) - - # send resource manager details - - resource_manager_details_message = ResourceManagerDetails( - message_id=get_unique_id(), - resource_id=get_unique_id(), - roles=[ - Role(role=RoleType.ENERGY_STORAGE, commodity=Commodity.ELECTRICITY) - ], - instruction_processing_delay=Duration(1), - available_control_types=[ - ControlType.FILL_RATE_BASED_CONTROL, - ControlType.NO_SELECTION, - ], - provides_forecast=True, - provides_power_measurement_types=[ - CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC - ], +async def report_fill_level( + ws, buffer: Buffer, interval: float = STATUS_INTERVAL +) -> None: + """Report where the buffer is every interval, which is what asks the CEM for a new schedule.""" + while True: + await asyncio.sleep(interval) + buffer.advance(datetime.now(timezone.utc)) + print(f"SENDING: FRBC.StorageStatus at {buffer.fill_level:.2f} kWh") + await ws.send_str( + FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=buffer.fill_level ).model_dump_json() + ) - print("SENDING: ResourceManagerDetails") - - await ws.send_str(resource_manager_details_message) - - response = await ws.receive() - - print("RECEIVING: ", response.json()) - - response = await ws.receive() - - print("RECEIVING: ", response.json()) - - # Let the server activate the control type - control_type = await ws.receive() - - control_type = control_type.json() - - print("RECEIVING: ", control_type) - message = ReceptionStatus( - subject_message_id=control_type["message_id"], - status=ReceptionStatusValues.OK, - ) +async def refresh_forecasts( + ws, leakage: FRBCLeakageBehaviour, interval: float = FORECAST_INTERVAL +) -> None: + """Send the leakage, the usage forecast and the comfort profile again every interval, each from now on. - print("SENDING: ReceptionStatus") + Sent only once, they would cover the first day, and the CEM would plan without them from then on. + """ + while True: + await asyncio.sleep(interval) + now = datetime.now(timezone.utc) + for message in ( + leakage.model_copy( + update={"message_id": get_unique_id(), "valid_from": now} + ), + usage_forecast(now), + fill_level_target_profile(now), + ): + print(f"SENDING: {message.message_type}") await ws.send_str(message.model_dump_json()) - electric_power = CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC - # send storage status - storage_status = FRBCStorageStatus( - message_id=get_unique_id(), present_fill_level=0.4 - ) +async def receive_until(ws, message_type: str) -> dict: + """The next message of a type, past the reception statuses that arrive in between.""" + while True: + message = (await ws.receive()).json() + print("RECEIVED: ", message) + if message.get("message_type") == message_type: + return message - print("SENDING: FRBC.StorageStatus") - await ws.send_str(storage_status.model_dump_json()) +async def run_resource_manager( + ws, interval: float = STATUS_INTERVAL, forecast_interval: float = FORECAST_INTERVAL +) -> Buffer: + """Introduce the heat pump, declare what it can do and what its buffer needs, then follow and report. - response = await ws.receive() - - print("RECEIVING: ", response.json()) - - # send system description - operation_mode_element = FRBCOperationModeElement( - fill_level_range=NumberRange(start_of_range=0.0, end_of_range=0.5), - fill_rate=NumberRange(start_of_range=-0.5, end_of_range=0.5), - power_ranges=[ - PowerRange( - start_of_range=-0.5, - end_of_range=0.5, - commodity_quantity=electric_power, - ) - ], - ) - - operation_mode = FRBCOperationMode( - id=get_unique_id(), - elements=[operation_mode_element], - abnormal_condition_only=False, - diagnostic_label="full operating range of a perfectly efficiency battery", - ) - - actuator = FRBCActuatorDescription( - id=get_unique_id(), - supported_commodities=[Commodity.ELECTRICITY], - operation_modes=[operation_mode], - transitions=[], - timers=[], - ) - - storage = FRBCStorageDescription( - provides_leakage_behaviour=False, - provides_fill_level_target_profile=False, - provides_usage_forecast=False, - fill_level_range=NumberRange(start_of_range=0.05, end_of_range=0.45), - ) - - valid_from = datetime(2026, 1, 15, 20, tzinfo=ZoneInfo("Europe/Amsterdam")) - - system_description_message = FRBCSystemDescription( - message_id=get_unique_id(), - valid_from=valid_from, - actuators=[actuator], - storage=storage, - ).model_dump_json() + In the order S2 suggests: the system description first, then the storage's status. + Returns the simulated buffer, for whoever wants to look at where the instructions took it. + """ + await ws.send_str( + Handshake( + message_id=get_unique_id(), + role=EnergyManagementRole.RM, + supported_protocol_versions=["0.1.0"], + ).model_dump_json() + ) + await receive_until(ws, "HandshakeResponse") + + await ws.send_str( + ResourceManagerDetails( + message_id=get_unique_id(), + resource_id=get_unique_id(), + name="heat pump with a hot-water buffer", + # It draws electricity and stores heat, so it is a consumer of the commodity it is planned in. + roles=[ + Role(role=RoleType.ENERGY_CONSUMER, commodity=Commodity.ELECTRICITY) + ], + instruction_processing_delay=Duration(root=1000), + available_control_types=[ + ControlType.FILL_RATE_BASED_CONTROL, + ControlType.NO_SELECTION, + ], + # It sends no PowerForecast. It could measure its power, which is what the next field says, + # and S2 requires at least one type there. + provides_forecast=False, + provides_power_measurement_types=[ELECTRIC_POWER], + ).model_dump_json() + ) + select = await receive_until(ws, "SelectControlType") + await ws.send_str( + ReceptionStatus( + subject_message_id=select["message_id"], status=ReceptionStatusValues.OK + ).model_dump_json() + ) - print("SENDING: FRBC.SystemDescription") + now = datetime.now(timezone.utc) + modes = operation_modes() + leakage = leakage_behaviour(now) + buffer = Buffer(modes={str(m.id): m for m in modes}, leakage=leakage, at=now) + for message in ( + system_description(modes, now), + leakage, + usage_forecast(now), + fill_level_target_profile(now), + FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=buffer.fill_level + ), + ): + print(f"SENDING: {message.message_type}") + await ws.send_str(message.model_dump_json()) + + background = [ + asyncio.create_task(report_fill_level(ws, buffer, interval)), + asyncio.create_task(refresh_forecasts(ws, leakage, forecast_interval)), + ] + try: + await answer_instructions(ws, buffer) + finally: + for task in background: + task.cancel() + return buffer - await ws.send_str(system_description_message) - await answer_instructions(ws) +async def main_s2(): + async with aiohttp.ClientSession() as session: + async with session.ws_connect("http://localhost:8080/ws") as ws: + await run_resource_manager(ws) async def main_websocket(): diff --git a/src/flexmeasures_client/s2/script/websockets_server.py b/src/flexmeasures_client/s2/script/websockets_server.py index 354adb18..591540bd 100644 --- a/src/flexmeasures_client/s2/script/websockets_server.py +++ b/src/flexmeasures_client/s2/script/websockets_server.py @@ -3,6 +3,7 @@ import logging import os from datetime import datetime, timedelta +from typing import Any from urllib.parse import urlparse from zoneinfo import ZoneInfo @@ -11,8 +12,19 @@ from s2python.common import ControlType from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.exceptions import ( + ContentTypeError, + JobFailedError, + WrongAPIVersionError, +) from flexmeasures_client.s2.cem import CEM from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple +from flexmeasures_client.s2.measurements import MeasurementRecorder +from flexmeasures_client.s2.site import ( + FRBC_SITE_SENSORS, + FRBCSiteConfigurator, + ensure_site, +) log_level = os.getenv("LOGGING_LEVEL", "WARNING").upper() logging.basicConfig( @@ -23,13 +35,40 @@ SITE_NAME = "My CEM" SITE_TIMEZONE = "Europe/Amsterdam" -# Enough days of demo prices to cover what the CEM plans for: a schedule starts a day after the -# system description that asked for it and runs for twelve hours, so two days would just do. +# Enough days of demo prices to cover what the CEM plans for, which is twelve hours from now, +# with the seeding repeated every SEED_PRICES_EVERY so that a demo left running never runs out. SEED_PRICE_DAYS = 3 +SEED_PRICES_EVERY = timedelta(hours=12) + +#: Whether resource managers may send messages wrapped with a simulated time, as a co-simulation does. +ACCEPT_SIMULATION_METADATA = os.getenv("S2_ACCEPT_SIMULATION_METADATA", "").lower() in ( + "1", + "true", + "yes", +) + +#: Each sensor the site needs, which is the library's table of what an FRBC site records. +SITE_SENSORS = FRBC_SITE_SENSORS + +#: What every site asset is made with: a building, whose connection takes three phases of 25 A at 230 V. +SITE_ASSET_SPECS: dict[str, Any] = dict( + generic_asset_type_id=6, + flex_model={"power-capacity": f"{3 * 25 * 230} VA"}, +) + +#: Whether each resource manager gets a site of its own, named after it, as in a co-simulation of a community, +#: rather than all of them sharing the one site this server sets up at startup. +SITE_PER_RESOURCE_MANAGER = os.getenv("S2_SITE_PER_RESOURCE_MANAGER", "").lower() in ( + "1", + "true", + "yes", +) # Keys under which the configured site is kept on the aiohttp application. FM_CLIENT = web.AppKey("fm_client", FlexMeasuresClient) -SENSORS = web.AppKey("sensors", tuple) +SENSORS = web.AppKey("sensors", dict) +CONFIGURATOR = web.AppKey("configurator", FRBCSiteConfigurator) +RECORDER = web.AppKey("recorder", MeasurementRecorder) async def rm_details_watchdog(ws, cem: CEM): @@ -53,7 +92,8 @@ async def rm_details_watchdog(ws, cem: CEM): cem._logger.debug(f"CONTROL TYPE: {cem._control_type}") - # after this, schedule will be triggered on reception of a new system description + # From here on, a schedule is asked for on each storage status, + # and on a system description that arrives after one. async def websocket_producer(ws, cem: CEM): @@ -102,91 +142,123 @@ async def websocket_handler(request): # calls, which takes about ten seconds against a local server, while s2-python's # reception-status timeout is five: a resource manager gave up before the CEM had read its # first message. - fm_client = request.app[FM_CLIENT] - price_sensor, power_sensor, soc_sensor, rm_discharge_sensor = request.app[SENSORS] + if SITE_PER_RESOURCE_MANAGER: + await serve_resource_manager( + ws, + request.app[FM_CLIENT], + configurator=request.app[CONFIGURATOR], + recorder=request.app[RECORDER], + ) + else: + await serve_resource_manager( + ws, + request.app[FM_CLIENT], + request.app[SENSORS], + recorder=request.app[RECORDER], + ) + return ws + + +async def serve_resource_manager( + ws, + fm_client: FlexMeasuresClient, + sensors: dict[str, dict] | None = None, + configurator: FRBCSiteConfigurator | None = None, + recorder: MeasurementRecorder | None = None, +) -> None: + """Act as the CEM for one resource manager connected over ws, until it disconnects. + On the site whose sensors are given, or, with a configurator, on a site of the resource manager's own, + which is set up once its details arrive. + """ cem = CEM( - sensor_id=power_sensor["id"], fm_client=fm_client, logger=LOGGER, + site_configurator=configurator, + # The server's, which outlives a connection, so a resource manager that reconnects is not recorded twice. + measurement_recorder=recorder, + # Where what the resource manager measures is recorded, on the fixed site; a configured site wires its own. + power_sensor_id=( + {"ELECTRIC.POWER.L1": sensors["measured-power"]["id"]} + if sensors is not None and "measured-power" in sensors + else None + ), + # A co-simulation's resource manager sends its messages wrapped with the simulated time, which this sets the clock to. + accept_simulation_metadata=ACCEPT_SIMULATION_METADATA, ) - frbc = FRBCSimple( - power_sensor_id=power_sensor["id"], - price_sensor_id=price_sensor["id"], - soc_sensor_id=soc_sensor["id"], - rm_discharge_sensor_id=rm_discharge_sensor["id"], - ) - cem.register_control_type(frbc) + if sensors is not None: + cem.register_control_type(fixed_site_handler(sensors)) + await serve(ws, cem) - # create "parallel" tasks for the message producer and consumer - await asyncio.gather( - websocket_consumer(ws, cem), - websocket_producer(ws, cem), - rm_details_watchdog(ws, cem), - ) - return ws +def fixed_site_handler(sensors: dict[str, dict]) -> FRBCSimple: + """The handler for a resource manager on the one site this server sets up at startup.""" + frbc = FRBCSimple( + power_sensor_id=sensors["power"]["id"], + price_sensor_id=sensors["price"]["id"], + soc_sensor_id=sensors["state of charge"]["id"], + rm_discharge_sensor_id=sensors["RM discharge"]["id"], + # Where the resource manager's declarations are recorded, so that each can be seen next to the schedule planned against it. + fill_rate_sensor_id=sensors["fill rate"]["id"], + soc_minima_sensor_id=sensors["soc minima"]["id"], + soc_maxima_sensor_id=sensors["soc maxima"]["id"], + usage_forecast_sensor_id=sensors["usage forecast"]["id"], + leakage_behaviour_sensor_id=sensors["storage efficiency"]["id"], + charging_efficiency_sensor_id=sensors["charging efficiency"]["id"], + timezone=SITE_TIMEZONE, + # Plan from now, rather than from a day after the description, so that a demo shows its instructions being followed. + valid_from_shift=timedelta(0), + ) + # A newer plan revokes what is left of the one before, so the resource manager does not follow both. + frbc.send_revocations = True + return frbc + + +async def serve(ws, cem: CEM) -> None: + """Serve one resource manager with this CEM, for as long as its connection lasts.""" + # The session lasts as long as the resource manager's messages do. + # When they end, the CEM is closed, which stops what its handlers still have in flight, + # and the producer and the watchdog stop with it: left running, they outlived the connection, + # and a resource manager that reconnected was served by two sessions planning for it at once. + consumer = asyncio.create_task(websocket_consumer(ws, cem)) + side_tasks = [ + asyncio.create_task(websocket_producer(ws, cem)), + asyncio.create_task(rm_details_watchdog(ws, cem)), + ] + try: + await consumer + finally: + await cem.close() + for task in side_tasks: + task.cancel() + await asyncio.gather(*side_tasks, return_exceptions=True) async def configure_site( site_name: str, fm_client: FlexMeasuresClient -) -> tuple[dict, dict, dict, dict]: - """Create or update the site asset and its sensors, and seed it with demo prices.""" - account = await fm_client.get_account() - assets = await fm_client.get_assets(parse_json_fields=True) - - site_asset = None - for asset in assets: - if asset["name"] == site_name: - site_asset = asset - break - - site_asset_specs = dict( - latitude=0, - longitude=0, - generic_asset_type_id=6, # Building asset type - flex_model={ - "power-capacity": f"{3 * 25 * 230} VA", - }, +) -> dict[str, dict]: + """Create or update the site asset, its sensors and its graphs, and seed it with demo prices.""" + _, sensors = await ensure_site( + fm_client, + site_name, + asset_specs=SITE_ASSET_SPECS, + sensors=SITE_SENSORS, + timezone=SITE_TIMEZONE, + # The demo's own graphs, on every start, so changes made to them in the UI do not outlive a restart. + reset_graphs=True, ) + await seed_prices(fm_client, sensors["price"]["id"]) + return sensors - if not site_asset: - site_asset = await fm_client.add_asset( - name=site_name, account_id=account["id"], **site_asset_specs - ) - # Update site asset with the latest specs - await fm_client.update_asset( - site_asset["id"], site_asset_specs, parse_json_fields=True - ) - sensors = site_asset.get("sensors", []) - price_sensor = None - power_sensor = None - soc_sensor = None - rm_discharge_sensor = None - for sensor in sensors: - if sensor["name"] == "price": - price_sensor = sensor - elif sensor["name"] == "power": - power_sensor = sensor - elif sensor["name"] == "state of charge": - soc_sensor = sensor - elif sensor["name"] == "RM discharge": - rm_discharge_sensor = sensor - - if price_sensor is None: - price_sensor = await fm_client.add_sensor( - name="price", - event_resolution="PT15M", - unit="EUR/kWh", - generic_asset_id=site_asset["id"], - timezone=SITE_TIMEZONE, - ) - # Demo prices, so that a freshly configured site has something to schedule against. - # One day's hourly shape, repeated over SEED_PRICE_DAYS from this morning, because the CEM - # plans a day and a half ahead of the moment a resource manager sends its system description: - # a fixed date here goes stale the day after it is written, and the scheduler then fails the - # job with UnknownPricesException rather than returning a schedule. +async def seed_prices(fm_client: FlexMeasuresClient, price_sensor_id: int) -> None: + """Demo prices, so that a site has something to schedule against. + + One day's hourly shape, repeated over SEED_PRICE_DAYS from this morning, because the CEM + plans ahead of the moment a resource manager reports its storage: + a fixed date here goes stale the day after it is written, and the scheduler then fails the + job with UnknownPricesException rather than returning a schedule. + """ one_day = [ 0.10, 0.11, 0.12, 0.15, 0.18, 0.17, 0.11, 0.09, 0.10, 0.09, 0.09, 0.10, 0.08, 0.05, 0.04, 0.04, @@ -196,42 +268,42 @@ async def configure_site( hour=0, minute=0, second=0, microsecond=0 ) await fm_client.post_sensor_data( - sensor_id=price_sensor["id"], + sensor_id=price_sensor_id, start=today, duration=timedelta(days=SEED_PRICE_DAYS), values=one_day * SEED_PRICE_DAYS, unit="EUR/kWh", ) - if power_sensor is None: - power_sensor = await fm_client.add_sensor( - name="power", - event_resolution="PT15M", - unit="kW", - generic_asset_id=site_asset["id"], - timezone=SITE_TIMEZONE, - # S2 states a power as positive when it is consumed, - # and this sensor holds both an actuator's power and the schedule planned for it. - # Left unsaid, FlexMeasures reads a power sensor the other way round, - # and flips a schedule's sign on the way in, which puts the two in opposite directions. - attributes={"consumption_is_positive": True}, - ) - if soc_sensor is None: - soc_sensor = await fm_client.add_sensor( - name="state of charge", - event_resolution="PT0M", - unit="kWh", - generic_asset_id=site_asset["id"], - timezone=SITE_TIMEZONE, - ) - if rm_discharge_sensor is None: - rm_discharge_sensor = await fm_client.add_sensor( - name="RM discharge", - event_resolution="PT15M", - unit="dimensionless", - generic_asset_id=site_asset["id"], - timezone=SITE_TIMEZONE, - ) - return price_sensor, power_sensor, soc_sensor, rm_discharge_sensor + + +async def keep_prices_seeded(app: web.Application): + """Seed the prices again every SEED_PRICES_EVERY for as long as the server runs, so a demo never runs out of them. + + Not where each resource manager has a site of its own, whose prices are whoever runs them to post. + """ + if SITE_PER_RESOURCE_MANAGER: + yield + return + + async def reseed(): + while True: + await asyncio.sleep(SEED_PRICES_EVERY.total_seconds()) + try: + await seed_prices(app[FM_CLIENT], app[SENSORS]["price"]["id"]) + except ( + ConnectionError, + ValueError, + ContentTypeError, + JobFailedError, + WrongAPIVersionError, + ): + LOGGER.exception( + "Could not seed the demo prices again; trying at the next interval." + ) + + task = asyncio.create_task(reseed()) + yield + task.cancel() async def configure_site_once(app: web.Application) -> None: @@ -250,6 +322,23 @@ async def configure_site_once(app: web.Application) -> None: ssl=parsed.scheme == "https", ) app[FM_CLIENT] = fm_client + app[RECORDER] = MeasurementRecorder(fm_client, logger=LOGGER) + if SITE_PER_RESOURCE_MANAGER: + # One configurator for the server's lifetime, so a resource manager that reconnects finds its site again by id. + app[CONFIGURATOR] = FRBCSiteConfigurator( + fm_client, + asset_type_id=SITE_ASSET_SPECS["generic_asset_type_id"], + asset_specs={"flex_model": SITE_ASSET_SPECS["flex_model"]}, + timezone=SITE_TIMEZONE, + # The co-simulation's resource managers are heat pumps, which cannot feed power back: + # FlexMeasures then keeps their production at zero even where constraints are relaxed. + power_sensor_attributes={"is_strictly_non_positive": True}, + # A community's capacity is a target set by its members' steering, breached at a price, as #188 asked for. + handler_options={"relax_site_capacity": True}, + logger=LOGGER, + ) + LOGGER.info("Each resource manager gets a site of its own; ready for them.") + return app[SENSORS] = await configure_site(SITE_NAME, fm_client) LOGGER.info("Site '%s' is configured; ready for resource managers.", SITE_NAME) @@ -260,6 +349,7 @@ async def close_fm_client(app: web.Application) -> None: app = web.Application() app.on_startup.append(configure_site_once) +app.cleanup_ctx.append(keep_prices_seeded) app.on_cleanup.append(close_fm_client) app.add_routes([web.get("/ws", websocket_handler)]) diff --git a/src/flexmeasures_client/s2/site.py b/src/flexmeasures_client/s2/site.py new file mode 100644 index 00000000..f94185f0 --- /dev/null +++ b/src/flexmeasures_client/s2/site.py @@ -0,0 +1,271 @@ +"""A FlexMeasures site for an S2 resource manager: its asset, its sensors, its graphs, and the handler wired to them. + +The example server keeps one site for whichever resource manager connects (see ensure_site). +A CEM serving many resource managers, as a co-simulation of a community does, gives each one a site of its own, +named after it, with FRBCSiteConfigurator. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import timedelta +from logging import Logger, getLogger +from typing import Any + +try: + from s2python.common import ResourceManagerDetails +except ImportError: + raise ImportError( + "The 's2-python' package is required for this functionality. " + "Install it using `pip install flexmeasures-client[s2]`." + ) + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple + +_LOGGER = getLogger(__name__) + + +@dataclass(frozen=True) +class SiteSensor: + """What a site's sensor is made with: its unit, its resolution, and any attributes.""" + + unit: str + resolution: str + attributes: dict = field(default_factory=dict) + + +#: The sensors an FRBC site needs, by name. +#: Their units are the ones the CEM records in: fill levels in kWh, rates of fill level per second, as S2 states them. +#: The names are a contract with whoever reads the site, a co-simulation among them, +#: which looks up "price", "production price", "power", "state of charge" and "measured-power" by name. +FRBC_SITE_SENSORS: dict[str, SiteSensor] = { + "price": SiteSensor("EUR/kWh", "PT15M"), + "production price": SiteSensor("EUR/kWh", "PT15M"), + # S2 states a power as positive when it is consumed, + # and this sensor holds both an actuator's power and the schedule planned for it. + # Left unsaid, FlexMeasures reads a power sensor the other way round, + # and flips a schedule's sign on the way in, which puts the two in opposite directions. + "power": SiteSensor("kW", "PT15M", {"consumption_is_positive": True}), + # What the resource manager measures its site drawing, kept apart from the schedule on "power". + "measured-power": SiteSensor("kW", "PT15M", {"consumption_is_positive": True}), + "state of charge": SiteSensor("kWh", "PT0M"), + "RM discharge": SiteSensor("dimensionless", "PT15M"), + "fill rate": SiteSensor("kWh/s", "PT15M"), + "soc minima": SiteSensor("kWh", "PT15M"), + "soc maxima": SiteSensor("kWh", "PT15M"), + "usage forecast": SiteSensor("kWh/s", "PT15M"), + "storage efficiency": SiteSensor("%", "PT15M"), + "charging efficiency": SiteSensor("dimensionless", "PT15M"), +} + + +def frbc_site_graphs(sensors: dict[str, dict]) -> list[dict]: + """A site's graphs, top to bottom, as FlexMeasures takes an asset's sensors_to_show. + + In the form with a title and a list of sensors per graph, which every FlexMeasures version reads, + rather than the newer one with plots, which needs v0.32 or later and draws the same graphs. + A graph whose sensors the site does not have is left out. + """ + # Titled as #188 titled them, since a co-simulation's controller adds to the graph titled "Power". + graphs = [ + ("Prices", ["price", "production price"]), + ("Power", ["power", "measured-power"]), + ("State of charge", ["state of charge", "soc minima", "soc maxima"]), + ("Fill rate and usage", ["fill rate", "usage forecast"]), + ] + shown = [] + for title, names in graphs: + ids = [sensors[name]["id"] for name in names if name in sensors] + if ids: + shown.append({"title": title, "sensors": ids}) + return shown + + +async def ensure_site( + fm_client: FlexMeasuresClient, + name: str, + *, + asset_specs: dict[str, Any], + sensors: dict[str, SiteSensor], + timezone: str, + known_asset_id: int | None = None, + reset_graphs: bool = False, +) -> tuple[dict, dict[str, dict]]: + """Find or make the site asset of this name, with its specs and every one of these sensors, and its graphs. + + It is safe to run again: what exists is kept, and what is missing is made. + A flex model in the specs is merged into the one the asset has, since others write to it too, + such as a co-simulation's controller setting what its stock is worth at the end. + Graphs are set where the asset has none, since others add to them too, unless asked to reset them. + + :param known_asset_id: the asset's id where it was found before, which spares a listing of every asset. + :param reset_graphs: whether to set the graphs even where the asset has some. + :returns: the asset, and its sensors by name. + """ + site_asset = None + if known_asset_id is not None: + try: + candidate = await fm_client.get_asset( + known_asset_id, parse_json_fields=True + ) + except (ValueError, ConnectionError): + candidate = None + if candidate is not None and candidate.get("name") == name: + site_asset = candidate + if site_asset is None: + # The newest of any of that name, as a co-simulation picks it too. + named = [ + asset + for asset in await fm_client.get_assets(parse_json_fields=True) + if asset["name"] == name + ] + site_asset = max(named, key=lambda asset: asset["id"]) if named else None + created = site_asset is None + if site_asset is None: + account = await fm_client.get_account() + if account is None: + raise ValueError( + f"The user belongs to no organisation, so there is nowhere to create the site {name!r}." + ) + site_asset = await fm_client.add_asset( + name=name, account_id=account["id"], **asset_specs + ) + else: + specs = dict(asset_specs) + if isinstance(specs.get("flex_model"), dict) and isinstance( + site_asset.get("flex_model"), dict + ): + specs["flex_model"] = {**site_asset["flex_model"], **specs["flex_model"]} + await fm_client.update_asset(site_asset["id"], specs, parse_json_fields=True) + + existing = {sensor["name"]: sensor for sensor in site_asset.get("sensors", [])} + made: dict[str, dict] = {} + for sensor_name, spec in sensors.items(): + made[sensor_name] = existing.get(sensor_name) or await fm_client.add_sensor( + name=sensor_name, + event_resolution=spec.resolution, + unit=spec.unit, + generic_asset_id=site_asset["id"], + timezone=timezone, + **({"attributes": spec.attributes} if spec.attributes else {}), + ) + + if created or reset_graphs or not site_asset.get("sensors_to_show"): + await fm_client.update_asset( + site_asset["id"], + {"sensors_to_show": frbc_site_graphs(made)}, + parse_json_fields=True, + ) + return site_asset, made + + +@dataclass +class ConfiguredSite: + """What a CEM needs to serve one resource manager: its handler, and where its measured power goes.""" + + handler: FRBCSimple + #: The sensor each commodity quantity's measured power is recorded on, as the CEM's power_sensor_id takes it. + power_sensor_ids: dict[str, int] + asset_id: int + + +class FRBCSiteConfigurator: + """Gives each resource manager a site of its own, named after it, and an FRBC handler wired to its sensors. + + Made once per server and handed to each CEM it makes (see CEM's site_configurator), + so that a resource manager that reconnects finds its site by the id remembered here, + rather than by a listing of every asset on the server. + """ + + def __init__( + self, + fm_client: FlexMeasuresClient, + *, + asset_type_id: int, + timezone: str = "UTC", + asset_specs: dict[str, Any] | None = None, + sensors: dict[str, SiteSensor] | None = None, + power_sensor_attributes: dict[str, Any] | None = None, + handler_options: dict[str, Any] | None = None, + send_revocations: bool = True, + logger: Logger | None = None, + ) -> None: + """ + :param asset_type_id: the FlexMeasures asset type each site is made with. + :param timezone: the time zone of each site's sensors, and of its handler. + :param asset_specs: anything else each site asset is made or updated with, such as a flex model. + :param sensors: the sensors each site gets; FRBC_SITE_SENSORS by default. + :param power_sensor_attributes: attributes added to the "power" sensor's, which a device's nature decides, + such as that it cannot produce. + :param handler_options: keyword arguments for each FRBCSimple beyond its sensors. + By default it plans from now, over 24 hours, as a co-simulation steps through days. + :param send_revocations: whether a newer plan revokes what is left of the one before. + """ + self._fm_client = fm_client + self._timezone = timezone + self._asset_specs = { + "generic_asset_type_id": asset_type_id, + **(asset_specs or {}), + } + sensors = dict(sensors if sensors is not None else FRBC_SITE_SENSORS) + if power_sensor_attributes: + power = sensors["power"] + sensors["power"] = SiteSensor( + power.unit, + power.resolution, + {**power.attributes, **power_sensor_attributes}, + ) + self._sensors = sensors + self._handler_options = { + "valid_from_shift": timedelta(0), + "schedule_duration": timedelta(hours=24), + **(handler_options or {}), + } + self._send_revocations = send_revocations + self._asset_ids: dict[str, int] = {} + self._logger = logger or _LOGGER + + async def __call__(self, details: ResourceManagerDetails) -> ConfiguredSite: + name = details.name + if not name: + raise ValueError( + "A resource manager without a name cannot be given a site of its own, which is found by its name." + ) + asset, sensors = await ensure_site( + self._fm_client, + name, + asset_specs=self._asset_specs, + sensors=self._sensors, + timezone=self._timezone, + known_asset_id=self._asset_ids.get(name), + ) + self._asset_ids[name] = asset["id"] + self._logger.info("Resource manager %r has site %s.", name, asset["id"]) + + def sensor_id(sensor_name: str) -> int | None: + sensor = sensors.get(sensor_name) + return sensor["id"] if sensor else None + + handler = FRBCSimple( + power_sensor_id=sensors["power"]["id"], + price_sensor_id=sensors["price"]["id"], + production_price_sensor_id=sensor_id("production price"), + soc_sensor_id=sensors["state of charge"]["id"], + rm_discharge_sensor_id=sensors["RM discharge"]["id"], + fill_rate_sensor_id=sensor_id("fill rate"), + soc_minima_sensor_id=sensor_id("soc minima"), + soc_maxima_sensor_id=sensor_id("soc maxima"), + usage_forecast_sensor_id=sensor_id("usage forecast"), + leakage_behaviour_sensor_id=sensor_id("storage efficiency"), + charging_efficiency_sensor_id=sensor_id("charging efficiency"), + timezone=self._timezone, + **self._handler_options, + ) + handler.send_revocations = self._send_revocations + measured = sensor_id("measured-power") + return ConfiguredSite( + handler=handler, + power_sensor_ids={"ELECTRIC.POWER.L1": measured} if measured else {}, + asset_id=asset["id"], + ) diff --git a/src/flexmeasures_client/s2/utils.py b/src/flexmeasures_client/s2/utils.py index 50eabaab..cfdcd55e 100644 --- a/src/flexmeasures_client/s2/utils.py +++ b/src/flexmeasures_client/s2/utils.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections import OrderedDict -from typing import Mapping, TypeVar +from typing import Any, Mapping, Protocol, TypeVar from uuid import uuid4 import pydantic @@ -49,15 +49,6 @@ def get_unique_id() -> str: return str(uuid4()) -def get_validation_error_summary(error: pydantic.ValidationError) -> str: - error_summary = "" - - for i, e in enumerate(error): - error_summary += f"\nValidationEror {i} -> \t {e['msg']}" - - return error_summary[1:] # skipping the first \n - - def get_message_id(message: pydantic.BaseModel) -> str | None: """ This function returns the message_id if it is found in the message, @@ -71,8 +62,14 @@ def get_message_id(message: pydantic.BaseModel) -> str | None: return None +class _HasMessageId(Protocol): + """Any S2 message: every one of them carries a message id.""" + + message_id: Any + + def get_reception_status( - subject_message: pydantic.BaseModel, + subject_message: _HasMessageId, status: ReceptionStatusValues = ReceptionStatusValues.OK, ): """ diff --git a/tests/client/test_asset.py b/tests/client/test_asset.py index 25101873..85c4b238 100644 --- a/tests/client/test_asset.py +++ b/tests/client/test_asset.py @@ -560,3 +560,61 @@ async def test_delete_asset_confirm_no(): (k[0], str(k[1])) for k in m.requests.keys() ] await client.close() + + +def _posted_asset(m) -> dict: + """The body of the one asset post aioresponses saw.""" + ((_, calls),) = [ + (key, calls) for key, calls in m.requests.items() if key[0] == "POST" + ] + return calls[0].kwargs["json"] + + +@pytest.mark.asyncio +async def test_an_asset_can_be_added_without_a_location(): + """FlexMeasures takes no location for an asset that has none, such as one standing for a community.""" + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + m.post("http://localhost:5000/api/v3_0/assets", status=201, payload={"id": 27}) + await client.add_asset(name="community", account_id=1, generic_asset_type_id=5) + posted = _posted_asset(m) + assert "latitude" not in posted and "longitude" not in posted + await client.close() + + +@pytest.mark.asyncio +async def test_a_location_is_posted_as_numbers_where_given(): + with aioresponses() as m: + client = FlexMeasuresClient(email="test@test.test", password="test") + client.access_token = "test-token" + m.post("http://localhost:5000/api/v3_0/assets", status=201, payload={"id": 28}) + await client.add_asset( + name="home", + account_id=1, + latitude=52.0, + longitude=4.0, + generic_asset_type_id=5, + ) + posted = _posted_asset(m) + assert (posted["latitude"], posted["longitude"]) == (52.0, 4.0) + await client.close() + + +@pytest.mark.asyncio +async def test_an_asset_needs_a_type(): + client = FlexMeasuresClient(email="test@test.test", password="test") + with pytest.raises(ValueError, match="generic_asset_type_id"): + await client.add_asset(name="untyped", account_id=1) + await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("location", [dict(latitude=52.0), dict(longitude=4.0)]) +async def test_half_a_location_is_refused(location): + client = FlexMeasuresClient(email="test@test.test", password="test") + with pytest.raises(ValueError, match="both a latitude and a longitude"): + await client.add_asset( + name="half", account_id=1, generic_asset_type_id=5, **location + ) + await client.close() diff --git a/tests/client/test_init.py b/tests/client/test_init.py index 0647b16e..a2b14436 100644 --- a/tests/client/test_init.py +++ b/tests/client/test_init.py @@ -21,11 +21,7 @@ _parse_sensor_json_fields, ) from flexmeasures_client.constants import CLIENT_VERSION_HEADER -from flexmeasures_client.response_handling import ( - check_content_type, - check_for_status, - parse_retry_after, -) +from flexmeasures_client.response_handling import check_for_status, parse_retry_after @pytest.mark.parametrize( @@ -837,15 +833,27 @@ def test_parse_sensor_json_fields(): assert sensor["attributes"]["key"] == "value" -def test_check_content_type_failure(): - """check_content_type raises on non-JSON content type.""" - from aiohttp import ContentTypeError as AiohttpContentTypeError +@pytest.mark.asyncio +async def test_an_answer_that_is_not_json_is_a_connection_error() -> None: + """A proxy's HTML error page, as during a server restart, is reported like every other failure to reach the API. - response = MagicMock() - response.headers = {"Content-Type": "text/html"} - response.text.return_value = "some html" - with pytest.raises(AiohttpContentTypeError): - check_content_type(response) + aiohttp's own json() refuses it inside the request loop, which is where this is handled. + The separate content-type check that used to follow the loop is gone: + it could only fire on the JSON variants aiohttp accepts, such as application/problem+json, and it should not have refused those. + """ + with aioresponses() as m: + m.get( + "http://localhost:5000/api/", + status=200, + body="502 Bad Gateway", + content_type="text/html", + ) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test", access_token="skip-auth" + ) + with pytest.raises(ConnectionError, match="text/html"): + await flexmeasures_client.get_versions() + await flexmeasures_client.close() def test_check_for_status_failure(): @@ -957,3 +965,65 @@ async def test_server_version_at_least_looks_the_version_up() -> None: assert await flexmeasures_client.server_version_at_least("0.27.0") is True assert await flexmeasures_client.server_version_at_least("1.1.0") is False await flexmeasures_client.close() + + +@pytest.mark.asyncio +async def test_an_auth_token_answer_that_is_not_an_object_is_refused() -> None: + """A list where the token should be says what came back, instead of failing on the lookup.""" + with aioresponses() as m: + m.post( + "http://localhost:5000/api/requestAuthToken", + status=200, + payload=["auth_token"], + ) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test" + ) + with pytest.raises(ValueError, match="requestAuthToken, but got a list"): + await flexmeasures_client.get_access_token() + assert flexmeasures_client.access_token is None + await flexmeasures_client.close() + + +@pytest.mark.asyncio +async def test_the_access_token_is_returned_as_well_as_stored() -> None: + with aioresponses() as m: + m.post( + "http://localhost:5000/api/requestAuthToken", + status=200, + payload={"auth_token": "a-token"}, + ) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test" + ) + assert await flexmeasures_client.get_access_token() == "a-token" + assert flexmeasures_client.access_token == "a-token" + await flexmeasures_client.close() + + +@pytest.mark.asyncio +async def test_a_version_answer_that_is_not_an_object_is_refused() -> None: + with aioresponses() as m: + m.get("http://localhost:5000/api/", status=200, payload=["v3_0"]) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test", access_token="skip-auth" + ) + with pytest.raises(ValueError, match="API root, but got a list"): + await flexmeasures_client.get_versions() + await flexmeasures_client.close() + + +@pytest.mark.asyncio +async def test_a_client_without_a_token_authorises_with_the_one_it_fetches() -> None: + with aioresponses() as m: + m.post( + "http://localhost:5000/api/requestAuthToken", + status=200, + payload={"auth_token": "a-token"}, + ) + flexmeasures_client = FlexMeasuresClient( + email="test@test.test", password="test" + ) + headers = await flexmeasures_client.get_headers(include_auth=True) + assert headers["Authorization"] == "a-token" + await flexmeasures_client.close() diff --git a/tests/s2/test_cem.py b/tests/s2/test_cem.py index 37f3aaff..4299fa8a 100644 --- a/tests/s2/test_cem.py +++ b/tests/s2/test_cem.py @@ -4,6 +4,7 @@ from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.control_types import ControlTypeHandler from flexmeasures_client.s2.control_types.FRBC import FRBCTest @@ -242,3 +243,164 @@ async def test_automatic_change_control_type(resource_manager_details, rm_handsh # CEM response is ReceptionStatus with an OK status assert response["message_type"] == "ReceptionStatus" assert response["status"] == "OK" + + +class _PlainPEBC(ControlTypeHandler): + """A handler for a second control type, which keeps no background tasks.""" + + _control_type = ControlType.POWER_ENVELOPE_BASED_CONTROL + + +async def _details_received(cem, rm_handshake, resource_manager_details) -> None: + """Handshake and details, with every answer taken off the queue: the CEM selects no control type yet.""" + await cem.handle_message(rm_handshake) + await cem.handle_message(resource_manager_details) + while not cem._sending_queue.empty(): + await cem.get_message() + assert cem.control_type == ControlType.NO_SELECTION + + +@pytest.mark.asyncio +async def test_a_message_of_the_control_type_being_selected_counts_as_its_confirmation( + frbc_system_description, resource_manager_details, rm_handshake +): + """A resource manager sends a control type's messages only once it is working in that control type. + + #188 saw a CEM stay in NO_SELECTION while a reconnecting resource manager kept sending FRBC messages, + each answered with a temporary error and retried. + """ + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await _details_received(cem, rm_handshake, resource_manager_details) + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + selection = await cem.get_message() + + await cem.handle_message(frbc_system_description) + + assert cem.control_type == ControlType.FILL_RATE_BASED_CONTROL + response = await cem.get_message() + assert response["status"] == ReceptionStatusValues.OK.value + assert selection["message_id"] not in cem.success_callbacks # no longer waited for + + +@pytest.mark.asyncio +async def test_a_message_of_another_control_type_is_not_taken_as_a_confirmation( + frbc_system_description, resource_manager_details, rm_handshake +): + """Selecting one control type and receiving another's message switches to neither.""" + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + cem.register_control_type(_PlainPEBC()) + await _details_received(cem, rm_handshake, resource_manager_details) + cem._selecting = (ControlType.POWER_ENVELOPE_BASED_CONTROL, "some-selection") + + await cem.handle_message(frbc_system_description) + + assert cem.control_type == ControlType.NO_SELECTION + + +@pytest.mark.asyncio +async def test_a_switch_between_control_types_is_left_to_its_confirmation( + frbc_system_description, resource_manager_details, rm_handshake +): + """While one control type is active, a message of the one being switched to is still that of an unconfirmed switch. + + Its confirmation is awaited by the active handler, which this CEM would not clear, so it is left to arrive. + """ + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + cem.register_control_type(_PlainPEBC()) + await _details_received(cem, rm_handshake, resource_manager_details) + cem._control_type = ControlType.POWER_ENVELOPE_BASED_CONTROL + cem._selecting = (ControlType.FILL_RATE_BASED_CONTROL, "some-selection") + + await cem.handle_message(frbc_system_description) + + assert cem.control_type == ControlType.POWER_ENVELOPE_BASED_CONTROL + + +@pytest.mark.asyncio +async def test_a_confirmed_selection_is_no_longer_pending( + resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await _details_received(cem, rm_handshake, resource_manager_details) + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + selection = await cem.get_message() + await cem.handle_message( + ReceptionStatus( + subject_message_id=selection["message_id"], + status=ReceptionStatusValues.OK, + ) + ) + assert cem.control_type == ControlType.FILL_RATE_BASED_CONTROL + assert cem._selecting is None + + +@pytest.mark.asyncio +async def test_a_control_type_without_a_handler_is_not_activated_implicitly( + frbc_system_description, resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + await _details_received(cem, rm_handshake, resource_manager_details) + cem._selecting = (ControlType.FILL_RATE_BASED_CONTROL, "some-selection") + + await cem.handle_message(frbc_system_description) + + assert cem.control_type == ControlType.NO_SELECTION + response = await cem.get_message() + assert response["status"] == ReceptionStatusValues.TEMPORARY_ERROR.value + + +@pytest.mark.asyncio +async def test_a_message_before_the_details_activates_nothing(frbc_system_description): + """Before its details, a resource manager has not been offered a control type, so it cannot have accepted one.""" + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await cem.handle_message(frbc_system_description) + assert cem.control_type is None + + +@pytest.mark.asyncio +async def test_closing_the_cem_stops_what_its_handlers_have_in_flight(): + """A schedule still being planned for a session that has ended would post and plan for a resource manager no longer there.""" + import asyncio + + cem = CEM(fm_client=None) + frbc = FRBCTest() + cem.register_control_type(frbc) + planning = asyncio.create_task(asyncio.sleep(3600)) + frbc.background_tasks.add(planning) + + await cem.close() + await asyncio.wait([planning], timeout=1) + + assert planning.cancelled() + planning.cancel() # in case it was not, so the test does not leave it running + + +@pytest.mark.asyncio +async def test_a_message_without_a_type_activates_nothing( + resource_manager_details, rm_handshake +): + cem = CEM(fm_client=None) + cem.register_control_type(FRBCTest()) + await _details_received(cem, rm_handshake, resource_manager_details) + cem._selecting = (ControlType.FILL_RATE_BASED_CONTROL, "some-selection") + from flexmeasures_client.s2.utils import get_unique_id + + await cem.handle_message({"message_id": get_unique_id()}) + assert cem.control_type == ControlType.NO_SELECTION + + +@pytest.mark.asyncio +async def test_a_handler_without_background_tasks_closes(): + """Closing every handler is what ends a session, so one that keeps no tasks must close too.""" + cem = CEM(fm_client=None) + plain = _PlainPEBC() + cem.register_control_type(plain) + if hasattr(plain, "background_tasks"): + del plain.background_tasks + await cem.close() + assert cem.is_closed() diff --git a/tests/s2/test_clock.py b/tests/s2/test_clock.py new file mode 100644 index 00000000..dd3a4361 --- /dev/null +++ b/tests/s2/test_clock.py @@ -0,0 +1,249 @@ +"""The time an S2 CEM goes by, and a co-simulation's way of setting it.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from zoneinfo import ZoneInfo + +import pytest +from s2python.common import ReceptionStatusValues + +from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.clock import SimulatedClock, WallClock +from flexmeasures_client.s2.control_types.FRBC import FRBCTest + +SIMULATED = datetime(2022, 3, 1, 12, 0, tzinfo=timezone.utc) + + +def test_a_simulated_clock_stands_still_between_settings(): + clock = SimulatedClock() + clock.set(SIMULATED) + assert clock.now() == SIMULATED + assert clock.now() == SIMULATED + + +def test_a_simulated_clock_goes_by_the_wall_clock_until_it_is_set(): + clock = SimulatedClock() + assert not clock.is_set + assert abs(clock.now() - WallClock().now()) < timedelta(seconds=5) + + +def test_a_simulated_time_without_a_zone_is_refused(): + """#188 took naive times as Europe/Amsterdam, which a simulation in UTC shifts by an hour or two.""" + with pytest.raises(ValueError, match="time zone"): + SimulatedClock().set(datetime(2022, 3, 1, 12, 0)) + + +def test_a_handler_registered_with_a_cem_goes_by_its_clock(): + clock = SimulatedClock(SIMULATED) + cem = CEM(fm_client=None, clock=clock) + frbc = FRBCTest() + cem.register_control_type(frbc) + assert frbc.now() == SIMULATED + assert cem.now() == SIMULATED + later = SIMULATED + timedelta(hours=3) + clock.set(later) + assert frbc.now() == later # the same clock, not a copy of its time + + +def test_now_is_given_in_the_handlers_time_zone(): + cem = CEM(fm_client=None, clock=SimulatedClock(SIMULATED)) + frbc = FRBCTest() + frbc._timezone = ZoneInfo("Europe/Amsterdam") + cem.register_control_type(frbc) + assert frbc.now().tzinfo == ZoneInfo("Europe/Amsterdam") + assert frbc.now() == SIMULATED + + +def test_accepting_simulation_metadata_needs_a_simulated_clock(): + with pytest.raises(ValueError, match="SimulatedClock"): + CEM(fm_client=None, clock=WallClock(), accept_simulation_metadata=True) + + +def _wrapped(message: dict, dt) -> dict: + return {"message": message, "metadata": {"dt": dt}} + + +@pytest.mark.asyncio +async def test_a_wrapped_message_sets_the_clock_and_is_handled(rm_handshake): + cem = CEM(fm_client=None, accept_simulation_metadata=True) + await cem.handle_message( + _wrapped(rm_handshake.model_dump(mode="json"), "2022-03-01T12:00:00+00:00") + ) + assert cem.now() == SIMULATED + reception = await cem.get_message() + assert reception["status"] == ReceptionStatusValues.OK.value + + +@pytest.mark.asyncio +async def test_a_wrapped_message_is_refused_by_a_cem_that_does_not_accept_it( + rm_handshake, +): + """The wrapper is not S2, and a live CEM does not let a resource manager set its time.""" + cem = CEM(fm_client=None) + before = cem.now() + message = rm_handshake.model_dump(mode="json") + await cem.handle_message(_wrapped(message, "2022-03-01T12:00:00+00:00")) + reception = await cem.get_message() + assert reception["status"] == ReceptionStatusValues.INVALID_MESSAGE.value + assert reception["subject_message_id"] == message["message_id"] + assert cem._sending_queue.empty() # the handshake itself was not answered + assert cem.now() - before < timedelta(seconds=5) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("dt", ["2022-03-01T12:00:00", "not a time"]) +async def test_a_wrapped_message_with_an_unusable_time_is_refused(rm_handshake, dt): + cem = CEM(fm_client=None, accept_simulation_metadata=True) + message = rm_handshake.model_dump(mode="json") + await cem.handle_message(_wrapped(message, dt)) + reception = await cem.get_message() + assert reception["status"] == ReceptionStatusValues.INVALID_MESSAGE.value + assert not cem._clock.is_set + + +@pytest.mark.asyncio +async def test_a_wrapper_around_no_message_is_refused_without_an_answer(): + """Nothing to answer: a reception status needs the id of the message it is about.""" + cem = CEM(fm_client=None) + await cem.handle_message({"message": "garbage", "metadata": {}}) + assert cem._sending_queue.empty() + + +def test_a_cem_timer_runs_on_simulated_time(): + # On a period boundary, which a co-simulation's times are: the ceiling of the moment is the moment itself. + clock = SimulatedClock(SIMULATED) + cem = CEM(fm_client=None, clock=clock) + assert cem._is_timer_due("power") + assert not cem._is_timer_due("power") # a clock standing still on a boundary + clock.set(SIMULATED + timedelta(hours=1)) + assert cem._is_timer_due("power") + + +def test_a_timer_handed_in_without_a_zone_is_read_as_the_hosts_local_time(): + """As it was compared before, with a zoneless datetime.now(), which is the host's local time.""" + local = (SIMULATED + timedelta(minutes=30)).astimezone().replace(tzinfo=None) + cem = CEM(fm_client=None, clock=SimulatedClock(SIMULATED), timers={"power": local}) + assert not cem._is_timer_due("power") + cem._clock.set(SIMULATED + timedelta(minutes=31)) + assert cem._is_timer_due("power") + + +def test_the_example_server_accepts_simulation_metadata_only_when_told_to(monkeypatch): + import importlib + + from flexmeasures_client.s2.script import websockets_server + + monkeypatch.setenv("S2_ACCEPT_SIMULATION_METADATA", "1") + assert importlib.reload(websockets_server).ACCEPT_SIMULATION_METADATA is True + monkeypatch.delenv("S2_ACCEPT_SIMULATION_METADATA") + assert importlib.reload(websockets_server).ACCEPT_SIMULATION_METADATA is False + + +def test_a_tunes_timer_runs_on_the_clock_its_cem_gives_it(): + from flexmeasures_client.s2.control_types.FRBC.frbc_tunes import ( + FillRateBasedControlTUNES, + ) + + start = SIMULATED + clock = SimulatedClock(start) + cem = CEM(fm_client=None, clock=clock) + tunes = FillRateBasedControlTUNES( + asset_id=1, + soc_minima_sensor_id=2, + soc_maxima_sensor_id=3, + rm_discharge_sensor_id=4, + fill_level_sensor_id=7, + thp_fill_rate_sensor_id=8, + thp_efficiency_sensor_id=9, + nes_fill_rate_sensor_id=10, + nes_efficiency_sensor_id=11, + usage_forecast_sensor_id=12, + fill_rate_sensor_id=13, + active_actuator_id_sensor_id=14, + leakage_behaviour_sensor_id=15, + production_price_sensor=16, + consumption_price_sensor=17, + state_of_charge_sensor_id=18, + ) + cem.register_control_type(tunes) + assert tunes._is_timer_due("usage_forecast") + assert not tunes._is_timer_due( + "usage_forecast" + ) # a clock standing still on a boundary + clock.set(start + timedelta(hours=1)) + assert tunes._is_timer_due("usage_forecast") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("inner", ["garbage", {"no": "type"}]) +async def test_a_wrapper_around_no_message_leaves_the_clock_alone(inner): + """With the flag on, too: the clock is set only for a message that will be handled.""" + cem = CEM(fm_client=None, accept_simulation_metadata=True) + await cem.handle_message( + {"message": inner, "metadata": {"dt": "2022-03-01T12:00:00+00:00"}} + ) + assert not cem._clock.is_set + assert cem._sending_queue.empty() + + +def test_a_tunes_timer_handed_in_without_a_zone_is_read_as_the_hosts_local_time(): + from flexmeasures_client.s2.control_types.FRBC.frbc_tunes import ( + FillRateBasedControlTUNES, + ) + + start = SIMULATED + clock = SimulatedClock(start) + cem = CEM(fm_client=None, clock=clock) + tunes = FillRateBasedControlTUNES( + asset_id=1, + soc_minima_sensor_id=2, + soc_maxima_sensor_id=3, + rm_discharge_sensor_id=4, + fill_level_sensor_id=7, + thp_fill_rate_sensor_id=8, + thp_efficiency_sensor_id=9, + nes_fill_rate_sensor_id=10, + nes_efficiency_sensor_id=11, + usage_forecast_sensor_id=12, + fill_rate_sensor_id=13, + active_actuator_id_sensor_id=14, + leakage_behaviour_sensor_id=15, + production_price_sensor=16, + consumption_price_sensor=17, + state_of_charge_sensor_id=18, + ) + cem.register_control_type(tunes) + tunes._timers["usage_forecast"] = ( + (start + timedelta(minutes=30)).astimezone().replace(tzinfo=None) + ) + assert not tunes._is_timer_due("usage_forecast") + + +@pytest.mark.asyncio +async def test_a_state_of_charge_is_known_as_of_the_simulated_time(): + """Without a prior, FlexMeasures stamped a simulated reading with the moment the request arrived, years later.""" + from unittest.mock import AsyncMock + + from s2python.frbc import FRBCStorageStatus + + from flexmeasures_client.client import FlexMeasuresClient + from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple + from flexmeasures_client.s2.utils import get_unique_id + + fm = AsyncMock(FlexMeasuresClient) + fm.get_sensor = AsyncMock(return_value={"event_resolution": "PT0M"}) + frbc = FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + ) + cem = CEM(fm_client=fm, clock=SimulatedClock(SIMULATED)) + cem.register_control_type(frbc) + await frbc.send_storage_status( + FRBCStorageStatus(message_id=get_unique_id(), present_fill_level=5.0) + ) + posted = fm.post_measurements.await_args.kwargs + assert posted["start"] == SIMULATED + assert posted["prior"] == SIMULATED diff --git a/tests/s2/test_example_end_to_end.py b/tests/s2/test_example_end_to_end.py new file mode 100644 index 00000000..12a64aa3 --- /dev/null +++ b/tests/s2/test_example_end_to_end.py @@ -0,0 +1,1000 @@ +"""The shipped example, both halves, end to end: the resource manager it ships, served by the CEM it ships. + +Only FlexMeasures is stood in for, and the websocket between the two is held in memory. +The example used to send no instruction at all, for two reasons the halves only show together: +the resource manager sent its storage status before its system description, which planned nothing, +and its description was dated ten months before the prices the server seeds. +""" + +from __future__ import annotations + +import asyncio +import gc +import json +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import aiohttp +import pandas as pd +import pytest + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.script import websockets_server as server +from flexmeasures_client.s2.script.websockets_client import run_resource_manager + + +class _Text: + type = aiohttp.WSMsgType.TEXT + + def __init__(self, data: str): + self.data = data + + def json(self) -> dict: + return json.loads(self.data) + + +#: Put on an end's inbox, it ends that end's stream of messages, as a websocket closed by its peer does. +_CLOSED = object() + + +class _End: + """One end of an in-memory websocket: what it sends arrives at the other end.""" + + def __init__(self, inbox: asyncio.Queue, outbox: asyncio.Queue): + self._inbox, self._outbox = inbox, outbox + self.received: list[dict] = [] + + async def send_str(self, data: str) -> None: + await self._outbox.put(data) + + async def send_json(self, message) -> None: + await self._outbox.put(json.dumps(message)) + + async def receive(self) -> _Text: + data = await self._inbox.get() + self.received.append(json.loads(data)) + return _Text(data) + + def __aiter__(self): + return self + + async def __anext__(self) -> _Text: + data = await self._inbox.get() + if data is _CLOSED: + raise StopAsyncIteration + self.received.append(json.loads(data)) + return _Text(data) + + async def close(self) -> None: + pass + + def of_type(self, message_type: str) -> list[dict]: + return [m for m in self.received if m.get("message_type") == message_type] + + +def _connected() -> tuple[_End, _End]: + to_resource_manager, to_cem = asyncio.Queue(), asyncio.Queue() + return _End(to_resource_manager, to_cem), _End(to_cem, to_resource_manager) + + +#: What the stand-in FlexMeasures plans, per quarter hour: idle, then full heat, then minimum heat. +SHAPE = [0.0, 0.0, 1500.0, 1500.0, 400.0, 400.0, 0.0, 0.0] + + +def _flexmeasures() -> AsyncMock: + fm = AsyncMock(FlexMeasuresClient) + fm.server_version_at_least = AsyncMock(return_value=True) + fm.get_sensor = AsyncMock( + return_value={"attributes": {"consumption_is_positive": True}} + ) + + def schedule(start, duration, **kwargs) -> dict: + steps = int(pd.Timedelta(duration) / pd.Timedelta(minutes=15)) + return { + "start": pd.Timestamp(start).isoformat(), + "duration": pd.Timedelta(duration).isoformat(), + "values": [SHAPE[i % len(SHAPE)] for i in range(steps)], + "scheduler_info": {"scheduler": "StorageScheduler"}, + } + + fm.trigger_and_get_schedule = AsyncMock(side_effect=lambda **kw: schedule(**kw)) + return fm + + +async def _until(condition, timeout: float = 15.0) -> None: + async def wait(): + while not condition(): + await asyncio.sleep(0.05) + + await asyncio.wait_for(wait(), timeout) + + +@pytest.mark.asyncio +async def test_the_shipped_resource_manager_is_planned_for_and_instructed(): + resource_manager, cem = _connected() + fm = _flexmeasures() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + unhandled: list[dict] = [] + loop = asyncio.get_running_loop() + loop.set_exception_handler(lambda _, context: unhandled.append(context)) + began = datetime.now(timezone.utc) + # The CEM's clock stands on the boundary the step under way began at, so that is where its schedules start, + # and the first instruction is due at once rather than at the next boundary, up to a quarter of an hour away. + on_a_boundary = began.replace( + minute=began.minute - began.minute % 15, second=0, microsecond=0 + ) + + from unittest.mock import patch + + clock = patch.object( + server.FRBCSimple, "now", lambda self: on_a_boundary.astimezone(self._timezone) + ) + clock.start() + tasks = [ + asyncio.create_task(server.serve_resource_manager(cem, fm, sensors)), + asyncio.create_task( + run_resource_manager(resource_manager, interval=0.3, forecast_interval=0.3) + ), + ] + try: + # Two storage statuses past the first, each a new plan, so that replanning is part of what is checked. + await _until(lambda: fm.trigger_and_get_schedule.await_count >= 3) + await _until(lambda: len(resource_manager.of_type("FRBC.Instruction")) >= 3) + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + clock.stop() + gc.collect() + loop.set_exception_handler(None) + + instructions = resource_manager.of_type("FRBC.Instruction") + distinct = { + (i["operation_mode"], round(i["operation_mode_factor"], 6)) + for i in instructions + } + assert len(distinct) >= 2, instructions + + refused = [ + m for m in resource_manager.of_type("ReceptionStatus") if m["status"] != "OK" + ] + assert len(cem.of_type("FRBC.UsageForecast")) >= 2 # sent again, not only once + assert refused == [] + + start = fm.trigger_and_get_schedule.await_args_list[0].kwargs["start"] + assert start == on_a_boundary + assert unhandled == [] + + # Every declaration reached the sensor the server made for it, to be seen next to the schedule planned against it. + posted_to = { + call.args[0] if call.args else call.kwargs.get("sensor_id") + for call in fm.post_sensor_data.await_args_list + + fm.post_measurements.await_args_list + } + for name in ( + "power", + "state of charge", + "fill rate", + "soc minima", + "soc maxima", + "usage forecast", + "storage efficiency", + "charging efficiency", + ): + assert sensors[name]["id"] in posted_to, name + + +# --------------------------------------------------------------------------- +# The example's own buffer, as a second opinion on the CEM's arithmetic +# --------------------------------------------------------------------------- + +from flexmeasures_client.s2.control_types.FRBC.utils import ( # noqa: E402 + compute_next_fill_level, + usage_rate_at, +) +from flexmeasures_client.s2.script.websockets_client import ( # noqa: E402 + Buffer, + heat, + leakage_behaviour, + operation_modes, + per_second, + usage_forecast, +) + +NOON = datetime(2024, 1, 1, 12, tzinfo=timezone.utc) + + +def _heating_buffer(fill_level: float) -> Buffer: + modes = operation_modes() + heating = next(m for m in modes if m.diagnostic_label == "heating") + return Buffer( + modes={str(m.id): m for m in modes}, + leakage=leakage_behaviour(NOON), + fill_level=fill_level, + mode_id=str(heating.id), + factor=1.0, + at=NOON, + ) + + +def test_the_example_buffer_follows_s2s_definitions(): + """A quarter hour at full heat from a cold 2 kWh: 6 kW of heat in, a tenth of a kWh an hour drawn, and a cold buffer's leakage.""" + buffer = _heating_buffer(2.0) + buffer.advance(NOON + timedelta(minutes=15)) + expected = 2.0 + (heat(6000) - per_second(0.1) - per_second(0.8 / 24)) * 900 + assert buffer.fill_level == pytest.approx(expected) + + +def test_the_example_and_the_cem_agree_on_where_the_buffer_goes(): + """Two integrators written separately from S2's definitions, one in the CEM and one in the example's buffer. + + While no boundary between elements is crossed they hold every rate constant over a step, and so must agree. + """ + buffer = _heating_buffer(2.0) + element_fill_rate = heat(6000) + cem_fill_level = 2.0 + for step in range(2): + moment = pd.Timestamp(NOON + step * timedelta(minutes=15)) + cem_fill_level = compute_next_fill_level( + cem_fill_level, + element_fill_rate, + timedelta(minutes=15), + leakage_behaviour=leakage_behaviour(NOON), + usage_rate=usage_rate_at(usage_forecast(NOON), moment), + ) + buffer.advance(NOON + timedelta(minutes=30)) + assert cem_fill_level < 6.0 # no element boundary crossed + assert buffer.fill_level == pytest.approx(cem_fill_level, rel=1e-12) + + +def test_a_warm_buffer_heats_at_the_warm_coefficient(): + """At 7 kWh the buffer is past the cold element, so full heat delivers 4.5 kW, and it leaks as a warm buffer does.""" + buffer = _heating_buffer(7.0) + buffer.advance(NOON + timedelta(minutes=15)) + expected = 7.0 + (heat(4500) - per_second(0.1) - per_second(1.5 / 24)) * 900 + assert buffer.fill_level == pytest.approx(expected) + + +def test_an_instruction_takes_over_from_where_the_last_one_left_the_buffer(): + """Idle for a quarter hour, losing only what is drawn and what leaks, then a quarter hour at full heat.""" + from s2python.frbc import FRBCInstruction + + from flexmeasures_client.s2.utils import get_unique_id + + buffer = _heating_buffer(4.0) + buffer.mode_id, buffer.factor = None, 0.0 # idle until told otherwise + heating = next(m for m in buffer.modes.values() if m.diagnostic_label == "heating") + buffer.follow( + FRBCInstruction( + message_id=get_unique_id(), + id=get_unique_id(), + actuator_id=get_unique_id(), + operation_mode=heating.id, + operation_mode_factor=1.0, + execution_time=NOON + timedelta(minutes=15), + abnormal_condition=False, + ), + NOON + timedelta(minutes=15), + ) + buffer.advance(NOON + timedelta(minutes=30)) + losses = per_second(0.1) + per_second(0.8 / 24) + expected = 4.0 - losses * 900 + (heat(6000) - losses) * 900 + assert buffer.fill_level == pytest.approx(expected) + + +@pytest.mark.asyncio +async def test_the_resource_manager_hands_back_its_buffer_when_the_cem_goes_away(): + resource_manager, cem = _connected() + fm = _flexmeasures() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + serving = asyncio.create_task(server.serve_resource_manager(cem, fm, sensors)) + managing = asyncio.create_task(run_resource_manager(resource_manager, interval=60)) + try: + await _until(lambda: len(resource_manager.of_type("FRBC.Instruction")) >= 1) + await resource_manager._inbox.put(_CLOSED) + buffer = await asyncio.wait_for(managing, timeout=5) + finally: + serving.cancel() + await asyncio.gather(serving, return_exceptions=True) + assert 0 <= buffer.fill_level <= 12 + + +@pytest.mark.asyncio +async def test_the_site_gets_a_sensor_for_every_declaration_and_prices_from_today(): + fm = AsyncMock(FlexMeasuresClient) + fm.get_account = AsyncMock(return_value={"id": 1}) + fm.get_assets = AsyncMock( + return_value=[ + { + "id": 5, + "name": server.SITE_NAME, + "sensors": [{"id": 50, "name": "price"}], + } + ] + ) + fm.add_sensor = AsyncMock( + side_effect=lambda **kwargs: { + "id": 100 + len(fm.add_sensor.await_args_list), + **kwargs, + } + ) + + sensors = await server.configure_site(server.SITE_NAME, fm) + + assert set(sensors) == set(server.SITE_SENSORS) + assert sensors["price"]["id"] == 50 # kept, not made again + made = {call.kwargs["name"]: call.kwargs for call in fm.add_sensor.await_args_list} + assert "price" not in made + assert made["power"]["attributes"] == {"consumption_is_positive": True} + assert made["fill rate"]["unit"] == "kWh/s" + assert made["soc minima"]["unit"] == "kWh" + prices = fm.post_sensor_data.await_args.kwargs + assert prices["sensor_id"] == 50 + assert prices["start"].date() == datetime.now(prices["start"].tzinfo).date() + + +def test_the_example_and_the_cem_agree_across_an_element_boundary(): + """The CEM follows a step exactly, the example's buffer a minute at a time: two discretisations of S2's definitions. + + Through the cold element and into the warm one they can differ only within the minute of the crossing, + by at most a minute of the difference between the two elements' fill rates. + """ + from flexmeasures_client.s2.control_types.FRBC.utils import ( + UsageProfile, + follow_operation_mode, + ) + + buffer = _heating_buffer(5.0) + heating = buffer.modes[buffer.mode_id] + cem_fill_level = 5.0 + for step in range(4): + cem_fill_level = follow_operation_mode( + cem_fill_level, + heating, + 1.0, + pd.Timestamp(NOON + step * timedelta(minutes=15)), + timedelta(minutes=15), + leakage_behaviour=leakage_behaviour(NOON), + usage=UsageProfile(usage_forecast(NOON)), + ) + buffer.advance(NOON + timedelta(hours=1)) + assert ( + cem_fill_level > 6.0 + ) # the boundary between the cold and the warm element was crossed + a_minute_of_the_difference = (heat(6000) - heat(4500)) * 60 + assert buffer.fill_level == pytest.approx( + cem_fill_level, abs=a_minute_of_the_difference + ) + + +# --------------------------------------------------------------------------- +# What the review of #259 found unpinned +# --------------------------------------------------------------------------- + +from flexmeasures_client.s2.script.websockets_client import ( # noqa: E402 + COLD_BELOW, + PHYSICAL_CAPACITY, + answer_instructions, + expected_usage, + fill_level_target_profile, + refresh_forecasts, + report_fill_level, + system_description, +) + + +class _Collect: + """A websocket that collects what is sent and hands over what it is given, then ends.""" + + def __init__(self, incoming: list[str] | None = None): + self.sent: list[dict] = [] + self._incoming = list(incoming or []) + + async def send_str(self, data: str) -> None: + self.sent.append(json.loads(data)) + + def __aiter__(self): + async def messages(): + for data in self._incoming: + yield _Text(data) + await asyncio.sleep(0) + + return messages() + + +def test_the_forecast_the_example_sends_is_the_usage_it_simulates(): + """Hour by hour, through a whole day, so that what is planned for is what is drawn.""" + for start_hour in (0, 13): + start = datetime(2024, 1, 1, start_hour, tzinfo=timezone.utc) + forecast = usage_forecast(start) + for h, element in enumerate(forecast.elements): + assert element.usage_rate_expected == pytest.approx( + per_second(expected_usage((start_hour + h) % 24)) + ) + + +def test_the_comfort_profile_preheats_before_the_morning_showers(): + profile = fill_level_target_profile(datetime(2024, 1, 1, tzinfo=timezone.utc)) + lows = [e.fill_level_range.start_of_range for e in profile.elements] + assert lows[6] == 6.0 and expected_usage(7) > expected_usage(6) + assert set(lows[:6] + lows[7:]) == {3.0} + + +def test_at_the_boundary_the_buffer_heats_at_the_warm_coefficient(): + buffer = _heating_buffer(COLD_BELOW) + fill, _, _ = buffer.rates(NOON) + assert fill == pytest.approx(heat(4500)) + + +def test_the_buffer_never_holds_more_than_it_physically_can(): + buffer = _heating_buffer(PHYSICAL_CAPACITY - 0.1) + buffer.advance(NOON + timedelta(hours=2)) + assert buffer.fill_level == PHYSICAL_CAPACITY + + +def test_the_example_declares_what_it_sends_and_how_it_can_switch(): + modes = operation_modes() + description = system_description(modes, NOON) + storage = description.storage + assert storage.provides_leakage_behaviour and storage.provides_usage_forecast + assert storage.provides_fill_level_target_profile + switches = {(str(t.from_), str(t.to)) for t in description.actuators[0].transitions} + ids = [str(m.id) for m in modes] + assert switches == {(ids[0], ids[1]), (ids[1], ids[0])} + + +@pytest.mark.asyncio +async def test_the_reported_fill_level_is_where_the_buffer_has_got_to(): + buffer = _heating_buffer(5.0) + buffer.at = datetime.now(timezone.utc) - timedelta(minutes=30) + ws = _Collect() + reporting = asyncio.create_task(report_fill_level(ws, buffer, interval=0.01)) + await _until(lambda: ws.sent) + reporting.cancel() + assert ( + ws.sent[0]["present_fill_level"] > 5.5 + ) # half an hour of heating, not where it started + + +@pytest.mark.asyncio +async def test_a_revoked_instruction_is_not_followed(): + """A newer plan replaced it; following it anyway left the heat pump in whatever the oldest plan said last.""" + from s2python.common import RevokableObjects, RevokeObject + from s2python.frbc import FRBCInstruction + + from flexmeasures_client.s2.utils import get_unique_id + + due_soon = FRBCInstruction( + message_id=get_unique_id(), + id=get_unique_id(), + actuator_id=get_unique_id(), + operation_mode=get_unique_id(), + operation_mode_factor=1.0, + execution_time=datetime.now(timezone.utc) + timedelta(milliseconds=200), + abnormal_condition=False, + ) + revoke = RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=due_soon.id, + ) + ws = _Collect([due_soon.model_dump_json(), revoke.model_dump_json()]) + await answer_instructions(ws) + await asyncio.sleep(0.4) + assert not [m for m in ws.sent if m["message_type"] == "FRBC.ActuatorStatus"] + acknowledged = { + m["subject_message_id"] + for m in ws.sent + if m["message_type"] == "ReceptionStatus" + } + assert str(revoke.message_id) in acknowledged + + +@pytest.mark.asyncio +async def test_the_forecasts_are_sent_again_from_now_on(): + ws = _Collect() + refreshing = asyncio.create_task( + refresh_forecasts(ws, leakage_behaviour(NOON), interval=0.01) + ) + await _until(lambda: len(ws.sent) >= 3) + refreshing.cancel() + assert [m["message_type"] for m in ws.sent[:3]] == [ + "FRBC.LeakageBehaviour", + "FRBC.UsageForecast", + "FRBC.FillLevelTargetProfile", + ] + assert pd.Timestamp(ws.sent[0]["valid_from"]) > NOON + + +@pytest.mark.asyncio +async def test_the_server_plans_from_now_in_the_site_time_zone_and_revokes_what_it_replaces(): + from unittest.mock import patch + + made = [] + real = server.FRBCSimple + + def capture(**kwargs): + frbc = real(**kwargs) + made.append((kwargs, frbc)) + return frbc + + resource_manager, cem = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with patch.object(server, "FRBCSimple", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager(cem, _flexmeasures(), sensors) + ) + await _until(lambda: made) + serving.cancel() + await asyncio.gather(serving, return_exceptions=True) + kwargs, frbc = made[0] + assert kwargs["timezone"] == server.SITE_TIMEZONE + assert kwargs["valid_from_shift"] == timedelta(0) + assert frbc.send_revocations is True + + +def test_the_site_sensors_are_what_the_cem_records_in(): + assert ( + server.SITE_SENSORS["state of charge"].unit, + server.SITE_SENSORS["state of charge"].resolution, + ) == ("kWh", "PT0M") + assert server.SITE_SENSORS["storage efficiency"].unit == "%" + assert server.SITE_SENSORS["usage forecast"].unit == "kWh/s" + + +@pytest.mark.asyncio +async def test_the_demo_prices_are_seeded_again_while_the_server_runs(): + from unittest.mock import patch + + from aiohttp import web + + app = web.Application() + fm = _flexmeasures() + app[server.FM_CLIENT] = fm + app[server.SENSORS] = {"price": {"id": 10}} + with patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)): + seeding = server.keep_prices_seeded(app) + await seeding.__anext__() + await _until(lambda: fm.post_sensor_data.await_count >= 2) + with pytest.raises(StopAsyncIteration): + await seeding.__anext__() + assert fm.post_sensor_data.await_args.kwargs["sensor_id"] == 10 + + +def test_the_loss_is_a_share_of_the_fill_level_a_schedule_starts_from(): + """1.5 kWh a day from a 5 kWh buffer is a larger share of it than of a 12 kWh one.""" + from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, + ) + + leakage = leakage_behaviour(NOON) + at_five = leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15), fill_level=7.0, share_of=5.0 + ) + rate = per_second(1.5 / 24) # the warm element, which applies at 7 + assert at_five == pytest.approx(100 * (1 - rate / 5.0) ** 900, rel=1e-12) + + +@pytest.mark.asyncio +async def test_a_failed_reseed_is_tried_again_at_the_next_interval(): + from unittest.mock import patch + + from aiohttp import web + + app = web.Application() + fm = _flexmeasures() + fm.post_sensor_data.side_effect = [ConnectionError("down"), None, None] + app[server.FM_CLIENT] = fm + app[server.SENSORS] = {"price": {"id": 10}} + with patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)): + seeding = server.keep_prices_seeded(app) + await seeding.__anext__() + await _until(lambda: fm.post_sensor_data.await_count >= 2) + with pytest.raises(StopAsyncIteration): + await seeding.__anext__() + + +def _instruction(mode, due_in: timedelta): + from s2python.frbc import FRBCInstruction + + from flexmeasures_client.s2.utils import get_unique_id + + return FRBCInstruction( + message_id=get_unique_id(), + id=get_unique_id(), + actuator_id=get_unique_id(), + operation_mode=mode, + operation_mode_factor=1.0, + execution_time=datetime.now(timezone.utc) + due_in, + abnormal_condition=False, + ) + + +@pytest.mark.asyncio +async def test_the_mode_reported_as_left_is_the_one_last_carried_out(): + """A revoked instruction was never carried out, so the next transition does not start from its mode.""" + from s2python.common import RevokableObjects, RevokeObject + + from flexmeasures_client.s2.utils import get_unique_id + + a, b = get_unique_id(), get_unique_id() + first = _instruction(a, timedelta(0)) + revoked = _instruction(b, timedelta(seconds=30)) + later = _instruction(a, timedelta(milliseconds=200)) + revoke = RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=revoked.id, + ) + ws = _Collect( + [ + first.model_dump_json(), + revoked.model_dump_json(), + revoke.model_dump_json(), + later.model_dump_json(), + ] + ) + await answer_instructions(ws) + await _until( + lambda: len([m for m in ws.sent if m["message_type"] == "FRBC.ActuatorStatus"]) + >= 2 + ) + statuses = [m for m in ws.sent if m["message_type"] == "FRBC.ActuatorStatus"] + assert statuses[1]["previous_operation_mode_id"] == str(a) + + +@pytest.mark.asyncio +async def test_the_example_tells_the_cem_where_each_instruction_stands(): + """Accepted on arrival, succeeded once carried out, revoked once revoked: the CEM revokes only what is not yet carried out.""" + from s2python.common import RevokableObjects, RevokeObject + + from flexmeasures_client.s2.utils import get_unique_id + + done = _instruction(get_unique_id(), timedelta(0)) + revoked = _instruction(get_unique_id(), timedelta(seconds=30)) + revoke = RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=revoked.id, + ) + ws = _Collect( + [done.model_dump_json(), revoked.model_dump_json(), revoke.model_dump_json()] + ) + await answer_instructions(ws) + await _until( + lambda: any( + m.get("status_type") == "SUCCEEDED" + for m in ws.sent + if m["message_type"] == "InstructionStatusUpdate" + ) + ) + updates = { + (m["instruction_id"], m["status_type"]) + for m in ws.sent + if m["message_type"] == "InstructionStatusUpdate" + } + assert updates == { + (str(done.id), "ACCEPTED"), + (str(done.id), "SUCCEEDED"), + (str(revoked.id), "ACCEPTED"), + (str(revoked.id), "REVOKED"), + } + + +@pytest.mark.asyncio +async def test_a_failed_ingestion_of_the_reseeded_prices_does_not_end_the_reseeding(): + from unittest.mock import patch + + from aiohttp import web + + from flexmeasures_client.exceptions import JobFailedError + + app = web.Application() + fm = _flexmeasures() + fm.post_sensor_data.side_effect = [JobFailedError("ingestion failed"), None, None] + app[server.FM_CLIENT] = fm + app[server.SENSORS] = {"price": {"id": 10}} + with patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)): + seeding = server.keep_prices_seeded(app) + await seeding.__anext__() + await _until(lambda: fm.post_sensor_data.await_count >= 2) + with pytest.raises(StopAsyncIteration): + await seeding.__anext__() + + +@pytest.mark.asyncio +async def test_a_revocation_arriving_once_an_instruction_is_due_is_too_late_to_stop_it(): + """It is carried out and reported as succeeded, not reported as revoked while the heat pump follows it.""" + from s2python.common import RevokableObjects, RevokeObject + + from flexmeasures_client.s2.script.websockets_client import apply_at_execution_time + from flexmeasures_client.s2.utils import get_unique_id + + instruction = _instruction(get_unique_id(), timedelta(0)) + became_due = [] + ws = _Collect() + await apply_at_execution_time( + ws, instruction, {"mode": None}, on_due=lambda: became_due.append(True) + ) + assert became_due == [True] + + revoke = RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=instruction.id, + ) + ws = _Collect([instruction.model_dump_json(), revoke.model_dump_json()]) + await answer_instructions(ws) + await _until(lambda: any(m.get("status_type") == "SUCCEEDED" for m in ws.sent)) + assert not [m for m in ws.sent if m.get("status_type") == "REVOKED"] + + +@pytest.mark.asyncio +async def test_the_site_graphs_show_the_buffer_with_its_band_and_what_fills_and_draws_it(): + """The graphs page showed only price and power, which says nothing about whether the plan kept the buffer warm.""" + fm = AsyncMock(FlexMeasuresClient) + fm.get_account = AsyncMock(return_value={"id": 1}) + fm.get_assets = AsyncMock( + return_value=[ + { + "id": 5, + "name": server.SITE_NAME, + "sensors": [], + "sensors_to_show": [{"title": "Edited in the UI", "sensors": [1]}], + } + ] + ) + fm.add_sensor = AsyncMock( + side_effect=lambda **kwargs: { + "id": 100 + len(fm.add_sensor.await_args_list), + **kwargs, + } + ) + sensors = await server.configure_site(server.SITE_NAME, fm) + shown = [ + call.args[1]["sensors_to_show"] + for call in fm.update_asset.await_args_list + if "sensors_to_show" in call.args[1] + ] + assert len(shown) == 1 + plotted = {graph["title"]: set(graph["sensors"]) for graph in shown[0]} + by_id = {sensor["id"]: name for name, sensor in sensors.items()} + named = {title: {by_id[i] for i in ids} for title, ids in plotted.items()} + assert {"state of charge", "soc minima", "soc maxima"} in named.values() + assert {"fill rate", "usage forecast"} in named.values() + assert {"power", "measured-power"} in named.values() + assert {"price", "production price"} in named.values() + + +def test_the_site_graphs_are_in_a_form_every_flexmeasures_reads(): + """A title and a list of sensors per graph; the form with plots is refused before FlexMeasures v0.32.""" + from flexmeasures_client.s2.site import frbc_site_graphs + + graphs = frbc_site_graphs( + {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + ) + for graph in graphs: + assert set(graph) == {"title", "sensors"} + assert all(isinstance(i, int) for i in graph["sensors"]) + + +@pytest.mark.asyncio +async def test_a_session_ends_with_its_connection(): + """Its producer, its watchdog and its planning stop, so a resource manager that reconnects is served by one session.""" + from unittest.mock import patch + + made = [] + real = server.CEM + + def capture(**kwargs): + cem = real(**kwargs) + made.append(cem) + return cem + + resource_manager, cem_end = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with patch.object(server, "CEM", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager(cem_end, _flexmeasures(), sensors) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0].is_closed() + leftover = [ + t + for t in asyncio.all_tasks() + if t is not asyncio.current_task() + and getattr(t.get_coro(), "__qualname__", "") + in ("websocket_producer", "rm_details_watchdog") + ] + assert leftover == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("accept", [False, True]) +async def test_the_server_hands_its_simulation_flag_to_each_cem(accept): + from unittest.mock import patch + + made = [] + real = server.CEM + + def capture(**kwargs): + made.append(kwargs) + return real(**kwargs) + + resource_manager, cem_end = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with ( + patch.object(server, "CEM", side_effect=capture), + patch.object(server, "ACCEPT_SIMULATION_METADATA", accept), + ): + serving = asyncio.create_task( + server.serve_resource_manager(cem_end, _flexmeasures(), sensors) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0]["accept_simulation_metadata"] is accept + + +@pytest.mark.asyncio +async def test_with_a_site_per_resource_manager_the_server_configures_none_at_startup(): + from unittest.mock import patch + + from aiohttp import web + + app = web.Application() + with ( + patch.object(server, "SITE_PER_RESOURCE_MANAGER", True), + patch.object(server, "configure_site", AsyncMock()) as configure, + ): + await server.configure_site_once(app) + configure.assert_not_awaited() + assert isinstance(app[server.CONFIGURATOR], server.FRBCSiteConfigurator) + assert ( + app[server.CONFIGURATOR] + ._sensors["power"] + .attributes["is_strictly_non_positive"] + ) + assert app[server.CONFIGURATOR]._handler_options["relax_site_capacity"] is True + # So that seeding, were it to happen, would reach FlexMeasures. + app[server.SENSORS] = {"price": {"id": 1}} + with ( + patch.object(server, "SITE_PER_RESOURCE_MANAGER", True), + patch.object(server, "SEED_PRICES_EVERY", timedelta(milliseconds=10)), + patch.object(server, "seed_prices", AsyncMock()) as seed, + ): + seeding = server.keep_prices_seeded(app) + await seeding.__anext__() # nothing to seed: the prices are the simulation's to post + await asyncio.sleep(0.05) + with pytest.raises(StopAsyncIteration): + await seeding.__anext__() + seed.assert_not_awaited() + await app[server.FM_CLIENT].close() + + +@pytest.mark.asyncio +async def test_a_resource_manager_with_a_site_of_its_own_is_served_once_it_is_configured(): + from unittest.mock import patch + + made = [] + real = server.CEM + + def capture(**kwargs): + cem = real(**kwargs) + made.append(cem) + return cem + + async def configurator(details): + raise AssertionError("not called in this test") + + resource_manager, cem_end = _connected() + with patch.object(server, "CEM", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager( + cem_end, _flexmeasures(), configurator=configurator + ) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0]._site_configurator is configurator + assert ( + made[0]._control_types_handlers == {} + ) # no handler until its site is configured + + +@pytest.mark.asyncio +async def test_on_the_fixed_site_measured_power_is_recorded_on_its_sensor(): + from unittest.mock import patch + + made = [] + real = server.CEM + + def capture(**kwargs): + made.append(kwargs) + return real(**kwargs) + + resource_manager, cem_end = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with patch.object(server, "CEM", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager(cem_end, _flexmeasures(), sensors) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0]["power_sensor_id"] == { + "ELECTRIC.POWER.L1": sensors["measured-power"]["id"] + } + + +@pytest.mark.asyncio +async def test_every_cem_the_server_makes_records_with_the_servers_recorder(): + """It outlives a connection, so what a reconnecting resource manager sends again is not recorded twice.""" + from unittest.mock import patch + + from aiohttp import web + + from flexmeasures_client.s2.measurements import MeasurementRecorder + + app = web.Application() + with patch.object(server, "SITE_PER_RESOURCE_MANAGER", True): + await server.configure_site_once(app) + assert isinstance(app[server.RECORDER], MeasurementRecorder) + await app[server.FM_CLIENT].close() + + made = [] + real = server.CEM + + def capture(**kwargs): + made.append(kwargs) + return real(**kwargs) + + recorder = MeasurementRecorder(_flexmeasures()) + resource_manager, cem_end = _connected() + sensors = {name: {"id": i} for i, name in enumerate(server.SITE_SENSORS, start=1)} + with patch.object(server, "CEM", side_effect=capture): + serving = asyncio.create_task( + server.serve_resource_manager( + cem_end, _flexmeasures(), sensors, recorder=recorder + ) + ) + await _until(lambda: made) + await cem_end._inbox.put(_CLOSED) + await asyncio.wait_for(serving, timeout=5) + assert made[0]["measurement_recorder"] is recorder + + +@pytest.mark.asyncio +@pytest.mark.parametrize("per_resource_manager", [False, True]) +async def test_a_connection_is_served_with_the_servers_recorder(per_resource_manager): + from unittest.mock import patch + + from aiohttp import web + from aiohttp.test_utils import TestClient, TestServer + + from flexmeasures_client.s2.measurements import MeasurementRecorder + + app = web.Application() + app.add_routes([web.get("/ws", server.websocket_handler)]) + app[server.FM_CLIENT] = _flexmeasures() + app[server.RECORDER] = MeasurementRecorder(app[server.FM_CLIENT]) + app[server.SENSORS] = {"power": {"id": 1}} + app[server.CONFIGURATOR] = server.FRBCSiteConfigurator( + app[server.FM_CLIENT], asset_type_id=6 + ) + served = AsyncMock() + with ( + patch.object(server, "serve_resource_manager", served), + patch.object(server, "SITE_PER_RESOURCE_MANAGER", per_resource_manager), + ): + async with TestClient(TestServer(app)) as client: + async with client.ws_connect("/ws"): + await _until(lambda: served.await_count) + assert served.await_args.kwargs["recorder"] is app[server.RECORDER] + if per_resource_manager: + assert served.await_args.kwargs["configurator"] is app[server.CONFIGURATOR] + else: + assert served.await_args.args[2] is app[server.SENSORS] diff --git a/tests/s2/test_frbc_charging_efficiency.py b/tests/s2/test_frbc_charging_efficiency.py index 34b382c6..2a899d82 100644 --- a/tests/s2/test_frbc_charging_efficiency.py +++ b/tests/s2/test_frbc_charging_efficiency.py @@ -301,20 +301,16 @@ async def test_a_fill_level_scale_is_applied_to_the_efficiency(): ] -@pytest.mark.asyncio -async def test_an_energy_unit_that_cannot_be_compared_to_watts_is_not_guessed_at( - caplog, -): - """A fill rate in a unit nothing converts is refused, rather than recorded thousandfold off.""" - system_description = _heater() - frbc = _frbc(system_description, energy_unit="therm") - - with caplog.at_level(logging.ERROR): - flex_model = await _asked_for(frbc, system_description) - - assert "charging-efficiency" not in flex_model - assert _recordings(frbc) == [] - assert "cannot be compared" in caplog.text +def test_an_energy_unit_that_cannot_be_compared_to_watts_is_refused_at_once(): + """A fill rate in a unit nothing converts would be recorded thousandfold off, so the handler is not made at all.""" + with pytest.raises(ValueError, match="energy_unit has to be one of"): + FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + energy_unit="therm", + ) def test_a_fill_rate_is_stated_in_watts_by_the_unit_the_fill_level_is_in(): @@ -326,8 +322,8 @@ def test_a_fill_rate_is_stated_in_watts_by_the_unit_the_fill_level_is_in(): fill_rate_in_watts(1, "therm") -def test_an_efficiency_is_taken_at_the_top_of_the_last_element(): - """A mode states an element per stretch of fill level, and the efficiency is the one at full tilt. +def test_an_efficiency_is_taken_at_the_top_of_the_element_that_applies(): + """A mode states an element per stretch of fill level, and the efficiency is that element's, at full tilt. The ratio, and not the slope over the element's range: a mode's ranges can be degenerate, where the actuator runs at one power and one fill rate only, @@ -353,6 +349,217 @@ def test_an_efficiency_is_taken_at_the_top_of_the_last_element(): ) ) - efficiency = operation_mode_conversion_efficiency(mode, 1, "kWh") + below, above = ( + operation_mode_conversion_efficiency(mode, 1, "kWh", fill_level) + for fill_level in (RANGE_TOP / 2, 1.5 * RANGE_TOP) + ) + + assert below == pytest.approx(HEATER_HEAT_IN_WATTS / HEATER_POWER) + assert above == pytest.approx(HEATER_HEAT_IN_WATTS / HEATER_POWER / 2) + + +@pytest.mark.asyncio +async def test_a_schedule_is_asked_for_against_the_efficiency_where_it_starts(): + """A heat pump that converts less once its buffer is warm is planned with the warm efficiency for a warm start.""" + cold = HEATER_HEAT_IN_WATTS / HEATER_POWER + warm = cold / 2 + mode = _operation_mode(HEATER_POWER, HEATER_FILL_RATE, "on") + mode.elements[0].fill_level_range.end_of_range = RANGE_TOP / 2 + mode.elements.append( + FRBCOperationModeElement( + fill_level_range=NumberRange( + start_of_range=RANGE_TOP / 2, end_of_range=RANGE_TOP + ), + fill_rate=NumberRange( + start_of_range=HEATER_FILL_RATE / 2, end_of_range=HEATER_FILL_RATE / 2 + ), + power_ranges=[ + PowerRange( + start_of_range=HEATER_POWER, + end_of_range=HEATER_POWER, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + ) + system_description = _system_description(mode) + frbc = _frbc(system_description) + # The latest status: a buffer warmed past the middle of its range since the one _frbc recorded. + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=0.75 * RANGE_TOP + ) + + await _asked_for(frbc, system_description) + + assert _recordings(frbc)[0]["values"] == [pytest.approx(warm)] + assert warm != pytest.approx(cold) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "energy_unit, soc_unit, per_unit", + [ + ("MJ", "kWh", 1 / 3.6), + ("Wh", "kWh", 1 / 1000), + ("J", "kWh", 1 / 3.6e6), + ("kWh", "kWh", 1.0), + ("MWh", "MWh", 1.0), + ], +) +async def test_a_state_of_charge_goes_out_in_a_unit_flexmeasures_takes( + energy_unit, soc_unit, per_unit +): + """FlexMeasures takes kWh or MWh only, so any other energy unit is converted at the edge, not refused by the server.""" + system_description = _heater() + frbc = _frbc(system_description, energy_unit=energy_unit) + flex_model = await _asked_for(frbc, system_description) + assert flex_model["soc-unit"] == soc_unit + assert flex_model["soc-at-start"] == pytest.approx(FILL_LEVEL_AT_START * per_unit) + assert flex_model["soc-max"] == pytest.approx(1.5 * RANGE_TOP * per_unit) + + +@pytest.mark.asyncio +async def test_a_lower_bound_below_zero_goes_out_in_the_same_unit(): + """Reported below its range, a storage widens the lower bound to take it in, which then needs the conversion too.""" + system_description = _heater() + frbc = _frbc(system_description, energy_unit="J") + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=-3.6e6 + ) + flex_model = await _asked_for(frbc, system_description) + assert flex_model["soc-at-start"] == pytest.approx(-1.0) + assert flex_model["soc-min"] == pytest.approx(-1.0) + + +@pytest.mark.asyncio +async def test_the_bounds_read_back_stay_in_the_recorded_unit(): + """Only what goes to FlexMeasures is converted: the schedule is matched against the operation modes in their own units.""" + from unittest.mock import patch - assert efficiency == pytest.approx(HEATER_HEAT_IN_WATTS / HEATER_POWER / 2) + system_description = _heater() + frbc = _frbc(system_description, energy_unit="MJ") + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule(system_description.message_id) + assert translate.call_args.kwargs["fill_level_bounds"] == ( + pytest.approx(0.0), + pytest.approx(1.5 * RANGE_TOP), + ) + + +@pytest.mark.asyncio +async def test_the_measured_state_of_charge_is_not_named_in_the_flex_model(): + """The state of charge goes out as a number, never as the sensor it is measured on. + + Named there, FlexMeasures would save the state of charge it plans onto that sensor, + beside the measurements, where a later schedule would read a plan as if it had been measured. + """ + system_description = _heater() + frbc = _frbc(system_description) + flex_model = await _asked_for(frbc, system_description) + assert "state-of-charge" not in flex_model + assert isinstance(flex_model["soc-at-start"], float) + + +@pytest.mark.asyncio +async def test_feeding_back_is_priced_by_its_own_sensor_where_there_is_one(): + system_description = _heater() + frbc = _frbc(system_description, production_price_sensor_id=42) + await frbc.trigger_schedule(system_description.message_id) + flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ + "flex_context" + ] + assert flex_context["production-price"] == {"sensor": 42} + assert flex_context["consumption-price"] == {"sensor": 2} + + +@pytest.mark.asyncio +async def test_without_its_own_sensor_feeding_back_is_priced_as_drawing(): + system_description = _heater() + frbc = _frbc(system_description) + await frbc.trigger_schedule(system_description.message_id) + flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ + "flex_context" + ] + assert flex_context["production-price"] == {"sensor": 2} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("relax", [False, True]) +async def test_the_site_capacity_is_relaxed_or_kept_hard_as_asked(relax): + """Sent either way: FlexMeasures reads a missing flag as its relax-constraints umbrella, which relaxes by default. + + So a home whose connection is a physical limit is only kept to it by an explicit False. + """ + system_description = _heater() + frbc = _frbc(system_description, relax_site_capacity=relax) + await frbc.trigger_schedule(system_description.message_id) + flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ + "flex_context" + ] + assert flex_context["relax-site-capacity-constraints"] is relax + + +@pytest.mark.asyncio +async def test_unset_the_site_capacity_is_left_to_flexmeasures(): + system_description = _heater() + frbc = _frbc(system_description) + await frbc.trigger_schedule(system_description.message_id) + flex_context = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs[ + "flex_context" + ] + assert "relax-site-capacity-constraints" not in flex_context + + +@pytest.mark.asyncio +async def test_a_schedule_overtaken_while_it_was_planned_is_not_translated(): + """A newer storage status arrived during the round trip, so turning this schedule into instructions is wasted.""" + from unittest.mock import patch + + from flexmeasures_client.s2.control_types.FRBC import STORAGE_STATUS_REQUESTS + + system_description = _heater() + frbc = _frbc(system_description) + request_number = frbc._count_request(STORAGE_STATUS_REQUESTS) + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def overtaken(**kwargs): + frbc._count_request(STORAGE_STATUS_REQUESTS) # a newer status, mid round trip + return planned + + frbc._fm_client.trigger_and_get_schedule = AsyncMock(side_effect=overtaken) + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions" + ) as translate: + await frbc.trigger_schedule( + system_description.message_id, request_number=request_number + ) + translate.assert_not_called() + frbc.send_instruction_batch.assert_not_awaited() + + +def test_a_batch_of_instructions_is_logged_once_and_by_name(caplog): + from flexmeasures_client.s2.control_types.FRBC.utils import ( + fm_schedule_to_instructions, + ) + + system_description = _heater() + with caplog.at_level(logging.DEBUG): + fm_schedule_to_instructions( + { + "start": "2024-01-01T00:00:00+00:00", + "duration": "PT1H", + "values": [0.0, HEATER_POWER, HEATER_POWER, 0.0], + }, + system_description, + initial_fill_level=FILL_LEVEL_AT_START, + ) + batches = [ + r.getMessage() + for r in caplog.records + if r.getMessage().startswith("Instructions: ") + ] + assert len(batches) == 1 + assert "'on'" in batches[0] and "'off'" in batches[0] diff --git a/tests/s2/test_frbc_declared_profiles.py b/tests/s2/test_frbc_declared_profiles.py index e783d876..201ed323 100644 --- a/tests/s2/test_frbc_declared_profiles.py +++ b/tests/s2/test_frbc_declared_profiles.py @@ -9,8 +9,9 @@ from __future__ import annotations +import asyncio import logging -from datetime import timedelta +from datetime import datetime, timedelta, timezone from unittest.mock import AsyncMock, patch import pandas as pd @@ -265,6 +266,7 @@ async def test_usage_forecast_is_recorded_from_a_resolution_boundary( async def test_leakage_behaviour_is_recorded_as_a_storage_efficiency( system_description, ): + """Recorded when a schedule is asked for, over the window it covers, rather than on arrival.""" frbc = _frbc(system_description, with_sensors=True) leakage = FRBCLeakageBehaviour( message_id=get_unique_id(), @@ -279,12 +281,22 @@ async def test_leakage_behaviour_is_recorded_as_a_storage_efficiency( ], ) - await frbc.send_leakage_behaviour(leakage) + await frbc.handle_message(leakage) + await asyncio.gather(*frbc.background_tasks) + assert not [ + c + for c in frbc._fm_client.post_sensor_data.await_args_list + if c.kwargs.get("sensor_id") == LEAKAGE_SENSOR_ID + ] # nothing on arrival + + start = datetime(2024, 1, 1, 12, 15, tzinfo=timezone.utc) + assert await frbc._record_storage_efficiency(start, (RANGE_BOTTOM + RANGE_TOP) / 2) posted = _posted(frbc, LEAKAGE_SENSOR_ID) assert posted["unit"] == "%" assert len(posted["values"]) == 1 - assert posted["start"].minute % 15 == 0 and posted["start"].second == 0 + assert posted["start"] == start + assert posted["duration"] == frbc._schedule_duration # --- Without the sensors: nothing recorded, and still no exception --- @@ -397,7 +409,7 @@ async def test_the_energy_unit_is_the_deployment_s_to_choose(system_description) async def test_a_schedule_is_asked_for_in_the_unit_the_fill_level_is_recorded_in( system_description, ): - """soc-unit has to follow energy_unit, or the scheduler plans against bounds a thousand times its measurements.""" + """soc-unit follows energy_unit where FlexMeasures takes it, so the values stated inline are in the unit they are recorded in.""" frbc = _frbc(system_description, with_sensors=True) # A schedule is only asked for once a storage status has said where to start from. frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( @@ -601,3 +613,58 @@ async def test_an_actuator_s_fill_rate_is_posted_in_the_fill_rate_unit( ) # The power sensor really does carry a power, and is untouched by this. assert _posted(frbc, 1)["unit"] == frbc.power_unit + + +@pytest.mark.asyncio +async def test_a_storage_efficiency_already_recorded_for_the_window_is_not_recorded_again( + system_description, +): + frbc = _frbc(system_description, with_sensors=True) + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from="2024-01-01T12:00:00+00:00", + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange( + start_of_range=RANGE_BOTTOM, end_of_range=RANGE_TOP + ), + leakage_rate=0.01, + ) + ], + ) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + start = datetime(2024, 1, 1, 12, 15, tzinfo=timezone.utc) + assert await frbc._record_storage_efficiency(start, 50.0) + assert await frbc._record_storage_efficiency(start, 50.0) + posts = [ + c + for c in frbc._fm_client.post_sensor_data.await_args_list + if c.kwargs.get("sensor_id") == LEAKAGE_SENSOR_ID + ] + assert len(posts) == 1 + + +@pytest.mark.asyncio +async def test_a_storage_efficiency_that_could_not_be_recorded_is_not_planned_against( + system_description, caplog +): + """And it is tried again next time, rather than remembered as recorded.""" + frbc = _frbc(system_description, with_sensors=True) + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from="2024-01-01T12:00:00+00:00", + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange( + start_of_range=RANGE_BOTTOM, end_of_range=RANGE_TOP + ), + leakage_rate=0.01, + ) + ], + ) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + frbc._fm_client.post_sensor_data.side_effect = ConnectionError("unreachable") + start = datetime(2024, 1, 1, 12, 15, tzinfo=timezone.utc) + assert await frbc._record_storage_efficiency(start, 50.0) is False + assert frbc._storage_efficiency_recorded is None + assert "could not record" in caplog.text diff --git a/tests/s2/test_frbc_element_selection.py b/tests/s2/test_frbc_element_selection.py new file mode 100644 index 00000000..992df47e --- /dev/null +++ b/tests/s2/test_frbc_element_selection.py @@ -0,0 +1,599 @@ +"""Which element of an operation mode, or of a leakage behaviour, answers a question. + +S2 lets those elements cover the fill-level range in pieces, and lets them arrive in any order: +s2python's own contiguity check sorts a copy of them and leaves the list as it was sent. +So no answer may depend on the order they are listed in. + +Where a fill level is known, the element that applies at it answers, +and a boundary two elements share belongs to the upper one. +Where FlexMeasures takes one number for a whole schedule, the most cautious element answers: +the one converting drawn power into stored energy least, or the one leaking most. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock + +import pytest +from s2python.common import Commodity, CommodityQuantity, NumberRange, PowerRange +from s2python.frbc import ( + FRBCActuatorDescription, + FRBCActuatorStatus, + FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, + FRBCOperationMode, + FRBCOperationModeElement, + FRBCStorageDescription, + FRBCStorageStatus, + FRBCSystemDescription, +) + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple +from flexmeasures_client.s2.control_types.FRBC.utils import ( + element_for_fill_level, + operation_mode_conversion_efficiency, + power_to_fill_rate_with_metrics, +) +from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, +) +from flexmeasures_client.s2.utils import get_unique_id + +POWER_SENSOR = 1 +CHARGING_EFFICIENCY_SENSOR = 7 +RANGE_TOP = 80.0 +KWH_PER_SECOND_IN_WATTS = 3.6e6 + + +def _element( + lowest: float, + highest: float, + power: float, + heat_in_watts: float, +) -> FRBCOperationModeElement: + """A stretch of fill level over which a mode runs from nothing up to a power, delivering up to some heat.""" + return FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=lowest, end_of_range=highest), + fill_rate=NumberRange( + start_of_range=0, end_of_range=heat_in_watts / KWH_PER_SECOND_IN_WATTS + ), + power_ranges=[ + PowerRange( + start_of_range=0, + end_of_range=power, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + + +#: A heat pump that derates as its buffer warms: 2000 W at a COP of 4 below 40, 1200 W at a COP of 3 above. +COLD = _element(0, 40, power=2000.0, heat_in_watts=8000.0) +WARM = _element(40, RANGE_TOP, power=1200.0, heat_in_watts=3600.0) + + +def _mode( + *elements: FRBCOperationModeElement, label: str = "heat" +) -> FRBCOperationMode: + return FRBCOperationMode( + id=get_unique_id(), + diagnostic_label=label, + elements=list(elements), + abnormal_condition_only=False, + ) + + +def _range(element) -> tuple[float, float]: + return ( + element.fill_level_range.start_of_range, + element.fill_level_range.end_of_range, + ) + + +# --------------------------------------------------------------------------- +# The selector itself +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("fill_level", [-5.0, 0.0, 20.0, 40.0, 60.0, 80.0, 95.0]) +def test_the_element_for_a_fill_level_does_not_depend_on_the_order_listed(fill_level): + assert _range(element_for_fill_level([COLD, WARM], fill_level)) == _range( + element_for_fill_level([WARM, COLD], fill_level) + ) + + +@pytest.mark.parametrize( + "fill_level, expected", + [ + (0.0, (0, 40)), + (39.9, (0, 40)), + # A boundary two elements share belongs to the upper one, + (40.0, (40, RANGE_TOP)), + # except at the very top, where there is no upper one. + (RANGE_TOP, (40, RANGE_TOP)), + # Outside the range, the nearest element answers: a storage may be outside its declared range. + (-5.0, (0, 40)), + (95.0, (40, RANGE_TOP)), + ], +) +def test_the_element_that_applies_at_a_fill_level(fill_level, expected): + for listed in ([COLD, WARM], [WARM, COLD]): + assert _range(element_for_fill_level(listed, fill_level)) == expected + + +def test_an_element_that_is_a_single_point_applies_at_that_point(): + """S2 accepts an element whose range is one fill level, as long as the elements stay contiguous. + + At the top of a buffer, a heat pump may run at its minimum only, and say so with such an element. + It is the upper of the two at their shared boundary, so it applies there. + """ + below = _element(0, RANGE_TOP, power=2000.0, heat_in_watts=8000.0) + at_the_top = _element(RANGE_TOP, RANGE_TOP, power=300.0, heat_in_watts=900.0) + for listed in ([below, at_the_top], [at_the_top, below]): + assert element_for_fill_level(listed, RANGE_TOP) is at_the_top + assert element_for_fill_level(listed, RANGE_TOP - 1) is below + + +def test_past_either_end_the_answer_is_that_at_the_end(): + """A storage outside its declared range is answered as at the nearest end, so the answer does not jump there.""" + at_the_bottom = _element(0, 0, power=0.0, heat_in_watts=0.0) + middle = _element(0, RANGE_TOP, power=2000.0, heat_in_watts=8000.0) + at_the_top = _element(RANGE_TOP, RANGE_TOP, power=300.0, heat_in_watts=900.0) + for listed in ( + [at_the_bottom, middle, at_the_top], + [at_the_top, middle, at_the_bottom], + ): + assert element_for_fill_level(listed, RANGE_TOP + 15) is at_the_top + assert element_for_fill_level(listed, -5.0) is element_for_fill_level( + listed, 0.0 + ) + # At the bottom, the point element shares its only fill level with the element above it, which is upper. + assert element_for_fill_level(listed, 0.0) is middle + + +def test_a_fill_level_in_a_gap_is_answered_by_the_nearer_element_and_on_a_tie_the_lower(): + """S2 rules gaps out, but the selector takes raw elements and should not depend on their order there either.""" + low = _element(0, 30, power=2000.0, heat_in_watts=8000.0) + high = _element(50, RANGE_TOP, power=1200.0, heat_in_watts=3600.0) + for listed in ([low, high], [high, low]): + assert element_for_fill_level(listed, 35.0) is low + assert element_for_fill_level(listed, 45.0) is high + assert element_for_fill_level(listed, 40.0) is low + + +def test_leakage_elements_are_selected_the_same_way(): + low = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=40), + leakage_rate=1e-5, + ) + high = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=40, end_of_range=RANGE_TOP), + leakage_rate=3e-5, + ) + assert element_for_fill_level([high, low], 10.0) is low + assert element_for_fill_level([low, high], 40.0) is high + + +# --------------------------------------------------------------------------- +# Where a fill level is known: the measured power, and the schedule translation +# --------------------------------------------------------------------------- + + +def _system_description(*modes: FRBCOperationMode) -> FRBCSystemDescription: + return FRBCSystemDescription( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + actuators=[ + FRBCActuatorDescription( + id=get_unique_id(), + supported_commodities=[Commodity.ELECTRICITY], + operation_modes=list(modes), + transitions=[], + timers=[], + ) + ], + storage=FRBCStorageDescription( + provides_leakage_behaviour=False, + provides_fill_level_target_profile=False, + provides_usage_forecast=False, + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + ), + ) + + +def _frbc(system_description: FRBCSystemDescription, **kwargs) -> FRBCSimple: + frbc = FRBCSimple( + power_sensor_id=POWER_SENSOR, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + **kwargs, + ) + frbc._fm_client = AsyncMock(FlexMeasuresClient) + frbc._logger = logging.getLogger("FRBCSimple") + frbc._system_description_history = { + system_description.message_id: system_description + } + # What the power sensor means by a sign is not the question here. + frbc._as_the_power_sensor_reads_it = AsyncMock(side_effect=lambda power: power) + return frbc + + +async def _measured_power(frbc: FRBCSimple, mode: FRBCOperationMode, factor: float): + actuator = list(frbc._system_description_history.values())[-1].actuators[0] + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=mode.id, + operation_mode_factor=factor, + ) + ) + posts = [ + call + for call in frbc._fm_client.post_sensor_data.await_args_list + if (call.args[:1] or (call.kwargs.get("sensor_id"),))[0] == POWER_SENSOR + ] + assert len(posts) == 1, posts + return posts[0].kwargs["values"][0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("listed", [(COLD, WARM), (WARM, COLD)]) +async def test_the_measured_power_is_that_of_the_element_at_the_present_fill_level( + listed, +): + """A warm buffer runs on the derated element, so full tilt there is 1200 W, not 2000 W.""" + mode = _mode(*listed) + frbc = _frbc(_system_description(mode)) + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=70.0 + ) + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(1200.0) + + +@pytest.mark.asyncio +async def test_without_a_fill_level_the_measured_power_is_a_guess_said_out_loud(caplog): + """Before any storage status, the bottom element answers, and the log says it was a guess.""" + mode = _mode(WARM, COLD) + frbc = _frbc(_system_description(mode)) + with caplog.at_level(logging.WARNING, logger="FRBCSimple"): + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(2000.0) + assert any( + "no usable FRBC.StorageStatus" in record.getMessage() + for record in caplog.records + ), caplog.records + + +@pytest.mark.parametrize("fill_level", [20.0, 40.0, 60.0]) +def test_a_schedule_is_translated_the_same_whichever_order_the_elements_come_in( + fill_level, +): + forward = power_to_fill_rate_with_metrics(_mode(COLD, WARM), 1000.0, fill_level) + backward = power_to_fill_rate_with_metrics(_mode(WARM, COLD), 1000.0, fill_level) + assert forward[:4] == pytest.approx(backward[:4]) + assert _range(forward[4]) == _range(backward[4]) + + +# --------------------------------------------------------------------------- +# Where one number covers a whole schedule: the charging and the storage efficiency +# --------------------------------------------------------------------------- + + +def _efficiency(mode: FRBCOperationMode, fill_level: float) -> float | None: + return operation_mode_conversion_efficiency( + mode, fill_level_scale=1, energy_unit="kWh", fill_level=fill_level + ) + + +@pytest.mark.parametrize("fill_level", [-5.0, 20.0, 40.0, 60.0, 95.0]) +def test_the_conversion_efficiency_does_not_depend_on_the_order_listed(fill_level): + assert _efficiency(_mode(COLD, WARM), fill_level) == _efficiency( + _mode(WARM, COLD), fill_level + ) + + +@pytest.mark.parametrize("fill_level, cop", [(20.0, 4.0), (60.0, 3.0)]) +def test_a_mode_states_the_efficiency_of_the_element_where_the_schedule_starts( + fill_level, cop +): + """A COP of 4 when cold and 3 when warm is 4 for a schedule starting cold, and 3 for one starting warm. + + Not the lowest of the two for both: that overstates what a cold buffer needs, + so the plan overshoots its upper band and buys more than it uses, + and against a usage it cannot relax it can find a problem the heat pump can meet infeasible. + The start is what matters most, since it runs before the next storage status asks for a new plan. + """ + assert _efficiency(_mode(COLD, WARM), fill_level) == pytest.approx(cop) + + +def test_an_element_that_tapers_off_applies_only_where_it_applies(): + """Near its highest flow temperature a heat pump may barely fill its buffer, and that does not hold for a cold one.""" + tapering = _element(50, 60, power=1200.0, heat_in_watts=10.0) + mode = _mode(_element(0, 50, power=1200.0, heat_in_watts=4800.0), tapering) + assert _efficiency(mode, 30.0) == pytest.approx(4.0) + assert _efficiency(mode, 55.0) == pytest.approx(10.0 / 1200.0) + + +@pytest.mark.parametrize( + "heat_in_watts", + [0.0, -50.0], + ids=[ + "standby, drawing power and delivering nothing", + "circulation, drawing power and losing heat", + ], +) +def test_a_mode_that_draws_power_without_filling_states_no_efficiency(heat_in_watts): + """Drawing power while the storage does not fill is not a conversion one ratio can state.""" + standby = FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + fill_rate=NumberRange( + start_of_range=heat_in_watts / KWH_PER_SECOND_IN_WATTS, + end_of_range=heat_in_watts / KWH_PER_SECOND_IN_WATTS, + ), + power_ranges=[ + PowerRange( + start_of_range=5.0, + end_of_range=5.0, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + assert _efficiency(_mode(standby), 10.0) is None + + +@pytest.mark.asyncio +async def test_a_standby_mode_does_not_drag_the_recorded_efficiency_to_zero(): + """Off, standby at 5 W, and on at a COP of 4.3: the scheduler is told 4.3, not 0. + + At 0, the stock could not rise at any power, so the cheapest plan would be never to run the heat pump, + and with the fill-level band priced rather than hard, that plan would be feasible rather than refused. + """ + off = _mode(_element(0, RANGE_TOP, power=0.0, heat_in_watts=0.0), label="off") + standby_element = FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + fill_rate=NumberRange(start_of_range=0, end_of_range=0), + power_ranges=[ + PowerRange( + start_of_range=5.0, + end_of_range=5.0, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + standby = _mode(standby_element, label="standby") + on = _mode(_element(0, RANGE_TOP, power=883.7, heat_in_watts=3800.0), label="on") + system_description = _system_description(off, standby, on) + frbc = _frbc( + system_description, charging_efficiency_sensor_id=CHARGING_EFFICIENCY_SENSOR + ) + start = datetime(2024, 1, 2, tzinfo=timezone.utc) + assert ( + await frbc._record_charging_efficiency(system_description, start, 30.0) is True + ) + recorded = [ + call.kwargs["values"] + for call in frbc._fm_client.post_sensor_data.await_args_list + if (call.args[:1] or (call.kwargs.get("sensor_id"),))[0] + == CHARGING_EFFICIENCY_SENSOR + ] + assert recorded == [[pytest.approx(3800.0 / 883.7)]] + + +def _leakage(*elements: FRBCLeakageBehaviourElement) -> FRBCLeakageBehaviour: + return FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=list(elements), + ) + + +LEAKS_LITTLE = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=40), leakage_rate=3e-5 +) +LEAKS_MOST = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=40, end_of_range=60), leakage_rate=9e-5 +) +LEAKS_SOME = FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=60, end_of_range=RANGE_TOP), + leakage_rate=6e-5, +) + + +# The storage efficiencies below are compared exactly, not approximately. +# They are the same arithmetic on the same numbers, so equal means equal, +# and a leakage rate per second becomes an efficiency so close to 100% that pytest.approx's default tolerance +# would call two different leakage rates the same, and pass with the defect these tests are about. + + +def test_the_storage_efficiency_does_not_depend_on_the_order_listed(): + efficiency = leakage_behaviour_to_storage_efficiency( + _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME), resolution=timedelta(minutes=15) + ) + assert efficiency == leakage_behaviour_to_storage_efficiency( + _leakage(LEAKS_SOME, LEAKS_LITTLE, LEAKS_MOST), + resolution=timedelta(minutes=15), + ) + + +def test_the_storage_efficiency_states_the_leakiest_element(): + """The leakiest element is not always the top one, so it is looked for rather than assumed to come last.""" + leakiest = leakage_behaviour_to_storage_efficiency( + _leakage(LEAKS_MOST), resolution=timedelta(minutes=15), fill_level_scale=1 + ) + # Alone, the leakiest element is measured against its own top; together, against the range's top. + together = leakage_behaviour_to_storage_efficiency( + _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME), + resolution=timedelta(minutes=15), + ) + top_only = leakage_behaviour_to_storage_efficiency( + _leakage( + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + leakage_rate=LEAKS_MOST.leakage_rate, + ) + ), + resolution=timedelta(minutes=15), + ) + assert together == top_only + assert leakiest != top_only + + +# --------------------------------------------------------------------------- +# Which fill level: the latest, as the resource manager states it +# --------------------------------------------------------------------------- + + +def _report_fill_level(frbc: FRBCSimple, fill_level: float) -> None: + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=fill_level + ) + + +@pytest.mark.asyncio +async def test_the_measured_power_follows_the_latest_storage_status(): + """A buffer that has warmed since its first status runs on the derated element now.""" + mode = _mode(COLD, WARM) + frbc = _frbc(_system_description(mode)) + _report_fill_level(frbc, 10.0) + _report_fill_level(frbc, 70.0) + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(1200.0) + + +@pytest.mark.asyncio +async def test_the_fill_level_is_compared_in_the_resource_managers_own_units(): + """The scale converts fill levels for FlexMeasures; the elements' ranges are still in the resource manager's units. + + At 7 in its own units the buffer is cold, though scaled by 10 it would read as 70, which is warm. + """ + mode = _mode(COLD, WARM) + frbc = _frbc(_system_description(mode), fill_level_scale=10) + _report_fill_level(frbc, 7.0) + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(2000.0) + + +@pytest.mark.asyncio +async def test_with_one_element_there_is_nothing_to_guess(caplog): + mode = _mode(_element(0, RANGE_TOP, power=2000.0, heat_in_watts=8000.0)) + frbc = _frbc(_system_description(mode)) + with caplog.at_level(logging.WARNING, logger="FRBCSimple"): + await _measured_power(frbc, mode, factor=1.0) + assert not caplog.records, caplog.records + + +@pytest.mark.asyncio +async def test_a_fill_level_that_is_not_a_number_counts_as_unknown(caplog): + """s2python lets a NaN fill level through, and no element applies at it, so it is treated as no fill level at all.""" + mode = _mode(WARM, COLD) + frbc = _frbc(_system_description(mode)) + _report_fill_level(frbc, float("nan")) + with caplog.at_level(logging.WARNING, logger="FRBCSimple"): + assert await _measured_power(frbc, mode, factor=1.0) == pytest.approx(2000.0) + assert any("no usable FRBC.StorageStatus" in r.getMessage() for r in caplog.records) + + +def test_no_element_applies_at_a_fill_level_that_is_not_a_number(): + with pytest.raises(ValueError, match="not a number"): + element_for_fill_level([COLD, WARM], float("nan")) + + +def test_the_storage_efficiency_at_a_fill_level_is_that_elements(): + """Where the fill level is known, the leakage element that applies there answers, not the leakiest.""" + leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) + at_20 = leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15), fill_level=20.0 + ) + only_little = leakage_behaviour_to_storage_efficiency( + _leakage( + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=RANGE_TOP), + leakage_rate=LEAKS_LITTLE.leakage_rate, + ) + ), + resolution=timedelta(minutes=15), + ) + assert at_20 == only_little + assert at_20 != leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15) + ) + + +@pytest.mark.asyncio +async def test_the_handler_records_leakage_at_the_latest_fill_level(): + leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) + frbc = _frbc(_system_description(_mode(COLD, WARM)), leakage_behaviour_sensor_id=9) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + await frbc._record_storage_efficiency( + datetime(2024, 1, 2, tzinfo=timezone.utc), 20.0 + ) + posted = [ + call.kwargs["values"] + for call in frbc._fm_client.post_sensor_data.await_args_list + if call.kwargs.get("sensor_id") == 9 + ] + assert posted == [ + [ + leakage_behaviour_to_storage_efficiency( + leakage, + resolution=timedelta(minutes=15), + fill_level=20.0, + share_of=max(20.0, _middle_of_the_range(frbc)), + ) + ] + ] + + +def _middle_of_the_range(frbc) -> float: + storage_range = list(frbc._system_description_history.values())[ + -1 + ].storage.fill_level_range + return (storage_range.start_of_range + storage_range.end_of_range) / 2 + + +@pytest.mark.asyncio +async def test_a_near_empty_start_takes_the_loss_as_a_share_of_the_middle_of_the_range(): + """A share of a near-empty start would leave almost nothing per step, and overstate the loss once the plan fills up.""" + leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) + frbc = _frbc(_system_description(_mode(COLD, WARM)), leakage_behaviour_sensor_id=9) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + start = datetime(2024, 1, 2, tzinfo=timezone.utc) + await frbc._record_storage_efficiency(start, 0.01) + (posted,) = [ + call.kwargs["values"][0] + for call in frbc._fm_client.post_sensor_data.await_args_list + if call.kwargs.get("sensor_id") == 9 + ] + middle = _middle_of_the_range(frbc) + assert middle > 0.01 + assert posted == leakage_behaviour_to_storage_efficiency( + leakage, + resolution=timedelta(minutes=15), + fill_level=0.01, + share_of=middle, + ) + assert posted > 99 + + +@pytest.mark.asyncio +async def test_a_start_above_the_middle_takes_the_loss_as_a_share_of_the_start(): + leakage = _leakage(LEAKS_LITTLE, LEAKS_MOST, LEAKS_SOME) + frbc = _frbc(_system_description(_mode(COLD, WARM)), leakage_behaviour_sensor_id=9) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage + storage_range = list(frbc._system_description_history.values())[ + -1 + ].storage.fill_level_range + high = (storage_range.end_of_range + _middle_of_the_range(frbc)) / 2 + await frbc._record_storage_efficiency( + datetime(2024, 1, 2, tzinfo=timezone.utc), high + ) + (posted,) = [ + call.kwargs["values"][0] + for call in frbc._fm_client.post_sensor_data.await_args_list + if call.kwargs.get("sensor_id") == 9 + ] + assert posted == leakage_behaviour_to_storage_efficiency( + leakage, resolution=timedelta(minutes=15), fill_level=high, share_of=high + ) diff --git a/tests/s2/test_frbc_simple_planning.py b/tests/s2/test_frbc_simple_planning.py index dab9231c..428c76ee 100644 --- a/tests/s2/test_frbc_simple_planning.py +++ b/tests/s2/test_frbc_simple_planning.py @@ -31,6 +31,8 @@ from flexmeasures_client.s2.utils import get_unique_id POWER_SENSOR_ID = 1 +#: A clock set before the descriptions below are valid, so that their own times decide where a schedule starts. +BEFORE_THE_EXAMPLES = datetime(2022, 12, 1, tzinfo=timezone.utc) SOC_SENSOR_ID = 3 @@ -175,6 +177,7 @@ async def test_a_storage_status_before_any_description_still_records_the_measure def test_a_schedule_starts_on_a_resolution_boundary(valid_from, expected_start): """An unaligned start leaves FlexMeasures with no prices for the window, and the job fails.""" frbc = frbc_simple() + frbc.now = lambda: BEFORE_THE_EXAMPLES start = frbc.schedule_start(datetime.fromisoformat(valid_from)) @@ -187,6 +190,7 @@ def test_a_schedule_starts_on_a_resolution_boundary(valid_from, expected_start): @pytest.mark.asyncio async def test_the_schedule_is_asked_for_from_an_aligned_start(): frbc = frbc_simple() + frbc.now = lambda: BEFORE_THE_EXAMPLES await frbc.handle_message(system_description("2022-12-02T09:07:33.512+00:00")) await settle(frbc) @@ -195,3 +199,488 @@ async def test_the_schedule_is_asked_for_from_an_aligned_start(): start = frbc._fm_client.trigger_and_get_schedule.await_args.kwargs["start"] assert start == datetime(2022, 12, 3, 9, 0, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +async def test_planning_without_a_storage_status_is_logged_not_printed(capsys, caplog): + """A handler runs unattended, so giving up on a schedule goes to the log, where it can be found. + + It used to go to standard output, which a deployment does not keep. + """ + frbc = frbc_simple() + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await settle(frbc) + system_description_id = list(frbc._system_description_history)[-1] + + with caplog.at_level("WARNING", logger="FRBCSimple"): + await frbc.trigger_schedule(system_description_id) + + frbc._fm_client.trigger_and_get_schedule.assert_not_awaited() + assert capsys.readouterr().out == "" + assert any( + record.levelname == "WARNING" and "FRBC.StorageStatus" in record.getMessage() + for record in caplog.records + ), caplog.records + + +@pytest.mark.asyncio +async def test_a_revoked_object_is_known_by_its_id_as_text(): + """Revocations are recorded by the id as text, which is how every other id in a handler is kept.""" + from s2python.common import RevokableObjects, RevokeObject + + frbc = frbc_simple() + object_id = get_unique_id() + await frbc.handle_message( + RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=object_id, + ) + ) + assert frbc.is_revoked(str(object_id)) + + +def test_a_handler_recognises_a_message_in_any_form_it_accepts(): + """supports_message takes a model, a dict or JSON text, and reads the type from each the same way.""" + frbc = frbc_simple() + description = system_description("2022-12-02T00:00:00+00:00") + as_dict = description.model_dump(mode="json") + assert frbc.supports_message(description) + assert frbc.supports_message(as_dict) + assert frbc.supports_message(description.model_dump_json()) + assert not frbc.supports_message(as_dict | {"message_type": "Handshake"}) + + +@pytest.mark.asyncio +async def test_the_cem_records_a_revocation_by_its_id_as_text(): + """With no control type active, the CEM itself takes the revocation and hands it to the handler it is for.""" + from s2python.common import RevokableObjects, RevokeObject + + from flexmeasures_client.s2.cem import CEM + + cem = CEM(fm_client=frbc_simple()._fm_client) + frbc = frbc_simple() + cem.register_control_type(frbc) + object_id = get_unique_id() + await cem.handle_message( + RevokeObject( + message_id=get_unique_id(), + object_type=RevokableObjects.FRBC_Instruction, + object_id=object_id, + ) + ) + assert frbc.is_revoked(str(object_id)) + + +def test_a_handler_turns_away_a_model_it_does_not_handle(): + from s2python.common import EnergyManagementRole, Handshake + + handshake = Handshake( + message_id=get_unique_id(), + role=EnergyManagementRole.CEM, + supported_protocol_versions=["0.0.2-beta"], + ) + assert not frbc_simple().supports_message(handshake) + + +def test_a_schedule_never_starts_before_now(): + """A description days old still plans from the next boundary, rather than a window that has already passed. + + Nor from the start of the step under way: the fill level reported now is taken as the level at the start, + so the plan would count on the whole step where only part of it is left. + """ + frbc = frbc_simple() + frbc.now = lambda: datetime(2022, 12, 10, 14, 7, tzinfo=timezone.utc) + assert frbc.schedule_start( + datetime(2022, 12, 2, 9, 0, tzinfo=timezone.utc) + ) == datetime(2022, 12, 10, 14, 15, tzinfo=timezone.utc) + + +def test_a_schedule_asked_for_on_a_boundary_starts_there(): + frbc = frbc_simple() + frbc.now = lambda: datetime(2022, 12, 10, 14, 15, tzinfo=timezone.utc) + assert frbc.schedule_start( + datetime(2022, 12, 2, 9, 0, tzinfo=timezone.utc) + ) == datetime(2022, 12, 10, 14, 15, tzinfo=timezone.utc) + + +def test_a_schedule_whose_shifted_start_is_in_the_step_under_way_starts_at_the_next_boundary(): + frbc = frbc_simple() + frbc._valid_from_shift = timedelta(0) + frbc.now = lambda: datetime(2022, 12, 10, 14, 7, tzinfo=timezone.utc) + assert frbc.schedule_start( + datetime(2022, 12, 10, 14, 6, tzinfo=timezone.utc) + ) == datetime(2022, 12, 10, 14, 15, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +async def test_a_status_before_its_description_is_planned_for_when_the_description_arrives(): + """The order the example resource manager used to send them in, which left it without a single instruction.""" + frbc = frbc_simple() + await frbc.handle_message(storage_status()) + await settle(frbc) + frbc._fm_client.trigger_and_get_schedule.assert_not_awaited() + + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await settle(frbc) + + frbc._fm_client.trigger_and_get_schedule.assert_awaited_once() + assert not frbc._sending_queue.empty() + + +@pytest.mark.asyncio +async def test_a_changed_description_is_planned_for_against_the_latest_status(): + """What the device can do changed, so the plan made for what it could do before no longer answers.""" + frbc = frbc_simple() + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await settle(frbc) + assert frbc._fm_client.trigger_and_get_schedule.await_count == 1 + + await frbc.handle_message(system_description("2022-12-02T06:00:00+00:00")) + await settle(frbc) + + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + + +@pytest.mark.asyncio +async def test_the_soc_is_not_reposted_when_a_description_asks_for_a_schedule(): + """The latest status is planned from, not recorded again as if it had been measured now.""" + frbc = frbc_simple() + await frbc.handle_message(storage_status()) + await settle(frbc) + posts_after_the_status = frbc._fm_client.post_measurements.await_count + + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await settle(frbc) + + assert frbc._fm_client.post_measurements.await_count == posts_after_the_status + + +def test_a_step_boundary_in_the_repeated_autumn_hour_is_not_an_hour_early(): + """At 02:07 the second time round, on the night the clocks go back, the step began at 02:00 of that second pass.""" + from zoneinfo import ZoneInfo + + frbc = frbc_simple() + amsterdam = ZoneInfo("Europe/Amsterdam") + second_pass = datetime(2026, 10, 25, 1, 7, tzinfo=timezone.utc).astimezone( + amsterdam + ) + assert second_pass.fold == 1 + aligned = frbc.align_to_resolution(second_pass) + assert aligned.astimezone(timezone.utc) == datetime( + 2026, 10, 25, 1, 0, tzinfo=timezone.utc + ) + + +@pytest.mark.asyncio +async def test_a_plan_asked_for_by_a_late_description_supersedes_one_in_flight(): + """A status's plan is still out when a changed description asks for its own: only the newer batch goes out.""" + frbc = frbc_simple() + release = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def slow_then_quick(**kwargs): + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await release.wait() + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = slow_then_quick + batches: list[tuple[int, int]] = [] + send = frbc.send_instruction_batch + + async def recording(*args, **kwargs): + sent = await send(*args, **kwargs) + batches.append((kwargs["request_number"], sent)) + return sent + + frbc.send_instruction_batch = recording + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + await frbc.handle_message(system_description("2022-12-02T06:00:00+00:00")) + await asyncio.sleep(0.05) + release.set() + await settle(frbc) + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + sent = dict(batches) + assert sent[2] > 0 # the description's plan, which is the newer request + # The status's plan, overtaken while it was out, sends nothing, whether it is dropped before or after translation. + assert sent.get(1, 0) == 0 + + +@pytest.mark.asyncio +async def test_a_state_of_charge_is_recorded_as_a_reading_at_an_instant(): + """On a sensor that records instants, and over a minute on one with a resolution, which refuses an instant.""" + for resolution, duration in ( + ("PT0M", timedelta(0)), + ("PT15M", timedelta(minutes=1)), + ): + frbc = frbc_simple() + frbc._fm_client.get_sensor.return_value = {"event_resolution": resolution} + await frbc.handle_message(storage_status()) + await settle(frbc) + posted = frbc._fm_client.post_measurements.await_args.kwargs + assert posted["duration"] == duration + + +@pytest.mark.asyncio +async def test_interleaved_statuses_about_one_step_get_belief_times_a_second_apart(): + """A status about step A, one about step B, then A again, all within a second: the two about A must not collide.""" + from s2python.frbc import FRBCActuatorStatus + + from flexmeasures_client.s2.utils import get_unique_id + + frbc = frbc_simple() + frbc._power_sensor_takes_consumption_as_positive = True + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + actuator = frbc._system_description_history[ + list(frbc._system_description_history)[-1] + ].actuators[0] + mode = actuator.operation_modes[0].id + a = datetime(2022, 12, 2, 0, 5, tzinfo=timezone.utc) + b = datetime(2022, 12, 2, 0, 20, tzinfo=timezone.utc) + for when in (a, b, a): + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=mode, + operation_mode_factor=1.0, + transition_timestamp=when, + ) + ) + priors = [ + c.kwargs["prior"] + for c in frbc._fm_client.post_sensor_data.await_args_list + if c.kwargs.get("start") == a + ] + assert len(priors) == 2 + assert priors[1] - priors[0] >= timedelta(seconds=1) + + +def test_a_step_boundary_is_on_the_local_clock_where_the_zone_is_half_an_hour_off_utc(): + """FlexMeasures floors a series on the sensor's clock, so an hourly step in Kolkata starts on the hour there.""" + from zoneinfo import ZoneInfo + + frbc = frbc_simple() + frbc._resolution = timedelta(hours=1) + moment = datetime(2026, 11, 5, 14, 40, tzinfo=ZoneInfo("Asia/Kolkata")) + assert frbc.align_to_resolution(moment) == datetime( + 2026, 11, 5, 14, 0, tzinfo=ZoneInfo("Asia/Kolkata") + ) + + +def test_a_boundary_in_the_repeated_hour_is_the_later_one_not_after_the_moment(): + """At two-hourly steps, 03:10 on the night the clocks go back is past 02:00 of the second pass, which is the boundary meant.""" + from zoneinfo import ZoneInfo + + frbc = frbc_simple() + frbc._resolution = timedelta(hours=2) + amsterdam = ZoneInfo("Europe/Amsterdam") + moment = datetime(2026, 10, 25, 3, 10, tzinfo=amsterdam) + aligned = frbc.align_to_resolution(moment) + assert aligned.astimezone(timezone.utc) == datetime( + 2026, 10, 25, 1, 0, tzinfo=timezone.utc + ) + + +def test_a_naive_moment_is_floored_on_its_clock(): + frbc = frbc_simple() + assert frbc.align_to_resolution(datetime(2026, 1, 1, 12, 7)) == datetime( + 2026, 1, 1, 12, 0 + ) + + +@pytest.mark.asyncio +async def test_a_state_of_charge_sensor_stating_no_duration_is_posted_over_a_minute( + caplog, +): + frbc = frbc_simple() + frbc._fm_client.get_sensor.return_value = {"event_resolution": "fortnightly"} + assert await frbc._soc_duration() == timedelta(minutes=1) + assert "not a duration" in caplog.text + + +@pytest.mark.asyncio +async def test_a_sensor_that_could_not_be_read_is_asked_again(): + frbc = frbc_simple() + frbc._fm_client.get_sensor.side_effect = [ + ConnectionError("down"), + {"event_resolution": "PT0M"}, + ] + assert await frbc._soc_duration() == timedelta(minutes=1) + assert await frbc._soc_duration() == timedelta(0) + + +@pytest.mark.asyncio +async def test_belief_times_are_forgotten_for_steps_more_than_a_day_old(): + from s2python.frbc import FRBCActuatorStatus + + from flexmeasures_client.s2.utils import get_unique_id + + frbc = frbc_simple() + frbc._power_sensor_takes_consumption_as_positive = True + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + actuator = frbc._system_description_history[ + list(frbc._system_description_history)[-1] + ].actuators[0] + for when in ( + datetime(2022, 12, 1, 0, 0, tzinfo=timezone.utc), + datetime(2022, 12, 2, 6, 0, tzinfo=timezone.utc), + ): + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=actuator.operation_modes[0].id, + operation_mode_factor=1.0, + transition_timestamp=when, + ) + ) + assert list(frbc._power_recorded_at) == [ + datetime(2022, 12, 2, 6, 0, tzinfo=timezone.utc) + ] + + +@pytest.mark.asyncio +async def test_a_description_replaced_while_its_plan_is_out_sends_only_the_newer_plan(): + """The plan for a description that a newer one replaced during the round trip is for a device that no longer is.""" + frbc = frbc_simple() + release = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def slow_then_quick(**kwargs): + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await release.wait() + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = slow_then_quick + batches: list[tuple[int, int]] = [] + send = frbc.send_instruction_batch + + async def recording(*args, **kwargs): + sent = await send(*args, **kwargs) + batches.append((kwargs["request_number"], sent)) + return sent + + frbc.send_instruction_batch = recording + await frbc.handle_message(storage_status()) + await settle(frbc) + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await asyncio.sleep(0.05) + await frbc.handle_message(system_description("2022-12-02T06:00:00+00:00")) + await asyncio.sleep(0.05) + release.set() + await settle(frbc) + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + sent = dict(batches) + newest = max(n for n, _ in batches) + assert sent[newest] > 0 + assert all(count == 0 for n, count in batches if n != newest) + + +@pytest.mark.asyncio +async def test_a_burst_of_statuses_costs_the_solve_in_flight_and_one_more(): + """Each status used to start a solve beside the one in flight, all but the last for nothing.""" + frbc = frbc_simple() + release = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + running = 0 + most = 0 + + async def solve(**kwargs): + nonlocal running, most + running += 1 + most = max(most, running) + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await release.wait() + running -= 1 + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = solve + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + for _ in range(4): + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + release.set() + await settle(frbc) + assert most == 1 + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + + +@pytest.mark.asyncio +async def test_a_request_overtaken_while_it_waits_for_the_solve_in_flight_is_dropped(): + """It was the newest when it began to wait, and is not when its turn comes.""" + frbc = frbc_simple() + release = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def solve(**kwargs): + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await release.wait() + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = solve + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + await frbc.handle_message(storage_status()) # waits for the solve in flight + await asyncio.sleep(0.05) + await frbc.handle_message(storage_status()) # overtakes the one waiting + await asyncio.sleep(0.05) + release.set() + await settle(frbc) + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + + +@pytest.mark.asyncio +async def test_a_request_out_of_date_makes_no_further_attempt_while_the_newest_waits( + caplog, +): + """It kept the newest waiting through every one of its retries, and the schedule they got was thrown away.""" + import logging + from datetime import timedelta as td + + caplog.set_level(logging.INFO) + + from flexmeasures_client.exceptions import JobFailedError + + frbc = frbc_simple() + frbc._schedule_attempts = 5 + frbc._schedule_retry_interval = td(milliseconds=10) + frbc._is_worth_asking_again = lambda failure: True + first_attempt = asyncio.Event() + planned = frbc._fm_client.trigger_and_get_schedule.return_value + + async def solve(**kwargs): + if frbc._fm_client.trigger_and_get_schedule.await_count == 1: + await first_attempt.wait() + raise JobFailedError("the worker crashed") + return planned + + frbc._fm_client.trigger_and_get_schedule.side_effect = solve + batches: list[int] = [] + send = frbc.send_instruction_batch + + async def recording(*args, **kwargs): + batches.append(kwargs["request_number"]) + return await send(*args, **kwargs) + + frbc.send_instruction_batch = recording + await frbc.handle_message(system_description("2022-12-02T00:00:00+00:00")) + await frbc.handle_message(storage_status()) + await asyncio.sleep(0.05) + await frbc.handle_message(storage_status()) # newer, while the first attempt is out + await asyncio.sleep(0.05) + first_attempt.set() + await settle(frbc) + # The failed attempt, then the newest. + assert frbc._fm_client.trigger_and_get_schedule.await_count == 2 + assert "Not asking again for planning request" in caplog.text + assert not [ + r for r in caplog.records if r.levelname == "ERROR" + ] # nothing translated for it + assert batches == [2] # nothing sent for the request that stood down diff --git a/tests/s2/test_frbc_soft_band.py b/tests/s2/test_frbc_soft_band.py index 1aa734bd..64055d78 100644 --- a/tests/s2/test_frbc_soft_band.py +++ b/tests/s2/test_frbc_soft_band.py @@ -156,7 +156,20 @@ async def test_the_usage_and_the_leakage_a_resource_manager_declares_are_planned system_description, ): """A storage that is drawn from and leaks is not planned for by its prices alone.""" + from s2python.frbc import FRBCLeakageBehaviour, FRBCLeakageBehaviourElement + frbc = _frbc(system_description) + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from="2024-01-01T00:00:00+00:00", + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + leakage_rate=0.001, + ) + ], + ) + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage flex_model, _ = await _asked_for(frbc, system_description) @@ -164,6 +177,16 @@ async def test_the_usage_and_the_leakage_a_resource_manager_declares_are_planned assert flex_model["storage-efficiency"] == {"sensor": LEAKAGE_SENSOR_ID} +@pytest.mark.asyncio +async def test_no_storage_efficiency_is_named_before_a_leakage_behaviour_is_known( + system_description, +): + """A sensor holding nothing for the window would be planned against as a storage that keeps everything.""" + frbc = _frbc(system_description) + flex_model, _ = await _asked_for(frbc, system_description) + assert "storage-efficiency" not in flex_model + + @pytest.mark.asyncio async def test_the_hard_bounds_are_wider_than_the_declared_range( system_description, diff --git a/tests/s2/test_frbc_time_base.py b/tests/s2/test_frbc_time_base.py new file mode 100644 index 00000000..eb940673 --- /dev/null +++ b/tests/s2/test_frbc_time_base.py @@ -0,0 +1,725 @@ +"""The time base under the instructions: every rate per second, as S2 states it. + +S2 states a fill rate, a usage rate and a leakage rate each as a change in fill level per second. +The schedule translation keeps a running fill level, to answer each step with the operation mode for the fill level +the storage is then at, and it used to advance that level by a rate per second times a step in hours, +so it moved 3600 times too little, and every step of a twelve-hour schedule was answered at the fill level it started from. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, patch + +import pandas as pd +import pytest +from s2python.common import ( + Commodity, + CommodityQuantity, + Duration, + NumberRange, + PowerRange, +) +from s2python.frbc import ( + FRBCActuatorDescription, + FRBCActuatorStatus, + FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, + FRBCOperationMode, + FRBCOperationModeElement, + FRBCStorageDescription, + FRBCStorageStatus, + FRBCSystemDescription, + FRBCUsageForecast, + FRBCUsageForecastElement, +) + +from flexmeasures_client.client import FlexMeasuresClient, _server_version_at_least +from flexmeasures_client.s2.control_types.FRBC.frbc_simple import FRBCSimple +from flexmeasures_client.s2.control_types.FRBC.utils import ( + compute_next_fill_level, + fm_schedule_to_instructions, + usage_rate_at, +) +from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, +) +from flexmeasures_client.s2.utils import get_unique_id + +START = datetime(2024, 1, 1, tzinfo=timezone.utc) +QUARTER_HOUR = timedelta(minutes=15) + + +def _leakage(*elements: tuple[float, float, float]) -> FRBCLeakageBehaviour: + return FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=START, + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=lo, end_of_range=hi), + leakage_rate=rate, + ) + for lo, hi, rate in elements + ], + ) + + +def _usage(*elements: tuple[timedelta, float]) -> FRBCUsageForecast: + return FRBCUsageForecast( + message_id=get_unique_id(), + start_time=START, + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=int(duration / timedelta(milliseconds=1))), + usage_rate_expected=rate, + ) + for duration, rate in elements + ], + ) + + +# --------------------------------------------------------------------------- +# The integrator's contract +# --------------------------------------------------------------------------- + + +def test_a_fill_rate_is_integrated_per_second(): + """0.001 per second for a quarter hour is 0.9, which the old integrator made 0.00025.""" + assert compute_next_fill_level(1.0, 0.001, QUARTER_HOUR) == pytest.approx(1.9) + + +def test_usage_lowers_the_fill_level_per_second(): + assert compute_next_fill_level( + 10.0, 0.0, QUARTER_HOUR, usage_rate=0.0005 + ) == pytest.approx(10.0 - 0.45) + + +def test_leakage_lowers_the_fill_level_at_the_rate_of_the_element_that_applies(): + """At 50, in the leakier upper element, an hour loses 3.6; the lower element would lose 0.36.""" + leakage = _leakage((50, 100, 0.001), (0, 50, 0.0001)) + assert compute_next_fill_level( + 50.0, 0.0, timedelta(hours=1), leakage_behaviour=leakage + ) == pytest.approx(46.4) + assert compute_next_fill_level( + 20.0, 0.0, timedelta(hours=1), leakage_behaviour=leakage + ) == pytest.approx(19.64) + + +@pytest.mark.parametrize( + "moment, expected", + [ + (START - timedelta(minutes=1), 0.0), + (START + timedelta(minutes=30), 0.001), + (START + timedelta(minutes=90), 0.002), + # Where one element ends, the next begins, and its rate applies. + (START + timedelta(minutes=60), 0.002), + (START + timedelta(minutes=150), 0.0), + ], +) +def test_the_usage_rate_a_forecast_expects_at_a_moment(moment, expected): + """Elements follow one another for their own durations; outside them nothing is forecast.""" + forecast = _usage((timedelta(hours=1), 0.001), (timedelta(hours=1), 0.002)) + assert usage_rate_at(forecast, pd.Timestamp(moment)) == pytest.approx(expected) + + +def test_without_a_forecast_nothing_is_used(): + assert usage_rate_at(None, pd.Timestamp(START)) == 0.0 + + +# --------------------------------------------------------------------------- +# The translation +# --------------------------------------------------------------------------- + + +def _element(lo, hi, power_top, fill_rate_top, power_bottom=0.0, fill_rate_bottom=0.0): + return FRBCOperationModeElement( + fill_level_range=NumberRange(start_of_range=lo, end_of_range=hi), + fill_rate=NumberRange( + start_of_range=fill_rate_bottom, end_of_range=fill_rate_top + ), + power_ranges=[ + PowerRange( + start_of_range=power_bottom, + end_of_range=power_top, + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, + ) + ], + ) + + +def _system_description(*elements) -> FRBCSystemDescription: + mode = FRBCOperationMode( + id=get_unique_id(), + diagnostic_label="heat", + elements=list(elements), + abnormal_condition_only=False, + ) + return FRBCSystemDescription( + message_id=get_unique_id(), + valid_from=START, + actuators=[ + FRBCActuatorDescription( + id=get_unique_id(), + supported_commodities=[Commodity.ELECTRICITY], + operation_modes=[mode], + transitions=[], + timers=[], + ) + ], + storage=FRBCStorageDescription( + provides_leakage_behaviour=True, + provides_fill_level_target_profile=False, + provides_usage_forecast=True, + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + ), + ) + + +#: Below 50 a mode draws up to 1000 W, above it up to 3000 W, filling at most 1/150 per second either way. +#: At a constant 500 W it fills 3 per quarter hour below 50, and 1 per quarter hour above it. +LOW = _element(0, 50, power_top=1000.0, fill_rate_top=1 / 150) +HIGH = _element(50, 100, power_top=3000.0, fill_rate_top=1 / 150) + + +def _schedule(values, step=QUARTER_HOUR) -> dict: + return { + "start": START.isoformat(), + "duration": pd.Timedelta(step * len(values)).isoformat(), + "values": values, + } + + +def _factors(instructions) -> list[tuple[datetime, float]]: + return [(i.execution_time, i.operation_mode_factor) for i in instructions] + + +def test_a_constant_power_is_instructed_anew_where_the_fill_level_crosses_into_another_element(): + """500 W is half of what the lower element draws at full tilt, and a sixth of the upper one's. + + From 45, three quarter hours of filling at 3 each take the buffer past 50, + where the same 500 W needs another factor, which is a new instruction although the power did not change. + """ + instructions = fm_schedule_to_instructions( + _schedule([500.0] * 4), _system_description(LOW, HIGH), initial_fill_level=45.0 + ) + assert _factors(instructions) == [ + (START, pytest.approx(0.5)), + (START + 2 * QUARTER_HOUR, pytest.approx(1 / 6)), + ] + + +def test_usage_brings_the_running_fill_level_down_across_a_boundary(): + """Filling 1 per quarter hour above 50 while 4 is used leaves 3 less each step, so from 55 the buffer drops below 50.""" + instructions = fm_schedule_to_instructions( + _schedule([500.0] * 4), + _system_description(LOW, HIGH), + initial_fill_level=55.0, + usage_forecast=_usage((timedelta(hours=1), 4 / 900)), + ) + assert _factors(instructions) == [ + (START, pytest.approx(1 / 6)), + (START + 2 * QUARTER_HOUR, pytest.approx(0.5)), + ] + + +def test_leakage_brings_the_running_fill_level_down_across_a_boundary(): + instructions = fm_schedule_to_instructions( + _schedule([500.0] * 4), + _system_description(LOW, HIGH), + initial_fill_level=55.0, + leakage_behaviour=_leakage((0, 100, 4 / 900)), + ) + assert _factors(instructions) == [ + (START, pytest.approx(1 / 6)), + (START + 2 * QUARTER_HOUR, pytest.approx(0.5)), + ] + + +def test_a_usage_column_takes_the_place_of_a_forecast(): + """A caller holding its usage per step in the frame, as TUNES does, passes it per second.""" + frame = pd.DataFrame( + {"schedule": [500.0] * 4, "usage_forecast": [4 / 900] * 4}, + index=pd.date_range(START, periods=4, freq=QUARTER_HOUR), + ) + instructions = fm_schedule_to_instructions( + frame, _system_description(LOW, HIGH), initial_fill_level=55.0 + ) + assert _factors(instructions)[1] == ( + START + 2 * QUARTER_HOUR, + pytest.approx(0.5), + ) + + +@pytest.mark.parametrize( + "step", [timedelta(minutes=5), QUARTER_HOUR, timedelta(hours=1)] +) +def test_a_schedule_is_read_at_the_resolution_it_comes_in(step): + """FlexMeasures answers at the power sensor's resolution, which was taken to be a quarter hour whatever it was. + + Twelve hourly values were laid on forty-eight quarter hours, and pandas refused, so no instruction went out. + """ + instructions = fm_schedule_to_instructions( + _schedule([0.0] * 6 + [1000.0] * 6, step=step), + _system_description(_element(0, 100, power_top=1000.0, fill_rate_top=1e-9)), + initial_fill_level=10.0, + ) + assert _factors(instructions) == [ + (START, pytest.approx(0.0)), + (START + 6 * step, pytest.approx(1.0)), + ] + + +def test_the_instruction_asks_for_the_factor_that_draws_the_power(): + """A mode running 800 to 1200 W at one fill rate, asked for 1200 W, is asked for factor 1, which draws 1200 W.""" + instructions = fm_schedule_to_instructions( + _schedule([1200.0]), + _system_description( + _element( + 0, + 100, + power_top=1200.0, + power_bottom=800.0, + fill_rate_top=2e-3, + fill_rate_bottom=2e-3, + ) + ), + initial_fill_level=10.0, + ) + assert _factors(instructions) == [(START, pytest.approx(1.0))] + + +# --------------------------------------------------------------------------- +# The storage efficiency +# --------------------------------------------------------------------------- + + +def test_the_storage_efficiency_compounds_a_leakage_rate_per_second(): + """A 10 kWh buffer losing 1.5 kWh a day keeps 99.844% per quarter hour, about 92.8% over twelve hours.""" + efficiency = leakage_behaviour_to_storage_efficiency( + _leakage((0, 10, 1.5 / 86400)), resolution=QUARTER_HOUR + ) + assert efficiency == pytest.approx(100 * (1 - 1.5 / 86400 / 10) ** 900, rel=1e-12) + assert 100 * (efficiency / 100) ** 48 == pytest.approx(92.774, abs=1e-3) + + +# --------------------------------------------------------------------------- +# The handler +# --------------------------------------------------------------------------- + + +def _frbc(system_description: FRBCSystemDescription, **kwargs) -> FRBCSimple: + frbc = FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + **kwargs, + ) + frbc._fm_client = AsyncMock(FlexMeasuresClient) + frbc._fm_client.server_version_at_least = AsyncMock( + side_effect=lambda minimum: _server_version_at_least("1.1.0", minimum) + ) + frbc._logger = logging.getLogger("FRBCSimple") + frbc._system_description_history = { + system_description.message_id: system_description + } + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=45.0 + ) + frbc._fm_client.trigger_and_get_schedule = AsyncMock( + return_value=_schedule([500.0] * 4) + ) + frbc.send_instruction_batch = AsyncMock(return_value=0) + frbc._as_the_power_sensor_reads_it = AsyncMock(side_effect=lambda power: power) + return frbc + + +@pytest.mark.asyncio +async def test_the_handler_translates_with_the_latest_leakage_and_usage(): + system_description = _system_description(LOW, HIGH) + frbc = _frbc(system_description, resolution=timedelta(minutes=5)) + older, latest = _leakage((0, 100, 1e-4)), _leakage((0, 100, 2e-4)) + frbc._leakage_behaviour_history[str(older.message_id)] = older + frbc._leakage_behaviour_history[str(latest.message_id)] = latest + forecast = _usage((timedelta(hours=1), 1e-3)) + frbc._usage_forecast_history[str(forecast.message_id)] = forecast + + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule(system_description.message_id) + + kwargs = translate.call_args.kwargs + assert kwargs["leakage_behaviour"] is latest + assert kwargs["usage_forecast"] is forecast + + +@pytest.mark.asyncio +async def test_a_measurement_covers_the_handlers_own_resolution(): + """A power measurement for one step of a five-minute grid is not posted as covering a quarter hour.""" + system_description = _system_description(LOW, HIGH) + frbc = _frbc( + system_description, resolution=timedelta(minutes=5), fill_rate_sensor_id=6 + ) + actuator = system_description.actuators[0] + await frbc.send_actuator_status( + FRBCActuatorStatus( + message_id=get_unique_id(), + actuator_id=actuator.id, + active_operation_mode_id=actuator.operation_modes[0].id, + operation_mode_factor=0.5, + ) + ) + durations = { + (call.args[:1] or (call.kwargs.get("sensor_id"),))[0]: call.kwargs["duration"] + for call in frbc._fm_client.post_sensor_data.await_args_list + } + assert durations == {1: timedelta(minutes=5), 6: timedelta(minutes=5)} + + +def test_a_schedule_without_values_asks_for_nothing(): + """Its step would be its duration over no values, so there is none, and no instruction either.""" + assert ( + fm_schedule_to_instructions( + {"start": START.isoformat(), "duration": "PT0H", "values": []}, + _system_description(LOW, HIGH), + initial_fill_level=45.0, + ) + == [] + ) + + +# --------------------------------------------------------------------------- +# Following a step as a resource manager does (from the review of #258) +# --------------------------------------------------------------------------- + +from flexmeasures_client.s2.control_types.FRBC.utils import ( # noqa: E402 + UsageProfile, + follow_operation_mode, +) + + +def _mode(*elements, label="heat") -> FRBCOperationMode: + return FRBCOperationMode( + id=get_unique_id(), + diagnostic_label=label, + elements=list(elements), + abnormal_condition_only=False, + ) + + +def _system_description_of(*modes) -> FRBCSystemDescription: + description = _system_description(LOW) + description.actuators[0].operation_modes = list(modes) + return description + + +def test_an_on_off_heat_pump_is_told_to_switch_off(): + """On and off each run at one power, so both ask for factor 0: only the mode tells them apart.""" + on = _mode( + _element( + 0, + 100, + power_top=883.7, + power_bottom=883.7, + fill_rate_top=1e-3, + fill_rate_bottom=1e-3, + ), + label="on", + ) + off = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0), label="off") + instructions = fm_schedule_to_instructions( + _schedule([883.7, 883.7, 0.0, 0.0]), + _system_description_of(on, off), + initial_fill_level=10.0, + ) + assert [(i.execution_time, str(i.operation_mode)) for i in instructions] == [ + (START, str(on.id)), + (START + 2 * QUARTER_HOUR, str(off.id)), + ] + + +def test_a_solvers_rounding_is_not_a_new_instruction(): + """At the bottom of a modulating range, 800 W give or take a ten-millionth is one instruction, not one per step.""" + modulating = _mode( + _element( + 0, + 100, + power_top=1200.0, + power_bottom=800.0, + fill_rate_top=2e-3, + fill_rate_bottom=1e-3, + ) + ) + noisy = [800.0 + (1e-7 if i % 2 else -1e-7) for i in range(48)] + instructions = fm_schedule_to_instructions( + _schedule(noisy), _system_description_of(modulating), initial_fill_level=10.0 + ) + assert len(instructions) == 1 + + +def test_usage_is_followed_through_the_step_not_sampled_at_its_start(): + """A five-minute draw in each quarter hour takes five minutes' worth, not none and not fifteen.""" + burst = _usage(*[(timedelta(minutes=5), 0.03), (timedelta(minutes=10), 0.0)] * 4) + idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) + assert follow_operation_mode( + 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage=UsageProfile(burst) + ) == pytest.approx(50.0 - 0.03 * 300) + + +def test_usage_counts_from_where_the_forecast_starts(): + """A forecast from 7 minutes past counts from then: 8 minutes of usage in a step that began on the hour.""" + late = FRBCUsageForecast( + message_id=get_unique_id(), + start_time=START + timedelta(minutes=7), + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=3600 * 1000), usage_rate_expected=0.01 + ) + ], + ) + idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) + assert follow_operation_mode( + 50.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage=UsageProfile(late) + ) == pytest.approx(50.0 - 0.01 * 480) + + +#: Fills 0.02 per second at full tilt below 50, half that above, as a heat pump does when its buffer warms. +STEEP = _element(0, 50, power_top=1000.0, fill_rate_top=0.02) +SHALLOW = _element(50, 100, power_top=2000.0, fill_rate_top=0.01) + + +def test_a_step_that_crosses_into_another_element_is_followed_across_it(): + """From 46, factor 1 reaches 50 after 200 s, then fills at half the rate: 57, not the 64 of holding the first element.""" + level = follow_operation_mode( + 46.0, _mode(STEEP, SHALLOW), 1.0, pd.Timestamp(START), QUARTER_HOUR + ) + assert level == pytest.approx(57.0) + + +def test_leakage_is_followed_across_an_element_boundary_too(): + """Leaking 0.01 per second above 50 and nothing below, a buffer at 54 settles at 50 rather than draining to 45.""" + leakage = _leakage((0, 50, 0.0), (50, 100, 0.01)) + idle = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) + level = follow_operation_mode( + 54.0, idle, 0.0, pd.Timestamp(START), QUARTER_HOUR, leakage_behaviour=leakage + ) + assert level == pytest.approx(50.0) + + +def test_a_gap_in_a_usage_column_is_no_usage_rather_than_a_buffer_full_to_the_top(): + index = pd.date_range(START, periods=4, freq=QUARTER_HOUR) + with_gap = pd.DataFrame( + {"schedule": [500.0] * 4, "usage_forecast": [float("nan")] * 4}, index=index + ) + without = pd.DataFrame( + {"schedule": [500.0] * 4, "usage_forecast": [0.0] * 4}, index=index + ) + description = _system_description(LOW, HIGH) + assert _factors( + fm_schedule_to_instructions(with_gap, description, initial_fill_level=45.0) + ) == _factors( + fm_schedule_to_instructions(without, description, initial_fill_level=45.0) + ) + + +def test_a_frame_is_read_at_its_own_step(): + """An hourly frame: an hour at 500 W fills 12, so the buffer is past 50 by the second row.""" + frame = pd.DataFrame( + {"schedule": [500.0] * 3}, + index=pd.date_range(START, periods=3, freq=timedelta(hours=1)), + ) + instructions = fm_schedule_to_instructions( + frame, _system_description(LOW, HIGH), initial_fill_level=45.0 + ) + assert _factors(instructions) == [ + (START, pytest.approx(0.5)), + (START + timedelta(hours=1), pytest.approx(1 / 6)), + ] + + +def test_a_schedule_duration_between_two_whole_steps_is_refused(): + """FlexMeasures answers with whole steps only, so such a schedule would be read at the wrong step.""" + with pytest.raises(ValueError, match="whole number of resolution steps"): + FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + schedule_duration=timedelta(hours=2, minutes=30), + resolution=timedelta(hours=1), + ) + + +@pytest.mark.asyncio +async def test_the_handler_translates_with_the_latest_usage_forecast(): + system_description = _system_description(LOW, HIGH) + frbc = _frbc(system_description) + older, latest = _usage((timedelta(hours=1), 1e-4)), _usage( + (timedelta(hours=1), 2e-4) + ) + frbc._usage_forecast_history[str(older.message_id)] = older + frbc._usage_forecast_history[str(latest.message_id)] = latest + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_simple.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule(system_description.message_id) + assert translate.call_args.kwargs["usage_forecast"] is latest + + +# --------------------------------------------------------------------------- +# Followed exactly, between the moments anything changes (second round of review) +# --------------------------------------------------------------------------- + +IDLE = _mode(_element(0, 100, power_top=0.0, fill_rate_top=0.0)) + + +@pytest.mark.parametrize("offset", [timedelta(0), timedelta(seconds=30)]) +def test_a_draw_shorter_than_a_minute_counts_exactly_once(offset): + """Half-minute draws every two minutes, starting on the minute or off it: a quarter hour holds 7 or 8 of them, exactly.""" + elements = [] + if offset: + elements.append((offset, 0.0)) + while sum((d for d, _ in elements), timedelta(0)) < QUARTER_HOUR: + elements += [(timedelta(seconds=30), 0.1), (timedelta(seconds=90), 0.0)] + pulses = _usage(*elements) + drawn_for = sum( + ( + min(QUARTER_HOUR, start + d) - start + for start, (d, rate) in zip( + [ + sum((e[0] for e in elements[:k]), timedelta(0)) + for k in range(len(elements)) + ], + elements, + ) + if rate and start < QUARTER_HOUR + ), + timedelta(0), + ) + level = follow_operation_mode( + 50.0, IDLE, 0.0, pd.Timestamp(START), QUARTER_HOUR, usage=UsageProfile(pulses) + ) + assert level == pytest.approx(50.0 - 0.1 * drawn_for.total_seconds()) + + +def test_a_level_held_at_a_boundary_stays_there_rather_than_flipping_across(): + """Filling below 50 and not above, while leaking everywhere: the level settles at 50, and one instruction serves 12 hours.""" + below = _element(0, 50, power_top=1000.0, fill_rate_top=0.02) + above = _element(50, 100, power_top=1000.0, power_bottom=200.0, fill_rate_top=0.0) + leaking = _leakage((0, 100, 0.01)) + instructions = fm_schedule_to_instructions( + _schedule([600.0] * 48), + _system_description_of(_mode(below, above)), + initial_fill_level=40.0, + leakage_behaviour=leaking, + ) + assert len(instructions) <= 2, _factors(instructions) + + +def test_an_element_narrower_than_a_minutes_travel_is_not_skipped(): + """Filling at 1 per second below 50 and not at all above: from 20, the level stops at 50, not 30 past it.""" + stop = _mode( + _element(0, 50, power_top=1000.0, fill_rate_top=1.0), + _element(50, 100, power_top=1000.0, fill_rate_top=0.0), + ) + assert follow_operation_mode( + 20.0, stop, 1.0, pd.Timestamp(START), QUARTER_HOUR + ) == pytest.approx(50.0) + + +def test_a_step_that_is_not_a_whole_number_of_minutes_is_followed_to_its_end(): + level = follow_operation_mode( + 10.0, _mode(STEEP, SHALLOW), 1.0, pd.Timestamp(START), timedelta(seconds=90) + ) + assert level == pytest.approx(10.0 + 0.02 * 90) + + +def test_a_level_draining_to_the_lower_bound_is_held_there(): + drained = follow_operation_mode( + 5.0, + IDLE, + 0.0, + pd.Timestamp(START), + QUARTER_HOUR, + usage=UsageProfile(rate=0.01), + bounds=(0.0, 100.0), + ) + assert drained == 0.0 + + +def test_a_small_but_real_change_of_factor_is_instructed(): + """The tolerance absorbs conversion noise, not a change the schedule asks for: 600 W then 605 W is two instructions.""" + modulating = _mode(_element(0, 100, power_top=1000.0, fill_rate_top=1e-6)) + instructions = fm_schedule_to_instructions( + _schedule([600.0, 605.0]), + _system_description_of(modulating), + initial_fill_level=10.0, + ) + assert [round(f, 3) for _, f in _factors(instructions)] == [0.6, 0.605] + + +def test_a_usage_column_takes_precedence_over_a_forecast(): + """A caller that gives both means its column: 4 per quarter hour drawn, not the forecast's nothing.""" + frame = pd.DataFrame( + {"schedule": [500.0] * 4, "usage_forecast": [4 / 900] * 4}, + index=pd.date_range(START, periods=4, freq=QUARTER_HOUR), + ) + with_both = fm_schedule_to_instructions( + frame, + _system_description(LOW, HIGH), + initial_fill_level=55.0, + usage_forecast=_usage((timedelta(hours=1), 0.0)), + ) + assert _factors(with_both)[1] == (START + 2 * QUARTER_HOUR, pytest.approx(0.5)) + + +@pytest.mark.parametrize( + "resolution, duration", + [(timedelta(0), timedelta(hours=12)), (QUARTER_HOUR, timedelta(0))], +) +def test_a_handler_needs_a_positive_resolution_and_duration(resolution, duration): + with pytest.raises(ValueError, match="have to be positive"): + FRBCSimple( + power_sensor_id=1, + price_sensor_id=2, + soc_sensor_id=3, + rm_discharge_sensor_id=4, + resolution=resolution, + schedule_duration=duration, + ) + + +def test_a_long_schedule_against_a_detailed_forecast_is_translated_quickly(): + """48 hours, at quarter hours, against the largest forecast S2 allows: well under the 1.7 s that stepping each minute took.""" + import time + + detailed = _usage(*[(timedelta(minutes=10), 1e-4 * (i % 3)) for i in range(288)]) + started = time.perf_counter() + fm_schedule_to_instructions( + _schedule([500.0, 0.0] * 96), + _system_description(LOW, HIGH), + initial_fill_level=45.0, + usage_forecast=detailed, + leakage_behaviour=_leakage((0, 50, 1e-4), (50, 100, 2e-4)), + ) + assert time.perf_counter() - started < 0.5 + + +def test_a_level_held_at_its_lower_bound_rises_from_there_once_filling_wins(): + """Drained to 0 in the first 5 minutes, held there while the draw lasts, then filling from 0, not from below it.""" + draw_then_none = _usage((timedelta(minutes=10), 0.02), (timedelta(minutes=5), 0.0)) + filling = _mode(_element(0, 100, power_top=1000.0, fill_rate_top=0.01)) + level = follow_operation_mode( + 3.0, + filling, + 1.0, + pd.Timestamp(START), + QUARTER_HOUR, + usage=UsageProfile(draw_then_none), + bounds=(0.0, 100.0), + ) + assert level == pytest.approx(0.01 * 300) diff --git a/tests/s2/test_frbc_tunes.py b/tests/s2/test_frbc_tunes.py index 645692c9..8363afb0 100644 --- a/tests/s2/test_frbc_tunes.py +++ b/tests/s2/test_frbc_tunes.py @@ -5,6 +5,7 @@ from unittest.mock import AsyncMock import numpy as np +import pandas as pd import pytest import pytest_asyncio from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues @@ -288,3 +289,259 @@ async def test_fill_rate_relay(cem_in_frbc_control_type): assert third_call["values"][0] == frbc._nes_fill_rate_sensor_id await cem.close() + + +@pytest.mark.asyncio +async def test_a_leakage_behaviour_is_recorded_rather_than_refused( + cem_in_frbc_control_type, +): + """A leakage behaviour is posted as a storage efficiency, and not answered as a permanent error. + + The handler read its fill-level scale from an attribute whose name was pasted in twice, + and the AttributeError that raised was caught as if the post had failed, + so every leakage behaviour was refused and none was recorded. + """ + from s2python.common import NumberRange + from s2python.frbc import FRBCLeakageBehaviour, FRBCLeakageBehaviourElement + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + frbc.send_message = AsyncMock() + + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + leakage_rate=0.001, + ) + ], + ) + await frbc.send_leakage_behaviour(leakage) + + sent = [call.args[0] for call in frbc.send_message.await_args_list] + assert not any( + getattr(message, "status", None) == ReceptionStatusValues.PERMANENT_ERROR + for message in sent + ), sent + posts = [ + call + for call in fm_client.post_sensor_data.await_args_list + if (call.args[0] if call.args else call.kwargs.get("sensor_id")) == 15 + ] + assert len(posts) == 1, fm_client.post_sensor_data.await_args_list + # What the translation makes of this message on a quarter-hour grid, which is what TUNES says it posts. + # The fill-level scale is not exercised here: it multiplies both the leakage and the range it is a share of, and cancels. + from flexmeasures_client.s2.control_types.translations import ( + leakage_behaviour_to_storage_efficiency, + ) + + assert posts[0].kwargs["values"] == [ + leakage_behaviour_to_storage_efficiency( + message=leakage, resolution=timedelta(minutes=15), fill_level_scale=1 + ) + ] + assert posts[0].kwargs["unit"] == "%" + + +@pytest.mark.asyncio +async def test_an_operation_mode_without_a_label_does_not_stop_tunes( + cem_in_frbc_control_type, +): + """S2 makes a diagnostic label optional, and TUNES tells its modes apart by label. + + It looked for "THP" and "NES" inside a label that may be None, which raised a TypeError, + so one unlabelled mode stopped the conversion efficiencies of every mode from being recorded. + """ + cem, fm_client, system_description = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + unlabelled = system_description.model_copy(deep=True) + unlabelled.actuators[0].operation_modes[-1].diagnostic_label = None + fm_client.post_sensor_data.reset_mock() + + await frbc.send_conversion_efficiencies(unlabelled) + + assert fm_client.post_sensor_data.await_count > 0 + + +@pytest.mark.asyncio +async def test_planning_does_not_stop_on_an_unlabelled_operation_mode( + cem_in_frbc_control_type, +): + """The same optional label, where TUNES picks the mode to plan with.""" + from s2python.frbc import FRBCStorageStatus + + cem, fm_client, system_description = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + unlabelled = system_description.model_copy(deep=True) + for operation_mode in unlabelled.actuators[0].operation_modes: + operation_mode.diagnostic_label = None + frbc._system_description_history[str(unlabelled.message_id)] = unlabelled + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=10.0 + ) + + await frbc.trigger_schedule() + + +@pytest.mark.asyncio +async def test_a_usage_forecast_is_posted_per_hour(cem_in_frbc_control_type): + """S2 states usage per second; TUNES posts it per hour, labelled MW, on a quarter-hour grid, for as long as it was forecast. + + FlexMeasures reads soc-usage as a power, a rate per hour, so a value per quarter hour would be planned as a quarter of the draw. + """ + from s2python.common import Duration + from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + fm_client.post_sensor_data.reset_mock() + + await frbc.send_usage_forecast( + FRBCUsageForecast( + message_id=get_unique_id(), + start_time=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=2 * 3600 * 1000), + usage_rate_expected=0.001, + ) + ], + ) + ) + + posts = [ + call + for call in fm_client.post_sensor_data.await_args_list + if call.kwargs.get("sensor_id") == 12 + ] + assert len(posts) == 1, fm_client.post_sensor_data.await_args_list + values = posts[0].kwargs["values"] + assert values == [pytest.approx(0.001 * 3600)] * len(values) + assert posts[0].kwargs["duration"] == str(pd.Timedelta(minutes=15) * len(values)) + + +@pytest.mark.asyncio +async def test_a_usage_forecast_is_posted_from_the_quarter_hour_it_starts_in( + cem_in_frbc_control_type, +): + from s2python.common import Duration + from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + fm_client.post_sensor_data.reset_mock() + + await frbc.send_usage_forecast( + FRBCUsageForecast( + message_id=get_unique_id(), + start_time=datetime(2024, 1, 1, 0, 7, tzinfo=timezone.utc), + elements=[ + FRBCUsageForecastElement( + duration=Duration(root=3600 * 1000), usage_rate_expected=0.001 + ) + ], + ) + ) + + posts = [ + c + for c in fm_client.post_sensor_data.await_args_list + if c.kwargs.get("sensor_id") == 12 + ] + assert posts[0].kwargs["start"] == datetime(2024, 1, 1, 0, 0, tzinfo=timezone.utc) + + +@pytest.mark.asyncio +async def test_a_usage_forecast_is_averaged_within_each_quarter_hour( + cem_in_frbc_control_type, +): + """Two halves of a quarter hour, at 1 and 3 thousandths per second, are posted as 2 thousandths per second, per hour.""" + from s2python.common import Duration + from s2python.frbc import FRBCUsageForecast, FRBCUsageForecastElement + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + fm_client.post_sensor_data.reset_mock() + half = Duration(root=int(7.5 * 60 * 1000)) + + await frbc.send_usage_forecast( + FRBCUsageForecast( + message_id=get_unique_id(), + start_time=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCUsageForecastElement(duration=half, usage_rate_expected=0.001), + FRBCUsageForecastElement(duration=half, usage_rate_expected=0.003), + ], + ) + ) + + posts = [ + c + for c in fm_client.post_sensor_data.await_args_list + if c.kwargs.get("sensor_id") == 12 + ] + assert posts[0].kwargs["values"][0] == pytest.approx(0.002 * 3600) + + +@pytest.mark.asyncio +async def test_planning_reads_its_usage_forecast_back_per_second( + cem_in_frbc_control_type, +): + """TUNES posts a usage forecast per hour; the schedule translation takes a rate per second, as S2 states it.""" + from unittest.mock import patch + + from s2python.common import NumberRange + from s2python.frbc import ( + FRBCLeakageBehaviour, + FRBCLeakageBehaviourElement, + FRBCStorageStatus, + ) + + cem, fm_client, _ = cem_in_frbc_control_type + frbc = cem._control_types_handlers[cem.control_type] + frbc._storage_status_history[get_unique_id()] = FRBCStorageStatus( + message_id=get_unique_id(), present_fill_level=10.0 + ) + + def sensor_data(sensor_id, start=None, duration=None, **kwargs): + # As many quarter hours as were asked for: the fixture's fixed day of values fits no other window. + steps = int(pd.Timedelta(duration) / pd.Timedelta(minutes=15)) + # The usage forecast as posted, 3600 per hour, and anything else as a plain efficiency. + value = 3600.0 if sensor_id == 12 else 1.0 + return { + "start": str(start), + "duration": str(duration), + "values": [value] * steps, + } + + fm_client.get_sensor_data.side_effect = sensor_data + # Not 1, which would hide a missing division: TUNES posts usage scaled into FlexMeasures' units. + frbc._fill_level_scale = 0.1 + leakage = FRBCLeakageBehaviour( + message_id=get_unique_id(), + valid_from=datetime(2024, 1, 1, tzinfo=timezone.utc), + elements=[ + FRBCLeakageBehaviourElement( + fill_level_range=NumberRange(start_of_range=0, end_of_range=100), + leakage_rate=0.001, + ) + ], + ) + older = leakage.model_copy(update={"message_id": get_unique_id()}) + frbc._leakage_behaviour_history[str(older.message_id)] = older + frbc._leakage_behaviour_history[str(leakage.message_id)] = leakage # the latest + # TUNES asks for a schedule by trigger and get, rather than trigger_and_get_schedule, over six hours. + fm_client.get_schedule.return_value = {"values": [0.0] * 24} + with patch( + "flexmeasures_client.s2.control_types.FRBC.frbc_tunes.fm_schedule_to_instructions", + return_value=[], + ) as translate: + await frbc.trigger_schedule() + + frame = translate.call_args.args[0] + usage = list(frame["usage_forecast"].dropna()) + # 3600 per hour, scaled by 0.1 on the way out: 10 per second in the resource manager's own units. + assert usage and usage == [pytest.approx(10.0)] * len(usage) + assert translate.call_args.kwargs["leakage_behaviour"] is leakage diff --git a/tests/s2/test_frbc_utils.py b/tests/s2/test_frbc_utils.py index e9461821..26061aa8 100644 --- a/tests/s2/test_frbc_utils.py +++ b/tests/s2/test_frbc_utils.py @@ -12,7 +12,6 @@ clamp_distance, fm_schedule_to_instructions, get_unique_id, - op_mode_compute_factor, power_to_fill_rate_with_metrics, ) @@ -39,28 +38,45 @@ def example_op_mode_elem(default_power_range): @pytest.mark.parametrize( - "fill_rate_range, input_fill_rate, expected_factor", + "power_range, fill_rate_range, power, expected_factor", [ - ((0.0, 1.0), 0.0, 0.0), - ((0.0, 1.0), 0.5, 0.5), - ((0.0, 1.0), 1.0, 1.0), - ((2.0, 4.0), 3.0, 0.5), - ((1.0, 1.0), 1.0, 0.0), + ((100.0, 200.0), (0.0, 1.0), 100.0, 0.0), + ((100.0, 200.0), (0.0, 1.0), 150.0, 0.5), + ((100.0, 200.0), (0.0, 1.0), 200.0, 1.0), + ((100.0, 200.0), (0.0, 1.0), 250.0, 1.0), + ((100.0, 200.0), (0.0, 1.0), 50.0, 0.0), + # One fill rate, a range of power: the factor still follows from the power. + # Worked back out of the fill rate instead, this asked for the bottom of the power range. + ((800.0, 1200.0), (2.0, 2.0), 1200.0, 1.0), + ((800.0, 1200.0), (2.0, 2.0), 1000.0, 0.5), + # One power, a range of fill rate: every factor draws this power, + # so the top of the fill-rate range is asked for, which is where the efficiency FlexMeasures plans with is declared. + ((1000.0, 1000.0), (1.0, 2.0), 1000.0, 1.0), + ((1000.0, 1000.0), (2.0, 1.0), 1000.0, 1.0), + # One power and one fill rate, as an on-off actuator states them: the factor makes no difference. + ((1000.0, 1000.0), (2.0, 2.0), 1000.0, 0.0), ], ) -def test_op_mode_compute_factor( - fill_rate_range, input_fill_rate, expected_factor, default_power_range +def test_the_factor_is_the_one_that_draws_the_power_asked_for( + power_range, fill_rate_range, power, expected_factor ): - op_mode_elem = FRBCOperationModeElement( + element = FRBCOperationModeElement( fill_level_range=NumberRange(start_of_range=0.0, end_of_range=1.0), fill_rate=NumberRange( start_of_range=fill_rate_range[0], end_of_range=fill_rate_range[1] ), - power_ranges=default_power_range, - running_costs=NumberRange(start_of_range=1.0, end_of_range=1.0), + power_ranges=[ + PowerRange( + start_of_range=power_range[0], + end_of_range=power_range[1], + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_3_PHASE_SYMMETRIC, + ) + ], ) - - factor = op_mode_compute_factor(op_mode_elem, fill_rate=input_fill_rate) + operation_mode = FRBCOperationMode( + id=get_unique_id(), elements=[element], abnormal_condition_only=False + ) + *_, factor = power_to_fill_rate_with_metrics(operation_mode, power, 0.5) assert np.isclose(factor, expected_factor) @@ -150,7 +166,7 @@ def test_power_to_fill_rate_with_metrics_comprehensive( ) # --- Call the method under test --- - fill_rate, fill_penalty, power_penalty, efficiency, element = ( + fill_rate, fill_penalty, power_penalty, efficiency, element, _factor = ( power_to_fill_rate_with_metrics(op_mode, test_power, test_fill_level) ) diff --git a/tests/s2/test_measurements.py b/tests/s2/test_measurements.py new file mode 100644 index 00000000..1fb3182e --- /dev/null +++ b/tests/s2/test_measurements.py @@ -0,0 +1,293 @@ +"""Recording a resource manager's measured power, live and in a co-simulation.""" + +from __future__ import annotations + +import asyncio +from datetime import datetime, timedelta, timezone +from unittest.mock import AsyncMock, patch +from zoneinfo import ZoneInfo + +import pytest + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2 import measurements +from flexmeasures_client.s2.measurements import MeasurementRecorder, step_of + +T = datetime(2022, 3, 1, 12, 0, tzinfo=timezone.utc) +Q = timedelta(minutes=15) + + +def _recorder(resolution: str = "PT15M") -> tuple[MeasurementRecorder, AsyncMock]: + fm = AsyncMock(FlexMeasuresClient) + fm.get_sensor = AsyncMock(return_value={"event_resolution": resolution}) + return MeasurementRecorder(fm), fm + + +def _posts(fm) -> list[dict]: + return [ + c.kwargs | {"sensor_id": c.args[0]} for c in fm.post_sensor_data.await_args_list + ] + + +def _recorded(fm) -> dict[datetime, float]: + return {p["start"]: p["values"][0] for p in _posts(fm)} + + +@pytest.mark.asyncio +async def test_a_co_simulations_burst_of_past_quarter_hours_is_recorded(): + """Its resource manager sends a window's measurements once the window has passed, each stamped at its quarter hour's start. + + Binned by the moment they arrived, none of them fell in the step the CEM's clock was in, and none was recorded. + """ + recorder, fm = _recorder() + window_end = T + 4 * Q + for i, watts in enumerate([1000, 2000, 0, 500]): + await recorder.add(7, T + i * Q, watts, now=window_end) + await recorder.settle() + assert _recorded(fm) == {T: 1000, T + Q: 2000, T + 2 * Q: 0, T + 3 * Q: 500} + for post in _posts(fm): + assert ( + post["unit"] == "W" + ) # as S2 states power; it went out labelled kW, a thousand times too large + assert post["duration"] == Q + assert ( + post["prior"] == post["start"] + Q + ) # each known at the end of what it measures + + +@pytest.mark.asyncio +async def test_a_live_resource_managers_samples_are_averaged_per_step_once_it_is_over(): + recorder, fm = _recorder() + await recorder.add(7, T + timedelta(minutes=1), 1000, now=T + timedelta(minutes=1)) + await recorder.add(7, T + timedelta(minutes=8), 2000, now=T + timedelta(minutes=8)) + await recorder.settle() + fm.post_sensor_data.assert_not_awaited() # the step is not over + await recorder.add( + 7, T + Q + timedelta(minutes=1), 800, now=T + Q + timedelta(minutes=1) + ) + await recorder.settle() + assert _recorded(fm) == {T: 1500} + + +@pytest.mark.asyncio +async def test_a_step_sent_again_is_not_recorded_again(): + """A co-simulation's resource manager reconnects every round and sends its series again.""" + recorder, fm = _recorder() + await recorder.add(7, T, 1000, now=T + Q) + await recorder.settle() + await recorder.add(7, T, 1000, now=T + Q) + await recorder.add( + 7, T, 999, now=T + Q + ) # the first value recorded for a step stands + await recorder.settle() + assert len(_posts(fm)) == 1 + + +@pytest.mark.asyncio +async def test_a_step_that_could_not_be_recorded_is_tried_again_without_another_measurement( + caplog, +): + """The final window of a co-simulation has no measurement after it to try again with.""" + recorder, fm = _recorder() + fm.post_sensor_data.side_effect = [ConnectionError("down"), None] + with patch.object(measurements, "RETRY_AFTER", timedelta(milliseconds=10)): + await recorder.add(7, T, 1000, now=T + Q) + await recorder.settle() + assert "trying again later" in caplog.text + assert [p["start"] for p in _posts(fm)] == [T, T] + + +@pytest.mark.asyncio +async def test_a_step_that_keeps_failing_is_given_up_on_and_does_not_hold_up_the_rest( + caplog, +): + """Posted together, one step the server always refused failed every request after it, and nothing was recorded again.""" + recorder, fm = _recorder() + + async def refuse_the_first_step(sensor_id, **kwargs): + if kwargs["start"] == T: + raise ValueError("422") + + fm.post_sensor_data.side_effect = refuse_the_first_step + with patch.object(measurements, "RETRY_AFTER", timedelta(milliseconds=10)): + await recorder.add(7, T, 1000, now=T + Q) + await recorder.add(7, T + Q, 2000, now=T + 2 * Q) + await recorder.settle() + assert [p["start"] for p in _posts(fm)].count(T) == measurements.ATTEMPTS + assert T + Q in [p["start"] for p in _posts(fm)] + assert "Giving up" in caplog.text + assert recorder._pending[7] == {} + + +@pytest.mark.asyncio +async def test_two_measurements_for_one_sensor_at_once_post_each_step_once(): + """Two CEMs on one sensor, or a sample arriving while its step's post is in flight, posted it twice and then raised.""" + recorder, fm = _recorder() + release = asyncio.Event() + + async def slow(sensor_id, **kwargs): + await release.wait() + + fm.post_sensor_data.side_effect = slow + await recorder.add(7, T, 100, now=T + Q) + await asyncio.sleep(0) + await recorder.add(7, T + timedelta(minutes=5), 200, now=T + Q) + release.set() + await recorder.settle() + assert [p["start"] for p in _posts(fm)].count(T) == 1 + + +@pytest.mark.asyncio +async def test_each_sensor_is_recorded_at_its_own_resolution(): + recorder, fm = _recorder("PT1H") + await recorder.add(7, T + timedelta(minutes=20), 10, now=T + timedelta(minutes=20)) + await recorder.add(7, T + timedelta(minutes=40), 30, now=T + timedelta(hours=1)) + await recorder.settle() + (post,) = _posts(fm) + assert (post["start"], post["values"], post["duration"]) == ( + T, + [20], + timedelta(hours=1), + ) + fm.get_sensor.assert_awaited_once() # asked once + + +@pytest.mark.asyncio +async def test_on_an_instantaneous_sensor_each_measurement_is_its_own_reading(): + recorder, fm = _recorder("PT0M") + await recorder.add(7, T, 10, now=T) + await recorder.add(7, T + timedelta(seconds=30), 20, now=T + timedelta(seconds=30)) + await recorder.settle() + assert [(p["start"], p["values"], p["duration"]) for p in _posts(fm)] == [ + (T, [10], timedelta(0)), + (T + timedelta(seconds=30), [20], timedelta(0)), + ] + + +@pytest.mark.asyncio +async def test_a_sensor_stating_no_resolution_cannot_take_measurements(): + recorder, fm = _recorder() + fm.get_sensor = AsyncMock(return_value={}) + with pytest.raises(ValueError, match="no resolution"): + await recorder.add(7, T, 10, now=T + Q) + + +@pytest.mark.asyncio +async def test_what_was_recorded_is_forgotten_after_a_week(): + recorder, fm = _recorder() + await recorder.add(7, T, 1000, now=T + Q) + await recorder.settle() + await recorder.add(7, T + timedelta(days=8), 1, now=T + timedelta(days=8)) + await recorder.settle() + assert T not in recorder._recorded[7] + await recorder.add(7, T, 1000, now=T + timedelta(days=8)) + await recorder.settle() + assert [p["start"] for p in _posts(fm)].count(T) == 2 + + +def test_a_step_in_the_repeated_autumn_hour_is_found_in_the_pass_the_moment_is_in(): + """pandas cannot floor such a moment by its pass alone, and the CEM answered every resend with a temporary error.""" + amsterdam = ZoneInfo("Europe/Amsterdam") + second_pass = datetime(2026, 10, 25, 1, 37, tzinfo=timezone.utc).astimezone( + amsterdam + ) + assert second_pass.fold == 1 + step = step_of(second_pass, Q) + assert step.astimezone(timezone.utc) == datetime( + 2026, 10, 25, 1, 30, tzinfo=timezone.utc + ) + + +def test_steps_are_on_the_moments_own_clock(): + kolkata = ZoneInfo("Asia/Kolkata") + assert step_of( + datetime(2026, 11, 5, 14, 40, tzinfo=kolkata), timedelta(hours=1) + ) == datetime(2026, 11, 5, 14, 0, tzinfo=kolkata) + + +def _measurement(*values: tuple[str, float]): + from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue + + from flexmeasures_client.s2.utils import get_unique_id + + return PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=T, + values=[ + PowerValue(commodity_quantity=CommodityQuantity(q), value=v) + for q, v in values + ], + ) + + +@pytest.mark.asyncio +async def test_the_cem_hands_measured_power_to_its_recorder_on_its_own_clock(): + from s2python.common import ReceptionStatusValues + + from flexmeasures_client.s2.cem import CEM + from flexmeasures_client.s2.clock import SimulatedClock + + recorder, fm = _recorder() + clock = SimulatedClock(T + timedelta(minutes=5)) + cem = CEM( + fm_client=fm, + clock=clock, + power_sensor_id={"ELECTRIC.POWER.L1": 7}, + measurement_recorder=recorder, + ) + await cem.handle_message(_measurement(("ELECTRIC.POWER.L1", 750.0))) + assert (await cem.get_message())["status"] == ReceptionStatusValues.OK.value + await recorder.settle() + fm.post_sensor_data.assert_not_awaited() # its step is not over by the simulated clock + clock.set(T + Q) + await cem.handle_message(_measurement(("ELECTRIC.POWER.L1", 750.0))) + await cem.get_message() + await recorder.settle() + assert _recorded(fm) == {T: 750.0} + + +@pytest.mark.asyncio +async def test_a_message_the_cem_cannot_take_in_full_is_taken_not_at_all(): + """Asked for again, a message taken in part had the part taken twice, which skewed its average.""" + from s2python.common import ReceptionStatusValues + + from flexmeasures_client.s2.cem import CEM + from flexmeasures_client.s2.clock import SimulatedClock + + recorder, fm = _recorder() + + async def readable(sensor_id, **kwargs): + if sensor_id == 8: + raise ConnectionError("down") + return {"event_resolution": "PT15M"} + + fm.get_sensor = AsyncMock(side_effect=readable) + cem = CEM( + fm_client=fm, + clock=SimulatedClock(T + timedelta(minutes=5)), + power_sensor_id={"ELECTRIC.POWER.L1": 7, "ELECTRIC.POWER.L2": 8}, + measurement_recorder=recorder, + ) + await cem.handle_message( + _measurement(("ELECTRIC.POWER.L1", 750.0), ("ELECTRIC.POWER.L2", 10.0)) + ) + assert (await cem.get_message())[ + "status" + ] == ReceptionStatusValues.TEMPORARY_ERROR.value + assert recorder._pending[7] == {} + + +def test_a_cem_has_a_recorder_of_its_own_unless_given_one(): + from flexmeasures_client.s2.cem import CEM + + assert isinstance(CEM(fm_client=None)._measurement_recorder, MeasurementRecorder) + + +@pytest.mark.asyncio +async def test_a_defect_while_recording_is_logged_rather_than_lost(caplog): + """An exception no post is expected to raise surfaces in the log, not as a task exception never retrieved.""" + recorder, fm = _recorder() + fm.post_sensor_data.side_effect = TypeError("a malformed error payload") + await recorder.add(7, T, 1000, now=T + Q) + await recorder.settle() + assert "Recording measured power failed" in caplog.text diff --git a/tests/s2/test_site.py b/tests/s2/test_site.py new file mode 100644 index 00000000..c67dbad9 --- /dev/null +++ b/tests/s2/test_site.py @@ -0,0 +1,402 @@ +"""A site of its own for each resource manager, and a CEM that waits for it.""" + +from __future__ import annotations + +import asyncio +from datetime import timedelta +from unittest.mock import AsyncMock + +import pytest +from s2python.common import ControlType, ReceptionStatus, ReceptionStatusValues + +from flexmeasures_client.client import FlexMeasuresClient +from flexmeasures_client.s2.cem import CEM +from flexmeasures_client.s2.site import ( + FRBC_SITE_SENSORS, + FRBCSiteConfigurator, + SiteSensor, + ensure_site, + frbc_site_graphs, +) + + +def _flexmeasures(existing: list[dict] | None = None) -> AsyncMock: + fm = AsyncMock(FlexMeasuresClient) + fm.get_account = AsyncMock(return_value={"id": 1}) + fm.get_assets = AsyncMock(return_value=existing or []) + fm.add_asset = AsyncMock(side_effect=lambda **kw: {"id": 50, "sensors": [], **kw}) + fm.made = {} + + def add_sensor(**kw): + sensor = {"id": 100 + len(fm.made), **kw} + fm.made[kw["name"]] = sensor + return sensor + + fm.add_sensor = AsyncMock(side_effect=add_sensor) + return fm + + +@pytest.fixture +def named_details(resource_manager_details): + return resource_manager_details.model_copy(update={"name": "home A"}) + + +@pytest.mark.asyncio +async def test_a_site_is_made_with_every_sensor_and_its_graphs(): + fm = _flexmeasures() + asset, sensors = await ensure_site( + fm, + "home A", + asset_specs={"generic_asset_type_id": 6}, + sensors=FRBC_SITE_SENSORS, + timezone="UTC", + ) + assert asset["id"] == 50 + assert set(sensors) == set(FRBC_SITE_SENSORS) + made = {c.kwargs["name"]: c.kwargs for c in fm.add_sensor.await_args_list} + assert made["power"]["attributes"] == {"consumption_is_positive": True} + assert made["state of charge"]["event_resolution"] == "PT0M" + graphs = fm.update_asset.await_args.args[1]["sensors_to_show"] + assert graphs == frbc_site_graphs(sensors) + + +@pytest.mark.asyncio +async def test_an_existing_site_keeps_its_sensors_and_gets_the_missing_ones(): + fm = _flexmeasures( + [ + {"id": 8, "name": "home B", "sensors": []}, + {"id": 7, "name": "home A", "sensors": [{"id": 70, "name": "price"}]}, + ] + ) + asset, sensors = await ensure_site( + fm, + "home A", + asset_specs={"generic_asset_type_id": 6}, + sensors=FRBC_SITE_SENSORS, + timezone="UTC", + ) + assert asset["id"] == 7 + assert sensors["price"]["id"] == 70 + assert "price" not in {c.kwargs["name"] for c in fm.add_sensor.await_args_list} + fm.add_asset.assert_not_awaited() + specs_update = fm.update_asset.await_args_list[0] + assert specs_update.args == (7, {"generic_asset_type_id": 6}) + + +@pytest.mark.asyncio +async def test_a_known_site_is_fetched_by_id_rather_than_found_in_a_listing(): + """A resource manager reconnects every round of a co-simulation; a listing of every asset each time is slow.""" + fm = _flexmeasures() + fm.get_asset = AsyncMock(return_value={"id": 7, "name": "home A", "sensors": []}) + await ensure_site( + fm, + "home A", + asset_specs={}, + sensors={"price": SiteSensor("EUR/kWh", "PT15M")}, + timezone="UTC", + known_asset_id=7, + ) + fm.get_assets.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_a_known_id_that_now_names_another_asset_is_looked_up_again(): + fm = _flexmeasures([{"id": 9, "name": "home A", "sensors": []}]) + fm.get_asset = AsyncMock( + return_value={"id": 7, "name": "someone else", "sensors": []} + ) + asset, _ = await ensure_site( + fm, "home A", asset_specs={}, sensors={}, timezone="UTC", known_asset_id=7 + ) + assert asset["id"] == 9 + + +@pytest.mark.asyncio +async def test_a_site_needs_an_organisation_to_be_made_in(): + fm = _flexmeasures() + fm.get_account = AsyncMock(return_value=None) + with pytest.raises(ValueError, match="no organisation"): + await ensure_site(fm, "home A", asset_specs={}, sensors={}, timezone="UTC") + + +def test_a_graph_without_sensors_on_the_site_is_left_out(): + graphs = frbc_site_graphs({"price": {"id": 1}, "power": {"id": 2}}) + assert graphs == [ + {"title": "Prices", "sensors": [1]}, + {"title": "Power", "sensors": [2]}, + ] + + +@pytest.mark.asyncio +async def test_each_resource_manager_gets_a_site_named_after_it_and_a_handler_wired_to_it( + named_details, +): + fm = _flexmeasures() + configurator = FRBCSiteConfigurator( + fm, asset_type_id=6, power_sensor_attributes={"is_strictly_non_positive": True} + ) + site = await configurator(named_details) + assert fm.add_asset.await_args.kwargs["name"] == "home A" + made = fm.made + assert made["power"]["attributes"] == { + "consumption_is_positive": True, + "is_strictly_non_positive": True, + } + handler = site.handler + assert handler._power_sensor_id == made["power"]["id"] + assert handler._price_sensor_id == made["price"]["id"] + assert handler._production_price_sensor_id == made["production price"]["id"] + assert handler._soc_sensor_id == made["state of charge"]["id"] + assert handler._valid_from_shift == timedelta(0) + assert handler._schedule_duration == timedelta(hours=24) + assert handler.send_revocations is True + assert site.power_sensor_ids == {"ELECTRIC.POWER.L1": made["measured-power"]["id"]} + + +@pytest.mark.asyncio +async def test_a_reconnecting_resource_manager_finds_its_site_by_id(named_details): + fm = _flexmeasures() + configurator = FRBCSiteConfigurator(fm, asset_type_id=6) + await configurator(named_details) + fm.get_asset = AsyncMock( + return_value={"id": 50, "name": "home A", "sensors": list(fm.made.values())} + ) + fm.get_assets.reset_mock() + await configurator(named_details) + fm.get_assets.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_a_resource_manager_without_a_name_gets_no_site(resource_manager_details): + configurator = FRBCSiteConfigurator(_flexmeasures(), asset_type_id=6) + with pytest.raises(ValueError, match="without a name"): + await configurator(resource_manager_details.model_copy(update={"name": None})) + + +# --- The CEM, while the site is being configured --- + + +async def _details_then_selection(cem, rm_handshake, resource_manager_details): + await cem.handle_message(rm_handshake) + await cem.handle_message(resource_manager_details) + await cem.activate_control_type(ControlType.FILL_RATE_BASED_CONTROL) + messages = [] + while not cem._sending_queue.empty(): + messages.append(await cem.get_message()) + return [m for m in messages if m["message_type"] == "SelectControlType"][0] + + +@pytest.mark.asyncio +async def test_messages_are_answered_with_a_temporary_error_until_the_site_is_configured( + rm_handshake, named_details, frbc_system_description +): + """The resource manager retries a temporary error, so it is served once its handler exists.""" + release = asyncio.Event() + fm = _flexmeasures() + configurator = FRBCSiteConfigurator(fm, asset_type_id=6) + + async def slow(details): + await release.wait() + return await configurator(details) + + cem = CEM(fm_client=fm, site_configurator=slow) + selection = await _details_then_selection(cem, rm_handshake, named_details) + await cem.handle_message( + ReceptionStatus( + subject_message_id=selection["message_id"], status=ReceptionStatusValues.OK + ) + ) + assert cem.control_type == ControlType.FILL_RATE_BASED_CONTROL + + await cem.handle_message(frbc_system_description) + assert (await cem.get_message())[ + "status" + ] == ReceptionStatusValues.TEMPORARY_ERROR.value + + release.set() + await cem._configuring + assert ControlType.FILL_RATE_BASED_CONTROL in cem._control_types_handlers + assert cem.power_sensor_id and "ELECTRIC.POWER.L1" in cem.power_sensor_id + await cem.handle_message(frbc_system_description) + answer = await cem.get_message() + assert answer["status"] == ReceptionStatusValues.OK.value + + +@pytest.mark.asyncio +async def test_a_site_that_could_not_be_set_up_is_tried_again_on_the_next_message( + rm_handshake, named_details, frbc_system_description, caplog +): + """A transient failure, such as a server error under load, would otherwise leave the resource manager retrying forever.""" + fm = _flexmeasures() + configurator = FRBCSiteConfigurator(fm, asset_type_id=6) + attempts = [] + + async def flaky(details): + attempts.append(details.name) + if len(attempts) == 1: + raise KeyError("a sensor the table lacks") + return await configurator(details) + + cem = CEM(fm_client=fm, site_configurator=flaky) + await cem.handle_message(rm_handshake) + await cem.handle_message(named_details) + await asyncio.wait([cem._configuring]) + assert "trying again on its next message" in caplog.text + assert cem._control_types_handlers == {} + + cem._control_type = ControlType.FILL_RATE_BASED_CONTROL + await cem.handle_message(frbc_system_description) + await asyncio.wait([cem._configuring]) + assert attempts == ["home A", "home A"] + assert ControlType.FILL_RATE_BASED_CONTROL in cem._control_types_handlers + + +@pytest.mark.asyncio +async def test_a_measurement_before_the_site_is_set_up_is_asked_for_again( + rm_handshake, named_details +): + """It used to be recorded on sensor 357, whichever site that belonged to, and answered OK, so it was never resent.""" + from datetime import datetime, timezone + + from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue + + from flexmeasures_client.s2.utils import get_unique_id + + release = asyncio.Event() + + async def slow(details): + await release.wait() + + fm = _flexmeasures() + cem = CEM(fm_client=fm, site_configurator=slow) + await cem.handle_message(rm_handshake) + await cem.handle_message(named_details) + while not cem._sending_queue.empty(): + await cem.get_message() + await cem.handle_message( + PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=datetime(2022, 3, 1, 12, tzinfo=timezone.utc), + values=[ + PowerValue( + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=500.0 + ) + ], + ) + ) + answer = await cem.get_message() + assert answer["status"] == ReceptionStatusValues.TEMPORARY_ERROR.value + fm.post_sensor_data.assert_not_awaited() + release.set() + + +@pytest.mark.asyncio +async def test_without_a_sensor_for_it_a_measurement_is_not_recorded(): + from datetime import datetime, timezone + + from s2python.common import CommodityQuantity, PowerMeasurement, PowerValue + + from flexmeasures_client.s2.utils import get_unique_id + + fm = _flexmeasures() + cem = CEM(fm_client=fm) + await cem.handle_message( + PowerMeasurement( + message_id=get_unique_id(), + measurement_timestamp=datetime(2022, 3, 1, 12, tzinfo=timezone.utc), + values=[ + PowerValue( + commodity_quantity=CommodityQuantity.ELECTRIC_POWER_L1, value=500.0 + ) + ], + ) + ) + fm.post_sensor_data.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_closing_the_cem_stops_a_site_being_configured( + rm_handshake, resource_manager_details +): + async def forever(details): + await asyncio.sleep(3600) + + cem = CEM(fm_client=_flexmeasures(), site_configurator=forever) + await cem.handle_message(rm_handshake) + await cem.handle_message(resource_manager_details) + configuring = cem._configuring + await cem.close() + await asyncio.wait([configuring], timeout=1) + assert configuring.cancelled() + + +@pytest.mark.asyncio +async def test_a_known_site_that_cannot_be_fetched_is_looked_up_again(): + """Deleted since, say: the listing finds it by name, or it is made again.""" + fm = _flexmeasures([{"id": 9, "name": "home A", "sensors": []}]) + fm.get_asset = AsyncMock(side_effect=ValueError("404")) + asset, _ = await ensure_site( + fm, "home A", asset_specs={}, sensors={}, timezone="UTC", known_asset_id=7 + ) + assert asset["id"] == 9 + + +@pytest.mark.asyncio +async def test_a_reconnect_keeps_what_others_wrote_to_the_flex_model_and_the_graphs(): + """A co-simulation's controller writes its end-of-day stock value into the flex model, and adds to the graphs.""" + fm = _flexmeasures( + [ + { + "id": 7, + "name": "home A", + "sensors": [], + "flex_model": { + "soc-value-at-end": "0.1 EUR/kWh", + "power-capacity": "1 kVA", + }, + "sensors_to_show": [{"title": "Gross vs net", "sensors": [1]}], + } + ] + ) + await ensure_site( + fm, + "home A", + asset_specs={"flex_model": {"power-capacity": "17250 VA"}}, + sensors={}, + timezone="UTC", + ) + specs = fm.update_asset.await_args_list[0].args[1] + assert specs["flex_model"] == { + "soc-value-at-end": "0.1 EUR/kWh", + "power-capacity": "17250 VA", + } + assert all( + "sensors_to_show" not in call.args[1] + for call in fm.update_asset.await_args_list + ) + + +@pytest.mark.asyncio +async def test_the_demo_resets_its_graphs_where_asked(): + fm = _flexmeasures( + [{"id": 7, "name": "home A", "sensors": [], "sensors_to_show": [1]}] + ) + await ensure_site( + fm, "home A", asset_specs={}, sensors={}, timezone="UTC", reset_graphs=True + ) + assert "sensors_to_show" in fm.update_asset.await_args.args[1] + + +@pytest.mark.asyncio +async def test_of_several_sites_with_one_name_the_newest_is_taken(): + fm = _flexmeasures( + [ + {"id": 9, "name": "home A", "sensors": []}, + {"id": 12, "name": "home A", "sensors": []}, + {"id": 3, "name": "home A", "sensors": []}, + ] + ) + asset, _ = await ensure_site( + fm, "home A", asset_specs={}, sensors={}, timezone="UTC" + ) + assert asset["id"] == 12