feat(S2): #188 finished, reconciled onto main (1 of 2)

.coveragerc

@@ -2,9 +2,6 @@
 [run]
 branch = True
 source = flexmeasures_client
-omit =
-    src/flexmeasures_client/s2/*
-    */site-packages/flexmeasures_client/s2/*
 
 [paths]
 source =

.github/workflows/ci.yml

@@ -104,7 +104,19 @@ jobs:
       - name: Install dependencies
         run: uv sync --frozen --extra s2 --group test
       - name: Run S2 tests
-        run: uv run pytest tests/s2 -rFEx --durations 10 --color yes
+        run: uv run pytest tests/s2 -rFEx --durations 10 --color yes --cov
+      - name: Show coverage report
+        run: uv run coverage report
+      - name: Generate coverage report
+        run: uv run coverage lcov -o coverage.lcov
+      - name: Upload partial coverage report
+        uses: coverallsapp/github-action@v2
+        with:
+          fail-on-error: false
+          path-to-lcov: coverage.lcov
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          flag-name: s2 - py${{ matrix.python }}
+          parallel: true
 
   finalize:
     needs: [ test, test-s2]

.pre-commit-config.yaml

@@ -53,7 +53,8 @@ repos:
     name: mypy (static typing)
     pass_filenames: false
     language: system
-    entry: uv run poe type-check
+    # With the s2 extra, so that the S2 message types are there to check against.
+    entry: uv run --extra s2 poe type-check
     verbose: true
 
 - repo: local
@@ -66,3 +67,14 @@ repos:
         Put screenshots in the FlexMeasures/screenshots repo, or leave them
         untracked and attach them to the PR/issue manually.
       files: '\.(png|jpe?g|gif|bmp|tiff?|webp|ico|psd)$'
+    - id: no-print-in-unattended-code
+      name: No print() in code that runs unattended
+      # A CEM's handlers run for days without anyone watching standard output,
+      # so what they have to say goes to the log, where a deployment keeps it.
+      # The example scripts print on purpose, and so do the client's interactive confirmations, which live outside s2/.
+      # Matched anywhere on a line, not only at its start, and with a space before the parenthesis too,
+      # so that `if x: print(...)`, `return print(...)`, `print (...)`, pprint and writes to stdout or stderr are caught.
+      language: pygrep
+      entry: '\b(print|pprint)\s*\(|sys\.std(out|err)\.write'
+      files: '^src/flexmeasures_client/s2/.*\.py$'
+      exclude: '^src/flexmeasures_client/s2/script/'

CHANGELOG.rst

@@ -27,8 +27,14 @@ Added
 Changed
 -------
 
+* ``add_asset`` no longer requires a ``latitude`` and ``longitude``, which FlexMeasures does not require either, for an asset with no location, such as one standing for a community. ``generic_asset_type_id`` stays required, and half a location is refused [`#261 <https://github.com/FlexMeasures/flexmeasures-client/pull/261>`__]
 * ``post_sensor_data``, and the deprecated ``post_measurements`` it was renamed from, wait for a server of FlexMeasures 0.33.0 or above that ingests the posted data asynchronously, answering ``202 Accepted`` with a job id, to finish doing so, so that a caller can read the data back or schedule over it; a failed job raises ``JobFailedError``, while a job still pending at the timeout, or one that cannot be looked up, is logged as an error and leaves the post successful. Pass ``await_ingestion=False`` to return as soon as the data is accepted [`#242 <https://github.com/FlexMeasures/flexmeasures-client/pull/242>`__]
 
+Removed
+-------
+
+* ``response_handling.check_content_type``, which refused an answer in a JSON variant that aiohttp parses, such as ``application/problem+json``, with an error that raised an ``AttributeError`` of its own when printed. Such an answer is now read like any other. A content type that is not JSON at all is still reported as a ``ConnectionError``, by aiohttp's own check [`#257 <https://github.com/FlexMeasures/flexmeasures-client/pull/257>`__]
+
 Fixed
 -----
 

examples/S2-co-simulation/README.md

@@ -0,0 +1,71 @@
+# A FlexMeasures stack for an S2 co-simulation
+
+`docker-compose.cem.yml` runs FlexMeasures with one S2 CEM per simulated home,
+for a co-simulation whose resource managers connect over websockets, such as a
+community of homes steered at once. How to start it is at the top of the file.
+
+Each CEM is the example server (`src/flexmeasures_client/s2/script/websockets_server.py`)
+with two switches on:
+
+- `S2_ACCEPT_SIMULATION_METADATA`: a resource manager wraps each message with the
+  simulated time, `{"message": ..., "metadata": {"dt": ...}}`, and the CEM, and every
+  handler it registers, go by that time (see `flexmeasures_client.s2.clock`).
+  `dt` has to name its time zone. Without the switch, a wrapped message is refused
+  as invalid, from the handshake on.
+- `S2_SITE_PER_RESOURCE_MANAGER`: each resource manager gets a site of its own,
+  named after it, with the sensors in `flexmeasures_client.s2.site.FRBC_SITE_SENSORS`.
+  Messages are answered with a temporary error until its site is set up, which a
+  resource manager retries. Its schedules start from the simulated now and cover
+  24 hours. The site's prices are for whoever runs the simulation to post.
+
+FlexMeasures reads three settings a co-simulation needs from its instance config
+(`flexmeasures-instance/flexmeasures.cfg` in the flexmeasures checkout), not from the environment:
+
+```python
+# A steering round replans a window at the same simulated moment, and that plan replaces the one before.
+# Without this, the scheduler cannot save the second plan, and the home gets no instructions for its window.
+FLEXMEASURES_ALLOW_DATA_OVERWRITE = True
+# Every home triggers a schedule each round, all from one account, far above the default of 10 per 5 minutes,
+# and every trigger past the limit waits minutes before it is retried.
+FLEXMEASURES_API_TRIGGER_RATE_LIMIT = "1000 per minute"
+# A bound on each solve, since a tail of degenerate problems otherwise proves optimality past the job timeout.
+FLEXMEASURES_LP_SOLVER_OPTIONS = {"time_limit": "15"}
+```
+
+A co-simulation reads the sites by sensor name: `price`, `production price`,
+`power` (the schedule), `measured-power` (what the home measured) and
+`state of charge`.
+
+The stack uses FlexMeasures' own development database service, `dev-db`,
+whose data FlexMeasures' compose file keeps in `docker-compose-data/dev-db`
+inside the flexmeasures checkout it runs from.
+A compose project name of its own does not separate that.
+To keep a development database you care about out of it,
+run it from a folder of its own holding checkouts named `flexmeasures` and `flexmeasures-client`:
+the overlay mounts `../flexmeasures` and `../flexmeasures-client`, so those names are what it runs.
+Git worktrees will do, with one thing to add: FlexMeasures' image reads its version from git, which it cannot through a worktree.
+Give it as the build argument `FLEXMEASURES_VERSION` to each service built from FlexMeasures' Dockerfile, in one more overlay:
+
+```yaml
+x-version: &version
+  build:
+    context: .
+    dockerfile: Dockerfile
+    args:
+      FLEXMEASURES_VERSION: "1.1.0"
+
+services:
+  server:
+    <<: *version
+  worker:
+    <<: *version
+  # and likewise worker-2, worker-3, automation-runner and each cem
+```
+
+Its host ports, 5000, 5433, 6380, 1025, 8025 and 8080 to 8084, have to be free too;
+where they are not, an extra overlay can move them with compose's `!override`.
+
+It runs FlexMeasures from the flexmeasures checkout, which is mounted over the code the image installed,
+with this client installed into the image's virtualenv, which is what FlexMeasures and its plugins run from.
+The UI's third-party libraries are fetched into the installed package when the image is built, so a UI served from the mounted checkout may lack them; the API does not need them.
+The server trusts the name the CEMs reach it by, `server`, alongside the loopback names a development server trusts by default.

examples/S2-co-simulation/docker-compose.cem.yml

@@ -0,0 +1,175 @@
+# ------------------------------------------------------------------
+# A FlexMeasures stack with one S2 CEM per simulated home,
+# for a co-simulation whose resource managers connect over websockets,
+# such as the community simulations that steer several homes at once.
+#
+# It is an overlay on FlexMeasures' own docker-compose.yml,
+# and runs FlexMeasures from the flexmeasures checkout and the CEMs from this one, both mounted.
+# With checkouts named flexmeasures and flexmeasures-client side by side,
+# run it from the flexmeasures folder, which holds the Dockerfile:
+#
+#   docker compose \
+#     -f docker-compose.yml \
+#     -f ../flexmeasures-client/examples/S2-co-simulation/docker-compose.cem.yml \
+#     up
+#
+# Resource managers connect to ws://localhost:8080/ws, ws://localhost:8081/ws and so on, one per home.
+# Each gets a site of its own, named after it,
+# and the CEM goes by the simulated time its messages carry (see README.md).
+# ------------------------------------------------------------------
+
+# Shared definition for every CEM.
+# Each listens on port 8080 inside its own container; only the host port differs.
+x-cem: &cem
+  build:
+    context: .
+    dockerfile: Dockerfile
+  image: flexmeasures-client-cem
+  depends_on:
+    - server
+  restart: always
+  environment:
+    FLEXMEASURES_BASE_URL: http://server:5000
+    FLEXMEASURES_USER: toy-user@flexmeasures.io
+    FLEXMEASURES_PASSWORD: toy-password
+    LOGGING_LEVEL: INFO
+    # A resource manager in a co-simulation wraps each message with the simulated time.
+    S2_ACCEPT_SIMULATION_METADATA: "1"
+    # And each home is a site of its own, rather than all of them sharing one.
+    S2_SITE_PER_RESOURCE_MANAGER: "1"
+  volumes:
+    - ../flexmeasures-client:/app/flexmeasures-client:rw
+  entrypoint: ["/bin/sh", "-c"]
+  command:
+  - |
+    # Install and run with the same interpreter (`python3 -m pip`, not bare `pip`):
+    # in images built from FlexMeasures' uv-based Dockerfile, `python3` is the /app/.venv virtualenv,
+    # which may come without pip, and cannot see what the system pip installs.
+    python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade
+    # s2-python as this repo locks it, and as the co-simulation's resource managers use it.
+    # The checkout may be mounted without git history to read its version from, as a git worktree is,
+    # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one.
+    SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install --break-system-packages -e "/app/flexmeasures-client[s2]" "s2-python==0.8.1"
+    exec python3 /app/flexmeasures-client/src/flexmeasures_client/s2/script/websockets_server.py
+
+# Shared definition for every RQ worker: one per home simulated at once,
+# like the CEMs and the server's gunicorn workers below,
+# since every home's scheduling jobs otherwise queue behind one worker process.
+x-worker: &worker
+  # Self-contained, since worker-2 and worker-3 do not exist in the base file to merge with.
+  build:
+    context: .
+    dockerfile: Dockerfile
+  depends_on:
+    - dev-db
+    - queue-db
+    - mailhog
+  restart: on-failure
+  volumes:
+    - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:rw
+    - ../flexmeasures/flexmeasures:/app/flexmeasures:rw
+    - ./flexmeasures-instance/:/app/instance/:rw
+    - ../flexmeasures-client:/app/flexmeasures-client:rw
+  entrypoint: ["/bin/sh", "-c"]
+  environment:
+    SQLALCHEMY_DATABASE_URI: "postgresql://fm-dev-db-user:fm-dev-db-pass@dev-db:5432/fm-dev-db"
+    FLEXMEASURES_REDIS_URL: queue-db
+    FLEXMEASURES_REDIS_PASSWORD: fm-redis-pass
+    SECRET_KEY: notsecret
+    SECURITY_TOTP_SECRETS: '{"1": "something-secret"}'
+    FLEXMEASURES_ENV: development
+    MAIL_SERVER: mailhog
+    MAIL_PORT: 1025
+    LOGGING_LEVEL: INFO
+  command:
+    - |
+      # Into the image's virtualenv, which FlexMeasures and its plugins run from, with that interpreter's own pip.
+      python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade
+      python3 -m pip install -r /usr/var/flexmeasures-instance/requirements.txt
+      # The checkout may be mounted without git history to read its version from, as a git worktree is,
+      # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one.
+      SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]"
+      # Drop any lingering registration for the ingestion queue before this worker starts.
+      # FlexMeasures ingests posted sensor data asynchronously when Redis lists a worker for that queue,
+      # and a dead worker's entry counts too: posts then wait on a queue nobody serves, and a run hangs without an error.
+      # No worker here serves that queue, so removing those entries on every start is safe, and idempotent.
+      python3 - <<'PYGUARD'
+      import os, redis
+      r = redis.Redis(host=os.environ.get("FLEXMEASURES_REDIS_URL", "queue-db"),
+                      port=int(os.environ.get("FLEXMEASURES_REDIS_PORT", 6379)),
+                      password=os.environ.get("FLEXMEASURES_REDIS_PASSWORD"))
+      try:
+          for name in r.smembers("rq:workers"):
+              key = name.decode()
+              queues = (r.hget(key, "queues") or b"").decode()
+              if "ingestion" in queues or not queues:
+                  r.srem("rq:workers", name); r.delete(key)
+          r.delete("rq:workers:ingestion")
+      except redis.RedisError as exc:
+          print(f"ingestion-registration guard skipped: {exc}")
+      PYGUARD
+      # Named after the container, which is unique per worker.
+      # Every queue the base file's worker serves but ingestion, which is left to the server to do as it receives the data.
+      flexmeasures jobs run-worker --name "flexmeasures-worker-$$(hostname)" --queue scheduling\|forecasting\|reporting
+
+services:
+  dev-db:
+    ports:
+      - "5433:5432"
+  queue-db:
+    ports:
+      - "6380:6379"
+  server:
+    environment:
+      # The CEMs reach the server by its service name, which a development server does not trust by default.
+      TRUSTED_HOSTS: "localhost,.localhost,127.0.0.1,[::1],server"
+    volumes:
+      - ./flexmeasures-instance/:/usr/var/flexmeasures-instance/:rw
+      - ./flexmeasures-instance/:/app/instance/:rw
+      - ../flexmeasures/flexmeasures:/app/flexmeasures:rw
+      - ../flexmeasures-client:/app/flexmeasures-client:rw
+    command:
+      - |
+        # Into the image's virtualenv, which FlexMeasures and its plugins run from, with that interpreter's own pip.
+        python3 -m pip --version >/dev/null 2>&1 || python3 -m ensurepip --upgrade
+        python3 -m pip install -r /usr/var/flexmeasures-instance/requirements.txt
+        # The checkout may be mounted without git history to read its version from, as a git worktree is,
+        # so a version is given, in the plain variable, since hatch-vcs cannot match the per-distribution one.
+        SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 python3 -m pip install -e "/app/flexmeasures-client[s2]"
+        flexmeasures db upgrade
+        if ! flexmeasures show accounts | grep -q "Docker Toy Account"; then
+            flexmeasures add toy-account --name 'Docker Toy Account'
+        fi
+        # Worker processes rather than threads: a slow, CPU-bound request held its worker's GIL,
+        # and under several threads that starved the other requests into client-side timeouts.
+        # One worker per CEM below, since every home's CEM can be mid-request at once;
+        # scale the two together.
+        gunicorn --bind 0.0.0.0:5000 --worker-tmp-dir /dev/shm --workers 5 --threads 1 --timeout 120 wsgi:application
+  # One worker per home simulated at once; add or remove them along with the CEMs.
+  worker:
+    <<: *worker
+  worker-2:
+    <<: *worker
+  worker-3:
+    <<: *worker
+  # One CEM per simulated home; add one by copying an entry and taking the next host port.
+  cem:
+    <<: *cem
+    ports:
+      - "8080:8080"
+  cem-2:
+    <<: *cem
+    ports:
+      - "8081:8080"
+  cem-3:
+    <<: *cem
+    ports:
+      - "8082:8080"
+  cem-4:
+    <<: *cem
+    ports:
+      - "8083:8080"
+  cem-5:
+    <<: *cem
+    ports:
+      - "8084:8080"

pyproject.toml

@@ -85,15 +85,17 @@ help = "Run the test suite (excluding S2) and generate coverage report."
 cmd = "pytest --ignore=tests/s2 --cov --cov-report="
 
 [tool.poe.tasks.test-s2]
-help = "Run S2 tests."
-cmd = "pytest tests/s2"
+help = "Run S2 tests and generate coverage report."
+cmd = "pytest tests/s2 --cov --cov-report="
 
 [tool.poe.tasks.type-check]
-help = "Run mypy on files with type hints."
-shell = """
-    files=$(find src -name \\*.py | xargs grep -l "from typing import")
-    mypy --follow-imports skip --ignore-missing-imports $files
-"""
+help = "Run mypy on every source file, including the bodies of functions without type hints, against the types of what they import."
+cmd = "mypy --ignore-missing-imports --check-untyped-defs src"
+
+[tool.mypy]
+# The S2 messages are pydantic models, so without the plugin mypy misreads their constructors and fields.
+# It also makes a missing s2 extra fail loudly, rather than leaving every S2 type silently Any.
+plugins = ["pydantic.mypy"]
 
 [tool.poe.tasks.lint]
 help = "Run pre-commit hooks on all files."