diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index eae875c892..f53dc16f2e 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -53,6 +53,9 @@ jobs: >> .env - name: Keep running flexmeasures container in background run: docker run -t -d --env-file .env --network=host --name fm-container flexmeasures:latest + - name: Check that the image serves the UI's libraries itself + run: docker exec fm-container python -c + "from flexmeasures.ui.utils.vendor_assets import missing_files; assert not missing_files(), missing_files()" - name: Execute database upgrade run: docker exec --env-file .env fm-container flexmeasures db upgrade diff --git a/.gitignore b/.gitignore index d8f876b03e..5035b2f6e6 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,8 @@ __pycache__ notebooks/.ipynb_checkpoints/ flexmeasures/ui/static/documentation +# The UI's third-party libraries, when fetched with: python -m flexmeasures.ui.utils.fetch_vendor_assets +flexmeasures/ui/static/vendor/ documentation/img/screenshot_* documentation/_autosummary/ generic_asset_fm_user_ownership.sql diff --git a/Dockerfile b/Dockerfile index cdc2a17bd5..af0d86002a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,6 +50,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \ SETUPTOOLS_SCM_PRETEND_VERSION="${FLEXMEASURES_VERSION}" \ uv sync --frozen --reinstall-package flexmeasures --no-dev --no-editable +# Fetch the UI's third-party libraries into the installed package, so the UI also works without internet access. +# Each file is checked against the hash recorded in flexmeasures/ui/utils/vendor_assets.py, and the build fails on a mismatch. +# It runs outside /app, because python -m would otherwise import the source copy in /app/flexmeasures, which the runtime image leaves behind. +RUN cd / && python -m flexmeasures.ui.utils.fetch_vendor_assets + # Install gunicorn separately since it's not a dependency of the project RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install gunicorn==25.0.3 diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 2afd3e1423..52f458f078 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -40,6 +40,7 @@ New features Infrastructure / Support ------------------------- +* The UI can work without internet access: the Docker image serves the UI's JavaScript and CSS libraries itself, and a pip-installed FlexMeasures can fetch them once (see :ref:`install-ui-libraries`), while otherwise browsers keep loading them from CDNs; the ``FLEXMEASURES_JS_VERSIONS`` setting now only lists the versions a host wants to differ, optionally with the hashes of their files, so plugins reading it should read the ``js_versions`` template variable instead [see `PR #2711 `_] * ``add_beliefs_to_db_and_enqueue_forecasting_jobs`` now raises ``BeliefReplacementNotAllowed`` rather than an ``IntegrityError`` when saving would change existing values, so plugins catching ``IntegrityError`` around it should catch the new exception instead [see `PR #2722 `_] * ``[GET] /sensors/(id)`` now answers a sensor which does not exist with a 404, as the asset endpoints do, rather than with a 422, so that a client can tell a sensor which is gone apart from a request it got wrong [see `PR #2650 `_] * Speed up preparing a storage schedule by validating each device's constraints with vectorised comparisons instead of one ``DataFrame.eval`` per check (from ~25 ms to ~3 ms per device); the helpers ``validate_constraint`` and ``sanitize_expression`` in ``flexmeasures.data.models.planning.storage``, which nothing in FlexMeasures called any more, are removed, with ``validate_constraints`` replacing them [see `PR #2641 `_] diff --git a/documentation/configuration.rst b/documentation/configuration.rst index 519f382108..0df349c0bb 100644 --- a/documentation/configuration.rst +++ b/documentation/configuration.rst @@ -267,7 +267,30 @@ Default: ``{"renewables": ["solar", "wind"], "EVSE": ["one-way_evse", "two-way_e FLEXMEASURES_JS_VERSIONS ^^^^^^^^^^^^^^^^^^^^^^^^ -Default: ``{"vega": "5.22.1", "vegaembed": "6.20.8", "vegalite": "5.2.0"}`` +Versions of the UI's JavaScript and CSS libraries to load, in place of the versions listed in ``flexmeasures/ui/utils/vendor_assets.py``. +Only list the libraries you want to differ, as a listed version stays in place when an upgrade of FlexMeasures bumps that library. +A library's key maps to its version, optionally together with the SHA-256 hashes of its files: + +.. code-block:: python + + FLEXMEASURES_JS_VERSIONS = { + # A version only: browsers load the library from its CDN without checking it. + "vega": "5.33.1", + # A version and the hash of the library's one file. + "vegaembed": ("6.26.0", ""), + # A version and the hash of each of the library's files. + "leaflet": ("1.9.5", {"dist/leaflet.css": "", "dist/leaflet-src.min.js": "", ...}), + } + +With hashes, browsers check each file they load from a CDN, and the library can be served by FlexMeasures itself (see :ref:`install-ui-libraries`). +FlexMeasures expects another version to have the same files as the listed one. +This prints the setting for another version, with the hashes of its files: + +.. code-block:: bash + + $ python -m flexmeasures.ui.utils.fetch_vendor_assets --print-hashes vega --version 5.33.1 + +Default: ``{}`` Timing diff --git a/documentation/host/installation.rst b/documentation/host/installation.rst index 466e76116a..8a4d5d0afd 100644 --- a/documentation/host/installation.rst +++ b/documentation/host/installation.rst @@ -451,6 +451,26 @@ This suffices for a quick start. For an actually secure secret, here is a Python Finally, you should set ``SECURITY_TWO_FACTOR_RESCUE_MAIL`` to an email address under which you can be reached by users who cannot use their 2nd means of authentication. +.. _install-ui-libraries: + +Serve the UI's libraries without internet access +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The UI uses third-party JavaScript and CSS libraries, such as Bootstrap and Vega. +The Docker image comes with them, so FlexMeasures serves them itself. +A FlexMeasures installed via `pip` does not, so browsers load them from public CDNs, checking each file against its expected hash. + +To serve them from FlexMeasures instead, for example on a network without internet access, fetch them once: + +.. code-block:: bash + + $ python -m flexmeasures.ui.utils.fetch_vendor_assets + +This downloads each library's files and licence into FlexMeasures' installation, and fails on any file that does not match its expected hash. +It fetches the versions set by :ref:`FLEXMEASURES_JS_VERSIONS ` too, read from the config file FlexMeasures reads (or from the one passed with ``--config``), except for versions set without hashes. +Run it again after upgrading FlexMeasures, as an upgrade can bump a library's version. + + Where to go from here? ------------------------ diff --git a/documentation/tut/building_uis.rst b/documentation/tut/building_uis.rst index 0b6aa4645b..4c8b6598bf 100644 --- a/documentation/tut/building_uis.rst +++ b/documentation/tut/building_uis.rst @@ -212,8 +212,8 @@ First, we define a div tag for the chart and a basic layout (full width). We als - - + + +{% import "includes/vendor.html" as vendor %} +{{ vendor.echarts() }} +{% import "includes/vendor.html" as vendor %} +{{ vendor.leaflet_js() }} - - - +{% import "includes/vendor.html" as vendor %} +{{ vendor.litepicker() }} {% include 'includes/graphs.html' %} diff --git a/flexmeasures/ui/templates/assets/asset_new.html b/flexmeasures/ui/templates/assets/asset_new.html index 59d6749053..0353d7c710 100644 --- a/flexmeasures/ui/templates/assets/asset_new.html +++ b/flexmeasures/ui/templates/assets/asset_new.html @@ -188,7 +188,8 @@

Location

- +{% import "includes/vendor.html" as vendor %} +{{ vendor.leaflet_js() }} - - - + {% import "includes/vendor.html" as vendor %} + {{ vendor.litepicker() }} - + {{ vendor.leaflet_js() }} diff --git a/flexmeasures/ui/templates/base.html b/flexmeasures/ui/templates/base.html index 37ce123b83..398724ba8b 100644 --- a/flexmeasures/ui/templates/base.html +++ b/flexmeasures/ui/templates/base.html @@ -58,31 +58,17 @@ cursor: default; } - - - - - - - - - - - + {% import "includes/vendor.html" as vendor %} + {{ vendor.head() }} - - {% if active_page == "tasks" %} {% elif active_page in ("assets", "users", "accounts") %} - - + {{ vendor.datatables_css() }} {% endif %} {% if extra_css %} @@ -105,8 +91,8 @@ {% include "security/_messages.html" %} - - + {% import "includes/vendor.html" as vendor %} + {{ vendor.jquery() }} {# Div blocks that child pages can reference #} {% block divs %} @@ -176,9 +162,7 @@ {#- Scripts used by all views (e.g. by navigation menu) -#} - - - + {{ vendor.bootstrap_js() }} - - {% if js_versions %} - - - - - {# Workaround for loading a NodeJS module without NodeJS #} - - - - {% endif %} + {% import "includes/vendor.html" as vendor %} + {{ vendor.scripts() }} @@ -261,9 +235,8 @@ {% block paginate_tables_script %} - - + {% import "includes/vendor.html" as vendor %} + {{ vendor.datatables_js() }} {% endblock paginate_tables_script %} {% endblock body %} diff --git a/flexmeasures/ui/templates/dashboard.html b/flexmeasures/ui/templates/dashboard.html index afc1bcface..49217e90a9 100644 --- a/flexmeasures/ui/templates/dashboard.html +++ b/flexmeasures/ui/templates/dashboard.html @@ -90,10 +90,10 @@ - +{% import "includes/vendor.html" as vendor %} +{{ vendor.leaflet_js() }} - - +{{ vendor.markercluster_js() }} + + + + + + + + {#- These rules refer to vendored fonts and images, so they live here rather than in flexmeasures.css, which cannot tell whether those were fetched. -#} + +{% endmacro %} + +{% macro datatables_css() %} + +{% endmacro %} + +{#- Early in , as inline scripts in page content use jQuery. -#} +{% macro jquery() %} + + +{% endmacro %} + +{% macro bootstrap_js() %} + +{% endmacro %} + +{% macro echarts() %} + +{% endmacro %} + +{#- Chart and widget libraries, in the order base.html loaded them from CDNs. -#} +{% macro scripts() %} + + + + + + {{ echarts() }} + {# Workaround for loading a NodeJS module without NodeJS #} + + + +{% endmacro %} + +{% macro datatables_js() %} + + +{% endmacro %} + +{% macro leaflet_js() %} + +{% endmacro %} + +{% macro markercluster_js() %} + + +{% endmacro %} + +{#- Date range picker on the asset graph, asset properties and sensor pages. -#} +{% macro litepicker() %} + + + + +{% endmacro %} diff --git a/flexmeasures/ui/templates/sensors/index.html b/flexmeasures/ui/templates/sensors/index.html index 8a3750683f..f36132a63f 100644 --- a/flexmeasures/ui/templates/sensors/index.html +++ b/flexmeasures/ui/templates/sensors/index.html @@ -521,10 +521,8 @@
- - - - + {% import "includes/vendor.html" as vendor %} + {{ vendor.litepicker() }} {% include 'includes/graphs.html' %} diff --git a/flexmeasures/ui/tests/test_vendor_assets.py b/flexmeasures/ui/tests/test_vendor_assets.py new file mode 100644 index 0000000000..cf5462e592 --- /dev/null +++ b/flexmeasures/ui/tests/test_vendor_assets.py @@ -0,0 +1,485 @@ +"""The UI loads its third-party libraries from FlexMeasures when they were fetched, and from their CDNs otherwise.""" + +from __future__ import annotations + +import ast +import base64 +import hashlib +import logging +import re +from pathlib import Path + +import pytest +import urllib.error +from flask import url_for +from jinja2.runtime import Macro + +from flexmeasures.ui import ( + add_jinja_variables, + flexmeasures_ui, + load_vendor_libraries, + log_where_vendor_files_load_from, +) +from flexmeasures.ui.utils import fetch_vendor_assets, vendor_assets +from flexmeasures.ui.utils.vendor_assets import ( + APP_EXTENSION, + JS_VERSIONS, + VENDOR_LIBRARIES, + VendorLibrary, + libraries_with_overrides, +) + +UI_DIR = Path(flexmeasures_ui.root_path) +STATIC_DIR = UI_DIR / "static" +TEMPLATE_DIR = UI_DIR / "templates" + +TAG = re.compile(r"""<(?:script|link)\b[^>]*>""") +TAG_URL = re.compile(r"""(?:src|href)="([^"]+)\"""") +INTEGRITY = re.compile(r"""integrity="([^"]+)\"""") +CSS_URL = re.compile(r"""url\(\s*["']?([^"')]+?)["']?\s*\)""") + +HASH = "ab" * 32 + +LEGACY_KEYS = { + "vega", + "vegaembed", + "vegalite", + "echarts", + "currencysymbolmap", + "jsoneditor", + "leaflet", + "leafletmarkercluster", + "leafletmarkerclusterlayersupport", +} + + +@pytest.fixture +def not_fetched(tmp_path, monkeypatch): + """No library files were fetched, as in a fresh checkout or a pip-installed FlexMeasures.""" + monkeypatch.setattr(vendor_assets, "VENDOR_DIR", tmp_path / "vendor") + + +@pytest.fixture +def fetched(tmp_path, monkeypatch): + """Every library file was fetched, as in the Docker image. + + The files hold placeholder content, as these tests are about where files load from, not what is in them. + """ + static_dir = tmp_path / "static" + monkeypatch.setattr(vendor_assets, "VENDOR_DIR", static_dir / "vendor") + monkeypatch.setattr(flexmeasures_ui, "static_folder", str(static_dir)) + for key, library in VENDOR_LIBRARIES.items(): + for path in [*library.files, library.license_path]: + file = vendor_assets.local_file(key, path) + file.parent.mkdir(parents=True, exist_ok=True) + file.write_text(f"{key} {path}") + + +def render_vendor_macros(app) -> tuple[list[Macro], str]: + with app.test_request_context(): + module = app.jinja_env.get_template("includes/vendor.html").module + macros = [m for m in vars(module).values() if isinstance(m, Macro)] + return macros, "".join(str(macro()) for macro in macros) + + +def manifest_entry( + url: str, libraries: dict[str, VendorLibrary] = VENDOR_LIBRARIES +) -> tuple[str, str]: + """The library key and file path that a CDN URL was made from.""" + for key, library in libraries.items(): + for path in library.files: + if library.url(path) == url: + return key, path + raise AssertionError(f"{url} is not in the libraries") + + +@pytest.fixture +def js_versions_setting(app, monkeypatch): + """Start the app as with a FLEXMEASURES_JS_VERSIONS setting, and restore it afterwards.""" + monkeypatch.setitem(app.extensions, APP_EXTENSION, VENDOR_LIBRARIES) + monkeypatch.setitem(app.jinja_env.globals, "js_versions", JS_VERSIONS) + + def set_setting(setting: dict): + monkeypatch.setitem(app.config, "FLEXMEASURES_JS_VERSIONS", setting) + load_vendor_libraries(app) + add_jinja_variables(app) + return app.extensions[APP_EXTENSION] + + return set_setting + + +def test_js_versions_global_keeps_legacy_keys(app): + """Plugin templates that read `js_versions.vega` and the like keep working.""" + assert app.jinja_env.globals["js_versions"] == JS_VERSIONS + assert LEGACY_KEYS <= JS_VERSIONS.keys() + + +def test_js_versions_global_follows_the_setting(app, js_versions_setting): + """Plugin templates read the versions that the host set.""" + js_versions_setting({"vega": "9.9.9"}) + assert app.jinja_env.globals["js_versions"]["vega"] == "9.9.9" + assert app.jinja_env.globals["js_versions"]["jquery"] == JS_VERSIONS["jquery"] + + +@pytest.mark.parametrize( + "setting, expected_files", + [ + # A copy of the setting's old defaults keeps the recorded hashes. + ({"vega": JS_VERSIONS["vega"]}, VENDOR_LIBRARIES["vega"].files), + ({"vega": "9.9.9"}, {"build/vega.min.js": None}), + ({"vega": ("9.9.9", HASH)}, {"build/vega.min.js": HASH}), + ( + { + "leaflet": ( + "9.9.9", + dict.fromkeys(VENDOR_LIBRARIES["leaflet"].files, HASH), + ) + }, + dict.fromkeys(VENDOR_LIBRARIES["leaflet"].files, HASH), + ), + ], +) +def test_js_versions_setting_forms(setting, expected_files): + """A version can be set on its own, with the hash of a library's one file, or with the hash of each of its files.""" + ((key, value),) = setting.items() + library = libraries_with_overrides(setting)[key] + assert library.version == (value if isinstance(value, str) else value[0]) + assert library.files == expected_files + if library.version != VENDOR_LIBRARIES[key].version: + assert library.license_sha256 is None + + +def test_js_versions_setting_ignores_unknown_libraries(caplog): + with caplog.at_level(logging.WARNING): + libraries = libraries_with_overrides({"vegga": "9.9.9"}) + assert libraries == VENDOR_LIBRARIES + assert "FLEXMEASURES_JS_VERSIONS sets vegga" in caplog.text + + +@pytest.mark.parametrize( + "setting, message", + [ + ({"leaflet": ("9.9.9", HASH)}, "has 7 files"), + ({"leaflet": ("9.9.9", {"dist/leaflet.css": HASH})}, "needs them for exactly"), + ({"vega": ("9.9.9", "sha256-abc")}, "not a SHA-256 hash"), + ({"vega": ("9.9.9",)}, "Set a version"), + ({"vega": ("9.9.9", 5)}, "Set a version"), + ({"vega": 5}, "Set a version"), + ], +) +def test_js_versions_setting_rejects_unusable_hashes(setting, message): + """A setting whose hashes cannot be matched to the library's files stops the app from starting, rather than loading files unchecked.""" + with pytest.raises(ValueError, match=message): + libraries_with_overrides(setting) + + +def test_overridden_versions_load_from_cdns(app, not_fetched, js_versions_setting): + """A version set without hashes loads unchecked, and one set with a hash is checked against it.""" + js_versions_setting({"vega": "9.9.9", "jquery": ("9.9.9", HASH)}) + _, html = render_vendor_macros(app) + tags = { + TAG_URL.findall(tag)[0]: tag for tag in TAG.findall(html) if TAG_URL.search(tag) + } + + vega_tag = tags["https://cdn.jsdelivr.net/npm/vega@9.9.9/build/vega.min.js"] + assert "integrity=" not in vega_tag + jquery_tag = tags["https://cdn.jsdelivr.net/npm/jquery@9.9.9/dist/jquery.min.js"] + digest = base64.b64encode(bytes.fromhex(HASH)).decode() + assert f'integrity="sha256-{digest}"' in jquery_tag + + +def test_libraries_load_from_cdns_with_integrity_when_not_fetched(app, not_fetched): + """Without fetched files, each tag loads its file from the library's CDN, and lets the browser check it against the recorded hash.""" + macros, html = render_vendor_macros(app) + tags = [ + tag for tag in TAG.findall(html) if TAG_URL.search(tag) + ] # leave out inline scripts + + assert len(macros) == 10 + assert tags + for tag in tags: + (url,) = TAG_URL.findall(tag) + key, path = manifest_entry(url) + digest = bytes.fromhex(VENDOR_LIBRARIES[key].files[path]) + assert INTEGRITY.findall(tag) == [ + f"sha256-{base64.b64encode(digest).decode()}" + ], tag + assert 'crossorigin="anonymous"' in tag, tag + # Fonts and images in the inline styles come from CDNs too. + for url in CSS_URL.findall(html): + manifest_entry(url) + + +def test_libraries_load_from_flexmeasures_when_fetched(app, client, fetched): + """With fetched files, each tag and inline style loads its file from FlexMeasures, which serves it.""" + _, html = render_vendor_macros(app) + urls = TAG_URL.findall(html) + CSS_URL.findall(html) + + assert "integrity=" not in html + assert urls + for url in sorted(set(urls)): + assert url.startswith("/ui/static/vendor/"), url + response = client.get(url) + assert response.status_code == 200, url + response.close() + + +def test_login_page_loads_libraries(client, not_fetched): + """Pages not rendered through render_flexmeasures_template, like the login page, also load the libraries.""" + page = client.get(url_for("security.login")) + assert page.status_code == 200 + assert b"https://cdn.jsdelivr.net/npm/jquery@" in page.data + assert b"https://cdn.jsdelivr.net/npm/bootstrap@" in page.data + + +@pytest.fixture +def fake_library(monkeypatch): + """A library with two files and a licence, whose downloads are served from memory.""" + contents = { + "https://cdn.example/lib@1.0/dist/lib.js": b"lib", + "https://cdn.example/lib@1.0/dist/lib.css": b"style", + "https://cdn.example/lib@1.0/LICENSE": b"licence", + } + library = VendorLibrary( + name="lib", + version="1.0", + base_url="https://cdn.example/{name}@{version}/", + files={ + "dist/lib.js": hashlib.sha256(b"lib").hexdigest(), + "dist/lib.css": hashlib.sha256(b"style").hexdigest(), + }, + license="LICENSE", + license_sha256=hashlib.sha256(b"licence").hexdigest(), + ) + monkeypatch.setattr(vendor_assets, "VENDOR_LIBRARIES", {"lib": library}) + downloads = [] + + def download(url: str) -> bytes: + downloads.append(url) + if url not in contents: + raise urllib.error.HTTPError(url, 404, "Not Found", {}, None) + return contents[url] + + return contents, downloads, download + + +def test_fetch_stores_files_and_licence(not_fetched, fake_library): + """Fetching stores a library's files and licence, and skips files that are already there.""" + contents, downloads, download = fake_library + libraries = vendor_assets.VENDOR_LIBRARIES + + assert vendor_assets.fetch(download=download, libraries=libraries) == 3 + assert ( + vendor_assets.local_file("lib", "dist/lib.js", libraries).read_bytes() == b"lib" + ) + assert ( + vendor_assets.local_file("lib", "dist/lib.css", libraries).read_bytes() + == b"style" + ) + assert ( + vendor_assets.local_file("lib", "LICENSE", libraries).read_bytes() == b"licence" + ) + assert vendor_assets.missing_files(libraries) == [] + + downloads.clear() + assert vendor_assets.fetch(download=download, libraries=libraries) == 0 + assert downloads == [] + + +def test_fetch_rejects_a_file_that_does_not_match_its_hash(not_fetched, fake_library): + """A file whose content differs from what was recorded is not stored, and fetching fails.""" + contents, _, download = fake_library + contents["https://cdn.example/lib@1.0/dist/lib.css"] = b"tampered" + libraries = vendor_assets.VENDOR_LIBRARIES + + with pytest.raises(ValueError, match="does not match the hash"): + vendor_assets.fetch(download=download, libraries=libraries) + assert not vendor_assets.local_file("lib", "dist/lib.css", libraries).exists() + assert not list(vendor_assets.VENDOR_DIR.rglob("*.part")) + + +def test_fetch_stores_an_overridden_version_with_hashes( + not_fetched, fake_library, caplog +): + """A version set with hashes is fetched and checked, while its licence, which is not executed, is fetched unchecked, and may be missing.""" + contents, _, download = fake_library + contents["https://cdn.example/lib@2.0/dist/lib.js"] = b"lib 2" + contents["https://cdn.example/lib@2.0/dist/lib.css"] = b"style 2" + libraries = libraries_with_overrides( + { + "lib": ( + "2.0", + { + "dist/lib.js": hashlib.sha256(b"lib 2").hexdigest(), + "dist/lib.css": hashlib.sha256(b"style 2").hexdigest(), + }, + ) + } + ) + + with caplog.at_level(logging.WARNING): + assert vendor_assets.fetch(download=download, libraries=libraries) == 2 + assert ( + vendor_assets.local_file("lib", "dist/lib.js", libraries).read_bytes() + == b"lib 2" + ) + assert "Could not fetch the licence of lib 2.0" in caplog.text + + contents["https://cdn.example/lib@2.0/dist/lib.css"] = b"tampered" + vendor_assets.local_file("lib", "dist/lib.css", libraries).unlink() + with pytest.raises(ValueError, match="does not match the hash"): + vendor_assets.fetch(download=download, libraries=libraries) + + +def test_fetch_skips_an_overridden_version_without_hashes(not_fetched, fake_library): + """A version set without hashes is never stored, so FlexMeasures does not serve unchecked files.""" + _, downloads, download = fake_library + libraries = libraries_with_overrides({"lib": "2.0"}) + + assert vendor_assets.fetch(download=download, libraries=libraries) == 0 + assert downloads == [] + assert vendor_assets.missing_files(libraries) == [] + assert vendor_assets.unhashed_libraries(libraries) == ["lib"] + + +@pytest.mark.parametrize("version", [None, "2.0"]) +def test_print_hashes(fake_library, capsys, version): + """Hashes are printed for VENDOR_LIBRARIES at the recorded version, and as a setting entry at another version.""" + contents, _, download = fake_library + contents["https://cdn.example/lib@2.0/dist/lib.js"] = b"lib 2" + contents["https://cdn.example/lib@2.0/dist/lib.css"] = b"style 2" + + vendor_assets.print_hashes("lib", download=download, version=version) + printed = capsys.readouterr().out + if version is None: + assert f'"dist/lib.js": "{hashlib.sha256(b"lib").hexdigest()}",' in printed + assert printed.startswith("files={") + else: + assert printed.startswith('"lib": ("2.0", {') + assert f'"dist/lib.js": "{hashlib.sha256(b"lib 2").hexdigest()}",' in printed + # What is printed is a valid entry of the setting. + setting = ast.literal_eval("{" + printed + "}") + assert libraries_with_overrides(setting)["lib"].files == { + "dist/lib.js": hashlib.sha256(b"lib 2").hexdigest(), + "dist/lib.css": hashlib.sha256(b"style 2").hexdigest(), + } + + +def test_fetch_command_reads_the_setting_from_the_config_file(tmp_path, monkeypatch): + """The fetch command fetches the versions which the host set in the config file.""" + config_file = tmp_path / "flexmeasures.cfg" + config_file.write_text( + 'FLEXMEASURES_JS_VERSIONS = {"vega": ("9.9.9", "' + HASH + '")}\n' + ) + fetched_with = {} + + def fake_fetch(keys, libraries): + fetched_with.update(libraries) + return 0 + + monkeypatch.setattr(vendor_assets, "fetch", fake_fetch) + assert fetch_vendor_assets.main(["--config", str(config_file)]) == 0 + assert fetched_with["vega"].version == "9.9.9" + assert fetched_with["vega"].files == {"build/vega.min.js": HASH} + + with pytest.raises(SystemExit): + fetch_vendor_assets.main(["--config", str(tmp_path / "missing.cfg")]) + + +def test_startup_log_warns_about_versions_set_without_hashes( + app, caplog, not_fetched, js_versions_setting +): + """Hosts hear which libraries browsers load unchecked, and how to get their hashes.""" + js_versions_setting({"vega": "9.9.9"}) + with caplog.at_level(logging.WARNING): + log_where_vendor_files_load_from(app) + assert "--print-hashes vega --version 9.9.9" in caplog.text + + +@pytest.mark.parametrize("fetch_first", [False, True]) +def test_startup_log_says_where_libraries_load_from(app, caplog, request, fetch_first): + """Hosts hear that browsers load the libraries from CDNs, and how to change that, unless they fetched them.""" + request.getfixturevalue("fetched" if fetch_first else "not_fetched") + with caplog.at_level(logging.INFO): + log_where_vendor_files_load_from(app) + + logged = "python -m flexmeasures.ui.utils.fetch_vendor_assets" in caplog.text + assert logged is not fetch_first + + +@pytest.mark.parametrize( + "css_file", + sorted((STATIC_DIR / "css").rglob("*.css")), + ids=lambda path: str(path.relative_to(STATIC_DIR)), +) +def test_css_references_resolve(css_file: Path): + """Fonts and images that our own stylesheets refer to exist next to them.""" + for ref in CSS_URL.findall(css_file.read_text()): + if ref.startswith(("data:", "#")): + continue + assert not re.match(r"https?://", ref), ref + target = ref.split("?")[0].split("#")[0] + assert (css_file.parent / target).resolve().is_file(), ref + + +EXTERNAL_ASSET = re.compile( + r"""<(?:script|link|img)\b[^>]*?\s(?:src|href)=["']https?://""" + r"""|url\(\s*["']?https?://""" +) + + +def test_no_external_asset_references(): + """No template or stylesheet names another host to fetch scripts, styles, fonts or images from, other than through VENDOR_LIBRARIES; plain links are fine.""" + offenders = [] + for path in [*TEMPLATE_DIR.rglob("*.html"), *(STATIC_DIR / "css").rglob("*.css")]: + text = path.read_text() + for match in EXTERNAL_ASSET.finditer(text): + line = text.count("\n", 0, match.start()) + 1 + offenders.append(f"{path.relative_to(UI_DIR)}:{line}") + assert offenders == [] + + +def test_asset_type_icons_are_served(client): + """The asset tree and asset lists show icons which FlexMeasures serves itself, rather than fetching them from iconify.""" + from flexmeasures.ui.utils.view_utils import FALLBACK_SVG_ICON, SVG_ICON_MAPPING + + for icon in sorted({*SVG_ICON_MAPPING.values(), FALLBACK_SVG_ICON}): + assert icon.startswith("/ui/static/icons/"), icon + response = client.get(icon) + assert response.status_code == 200, icon + assert response.mimetype == "image/svg+xml", icon + response.close() + + +def test_fetched_files_are_cached_long(client, fetched): + """Browsers keep fetched library files for a year, so a page view on a poor connection does not re-check about 30 files.""" + response = client.get( + f"/ui/static/vendor/jquery/{JS_VERSIONS['jquery']}/dist/jquery.min.js" + ) + assert response.status_code == 200 + assert response.cache_control.max_age == 31536000 + assert response.cache_control.immutable + response.close() + + +def test_fetched_files_of_an_overridden_version_are_cached_long( + client, fetched, js_versions_setting +): + """A version set through FLEXMEASURES_JS_VERSIONS gets its own path, so its files are cached long too.""" + libraries = js_versions_setting({"jquery": ("9.9.9", HASH)}) + file = vendor_assets.local_file("jquery", "dist/jquery.min.js", libraries) + file.parent.mkdir(parents=True) + file.write_text("jquery 9.9.9") + + response = client.get("/ui/static/vendor/jquery/9.9.9/dist/jquery.min.js") + assert response.status_code == 200 + assert response.cache_control.max_age == 31536000 + response.close() + + +def test_unversioned_files_keep_default_caching(client): + """Files without a version in their path keep Flask's default, so a changed file reaches browsers.""" + response = client.get("/ui/static/icons/mdi/battery.svg") + assert response.status_code == 200 + assert response.cache_control.max_age is None + response.close() diff --git a/flexmeasures/ui/utils/fetch_vendor_assets.py b/flexmeasures/ui/utils/fetch_vendor_assets.py new file mode 100644 index 0000000000..606a15b3eb --- /dev/null +++ b/flexmeasures/ui/utils/fetch_vendor_assets.py @@ -0,0 +1,80 @@ +"""Fetch the UI's third-party libraries, so FlexMeasures serves them itself rather than browsers loading them from CDNs. + +Run it as ``python -m flexmeasures.ui.utils.fetch_vendor_assets``; see flexmeasures/ui/utils/vendor_assets.py. +It fetches the versions that the FLEXMEASURES_JS_VERSIONS setting sets, read from the config file that FlexMeasures reads. +It lives apart from vendor_assets.py, which flexmeasures.ui imports, so running it does not import itself twice. +""" + +from __future__ import annotations + +import argparse +import os +import sys + +from flask import Config + +from flexmeasures.ui.utils import vendor_assets + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="python -m flexmeasures.ui.utils.fetch_vendor_assets", + description="Fetch the UI's third-party libraries, so FlexMeasures serves them itself rather than browsers loading them from CDNs.", + ) + parser.add_argument( + "keys", + nargs="*", + metavar="key", + help=f"Libraries to fetch (default: all), from: {', '.join(vendor_assets.VENDOR_LIBRARIES)}.", + ) + parser.add_argument( + "--print-hashes", + action="store_true", + help="Print the hashes of the given libraries' files at their recorded versions, instead of fetching them.", + ) + parser.add_argument( + "--version", + help="With --print-hashes and one library, print the hashes of this version instead, as an entry of the FLEXMEASURES_JS_VERSIONS setting.", + ) + parser.add_argument( + "--config", + help="The config file to read FLEXMEASURES_JS_VERSIONS from (default: the one FlexMeasures reads, ~/.flexmeasures.cfg or the instance folder's flexmeasures.cfg).", + ) + args = parser.parse_args(argv) + unknown = [key for key in args.keys if key not in vendor_assets.VENDOR_LIBRARIES] + if unknown: + parser.error(f"unknown libraries: {', '.join(unknown)}") + if args.version is not None and not (args.print_hashes and len(args.keys) == 1): + parser.error("--version needs --print-hashes and one library") + if args.config is not None and not os.path.isfile(args.config): + parser.error(f"cannot find config file {args.config}") + if args.print_hashes: + for key in args.keys: + print(f"{key}:") + vendor_assets.print_hashes(key, version=args.version) + return 0 + setting = read_js_versions_setting(args.config) + libraries = vendor_assets.libraries_with_overrides(setting) + written = vendor_assets.fetch(args.keys, libraries=libraries) + print(f"Fetched {written} file(s) into {vendor_assets.VENDOR_DIR}.") + for key in vendor_assets.unhashed_libraries(libraries): + print( + f"Skipped {key} {libraries[key].version}, as FLEXMEASURES_JS_VERSIONS sets it without hashes." + ) + return 0 + + +def read_js_versions_setting(path: str | None = None) -> dict | None: + """The FLEXMEASURES_JS_VERSIONS setting from a config file, by default the one that FlexMeasures reads.""" + from flexmeasures.utils.config_utils import find_flexmeasures_cfg + + path = path or find_flexmeasures_cfg() + if path is None or not os.path.isfile(path): + return None + config = Config(os.path.dirname(os.path.abspath(path))) + config.from_pyfile(os.path.abspath(path)) + return config.get("FLEXMEASURES_JS_VERSIONS") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/flexmeasures/ui/utils/vendor_assets.py b/flexmeasures/ui/utils/vendor_assets.py new file mode 100644 index 0000000000..096ab0d4f7 --- /dev/null +++ b/flexmeasures/ui/utils/vendor_assets.py @@ -0,0 +1,551 @@ +"""Third-party libraries which the UI loads, and where it loads them from. + +Each library is served by FlexMeasures from static/vendor/// when its files were fetched there, +and loaded from its CDN otherwise, with the hash recorded below as the tag's integrity attribute. +The Docker image comes with the files fetched. +To serve them from a pip-installed FlexMeasures, for example without internet access, fetch them once: + + python -m flexmeasures.ui.utils.fetch_vendor_assets + +Fetching also stores each library's licence next to its files, and fails on any file whose hash does not match. +Templates load these libraries through the macros in templates/includes/vendor.html. + +To bump a library, change its version below, then print the hashes of its files at that version, and paste them in: + + python -m flexmeasures.ui.utils.fetch_vendor_assets --print-hashes + +Hosts can load other versions through the FLEXMEASURES_JS_VERSIONS setting, see libraries_with_overrides. + +This is a leaf module, so flexmeasures/ui/__init__.py can import it without an import cycle. +""" + +from __future__ import annotations + +import base64 +import hashlib +import logging +import re +import urllib.error +import urllib.request +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Callable + +from flask import current_app, has_app_context, url_for +from markupsafe import Markup, escape + +VENDOR_DIR = Path(__file__).resolve().parents[1] / "static" / "vendor" +NPM_CDN = "https://cdn.jsdelivr.net/npm/{name}@{version}/" + + +@dataclass(frozen=True) +class VendorLibrary: + """A library's version, its files with their SHA-256 hashes, and where to download them.""" + + # The npm package name, or the library's own name if it does not come from npm. + name: str + version: str + # Path relative to the base URL -> SHA-256 hash, or None for a file that hosts load without a hash, see libraries_with_overrides. + files: dict[str, str | None] + license: str # path relative to the base URL, or a full URL + license_sha256: str | None + base_url: str = NPM_CDN # formatted with the name and version + + def url(self, path: str) -> str: + return self.base_url.format(name=self.name, version=self.version) + path + + @property + def license_url(self) -> str: + return self.license if "://" in self.license else self.url(self.license) + + @property + def license_path(self) -> str: + """Where the licence is stored, relative to the library's local directory.""" + return ( + self.license.rsplit("/", 1)[-1] if "://" in self.license else self.license + ) + + +VENDOR_LIBRARIES: dict[str, VendorLibrary] = { + "vega": VendorLibrary( + name="vega", + version="5.22.1", + files={ + "build/vega.min.js": "731f01b68116bc185a196322096ca342737424c4995c48a6674f57ebc967e9c1", + }, + license="LICENSE", + license_sha256="b75f7ed0af20dedadf92c52bc236161bcf0d294ff2e6e34ca76403203349f71d", + ), + "vegaembed": VendorLibrary( + name="vega-embed", + version="6.21.0", + files={ + "build/vega-embed.min.js": "fd72eab487018408e196630e6b3db75005f30604329b3eb40fbdfe3dfacadb8c", + }, + license="LICENSE", + license_sha256="32df67148f0fc3db0eb9e263a7b75d07f1eb14c61955005a4a39c6918d10d137", + ), + "vegalite": VendorLibrary( + name="vega-lite", + version="5.5.0", # "5.6.0" has a problematic bar chart: see our sensor page and https://github.com/vega/vega-lite/issues/8496 + files={ + "build/vega-lite.min.js": "4804d726d5eb7f7d0335ce234481c6a51ceb3ef2f0031d544a80f381d1f4163d", + }, + license="LICENSE", + license_sha256="f618900fd0d64046963b29f40590cdd1e341a2f41449f99110d82fd81fea808c", + ), + "echarts": VendorLibrary( + name="echarts", + version="5.6.0", # used for the fast (canvas-based) chart mode + files={ + "dist/echarts.min.js": "bf4a223524e40b77c304bec67e1222cf551f14880cf42c69dc046558e11c07b1", + }, + license="LICENSE", + license_sha256="634293835b43a6dd2094fa39182a3d9a6b9ca43b7fdb9ac354e8037af2a3093a", + ), + "currencysymbolmap": VendorLibrary( + name="currency-symbol-map", + version="5.1.0", + files={ + "map.js": "22c8c8381435b89abbc8786bb7d8c41ea06236113117c61bdb38c6c3a51b4e50", + }, + license="LICENSE.md", + license_sha256="db6bef538dd26785fb7b22de741b04a85121ccaa19568af310b97dcd3629a331", + ), + "jsoneditor": VendorLibrary( + name="@json-editor/json-editor", + version="2.15.2", + files={ + "dist/jsoneditor.min.js": "e1339ef686f3e750ee520f534a24512df9f0891e5848b271ae03cb9d8629428f", + }, + license="LICENSE", + license_sha256="2c3d6f6bad4123d4f2abf92666e17adc3fc30a956e3b03c78b108c2a29630252", + ), + "leaflet": VendorLibrary( + name="leaflet", + version="1.9.4", + files={ + "dist/leaflet.css": "a7837102824184820dfa198d1ebcd109ff6d0ff9a2672a074b9a1b4d147d04c6", + "dist/leaflet-src.min.js": "3c3ccb000ef205c785428a4385281c8ca0aadfc00df0a686d3ddd4332131f2ce", + "dist/images/layers-2x.png": "066daca850d8ffbef007af00b06eac0015728dee279c51f3cb6c716df7c42edf", + "dist/images/layers.png": "1dbbe9d028e292f36fcba8f8b3a28d5e8932754fc2215b9ac69e4cdecf5107c6", + "dist/images/marker-icon-2x.png": "00179c4c1ee830d3a108412ae0d294f55776cfeb085c60129a39aa6fc4ae2528", + "dist/images/marker-icon.png": "574c3a5cca85f4114085b6841596d62f00d7c892c7b03f28cbfa301deb1dc437", + "dist/images/marker-shadow.png": "264f5c640339f042dd729062cfc04c17f8ea0f29882b538e3848ed8f10edb4da", + }, + license="LICENSE", + license_sha256="53e8dc25862014e4324741ca18fbe3611e11d42ef69f59f86ea8c5389647d4cb", + ), + "leafletmarkercluster": VendorLibrary( + name="leaflet.markercluster", + version="1.5.3", + files={ + "dist/MarkerCluster.css": "614dea0a98ff3f4ead74f04918f6b1d1b9ba435c25b5fc23b21a394d1e3e4d87", + "dist/MarkerCluster.Default.css": "61258232d98d64dc2a7b1e02130d67421bc5b9bda5994eef70228ff97570c170", + "dist/leaflet.markercluster.js": "1e4e1d22972a3926f48598e0caf14e3fe7049835d428a344fed4f9e3665b3508", + }, + license="MIT-LICENCE.txt", + license_sha256="3133dacd38250bd1f524d8166cc60a016a382522e062fce143691c65bf5e4b3d", + ), + "leafletmarkerclusterlayersupport": VendorLibrary( + name="leaflet.markercluster.layersupport", + version="2.0.1", + files={ + "dist/leaflet.markercluster.layersupport.js": "7a61dfa8cb8829a81800a309ab8d4ebbe81fb309db22df670286dce66acd6bd8", + }, + license="LICENSE", + license_sha256="ca934ce716095f28879e54a71b5dddd742bdf7cb72ca0d7cc9d1a6892d66aca6", + ), + "jquery": VendorLibrary( + name="jquery", + version="3.5.1", + files={ + "dist/jquery.min.js": "f7f6a5894f1d19ddad6fa392b2ece2c5e578cbf7da4ea805b6885eb6985b6e3d", + }, + license="LICENSE.txt", + license_sha256="1ff603118efdb313799567cb4a408594954384f40843ca6f29c596f5b1f8907d", + ), + "floatthead": VendorLibrary( + name="floatthead", + version="2.2.1", + files={ + "dist/jquery.floatThead.min.js": "020bcbcff534803c840937a0eab0f671cd3ecb4240ed49656aaf76ae99bab730", + }, + license="LICENSE", + license_sha256="99c0cb82d16f11c4e3d92b856664ccc820f4d369a962c00ab16856926fe9accf", + ), + "bootstrap": VendorLibrary( + name="bootstrap", + version="5.3.3", + files={ + "dist/css/bootstrap.min.css": "3c8f27e6009ccfd710a905e6dcf12d0ee3c6f2ac7da05b0572d3e0d12e736fc8", + "dist/js/bootstrap.bundle.min.js": "0833b2e9c3a26c258476c46266e6877fc75218625162e0460be9a3a098a61c6c", + }, + license="LICENSE", + license_sha256="8c14611ae41ac6fd543c13349f22188eb12c69b3e59105c5eca3925a8e4eca3e", + ), + "bootstrapicons": VendorLibrary( + name="bootstrap-icons", + version="1.10.5", + files={ + "font/bootstrap-icons.css": "d8824f7067cdfea38afec7e9ffaf072125266824206d69ef1f112d72153a505e", + "font/fonts/bootstrap-icons.woff": "999550fafbfbc0e0474b3b5c945df287b079d2a23dc07dcc6240a7d2f18e081d", + "font/fonts/bootstrap-icons.woff2": "cfe45b981d1b91b173361a34cfce5f60893dbd1ac4af2c3ac11fc17552c5401f", + }, + license="LICENSE", + license_sha256="0623348b57e817b6897c41b6065380b599d5befe2839d83270566c605e430441", + ), + "fontawesome": VendorLibrary( + name="font-awesome", + version="4.7.0", + files={ + "css/font-awesome.min.css": "799aeb25cc0373fdee0e1b1db7ad6c2f6a0e058dfadaa3379689f583213190bd", + "fonts/FontAwesome.otf": "444dd4366615ffc4a16d012b2fa90137065d3ccb410fa6fd5e4ddd7b5e4ffcd5", + "fonts/fontawesome-webfont.eot": "7bfcab6db99d5cfbf1705ca0536ddc78585432cc5fa41bbd7ad0f009033b2979", + "fonts/fontawesome-webfont.svg": "ad6157926c1622ba4e1d03d478f1541368524bfc46f51e42fe0d945f7ef323e4", + "fonts/fontawesome-webfont.ttf": "aa58f33f239a0fb02f5c7a6c45c043d7a9ac9a093335806694ecd6d4edc0d6a8", + "fonts/fontawesome-webfont.woff": "ba0c59deb5450f5cb41b3f93609ee2d0d995415877ddfa223e8a8a7533474f07", + "fonts/fontawesome-webfont.woff2": "2adefcbc041e7d18fcf2d417879dc5a09997aa64d675b7a3c4b6ce33da13f3fe", + }, + license="README.md", + license_sha256="610427fe917f2b51ef69b5bfa9a7bfe3fcba59b1ccaf86a500a47e5fa1da322b", + ), + "ionrangeslider": VendorLibrary( + name="ion-rangeslider", + version="2.3.1", + files={ + "css/ion.rangeSlider.css": "683c3601c8c673010044e3fbad99c8da5741fbf5f2b93c1b647e43af5222b8ca", + "js/ion.rangeSlider.min.js": "67adfdac93b9ec1899cd00e55ac1b217e109dc5b379c3e2940f91f8a64f2dd2f", + }, + license="License.md", + license_sha256="eab00c5f6ec0a56b255565aa3f2e624e53e220bc916f1fab4cbdac14e051569e", + ), + "datatables": VendorLibrary( + name="DataTables", + version="1.10.22", + base_url="https://cdn.datatables.net/{version}/", + files={ + "js/jquery.dataTables.min.js": "c91c904fbfbe1fcb61c4e4cb955b35e8eb303f29d97a9f757c744fc6603a978a", + "css/jquery.dataTables.min.css": "ca31d3aa2364f2a372c1d9ed477be2b71852e7d3a98bb92476c19efe67024a48", + "images/sort_asc.png": "595704c3f3cf4cb65c7d9c8508a99e7480e150095473faed31a07c21b13389b8", + "images/sort_asc_disabled.png": "a65b8f4f84d6427a81c360282fc5394d51bf99dada5f159e6aa0fce3c396825c", + "images/sort_both.png": "3e016c23ae51417382b640ae2d19eb48047532c37ad53894bd185586559ccffb", + "images/sort_desc.png": "d08ed0e21f187dd309030d465224da8085119a15a17d616ba0e477bb50c6f10d", + "images/sort_desc_disabled.png": "6c0f0c1b21ef6807057afc8ddc1a925d1dbd21cb11e9270ec84ff4ac40d9a3fa", + }, + license="https://cdn.jsdelivr.net/npm/datatables.net@1.10.22/License.txt", + license_sha256="4f6610cece9940c3bf3de202a920e7b69491e7947d968330f03db592caa456dd", + ), + "datatablesconditionalpaging": VendorLibrary( + name="DataTables conditionalPaging plug-in", + version="1.10.22", + base_url="https://cdn.datatables.net/plug-ins/{version}/features/conditionalPaging/", + files={ + "dataTables.conditionalPaging.js": "336e6dd30fe2ba132c2980f057c359453c5e85ca046842717457505acdabc39c", + }, + license="https://cdn.jsdelivr.net/npm/datatables.net-plugins@1.10.22/License.txt", + license_sha256="46daf76193ec5496a09285dfc4c58127f68d895c4de20ded97aff6d58b452a35", + ), + "d3": VendorLibrary( + name="d3", + version="6.7.0", + files={ + "dist/d3.min.js": "fbb8da602a76f4ed49bac3561da62d8149fa121b8fd15685bacc21355d3a3322", + }, + license="LICENSE", + license_sha256="0747bebeb06b10954913149be9b9a8bdf6fad3e6fdcbd9f9524e7a94c13d2cea", + ), + "litepicker": VendorLibrary( + name="litepicker", + version="2.0.12", + files={ + "dist/litepicker.js": "7fa62995b2417110edd2dfc18d43256816f705f9107b290d758fb9c8705e3e27", + "dist/plugins/ranges.js": "ec92df7d80691bef3d935208ddf0e0d1b4d9efc58c1b1132358c3ee47b4de48c", + "dist/plugins/keyboardnav.js": "2f1f56813045d77aa790718fb31ce1d0da7c6e01f674ceb2d36d7137181a56f1", + }, + license="LICENSE.md", + license_sha256="a14d94572e372b6ec3bd884f394b43ea2e80ab068bd37a5a7cb243662b543922", + ), + "jstimezonedetect": VendorLibrary( + name="jstimezonedetect", + version="1.0.7", + files={ + "dist/jstz.js": "4323ac38f342b0b7b1eee332e7278c67f4d01fe21c35ab03a684d49679a4feee", + }, + license="LICENCE.txt", + license_sha256="70e94d1525a9cfeaec737aef0d21391b8585420add847742647cf1d916957555", + ), + "poppins": VendorLibrary( + name="Poppins", + version="v20", + base_url="https://fonts.gstatic.com/s/poppins/{version}/", + files={ + "pxiEyp8kv8JHgFVrJJbecmNE.woff2": "478b3ab728aaa25a76c8cd8bb04addde92c43d72e643d0fdc455340b44c3f074", + "pxiEyp8kv8JHgFVrJJnecmNE.woff2": "cb8bdeabc838774d9808eb7c4cfcea963b57855e34f84b54797076940c8e5986", + "pxiEyp8kv8JHgFVrJJfecg.woff2": "7d93459d86585bfcdbb7e0376056226adb25821ee54b96236fe2123e9560929f", + }, + license="https://cdn.jsdelivr.net/gh/google/fonts@7dc16b7de42db624b902f2292b68ed9e489e5053/ofl/poppins/OFL.txt", + license_sha256="6be04893d770899a015649c7aa3b582f871b272f8747a92b78b17c3e5c8b2573", + ), +} + +# Each library's version, by key, available in templates as `js_versions`. +# Plugin templates may read the keys which the deprecated FLEXMEASURES_JS_VERSIONS setting had, such as `js_versions.vega`. +JS_VERSIONS: dict[str, str] = { + key: library.version for key, library in VENDOR_LIBRARIES.items() +} + +# Where the app keeps its libraries, with the versions that FLEXMEASURES_JS_VERSIONS sets. +APP_EXTENSION = "flexmeasures_vendor_libraries" + +SHA256 = re.compile(r"[0-9a-f]{64}") + + +def _logger() -> logging.Logger: + """The app's logger, or this module's outside an app, such as in the fetch command. + + A module-level logger in an app would be disabled when FlexMeasures configures logging after importing this module. + """ + return current_app.logger if has_app_context() else logging.getLogger(__name__) + + +def libraries_with_overrides(setting: dict | None) -> dict[str, VendorLibrary]: + """The libraries to load, with the versions that the FLEXMEASURES_JS_VERSIONS setting sets in place of the recorded ones. + + The setting maps a library's key to its version, in one of three forms: + + - ``"5.33.1"``: browsers load the library from its CDN without checking it, and it is not fetched. + - ``("5.33.1", "")``: the hash of the library's one file. + - ``("1.9.5", {"dist/leaflet.css": "", ...})``: the hash of each of the library's files. + + With hashes, the library is handled like the recorded ones: browsers check its files, and it can be fetched. + The new version is expected to have the same files as the recorded one, as the templates name them. + ``python -m flexmeasures.ui.utils.fetch_vendor_assets --print-hashes --version `` prints the setting for a version. + """ + libraries = dict(VENDOR_LIBRARIES) + for key, value in (setting or {}).items(): + if key not in libraries: + _logger().warning( + f"FLEXMEASURES_JS_VERSIONS sets {key}, which is not one of the UI's libraries, so it is ignored. " + f"The libraries are: {', '.join(VENDOR_LIBRARIES)}." + ) + continue + library = libraries[key] + version, hashes = _unpack_override(key, value) + if hashes is None: + if version == library.version: + # For example, a copy of the setting's old defaults. + continue + files = dict.fromkeys(library.files) + elif isinstance(hashes, str): + if len(library.files) != 1: + raise ValueError( + f"FLEXMEASURES_JS_VERSIONS sets one hash for {key}, which has {len(library.files)} files. " + f"Set a dict with the hash of each of them instead: {', '.join(library.files)}." + ) + files = {path: hashes for path in library.files} + else: + if set(hashes) != set(library.files): + raise ValueError( + f"FLEXMEASURES_JS_VERSIONS sets hashes for {key}'s files {', '.join(sorted(hashes))}, " + f"but needs them for exactly these files: {', '.join(library.files)}." + ) + files = {path: hashes[path] for path in library.files} + for path, sha256 in files.items(): + if sha256 is not None and not SHA256.fullmatch(sha256): + raise ValueError( + f"FLEXMEASURES_JS_VERSIONS sets {sha256!r} as the hash of {key}'s {path}, which is not a SHA-256 hash in hexadecimal." + ) + libraries[key] = replace( + library, + version=version, + files=files, + # A licence is not executed, so a licence of another version is fetched without a hash. + license_sha256=( + library.license_sha256 if version == library.version else None + ), + ) + return libraries + + +def _unpack_override(key: str, value) -> tuple[str, str | dict | None]: + if isinstance(value, str): + return value, None + if ( + isinstance(value, (tuple, list)) + and len(value) == 2 + and isinstance(value[0], str) + and isinstance(value[1], (str, dict)) + ): + return value[0], value[1] + raise ValueError( + f"FLEXMEASURES_JS_VERSIONS sets {key} to {value!r}. " + "Set a version, a version and the hash of the library's file, or a version and a dict with the hash of each of its files." + ) + + +def active_libraries() -> dict[str, VendorLibrary]: + """The libraries which the current app loads, or the recorded ones outside an app.""" + if has_app_context(): + return current_app.extensions.get(APP_EXTENSION, VENDOR_LIBRARIES) + return VENDOR_LIBRARIES + + +def local_file( + key: str, path: str, libraries: dict[str, VendorLibrary] | None = None +) -> Path: + """Where a library's file is stored when it was fetched.""" + library = (active_libraries() if libraries is None else libraries)[key] + return VENDOR_DIR / key / library.version / path + + +def vendor_url(key: str, path: str) -> str: + """URL of a library's file: served by FlexMeasures if it was fetched, or else from the library's CDN.""" + library = active_libraries()[key] + if local_file(key, path).is_file(): + return url_for( + "flexmeasures_ui.static", + filename=f"vendor/{key}/{library.version}/{path}", + ) + return library.url(path) + + +def vendor_attrs(key: str, path: str, attribute: str = "src") -> Markup: + """Attributes for a