diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 4d6f33c605..cc112bbcf1 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -40,7 +40,7 @@ New features Infrastructure / Support ------------------------- -* The UI can work without internet access: the Docker image serves the UI's JavaScript and CSS libraries itself, and a pip-installed FlexMeasures can fetch them once (see :ref:`install-ui-libraries`), while otherwise browsers keep loading them from CDNs; the ``FLEXMEASURES_JS_VERSIONS`` setting now only lists the versions a host wants to differ, optionally with the hashes of their files, so plugins reading it should read the ``js_versions`` template variable instead [see `PR #2711 `_] +* The UI can work without internet access: the Docker image serves the UI's JavaScript and CSS libraries itself, and a pip-installed FlexMeasures can fetch them once (see :ref:`install-ui-libraries`), while otherwise browsers keep loading them from CDNs; the ``FLEXMEASURES_JS_VERSIONS`` setting now only lists the versions a host wants to differ, optionally with the hashes of their files, so plugins reading it should read the ``js_versions`` template variable instead [see `PR #2711 `_ and `PR #2747 `_] * Upgrade Vega, which draws the UI's charts, from 5.22.1 to 5.33.1, its latest 5.x release [see `PR #2743 `_] * ``add_beliefs_to_db_and_enqueue_forecasting_jobs`` now raises ``BeliefReplacementNotAllowed`` rather than an ``IntegrityError`` when saving would change existing values, so plugins catching ``IntegrityError`` around it should catch the new exception instead [see `PR #2722 `_] * ``[GET] /sensors/(id)`` now answers a sensor which does not exist with a 404, as the asset endpoints do, rather than with a 422, so that a client can tell a sensor which is gone apart from a request it got wrong [see `PR #2650 `_] diff --git a/documentation/host/installation.rst b/documentation/host/installation.rst index 569b4ca016..4ec2a8882b 100644 --- a/documentation/host/installation.rst +++ b/documentation/host/installation.rst @@ -472,7 +472,7 @@ You can also fetch them yourself, for example on a server without internet acces $ python -m flexmeasures.ui.utils.fetch_vendor_assets -Both download each library's files and licence into FlexMeasures' installation, and fail on any file that does not match its expected hash. +Both download each library's files and licence, with any notice file the licence requires, into FlexMeasures' installation, and fail on any file that does not match its expected hash. They fetch the versions set by :ref:`FLEXMEASURES_JS_VERSIONS ` too, except for versions set without hashes. The command reads that setting from the config file FlexMeasures reads, or from the one passed with ``--config``. To stop FlexMeasures from fetching the libraries itself, set :ref:`FLEXMEASURES_FETCH_UI_LIBRARIES ` to ``False``. diff --git a/flexmeasures/ui/tests/test_vendor_assets.py b/flexmeasures/ui/tests/test_vendor_assets.py index a1702d5f77..0d196790af 100644 --- a/flexmeasures/ui/tests/test_vendor_assets.py +++ b/flexmeasures/ui/tests/test_vendor_assets.py @@ -69,7 +69,7 @@ def fetched(tmp_path, monkeypatch): monkeypatch.setattr(vendor_assets, "VENDOR_DIR", static_dir / "vendor") monkeypatch.setattr(flexmeasures_ui, "static_folder", str(static_dir)) for key, library in VENDOR_LIBRARIES.items(): - for path in [*library.files, library.license_path]: + for path in [*library.files, *(path for path, _, _ in library.licence_files())]: file = vendor_assets.local_file(key, path) file.parent.mkdir(parents=True, exist_ok=True) file.write_text(f"{key} {path}") @@ -234,11 +234,12 @@ def test_login_page_loads_libraries(client, not_fetched): @pytest.fixture def fake_library(monkeypatch): - """A library with two files and a licence, whose downloads are served from memory.""" + """A library with two files, a licence and a notice, whose downloads are served from memory.""" contents = { "https://cdn.example/lib@1.0/dist/lib.js": b"lib", "https://cdn.example/lib@1.0/dist/lib.css": b"style", "https://cdn.example/lib@1.0/LICENSE": b"licence", + "https://cdn.example/lib@1.0/NOTICE": b"notice", } library = VendorLibrary( name="lib", @@ -250,6 +251,7 @@ def fake_library(monkeypatch): }, license="LICENSE", license_sha256=hashlib.sha256(b"licence").hexdigest(), + notices={"NOTICE": hashlib.sha256(b"notice").hexdigest()}, ) monkeypatch.setattr(vendor_assets, "VENDOR_LIBRARIES", {"lib": library}) downloads = [] @@ -264,11 +266,11 @@ def download(url: str) -> bytes: def test_fetch_stores_files_and_licence(not_fetched, fake_library): - """Fetching stores a library's files and licence, and skips files that are already there.""" + """Fetching stores a library's files, licence and notice, and skips files that are already there.""" contents, downloads, download = fake_library libraries = vendor_assets.VENDOR_LIBRARIES - assert vendor_assets.fetch(download=download, libraries=libraries) == 3 + assert vendor_assets.fetch(download=download, libraries=libraries) == 4 assert ( vendor_assets.local_file("lib", "dist/lib.js", libraries).read_bytes() == b"lib" ) @@ -279,6 +281,9 @@ def test_fetch_stores_files_and_licence(not_fetched, fake_library): assert ( vendor_assets.local_file("lib", "LICENSE", libraries).read_bytes() == b"licence" ) + assert ( + vendor_assets.local_file("lib", "NOTICE", libraries).read_bytes() == b"notice" + ) assert vendor_assets.missing_files(libraries) == [] downloads.clear() @@ -286,15 +291,18 @@ def test_fetch_stores_files_and_licence(not_fetched, fake_library): assert downloads == [] -def test_fetch_rejects_a_file_that_does_not_match_its_hash(not_fetched, fake_library): +@pytest.mark.parametrize("path", ["dist/lib.css", "NOTICE"]) +def test_fetch_rejects_a_file_that_does_not_match_its_hash( + not_fetched, fake_library, path +): """A file whose content differs from what was recorded is not stored, and fetching fails.""" contents, _, download = fake_library - contents["https://cdn.example/lib@1.0/dist/lib.css"] = b"tampered" + contents[f"https://cdn.example/lib@1.0/{path}"] = b"tampered" libraries = vendor_assets.VENDOR_LIBRARIES with pytest.raises(ValueError, match="does not match the hash"): vendor_assets.fetch(download=download, libraries=libraries) - assert not vendor_assets.local_file("lib", "dist/lib.css", libraries).exists() + assert not vendor_assets.local_file("lib", path, libraries).exists() assert not list(vendor_assets.VENDOR_DIR.rglob("*.part")) @@ -323,7 +331,8 @@ def test_fetch_stores_an_overridden_version_with_hashes( vendor_assets.local_file("lib", "dist/lib.js", libraries).read_bytes() == b"lib 2" ) - assert "Could not fetch the licence of lib 2.0" in caplog.text + assert "Could not fetch the licence file LICENSE of lib 2.0" in caplog.text + assert "Could not fetch the licence file NOTICE of lib 2.0" in caplog.text contents["https://cdn.example/lib@2.0/dist/lib.css"] = b"tampered" vendor_assets.local_file("lib", "dist/lib.css", libraries).unlink() @@ -354,6 +363,7 @@ def test_print_hashes(fake_library, capsys, version): if version is None: assert f'"dist/lib.js": "{hashlib.sha256(b"lib").hexdigest()}",' in printed assert printed.startswith("files={") + assert f'"NOTICE": "{hashlib.sha256(b"notice").hexdigest()}",' in printed else: assert printed.startswith('"lib": ("2.0", {') assert f'"dist/lib.js": "{hashlib.sha256(b"lib 2").hexdigest()}",' in printed @@ -574,7 +584,7 @@ def test_only_one_process_fetches(not_fetched, fake_library): fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) assert vendor_assets.fetch_unless_fetching(libraries, download) is None assert downloads == [] - assert vendor_assets.fetch_unless_fetching(libraries, download) == 3 + assert vendor_assets.fetch_unless_fetching(libraries, download) == 4 # Once nothing is missing, the lock is not even taken. with open(vendor_assets.VENDOR_DIR / vendor_assets.LOCK_FILE, "w") as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) diff --git a/flexmeasures/ui/utils/vendor_assets.py b/flexmeasures/ui/utils/vendor_assets.py index 326ba2a768..6976dc9f45 100644 --- a/flexmeasures/ui/utils/vendor_assets.py +++ b/flexmeasures/ui/utils/vendor_assets.py @@ -8,7 +8,7 @@ python -m flexmeasures.ui.utils.fetch_vendor_assets -Fetching also stores each library's licence next to its files, and fails on any file whose hash does not match. +Fetching also stores each library's licence next to its files, with any notice file the licence requires to be shipped along, and fails on any file whose hash does not match. Templates load these libraries through the macros in templates/includes/vendor.html. To bump a library, change its version below, then print the hashes of its files at that version, and paste them in: @@ -31,7 +31,7 @@ import re import urllib.error import urllib.request -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Callable @@ -59,6 +59,9 @@ class VendorLibrary: license: str # path relative to the base URL, or a full URL license_sha256: str | None base_url: str = NPM_CDN # formatted with the name and version + # Further files which the licence requires to be shipped with the library, such as an Apache-2.0 NOTICE file. + # Path relative to the base URL -> SHA-256 hash, or None for another version, see libraries_with_overrides. + notices: dict[str, str | None] = field(default_factory=dict) # For a library not served from npm, the npm package to look up advisories for. npm_package: str | None = None @@ -83,6 +86,13 @@ def license_path(self) -> str: self.license.rsplit("/", 1)[-1] if "://" in self.license else self.license ) + def licence_files(self) -> list[tuple[str, str, str | None]]: + """The licence and notices to store next to the library's files, each as its local path, URL and SHA-256 hash.""" + return [ + (self.license_path, self.license_url, self.license_sha256), + *((path, self.url(path), sha256) for path, sha256 in self.notices.items()), + ] + VENDOR_LIBRARIES: dict[str, VendorLibrary] = { "vega": VendorLibrary( @@ -121,6 +131,9 @@ def license_path(self) -> str: }, license="LICENSE", license_sha256="634293835b43a6dd2094fa39182a3d9a6b9ca43b7fdb9ac354e8037af2a3093a", + notices={ + "NOTICE": "fa99ac3af859d0e13166906dc53a73ad34a08898da7e8ae83407275496e9e30c", + }, ), "currencysymbolmap": VendorLibrary( name="currency-symbol-map", @@ -383,6 +396,11 @@ def libraries_with_overrides(setting: dict | None) -> dict[str, VendorLibrary]: license_sha256=( library.license_sha256 if version == library.version else None ), + notices=( + library.notices + if version == library.version + else dict.fromkeys(library.notices) + ), ) return libraries @@ -481,7 +499,7 @@ def fetch( download: Callable[[str], bytes] = download, libraries: dict[str, VendorLibrary] | None = None, ) -> int: - """Download the libraries' files and licences into static/vendor/, checking each against its hash. + """Download the libraries' files, licences and notices into static/vendor/, checking each against its hash. Files that are already there with the right hash are skipped, and so are libraries without hashes. Returns the number of files written. @@ -494,22 +512,16 @@ def fetch( continue for path, sha256 in library.files.items(): written += _store(key, path, library.url(path), sha256, download, libraries) - try: - written += _store( - key, - library.license_path, - library.license_url, - library.license_sha256, - download, - libraries, - ) - except urllib.error.HTTPError as exc: - if library.license_sha256 is not None: - raise - # A version set through FLEXMEASURES_JS_VERSIONS may keep its licence elsewhere. - _logger().warning( - f"Could not fetch the licence of {key} {library.version} from {library.license_url}: {exc}." - ) + for path, url, sha256 in library.licence_files(): + try: + written += _store(key, path, url, sha256, download, libraries) + except urllib.error.HTTPError as exc: + if sha256 is not None: + raise + # A version set through FLEXMEASURES_JS_VERSIONS may keep its licence elsewhere. + _logger().warning( + f"Could not fetch the licence file {path} of {key} {library.version} from {url}: {exc}." + ) return written @@ -583,6 +595,11 @@ def print_hashes( print(f' "{path}": "{_sha256(download(library.url(path)))}",') print("},") print(f'license_sha256="{_sha256(download(library.license_url))}",') + if library.notices: + print("notices={") + for path in library.notices: + print(f' "{path}": "{_sha256(download(library.url(path)))}",') + print("},") return library = replace(library, version=version) hashes = {path: _sha256(download(library.url(path))) for path in library.files}