Ship ECharts' NOTICE file with its licence

documentation/changelog.rst

@@ -40,7 +40,7 @@ New features
 
 Infrastructure / Support
 -------------------------
-* The UI can work without internet access: the Docker image serves the UI's JavaScript and CSS libraries itself, and a pip-installed FlexMeasures can fetch them once (see :ref:`install-ui-libraries`), while otherwise browsers keep loading them from CDNs; the ``FLEXMEASURES_JS_VERSIONS`` setting now only lists the versions a host wants to differ, optionally with the hashes of their files, so plugins reading it should read the ``js_versions`` template variable instead [see `PR #2711 <https://www.github.com/FlexMeasures/flexmeasures/pull/2711>`_]
+* The UI can work without internet access: the Docker image serves the UI's JavaScript and CSS libraries itself, and a pip-installed FlexMeasures can fetch them once (see :ref:`install-ui-libraries`), while otherwise browsers keep loading them from CDNs; the ``FLEXMEASURES_JS_VERSIONS`` setting now only lists the versions a host wants to differ, optionally with the hashes of their files, so plugins reading it should read the ``js_versions`` template variable instead [see `PR #2711 <https://www.github.com/FlexMeasures/flexmeasures/pull/2711>`_ and `PR #2747 <https://www.github.com/FlexMeasures/flexmeasures/pull/2747>`_]
 * Upgrade Vega, which draws the UI's charts, from 5.22.1 to 5.33.1, its latest 5.x release [see `PR #2743 <https://www.github.com/FlexMeasures/flexmeasures/pull/2743>`_]
 * ``add_beliefs_to_db_and_enqueue_forecasting_jobs`` now raises ``BeliefReplacementNotAllowed`` rather than an ``IntegrityError`` when saving would change existing values, so plugins catching ``IntegrityError`` around it should catch the new exception instead [see `PR #2722 <https://www.github.com/FlexMeasures/flexmeasures/pull/2722>`_]
 * ``[GET] /sensors/(id)`` now answers a sensor which does not exist with a 404, as the asset endpoints do, rather than with a 422, so that a client can tell a sensor which is gone apart from a request it got wrong [see `PR #2650 <https://www.github.com/FlexMeasures/flexmeasures/pull/2650>`_]

documentation/host/installation.rst

@@ -472,7 +472,7 @@ You can also fetch them yourself, for example on a server without internet acces
 
    $ python -m flexmeasures.ui.utils.fetch_vendor_assets
 
-Both download each library's files and licence into FlexMeasures' installation, and fail on any file that does not match its expected hash.
+Both download each library's files and licence, with any notice file the licence requires, into FlexMeasures' installation, and fail on any file that does not match its expected hash.
 They fetch the versions set by :ref:`FLEXMEASURES_JS_VERSIONS <configuration>` too, except for versions set without hashes.
 The command reads that setting from the config file FlexMeasures reads, or from the one passed with ``--config``.
 To stop FlexMeasures from fetching the libraries itself, set :ref:`FLEXMEASURES_FETCH_UI_LIBRARIES <configuration>` to ``False``.

flexmeasures/ui/tests/test_vendor_assets.py

@@ -69,7 +69,7 @@ def fetched(tmp_path, monkeypatch):
     monkeypatch.setattr(vendor_assets, "VENDOR_DIR", static_dir / "vendor")
     monkeypatch.setattr(flexmeasures_ui, "static_folder", str(static_dir))
     for key, library in VENDOR_LIBRARIES.items():
-        for path in [*library.files, library.license_path]:
+        for path in [*library.files, *(path for path, _, _ in library.licence_files())]:
             file = vendor_assets.local_file(key, path)
             file.parent.mkdir(parents=True, exist_ok=True)
             file.write_text(f"{key} {path}")
@@ -234,11 +234,12 @@ def test_login_page_loads_libraries(client, not_fetched):
 
 @pytest.fixture
 def fake_library(monkeypatch):
-    """A library with two files and a licence, whose downloads are served from memory."""
+    """A library with two files, a licence and a notice, whose downloads are served from memory."""
     contents = {
         "https://cdn.example/lib@1.0/dist/lib.js": b"lib",
         "https://cdn.example/lib@1.0/dist/lib.css": b"style",
         "https://cdn.example/lib@1.0/LICENSE": b"licence",
+        "https://cdn.example/lib@1.0/NOTICE": b"notice",
     }
     library = VendorLibrary(
         name="lib",
@@ -250,6 +251,7 @@ def fake_library(monkeypatch):
         },
         license="LICENSE",
         license_sha256=hashlib.sha256(b"licence").hexdigest(),
+        notices={"NOTICE": hashlib.sha256(b"notice").hexdigest()},
     )
     monkeypatch.setattr(vendor_assets, "VENDOR_LIBRARIES", {"lib": library})
     downloads = []
@@ -264,11 +266,11 @@ def download(url: str) -> bytes:
 
 
 def test_fetch_stores_files_and_licence(not_fetched, fake_library):
-    """Fetching stores a library's files and licence, and skips files that are already there."""
+    """Fetching stores a library's files, licence and notice, and skips files that are already there."""
     contents, downloads, download = fake_library
     libraries = vendor_assets.VENDOR_LIBRARIES
 
-    assert vendor_assets.fetch(download=download, libraries=libraries) == 3
+    assert vendor_assets.fetch(download=download, libraries=libraries) == 4
     assert (
         vendor_assets.local_file("lib", "dist/lib.js", libraries).read_bytes() == b"lib"
     )
@@ -279,22 +281,28 @@ def test_fetch_stores_files_and_licence(not_fetched, fake_library):
     assert (
         vendor_assets.local_file("lib", "LICENSE", libraries).read_bytes() == b"licence"
     )
+    assert (
+        vendor_assets.local_file("lib", "NOTICE", libraries).read_bytes() == b"notice"
+    )
     assert vendor_assets.missing_files(libraries) == []
 
     downloads.clear()
     assert vendor_assets.fetch(download=download, libraries=libraries) == 0
     assert downloads == []
 
 
-def test_fetch_rejects_a_file_that_does_not_match_its_hash(not_fetched, fake_library):
+@pytest.mark.parametrize("path", ["dist/lib.css", "NOTICE"])
+def test_fetch_rejects_a_file_that_does_not_match_its_hash(
+    not_fetched, fake_library, path
+):
     """A file whose content differs from what was recorded is not stored, and fetching fails."""
     contents, _, download = fake_library
-    contents["https://cdn.example/lib@1.0/dist/lib.css"] = b"tampered"
+    contents[f"https://cdn.example/lib@1.0/{path}"] = b"tampered"
     libraries = vendor_assets.VENDOR_LIBRARIES
 
     with pytest.raises(ValueError, match="does not match the hash"):
         vendor_assets.fetch(download=download, libraries=libraries)
-    assert not vendor_assets.local_file("lib", "dist/lib.css", libraries).exists()
+    assert not vendor_assets.local_file("lib", path, libraries).exists()
     assert not list(vendor_assets.VENDOR_DIR.rglob("*.part"))
 
 
@@ -323,7 +331,8 @@ def test_fetch_stores_an_overridden_version_with_hashes(
         vendor_assets.local_file("lib", "dist/lib.js", libraries).read_bytes()
         == b"lib 2"
     )
-    assert "Could not fetch the licence of lib 2.0" in caplog.text
+    assert "Could not fetch the licence file LICENSE of lib 2.0" in caplog.text
+    assert "Could not fetch the licence file NOTICE of lib 2.0" in caplog.text
 
     contents["https://cdn.example/lib@2.0/dist/lib.css"] = b"tampered"
     vendor_assets.local_file("lib", "dist/lib.css", libraries).unlink()
@@ -354,6 +363,7 @@ def test_print_hashes(fake_library, capsys, version):
     if version is None:
         assert f'"dist/lib.js": "{hashlib.sha256(b"lib").hexdigest()}",' in printed
         assert printed.startswith("files={")
+        assert f'"NOTICE": "{hashlib.sha256(b"notice").hexdigest()}",' in printed
     else:
         assert printed.startswith('"lib": ("2.0", {')
         assert f'"dist/lib.js": "{hashlib.sha256(b"lib 2").hexdigest()}",' in printed
@@ -574,7 +584,7 @@ def test_only_one_process_fetches(not_fetched, fake_library):
         fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
         assert vendor_assets.fetch_unless_fetching(libraries, download) is None
         assert downloads == []
-    assert vendor_assets.fetch_unless_fetching(libraries, download) == 3
+    assert vendor_assets.fetch_unless_fetching(libraries, download) == 4
     # Once nothing is missing, the lock is not even taken.
     with open(vendor_assets.VENDOR_DIR / vendor_assets.LOCK_FILE, "w") as lock:
         fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)

flexmeasures/ui/utils/vendor_assets.py

@@ -8,7 +8,7 @@
 
     python -m flexmeasures.ui.utils.fetch_vendor_assets
 
-Fetching also stores each library's licence next to its files, and fails on any file whose hash does not match.
+Fetching also stores each library's licence next to its files, with any notice file the licence requires to be shipped along, and fails on any file whose hash does not match.
 Templates load these libraries through the macros in templates/includes/vendor.html.
 
 To bump a library, change its version below, then print the hashes of its files at that version, and paste them in:
@@ -31,7 +31,7 @@
 import re
 import urllib.error
 import urllib.request
-from dataclasses import dataclass, replace
+from dataclasses import dataclass, field, replace
 from pathlib import Path
 from typing import Callable
 
@@ -59,6 +59,9 @@ class VendorLibrary:
     license: str  # path relative to the base URL, or a full URL
     license_sha256: str | None
     base_url: str = NPM_CDN  # formatted with the name and version
+    # Further files which the licence requires to be shipped with the library, such as an Apache-2.0 NOTICE file.
+    # Path relative to the base URL -> SHA-256 hash, or None for another version, see libraries_with_overrides.
+    notices: dict[str, str | None] = field(default_factory=dict)
     # For a library not served from npm, the npm package to look up advisories for.
     npm_package: str | None = None
 
@@ -83,6 +86,13 @@ def license_path(self) -> str:
             self.license.rsplit("/", 1)[-1] if "://" in self.license else self.license
         )
 
+    def licence_files(self) -> list[tuple[str, str, str | None]]:
+        """The licence and notices to store next to the library's files, each as its local path, URL and SHA-256 hash."""
+        return [
+            (self.license_path, self.license_url, self.license_sha256),
+            *((path, self.url(path), sha256) for path, sha256 in self.notices.items()),
+        ]
+
 
 VENDOR_LIBRARIES: dict[str, VendorLibrary] = {
     "vega": VendorLibrary(
@@ -121,6 +131,9 @@ def license_path(self) -> str:
         },
         license="LICENSE",
         license_sha256="634293835b43a6dd2094fa39182a3d9a6b9ca43b7fdb9ac354e8037af2a3093a",
+        notices={
+            "NOTICE": "fa99ac3af859d0e13166906dc53a73ad34a08898da7e8ae83407275496e9e30c",
+        },
     ),
     "currencysymbolmap": VendorLibrary(
         name="currency-symbol-map",
@@ -383,6 +396,11 @@ def libraries_with_overrides(setting: dict | None) -> dict[str, VendorLibrary]:
             license_sha256=(
                 library.license_sha256 if version == library.version else None
             ),
+            notices=(
+                library.notices
+                if version == library.version
+                else dict.fromkeys(library.notices)
+            ),
         )
     return libraries
 
@@ -481,7 +499,7 @@ def fetch(
     download: Callable[[str], bytes] = download,
     libraries: dict[str, VendorLibrary] | None = None,
 ) -> int:
-    """Download the libraries' files and licences into static/vendor/, checking each against its hash.
+    """Download the libraries' files, licences and notices into static/vendor/, checking each against its hash.
 
     Files that are already there with the right hash are skipped, and so are libraries without hashes.
     Returns the number of files written.
@@ -494,22 +512,16 @@ def fetch(
             continue
         for path, sha256 in library.files.items():
             written += _store(key, path, library.url(path), sha256, download, libraries)
-        try:
-            written += _store(
-                key,
-                library.license_path,
-                library.license_url,
-                library.license_sha256,
-                download,
-                libraries,
-            )
-        except urllib.error.HTTPError as exc:
-            if library.license_sha256 is not None:
-                raise
-            # A version set through FLEXMEASURES_JS_VERSIONS may keep its licence elsewhere.
-            _logger().warning(
-                f"Could not fetch the licence of {key} {library.version} from {library.license_url}: {exc}."
-            )
+        for path, url, sha256 in library.licence_files():
+            try:
+                written += _store(key, path, url, sha256, download, libraries)
+            except urllib.error.HTTPError as exc:
+                if sha256 is not None:
+                    raise
+                # A version set through FLEXMEASURES_JS_VERSIONS may keep its licence elsewhere.
+                _logger().warning(
+                    f"Could not fetch the licence file {path} of {key} {library.version} from {url}: {exc}."
+                )
     return written
 
 
@@ -583,6 +595,11 @@ def print_hashes(
             print(f'    "{path}": "{_sha256(download(library.url(path)))}",')
         print("},")
         print(f'license_sha256="{_sha256(download(library.license_url))}",')
+        if library.notices:
+            print("notices={")
+            for path in library.notices:
+                print(f'    "{path}": "{_sha256(download(library.url(path)))}",')
+            print("},")
         return
     library = replace(library, version=version)
     hashes = {path: _sha256(download(library.url(path))) for path in library.files}