Check the UI's third-party libraries for advisories every week

.github/workflows/ui-library-advisories.yml

@@ -0,0 +1,29 @@
+name: UI library advisories
+
+# Look up the UI's third-party libraries (see flexmeasures/ui/utils/vendor_assets.py) in the OSV database,
+# as Dependabot only reads uv.lock and so does not know about them.
+on:
+  schedule:
+    - cron: "0 6 * * 1"  # Mondays, 06:00 UTC
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "flexmeasures/ui/utils/vendor_assets.py"
+      - "ci/check_ui_library_advisories.py"
+      - ".github/workflows/ui-library-advisories.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version-file: ".python-version"
+      - name: Install Flask, which vendor_assets.py imports
+        run: pip install flask
+      - name: Check the UI's libraries for known advisories
+        run: python ci/check_ui_library_advisories.py

ci/check_ui_library_advisories.py

@@ -0,0 +1,125 @@
+"""Check the UI's third-party libraries for known security advisories, in the OSV database (https://osv.dev).
+
+Reads the libraries and their versions from flexmeasures/ui/utils/vendor_assets.py,
+and exits with 1 if any of them has an advisory that is not ignored below.
+Run by .github/workflows/ui-library-advisories.yml.
+It needs only Flask, which vendor_assets.py imports, so it does not install FlexMeasures.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import os
+import sys
+import urllib.request
+from pathlib import Path
+from typing import Callable
+
+VENDOR_ASSETS = (
+    Path(__file__).resolve().parents[1]
+    / "flexmeasures"
+    / "ui"
+    / "utils"
+    / "vendor_assets.py"
+)
+OSV_API = "https://api.osv.dev/v1"
+
+# Advisories which OSV reports for a version that the advisory itself shows is not affected,
+# by (advisory, npm package, version), with the reason.
+IGNORED: dict[tuple[str, str, str], str] = {
+    (
+        "GHSA-7f2v-3qq3-vvjf",
+        "vega",
+        "5.33.1",
+    ): "The advisory names vega-expression 5.2.1 as the fix outside ESM, and vega 5.33.1 depends on vega-expression ~5.2.1.",
+    (
+        "GHSA-fgmj-fm8m-jvvx",
+        "echarts",
+        "5.6.0",
+    ): "Only the tooltip of the 'lines' series type is affected, which FlexMeasures does not use (fast-chart.js draws 'line' and 'bar' series).",
+}
+
+
+def load_libraries() -> dict:
+    """VENDOR_LIBRARIES, read from vendor_assets.py without importing the flexmeasures package."""
+    spec = importlib.util.spec_from_file_location("vendor_assets", VENDOR_ASSETS)
+    module = importlib.util.module_from_spec(spec)
+    # Dataclasses look up their module while being defined.
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module.VENDOR_LIBRARIES
+
+
+def query_osv(packages: list[tuple[str, str]]) -> list[list[str]]:
+    """IDs of the advisories which affect each (npm package, version)."""
+    query = {
+        "queries": [
+            {"package": {"name": name, "ecosystem": "npm"}, "version": version}
+            for name, version in packages
+        ]
+    }
+    request = urllib.request.Request(
+        f"{OSV_API}/querybatch",
+        data=json.dumps(query).encode(),
+        headers={"Content-Type": "application/json"},
+    )
+    with urllib.request.urlopen(request, timeout=60) as response:
+        results = json.load(response)["results"]
+    return [[vuln["id"] for vuln in result.get("vulns", [])] for result in results]
+
+
+def summarize(advisory: str) -> str:
+    with urllib.request.urlopen(f"{OSV_API}/vulns/{advisory}", timeout=60) as response:
+        return json.load(response).get("summary", "")
+
+
+def find_advisories(
+    libraries: dict,
+    query: Callable[[list[tuple[str, str]]], list[list[str]]] = query_osv,
+) -> list[tuple[str, str, str, str]]:
+    """(key, npm package, version, advisory) for each advisory affecting a library, except the ignored ones."""
+    checked = [
+        (key, library.npm_name, library.version)
+        for key, library in libraries.items()
+        if library.npm_name
+    ]
+    results = query([(name, version) for _, name, version in checked])
+    return [
+        (key, name, version, advisory)
+        for (key, name, version), advisories in zip(checked, results)
+        for advisory in advisories
+        if (advisory, name, version) not in IGNORED
+    ]
+
+
+def main() -> int:
+    found = find_advisories(load_libraries())
+    if found:
+        lines = [
+            "## UI libraries with known advisories",
+            "",
+            "| Library | npm package | Version | Advisory | Summary |",
+            "|---|---|---|---|---|",
+        ]
+        for key, name, version, advisory in found:
+            lines.append(
+                f"| {key} | {name} | {version} | https://osv.dev/vulnerability/{advisory} | {summarize(advisory)} |"
+            )
+        lines.append("")
+        lines.append(
+            "Bump each library in flexmeasures/ui/utils/vendor_assets.py to a version with a fix, "
+            "or, if the advisory does not apply to that version, add it to IGNORED in ci/check_ui_library_advisories.py with the reason."
+        )
+    else:
+        lines = ["No known advisories affect the UI's libraries."]
+    report = "\n".join(lines)
+    print(report)
+    if os.environ.get("GITHUB_STEP_SUMMARY"):
+        with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary:
+            summary.write(report + "\n")
+    return 1 if found else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

flexmeasures/ui/tests/test_ui_library_advisories.py

@@ -0,0 +1,72 @@
+"""ci/check_ui_library_advisories.py looks up each of the UI's libraries under its npm name, and leaves out ignored advisories."""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+import pytest
+
+from flexmeasures.ui.utils.vendor_assets import VENDOR_LIBRARIES
+
+SCRIPT = Path(__file__).resolve().parents[3] / "ci" / "check_ui_library_advisories.py"
+
+
+@pytest.fixture
+def check():
+    spec = importlib.util.spec_from_file_location("check_ui_library_advisories", SCRIPT)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_script_reads_the_libraries_without_importing_flexmeasures(check):
+    """The workflow only installs Flask, so the script reads vendor_assets.py by path."""
+    libraries = check.load_libraries()
+    assert {key: library.version for key, library in libraries.items()} == {
+        key: library.version for key, library in VENDOR_LIBRARIES.items()
+    }
+
+
+def test_libraries_are_looked_up_under_their_npm_names(check):
+    """Libraries served from another CDN are looked up under their npm package, and fonts, which are not on npm, are skipped."""
+    queried = []
+
+    def query(packages):
+        queried.extend(packages)
+        return [[] for _ in packages]
+
+    assert check.find_advisories(VENDOR_LIBRARIES, query=query) == []
+    names = {name for name, _ in queried}
+    assert {
+        "vega",
+        "jquery",
+        "@json-editor/json-editor",
+        "datatables.net",
+        "datatables.net-plugins",
+    } <= names
+    assert "DataTables" not in names
+    assert "Poppins" not in names
+    assert len(queried) == len(VENDOR_LIBRARIES) - 1  # all but Poppins
+
+
+def test_ignored_advisories_are_left_out(check, monkeypatch):
+    """An advisory is only ignored for the package and version it was recorded for."""
+    monkeypatch.setattr(
+        check,
+        "IGNORED",
+        {("GHSA-1", "vega", VENDOR_LIBRARIES["vega"].version): "reason"},
+    )
+
+    def query(packages):
+        return [
+            ["GHSA-1", "GHSA-2"] if name in ("vega", "jquery") else []
+            for name, _ in packages
+        ]
+
+    found = check.find_advisories(VENDOR_LIBRARIES, query=query)
+    assert sorted((key, advisory) for key, _, _, advisory in found) == [
+        ("jquery", "GHSA-1"),
+        ("jquery", "GHSA-2"),
+        ("vega", "GHSA-2"),
+    ]

flexmeasures/ui/utils/vendor_assets.py

@@ -16,6 +16,8 @@
 
 Hosts can load other versions through the FLEXMEASURES_JS_VERSIONS setting, see libraries_with_overrides.
 
+Every week, .github/workflows/ui-library-advisories.yml looks up these versions in the OSV database of security advisories.
+
 This is a leaf module, so flexmeasures/ui/__init__.py can import it without an import cycle.
 """
 
@@ -50,6 +52,15 @@ class VendorLibrary:
     license: str  # path relative to the base URL, or a full URL
     license_sha256: str | None
     base_url: str = NPM_CDN  # formatted with the name and version
+    # For a library not served from npm, the npm package to look up advisories for.
+    npm_package: str | None = None
+
+    @property
+    def npm_name(self) -> str | None:
+        """The npm package which this library is published as, if any, which advisory databases know it by."""
+        if self.npm_package:
+            return self.npm_package
+        return self.name if self.base_url == NPM_CDN else None
 
     def url(self, path: str) -> str:
         return self.base_url.format(name=self.name, version=self.version) + path
@@ -225,6 +236,7 @@ def license_path(self) -> str:
         name="DataTables",
         version="1.13.11",
         base_url="https://cdn.datatables.net/{version}/",
+        npm_package="datatables.net",
         files={
             "js/jquery.dataTables.min.js": "a33146fad8c7228dc4dc90043e3e50d51cead8b2267aeac32b0a8f3be8a52b86",
             "css/jquery.dataTables.min.css": "b113506cbced802d49dd3f2fdec2d9de60c9cbfe53ab13b87337e34ef10200eb",
@@ -236,6 +248,7 @@ def license_path(self) -> str:
         name="DataTables conditionalPaging plug-in",
         version="1.13.6",
         base_url="https://cdn.datatables.net/plug-ins/{version}/features/conditionalPaging/",
+        npm_package="datatables.net-plugins",
         files={
             "dataTables.conditionalPaging.js": "b2474d8a4335060d3498248a0980471a2e772eb3655dae4d59e60dc26d12ff72",
         },