diff --git a/.github/workflows/ui-library-advisories.yml b/.github/workflows/ui-library-advisories.yml new file mode 100644 index 0000000000..909b37c379 --- /dev/null +++ b/.github/workflows/ui-library-advisories.yml @@ -0,0 +1,29 @@ +name: UI library advisories + +# Look up the UI's third-party libraries (see flexmeasures/ui/utils/vendor_assets.py) in the OSV database, +# as Dependabot only reads uv.lock and so does not know about them. +on: + schedule: + - cron: "0 6 * * 1" # Mondays, 06:00 UTC + workflow_dispatch: + pull_request: + paths: + - "flexmeasures/ui/utils/vendor_assets.py" + - "ci/check_ui_library_advisories.py" + - ".github/workflows/ui-library-advisories.yml" + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version-file: ".python-version" + - name: Install Flask, which vendor_assets.py imports + run: pip install flask + - name: Check the UI's libraries for known advisories + run: python ci/check_ui_library_advisories.py diff --git a/ci/check_ui_library_advisories.py b/ci/check_ui_library_advisories.py new file mode 100644 index 0000000000..ea208a97d2 --- /dev/null +++ b/ci/check_ui_library_advisories.py @@ -0,0 +1,125 @@ +"""Check the UI's third-party libraries for known security advisories, in the OSV database (https://osv.dev). + +Reads the libraries and their versions from flexmeasures/ui/utils/vendor_assets.py, +and exits with 1 if any of them has an advisory that is not ignored below. +Run by .github/workflows/ui-library-advisories.yml. +It needs only Flask, which vendor_assets.py imports, so it does not install FlexMeasures. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import sys +import urllib.request +from pathlib import Path +from typing import Callable + +VENDOR_ASSETS = ( + Path(__file__).resolve().parents[1] + / "flexmeasures" + / "ui" + / "utils" + / "vendor_assets.py" +) +OSV_API = "https://api.osv.dev/v1" + +# Advisories which OSV reports for a version that the advisory itself shows is not affected, +# by (advisory, npm package, version), with the reason. +IGNORED: dict[tuple[str, str, str], str] = { + ( + "GHSA-7f2v-3qq3-vvjf", + "vega", + "5.33.1", + ): "The advisory names vega-expression 5.2.1 as the fix outside ESM, and vega 5.33.1 depends on vega-expression ~5.2.1.", + ( + "GHSA-fgmj-fm8m-jvvx", + "echarts", + "5.6.0", + ): "Only the tooltip of the 'lines' series type is affected, which FlexMeasures does not use (fast-chart.js draws 'line' and 'bar' series).", +} + + +def load_libraries() -> dict: + """VENDOR_LIBRARIES, read from vendor_assets.py without importing the flexmeasures package.""" + spec = importlib.util.spec_from_file_location("vendor_assets", VENDOR_ASSETS) + module = importlib.util.module_from_spec(spec) + # Dataclasses look up their module while being defined. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module.VENDOR_LIBRARIES + + +def query_osv(packages: list[tuple[str, str]]) -> list[list[str]]: + """IDs of the advisories which affect each (npm package, version).""" + query = { + "queries": [ + {"package": {"name": name, "ecosystem": "npm"}, "version": version} + for name, version in packages + ] + } + request = urllib.request.Request( + f"{OSV_API}/querybatch", + data=json.dumps(query).encode(), + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=60) as response: + results = json.load(response)["results"] + return [[vuln["id"] for vuln in result.get("vulns", [])] for result in results] + + +def summarize(advisory: str) -> str: + with urllib.request.urlopen(f"{OSV_API}/vulns/{advisory}", timeout=60) as response: + return json.load(response).get("summary", "") + + +def find_advisories( + libraries: dict, + query: Callable[[list[tuple[str, str]]], list[list[str]]] = query_osv, +) -> list[tuple[str, str, str, str]]: + """(key, npm package, version, advisory) for each advisory affecting a library, except the ignored ones.""" + checked = [ + (key, library.npm_name, library.version) + for key, library in libraries.items() + if library.npm_name + ] + results = query([(name, version) for _, name, version in checked]) + return [ + (key, name, version, advisory) + for (key, name, version), advisories in zip(checked, results) + for advisory in advisories + if (advisory, name, version) not in IGNORED + ] + + +def main() -> int: + found = find_advisories(load_libraries()) + if found: + lines = [ + "## UI libraries with known advisories", + "", + "| Library | npm package | Version | Advisory | Summary |", + "|---|---|---|---|---|", + ] + for key, name, version, advisory in found: + lines.append( + f"| {key} | {name} | {version} | https://osv.dev/vulnerability/{advisory} | {summarize(advisory)} |" + ) + lines.append("") + lines.append( + "Bump each library in flexmeasures/ui/utils/vendor_assets.py to a version with a fix, " + "or, if the advisory does not apply to that version, add it to IGNORED in ci/check_ui_library_advisories.py with the reason." + ) + else: + lines = ["No known advisories affect the UI's libraries."] + report = "\n".join(lines) + print(report) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary: + summary.write(report + "\n") + return 1 if found else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/flexmeasures/ui/tests/test_ui_library_advisories.py b/flexmeasures/ui/tests/test_ui_library_advisories.py new file mode 100644 index 0000000000..a68a6887e5 --- /dev/null +++ b/flexmeasures/ui/tests/test_ui_library_advisories.py @@ -0,0 +1,72 @@ +"""ci/check_ui_library_advisories.py looks up each of the UI's libraries under its npm name, and leaves out ignored advisories.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +from flexmeasures.ui.utils.vendor_assets import VENDOR_LIBRARIES + +SCRIPT = Path(__file__).resolve().parents[3] / "ci" / "check_ui_library_advisories.py" + + +@pytest.fixture +def check(): + spec = importlib.util.spec_from_file_location("check_ui_library_advisories", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_script_reads_the_libraries_without_importing_flexmeasures(check): + """The workflow only installs Flask, so the script reads vendor_assets.py by path.""" + libraries = check.load_libraries() + assert {key: library.version for key, library in libraries.items()} == { + key: library.version for key, library in VENDOR_LIBRARIES.items() + } + + +def test_libraries_are_looked_up_under_their_npm_names(check): + """Libraries served from another CDN are looked up under their npm package, and fonts, which are not on npm, are skipped.""" + queried = [] + + def query(packages): + queried.extend(packages) + return [[] for _ in packages] + + assert check.find_advisories(VENDOR_LIBRARIES, query=query) == [] + names = {name for name, _ in queried} + assert { + "vega", + "jquery", + "@json-editor/json-editor", + "datatables.net", + "datatables.net-plugins", + } <= names + assert "DataTables" not in names + assert "Poppins" not in names + assert len(queried) == len(VENDOR_LIBRARIES) - 1 # all but Poppins + + +def test_ignored_advisories_are_left_out(check, monkeypatch): + """An advisory is only ignored for the package and version it was recorded for.""" + monkeypatch.setattr( + check, + "IGNORED", + {("GHSA-1", "vega", VENDOR_LIBRARIES["vega"].version): "reason"}, + ) + + def query(packages): + return [ + ["GHSA-1", "GHSA-2"] if name in ("vega", "jquery") else [] + for name, _ in packages + ] + + found = check.find_advisories(VENDOR_LIBRARIES, query=query) + assert sorted((key, advisory) for key, _, _, advisory in found) == [ + ("jquery", "GHSA-1"), + ("jquery", "GHSA-2"), + ("vega", "GHSA-2"), + ] diff --git a/flexmeasures/ui/utils/vendor_assets.py b/flexmeasures/ui/utils/vendor_assets.py index 94c4ae15b0..ccc75a0010 100644 --- a/flexmeasures/ui/utils/vendor_assets.py +++ b/flexmeasures/ui/utils/vendor_assets.py @@ -16,6 +16,8 @@ Hosts can load other versions through the FLEXMEASURES_JS_VERSIONS setting, see libraries_with_overrides. +Every week, .github/workflows/ui-library-advisories.yml looks up these versions in the OSV database of security advisories. + This is a leaf module, so flexmeasures/ui/__init__.py can import it without an import cycle. """ @@ -50,6 +52,15 @@ class VendorLibrary: license: str # path relative to the base URL, or a full URL license_sha256: str | None base_url: str = NPM_CDN # formatted with the name and version + # For a library not served from npm, the npm package to look up advisories for. + npm_package: str | None = None + + @property + def npm_name(self) -> str | None: + """The npm package which this library is published as, if any, which advisory databases know it by.""" + if self.npm_package: + return self.npm_package + return self.name if self.base_url == NPM_CDN else None def url(self, path: str) -> str: return self.base_url.format(name=self.name, version=self.version) + path @@ -225,6 +236,7 @@ def license_path(self) -> str: name="DataTables", version="1.13.11", base_url="https://cdn.datatables.net/{version}/", + npm_package="datatables.net", files={ "js/jquery.dataTables.min.js": "a33146fad8c7228dc4dc90043e3e50d51cead8b2267aeac32b0a8f3be8a52b86", "css/jquery.dataTables.min.css": "b113506cbced802d49dd3f2fdec2d9de60c9cbfe53ab13b87337e34ef10200eb", @@ -236,6 +248,7 @@ def license_path(self) -> str: name="DataTables conditionalPaging plug-in", version="1.13.6", base_url="https://cdn.datatables.net/plug-ins/{version}/features/conditionalPaging/", + npm_package="datatables.net-plugins", files={ "dataTables.conditionalPaging.js": "b2474d8a4335060d3498248a0980471a2e772eb3655dae4d59e60dc26d12ff72", },