From aeca0ed8b780a153e3dde2fa7272b259896cff93 Mon Sep 17 00:00:00 2001 From: Fabio Kayser Date: Thu, 22 Oct 2026 11:39:48 +0200 Subject: [PATCH 1/3] fix: address dependency and CodeQL security alerts --- pyproject.toml | 6 ++- scripts/check-docs.mjs | 9 +++- scripts/check-docs.test.mjs | 7 ++++ src/datam8/secrets.py | 9 ---- .../test_017_plugin_secret_sql_regressions.py | 28 +++++++++++++ uv.lock | 41 +++++++++++-------- 6 files changed, 70 insertions(+), 30 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2608a9d..50515bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,8 +47,8 @@ Issues = "https://github.com/oraylis/datam8/issues" [dependency-groups] dev = [ - "pytest~=8.3.0", # TODO: upgrade to 9.x.x - "pytest-cases~=3.8.0", + "pytest~=9.0.3", + "pytest-cases~=3.10.1", "coverage~=7.13.5", "hatchling~=1.27.0", "httpx~=0.25.0", @@ -61,6 +61,8 @@ build-backend = "hatchling.build" [tool.uv] exclude-newer = "1 weeks" +# Allow critical CA trust-store fixes without waiting for the global cooldown (GHSA-6q48-6c7w-9gw2). +exclude-newer-package = { certifi = false } ############# # tool.ruff # diff --git a/scripts/check-docs.mjs b/scripts/check-docs.mjs index d7af701..d1e0a69 100644 --- a/scripts/check-docs.mjs +++ b/scripts/check-docs.mjs @@ -23,7 +23,14 @@ export function headingAnchors(source) { for (const line of proseLines(source)) { const heading = line.match(/^#{1,6}\s+(.+?)\s*#*$/)?.[1]; if (!heading) continue; - const slug = heading.toLowerCase().replace(/<[^>]*>/g, "") + let text = heading.toLowerCase(); + let previous; + // Remove innermost tags until stripping cannot expose another tag. + do { + previous = text; + text = text.replace(/<[^<>]*>/g, ""); + } while (text !== previous); + const slug = text .replace(/[^\p{L}\p{N}\p{M}_\-\s]/gu, "").replace(/\s/g, "-"); const count = counts.get(slug) || 0; counts.set(slug, count + 1); diff --git a/scripts/check-docs.test.mjs b/scripts/check-docs.test.mjs index 03136e7..e7b311b 100644 --- a/scripts/check-docs.test.mjs +++ b/scripts/check-docs.test.mjs @@ -25,6 +25,13 @@ test("reports empty alt text and heading spacing", () => { assert.equal(errors.length, 2); }); +test("heading anchors remove inline and reintroduced HTML tags", () => { + const source = "# Guide\n\n## Überblick 中文\n\n## Save\n\n## Save\n\n## t>Read\n\n## is removedt>alert(123)\n"; + assert.deepEqual([...headingAnchors(source)], [ + "guide", "überblick-中文", "save", "save-1", "read", "alert123", + ]); +}); + test("capture evidence rejects incomplete exports, missing images and failed acceptance", () => { const manifest = { frontendCommit: "a".repeat(40), backendCommit: "b".repeat(40), diff --git a/src/datam8/secrets.py b/src/datam8/secrets.py index 4220477..f2c7523 100644 --- a/src/datam8/secrets.py +++ b/src/datam8/secrets.py @@ -107,8 +107,6 @@ def __register_secret(self, path: PurePosixPath, /) -> None: secrets = self.__get_password(service_name) posix_path = path.as_posix() - logger.debug(f"Before register: {secrets}") - if secrets is None or secrets == "": secrets = posix_path elif f",{path.as_posix()}," in f",{secrets},": @@ -118,7 +116,6 @@ def __register_secret(self, path: PurePosixPath, /) -> None: secrets = f"{secrets},{path.as_posix()}" self.__set_password(service_name, secrets) - logger.debug(f"After register: {secrets}") def __unregister_secret(self, path: PurePosixPath, /) -> None: service_name = self.__create_service_name() @@ -126,8 +123,6 @@ def __unregister_secret(self, path: PurePosixPath, /) -> None: current_secrets = [] if current_secrets is None else current_secrets.split(",") posix_path = path.as_posix() - logger.debug(f"Before unregister: {current_secrets}") - if posix_path not in current_secrets: logger.warning("Trying to unregister non existing secret") return @@ -135,8 +130,6 @@ def __unregister_secret(self, path: PurePosixPath, /) -> None: path_index = current_secrets.index(posix_path) current_secrets.pop(path_index) - logger.debug(f"After unregister: {current_secrets}") - self.__set_password(service_name, ",".join(current_secrets)) def __create_service_name(self, path: PurePosixPath | None = None, /) -> str: @@ -191,8 +184,6 @@ def list_secrets(self) -> list[PurePosixPath]: secret_list = secrets.split(",") - logger.debug("Secret registry: '%s' [%s]", secrets, len(secret_list)) - return [PurePosixPath(p) for p in secret_list] def get_secret(self, path: PurePosixPath | str, /) -> str | None: diff --git a/tests/test_017_plugin_secret_sql_regressions.py b/tests/test_017_plugin_secret_sql_regressions.py index 5af05a6..98d03f0 100644 --- a/tests/test_017_plugin_secret_sql_regressions.py +++ b/tests/test_017_plugin_secret_sql_regressions.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import logging from pathlib import PurePosixPath from types import SimpleNamespace @@ -67,6 +68,33 @@ def test_secret_route_upserts_without_duplicate_registry_entries(fake_secret_bac assert SecretResolver().list_secrets() == [PurePosixPath(path)] +def test_secret_operations_do_not_log_values_or_registry( + fake_secret_backend, caplog: pytest.LogCaptureFixture +) -> None: + resolver = SecretResolver() + path = "datasources/security-regression/password" + first_value = "initial-password-sentinel" + updated_value = "updated-password-sentinel" + registry_marker = "registry-only-sentinel" + # Register a distinct marker before capture so it only appears in registry + # contents, rather than the service paths logged by subsequent operations. + resolver.set_secret(registry_marker, "unused-registry-password") + + with caplog.at_level(logging.DEBUG, logger="datam8.secrets"): + resolver.set_secret(path, first_value) + assert resolver.get_secret(path) == first_value + resolver.set_secret(path, updated_value, force=True) + assert resolver.get_secret(path) == updated_value + assert resolver.list_secrets() == [PurePosixPath(registry_marker), PurePosixPath(path)] + resolver.unset_secret(path) + assert resolver.get_secret(path) is None + assert resolver.list_secrets() == [PurePosixPath(registry_marker)] + + assert caplog.records + for sensitive in (first_value, updated_value, registry_marker): + assert sensitive not in caplog.text + + def _make_sql_plugin() -> SqlServer: plugin = SqlServer.__new__(SqlServer) plugin._data_source = SimpleNamespace(name="AdventureWorks") # type: ignore[ty:invalid-assignment] diff --git a/uv.lock b/uv.lock index bc92b5a..4580c25 100644 --- a/uv.lock +++ b/uv.lock @@ -7,9 +7,12 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-10-07T07:21:35.5030511Z" +exclude-newer = "2026-10-15T09:38:04.3671827Z" exclude-newer-span = "P1W" +[options.exclude-newer-package] +certifi = false + [[package]] name = "annotated-doc" version = "0.0.5" @@ -218,11 +221,11 @@ wheels = [ [[package]] name = "certifi" -version = "2026.7.22" +version = "2026.10.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]] @@ -616,8 +619,8 @@ dev = [ { name = "coverage", specifier = "~=7.13.5" }, { name = "hatchling", specifier = "~=1.27.0" }, { name = "httpx", specifier = "~=0.25.0" }, - { name = "pytest", specifier = "~=8.3.0" }, - { name = "pytest-cases", specifier = "~=3.8.0" }, + { name = "pytest", specifier = "~=9.0.3" }, + { name = "pytest-cases", specifier = "~=3.10.1" }, ] [[package]] @@ -1251,11 +1254,11 @@ wheels = [ [[package]] name = "pyjwt" -version = "2.13.0" +version = "2.15.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } +sdist = { url = "https://files.pythonhosted.org/packages/02/a5/5197bfd06417837ac079921c66fa6393f1dea3557272a263cebfef69e432/pyjwt-2.15.0.tar.gz", hash = "sha256:b11c5f9791d7bf51c2b39a81ed669f6b2dbbd669df2942f6c60167e9e3d1abe4", size = 120513, upload-time = "2026-09-23T16:56:00.689Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, + { url = "https://files.pythonhosted.org/packages/e8/55/40e45bf052ee8ee12a4dfd785519660f8effa7b065442b91646ec6828619/pyjwt-2.15.0-py3-none-any.whl", hash = "sha256:7a3742debf6b879e912dbb9819ceec1594be812452b78c5f2e2dfc56564954f8", size = 33680, upload-time = "2026-09-23T16:55:59.241Z" }, ] [package.optional-dependencies] @@ -1265,31 +1268,33 @@ crypto = [ [[package]] name = "pytest" -version = "8.3.5" +version = "9.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, + { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ae/3c/c9d525a414d506893f0cd8a8d0de7706446213181570cdbd766691164e40/pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845", size = 1450891, upload-time = "2025-03-02T12:54:54.503Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", size = 343634, upload-time = "2025-03-02T12:54:52.069Z" }, + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, ] [[package]] name = "pytest-cases" -version = "3.8.6" +version = "3.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "decopatch" }, { name = "makefun" }, { name = "packaging" }, + { name = "pytest" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/15/cb/d0d0810f9771b7daf35be7f283035c7a24c8c53b0272be458aebc2169ced/pytest_cases-3.8.6.tar.gz", hash = "sha256:5c24e0ab0cb6f8e802a469b7965906a333d3babb874586ebc56f7e2cbe1a7c44", size = 1092150, upload-time = "2024-09-26T22:02:17.995Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c1/a2/c7abc3b606125cf3732e8e613092b0e2549365e824b5f37972125c22806b/pytest_cases-3.10.1.tar.gz", hash = "sha256:451f9e3ecd5d2d81a4362c10c441126f5b3d1ae3a7efaf59f60b1fb930df2d69", size = 1095867, upload-time = "2026-03-02T23:05:33.824Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/78/63/965396a32be6f2cd6d16650ef1e103caf5df92a530087bb0cf0b2509a04e/pytest_cases-3.8.6-py2.py3-none-any.whl", hash = "sha256:564c722492ea7e7ec3ac433fd14070180e65866f49fa35bfe938c0d5d9afba67", size = 107283, upload-time = "2024-09-26T22:02:16.034Z" }, + { url = "https://files.pythonhosted.org/packages/61/f2/7a29fb0571562034b05c38dceabba48dcc622be5d6c5448db80779e55de7/pytest_cases-3.10.1-py2.py3-none-any.whl", hash = "sha256:0deb8a85b6132e44adbc1cfc57897c6a624ec23f48ab445a43c7d56a6b9315a4", size = 108870, upload-time = "2026-03-02T23:05:32.663Z" }, ] [[package]] @@ -1585,11 +1590,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.7.0" +version = "2.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] [[package]] From c3cfe4a6be7d61831fe1ad0bd79bbeeb1b16615d Mon Sep 17 00:00:00 2001 From: Fabio Kayser Date: Thu, 22 Oct 2026 12:54:43 +0200 Subject: [PATCH 2/3] fix: complete secret metadata and schema security hardening --- datam8-model | 2 +- scripts/check-docs.test.mjs | 6 ++++++ src/datam8/secrets.py | 6 +++--- .../test_017_plugin_secret_sql_regressions.py | 21 ++++++++++++------- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/datam8-model b/datam8-model index 591b796..3c71ddd 160000 --- a/datam8-model +++ b/datam8-model @@ -1 +1 @@ -Subproject commit 591b796a962117ef2a1f9d807074bf3a4f5cc159 +Subproject commit 3c71ddd2c756ea882b9b00f39b53f0e078726da5 diff --git a/scripts/check-docs.test.mjs b/scripts/check-docs.test.mjs index e7b311b..7200d39 100644 --- a/scripts/check-docs.test.mjs +++ b/scripts/check-docs.test.mjs @@ -32,6 +32,12 @@ test("heading anchors remove inline and reintroduced HTML tags", () => { ]); }); +test("heading anchors handle exposed and incomplete tags", () => { + assert.ok(headingAnchors("# ipt>Save\n").has("save")); + assert.ok(headingAnchors("# <em>Save\n").has("save")); + assert.ok(headingAnchors("#