From 03b2e2da7ccbdabcc155a56b4a8891ae89cbb6ae Mon Sep 17 00:00:00 2001 From: Fabio Kayser Date: Thu, 5 Nov 2026 08:34:18 +0100 Subject: [PATCH 1/6] fix: make the public sample use the bundled SQL Server connector --- Base/DataSourceTypes.json | 132 ++++-------------- .../__modules/metadata_utils.py | 5 + Generate/databricks-lake/__modules/payload.py | 5 +- plugins.lock.json | 14 -- pyproject.toml | 5 +- scripts/check_generation.py | 3 + scripts/install_plugins.py | 51 ------- tests/test_public_entrypoint.py | 64 +++++++++ uv.lock | 77 ++++------ 9 files changed, 136 insertions(+), 220 deletions(-) delete mode 100644 plugins.lock.json delete mode 100644 scripts/install_plugins.py create mode 100644 tests/test_public_entrypoint.py diff --git a/Base/DataSourceTypes.json b/Base/DataSourceTypes.json index 65df5a4..91036fc 100644 --- a/Base/DataSourceTypes.json +++ b/Base/DataSourceTypes.json @@ -123,104 +123,65 @@ "targetType": "string" } ], - "pluginId": "sqlserver", + "pluginId": "builtin:SQLServer", "connectionProperties": [ { "name": "authMode", - "displayName": "authMode", - "required": true, "type": "string", - "default": "sql_user" + "required": true }, { "name": "host", - "displayName": "host", - "required": false, - "type": "string" + "type": "string", + "required": true }, { "name": "database", - "displayName": "database", - "required": false, - "type": "string" + "type": "string", + "required": true }, { "name": "port", - "displayName": "port", - "required": false, "type": "number", + "required": true, "default": 1433 }, { "name": "username", - "displayName": "username", - "required": false, - "type": "string" + "type": "string", + "required": false }, { "name": "password", - "displayName": "password", - "required": false, - "type": "secret" - }, - { - "name": "encrypt", - "displayName": "encrypt", - "required": false, - "description": "Enable encrypted transport", - "type": "boolean", - "default": true + "type": "secret", + "required": false }, { - "name": "trusted_connection", - "displayName": "trusted_connection", + "name": "trust_server_certificate", "required": false, - "description": "Use Windows authentication", + "default": false, "type": "boolean", - "default": false - }, - { - "name": "workspace_url", - "displayName": "workspace_url", - "required": false, - "type": "string" - }, - { - "name": "warehouse_id", - "displayName": "warehouse_id", - "required": false, - "type": "string" + "description": "Trust unverifyable certificates, e.g. self-signed" }, { - "name": "uc_connection_name", - "displayName": "uc_connection_name", + "name": "trust_server_certificate_ca", "required": false, - "type": "string" - }, - { - "name": "token", - "displayName": "token", - "required": false, - "type": "secret" - }, - { - "name": "client_id", - "displayName": "client_id", - "required": false, - "type": "string" + "type": "string", + "description": "Path to a root ca to verify the server certificate against" }, { - "name": "client_secret", - "displayName": "client_secret", + "name": "encrypt", "required": false, - "type": "secret" + "default": true, + "type": "boolean", + "description": "Activate SSL encryption" }, { - "name": "auth_scope", - "displayName": "auth_scope", + "name": "trusted_connection", "required": false, - "type": "string", - "default": "all-apis" + "default": false, + "type": "boolean", + "description": "Enables Windows Authentication" } ], "authModes": [ @@ -228,59 +189,26 @@ "name": "sql_user", "displayName": "Username / Password", "required": [ - "authMode", - "host", - "database", - "port", "username", "password" ], "optional": [ + "trust_server_certificate", + "trust_server_certificate_ca", "encrypt" ] }, { "name": "windows", - "displayName": "Windows", + "displayName": "Windows Authentication", "required": [ - "authMode", - "host", - "database", - "port", "trusted_connection" ], "optional": [ + "trust_server_certificate", + "trust_server_certificate_ca", "encrypt" ] - }, - { - "name": "databricks_pat", - "displayName": "Databricks (PAT)", - "required": [ - "authMode", - "database", - "workspace_url", - "warehouse_id", - "uc_connection_name", - "token" - ], - "optional": [] - }, - { - "name": "databricks_oauth", - "displayName": "Databricks (OAuth Client Credentials)", - "required": [ - "authMode", - "database", - "workspace_url", - "warehouse_id", - "uc_connection_name", - "client_id", - "client_secret" - ], - "optional": [ - "auth_scope" - ] } ] } diff --git a/Generate/databricks-lake/__modules/metadata_utils.py b/Generate/databricks-lake/__modules/metadata_utils.py index d485f5c..c6826ce 100644 --- a/Generate/databricks-lake/__modules/metadata_utils.py +++ b/Generate/databricks-lake/__modules/metadata_utils.py @@ -15,6 +15,11 @@ TARGET_NAME = "databricks" + +def source_reader_kind(plugin_id: str | None) -> str | None: + """Keep the generated SQL reader contract independent of host plugin IDs.""" + return "sqlserver" if plugin_id == "builtin:SQLServer" else plugin_id + # Common aliases so source / canonical names converge to those defined in DataTypes.json. TYPE_ALIASES: dict[str, str] = { "integer": "int", diff --git a/Generate/databricks-lake/__modules/payload.py b/Generate/databricks-lake/__modules/payload.py index f6e366e..68f6304 100644 --- a/Generate/databricks-lake/__modules/payload.py +++ b/Generate/databricks-lake/__modules/payload.py @@ -19,6 +19,7 @@ from metadata_utils import ( MetadataResolver, + source_reader_kind, attribute_property_equals, build_business_key_partitions, build_delta_table_properties, @@ -532,7 +533,7 @@ def generate_raw_dml_notebooks(model: Model, cache: Cache) -> Sequence[IPayload] data_source_name=data_source_name, ) source_type = next((item.entity for item in model.dataSourceTypes.values() if item.entity.name == data_source_entry.type), None) - plugin = getattr(source_type, "pluginId", None) + plugin = source_reader_kind(getattr(source_type, "pluginId", None)) if plugin not in {"sqlserver", "oracle", "mysql", "postgresql", "http_api"}: raise ValueError(f"Unsupported UC source plugin for {data_source_name}: {plugin}") extended = data_source_entry.extendedProperties or {} @@ -832,7 +833,7 @@ def generation_manifest(model: Model, cache: Cache) -> Sequence[IPayload]: raw_key = "raw." + raw["full_table_name"].lower() data_source = resolver.data_sources[external.dataSource] source_type = next(item.entity for item in model.dataSourceTypes.values() if item.entity.name == data_source.type) - plugin = source_type.pluginId + plugin = source_reader_kind(source_type.pluginId) settings = data_source.extendedProperties or {} extract_mode = next((reference.value for reference in external.properties or [] if reference.property == "extract_mode"), "full") external_spec = {"source": data_source.name, "plugin": plugin, "connection_suffix": settings.get("unityCatalogConnection", plugin), "location": external.sourceLocation, "database": settings.get("database"), "service_name": settings.get("service_name"), "extract_mode": extract_mode} diff --git a/plugins.lock.json b/plugins.lock.json deleted file mode 100644 index aabb007..0000000 --- a/plugins.lock.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "repository": "https://github.com/oraylis/datam8-plugins", - "commit": "5fec63282f0c94727031a5ebe15ed6e86247bb5b", - "plugins": [ - { - "id": "sqlserver", - "version": "1.2.0", - "sha256": { - "plugin.json": "d59d3c622209d01ace0a21ffeff4ae18384f483fb75d4b06dc0ca4ffebbe13e2", - "plugin.py": "d69f44b96795f65f4fb47674e4d8e6447d6c3504cc1684f6b0c024b3234a0490" - } - } - ] -} diff --git a/pyproject.toml b/pyproject.toml index f89651a..89404c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,12 @@ [project] name = "datam8-sample-solution" version = "2.1.0" -description = "Portable DataM8 multi-source sample solution" +description = "Public AdventureWorks metadata and local generation example" readme = "README.md" requires-python = ">=3.12" dependencies = [ - "datam8", + "datam8[sql]", "PyYAML==6.0.3", - "pyodbc==5.3.0", ] [project.urls] diff --git a/scripts/check_generation.py b/scripts/check_generation.py index 324a4ef..250e94e 100644 --- a/scripts/check_generation.py +++ b/scripts/check_generation.py @@ -36,6 +36,9 @@ def main(): if name=="databricks": resources=generated_resources(output.parent) manifest=json.loads((output/"manifest.json").read_text()) + external_sources=[item["external"] for item in manifest["datasets"] if item.get("external")] + if not external_sources or any(item["plugin"] != "sqlserver" for item in external_sources): + raise AssertionError("The public SQL Server sample must generate SQL Server source readers") execution_plan(resources,manifest,required_phases=manifest["lifecycle"]) for job in resources["jobs"].values():check_graph(job["tasks"]) required_assets(output.parent) diff --git a/scripts/install_plugins.py b/scripts/install_plugins.py deleted file mode 100644 index f60c4cd..0000000 --- a/scripts/install_plugins.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Install the pinned metadata plugins locally; plugin sources stay Git-ignored.""" -from __future__ import annotations - -import argparse -import hashlib -import json -import os -from pathlib import Path -from urllib.request import Request, urlopen - -ROOT = Path(__file__).resolve().parents[1] - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--repository", type=Path, help="Optional local datam8-plugins checkout") - args = parser.parse_args() - lock = json.loads((ROOT / "plugins.lock.json").read_text()) - token = os.getenv("DATAM8_PLUGINS_READ_TOKEN") - if not args.repository and not token: - raise RuntimeError( - "datam8-plugins is private. Set DATAM8_PLUGINS_READ_TOKEN or pass a local --repository checkout." - ) - files = [] - for plugin in lock["plugins"]: - for name, expected in plugin["sha256"].items(): - relative = f"plugins/{plugin['id']}/{name}" - if args.repository: - import subprocess - content = subprocess.check_output(["git", "-C", str(args.repository), "show", f"{lock['commit']}:{relative}"]) - else: - repository = lock["repository"].removeprefix("https://github.com/") - request = Request( - f"https://raw.githubusercontent.com/{repository}/{lock['commit']}/{relative}", - headers={"Authorization": f"Bearer {token}"}, - ) - with urlopen(request, timeout=60) as response: - content = response.read() - if hashlib.sha256(content).hexdigest() != expected: - raise ValueError("Plugin checksum mismatch: " + relative) - if name == "plugin.json" and json.loads(content)["version"] != plugin["version"]: - raise ValueError("Plugin version mismatch: " + relative) - files.append((ROOT / "Plugins" / plugin["id"] / name, content)) - for path, content in files: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(content) - print(f"Installed {len(lock['plugins'])} verified plugins at {lock['commit']}") - - -if __name__ == "__main__": - main() diff --git a/tests/test_public_entrypoint.py b/tests/test_public_entrypoint.py new file mode 100644 index 0000000..e345b7d --- /dev/null +++ b/tests/test_public_entrypoint.py @@ -0,0 +1,64 @@ +"""Exercise the public sample binding against the real bundled SQL Server host.""" +import json +from pathlib import Path +import sys +import unittest +from unittest.mock import patch + +import polars as pl + +from datam8.plugins import PluginManager, init_builtin_plugins +from datam8_model.data_source import DataSource, DataSourceType + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "Generate/databricks-lake/__modules")) +from metadata_utils import source_reader_kind + + +class PublicEntrypointTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + source_types = json.loads((ROOT / "Base/DataSourceTypes.json").read_text()) + cls.source_type = DataSourceType.model_validate(source_types["dataSourceTypes"][0]) + init_builtin_plugins(plugin_id=cls.source_type.pluginId) + cls.plugin_class = PluginManager().get_plugin(cls.source_type.pluginId) + + def test_saved_connection_forms_match_the_public_builtin(self): + self.assertEqual(self.source_type.pluginId, "builtin:SQLServer") + properties = {item.name: item for item in self.source_type.connectionProperties} + for expected in self.plugin_class.get_connection_properties(): + actual = properties.pop(expected.name) + self.assertEqual((actual.type, actual.required, actual.default), + (expected.type, expected.required, expected.default)) + self.assertFalse(properties) + self.assertEqual(self.source_type.authModes, self.plugin_class.get_auth_modes()) + + def test_real_builtin_metadata_preserves_source_types_and_sample_mappings(self): + sources = json.loads((ROOT / "Base/DataSources.json").read_text()) + source = DataSource.model_validate(sources["dataSources"][0]) + source.extendedProperties["password"] = "local-fixture-only" + plugin = self.plugin_class(self.plugin_class.manifest(), source, self.source_type) + rows = pl.DataFrame({ + "ordinal": [1, 2], "name": ["CustomerID", "Amount"], + "dataType": ["int", "decimal"], "maxLength": [None, None], + "numericPrecision": [10, 18], "numericScale": [0, 2], + "isNullable": ["NO", "YES"], "isPrimaryKey": [1, 0], + }) + with patch.object(plugin, "_execute_query", return_value=rows) as query: + fields = list(plugin.get_table_metadata("[SalesLT].[Fixture]").iter_source_fields()) + query.assert_called_once() + self.assertEqual([(field.name, field.dataType) for field in fields], + [("CustomerID", "int"), ("Amount", "decimal")]) + self.assertEqual(fields[1].numbericScale, 2) + mappings = {item.sourceType: item.targetType for item in self.source_type.dataTypeMapping} + self.assertEqual([mappings[field.dataType] for field in fields], ["int", "decimal"]) + + def test_host_binding_keeps_the_existing_generated_reader_kind(self): + self.assertEqual(source_reader_kind(self.source_type.pluginId), "sqlserver") + self.assertEqual(source_reader_kind("sqlserver"), "sqlserver") + self.assertEqual(source_reader_kind("oracle"), "oracle") + self.assertEqual(source_reader_kind("builtin:CsvFile"), "builtin:CsvFile") + + +if __name__ == "__main__": + unittest.main() diff --git a/uv.lock b/uv.lock index 0ae6db7..72ad102 100644 --- a/uv.lock +++ b/uv.lock @@ -158,6 +158,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "connectorx" +version = "0.4.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/74/c3/77aebad14179cf1a8cfdc5e84ea1bac4efb82de270ca6fc7ff914f8ec601/connectorx-0.4.5-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:25efda2317f40e6536582c3dd4f57a8a31c7e5969d708a674272c05591e6f5a2", size = 37900333, upload-time = "2026-01-18T01:31:27.545Z" }, + { url = "https://files.pythonhosted.org/packages/23/5c/e1e82bcb235fa52280696b01a975535800c0b8c3c12af7c5ddbb42a39010/connectorx-0.4.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:27539e03408705f318572b163c419572a114fdc9baf4d1e6cd746bb87f573cf2", size = 35971550, upload-time = "2026-01-18T01:31:43.775Z" }, + { url = "https://files.pythonhosted.org/packages/14/be/fa9c3a14b6c10c899d0fb93f8ea549285db271c9899900f7eb21c3cdeb9a/connectorx-0.4.5-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c68cc9c6bff737d3c9fb8735b27ecc8474238ef640abb701ee0ab213c6c95f8c", size = 43720741, upload-time = "2026-01-18T01:30:52.897Z" }, + { url = "https://files.pythonhosted.org/packages/3e/4c/54cbfabd1866f3f8657e348f3a496fbde0a138d66a9f2f024b28b4db1c2e/connectorx-0.4.5-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:3863bc71677d6314b60cb1e1489a650114d37d8d9f58f2df038cae4a82d2ffc5", size = 43751056, upload-time = "2026-01-18T01:31:10.534Z" }, + { url = "https://files.pythonhosted.org/packages/e0/03/350aafe6bc38a3851744bfab7c4d61bd16500ff6b20dfcb98054b7adb56d/connectorx-0.4.5-cp312-none-win_amd64.whl", hash = "sha256:0737254429e22e5012e1fe6a849112da38abb9b56743b3b8c8a1f902e5270e75", size = 34642277, upload-time = "2026-01-18T01:59:38.121Z" }, + { url = "https://files.pythonhosted.org/packages/10/de/65de3629f3bbb0597cc3393078085a27b5b52a9fa5b701a60c1c11a9868c/connectorx-0.4.5-cp313-cp313-macosx_10_7_x86_64.whl", hash = "sha256:ff2f4236a0fc14cd724b03df1f11c03b714442f4381575465f7d0f4f91135766", size = 37900697, upload-time = "2026-01-18T01:31:30.664Z" }, + { url = "https://files.pythonhosted.org/packages/88/59/6a4542bc57c53e99b366a8f377ebb8c9b9915d08a8915726c4a5f0fd8219/connectorx-0.4.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f5d4754069644a712bd3105345e4f7c680420c5bb1d1264070cda058c7f07fb3", size = 35972573, upload-time = "2026-01-18T01:31:46.945Z" }, + { url = "https://files.pythonhosted.org/packages/a6/8e/96abe0aecb5e121a80f64ded08c4d8b4115df85f1b246a2680c9f29d4d3d/connectorx-0.4.5-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:31a65ff4ec8fde7ea7aa2812f2b21e7a512a3216b1b22ca1b02d3975b0bf1e75", size = 43746476, upload-time = "2026-01-18T01:30:56.296Z" }, + { url = "https://files.pythonhosted.org/packages/0a/9d/3bae67718e0bfbefba41959f2f1dc0a5765392aea311b02d98457c8efd34/connectorx-0.4.5-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:ab1d62a26350055c5e901daa4d6dddb75b11addb923797158c809dffc4f0ac9e", size = 43761125, upload-time = "2026-01-18T01:31:14.253Z" }, + { url = "https://files.pythonhosted.org/packages/40/32/54a45bc796b2e5a572bb76c17ebaf971ec57bf9960ba23731d76a7f70962/connectorx-0.4.5-cp313-none-win_amd64.whl", hash = "sha256:50c20558beff2719be34ff325213526c1700c3a20743e9e0ba592774ebc9cc92", size = 34645527, upload-time = "2026-01-18T01:59:41.303Z" }, + { url = "https://files.pythonhosted.org/packages/79/76/8f89e0d1c973af07eed12c584c52c33b51dc119b7ed85e3c0f91615bfec6/connectorx-0.4.5-cp314-cp314-macosx_10_7_x86_64.whl", hash = "sha256:3ddfe372065b974365bff3b383e39c29cad468c0e7556543dd23753446c441ed", size = 37898985, upload-time = "2026-01-18T01:31:34.162Z" }, + { url = "https://files.pythonhosted.org/packages/f4/38/6243f6c83e9515ebab87c0b21be1c2599bd3f6148b8905802b12dba4bf83/connectorx-0.4.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3fd7788294417cbbb3811f8942e4fe3b4c190b80627a3c706ceae6c321824bcf", size = 35968795, upload-time = "2026-01-18T01:59:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/7f/1a/031079b5c597b83df8548012095c23c10d471e6a4c29617d55dd4cf5a9b8/connectorx-0.4.5-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:38ad8a032fddf25c36c6911d857fbe54220fe28439f02a4beb273b29bdef1eb8", size = 43742341, upload-time = "2026-01-18T01:31:00.55Z" }, + { url = "https://files.pythonhosted.org/packages/dc/97/525b11d7e3c3a286d83dc138f4e4ef948307338527b442be8fa3b1b379d7/connectorx-0.4.5-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:cc01ca122f649e62707f49f7220ba1ae67961b260e2dcff9e8647ea9915a01cf", size = 43757296, upload-time = "2026-01-18T01:31:17.503Z" }, + { url = "https://files.pythonhosted.org/packages/57/3a/5e1a7cb3b0175c249c232f487918494ec7e26c13f3d543a55c8c7752b993/connectorx-0.4.5-cp314-none-win_amd64.whl", hash = "sha256:2073970532a8e6e2a8a2c0b163497eb8e58216e28fdab6693fcd7e58bfc47bfc", size = 34641445, upload-time = "2026-01-18T01:59:44.351Z" }, +] + [[package]] name = "cryptography" version = "46.0.5" @@ -219,20 +241,23 @@ dependencies = [ { name = "typer" }, ] +[package.optional-dependencies] +sql = [ + { name = "connectorx" }, +] + [[package]] name = "datam8-sample-solution" version = "2.1.0" source = { virtual = "." } dependencies = [ - { name = "datam8" }, - { name = "pyodbc" }, + { name = "datam8", extra = ["sql"] }, { name = "pyyaml" }, ] [package.metadata] requires-dist = [ - { name = "datam8", git = "https://github.com/oraylis/datam8-cli?rev=85ff50de3c7fc8e20abd4edd021d63df02c31b91" }, - { name = "pyodbc", specifier = "==5.3.0" }, + { name = "datam8", extras = ["sql"], git = "https://github.com/oraylis/datam8-cli?rev=85ff50de3c7fc8e20abd4edd021d63df02c31b91" }, { name = "pyyaml", specifier = "==6.0.3" }, ] @@ -661,50 +686,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] -[[package]] -name = "pyodbc" -version = "5.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8f/85/44b10070a769a56bd910009bb185c0c0a82daff8d567cd1a116d7d730c7d/pyodbc-5.3.0.tar.gz", hash = "sha256:2fe0e063d8fb66efd0ac6dc39236c4de1a45f17c33eaded0d553d21c199f4d05", size = 121770, upload-time = "2025-10-17T18:04:09.43Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/0c/7ecf8077f4b932a5d25896699ff5c394ffc2a880a9c2c284d6a3e6ea5949/pyodbc-5.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ebf6b5d989395efe722b02b010cb9815698a4d681921bf5db1c0e1195ac1bde", size = 72994, upload-time = "2025-10-17T18:03:20.551Z" }, - { url = "https://files.pythonhosted.org/packages/03/78/9fbde156055d88c1ef3487534281a5b1479ee7a2f958a7e90714968749ac/pyodbc-5.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:197bb6ddafe356a916b8ee1b8752009057fce58e216e887e2174b24c7ab99269", size = 72535, upload-time = "2025-10-17T18:03:21.423Z" }, - { url = "https://files.pythonhosted.org/packages/9f/f9/8c106dcd6946e95fee0da0f1ba58cd90eb872eebe8968996a2ea1f7ac3c1/pyodbc-5.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6ccb5315ec9e081f5cbd66f36acbc820ad172b8fa3736cf7f993cdf69bd8a96", size = 333565, upload-time = "2025-10-17T18:03:22.695Z" }, - { url = "https://files.pythonhosted.org/packages/4b/30/2c70f47a76a4fafa308d148f786aeb35a4d67a01d41002f1065b465d9994/pyodbc-5.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5dd3d5e469f89a3112cf8b0658c43108a4712fad65e576071e4dd44d2bd763c7", size = 340283, upload-time = "2025-10-17T18:03:23.691Z" }, - { url = "https://files.pythonhosted.org/packages/7d/b2/0631d84731606bfe40d3b03a436b80cbd16b63b022c7b13444fb30761ca8/pyodbc-5.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b180bc5e49b74fd40a24ef5b0fe143d0c234ac1506febe810d7434bf47cb925b", size = 1302767, upload-time = "2025-10-17T18:03:25.311Z" }, - { url = "https://files.pythonhosted.org/packages/74/b9/707c5314cca9401081b3757301241c167a94ba91b4bd55c8fa591bf35a4a/pyodbc-5.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e3c39de3005fff3ae79246f952720d44affc6756b4b85398da4c5ea76bf8f506", size = 1361251, upload-time = "2025-10-17T18:03:26.538Z" }, - { url = "https://files.pythonhosted.org/packages/97/7c/893036c8b0c8d359082a56efdaa64358a38dda993124162c3faa35d1924d/pyodbc-5.3.0-cp312-cp312-win32.whl", hash = "sha256:d32c3259762bef440707098010035bbc83d1c73d81a434018ab8c688158bd3bb", size = 63413, upload-time = "2025-10-17T18:03:27.903Z" }, - { url = "https://files.pythonhosted.org/packages/c0/70/5e61b216cc13c7f833ef87f4cdeab253a7873f8709253f5076e9bb16c1b3/pyodbc-5.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:fe77eb9dcca5fc1300c9121f81040cc9011d28cff383e2c35416e9ec06d4bc95", size = 70133, upload-time = "2025-10-17T18:03:28.746Z" }, - { url = "https://files.pythonhosted.org/packages/aa/85/e7d0629c9714a85eb4f85d21602ce6d8a1ec0f313fde8017990cf913e3b4/pyodbc-5.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:afe7c4ac555a8d10a36234788fc6cfc22a86ce37fc5ba88a1f75b3e6696665dc", size = 64700, upload-time = "2025-10-17T18:03:29.638Z" }, - { url = "https://files.pythonhosted.org/packages/0c/1d/9e74cbcc1d4878553eadfd59138364b38656369eb58f7e5b42fb344c0ce7/pyodbc-5.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7e9ab0b91de28a5ab838ac4db0253d7cc8ce2452efe4ad92ee6a57b922bf0c24", size = 72975, upload-time = "2025-10-17T18:03:30.466Z" }, - { url = "https://files.pythonhosted.org/packages/37/c7/27d83f91b3144d3e275b5b387f0564b161ddbc4ce1b72bb3b3653e7f4f7a/pyodbc-5.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6132554ffbd7910524d643f13ce17f4a72f3a6824b0adef4e9a7f66efac96350", size = 72541, upload-time = "2025-10-17T18:03:31.348Z" }, - { url = "https://files.pythonhosted.org/packages/1b/33/2bb24e7fc95e98a7b11ea5ad1f256412de35d2e9cc339be198258c1d9a76/pyodbc-5.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1629af4706e9228d79dabb4863c11cceb22a6dab90700db0ef449074f0150c0d", size = 343287, upload-time = "2025-10-17T18:03:32.287Z" }, - { url = "https://files.pythonhosted.org/packages/fa/24/88cde8b6dc07a93a92b6c15520a947db24f55db7bd8b09e85956642b7cf3/pyodbc-5.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ceaed87ba2ea848c11223f66f629ef121f6ebe621f605cde9cfdee4fd9f4b68", size = 350094, upload-time = "2025-10-17T18:03:33.336Z" }, - { url = "https://files.pythonhosted.org/packages/c2/99/53c08562bc171a618fa1699297164f8885e66cde38c3b30f454730d0c488/pyodbc-5.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3cc472c8ae2feea5b4512e23b56e2b093d64f7cbc4b970af51da488429ff7818", size = 1301029, upload-time = "2025-10-17T18:03:34.561Z" }, - { url = "https://files.pythonhosted.org/packages/d8/10/68a0b5549876d4b53ba4c46eed2a7aca32d589624ed60beef5bd7382619e/pyodbc-5.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c79df54bbc25bce9f2d87094e7b39089c28428df5443d1902b0cc5f43fd2da6f", size = 1361420, upload-time = "2025-10-17T18:03:35.958Z" }, - { url = "https://files.pythonhosted.org/packages/41/0f/9dfe4987283ffcb981c49a002f0339d669215eb4a3fe4ee4e14537c52852/pyodbc-5.3.0-cp313-cp313-win32.whl", hash = "sha256:c2eb0b08e24fe5c40c7ebe9240c5d3bd2f18cd5617229acee4b0a0484dc226f2", size = 63399, upload-time = "2025-10-17T18:03:36.931Z" }, - { url = "https://files.pythonhosted.org/packages/56/03/15dcefe549d3888b649652af7cca36eda97c12b6196d92937ca6d11306e9/pyodbc-5.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:01166162149adf2b8a6dc21a212718f205cabbbdff4047dc0c415af3fd85867e", size = 70133, upload-time = "2025-10-17T18:03:38.47Z" }, - { url = "https://files.pythonhosted.org/packages/c4/c1/c8b128ae59a14ecc8510e9b499208e342795aecc3af4c3874805c720b8db/pyodbc-5.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:363311bd40320b4a61454bebf7c38b243cd67c762ed0f8a5219de3ec90c96353", size = 64683, upload-time = "2025-10-17T18:03:39.68Z" }, - { url = "https://files.pythonhosted.org/packages/ab/f2/c26d82a7ce1e90b8bbb8731d3d53de73814e2f6606b9db9d978303aa8d5f/pyodbc-5.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:3f1bdb3ce6480a17afaaef4b5242b356d4997a872f39e96f015cabef00613797", size = 73513, upload-time = "2025-10-17T18:03:40.536Z" }, - { url = "https://files.pythonhosted.org/packages/82/d5/1ab1b7c4708cbd701990a8f7183c5bb5e0712d5e8479b919934e46dadab4/pyodbc-5.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:7713c740a10f33df3cb08f49a023b7e1e25de0c7c99650876bbe717bc95ee780", size = 72631, upload-time = "2025-10-17T18:03:41.713Z" }, - { url = "https://files.pythonhosted.org/packages/b1/f1/7e3831eeac2b09b31a77e6b3495491ce162035ff2903d7261b49d35aa3c2/pyodbc-5.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cf18797a12e70474e1b7f5027deeeccea816372497e3ff2d46b15bec2d18a0cc", size = 344580, upload-time = "2025-10-17T18:03:42.67Z" }, - { url = "https://files.pythonhosted.org/packages/a2/a6/71d26d626a3c45951620b7ff356ec920e420f0e09b0a924123682aa5e4ab/pyodbc-5.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:08b2439500e212625471d32f8fde418075a5ddec556e095e5a4ba56d61df2dc6", size = 350224, upload-time = "2025-10-17T18:03:43.731Z" }, - { url = "https://files.pythonhosted.org/packages/93/14/f702c5e8c2d595776266934498505f11b7f1545baf21ffec1d32c258e9d3/pyodbc-5.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:729c535341bb09c476f219d6f7ab194bcb683c4a0a368010f1cb821a35136f05", size = 1301503, upload-time = "2025-10-17T18:03:45.013Z" }, - { url = "https://files.pythonhosted.org/packages/d9/b2/ad92ebdd1b5c7fec36b065e586d1d34b57881e17ba5beec5c705f1031058/pyodbc-5.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c67e7f2ce649155ea89beb54d3b42d83770488f025cf3b6f39ca82e9c598a02e", size = 1361050, upload-time = "2025-10-17T18:03:46.298Z" }, - { url = "https://files.pythonhosted.org/packages/19/40/dc84e232da07056cb5aaaf5f759ba4c874bc12f37569f7f1670fc71e7ae1/pyodbc-5.3.0-cp314-cp314-win32.whl", hash = "sha256:a48d731432abaee5256ed6a19a3e1528b8881f9cb25cb9cf72d8318146ea991b", size = 65670, upload-time = "2025-10-17T18:03:56.414Z" }, - { url = "https://files.pythonhosted.org/packages/b8/79/c48be07e8634f764662d7a279ac204f93d64172162dbf90f215e2398b0bd/pyodbc-5.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:58635a1cc859d5af3f878c85910e5d7228fe5c406d4571bffcdd281375a54b39", size = 72177, upload-time = "2025-10-17T18:03:57.296Z" }, - { url = "https://files.pythonhosted.org/packages/fc/79/e304574446b2263f428ce14df590ba52c2e0e0205e8d34b235b582b7d57e/pyodbc-5.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:754d052030d00c3ac38da09ceb9f3e240e8dd1c11da8906f482d5419c65b9ef5", size = 66668, upload-time = "2025-10-17T18:03:58.174Z" }, - { url = "https://files.pythonhosted.org/packages/43/17/f4eabf443b838a2728773554017d08eee3aca353102934a7e3ba96fb0e31/pyodbc-5.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:f927b440c38ade1668f0da64047ffd20ec34e32d817f9a60d07553301324b364", size = 75780, upload-time = "2025-10-17T18:03:47.273Z" }, - { url = "https://files.pythonhosted.org/packages/59/ea/e79e168c3d38c27d59d5d96273fd9e3c3ba55937cc944c4e60618f51de90/pyodbc-5.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:25c4cfb2c08e77bc6e82f666d7acd52f0e52a0401b1876e60f03c73c3b8aedc0", size = 75503, upload-time = "2025-10-17T18:03:48.171Z" }, - { url = "https://files.pythonhosted.org/packages/90/81/d1d7c125ec4a20e83fdc28e119b8321192b2bd694f432cf63e1199b2b929/pyodbc-5.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc834567c2990584b9726cba365834d039380c9dbbcef3030ddeb00c6541b943", size = 398356, upload-time = "2025-10-17T18:03:49.131Z" }, - { url = "https://files.pythonhosted.org/packages/5e/fc/f6be4b3cc3910f8c2aba37aa41671121fd6f37b402ae0fefe53a70ac7cd5/pyodbc-5.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8339d3094858893c1a68ee1af93efc4dff18b8b65de54d99104b99af6306320d", size = 397291, upload-time = "2025-10-17T18:03:50.18Z" }, - { url = "https://files.pythonhosted.org/packages/03/2e/0610b1ed05a5625528d52f6cece9610e84617d35f475c89c2a52f66d13f7/pyodbc-5.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74528fe148980d0c735c0ebb4a4dc74643ac4574337c43c1006ac4d09593f92d", size = 1353900, upload-time = "2025-10-17T18:03:51.339Z" }, - { url = "https://files.pythonhosted.org/packages/1d/f1/43497e1d37f9f71b43b2b3172e7b1bdf50851e278390c3fb6b46a3630c53/pyodbc-5.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d89a7f2e24227150c13be8164774b7e1f9678321a4248f1356a465b9cc17d31e", size = 1406062, upload-time = "2025-10-17T18:03:52.546Z" }, - { url = "https://files.pythonhosted.org/packages/9e/8b/88a1277c2f7d9ab1cec0a71e074ba24fd4a1710a43974682546da90a1343/pyodbc-5.3.0-cp314-cp314t-win32.whl", hash = "sha256:af4d8c9842fc4a6360c31c35508d6594d5a3b39922f61b282c2b4c9d9da99514", size = 70132, upload-time = "2025-10-17T18:03:53.715Z" }, - { url = "https://files.pythonhosted.org/packages/ba/c7/ee98c62050de4aa8bafb6eb1e11b95e0b0c898bd5930137c6dc776e06a9b/pyodbc-5.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:bfeb3e34795d53b7d37e66dd54891d4f9c13a3889a8f5fe9640e56a82d770955", size = 79452, upload-time = "2025-10-17T18:03:54.664Z" }, - { url = "https://files.pythonhosted.org/packages/4b/8f/d8889efd96bbe8e5d43ff9701f6b1565a8e09c3e1f58c388d550724f777b/pyodbc-5.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:13656184faa3f2d5c6f19b701b8f247342ed581484f58bf39af7315c054e69db", size = 70142, upload-time = "2025-10-17T18:03:55.551Z" }, -] - [[package]] name = "pywin32-ctypes" version = "0.2.3" From 41eea0a1252449a71c542426a293dd4b746460d0 Mon Sep 17 00:00:00 2001 From: Fabio Kayser Date: Thu, 5 Nov 2026 08:36:30 +0100 Subject: [PATCH 2/6] docs: add task-oriented entrypoints and clear usage boundaries --- AGENTS.md | 6 +- Generate/databricks-lake/DEPLOYMENT.md | 16 ++-- Generate/databricks-lake/README.md | 16 ++-- README.md | 73 ++++++++++++++----- docs/documentation-handoff.md | 8 +- docs/getting-started.md | 31 ++++---- docs/index.md | 26 ++++--- docs/integration-tests.md | 38 +++++++++- docs/model-properties.md | 1 - docs/quickstart.md | 17 ++++- docs/reference/targets.md | 5 +- ...solutions-cleanup.md => sample-cleanup.md} | 14 ++-- docs/template-development.md | 4 +- 13 files changed, 176 insertions(+), 79 deletions(-) rename docs/{solutions-cleanup.md => sample-cleanup.md} (69%) diff --git a/AGENTS.md b/AGENTS.md index 0e005b5..c96b4f9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,12 +1,12 @@ # DataM8 public sample This repository owns a small AdventureWorks Base/Model example, business functions and the -classic Databricks, Docs and Power BI generation references. Large multi-source models, Lakeflow, Fabric and -all cloud deployment/integration belong to the private datam8-solutions repo. +classic Databricks, Docs and Power BI generation references. Only local generation +is included; the repository has no cloud deployment or integration runner. - Preserve unrelated local changes. Never commit installed plugins or credentials. - Do not add live workspace names, IDs, hosts, catalog or principal bindings. - The migration is awaiting manual review: do not run tests, CI checks, cloud deployments or integration until the user explicitly approves the exact commit. - Regeneration and manual inspection for the review are allowed. -- The maintainer merges feature branches manually; Solutions must merge first. +- The maintainer merges feature branches manually. diff --git a/Generate/databricks-lake/DEPLOYMENT.md b/Generate/databricks-lake/DEPLOYMENT.md index bff7e23..c2a5e07 100644 --- a/Generate/databricks-lake/DEPLOYMENT.md +++ b/Generate/databricks-lake/DEPLOYMENT.md @@ -1,7 +1,11 @@ -# Classic target: generation only +# Classic Target: Generation Only -This public target renders notebooks and resource definitions. Environment-bound -bundle entrypoints and cloud execution live in the private -[Solutions repository](https://github.com/oraylis/datam8-solutions). -The utilities retained under Output are neutral runtime source assets required -by generated references, not deployment configuration. +This target renders notebooks and resource definitions. It contains no +environment-bound bundle entrypoint or cloud execution workflow. Configure your +own deployment process after reviewing platform, source and identity prerequisites; +generated files are not deployment evidence. + +The utilities retained under `Output/databricks-lake/utils` are neutral runtime +source assets required by generated references, not deployment configuration. +Keep them separate from cleaned output. See [local generation](../../docs/getting-started.md) +and [deployment boundaries](../../docs/quickstart.md). diff --git a/Generate/databricks-lake/README.md b/Generate/databricks-lake/README.md index b7d0acb..c77cf90 100644 --- a/Generate/databricks-lake/README.md +++ b/Generate/databricks-lake/README.md @@ -86,7 +86,6 @@ beneath it: ```text Output/databricks-lake/ - databricks.yml # Maintained bundle entry point utils/ # Shared runtime notebooks/frameworks generated/ notebooks//ddl///.py @@ -104,19 +103,18 @@ The sample therefore generates `load_sales_daily.yml` and assignments and the other target-specific properties. The exact notebook subdirectories follow the model's folder hierarchy. Cluster -settings are emitted when cluster metadata is configured. The existing -[bundle entry point](https://github.com/oraylis/datam8-solutions/tree/main/deployment) includes the -generated resources and the shared `utils` directory. Maintain those deployment -files alongside the generated output. See [deployment and integration](DEPLOYMENT.md) -for the configured test environments and local run command. +settings are emitted when cluster metadata is configured. Generated notebooks +reference the authored `utils` directory. The sample contains no environment-bound +bundle entrypoint; review [deployment boundaries](DEPLOYMENT.md) before configuring +your own platform delivery process. ## Generate and use the result -Use **DataM8 app 2.0.0**, select **databricks** in the generator and choose -**Generate**. Or run **DataM8 CLI 2.0.0** from the solution root: +Use a compatible DataM8 app, select **databricks** in the generator and choose +**Generate**. Or use the pinned CLI from the solution root: ```sh -datam8 generate databricks --solution ./ORAYLISDatabricksSample.dm8s +uv run datam8 generate databricks --solution ./ORAYLISDatabricksSample.dm8s ``` Review the notebooks, job graphs and generated paths. Configure your workspace, diff --git a/README.md b/README.md index c00902b..848d058 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,41 @@ # DataM8 Sample Solution -This public repository owns the small AdventureWorks metadata model and its -business transformations. It supports local generation without cloud credentials. +Learn metadata-driven generation with a small AdventureWorks customer, address +and order model. Open it in the DataM8 app or use the CLI to inspect and edit +metadata, then generate model documentation, Databricks notebooks or a Power BI +semantic model. The first exercise runs locally without a database or cloud account. + +## Start with your task + +| I want to… | Begin here | First result | +| --- | --- | --- | +| Learn visual modeling | [Open the sample in the app](docs/getting-started.md#use-the-app) | Change a Customer description and confirm it survives reload | +| Validate and generate from the command line | [Local CLI setup](docs/getting-started.md#use-the-cli) | Generate and inspect a fresh `Output/docs/index.md` | +| Change templates or target conventions | [Template development](docs/template-development.md) and [configured targets](docs/reference/targets.md) | Change one template and compare the generated output | +| Understand the example | [Model documentation target](Generate/docs/README.md) and [property references](docs/model-properties.md) | Follow Customer through Stage, Core, Curated and Consumer | + +**Recommended first result:** generate `docs` and open its index. Use a disposable +copy: autosave writes metadata, and generation can replace a target's output. +Install the [stable app or CLI](https://github.com/oraylis/datam8/blob/main/docs/installation.md) +for product use; the local project below pins a public CLI revision for reproducibility. + +## Try the local CLI path + +Clone or download the repository, copy the complete solution to an exercise +directory, and run these commands there with uv and a compatible Python: + +```powershell +uv sync --frozen +uv run datam8 validate --solution ./ORAYLISDatabricksSample.dm8s +uv run datam8 generate docs --solution ./ORAYLISDatabricksSample.dm8s +``` + +Open `Output/docs/index.md`; it should contain Customer and links to entity pages +and a relationship diagram. The SQL Server metadata connector is bundled with +the pinned CLI and installed through its `sql` extra. No connector checkout, +download token or source connection is needed for this exercise. + +## What is included | Target | Source | Output | | --- | --- | --- | @@ -9,23 +43,26 @@ business transformations. It supports local generation without cloud credentials | powerbi | `Generate/powerbi-tabular` | `Output/powerbi-tabular/generated` | | docs | `Generate/docs` | `Output/docs` | -The large multi-source models, Lakeflow, Fabric, deployment bindings and cloud integration belong to -[DataM8 Solutions](https://github.com/oraylis/datam8-solutions). That repository -is private; its links require the corresponding access. The classic target here -remains a generate-only reference. No cloud resources are removed by this cleanup. +The model follows AdventureWorks customers, addresses and orders through Stage, +Core and Curated, including DimCustomer, DimDate and FactSalesOrder. Consumer +entities retain Customer, Date, Sales Order and Total Sales. The SQL example +uses the standard SalesLT.Customer table; no custom language-specific views are +required. Data endpoints and secret references are neutral placeholders. -Use the pinned CLI with `uv sync --frozen` and -`uv run datam8 generate -s ORAYLISDatabricksSample.dm8s databricks --clean-output`. -The Generate-only workflow retains offline target and documentation checks. +This is a generate-only example. Deployment bindings, cloud resources and live +execution are not included. Generating notebooks or TMDL does not demonstrate +successful deployment or processing. See [output boundaries](docs/quickstart.md). -All source endpoints and credential references must be neutral examples. Never -commit live workspace names/IDs, hosts, warehouse IDs, principals or credentials. -See [the cleanup handoff](docs/solutions-cleanup.md) and [security](SECURITY.md). +## Learn more and contribute -Tests are intentionally deferred until the user reviews both migration PRs. +- [Sample documentation](docs/index.md) +- [Central illustrated handbook](https://github.com/oraylis/datam8/blob/main/docs/user-guide/README.md) +- [Local checks and review requirements](docs/integration-tests.md) +- [Sample cleanup history](docs/sample-cleanup.md) +- [Source and credential policy](SECURITY.md) -The model follows customers, addresses and orders through Stage, Core and Curated, -including DimCustomer, DimDate and FactSalesOrder. The consumer layer retains -Customer, Date, Sales Order and Total Sales. Product/procurement/inventory/commerce/ -shipping examples belong to Solutions. Customer reads the standard SalesLT.Customer -table; no Customer_DE/Customer_EN source views are required. +Follow [repository instructions](AGENTS.md) and the exact-commit review boundary +before running tests or CI checks. Propose changes through a feature PR; the +maintainer merges manually. Report product issues in +[DataM8](https://github.com/oraylis/datam8/issues), with no credentials or live +environment details. diff --git a/docs/documentation-handoff.md b/docs/documentation-handoff.md index 6ca005c..9f7d44a 100644 --- a/docs/documentation-handoff.md +++ b/docs/documentation-handoff.md @@ -1,4 +1,8 @@ -# Sample Documentation Handoff +# Historical Sample Documentation Handoff + +This page records the original documentation transfer. Publication and branch +statements below are historical; current contributor checks follow +[repository instructions](../AGENTS.md). This is a documentation-only package on the agreed feature branch. No product API, schema JSON, dependency pin or target implementation is changed here. @@ -6,7 +10,7 @@ schema JSON, dependency pin or target implementation is changed here. Run `node --test scripts/check-docs.test.mjs` and `node scripts/check-docs.mjs` with Node 24. CI runs those checks without a frontend or sample checkout. -Read the [central handoff](https://github.com/oraylis/datam8/blob/codex/documentation-v2/docs/handoff.md) for branch/workspace mapping, +Read the [central handoff](https://github.com/oraylis/datam8/blob/main/docs/handoff.md) for branch/workspace mapping, results and manual merge order. The new central and component pages are local until their branches are published. After manual merges, update development links to main; keep tutorial evidence bound to commits. Public user workflows belong in the handbook. diff --git a/docs/getting-started.md b/docs/getting-started.md index dd89bd1..25562bf 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,10 +1,10 @@ -# Getting Started with DataM8 2.0.0 +# Get Started with the Public Sample Choose how you want to work with the sample: -- **DataM8 app 2.0.0:** create and edit model entities, then generate output in the +- **DataM8 app:** create and edit model entities, then generate output in the application. The app includes the generator and its runtime. -- **DataM8 CLI 2.0.0:** edit the model and templates in your preferred editor, then +- **DataM8 CLI:** edit the model and templates in your preferred editor, then validate and generate from the command line. ## Get the sample @@ -22,10 +22,11 @@ supporting configuration alongside `ORAYLISDatabricksSample.dm8s`. Copy the whol solution when creating another exercise directory. Autosave writes your edits to disk, and generation can replace files in a target output directory. -## Option 1: Use the DataM8 app +## Use the app -Install **DataM8 app 2.0.0** from the -[desktop release](https://github.com/oraylis/datam8-frontend/releases/tag/v2.0.0). +Install the app from the +[current stable desktop release](https://github.com/oraylis/datam8-frontend/releases/latest), +following its [installation and verification guide](https://github.com/oraylis/datam8-frontend/blob/main/docs/desktop-installation.md). The app bundles generator functionality; you can work with the model and generate output without installing Python or the CLI separately. @@ -38,25 +39,27 @@ output without installing Python or the CLI separately. 5. Open **Toggle generator**, select **docs**, and choose **Generate**. 6. Open `Output/docs/index.md` and inspect the generated model documentation. -Follow the [illustrated user guide](https://github.com/oraylis/datam8/blob/codex/documentation-v2/docs/user-guide/README.md) +Follow the [illustrated user guide](https://github.com/oraylis/datam8/blob/main/docs/user-guide/README.md) for the model editors, relationships, sources and generation workflow. -## Option 2: Use the DataM8 CLI +## Use the CLI -Use Python 3.13 or newer and uv. This sample pins the verified CLI 2.1.0 -source revision and model dependency in its project and lockfile. +Use uv and a Python version supported by [the project](../pyproject.toml). The +lockfile pins a public CLI revision and its SQL extra for `builtin:SQLServer`. +The app has its own bundled runtime; consult its release compatibility notes. From the local sample checkout: ```powershell uv sync --frozen -uv run python scripts/install_plugins.py uv run datam8 validate --solution ./ORAYLISDatabricksSample.dm8s uv run datam8 generate docs --solution ./ORAYLISDatabricksSample.dm8s ``` -The CLI operates directly on the persisted local solution. Installed plugins -remain local dependencies. See [integration tests](integration-tests.md) for -exact revisions, checksums, target selection and deployment commands. +The CLI operates directly on the persisted local solution. No separately +installed solution connector or repository-access token is required. See +[local checks](integration-tests.md) for the exact-commit contributor review +boundary. Source import and preview need your own source settings; local +metadata validation and generation do not need a source connection. Check the exit codes, then open `Output/docs/index.md`. It should contain Customer and links to entity pages and the relationship diagram. Edit the model or templates diff --git a/docs/index.md b/docs/index.md index 8cfe758..401b79c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,20 +1,22 @@ # Sample Documentation -Start with **DataM8 app 2.0.0**, which bundles the generator, or install -**the pinned DataM8 CLI 2.1.0** for command-line validation and generation. +Start with [the local exercise](getting-started.md): open the sample in the app or +use the pinned public CLI, change metadata and generate documentation. A source +connection or cloud account is not required for that path. - [Local getting started](getting-started.md) - [Configured targets and output paths](reference/targets.md) - [Model properties by target](model-properties.md) -- [Databricks: layered lakehouse, notebooks and jobs](../Generate/databricks-lake/README.md) -- [Databricks Lakeflow: ingestion and modeled processing](https://github.com/oraylis/datam8-solutions/tree/main/databricks/Generate/databricks-lakeflow/README.md) -- [Power BI: consumer semantic model](../Generate/powerbi-tabular/README.md) -- [Documentation: model catalog and relationship diagram](../Generate/docs/README.md) +- [Databricks notebooks and job definitions](../Generate/databricks-lake/README.md) +- [Power BI consumer semantic model](../Generate/powerbi-tabular/README.md) +- [Documentation catalog and relationship diagram](../Generate/docs/README.md) - [Template and payload development](template-development.md) -- [Optional deployment preparation](quickstart.md) -- [Integration tests, repeat checks and resume](integration-tests.md) -- [Documentation handoff](documentation-handoff.md) -- [Central user handbook](https://github.com/oraylis/datam8/blob/codex/documentation-v2/docs/user-guide/README.md) +- [Generation and deployment boundaries](quickstart.md) +- [Local checks and review requirements](integration-tests.md) +- [Historical documentation handoff](documentation-handoff.md) +- [Sample cleanup history](sample-cleanup.md) +- [Central user handbook](https://github.com/oraylis/datam8/blob/main/docs/user-guide/README.md) -The getting-started guide covers both routes. Explore and edit the model, then -generate documentation locally before setting up source connections or deployment. +General app installation follows the stable release. The project and lockfile +identify the sample's pinned CLI. Read technical references for another installed +release at its matching version tag. diff --git a/docs/integration-tests.md b/docs/integration-tests.md index 0742b4a..8a6d97a 100644 --- a/docs/integration-tests.md +++ b/docs/integration-tests.md @@ -1,3 +1,37 @@ -# Deployment and large models moved +# Local Generation Checks and Review -Large platform-specific models and cloud operations belong to [DataM8 Solutions](https://github.com/oraylis/datam8-solutions). This public repository retains only the small AdventureWorks customer/address/order example and local classic/Docs/Power BI generation. +This repository checks the AdventureWorks model and local generation of +Databricks, documentation and Power BI artifacts. It has no cloud deployment or +integration runner. Generated files do not establish deployment acceptance. + +## Review boundary + +Follow [repository instructions](../AGENTS.md). Do not run tests or CI checks for +a changed sample until the user approves its exact commit. The manual workflow +and generation checker require that revision; do not set approval on the user's +behalf. Use an independently assigned test model when required by the review +handoff. Generating artifacts for source review remains separate. + +## After review + +On a disposable complete copy, install the locked public dependencies: + +```powershell +uv sync --frozen +node --test scripts/check-docs.test.mjs +node scripts/check-docs.mjs +uv run python -m unittest discover -s tests +uv run python scripts/check_generation.py --target all +``` + +The approved exact-commit marker must already be supplied for the final command. +The suite covers model references, public source placeholders, SQL Server builtin +forms/metadata and generated reader compatibility. The generation checker validates +all targets, Python syntax, notebook cells, job graphs and required runtime assets. +It does not contact source systems or deploy generated artifacts. + +Public-entrypoint acceptance uses a fresh copy with an empty Plugins folder and +no repository-access token. Validate and generate all targets; inspect newly +written output. SQL fixtures mock database I/O while using the real bundled +connector. Live SQL, native app behavior and cloud execution need separate +authorized acceptance. diff --git a/docs/model-properties.md b/docs/model-properties.md index 83b457f..f0de4b6 100644 --- a/docs/model-properties.md +++ b/docs/model-properties.md @@ -3,7 +3,6 @@ Choose the target reference for property locations, values and generated behavior: - [Databricks Lake](../Generate/databricks-lake/PROPERTIES.md) -- [Databricks Lakeflow](https://github.com/oraylis/datam8-solutions/tree/main/databricks/Generate/databricks-lakeflow/PROPERTIES.md) - [Power BI](../Generate/powerbi-tabular/PROPERTIES.md) - [Documentation target](../Generate/docs/PROPERTIES.md) diff --git a/docs/quickstart.md b/docs/quickstart.md index a7f901a..8fd7a3f 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,3 +1,16 @@ -# Generate-only example +# From Generated Files to Deployment -Start with [local generation](getting-started.md). The public repository has no deployment runner, environment bindings or active Azure pipeline. Cloud deployment and integration belong to the private [Solutions repository](https://github.com/oraylis/datam8-solutions). Generated notebooks/TMDL are not evidence of a running deployment. +Start with [local generation](getting-started.md). This public example contains +metadata and templates, not a deployment runner, environment bindings or an active +Azure pipeline. Generated notebooks and TMDL are files for review, not evidence of +a running data platform. + +Before any deployment, review your platform's workspace, catalog, storage, +identities, source connections and secret integration in your own delivery process. +Keep those bindings outside this sample and preserve the authored runtime utilities +referenced by generated notebooks. Verify platform prerequisites independently. + +The [Databricks boundary](../Generate/databricks-lake/DEPLOYMENT.md) and +[configured outputs](reference/targets.md) identify the artifacts supplied here. +The local exercise creates no cloud resources and starts no data processing. +Never commit live values in place of neutral source examples. diff --git a/docs/reference/targets.md b/docs/reference/targets.md index 3228819..b376ab7 100644 --- a/docs/reference/targets.md +++ b/docs/reference/targets.md @@ -8,5 +8,6 @@ The solution descriptor contains exactly three generate-only targets. | powerbi | `Generate/powerbi-tabular` | `Output/powerbi-tabular/generated` | | docs | `Generate/docs` | `Output/docs` | -Lakeflow and all deployment/integration operations moved to the private -[Solutions repository](https://github.com/oraylis/datam8-solutions). +Each target writes files locally. The sample includes no deployment runner, +environment bindings or live integration workflow. Review the +[deployment boundary](../quickstart.md) before using generated artifacts elsewhere. diff --git a/docs/solutions-cleanup.md b/docs/sample-cleanup.md similarity index 69% rename from docs/solutions-cleanup.md rename to docs/sample-cleanup.md index b21930c..3f3024e 100644 --- a/docs/solutions-cleanup.md +++ b/docs/sample-cleanup.md @@ -1,7 +1,8 @@ -# Solutions migration and sample cleanup +# Sample Cleanup Handoff -Branch: `feature/sample-solutions-cleanup`, based on the sample main revision. -The corresponding Solutions migration must be accepted before this cleanup merges. +This historical handoff records the reduction to a public, generate-only sample. +Its branch-state and review statements describe that cleanup. New revisions +retain their own exact-commit review boundary. Retained: the small AdventureWorks customer/address/order model, its linked Core/Curated entities, classic databricks-lake, Docs, Power BI and Generate-only. Removed: Lakeflow target/bundle, live bundle bindings, cloud integration runner, @@ -14,7 +15,7 @@ No tests have been run for this branch. Before the user's explicit review, Generate-only is dispatch-only and guarded by an approved exact commit setting. The independent test model subsequently checks generation, links, source privacy and the absence of imports of removed integration modules. PR/main triggers are -restored only after that approval. Both branches must be reviewed together. +restored only after that approval. Each new sample revision requires its own explicit review. ## Prior public exposure @@ -27,5 +28,6 @@ and artifacts. This is an inventory, not a completed historical credential audit No history rewrite, artifact deletion, credential rotation or cloud destruction is implicitly performed. Any actual credential exposure requires separate handling. -The full imported model is preserved independently under databricks/ and fabric/ in Solutions. Solutions deployments read their own platform models; they do not fetch this reduced public model. The remaining metadata intentionally contains only -AdventureWorks and the Customer, Address, Order and Calendar dependency closure. +The retained metadata intentionally contains only AdventureWorks and the Customer, +Address, Order and Calendar dependency closure. The target reference documents +the three local generation targets. diff --git a/docs/template-development.md b/docs/template-development.md index e5959d4..1e9683d 100644 --- a/docs/template-development.md +++ b/docs/template-development.md @@ -20,11 +20,11 @@ def documentation_index(model, cache): This excerpt belongs inside the existing module with its imports and builder; it is not a complete standalone plugin. Inspect the backend's -[generate implementation](https://github.com/oraylis/datam8-cli/blob/feature/external_refresh/src/datam8/generate.py) for engine behavior. +[generate implementation](https://github.com/oraylis/datam8-cli/blob/main/src/datam8/generate.py) for engine behavior. ## Change and verify -1. Work on a complete sample copy using DataM8 app 2.0.0 or CLI 2.0.0. +1. Work on a complete sample copy using a compatible app or the pinned CLI. 2. Add/shape a field in Python and update its Jinja consumer. 3. Generate that target in the app or with the [CLI](getting-started.md). 4. Assert new output content, links, ordering and paths; compare unchanged output From 48e7c84e2883b96da354681661dfcdcb583cc631 Mon Sep 17 00:00:00 2001 From: Fabio Kayser Date: Thu, 5 Nov 2026 09:21:29 +0100 Subject: [PATCH 3/6] fix(sample): align SQL builtin forms and offline secret fixture --- Base/DataSourceTypes.json | 2 +- tests/test_public_entrypoint.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Base/DataSourceTypes.json b/Base/DataSourceTypes.json index 91036fc..a36eb35 100644 --- a/Base/DataSourceTypes.json +++ b/Base/DataSourceTypes.json @@ -166,7 +166,7 @@ { "name": "trust_server_certificate_ca", "required": false, - "type": "string", + "type": "boolean", "description": "Path to a root ca to verify the server certificate against" }, { diff --git a/tests/test_public_entrypoint.py b/tests/test_public_entrypoint.py index e345b7d..42a53ca 100644 --- a/tests/test_public_entrypoint.py +++ b/tests/test_public_entrypoint.py @@ -36,8 +36,11 @@ def test_saved_connection_forms_match_the_public_builtin(self): def test_real_builtin_metadata_preserves_source_types_and_sample_mappings(self): sources = json.loads((ROOT / "Base/DataSources.json").read_text()) source = DataSource.model_validate(sources["dataSources"][0]) - source.extendedProperties["password"] = "local-fixture-only" - plugin = self.plugin_class(self.plugin_class.manifest(), source, self.source_type) + source.extendedProperties["password"] = "fixture:sql-password" + with patch("datam8.plugins.base.SecretResolver") as secrets: + secrets.return_value.get_secret.return_value = "local-fixture-only" + plugin = self.plugin_class(self.plugin_class.manifest(), source, self.source_type) + secrets.return_value.get_secret.assert_called_once_with("fixture:sql-password") rows = pl.DataFrame({ "ordinal": [1, 2], "name": ["CustomerID", "Amount"], "dataType": ["int", "decimal"], "maxLength": [None, None], From a8aaf3b07990024d821d3316f43c5b9563de3a32 Mon Sep 17 00:00:00 2001 From: Fabio Kayser Date: Thu, 5 Nov 2026 09:21:29 +0100 Subject: [PATCH 4/6] chore: allow local verification without migration approval gates --- .github/workflows/generation.yml | 10 ++++------ AGENTS.md | 32 ++++++++++++++++++++++---------- CLAUDE.md | 3 ++- README.md | 4 ++-- docs/getting-started.md | 3 +-- docs/integration-tests.md | 31 +++++++++++++++---------------- docs/sample-cleanup.md | 11 ++++++----- scripts/check_generation.py | 4 ---- 8 files changed, 52 insertions(+), 46 deletions(-) diff --git a/.github/workflows/generation.yml b/.github/workflows/generation.yml index 12b8f50..7dff892 100644 --- a/.github/workflows/generation.yml +++ b/.github/workflows/generation.yml @@ -1,6 +1,10 @@ name: Generate sample targets on: + pull_request: + branches: [main] + push: + branches: [main] workflow_dispatch: inputs: target: @@ -15,14 +19,8 @@ permissions: jobs: generate: runs-on: ubuntu-24.04 - env: - SAMPLE_REVIEW_COMMIT: ${{ vars.SAMPLE_REVIEW_COMMIT }} steps: - uses: actions/checkout@v6 - - name: Require manual review before any checks - env: - APPROVED_COMMIT: ${{ vars.SAMPLE_REVIEW_COMMIT }} - run: '[[ "$APPROVED_COMMIT" == "$GITHUB_SHA" ]] || { echo "Manual review required"; exit 1; }' - uses: actions/setup-node@v6 with: node-version: 24 diff --git a/AGENTS.md b/AGENTS.md index c96b4f9..540ba2e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,12 +1,24 @@ -# DataM8 public sample +# Working in the DataM8 Public Sample -This repository owns a small AdventureWorks Base/Model example, business functions and the -classic Databricks, Docs and Power BI generation references. Only local generation -is included; the repository has no cloud deployment or integration runner. +This repository owns the small AdventureWorks Base/Model example, business +functions and classic Databricks, Docs and Power BI generation references. +It supports local generation; no cloud deployment or integration runner is included. -- Preserve unrelated local changes. Never commit installed plugins or credentials. -- Do not add live workspace names, IDs, hosts, catalog or principal bindings. -- The migration is awaiting manual review: do not run tests, CI checks, cloud - deployments or integration until the user explicitly approves the exact commit. -- Regeneration and manual inspection for the review are allowed. -- The maintainer merges feature branches manually. +- Start with [getting started](docs/getting-started.md) and the + [local checks](docs/integration-tests.md). Read the affected target's reference + before changing its templates or payloads. +- Preserve the model's dependency closure, saved type mappings and three targets. + Schema and CLI APIs remain canonical upstream; do not patch them here. +- Use disposable complete copies for editing and generation. Output can be replaced. + Use local fixtures for SQL metadata tests; do not contact a live source. +- Continue authorized local edits, dependency setup, tests and generation without + a separate commit approval or mandatory model handoff. +- Never commit installed plugins, credentials or live workspace/source bindings. + Public entry points and dependencies must work without private repository access. +- With Node 24 and the Python version in `pyproject.toml`, run: + `uv sync --frozen`, `node --test scripts/check-docs.test.mjs`, + `node scripts/check-docs.mjs`, `uv run python -m unittest discover -s tests` + and `uv run python scripts/check_generation.py --target all`. +- Report passes, failures and skips separately. Local generation is not evidence + of live source, app or cloud acceptance. Live work requires separate authorization. +- Keep unrelated changes intact and propose feature PRs; the maintainer merges manually. diff --git a/CLAUDE.md b/CLAUDE.md index e9c5806..92979fa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,3 +1,4 @@ # Working rules -Read [AGENTS.md](AGENTS.md). This public sample is generate-only. The two migration branches await manual review before any tests or deployment work. +Read [AGENTS.md](AGENTS.md) for repository ownership, local verification commands +and cloud/source boundaries. This public sample supports local generation only. diff --git a/README.md b/README.md index 848d058..3e13a19 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,8 @@ successful deployment or processing. See [output boundaries](docs/quickstart.md) - [Sample cleanup history](docs/sample-cleanup.md) - [Source and credential policy](SECURITY.md) -Follow [repository instructions](AGENTS.md) and the exact-commit review boundary -before running tests or CI checks. Propose changes through a feature PR; the +Follow [repository instructions](AGENTS.md) and run the relevant local checks +during implementation. Propose changes through a feature PR; the maintainer merges manually. Report product issues in [DataM8](https://github.com/oraylis/datam8/issues), with no credentials or live environment details. diff --git a/docs/getting-started.md b/docs/getting-started.md index 25562bf..b5b71b5 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -57,8 +57,7 @@ uv run datam8 generate docs --solution ./ORAYLISDatabricksSample.dm8s The CLI operates directly on the persisted local solution. No separately installed solution connector or repository-access token is required. See -[local checks](integration-tests.md) for the exact-commit contributor review -boundary. Source import and preview need your own source settings; local +[local checks](integration-tests.md) for contributor verification. Source import and preview need your own source settings; local metadata validation and generation do not need a source connection. Check the exit codes, then open `Output/docs/index.md`. It should contain Customer diff --git a/docs/integration-tests.md b/docs/integration-tests.md index 8a6d97a..e3f9cc3 100644 --- a/docs/integration-tests.md +++ b/docs/integration-tests.md @@ -4,17 +4,11 @@ This repository checks the AdventureWorks model and local generation of Databricks, documentation and Power BI artifacts. It has no cloud deployment or integration runner. Generated files do not establish deployment acceptance. -## Review boundary +## Local checks -Follow [repository instructions](../AGENTS.md). Do not run tests or CI checks for -a changed sample until the user approves its exact commit. The manual workflow -and generation checker require that revision; do not set approval on the user's -behalf. Use an independently assigned test model when required by the review -handoff. Generating artifacts for source review remains separate. - -## After review - -On a disposable complete copy, install the locked public dependencies: +Run these checks during implementation, without a separate commit approval or +mandatory model handoff. Follow [repository instructions](../AGENTS.md) and use +a disposable complete copy with Node 24 and the Python version in the project: ```powershell uv sync --frozen @@ -24,14 +18,19 @@ uv run python -m unittest discover -s tests uv run python scripts/check_generation.py --target all ``` -The approved exact-commit marker must already be supplied for the final command. The suite covers model references, public source placeholders, SQL Server builtin forms/metadata and generated reader compatibility. The generation checker validates all targets, Python syntax, notebook cells, job graphs and required runtime assets. It does not contact source systems or deploy generated artifacts. -Public-entrypoint acceptance uses a fresh copy with an empty Plugins folder and -no repository-access token. Validate and generate all targets; inspect newly -written output. SQL fixtures mock database I/O while using the real bundled -connector. Live SQL, native app behavior and cloud execution need separate -authorized acceptance. +The generation workflow runs on pull requests, pushes to main and manual dispatch. +It uses read-only repository permissions and public locked dependencies; no review +variable, source credential or repository-access token is needed. + +## Public-entrypoint acceptance + +Use a fresh copy with an empty Plugins folder and no repository-access token. +Validate and generate all targets; inspect newly written output. SQL fixtures mock +database I/O while using the real bundled connector. Record passes, failures and +skips separately. Native app behavior, live SQL and cloud execution require their +own authorized acceptance; local checks do not establish those results. diff --git a/docs/sample-cleanup.md b/docs/sample-cleanup.md index 3f3024e..c051005 100644 --- a/docs/sample-cleanup.md +++ b/docs/sample-cleanup.md @@ -1,21 +1,22 @@ # Sample Cleanup Handoff This historical handoff records the reduction to a public, generate-only sample. -Its branch-state and review statements describe that cleanup. New revisions -retain their own exact-commit review boundary. +Its branch-state and review statements describe that cleanup. Current local +verification follows [local checks](integration-tests.md); the historical test +hold below is no longer the working policy. Retained: the small AdventureWorks customer/address/order model, its linked Core/Curated entities, classic databricks-lake, Docs, Power BI and Generate-only. Removed: Lakeflow target/bundle, live bundle bindings, cloud integration runner, integration-only tests and workflows, and the obsolete Azure pipeline. Pure graph/asset checks remain for local generation. -## Manual review boundary +## Historical manual review boundary -No tests have been run for this branch. Before the user's explicit review, +At the original cleanup handoff, no tests had been run for that branch. Before the user's explicit review, Generate-only is dispatch-only and guarded by an approved exact commit setting. The independent test model subsequently checks generation, links, source privacy and the absence of imports of removed integration modules. PR/main triggers are -restored only after that approval. Each new sample revision requires its own explicit review. +restored only after that approval. This section records the former migration process. ## Prior public exposure diff --git a/scripts/check_generation.py b/scripts/check_generation.py index 250e94e..e217969 100644 --- a/scripts/check_generation.py +++ b/scripts/check_generation.py @@ -3,7 +3,6 @@ import argparse import ast import json -import os from pathlib import Path import subprocess import sys @@ -18,9 +17,6 @@ def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument("--target",default="all") args=parser.parse_args() - revision=subprocess.check_output(["git","rev-parse","HEAD"],cwd=ROOT,text=True).strip() - if os.environ.get("SAMPLE_REVIEW_COMMIT") != revision: - parser.error("Explicit manual review of this exact sample commit is required before checks") descriptor=ROOT/"ORAYLISDatabricksSample.dm8s" targets=json.loads(descriptor.read_text())["generatorTargets"] names=[t["name"] for t in targets] From 5a4b22f0d3aaa2c0fd6a75bc887ae420af01ed0d Mon Sep 17 00:00:00 2001 From: Fabio Kayser Date: Thu, 5 Nov 2026 10:17:29 +0100 Subject: [PATCH 5/6] refactor(sample): use the SQL builtin ID directly in source readers --- .../databricks-lake/__modules/metadata_utils.py | 4 ---- Generate/databricks-lake/__modules/payload.py | 7 +++---- Generate/shared/uc_sources.py | 4 ++-- scripts/check_generation.py | 2 +- tests/test_public_entrypoint.py | 9 --------- tests/test_uc_sources.py | 17 ++++++++++++++--- 6 files changed, 20 insertions(+), 23 deletions(-) diff --git a/Generate/databricks-lake/__modules/metadata_utils.py b/Generate/databricks-lake/__modules/metadata_utils.py index c6826ce..0e51713 100644 --- a/Generate/databricks-lake/__modules/metadata_utils.py +++ b/Generate/databricks-lake/__modules/metadata_utils.py @@ -16,10 +16,6 @@ TARGET_NAME = "databricks" -def source_reader_kind(plugin_id: str | None) -> str | None: - """Keep the generated SQL reader contract independent of host plugin IDs.""" - return "sqlserver" if plugin_id == "builtin:SQLServer" else plugin_id - # Common aliases so source / canonical names converge to those defined in DataTypes.json. TYPE_ALIASES: dict[str, str] = { "integer": "int", diff --git a/Generate/databricks-lake/__modules/payload.py b/Generate/databricks-lake/__modules/payload.py index 68f6304..2f869c1 100644 --- a/Generate/databricks-lake/__modules/payload.py +++ b/Generate/databricks-lake/__modules/payload.py @@ -19,7 +19,6 @@ from metadata_utils import ( MetadataResolver, - source_reader_kind, attribute_property_equals, build_business_key_partitions, build_delta_table_properties, @@ -533,8 +532,8 @@ def generate_raw_dml_notebooks(model: Model, cache: Cache) -> Sequence[IPayload] data_source_name=data_source_name, ) source_type = next((item.entity for item in model.dataSourceTypes.values() if item.entity.name == data_source_entry.type), None) - plugin = source_reader_kind(getattr(source_type, "pluginId", None)) - if plugin not in {"sqlserver", "oracle", "mysql", "postgresql", "http_api"}: + plugin = getattr(source_type, "pluginId", None) + if plugin not in {"builtin:SQLServer", "oracle", "mysql", "postgresql", "http_api"}: raise ValueError(f"Unsupported UC source plugin for {data_source_name}: {plugin}") extended = data_source_entry.extendedProperties or {} connection_name = extended.get("unityCatalogConnection", plugin) @@ -833,7 +832,7 @@ def generation_manifest(model: Model, cache: Cache) -> Sequence[IPayload]: raw_key = "raw." + raw["full_table_name"].lower() data_source = resolver.data_sources[external.dataSource] source_type = next(item.entity for item in model.dataSourceTypes.values() if item.entity.name == data_source.type) - plugin = source_reader_kind(source_type.pluginId) + plugin = source_type.pluginId settings = data_source.extendedProperties or {} extract_mode = next((reference.value for reference in external.properties or [] if reference.property == "extract_mode"), "full") external_spec = {"source": data_source.name, "plugin": plugin, "connection_suffix": settings.get("unityCatalogConnection", plugin), "location": external.sourceLocation, "database": settings.get("database"), "service_name": settings.get("service_name"), "extract_mode": extract_mode} diff --git a/Generate/shared/uc_sources.py b/Generate/shared/uc_sources.py index a932636..ed46d27 100644 --- a/Generate/shared/uc_sources.py +++ b/Generate/shared/uc_sources.py @@ -24,7 +24,7 @@ def database_query(connection, location, database, plugin): if match and not re.fullmatch(identifier + r'(?:\s*\.\s*' + identifier + r')*', table): table, predicate = location.strip(), None if not re.match(r"(?i)^select\b", table): - table = table.replace("[", "").replace("]", "") if plugin == "sqlserver" else table + table = table.replace("[", "").replace("]", "") if plugin == "builtin:SQLServer" else table selection = "dbtable => " + sql_literal(table) else: selection = "query => " + sql_literal(table) @@ -35,7 +35,7 @@ def read_database(spark, connection, location, database, plugin): query, predicate = database_query(connection, location, database, plugin) frame = spark.sql(query) if predicate: - frame = frame.filter(predicate.replace("[", "`").replace("]", "`") if plugin == "sqlserver" else predicate) + frame = frame.filter(predicate.replace("[", "`").replace("]", "`") if plugin == "builtin:SQLServer" else predicate) return frame diff --git a/scripts/check_generation.py b/scripts/check_generation.py index e217969..e909a33 100644 --- a/scripts/check_generation.py +++ b/scripts/check_generation.py @@ -33,7 +33,7 @@ def main(): resources=generated_resources(output.parent) manifest=json.loads((output/"manifest.json").read_text()) external_sources=[item["external"] for item in manifest["datasets"] if item.get("external")] - if not external_sources or any(item["plugin"] != "sqlserver" for item in external_sources): + if not external_sources or any(item["plugin"] != "builtin:SQLServer" for item in external_sources): raise AssertionError("The public SQL Server sample must generate SQL Server source readers") execution_plan(resources,manifest,required_phases=manifest["lifecycle"]) for job in resources["jobs"].values():check_graph(job["tasks"]) diff --git a/tests/test_public_entrypoint.py b/tests/test_public_entrypoint.py index 42a53ca..6342a81 100644 --- a/tests/test_public_entrypoint.py +++ b/tests/test_public_entrypoint.py @@ -1,7 +1,6 @@ """Exercise the public sample binding against the real bundled SQL Server host.""" import json from pathlib import Path -import sys import unittest from unittest.mock import patch @@ -11,8 +10,6 @@ from datam8_model.data_source import DataSource, DataSourceType ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "Generate/databricks-lake/__modules")) -from metadata_utils import source_reader_kind class PublicEntrypointTests(unittest.TestCase): @@ -56,12 +53,6 @@ def test_real_builtin_metadata_preserves_source_types_and_sample_mappings(self): mappings = {item.sourceType: item.targetType for item in self.source_type.dataTypeMapping} self.assertEqual([mappings[field.dataType] for field in fields], ["int", "decimal"]) - def test_host_binding_keeps_the_existing_generated_reader_kind(self): - self.assertEqual(source_reader_kind(self.source_type.pluginId), "sqlserver") - self.assertEqual(source_reader_kind("sqlserver"), "sqlserver") - self.assertEqual(source_reader_kind("oracle"), "oracle") - self.assertEqual(source_reader_kind("builtin:CsvFile"), "builtin:CsvFile") - if __name__ == "__main__": unittest.main() diff --git a/tests/test_uc_sources.py b/tests/test_uc_sources.py index c4cbb53..e200c6a 100644 --- a/tests/test_uc_sources.py +++ b/tests/test_uc_sources.py @@ -2,10 +2,10 @@ from pathlib import Path from types import SimpleNamespace import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "Generate/shared")) -from uc_sources import database_query, fetch_http_rows +from uc_sources import database_query, fetch_http_rows, read_database class Proxy: @@ -44,7 +44,7 @@ def test_nonretryable_error_does_not_disclose_response(self): self.assertNotIn("sensitive", str(error.exception)) def test_sqlserver_ordinary_table_and_filter(self): - query, predicate = database_query("sample_sqlserver", "SELECT * FROM [SalesLT].[Customer] WHERE CustomerID > 100", "adventureworks", "sqlserver") + query, predicate = database_query("sample_sqlserver", "SELECT * FROM [SalesLT].[Customer] WHERE CustomerID > 100", "adventureworks", "builtin:SQLServer") self.assertIn("dbtable => 'SalesLT.Customer'", query) self.assertEqual(predicate, "CustomerID > 100") self.assertNotIn("host", query) @@ -53,6 +53,17 @@ def test_oracle_uses_service_binding(self): query, _ = database_query("sample_oracle", '"APP"."SUPPLIERS"', "SAMPLEPDB", "oracle") self.assertIn("service_name => 'SAMPLEPDB'", query) + def test_sql_builtin_reader_handles_quoted_columns_without_id_translation(self): + spark = Mock() + result = read_database( + spark, "sample_sqlserver", + "SELECT * FROM [SalesLT].[Customer] WHERE [CustomerID] > 100", + "adventureworks", "builtin:SQLServer", + ) + self.assertIn("dbtable => 'SalesLT.Customer'", spark.sql.call_args.args[0]) + spark.sql.return_value.filter.assert_called_once_with("`CustomerID` > 100") + self.assertIs(result, spark.sql.return_value.filter.return_value) + def test_proxy_endpoint_cannot_escape_connection(self): with self.assertRaises(ValueError): fetch_http_rows("sample_shipping", "../other", Proxy()) From b65443f5e84a0781909fb9416cdf8d7634bad9a8 Mon Sep 17 00:00:00 2001 From: Fabio Kayser Date: Thu, 5 Nov 2026 10:17:30 +0100 Subject: [PATCH 6/6] docs: remove obsolete sample task handoffs --- README.md | 1 - docs/documentation-handoff.md | 16 ---------------- docs/index.md | 2 -- docs/sample-cleanup.md | 34 ---------------------------------- 4 files changed, 53 deletions(-) delete mode 100644 docs/documentation-handoff.md delete mode 100644 docs/sample-cleanup.md diff --git a/README.md b/README.md index 3e13a19..07546f8 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,6 @@ successful deployment or processing. See [output boundaries](docs/quickstart.md) - [Sample documentation](docs/index.md) - [Central illustrated handbook](https://github.com/oraylis/datam8/blob/main/docs/user-guide/README.md) - [Local checks and review requirements](docs/integration-tests.md) -- [Sample cleanup history](docs/sample-cleanup.md) - [Source and credential policy](SECURITY.md) Follow [repository instructions](AGENTS.md) and run the relevant local checks diff --git a/docs/documentation-handoff.md b/docs/documentation-handoff.md deleted file mode 100644 index 9f7d44a..0000000 --- a/docs/documentation-handoff.md +++ /dev/null @@ -1,16 +0,0 @@ -# Historical Sample Documentation Handoff - -This page records the original documentation transfer. Publication and branch -statements below are historical; current contributor checks follow -[repository instructions](../AGENTS.md). - -This is a documentation-only package on the agreed feature branch. No product API, -schema JSON, dependency pin or target implementation is changed here. - -Run `node --test scripts/check-docs.test.mjs` and `node scripts/check-docs.mjs` -with Node 24. CI runs those checks without a frontend or sample checkout. - -Read the [central handoff](https://github.com/oraylis/datam8/blob/main/docs/handoff.md) for branch/workspace mapping, -results and manual merge order. The new central and component pages are local until -their branches are published. After manual merges, update development links to main; -keep tutorial evidence bound to commits. Public user workflows belong in the handbook. diff --git a/docs/index.md b/docs/index.md index 401b79c..2801a2b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,8 +13,6 @@ connection or cloud account is not required for that path. - [Template and payload development](template-development.md) - [Generation and deployment boundaries](quickstart.md) - [Local checks and review requirements](integration-tests.md) -- [Historical documentation handoff](documentation-handoff.md) -- [Sample cleanup history](sample-cleanup.md) - [Central user handbook](https://github.com/oraylis/datam8/blob/main/docs/user-guide/README.md) General app installation follows the stable release. The project and lockfile diff --git a/docs/sample-cleanup.md b/docs/sample-cleanup.md deleted file mode 100644 index c051005..0000000 --- a/docs/sample-cleanup.md +++ /dev/null @@ -1,34 +0,0 @@ -# Sample Cleanup Handoff - -This historical handoff records the reduction to a public, generate-only sample. -Its branch-state and review statements describe that cleanup. Current local -verification follows [local checks](integration-tests.md); the historical test -hold below is no longer the working policy. - -Retained: the small AdventureWorks customer/address/order model, its linked Core/Curated entities, classic databricks-lake, Docs, Power BI and Generate-only. -Removed: Lakeflow target/bundle, live bundle bindings, cloud integration runner, -integration-only tests and workflows, and the obsolete Azure pipeline. -Pure graph/asset checks remain for local generation. - -## Historical manual review boundary - -At the original cleanup handoff, no tests had been run for that branch. Before the user's explicit review, -Generate-only is dispatch-only and guarded by an approved exact commit setting. -The independent test model subsequently checks generation, links, source privacy -and the absence of imports of removed integration modules. PR/main triggers are -restored only after that approval. This section records the former migration process. - -## Prior public exposure - -Previous main contained real workspace hosts, catalog names and principal IDs -in bundle.targets.yml, tests/integration/environments.json and deployment guides; -one bundle also contained a personal account. An unused system-properties module -contained infrastructure defaults. These are removed/neutralized in this branch. -Deleting them does not erase Git history, forks, caches or previous Actions logs -and artifacts. This is an inventory, not a completed historical credential audit. -No history rewrite, artifact deletion, credential rotation or cloud destruction -is implicitly performed. Any actual credential exposure requires separate handling. - -The retained metadata intentionally contains only AdventureWorks and the Customer, -Address, Order and Calendar dependency closure. The target reference documents -the three local generation targets.