diff --git a/.gitignore b/.gitignore index 8d88214..5e4f37e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,23 @@ +# Generated target outputs and local deployment state +output/ +.databricks/ +.env +.env.* +!.env.example + # DataM8 local files datam8.user index.json -# Python environment local files -.venv -__pycache__ - -# Visual Studio local files -.vs +# Local Python/editor caches +.venv/ +__pycache__/ +.pytest_cache/ +.ruff_cache/ +.vs/ +**/.pbi/localSettings.json +**/.pbi/cache.abf -# Installed source plugins remain local dependencies. -Plugins/* -!Plugins/.gitkeep +# Installed custom source plugins remain local dependencies +plugins/* +!plugins/.gitkeep diff --git a/.vscode/settings.json b/.vscode/settings.json index cd2cd2e..ebdcabf 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,35 +1,35 @@ { - "json.schemas": [ - { - "fileMatch": [ - "/Base/**/*.json", - ], - "url": "https://raw.githubusercontent.com/oraylis/datam8-model/refs/tags/v2.0.0-beta.1/schema/base.json" - }, - { - "fileMatch": [ - "/Model/**/*.json", - "!/Model/**/.properties.json", - ], - "url": "https://raw.githubusercontent.com/oraylis/datam8-model/refs/tags/v2.0.0-beta.1/schema/model.json" - }, - { - "fileMatch": [ - "/Model/**/*.json", - ], - "url": "https://raw.githubusercontent.com/oraylis/datam8-model/refs/tags/v2.0.0-beta.1/schema/folder.json" - }, - { - "fileMatch": [ - "/*.dm8s", - ], - "url": "https://raw.githubusercontent.com/oraylis/datam8-model/refs/tags/v2.0.0-beta.1/schema/solution.json" - }, - { - "fileMatch": [ - "/Plugins/**/*.json", - ], - "url": "https://raw.githubusercontent.com/oraylis/datam8-model/refs/tags/v2.0.0-beta.1/schema/plugin.json" - }, - ] + "json.schemas": [ + { + "fileMatch": [ + "/base/**/*.json" + ], + "url": "https://raw.githubusercontent.com/oraylis/datam8-model/002aa65f6f127f91d06c8dc0fc4ea50ce75616fc/schema/base.json" + }, + { + "fileMatch": [ + "/model/**/*.json", + "!/model/**/.properties.json" + ], + "url": "https://raw.githubusercontent.com/oraylis/datam8-model/002aa65f6f127f91d06c8dc0fc4ea50ce75616fc/schema/model.json" + }, + { + "fileMatch": [ + "/model/**/.properties.json" + ], + "url": "https://raw.githubusercontent.com/oraylis/datam8-model/002aa65f6f127f91d06c8dc0fc4ea50ce75616fc/schema/folder.json" + }, + { + "fileMatch": [ + "/*.dm8s" + ], + "url": "https://raw.githubusercontent.com/oraylis/datam8-model/002aa65f6f127f91d06c8dc0fc4ea50ce75616fc/schema/solution.json" + }, + { + "fileMatch": [ + "/plugins/**/plugin.json" + ], + "url": "https://raw.githubusercontent.com/oraylis/datam8-model/002aa65f6f127f91d06c8dc0fc4ea50ce75616fc/schema/plugin.json" + } + ] } diff --git a/AGENTS.md b/AGENTS.md index 540ba2e..092e29a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,9 @@ # Working in the DataM8 Public Sample -This repository owns the small AdventureWorks Base/Model example, business +This repository owns the small AdventureWorks base/model example, business functions and classic Databricks, Docs and Power BI generation references. -It supports local generation; no cloud deployment or integration runner is included. +It supports local generation and a neutral, operator-configured Databricks bundle. +No live integration runner is included; cloud execution requires separate authorization. - Start with [getting started](docs/getting-started.md) and the [local checks](docs/integration-tests.md). Read the affected target's reference diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 92979fa..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,4 +0,0 @@ -# Working rules - -Read [AGENTS.md](AGENTS.md) for repository ownership, local verification commands -and cloud/source boundaries. This public sample supports local generation only. diff --git a/Generate/databricks-lake/DEPLOYMENT.md b/Generate/databricks-lake/DEPLOYMENT.md deleted file mode 100644 index c2a5e07..0000000 --- a/Generate/databricks-lake/DEPLOYMENT.md +++ /dev/null @@ -1,11 +0,0 @@ -# Classic Target: Generation Only - -This target renders notebooks and resource definitions. It contains no -environment-bound bundle entrypoint or cloud execution workflow. Configure your -own deployment process after reviewing platform, source and identity prerequisites; -generated files are not deployment evidence. - -The utilities retained under `Output/databricks-lake/utils` are neutral runtime -source assets required by generated references, not deployment configuration. -Keep them separate from cleaned output. See [local generation](../../docs/getting-started.md) -and [deployment boundaries](../../docs/quickstart.md). diff --git a/ORAYLISDatabricksSample.dm8s b/ORAYLISDatabricksSample.dm8s index 76cd87b..e2106d7 100644 --- a/ORAYLISDatabricksSample.dm8s +++ b/ORAYLISDatabricksSample.dm8s @@ -1,26 +1,26 @@ { "schemaVersion": "2.0.0", - "basePath": "Base", - "modelPath": "Model", - "pluginsPath": "Plugins", + "basePath": "base", + "modelPath": "model", + "pluginsPath": "plugins", "generatorTargets": [ { "name": "databricks", "isDefault": true, - "sourcePath": "Generate/databricks-lake", - "outputPath": "Output/databricks-lake/generated" + "sourcePath": "generate/databricks-lake", + "outputPath": "output/databricks" }, { "name": "powerbi", "isDefault": false, - "sourcePath": "Generate/powerbi-tabular", - "outputPath": "Output/powerbi-tabular/generated" + "sourcePath": "generate/powerbi-tabular", + "outputPath": "output/powerbi" }, { "name": "docs", "isDefault": false, - "sourcePath": "Generate/docs", - "outputPath": "Output/docs" + "sourcePath": "generate/docs", + "outputPath": "output/docs" } ] } diff --git a/Output/.gitignore b/Output/.gitignore deleted file mode 100644 index 7470a60..0000000 --- a/Output/.gitignore +++ /dev/null @@ -1,12 +0,0 @@ -.databricks - -# remove all files -databricks/generated -databricks-lake/generated -databricks-lakeflow/generated -powerbi-tabular/generated -docs/ -e2e-test-reports/ - -# allow the gitignore file -!.gitignore diff --git a/Plugins/.gitkeep b/Plugins/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/README.md b/README.md index 07546f8..02b1121 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,10 @@ semantic model. The first exercise runs locally without a database or cloud acco | I want to… | Begin here | First result | | --- | --- | --- | | Learn visual modeling | [Open the sample in the app](docs/getting-started.md#use-the-app) | Change a Customer description and confirm it survives reload | -| Validate and generate from the command line | [Local CLI setup](docs/getting-started.md#use-the-cli) | Generate and inspect a fresh `Output/docs/index.md` | +| Validate and generate from the command line | [Local CLI setup](docs/getting-started.md#use-the-cli) | Generate and inspect a fresh `output/docs/index.md` | +| Build and deploy my own solution | [Extend the sample](docs/build-your-own-solution.md), then [deploy](docs/quickstart.md) | Extend your model, validate and generate it, then deploy to your own environment | | Change templates or target conventions | [Template development](docs/template-development.md) and [configured targets](docs/reference/targets.md) | Change one template and compare the generated output | -| Understand the example | [Model documentation target](Generate/docs/README.md) and [property references](docs/model-properties.md) | Follow Customer through Stage, Core, Curated and Consumer | +| Understand the example | [Model documentation target](generate/docs/README.md) and [property references](docs/model-properties.md) | Follow Customer through Stage, Core, Curated and Consumer | **Recommended first result:** generate `docs` and open its index. Use a disposable copy: autosave writes metadata, and generation can replace a target's output. @@ -30,7 +31,7 @@ uv run datam8 validate --solution ./ORAYLISDatabricksSample.dm8s uv run datam8 generate docs --solution ./ORAYLISDatabricksSample.dm8s ``` -Open `Output/docs/index.md`; it should contain Customer and links to entity pages +Open `output/docs/index.md`; it should contain Customer and links to entity pages and a relationship diagram. The SQL Server metadata connector is bundled with the pinned CLI and installed through its `sql` extra. No connector checkout, download token or source connection is needed for this exercise. @@ -39,9 +40,9 @@ download token or source connection is needed for this exercise. | Target | Source | Output | | --- | --- | --- | -| databricks | `Generate/databricks-lake` | `Output/databricks-lake/generated` | -| powerbi | `Generate/powerbi-tabular` | `Output/powerbi-tabular/generated` | -| docs | `Generate/docs` | `Output/docs` | +| databricks | `generate/databricks-lake` | `output/databricks` | +| powerbi | `generate/powerbi-tabular` | `output/powerbi` | +| docs | `generate/docs` | `output/docs` | The model follows AdventureWorks customers, addresses and orders through Stage, Core and Curated, including DimCustomer, DimDate and FactSalesOrder. Consumer @@ -49,9 +50,10 @@ entities retain Customer, Date, Sales Order and Total Sales. The SQL example uses the standard SalesLT.Customer table; no custom language-specific views are required. Data endpoints and secret references are neutral placeholders. -This is a generate-only example. Deployment bindings, cloud resources and live -execution are not included. Generating notebooks or TMDL does not demonstrate -successful deployment or processing. See [output boundaries](docs/quickstart.md). +A neutral Databricks bundle under `deployment/` and the [own-deployment guides](docs/quickstart.md) +cover the next steps. Supply your own identity, compute, catalog and source +connection. Credentials and live environment bindings are not included. Local +generation does not establish successful deployment, refresh or processing. ## Learn more and contribute diff --git a/SECURITY.md b/SECURITY.md index 9f7698b..813e86f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,5 +4,7 @@ Use the central [private vulnerability-report route](https://github.com/oraylis/ or DataM8@Oraylis.de if that route is unavailable. Include component versions, reproduction and impact; keep sensitive data out of public issues. -This documentation covers DataM8 2.0.0. Follow the -[central reporting guidance](https://github.com/oraylis/datam8/blob/codex/documentation-v2/SECURITY.md). +Follow the [central reporting guidance](https://github.com/oraylis/datam8/blob/main/SECURITY.md). +Versioned examples contain neutral placeholders and secret references only. Keep +Databricks authentication profiles, Power BI credentials, `.env` files and local +deployment state outside version control. Never add passwords, tokens or client secrets. diff --git a/Base/AttributeTypes.json b/base/AttributeTypes.json similarity index 100% rename from Base/AttributeTypes.json rename to base/AttributeTypes.json diff --git a/Base/DataProducts.json b/base/DataProducts.json similarity index 100% rename from Base/DataProducts.json rename to base/DataProducts.json diff --git a/Base/DataSourceTypes.json b/base/DataSourceTypes.json similarity index 100% rename from Base/DataSourceTypes.json rename to base/DataSourceTypes.json diff --git a/Base/DataSources.json b/base/DataSources.json similarity index 100% rename from Base/DataSources.json rename to base/DataSources.json diff --git a/Base/DataTypes.json b/base/DataTypes.json similarity index 100% rename from Base/DataTypes.json rename to base/DataTypes.json diff --git a/Base/Properties.json b/base/Properties.json similarity index 100% rename from Base/Properties.json rename to base/Properties.json diff --git a/Base/PropertyValues.json b/base/PropertyValues.json similarity index 100% rename from Base/PropertyValues.json rename to base/PropertyValues.json diff --git a/Base/Zones.json b/base/Zones.json similarity index 100% rename from Base/Zones.json rename to base/Zones.json diff --git a/deployment/databricks/databricks.yml b/deployment/databricks/databricks.yml new file mode 100644 index 0000000..94942ec --- /dev/null +++ b/deployment/databricks/databricks.yml @@ -0,0 +1,47 @@ +bundle: + name: datam8-sample + +include: + - ../../output/databricks/schemas/*.yml + - ../../output/databricks/clusters/*.yml + - ../../output/databricks/jobs/*.yml + - ../../output/databricks/jobs/*/*.yml + +# External paths expand the sync root to the repository root. Re-include output +# because .gitignore intentionally excludes generated files from source control. +sync: + paths: + - ../../output/databricks + - ../../utils + include: + - ../../output/databricks/** + - ../../utils/** + +variables: + environment: + default: dev + catalog_name: + description: Your isolated existing Unity Catalog catalog + owner: + description: Your own Databricks user name (table owner) + cluster_id: + description: Existing Unity Catalog compute running Databricks Runtime 18.3 or later + connection_prefix: + description: Prefix before the modeled sqlserver connection suffix + default: sample_ + date_start: + default: "2000-01-01" + date_end: + default: "2030-12-31" + unpause_schedules: + default: PAUSED + +targets: + dev: + default: true + mode: development + cluster_id: ${var.cluster_id} + workspace: + root_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/${bundle.name}/${bundle.target} + run_as: + user_name: ${workspace.current_user.userName} diff --git a/docs/build-your-own-solution.md b/docs/build-your-own-solution.md new file mode 100644 index 0000000..9319a91 --- /dev/null +++ b/docs/build-your-own-solution.md @@ -0,0 +1,85 @@ +# Build Your Own Solution + +Start with a complete fork or copy of this repository, including `base`, `model`, +`generate`, `plugins`, `utils`, `deployment` and the solution descriptor. Keep your +own solution in source control and make changes on a branch. Local model editing, +validation and generation do not require a source connection or cloud account. + +## Establish a reproducible starting point + +Follow [local setup](getting-started.md#use-the-cli), then commit your starting +point. Keep `pyproject.toml` and `uv.lock` together: the lockfile binds the public +CLI and schema dependencies. The editor schemas in `.vscode/settings.json` match +that dependency. For the app, use a release compatible with the solution schema; +its bundled runtime is independent of the uv environment. When upgrading, review +compatibility notes and regenerate on a branch before updating these pins. + +```powershell +uv sync --frozen +uv run datam8 validate --solution ./ORAYLISDatabricksSample.dm8s +uv run datam8 generate docs --solution ./ORAYLISDatabricksSample.dm8s --clean-output +uv run datam8 generate databricks --solution ./ORAYLISDatabricksSample.dm8s --clean-output +uv run datam8 generate powerbi --solution ./ORAYLISDatabricksSample.dm8s --clean-output +``` + +Open the generated documentation index and inspect the notebooks/jobs and TMDL +under the [three target output folders](reference/targets.md). Each generation +command replaces only its selected target's output, including stale artifacts. +Save a copy of this baseline outside `output/` to compare artifacts after editing. + +## Extend the model + +Use the DataM8 app or edit the persisted JSON and function files in your editor. +The [user handbook](https://github.com/oraylis/datam8/blob/main/docs/user-guide/README.md) +covers the individual editors and workflows. + +1. Choose the layer where the change belongs. Add or modify entities and + attributes with the appropriate types, keys, descriptions and properties. + Add new entities through the app or follow the existing schema when editing + JSON; every entity needs a unique ID. +2. Define where the data comes from. For an external source, configure its source + type and connection references. Source import and preview require your own + connection and credentials; stored model metadata can be edited offline. + For an existing modeled source, select the upstream entity and map its + attributes to the target attributes. +3. Update transformations and referenced function files when the change needs + new logic. Maintain relationships and downstream mappings across the affected + layers. Expose reporting fields in the Consumer layer when report authors + need them. +4. Save and reload the solution to confirm that the edits are persisted. Validate + before generating again. + +Keep credentials outside version control and use secret references in source +settings. For property behavior and target selection, consult the +[model properties](model-properties.md) and [target references](reference/targets.md). + +## Validate and review the generated changes + +Run the validation and generation commands above again, or select the affected +targets in the app. Check command exit codes and review the resulting artifacts: + +- **Documentation:** changed entities, attributes, mappings and relationships + appear in the catalog and entity pages. +- **Databricks:** affected Raw, Stage, Core and Curated notebooks and jobs reflect + the change, with the intended dependencies and table definitions. +- **Power BI:** affected Consumer tables, columns, relationships and measures + appear in the generated TMDL. + +The targets cover different layers, so a model change may affect only some of +them. Compare against your baseline and investigate unrelated differences. +Documentation timestamps change whenever documentation is regenerated. Commit +the authored model, template and configuration changes after review; output +remains disposable. When changing generator code, use the contributor +[local checks](integration-tests.md) as well. + +## Prepare your own environment + +Continue with [Databricks deployment](../generate/databricks-lake/DEPLOYMENT.md) +to create tables and load data, then [Power BI project integration](powerbi-deployment.md) +to build and publish your own report. For different output conventions, follow +[template development](template-development.md). + +Keep your authored reports and connection parameters in a separate PBIP project, +and maintain bundle configuration under `deployment/`. Local profiles, secrets +and deployment state belong outside version control. Review the generated +changes and the impact on existing data before deploying to your own environment. diff --git a/docs/getting-started.md b/docs/getting-started.md index b5b71b5..d49e327 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -17,7 +17,7 @@ git clone https://github.com/oraylis/datam8-sample-solution.git datam8-sample cd datam8-sample ``` -Use a working copy for your changes. Keep Base, Model, Plugins, Generate and the +Use a working copy for your changes. Keep `base`, `model`, `plugins`, `generate`, `utils`, `deployment` and the supporting configuration alongside `ORAYLISDatabricksSample.dm8s`. Copy the whole solution when creating another exercise directory. Autosave writes your edits to disk, and generation can replace files in a target output directory. @@ -37,7 +37,7 @@ output without installing Python or the CLI separately. change, edit the Stage Customer description and wait for autosave to finish. 4. Reload the solution and confirm that your change was saved. 5. Open **Toggle generator**, select **docs**, and choose **Generate**. -6. Open `Output/docs/index.md` and inspect the generated model documentation. +6. Open `output/docs/index.md` and inspect the generated model documentation. Follow the [illustrated user guide](https://github.com/oraylis/datam8/blob/main/docs/user-guide/README.md) for the model editors, relationships, sources and generation workflow. @@ -60,7 +60,7 @@ installed solution connector or repository-access token is required. See [local checks](integration-tests.md) for contributor verification. Source import and preview need your own source settings; local metadata validation and generation do not need a source connection. -Check the exit codes, then open `Output/docs/index.md`. It should contain Customer +Check the exit codes, then open `output/docs/index.md`. It should contain Customer and links to entity pages and the relationship diagram. Edit the model or templates in your editor, then validate and generate again to inspect your changes. @@ -74,5 +74,6 @@ paths. Deployment is a separate step after generation. ## Next steps -Read [targets](reference/targets.md), [template development](template-development.md) -and, only when needed, [deployment preparation](quickstart.md). +[Build your own solution](build-your-own-solution.md): extend entities, attributes +and mappings, validate and regenerate, then follow [deployment](quickstart.md). For generator work, +read [targets](reference/targets.md) and [template development](template-development.md). diff --git a/docs/index.md b/docs/index.md index 2801a2b..1dbbcda 100644 --- a/docs/index.md +++ b/docs/index.md @@ -5,13 +5,15 @@ use the pinned public CLI, change metadata and generate documentation. A source connection or cloud account is not required for that path. - [Local getting started](getting-started.md) +- [Build your own solution](build-your-own-solution.md): extend entities, attributes and mappings +- [Deploy to your own environment](quickstart.md): Databricks first, then Power BI - [Configured targets and output paths](reference/targets.md) - [Model properties by target](model-properties.md) -- [Databricks notebooks and job definitions](../Generate/databricks-lake/README.md) -- [Power BI consumer semantic model](../Generate/powerbi-tabular/README.md) -- [Documentation catalog and relationship diagram](../Generate/docs/README.md) +- [Databricks notebooks and job definitions](../generate/databricks-lake/README.md) +- [Power BI consumer semantic model](../generate/powerbi-tabular/README.md) +- [Documentation catalog and relationship diagram](../generate/docs/README.md) - [Template and payload development](template-development.md) -- [Generation and deployment boundaries](quickstart.md) +- [Power BI Desktop/PBIP integration](powerbi-deployment.md) - [Local checks and review requirements](integration-tests.md) - [Central user handbook](https://github.com/oraylis/datam8/blob/main/docs/user-guide/README.md) diff --git a/docs/integration-tests.md b/docs/integration-tests.md index e3f9cc3..e8f393f 100644 --- a/docs/integration-tests.md +++ b/docs/integration-tests.md @@ -1,8 +1,8 @@ # Local Generation Checks and Review This repository checks the AdventureWorks model and local generation of -Databricks, documentation and Power BI artifacts. It has no cloud deployment or -integration runner. Generated files do not establish deployment acceptance. +Databricks, documentation and Power BI artifacts and the neutral deployment bundle. It has no live integration runner. +Generated files do not establish deployment acceptance. ## Local checks @@ -20,16 +20,24 @@ uv run python scripts/check_generation.py --target all The suite covers model references, public source placeholders, SQL Server builtin forms/metadata and generated reader compatibility. The generation checker validates -all targets, Python syntax, notebook cells, job graphs and required runtime assets. +all targets, Python syntax, notebook cells, job graphs and runtime assets using +the actual bundle includes, variables and sync configuration. A temporary-copy +workflow test checks target-clean isolation and an internal Consumer extension fixture. It does not contact source systems or deploy generated artifacts. +The offline bundle check uses the resource includes, sync paths and Git-style +include/exclude patterns in the base bundle configuration. It checks referenced +assets and generated resource coverage, including additional authored resources. +Use the Databricks CLI to validate environment-specific target overrides before +deployment, as described in the [deployment guide](../generate/databricks-lake/DEPLOYMENT.md). + The generation workflow runs on pull requests, pushes to main and manual dispatch. It uses read-only repository permissions and public locked dependencies; no review variable, source credential or repository-access token is needed. ## Public-entrypoint acceptance -Use a fresh copy with an empty Plugins folder and no repository-access token. +Use a fresh copy with an empty `plugins` folder and no repository-access token. Validate and generate all targets; inspect newly written output. SQL fixtures mock database I/O while using the real bundled connector. Record passes, failures and skips separately. Native app behavior, live SQL and cloud execution require their diff --git a/docs/model-properties.md b/docs/model-properties.md index f0de4b6..a8edf84 100644 --- a/docs/model-properties.md +++ b/docs/model-properties.md @@ -2,8 +2,8 @@ Choose the target reference for property locations, values and generated behavior: -- [Databricks Lake](../Generate/databricks-lake/PROPERTIES.md) -- [Power BI](../Generate/powerbi-tabular/PROPERTIES.md) -- [Documentation target](../Generate/docs/PROPERTIES.md) +- [Databricks Lake](../generate/databricks-lake/PROPERTIES.md) +- [Power BI](../generate/powerbi-tabular/PROPERTIES.md) +- [Documentation target](../generate/docs/PROPERTIES.md) -Property definitions and reusable values live in `Base/Properties.json` and `Base/PropertyValues.json`. +Property definitions and reusable values live in `base/Properties.json` and `base/PropertyValues.json`. diff --git a/docs/powerbi-deployment.md b/docs/powerbi-deployment.md new file mode 100644 index 0000000..8004b81 --- /dev/null +++ b/docs/powerbi-deployment.md @@ -0,0 +1,63 @@ +# Use the Generated Model in Your Own Power BI Project + +This route uses Power BI Desktop, an authored PBIP project and manual publishing. +The generator writes TMDL to `output/powerbi`; it does not create reports or publish +through XMLA. See [Microsoft's semantic-model project reference](https://learn.microsoft.com/en-us/power-bi/developer/projects/projects-dataset) +for `definition.pbism`, TMDL and local cache behavior. + +## Create the project and integrate TMDL + +1. Use Power BI Desktop with PBIP and TMDL project support enabled. Create your + own report and save it as a **Power BI Project (.pbip)** outside `output/`. + If necessary, add a small local table first so Desktop creates a semantic model. + Save the model in TMDL format and close Desktop before editing files. +2. Generate the target from your solution root: + + ```powershell + uv run datam8 generate powerbi --solution ./ORAYLISDatabricksSample.dm8s --clean-output + ``` + +3. Back up or commit your PBIP project. Replace the contents of its semantic-model + `definition/` folder with the contents of `output/powerbi/`, including `tables/`. + Keep the project, report folder, `definition.pbism` and report-to-model binding. + Do not place a second `powerbi-tabular/` folder beneath `definition/`. +4. Keep the project free of a stale `.pbi/cache.abf` from the temporary model. + Microsoft documents that this cache can overwrite the definition on opening. + A project without a cache opens with metadata and no data. Ignore local cache + and settings in source control. +5. Open the PBIP in Desktop **without refresh**. Check Customer, Date, Sales Order, + measures and relationships in Model view, including any changes you made to + Consumer metadata. Opening validates local model metadata; it does not + establish source connectivity or successful refresh. + +## Connect, refresh and publish + +After [loading the Databricks tables](../generate/databricks-lake/DEPLOYMENT.md), +set these Power Query parameters in your authored project: + +| Parameter | Your value | +| --- | --- | +| `DBW_Catalog` | The catalog used for this solution's Databricks deployment | +| `DBW_Hostname` | Your Databricks SQL server hostname | +| `DBW_HTTP_Path` | Your compatible SQL warehouse's HTTP path | + +Generated defaults are neutral placeholders. Authenticate through Desktop's +Databricks connector using your own identity; never put credentials in TMDL. +You need warehouse access and permission to read the Curated tables. Refresh in +your own environment, build report visuals, then use Desktop **Publish** to select +your workspace. Configure the semantic model's credentials and scheduled refresh +in Power BI Service, and verify a refresh there. Workspace access, licensing and +tenant policies are operator prerequisites. + +## Regenerate without losing authored work + +Keep the PBIP outside the generated output. Regenerate into `output/powerbi`, +close Desktop and review the TMDL diff before copying it into `definition/` again. +Preserve your project's `expressions.tmdl` connection parameters, authored report +folder, project files and report-to-model binding. Review any new generated +parameters explicitly rather than overwriting existing values. Remove obsolete +generated table definitions when the model removes a table; merge any intentional +hand-authored measures or other TMDL customizations separately. + +Reopen without refresh to inspect metadata first. Refresh and publish only when +you intend to update your own environment. The sample's local checks never do so. diff --git a/docs/quickstart.md b/docs/quickstart.md index 8fd7a3f..7d1fbaa 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -1,16 +1,24 @@ # From Generated Files to Deployment -Start with [local generation](getting-started.md). This public example contains -metadata and templates, not a deployment runner, environment bindings or an active -Azure pipeline. Generated notebooks and TMDL are files for review, not evidence of -a running data platform. +First [build your own solution](build-your-own-solution.md): fork or copy the +complete sample, keep its dependency pins, extend the model and its mappings, +then validate and generate the affected targets. This path needs no credentials +or cloud account. -Before any deployment, review your platform's workspace, catalog, storage, -identities, source connections and secret integration in your own delivery process. -Keep those bindings outside this sample and preserve the authored runtime utilities -referenced by generated notebooks. Verify platform prerequisites independently. +Once those artifacts are ready, choose the next step: -The [Databricks boundary](../Generate/databricks-lake/DEPLOYMENT.md) and -[configured outputs](reference/targets.md) identify the artifacts supplied here. -The local exercise creates no cloud resources and starts no data processing. -Never commit live values in place of neutral source examples. +| Task | Guide | Expected result | +| --- | --- | --- | +| Run the data platform in your own environment | [Databricks deployment](../generate/databricks-lake/DEPLOYMENT.md) | Personal dev bundle, created tables and ordered successful loads | +| Create and publish a report | [Power BI Desktop/PBIP integration](powerbi-deployment.md) | Authored report using the generated semantic model and your own connection parameters | +| Share model documentation | [Docs target](../generate/docs/README.md) | Local Markdown catalog and editable relationship diagram | + +The neutral bundle lives under `deployment/databricks/`, authored runtime notebooks +under `utils/`, and disposable output under the [target folders](reference/targets.md). +No passwords, tokens or client secrets are supplied. Keep authentication local, +use secret references for source settings, and exclude local deployment state. + +Databricks deploy/run and Power BI refresh/publish are explicit operator actions +against your own environment. They are not part of local generation or CI; review +prerequisites and generated diffs before running them. The sample contains no live +integration runner or XMLA publisher. Local checks do not establish cloud acceptance. diff --git a/docs/reference/targets.md b/docs/reference/targets.md index b376ab7..9f01381 100644 --- a/docs/reference/targets.md +++ b/docs/reference/targets.md @@ -1,13 +1,14 @@ # Configured Targets and Output Paths -The solution descriptor contains exactly three generate-only targets. +The solution descriptor contains exactly three local generation targets. | Target | Template source | Configured output root | | --- | --- | --- | -| databricks | `Generate/databricks-lake` | `Output/databricks-lake/generated` | -| powerbi | `Generate/powerbi-tabular` | `Output/powerbi-tabular/generated` | -| docs | `Generate/docs` | `Output/docs` | +| databricks | `generate/databricks-lake` | `output/databricks` | +| powerbi | `generate/powerbi-tabular` | `output/powerbi` | +| docs | `generate/docs` | `output/docs` | -Each target writes files locally. The sample includes no deployment runner, -environment bindings or live integration workflow. Review the -[deployment boundary](../quickstart.md) before using generated artifacts elsewhere. +Each target writes only inside its own output folder. The authored runtime +notebooks in `utils/` and the neutral bundle in `deployment/databricks/` survive +clean generation. Supply your own environment settings and follow the +[deployment guides](../quickstart.md) after reviewing generated artifacts. diff --git a/docs/template-development.md b/docs/template-development.md index 1e9683d..15ce7bb 100644 --- a/docs/template-development.md +++ b/docs/template-development.md @@ -6,8 +6,8 @@ data. Put complex joins, ordering and target decisions in Python rather than tem ## Documentation target example -Read [payload.py](../Generate/docs/__modules/payload.py) and -[the architecture](../Generate/docs/ARCHITECTURE.md). The target builds one cached +Read [payload.py](../generate/docs/__modules/payload.py) and +[the architecture](../generate/docs/ARCHITECTURE.md). The target builds one cached documentation snapshot, then registers payloads for index, entity pages and a diagram. The existing index payload uses this shape: @@ -27,8 +27,8 @@ it is not a complete standalone plugin. Inspect the backend's 1. Work on a complete sample copy using a compatible app or the pinned CLI. 2. Add/shape a field in Python and update its Jinja consumer. 3. Generate that target in the app or with the [CLI](getting-started.md). -4. Assert new output content, links, ordering and paths; compare unchanged output - with the parity script when required. +4. Assert new output content, links, ordering and paths; compare unaffected artifacts + against a saved baseline outside the cleaned target output. 5. Update the affected target architecture and central user chapter if behavior changes. Generation prepares files. Deployment credentials, resource creation and target diff --git a/Generate/databricks-lake/ARCHITECTURE.md b/generate/databricks-lake/ARCHITECTURE.md similarity index 97% rename from Generate/databricks-lake/ARCHITECTURE.md rename to generate/databricks-lake/ARCHITECTURE.md index 81033c3..6f69bb4 100644 --- a/Generate/databricks-lake/ARCHITECTURE.md +++ b/generate/databricks-lake/ARCHITECTURE.md @@ -53,7 +53,7 @@ This target generates Databricks notebooks and bundle resources for: ## Validation Checklist -1. Run lint: `ruff check Generate/databricks-lake/__modules` +1. Run lint: `ruff check generate/databricks-lake/__modules` 2. Validate template rendering in local generation run. 3. Run parity check against baseline output: `scripts/verify_generation_parity.ps1` diff --git a/generate/databricks-lake/DEPLOYMENT.md b/generate/databricks-lake/DEPLOYMENT.md new file mode 100644 index 0000000..aad58b0 --- /dev/null +++ b/generate/databricks-lake/DEPLOYMENT.md @@ -0,0 +1,113 @@ +# Deploy to Your Own Databricks Development Environment + +Use a complete [own solution copy](../../docs/build-your-own-solution.md). The +[neutral bundle](../../deployment/databricks/databricks.yml) deploys generated +notebooks/jobs with your own user, an isolated dev workspace path and paused +schedules. It requires existing compatible compute; it does not provision a +workspace, source system, catalog, connection or service principal. + +## Prepare source, catalog and identity + +- Have an AdventureWorks SQL Server database containing the modeled SalesLT + tables, reachable from the compute. Preserve the sample's source/table names + or update the metadata and regenerate deliberately. +- Have a Unity Catalog-enabled workspace and an existing catalog reserved for + your solution. Your user needs `USE CATALOG`, `CREATE SCHEMA`, and permission + to create/read/write the solution's tables and assign their ownership. Existing + objects must already be accessible. The generated schema resources create the + Raw, Stage, Core and Curated schemas inside that catalog; managed storage must + already be configured by its owner. +- Have an existing Unity Catalog **SQL Server connection** to your database, + with credentials managed by its owner. The sample connection suffix is + `sqlserver`; with the default prefix `sample_`, the runtime uses + `sample_sqlserver`. Set your own prefix or change the modeled + `unityCatalogConnection` setting on your solution copy. This is independent + of the CLI's metadata connector: the latter uses `builtin:SQLServer` and + local secret references when you explicitly import/preview a source. +- Use existing Unity Catalog-compatible compute running **Databricks Runtime + 18.3 or later**, with network access to SQL Server and your user's ability to + attach. The generated reader uses `remote_query`, not JDBC. You need + `USE CONNECTION`; dedicated compute also requires `MANAGE` on the connection. + Confirm current runtime and access requirements in the + [manufacturer documentation](https://learn.microsoft.com/en-us/azure/databricks/query-federation/remote-queries). + Generated cluster metadata is illustrative; the mandatory dev cluster override + below avoids provisioning those clusters or relying on their example settings. + +Do not replace neutral examples with committed credentials. Databricks auth +profiles live in your personal environment; source passwords belong in the +connection's managed authentication. Local `.env`, `.databricks/` state and Power BI +credentials are excluded or stored outside the repository. + +## Generate and inspect locally + +From the solution root: + +```powershell +uv sync --frozen +uv run datam8 validate --solution ./ORAYLISDatabricksSample.dm8s +uv run datam8 generate databricks --solution ./ORAYLISDatabricksSample.dm8s --clean-output +``` + +Inspect `output/databricks/manifest.json`, the notebooks, and YAML resources. +Bundle `include` globs load every generated schema, cluster-variable and job YAML. +`sync.paths` selects only `output/databricks` and `utils`; its common ancestor is +the solution root. Explicit `sync.include` patterns re-include the ignored output. +The workspace files retain `output/databricks/...` and `utils/...`, so `%run` +references and Python runtime imports resolve together. `base`, `model`, other +targets and local configuration are not synchronized. See the +[bundle reference](https://docs.databricks.com/aws/en/dev-tools/bundles/reference#sync) +for path and ignore semantics. + +## Authenticate, validate and deploy + +Install the [Databricks CLI](https://docs.databricks.com/aws/en/dev-tools/cli/install) +and authenticate locally to your own workspace. The following commands contact +that workspace; run them only when you intend to deploy there. Replace the example +values in your shell, without committing them: + +```powershell +databricks auth login --host https://YOUR-WORKSPACE --profile sample-dev +$env:BUNDLE_VAR_catalog_name = "YOUR_ISOLATED_CATALOG" +$env:BUNDLE_VAR_owner = "YOUR_USER_NAME" +$env:BUNDLE_VAR_cluster_id = "YOUR_EXISTING_CLUSTER_ID" +$env:BUNDLE_VAR_connection_prefix = "sample_" +Set-Location deployment/databricks +databricks bundle validate --target dev --profile sample-dev +databricks bundle deploy --target dev --profile sample-dev +``` + +Validation should resolve all variables, resources and local notebook paths. +Deployment should create the schema/job resources and upload files beneath +`/Workspace/Users//.bundle/datam8-sample/dev`, with schedules paused. +The required `cluster_id` selects an existing cluster through development mode; +the `run_as` user is the authenticated current user. See +[development-mode behavior](https://docs.databricks.com/aws/en/dev-tools/bundles/deployment-modes). +Use a distinct bundle name when deploying multiple solution copies as the same +user; dev workspace isolation does not isolate data inside a shared catalog. + +## Create tables and run loads in order + +From `deployment/databricks`, using the same variables and profile: + +```powershell +databricks bundle run --target dev --profile sample-dev Create_All_Tables +databricks bundle run --target dev --profile sample-dev Load_sales_daily +databricks bundle run --target dev --profile sample-dev Load_sales_weekly +``` + +Wait for each command to finish successfully before starting the next. Create All +calls zone creation jobs in layer order. The current manifest makes the weekly +load depend on the daily load; after model changes, read `execution_plan` again +and follow its dependencies. Do not start downstream jobs after an upstream +failure. Schedules remain paused until you intentionally configure automation. + +Inspect run logs, row counts, keys and expected values in your isolated catalog. +The sample's offline tests do not check these live outcomes. On permission, +connection or runtime errors, correct your environment before retrying. Then +continue with [Power BI](../../docs/powerbi-deployment.md). + +For a shared or production deployment, define a separate target and review +`run_as`, grants, compute, data isolation and scheduling with your operators. +Service principals are optional advanced configuration; no fixed principal or +job-permission grant is emitted by the templates. Regenerate output before each +deploy, keeping the authored bundle and runtime utilities outside cleaned output. diff --git a/Generate/databricks-lake/PROPERTIES.md b/generate/databricks-lake/PROPERTIES.md similarity index 94% rename from Generate/databricks-lake/PROPERTIES.md rename to generate/databricks-lake/PROPERTIES.md index 4b650ff..fb57d8a 100644 --- a/Generate/databricks-lake/PROPERTIES.md +++ b/generate/databricks-lake/PROPERTIES.md @@ -21,7 +21,7 @@ Extraction settings do not implicitly filter a modeled Core source. - **External source:** set values in its `sources[]` entry, connected to a DataSource. - **Source mapping:** mark a cursor with `extract_column=delta`. - **Attribute:** business keys and SCD history are model fields. -- **Reusable values:** define them in `Base/PropertyValues.json`; jobs may reference schedules and clusters. +- **Reusable values:** define them in `base/PropertyValues.json`; jobs may reference schedules and clusters. Use the value's `name` in a property reference. Effects and precedence can differ by target; each target page documents its contract. @@ -32,13 +32,13 @@ The job contains the assigned entities across Raw, Stage, Core, and Curated. Dependencies between entities with the same `jobs` value come from their modeled source references. Cross-job dependencies remain in the execution manifest; a load job never invokes another load job or runs its tasks. Raw -extraction tasks stay in the job of their consuming Stage entity. The local/CI -runner runs each selected job once in dependency order. The Create All job and +extraction tasks stay in the job of their consuming Stage entity. Run each selected job once in dependency order using the manifest. +No live integration runner is included. The Create All job and its four zone-specific table-creation jobs are separate from these load jobs. | Property | Where and current values | Effect in this target | | --- | --- | --- | -| `jobs` | Entity or folder. Current load values: `sales_daily` → `Sales (Daily)`, `sales_weekly` → `Sales (Weekly)`. `create_all` configures table creation. | Assigns entities to the corresponding generated job. The model value's `displayName` is the job display name. Same-job source dependencies create task edges; cross-job dependencies are recorded for ordered runner execution. | +| `jobs` | Entity or folder. Current load values: `sales_daily` → `Sales (Daily)`, `sales_weekly` → `Sales (Weekly)`. `create_all` configures table creation. | Assigns entities to the corresponding generated job. The model value's `displayName` is the job display name. Same-job source dependencies create task edges; cross-job dependencies are recorded for ordered operator execution. | | `schedules` | Nested in a `jobs` property value; current schedule values are `daily` (`1 0`) and `weekly` (`0 0 2 ? * SUN`). | Attaches a Quartz schedule to the generated job. Time zone is `Europe/Amsterdam`; enablement is controlled by the bundle's `unpause_schedules` variable. | | `cluster` | Nested in a `jobs` property value. Values defined: `serverless`, `small`, `extra_small`. | Selects serverless job compute or a configured job cluster. Create All and the Sales Daily/Weekly jobs use `extra_small` (two `Standard_D4ds_v5` workers). | | `write_mode` | Entity or folder. Values: `overwrite`, `merge`, `append`. | Controls writes to modeled tables. `merge` requires business keys and uses the configured history fields; `append` adds rows; `overwrite` replaces the current output. | @@ -53,7 +53,7 @@ its four zone-specific table-creation jobs are separate from these load jobs. | `data_layout` | Entity. The model contains `liquid_clustering` examples. | **No effect:** the classic DDL/DML generator does not read this property or emit liquid clustering. | | `dataset_type` | Entity or folder. Values: `materialized_view`, `streaming_table`. | **No effect:** this target always generates Delta tables and notebook DML; it does not select Lakeflow object types. | | `sequence_by`, `delete_condition` | Entity or folder. | **No effect:** these are read only by the Lakeflow target. | -| `target` | Zone. Defined value: `power-bi`. | **No effect on this target's job or schema generation.** Databricks schema names come from each zone's `targetName` in `Base/Zones.json`. | +| `target` | Zone. Defined value: `power-bi`. | **No effect on this target's job or schema generation.** Databricks schema names come from each zone's `targetName` in `base/Zones.json`. | | Attribute `sensitive` | Attribute property; present on customer attributes. | Emitted as a Unity Catalog column tag by classic DDL; it does not mask or encrypt the data. | `validate_all` is a defined `jobs` value but no current model entity or generated diff --git a/Generate/databricks-lake/README.md b/generate/databricks-lake/README.md similarity index 84% rename from Generate/databricks-lake/README.md rename to generate/databricks-lake/README.md index c77cf90..ae3a0e2 100644 --- a/Generate/databricks-lake/README.md +++ b/generate/databricks-lake/README.md @@ -30,7 +30,7 @@ the sources and transformations you model. | Core | Integrate data into reusable business entities and apply business transformations. | DDL and DML notebooks, plus referenced transformation scripts | | Curated | Prepare analytical entities for consumption, such as dimensions and facts. | DDL and DML notebooks for the modeled analytical tables | -Layer definitions live in [Base/Zones.json](../../Base/Zones.json). Notebook +Layer definitions live in [base/Zones.json](../../base/Zones.json). Notebook directories follow each zone's local folder, with product and module folders preserved beneath it. The sample uses `010-Stage`, `020-Core` and `030-Curated`; Raw notebooks are derived from the external sources. @@ -72,29 +72,30 @@ finishes with `Complete_Load`. Source references become task dependencies; relationships used by `lookup_dimensions` also gate the dependent entity task. Cross-job dependencies remain in the generated execution manifest. A load job does not start another model job or process its entities; it reads the existing -upstream data. The local and CI integration runner runs every selected model -job once in dependency order and stops before downstream jobs if a prerequisite -fails. A manually started downstream job expects its upstream data to be current. +upstream data. Run jobs in the manifest dependency order and wait for each +upstream job to succeed before starting a dependent job. No live integration runner +is included. A downstream job expects its upstream data to be current. Schedule and compute settings come from the modeled job values and bundle configuration. ## Generated layout The target is selected as **databricks**. Its configured output root is -`Output/databricks-lake/generated`; payloads write notebooks and bundle YAML +`output/databricks`; payloads write notebooks and bundle YAML beneath it: ```text -Output/databricks-lake/ - utils/ # Shared runtime notebooks/frameworks - generated/ - notebooks//ddl///.py - notebooks//dml///.py - schemas/schema.yml - clusters/clusters.yml - jobs/create_all_tables.yml - jobs//Create_.yml # One parallel DDL task group per zone - jobs/load_.yml +output/databricks/ + notebooks//ddl///.py + notebooks//dml///.py + runtime/uc_sources.py + schemas/schema.yml + clusters/clusters.yml + jobs/create_all_tables.yml + jobs//Create_.yml + jobs/load_.yml +utils/ # Authored runtime notebooks +deployment/databricks/databricks.yml # Authored bundle configuration ``` The sample therefore generates `load_sales_daily.yml` and @@ -104,9 +105,9 @@ assignments and the other target-specific properties. The exact notebook subdirectories follow the model's folder hierarchy. Cluster settings are emitted when cluster metadata is configured. Generated notebooks -reference the authored `utils` directory. The sample contains no environment-bound -bundle entrypoint; review [deployment boundaries](DEPLOYMENT.md) before configuring -your own platform delivery process. +reference the authored `utils` directory. The neutral bundle includes only this +target output and `utils/`. Follow [deployment](DEPLOYMENT.md) to supply your own +identity, catalog, source connection and existing compatible compute. ## Generate and use the result diff --git a/Generate/databricks-lake/__modules/jobs_helpers.py b/generate/databricks-lake/__modules/jobs_helpers.py similarity index 100% rename from Generate/databricks-lake/__modules/jobs_helpers.py rename to generate/databricks-lake/__modules/jobs_helpers.py diff --git a/Generate/databricks-lake/__modules/metadata_utils.py b/generate/databricks-lake/__modules/metadata_utils.py similarity index 99% rename from Generate/databricks-lake/__modules/metadata_utils.py rename to generate/databricks-lake/__modules/metadata_utils.py index 0e51713..a6bb5a7 100644 --- a/Generate/databricks-lake/__modules/metadata_utils.py +++ b/generate/databricks-lake/__modules/metadata_utils.py @@ -128,7 +128,7 @@ def _select_expr(source_expr: str, target: str) -> str: @dataclass(frozen=True) class ZoneMetadata: - """Represents a zone entry from Base/Zones.json.""" + """Represents a zone entry from base/Zones.json.""" name: str display_name: str @@ -349,7 +349,7 @@ def zone_folder_name(self, zone: ZoneMetadata) -> str: return zone.local_folder or zone.name def zones(self) -> list[ZoneMetadata]: - """Return all known zones defined in Base/Zones.json.""" + """Return all known zones defined in base/Zones.json.""" _, name_map = self._zones_maps # name_map already keyed by lower-case zone name; values are the ZoneMetadata instances. # Preserve insertion order from the JSON file by relying on dict value order. diff --git a/Generate/databricks-lake/__modules/payload.py b/generate/databricks-lake/__modules/payload.py similarity index 100% rename from Generate/databricks-lake/__modules/payload.py rename to generate/databricks-lake/__modules/payload.py diff --git a/Generate/databricks-lake/__modules/payload_helpers.py b/generate/databricks-lake/__modules/payload_helpers.py similarity index 100% rename from Generate/databricks-lake/__modules/payload_helpers.py rename to generate/databricks-lake/__modules/payload_helpers.py diff --git a/Generate/databricks-lake/clusters/clusters.yml.jinja2 b/generate/databricks-lake/clusters/clusters.yml.jinja2 similarity index 100% rename from Generate/databricks-lake/clusters/clusters.yml.jinja2 rename to generate/databricks-lake/clusters/clusters.yml.jinja2 diff --git a/Generate/databricks-lake/columns.py.jinja2 b/generate/databricks-lake/columns.py.jinja2 similarity index 100% rename from Generate/databricks-lake/columns.py.jinja2 rename to generate/databricks-lake/columns.py.jinja2 diff --git a/Generate/databricks-lake/data_sources.md.jinja2 b/generate/databricks-lake/data_sources.md.jinja2 similarity index 100% rename from Generate/databricks-lake/data_sources.md.jinja2 rename to generate/databricks-lake/data_sources.md.jinja2 diff --git a/Generate/databricks-lake/ddl_notebook.py.jinja2 b/generate/databricks-lake/ddl_notebook.py.jinja2 similarity index 93% rename from Generate/databricks-lake/ddl_notebook.py.jinja2 rename to generate/databricks-lake/ddl_notebook.py.jinja2 index 0af0530..d57799b 100644 --- a/Generate/databricks-lake/ddl_notebook.py.jinja2 +++ b/generate/databricks-lake/ddl_notebook.py.jinja2 @@ -31,13 +31,13 @@ from delta.tables import DeltaTable # COMMAND ---------- -# MAGIC %run ../../../../../../utils/MigrationFramework +# MAGIC %run ../../../../../../../utils/MigrationFramework # COMMAND ---------- dbutils.widgets.text("env", "dev", "Environment") -dbutils.widgets.text("catalog_name", "datam8_campus_dev_fka", "Catalog Name") -dbutils.widgets.text("owner", "datam8_sample_dev_owner", "Owner") +dbutils.widgets.text("catalog_name", "", "Catalog Name") +dbutils.widgets.text("owner", "", "Owner") dbutils.widgets.text("job_run_id", "", "Job Run ID") # dbutils.widgets.dropdown( @@ -54,6 +54,8 @@ dbutils.widgets.text("job_run_id", "", "Job Run ID") env = dbutils.widgets.get("env") catalog_name = dbutils.widgets.get("catalog_name") owner = dbutils.widgets.get("owner") +if not catalog_name or not owner: + raise ValueError("Set catalog_name and owner explicitly before running this notebook") job_run_id = dbutils.widgets.get("job_run_id") # run_mode = dbutils.widgets.get("run_mode") # sandbox = dbutils.widgets.get("sandbox") diff --git a/Generate/databricks-lake/ddl_notebook_raw.py.jinja2 b/generate/databricks-lake/ddl_notebook_raw.py.jinja2 similarity index 93% rename from Generate/databricks-lake/ddl_notebook_raw.py.jinja2 rename to generate/databricks-lake/ddl_notebook_raw.py.jinja2 index ecd7a2b..ee14e63 100644 --- a/Generate/databricks-lake/ddl_notebook_raw.py.jinja2 +++ b/generate/databricks-lake/ddl_notebook_raw.py.jinja2 @@ -31,13 +31,13 @@ from delta.tables import DeltaTable # COMMAND ---------- -# MAGIC %run ../../../../../../utils/MigrationFramework +# MAGIC %run ../../../../../../../utils/MigrationFramework # COMMAND ---------- dbutils.widgets.text("env", "dev", "Environment") -dbutils.widgets.text("catalog_name", "datam8_campus_dev_fka", "Catalog Name") -dbutils.widgets.text("owner", "datam8_sample_dev_owner", "Owner") +dbutils.widgets.text("catalog_name", "", "Catalog Name") +dbutils.widgets.text("owner", "", "Owner") dbutils.widgets.text("job_run_id", "", "Job Run ID") # dbutils.widgets.dropdown( @@ -54,6 +54,8 @@ dbutils.widgets.text("job_run_id", "", "Job Run ID") env = dbutils.widgets.get("env") catalog_name = dbutils.widgets.get("catalog_name") owner = dbutils.widgets.get("owner") +if not catalog_name or not owner: + raise ValueError("Set catalog_name and owner explicitly before running this notebook") job_run_id = dbutils.widgets.get("job_run_id") # run_mode = dbutils.widgets.get("run_mode") # sandbox = dbutils.widgets.get("sandbox") diff --git a/Generate/databricks-lake/dml_function.py.jinja2 b/generate/databricks-lake/dml_function.py.jinja2 similarity index 100% rename from Generate/databricks-lake/dml_function.py.jinja2 rename to generate/databricks-lake/dml_function.py.jinja2 diff --git a/Generate/databricks-lake/dml_notebook.py.jinja2 b/generate/databricks-lake/dml_notebook.py.jinja2 similarity index 98% rename from Generate/databricks-lake/dml_notebook.py.jinja2 rename to generate/databricks-lake/dml_notebook.py.jinja2 index a23372e..429cb1a 100644 --- a/Generate/databricks-lake/dml_notebook.py.jinja2 +++ b/generate/databricks-lake/dml_notebook.py.jinja2 @@ -17,15 +17,15 @@ from delta import DeltaTable # noqa: F401 # DBTITLE 1,Initialize Migration Framework # MAGIC -# MAGIC %run ../../../../../../utils/MigrationFramework +# MAGIC %run ../../../../../../../utils/MigrationFramework # COMMAND ---------- dbutils.widgets.text("date_start", "2000-01-01", "Calendar start") dbutils.widgets.text("date_end", "2030-12-31", "Calendar end") dbutils.widgets.text("env", "dev", "Environment") -dbutils.widgets.text("catalog_name", "datam8_campus_dev_fka", "Catalog Name") -dbutils.widgets.text("owner", "datam8_sample_dev_owner", "Owner") +dbutils.widgets.text("catalog_name", "", "Catalog Name") +dbutils.widgets.text("owner", "", "Owner") dbutils.widgets.text("job_run_id", "", "Job Run ID") # dbutils.widgets.dropdown( @@ -45,6 +45,8 @@ sample_date_end = dbutils.widgets.get("date_end") env = dbutils.widgets.get("env") catalog_name = dbutils.widgets.get("catalog_name") owner = dbutils.widgets.get("owner") +if not catalog_name or not owner: + raise ValueError("Set catalog_name and owner explicitly before running this notebook") job_run_id = dbutils.widgets.get("job_run_id") # run_mode = dbutils.widgets.get("run_mode") diff --git a/Generate/databricks-lake/dml_notebook_raw.py.jinja2 b/generate/databricks-lake/dml_notebook_raw.py.jinja2 similarity index 93% rename from Generate/databricks-lake/dml_notebook_raw.py.jinja2 rename to generate/databricks-lake/dml_notebook_raw.py.jinja2 index 8344d22..33a8cd6 100644 --- a/Generate/databricks-lake/dml_notebook_raw.py.jinja2 +++ b/generate/databricks-lake/dml_notebook_raw.py.jinja2 @@ -13,14 +13,14 @@ from pyspark.sql import functions as F # noqa: F401 # COMMAND ---------- -# MAGIC %run ../../../../../../utils/MigrationFramework +# MAGIC %run ../../../../../../../utils/MigrationFramework # COMMAND ---------- dbutils.widgets.text("connection_prefix", "sample_", "UC Connection Prefix") dbutils.widgets.text("env", "dev", "Environment") -dbutils.widgets.text("catalog_name", "datam8_campus_dev_fka", "Catalog Name") -dbutils.widgets.text("owner", "datam8_sample_dev_owner", "Owner") +dbutils.widgets.text("catalog_name", "", "Catalog Name") +dbutils.widgets.text("owner", "", "Owner") dbutils.widgets.text("job_run_id", "", "Job Run ID") # dbutils.widgets.dropdown( @@ -37,6 +37,8 @@ dbutils.widgets.text("job_run_id", "", "Job Run ID") env = dbutils.widgets.get("env") catalog_name = dbutils.widgets.get("catalog_name") owner = dbutils.widgets.get("owner") +if not catalog_name or not owner: + raise ValueError("Set catalog_name and owner explicitly before running this notebook") job_run_id = dbutils.widgets.get("job_run_id") # run_mode = dbutils.widgets.get("run_mode") # sandbox = dbutils.widgets.get("sandbox") @@ -108,7 +110,7 @@ if not source_location: from pathlib import Path import sys notebook_folder = Path(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()).parent -# DML notebooks are generated/notebooks//dml///. +# DML notebooks are output/databricks/notebooks//dml///. sys.path.insert(0, str(Path("/Workspace") / notebook_folder.relative_to("/").parents[4] / "runtime")) from uc_sources import read_database, read_http {% if data.plugin == "http_api" %} diff --git a/Generate/databricks-lake/jobs/create_all.yml.jinja2 b/generate/databricks-lake/jobs/create_all.yml.jinja2 similarity index 82% rename from Generate/databricks-lake/jobs/create_all.yml.jinja2 rename to generate/databricks-lake/jobs/create_all.yml.jinja2 index 3988fa0..f280959 100644 --- a/Generate/databricks-lake/jobs/create_all.yml.jinja2 +++ b/generate/databricks-lake/jobs/create_all.yml.jinja2 @@ -2,17 +2,12 @@ resources: jobs: {{ data.job_key }}: name: {{ ("[${var.environment}] " + data.job_name) | tojson }} - permissions: - - service_principal_name: ${var.deploy_service_principal} - level: IS_OWNER - - service_principal_name: ${var.run_as_service_principal} - level: CAN_MANAGE_RUN max_concurrent_runs: 1 timeout_seconds: 1800 tags: platform: databricks-lake parameters: - - name: environment + - name: env default: ${var.environment} - name: date_start default: ${var.date_start} @@ -44,7 +39,7 @@ resources: job_id: ${resources.jobs.{{ task.job_ref }}.id} {% else %} notebook_task: - notebook_path: ${workspace.root_path}/${var.notebook_path}/{{ task.notebook_path }} + notebook_path: ${workspace.file_path}/output/databricks/notebooks/{{ task.notebook_path }} source: WORKSPACE {%- set task_cluster_ref = task.cluster_ref or data.cluster_ref %} {%- if task_cluster_ref %} diff --git a/Generate/databricks-lake/jobs/create_zone.yml.jinja2 b/generate/databricks-lake/jobs/create_zone.yml.jinja2 similarity index 84% rename from Generate/databricks-lake/jobs/create_zone.yml.jinja2 rename to generate/databricks-lake/jobs/create_zone.yml.jinja2 index 23d4e50..7d9b199 100644 --- a/Generate/databricks-lake/jobs/create_zone.yml.jinja2 +++ b/generate/databricks-lake/jobs/create_zone.yml.jinja2 @@ -2,16 +2,11 @@ resources: jobs: {{ data.job_key }}: name: {{ ("[${var.environment}] " + data.job_name) | tojson }} - permissions: - - service_principal_name: ${var.deploy_service_principal} - level: IS_OWNER - - service_principal_name: ${var.run_as_service_principal} - level: CAN_MANAGE_RUN tags: platform: databricks-lake max_concurrent_runs: 1 parameters: - - name: environment + - name: env default: ${var.environment} - name: date_start default: ${var.date_start} @@ -52,7 +47,7 @@ resources: {% endfor %} {% endif %} notebook_task: - notebook_path: ${workspace.root_path}/${var.notebook_path}/{{ task.notebook_path }} + notebook_path: ${workspace.file_path}/output/databricks/notebooks/{{ task.notebook_path }} base_parameters: replication_integration_status_id: '1' execution_status_id: '1' diff --git a/Generate/databricks-lake/jobs/load_job_group.yml.jinja2 b/generate/databricks-lake/jobs/load_job_group.yml.jinja2 similarity index 86% rename from Generate/databricks-lake/jobs/load_job_group.yml.jinja2 rename to generate/databricks-lake/jobs/load_job_group.yml.jinja2 index fc5f4b3..530eb09 100644 --- a/Generate/databricks-lake/jobs/load_job_group.yml.jinja2 +++ b/generate/databricks-lake/jobs/load_job_group.yml.jinja2 @@ -2,16 +2,11 @@ resources: jobs: {{ data.job_key }}: name: {{ ("[${var.environment}] " + data.job_name) | tojson }} - permissions: - - service_principal_name: ${var.deploy_service_principal} - level: IS_OWNER - - service_principal_name: ${var.run_as_service_principal} - level: CAN_MANAGE_RUN tags: platform: databricks-lake max_concurrent_runs: 1 parameters: - - name: environment + - name: env default: ${var.environment} - name: date_start default: ${var.date_start} @@ -51,7 +46,7 @@ resources: tasks: - task_key: Start_Load notebook_task: - notebook_path: ${workspace.root_path}/files/utils/Start_Load + notebook_path: ${workspace.file_path}/utils/Start_Load {% if data.job_value %} base_parameters: target_zone: {{ data.job_value }} @@ -71,7 +66,7 @@ resources: {%- endfor %} {%- endif %} notebook_task: - notebook_path: ${workspace.root_path}/${var.notebook_path}/{{ task.notebook_path }} + notebook_path: ${workspace.file_path}/output/databricks/notebooks/{{ task.notebook_path }} base_parameters: replication_integration_status_id: '1' execution_status_id: '1' @@ -91,7 +86,7 @@ resources: {%- endfor %} {%- endif %} notebook_task: - notebook_path: ${workspace.root_path}/${var.notebook_path}/{{ task.notebook_path }} + notebook_path: ${workspace.file_path}/output/databricks/notebooks/{{ task.notebook_path }} base_parameters: replication_integration_status_id: '1' execution_status_id: '1' @@ -110,7 +105,7 @@ resources: {%- endfor %} run_if: ALL_DONE notebook_task: - notebook_path: ${workspace.root_path}/files/utils/Complete_Load + notebook_path: ${workspace.file_path}/utils/Complete_Load source: WORKSPACE {% if data.serverless %} environment_key: serverless diff --git a/Generate/databricks-lake/runtime_file.jinja2 b/generate/databricks-lake/runtime_file.jinja2 similarity index 100% rename from Generate/databricks-lake/runtime_file.jinja2 rename to generate/databricks-lake/runtime_file.jinja2 diff --git a/Generate/databricks-lake/schema.yml.jinja2 b/generate/databricks-lake/schema.yml.jinja2 similarity index 100% rename from Generate/databricks-lake/schema.yml.jinja2 rename to generate/databricks-lake/schema.yml.jinja2 diff --git a/Generate/databricks-lake/zones.jinja2 b/generate/databricks-lake/zones.jinja2 similarity index 100% rename from Generate/databricks-lake/zones.jinja2 rename to generate/databricks-lake/zones.jinja2 diff --git a/Generate/docs/ARCHITECTURE.md b/generate/docs/ARCHITECTURE.md similarity index 96% rename from Generate/docs/ARCHITECTURE.md rename to generate/docs/ARCHITECTURE.md index d3cfc7e..00410b1 100644 --- a/Generate/docs/ARCHITECTURE.md +++ b/generate/docs/ARCHITECTURE.md @@ -39,7 +39,7 @@ This target produces model documentation artifacts: ## Validation Checklist -1. Run lint: `ruff check Generate/docs/__modules` +1. Run lint: `ruff check generate/docs/__modules` 2. Generate docs and verify: - entity links are correct - diagram opens in draw.io diff --git a/Generate/docs/PROPERTIES.md b/generate/docs/PROPERTIES.md similarity index 94% rename from Generate/docs/PROPERTIES.md rename to generate/docs/PROPERTIES.md index 6bf0670..17e2914 100644 --- a/Generate/docs/PROPERTIES.md +++ b/generate/docs/PROPERTIES.md @@ -7,7 +7,7 @@ - **External source:** set values in its `sources[]` entry, connected to a DataSource. - **Source mapping:** mark a cursor with `extract_column=delta`. - **Attribute:** business keys and SCD history are model fields. -- **Reusable values:** define them in `Base/PropertyValues.json`; jobs may reference schedules and clusters. +- **Reusable values:** define them in `base/PropertyValues.json`; jobs may reference schedules and clusters. Use the value's `name` in a property reference. Effects and precedence can differ by target; each target page documents its contract. diff --git a/Generate/docs/README.md b/generate/docs/README.md similarity index 84% rename from Generate/docs/README.md rename to generate/docs/README.md index 74af656..0c107a4 100644 --- a/Generate/docs/README.md +++ b/generate/docs/README.md @@ -23,7 +23,7 @@ model changes so that the reference reflects the current metadata. ## Generated artifacts -The target name is **docs** and its configured output root is `Output/docs`. +The target name is **docs** and its configured output root is `output/docs`. | Artifact | Role | | --- | --- | @@ -36,15 +36,15 @@ generated pages, so it is the starting point for browsing the reference. ## Generate and use the result -Use **DataM8 app 2.0.0**, select **docs** in the generator and choose **Generate**. -Or run **DataM8 CLI 2.0.0** from the solution root: +Use a compatible **DataM8 app**, select **docs** in the generator and choose **Generate**. +Or run the pinned **DataM8 CLI** from the solution root: ```sh -datam8 generate docs --solution ./ORAYLISDatabricksSample.dm8s +uv run datam8 generate docs --solution ./ORAYLISDatabricksSample.dm8s ``` -Open `Output/docs/index.md` in a Markdown viewer and follow its entity links. Open -`Output/docs/diagrams/entity-relationships.drawio` in draw.io to explore or arrange +Open `output/docs/index.md` in a Markdown viewer and follow its entity links. Open +`output/docs/diagrams/entity-relationships.drawio` in draw.io to explore or arrange the diagram. Keep the generated Markdown and diagram together when sharing the reference or including it in your project's documentation. diff --git a/Generate/docs/__modules/documentation.py b/generate/docs/__modules/documentation.py similarity index 99% rename from Generate/docs/__modules/documentation.py rename to generate/docs/__modules/documentation.py index e7d5076..d65e6f3 100644 --- a/Generate/docs/__modules/documentation.py +++ b/generate/docs/__modules/documentation.py @@ -2,7 +2,7 @@ The builder denormalizes entities, sources, relationships, transformations, and diagram coordinates into a single `DocumentationResult` consumed by Jinja -templates in `Generate/docs`. +templates in `generate/docs`. """ from __future__ import annotations diff --git a/Generate/docs/__modules/payload.py b/generate/docs/__modules/payload.py similarity index 100% rename from Generate/docs/__modules/payload.py rename to generate/docs/__modules/payload.py diff --git a/Generate/docs/entity.md.jinja2 b/generate/docs/entity.md.jinja2 similarity index 100% rename from Generate/docs/entity.md.jinja2 rename to generate/docs/entity.md.jinja2 diff --git a/Generate/docs/er_diagram.drawio.jinja2 b/generate/docs/er_diagram.drawio.jinja2 similarity index 100% rename from Generate/docs/er_diagram.drawio.jinja2 rename to generate/docs/er_diagram.drawio.jinja2 diff --git a/Generate/docs/index.md.jinja2 b/generate/docs/index.md.jinja2 similarity index 100% rename from Generate/docs/index.md.jinja2 rename to generate/docs/index.md.jinja2 diff --git a/Generate/powerbi-tabular/ARCHITECTURE.md b/generate/powerbi-tabular/ARCHITECTURE.md similarity index 96% rename from Generate/powerbi-tabular/ARCHITECTURE.md rename to generate/powerbi-tabular/ARCHITECTURE.md index 261f548..8272d45 100644 --- a/Generate/powerbi-tabular/ARCHITECTURE.md +++ b/generate/powerbi-tabular/ARCHITECTURE.md @@ -47,5 +47,5 @@ This target generates TMDL artifacts for consumer entities: ## Validation Checklist -1. Run lint: `ruff check Generate/powerbi-tabular/__modules` +1. Run lint: `ruff check generate/powerbi-tabular/__modules` 2. Generate output and verify table/relationship names with spaces still quote correctly. diff --git a/Generate/powerbi-tabular/PROPERTIES.md b/generate/powerbi-tabular/PROPERTIES.md similarity index 95% rename from Generate/powerbi-tabular/PROPERTIES.md rename to generate/powerbi-tabular/PROPERTIES.md index 537cef2..72e62bb 100644 --- a/Generate/powerbi-tabular/PROPERTIES.md +++ b/generate/powerbi-tabular/PROPERTIES.md @@ -7,7 +7,7 @@ - **External source:** set values in its `sources[]` entry, connected to a DataSource. - **Source mapping:** mark a cursor with `extract_column=delta`. - **Attribute:** business keys and SCD history are model fields. -- **Reusable values:** define them in `Base/PropertyValues.json`; jobs may reference schedules and clusters. +- **Reusable values:** define them in `base/PropertyValues.json`; jobs may reference schedules and clusters. Use the value's `name` in a property reference. Effects and precedence can differ by target; each target page documents its contract. diff --git a/Generate/powerbi-tabular/README.md b/generate/powerbi-tabular/README.md similarity index 82% rename from Generate/powerbi-tabular/README.md rename to generate/powerbi-tabular/README.md index bac3ed5..733c436 100644 --- a/Generate/powerbi-tabular/README.md +++ b/generate/powerbi-tabular/README.md @@ -2,7 +2,7 @@ Build the semantic model that turns your consumer-layer metadata into tables, relationships and measures for Power BI reporting. This target generates a TMDL -model definition from the entities in `Model/040-Consumer`, keeping the reporting +model definition from the entities in `model/040-Consumer`, keeping the reporting model connected to the same metadata used for your data platform. ## What you can build @@ -30,9 +30,7 @@ lakehouse tables; this target supplies their semantic reporting layer. ## Generated artifacts The target name is **powerbi** and its configured output root is -`Output/powerbi-tabular/generated`. Payloads add a `powerbi-tabular` subdirectory, -so the model definition is written to -`Output/powerbi-tabular/generated/powerbi-tabular/`. +`output/powerbi`. TMDL files and `tables/` are written directly inside it. | Artifact | Role | | --- | --- | @@ -44,15 +42,15 @@ so the model definition is written to ## Generate and use the result -Use **DataM8 app 2.0.0**, select **powerbi** in the generator and choose -**Generate**. Or run **DataM8 CLI 2.0.0** from the solution root: +Use a compatible **DataM8 app**, select **powerbi** in the generator and choose +**Generate**. Or run the pinned **DataM8 CLI** from the solution root: ```sh -datam8 generate powerbi --solution ./ORAYLISDatabricksSample.dm8s +uv run datam8 generate powerbi --solution ./ORAYLISDatabricksSample.dm8s ``` Inspect the generated tables, relationship columns and measures. Integrate the -TMDL definition into your semantic-model project and supply the Databricks +TMDL definition into [your own Desktop/PBIP project](../../docs/powerbi-deployment.md) and supply the Databricks connection parameters for your environment. Publish and configure refresh through your Power BI deployment workflow; reports can then use the shared semantic model. diff --git a/Generate/powerbi-tabular/__modules/payload.py b/generate/powerbi-tabular/__modules/payload.py similarity index 98% rename from Generate/powerbi-tabular/__modules/payload.py rename to generate/powerbi-tabular/__modules/payload.py index 5d1d077..dc0422c 100644 --- a/Generate/powerbi-tabular/__modules/payload.py +++ b/generate/powerbi-tabular/__modules/payload.py @@ -131,6 +131,7 @@ def _map_data_type(attribute_type: str, logical_type: str) -> str: mapping: dict[str, str] = { "long": "int64", "int": "int64", + "integer": "int64", "short": "int64", "double": "double", "decimal": "decimal", @@ -519,7 +520,7 @@ def database_payload(model: Model, cache: Cache) -> Sequence[IPayload]: return [ BasePayload( data=data, - output_path=Path("powerbi-tabular", "database.tmdl"), + output_path=Path("database.tmdl"), ) ] @@ -539,7 +540,7 @@ def model_payload(model: Model, cache: Cache) -> Sequence[IPayload]: return [ BasePayload( data=data, - output_path=Path("powerbi-tabular", "model.tmdl"), + output_path=Path("model.tmdl"), ) ] @@ -579,7 +580,7 @@ def expressions_payload(model: Model, cache: Cache) -> Sequence[IPayload]: return [ BasePayload( data={"expressions": expressions}, - output_path=Path("powerbi-tabular", "expressions.tmdl"), + output_path=Path("expressions.tmdl"), ) ] @@ -593,7 +594,7 @@ def relationships_payload(model: Model, cache: Cache) -> Sequence[IPayload]: return [ BasePayload( data={"relationships": relationships}, - output_path=Path("powerbi-tabular", "relationships.tmdl"), + output_path=Path("relationships.tmdl"), ) ] @@ -608,7 +609,7 @@ def table_payloads(model: Model, cache: Cache) -> Sequence[IPayload]: payloads.append( BasePayload( data={"table": table}, - output_path=Path("powerbi-tabular", "tables", f"{table.name}.tmdl"), + output_path=Path("tables", f"{table.name}.tmdl"), ) ) diff --git a/Generate/powerbi-tabular/database.tmdl.jinja2 b/generate/powerbi-tabular/database.tmdl.jinja2 similarity index 100% rename from Generate/powerbi-tabular/database.tmdl.jinja2 rename to generate/powerbi-tabular/database.tmdl.jinja2 diff --git a/Generate/powerbi-tabular/expressions.tmdl.jinja2 b/generate/powerbi-tabular/expressions.tmdl.jinja2 similarity index 100% rename from Generate/powerbi-tabular/expressions.tmdl.jinja2 rename to generate/powerbi-tabular/expressions.tmdl.jinja2 diff --git a/Generate/powerbi-tabular/model.tmdl.jinja2 b/generate/powerbi-tabular/model.tmdl.jinja2 similarity index 100% rename from Generate/powerbi-tabular/model.tmdl.jinja2 rename to generate/powerbi-tabular/model.tmdl.jinja2 diff --git a/Generate/powerbi-tabular/relationships.tmdl.jinja2 b/generate/powerbi-tabular/relationships.tmdl.jinja2 similarity index 100% rename from Generate/powerbi-tabular/relationships.tmdl.jinja2 rename to generate/powerbi-tabular/relationships.tmdl.jinja2 diff --git a/Generate/powerbi-tabular/table.tmdl.jinja2 b/generate/powerbi-tabular/table.tmdl.jinja2 similarity index 100% rename from Generate/powerbi-tabular/table.tmdl.jinja2 rename to generate/powerbi-tabular/table.tmdl.jinja2 diff --git a/Generate/shared/uc_sources.py b/generate/shared/uc_sources.py similarity index 100% rename from Generate/shared/uc_sources.py rename to generate/shared/uc_sources.py diff --git a/main.py b/main.py deleted file mode 100644 index 9656873..0000000 --- a/main.py +++ /dev/null @@ -1,6 +0,0 @@ -def main(): - print("Hello from datam8-sample-solution!") - - -if __name__ == "__main__": - main() diff --git a/Model/010-Stage/AdventureWorks/.properties.json b/model/010-Stage/AdventureWorks/.properties.json similarity index 100% rename from Model/010-Stage/AdventureWorks/.properties.json rename to model/010-Stage/AdventureWorks/.properties.json diff --git a/Model/010-Stage/AdventureWorks/SalesLT/.properties.json b/model/010-Stage/AdventureWorks/SalesLT/.properties.json similarity index 100% rename from Model/010-Stage/AdventureWorks/SalesLT/.properties.json rename to model/010-Stage/AdventureWorks/SalesLT/.properties.json diff --git a/Model/010-Stage/AdventureWorks/SalesLT/Address.json b/model/010-Stage/AdventureWorks/SalesLT/Address.json similarity index 100% rename from Model/010-Stage/AdventureWorks/SalesLT/Address.json rename to model/010-Stage/AdventureWorks/SalesLT/Address.json diff --git a/Model/010-Stage/AdventureWorks/SalesLT/Customer.json b/model/010-Stage/AdventureWorks/SalesLT/Customer.json similarity index 100% rename from Model/010-Stage/AdventureWorks/SalesLT/Customer.json rename to model/010-Stage/AdventureWorks/SalesLT/Customer.json diff --git a/Model/010-Stage/AdventureWorks/SalesLT/CustomerAddress.json b/model/010-Stage/AdventureWorks/SalesLT/CustomerAddress.json similarity index 100% rename from Model/010-Stage/AdventureWorks/SalesLT/CustomerAddress.json rename to model/010-Stage/AdventureWorks/SalesLT/CustomerAddress.json diff --git a/Model/010-Stage/AdventureWorks/SalesLT/SalesOrderDetail.json b/model/010-Stage/AdventureWorks/SalesLT/SalesOrderDetail.json similarity index 100% rename from Model/010-Stage/AdventureWorks/SalesLT/SalesOrderDetail.json rename to model/010-Stage/AdventureWorks/SalesLT/SalesOrderDetail.json diff --git a/Model/010-Stage/AdventureWorks/SalesLT/SalesOrderHeader.json b/model/010-Stage/AdventureWorks/SalesLT/SalesOrderHeader.json similarity index 100% rename from Model/010-Stage/AdventureWorks/SalesLT/SalesOrderHeader.json rename to model/010-Stage/AdventureWorks/SalesLT/SalesOrderHeader.json diff --git a/Model/020-Core/MasterData/.properties.json b/model/020-Core/MasterData/.properties.json similarity index 100% rename from Model/020-Core/MasterData/.properties.json rename to model/020-Core/MasterData/.properties.json diff --git a/Model/020-Core/MasterData/Address/Address.json b/model/020-Core/MasterData/Address/Address.json similarity index 100% rename from Model/020-Core/MasterData/Address/Address.json rename to model/020-Core/MasterData/Address/Address.json diff --git a/Model/020-Core/MasterData/Address/Address/Address.py b/model/020-Core/MasterData/Address/Address/Address.py similarity index 100% rename from Model/020-Core/MasterData/Address/Address/Address.py rename to model/020-Core/MasterData/Address/Address/Address.py diff --git a/Model/020-Core/MasterData/Customer/Customer.json b/model/020-Core/MasterData/Customer/Customer.json similarity index 100% rename from Model/020-Core/MasterData/Customer/Customer.json rename to model/020-Core/MasterData/Customer/Customer.json diff --git a/Model/020-Core/MasterData/Customer/Customer/CustomerNew.py b/model/020-Core/MasterData/Customer/Customer/CustomerNew.py similarity index 100% rename from Model/020-Core/MasterData/Customer/Customer/CustomerNew.py rename to model/020-Core/MasterData/Customer/Customer/CustomerNew.py diff --git a/Model/020-Core/MasterData/Customer/CustomerAddress.json b/model/020-Core/MasterData/Customer/CustomerAddress.json similarity index 100% rename from Model/020-Core/MasterData/Customer/CustomerAddress.json rename to model/020-Core/MasterData/Customer/CustomerAddress.json diff --git a/Model/020-Core/MasterData/Customer/CustomerAddress/CustomerAddress.py b/model/020-Core/MasterData/Customer/CustomerAddress/CustomerAddress.py similarity index 100% rename from Model/020-Core/MasterData/Customer/CustomerAddress/CustomerAddress.py rename to model/020-Core/MasterData/Customer/CustomerAddress/CustomerAddress.py diff --git a/Model/030-Curated/MasterData/.properties.json b/model/030-Curated/MasterData/.properties.json similarity index 100% rename from Model/030-Curated/MasterData/.properties.json rename to model/030-Curated/MasterData/.properties.json diff --git a/Model/030-Curated/MasterData/Calendar/DimDate.json b/model/030-Curated/MasterData/Calendar/DimDate.json similarity index 100% rename from Model/030-Curated/MasterData/Calendar/DimDate.json rename to model/030-Curated/MasterData/Calendar/DimDate.json diff --git a/Model/030-Curated/MasterData/Calendar/DimDate/GenerateDates.py b/model/030-Curated/MasterData/Calendar/DimDate/GenerateDates.py similarity index 100% rename from Model/030-Curated/MasterData/Calendar/DimDate/GenerateDates.py rename to model/030-Curated/MasterData/Calendar/DimDate/GenerateDates.py diff --git a/Model/030-Curated/MasterData/Customer/DimCustomer.json b/model/030-Curated/MasterData/Customer/DimCustomer.json similarity index 100% rename from Model/030-Curated/MasterData/Customer/DimCustomer.json rename to model/030-Curated/MasterData/Customer/DimCustomer.json diff --git a/Model/030-Curated/MasterData/Customer/DimCustomer/DimCustomer.py b/model/030-Curated/MasterData/Customer/DimCustomer/DimCustomer.py similarity index 100% rename from Model/030-Curated/MasterData/Customer/DimCustomer/DimCustomer.py rename to model/030-Curated/MasterData/Customer/DimCustomer/DimCustomer.py diff --git a/Model/030-Curated/Sales/.properties.json b/model/030-Curated/Sales/.properties.json similarity index 100% rename from Model/030-Curated/Sales/.properties.json rename to model/030-Curated/Sales/.properties.json diff --git a/Model/030-Curated/Sales/Orders/FactSalesOrder.json b/model/030-Curated/Sales/Orders/FactSalesOrder.json similarity index 100% rename from Model/030-Curated/Sales/Orders/FactSalesOrder.json rename to model/030-Curated/Sales/Orders/FactSalesOrder.json diff --git a/Model/030-Curated/Sales/Orders/FactSalesOrder/FactSalesOrder.py b/model/030-Curated/Sales/Orders/FactSalesOrder/FactSalesOrder.py similarity index 100% rename from Model/030-Curated/Sales/Orders/FactSalesOrder/FactSalesOrder.py rename to model/030-Curated/Sales/Orders/FactSalesOrder/FactSalesOrder.py diff --git a/Model/040-Consumer/MasterData/.properties.json b/model/040-Consumer/MasterData/.properties.json similarity index 100% rename from Model/040-Consumer/MasterData/.properties.json rename to model/040-Consumer/MasterData/.properties.json diff --git a/Model/040-Consumer/MasterData/Calendar/Date.json b/model/040-Consumer/MasterData/Calendar/Date.json similarity index 100% rename from Model/040-Consumer/MasterData/Calendar/Date.json rename to model/040-Consumer/MasterData/Calendar/Date.json diff --git a/Model/040-Consumer/MasterData/Customer/Customer.json b/model/040-Consumer/MasterData/Customer/Customer.json similarity index 100% rename from Model/040-Consumer/MasterData/Customer/Customer.json rename to model/040-Consumer/MasterData/Customer/Customer.json diff --git a/Model/040-Consumer/MasterData/Reporting/Measure.json b/model/040-Consumer/MasterData/Reporting/Measure.json similarity index 100% rename from Model/040-Consumer/MasterData/Reporting/Measure.json rename to model/040-Consumer/MasterData/Reporting/Measure.json diff --git a/Model/040-Consumer/Sales/.properties.json b/model/040-Consumer/Sales/.properties.json similarity index 100% rename from Model/040-Consumer/Sales/.properties.json rename to model/040-Consumer/Sales/.properties.json diff --git a/Model/040-Consumer/Sales/Orders/Sales Order.json b/model/040-Consumer/Sales/Orders/Sales Order.json similarity index 100% rename from Model/040-Consumer/Sales/Orders/Sales Order.json rename to model/040-Consumer/Sales/Orders/Sales Order.json diff --git a/Diagram/.gitkeep b/plugins/.gitkeep similarity index 100% rename from Diagram/.gitkeep rename to plugins/.gitkeep diff --git a/pyproject.toml b/pyproject.toml index 89404c9..3e7882f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ readme = "README.md" requires-python = ">=3.12" dependencies = [ "datam8[sql]", + "pathspec==1.1.1", "PyYAML==6.0.3", ] @@ -16,10 +17,13 @@ Homepage = "https://github.com/oraylis/datam8-sample-solution" datam8 = { git = "https://github.com/oraylis/datam8-cli", rev = "85ff50de3c7fc8e20abd4edd021d63df02c31b91" } [tool.ruff] -line-length = 100 +line-length = 120 +fix = false +builtins = ["spark", "sqlContext", "dbutils", "Catalog", "Table"] [tool.ruff.lint] -select = ["I", "E4", "E7", "E9", "W", "F", "Q", "UP"] +select = ["E", "F"] +unfixable = ["F401"] [tool.ruff.format] quote-style = "double" diff --git a/ruff.toml b/ruff.toml deleted file mode 100644 index 166fcde..0000000 --- a/ruff.toml +++ /dev/null @@ -1,19 +0,0 @@ -line-length = 120 - -# Enable pycodestyle rules. -lint.select = [ - # pycodestyle - "E", - # Pyflakes - "F" -] - -builtins = ["spark","sqlContext","dbutils", "Catalog", "Table"] - -# Never enforce... -# F821 = undefined name -# ignore = ["F821"] - -# Always autofix, but never try to fix `F401` (unused imports). -fix = false -lint.unfixable = ["F401"] diff --git a/scripts/bundle_assets.py b/scripts/bundle_assets.py index e2e76d7..8f0d9ca 100644 --- a/scripts/bundle_assets.py +++ b/scripts/bundle_assets.py @@ -1,34 +1,83 @@ -"""Check generated local assets without environment-specific bundle files.""" +"""Check the actual bundle configuration and its local asset closure offline.""" from pathlib import Path -import fnmatch +import glob +import os import re +from pathspec import GitIgnoreSpec import yaml -from databricks_jobs import generated_resources +def bundle_configuration(bundle: Path, output: Path | None = None): + """Load configured resource includes; check generated coverage when supplied.""" + bundle = bundle.resolve() + configuration = yaml.safe_load((bundle / "databricks.yml").read_text(encoding="utf-8")) + included = set() + for pattern in configuration.get("include", []): + matches = {Path(name).resolve() for name in glob.glob(str(bundle / pattern), recursive=True) + if Path(name).is_file()} + if not matches: + raise ValueError("Missing resource include: " + pattern) + included.update(matches) + if output is not None: + expected = {path.resolve() for path in output.rglob("*") if path.suffix in {".yml", ".yaml"}} + if expected - included: + raise ValueError("Bundle includes must cover every generated resource YAML") + for path in sorted(included): + addition = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + for section in ("variables", "resources"): + destination = configuration.setdefault(section, {}) + for key, value in addition.get(section, {}).items(): + if section == "resources": + group = destination.setdefault(key, {}) + if group.keys() & value.keys(): + raise ValueError("Duplicate resource in " + str(path)) + group.update(value) + else: + if key in destination: + raise ValueError("Duplicate variable: " + key) + destination[key] = value + references = set(re.findall(r"\$\{var\.([^}]+)\}", yaml.safe_dump(configuration))) + missing = references - configuration.get("variables", {}).keys() + if missing: + raise ValueError("Undeclared bundle variables: " + str(sorted(missing))) + return configuration -def required_assets(bundle: Path): - configuration = {"variables": {"notebook_path": {"default": "files/generated/notebooks"}}, "sync": {"include": ["generated/**", "utils/**"]}} - variables = {key: value.get("default") for key, value in configuration.get("variables", {}).items()} +def required_assets(bundle: Path, output: Path | None = None): + bundle = bundle.resolve() + configuration = bundle_configuration(bundle, output) + sync = configuration.get("sync", {}) + roots = {(bundle / value).resolve() for value in sync.get("paths", ["."])} + for path in roots: + if not path.exists(): + raise ValueError("Missing sync path: " + str(path)) + root = Path(os.path.commonpath([bundle, *roots])) required = set() def local(value): - for name, default in variables.items(): - if default is not None: - value = value.replace("${var." + name + "}", str(default)) - for prefix in ("${workspace.root_path}/files/", "${workspace.file_path}/"): + variables = configuration.get("variables", {}) + for _ in range(len(variables) + 1): + resolved = value + for name, definition in variables.items(): + if "default" in definition: + resolved = resolved.replace("${var." + name + "}", str(definition["default"])) + if resolved == value: + break + value = resolved + for prefix in ("${workspace.file_path}/", "${workspace.root_path}/files/"): if value.startswith(prefix): - path = bundle / value[len(prefix):] + if "${" in value[len(prefix):]: + raise ValueError("Unresolved bundle asset path: " + value) + path = root / value[len(prefix):] if not path.is_file() and path.with_suffix(".py").is_file(): path = path.with_suffix(".py") - if not path.is_file() or not path.resolve().is_relative_to(bundle.resolve()): + if not path.is_file() or not path.resolve().is_relative_to(root): raise ValueError("Missing or external bundle asset: " + value) required.add(path.resolve()) return raise ValueError("Unresolved bundle asset path: " + value) - resources = generated_resources(bundle) + resources = configuration.get("resources", {}) for job in resources.get("jobs", {}).values(): for task in job.get("tasks", []): if "notebook_task" in task: @@ -38,21 +87,42 @@ def local(value): for kind in ("file", "notebook"): if kind in library: local(library[kind]["path"]) - required.update(path.resolve() for path in (bundle / "generated/runtime").glob("*.py")) + if output is not None: + required.update(path.resolve() for path in (output / "runtime").glob("*.py")) pending = list(required) while pending: path = pending.pop() for target in re.findall(r"(?m)^# MAGIC %run\s+([^\s]+)", path.read_text(encoding="utf-8")): dependency = (path.parent / target).with_suffix(".py").resolve() - if not dependency.is_relative_to(bundle.resolve()) or not dependency.is_file(): - raise ValueError("Missing %run dependency in " + str(path.relative_to(bundle))) + if not dependency.is_relative_to(root) or not dependency.is_file(): + raise ValueError("Missing %run dependency in " + str(path.relative_to(root))) if dependency not in required: required.add(dependency) pending.append(dependency) - include = configuration.get("sync", {}).get("include", []) - exclude = configuration.get("sync", {}).get("exclude", []) + include = GitIgnoreSpec.from_lines(sync.get("include", [])) + exclude = GitIgnoreSpec.from_lines(sync.get("exclude", [])) for path in required: - relative = path.relative_to(bundle.resolve()).as_posix() - if not any(fnmatch.fnmatchcase(relative, pattern) for pattern in include) or any(fnmatch.fnmatchcase(relative, pattern) for pattern in exclude): - raise ValueError("Referenced asset excluded from bundle sync: " + relative) - return sorted(path.relative_to(bundle.resolve()).as_posix() for path in required) + relative = Path(os.path.relpath(path, bundle)).as_posix() + if (not any(path.is_relative_to(folder) for folder in roots) + or (_ignored(path, root) and not include.match_file(relative)) + or exclude.match_file(relative)): + raise ValueError("Referenced asset excluded from bundle sync: " + path.relative_to(root).as_posix()) + return sorted(path.relative_to(root).as_posix() for path in required) + + +def _ignored(path: Path, root: Path): + """Apply scoped .gitignore rules, including excluded parent directories.""" + ignored = False + for folder in reversed([parent for parent in path.parents if parent.is_relative_to(root)]): + rules = folder / ".gitignore" + if not rules.is_file(): + continue + spec = GitIgnoreSpec.from_lines(rules.read_text(encoding="utf-8").splitlines()) + for parent in path.parents: + if parent != folder and parent.is_relative_to(folder): + if spec.match_file(parent.relative_to(folder).as_posix() + "/"): + return True + decision = spec.check_file(path.relative_to(folder).as_posix()).include + if decision is not None: + ignored = decision + return ignored diff --git a/scripts/check-docs.test.mjs b/scripts/check-docs.test.mjs index 03136e7..848cf5e 100644 --- a/scripts/check-docs.test.mjs +++ b/scripts/check-docs.test.mjs @@ -42,8 +42,8 @@ test("capture evidence rejects incomplete exports, missing images and failed acc }); test("target reference detects stale output paths and omitted targets", () => { - const solution = { generatorTargets: [{ name: "docs", sourcePath: "Generate/docs", outputPath: "Output/docs" }] }; - assert.deepEqual(checkTargetDocumentation("| docs | `Generate/docs` | `Output/docs` |\n", solution), []); - assert.equal(checkTargetDocumentation("| docs | `Generate/docs` | `Output/old` |\n", solution).length, 1); + const solution = { generatorTargets: [{ name: "docs", sourcePath: "generate/docs", outputPath: "output/docs" }] }; + assert.deepEqual(checkTargetDocumentation("| docs | `generate/docs` | `output/docs` |\n", solution), []); + assert.equal(checkTargetDocumentation("| docs | `generate/docs` | `Output/old` |\n", solution).length, 1); assert.equal(checkTargetDocumentation("No target rows\n", solution).length, 2); }); diff --git a/scripts/check_generation.py b/scripts/check_generation.py index e909a33..c9ac6bf 100644 --- a/scripts/check_generation.py +++ b/scripts/check_generation.py @@ -30,14 +30,14 @@ def main(): for path in output.rglob("*.py"):ast.parse(path.read_text(encoding="utf-8"),filename=str(path)) check_notebook_markdown_cells(output) if name=="databricks": - resources=generated_resources(output.parent) + resources=generated_resources(output) manifest=json.loads((output/"manifest.json").read_text()) external_sources=[item["external"] for item in manifest["datasets"] if item.get("external")] if not external_sources or any(item["plugin"] != "builtin:SQLServer" for item in external_sources): raise AssertionError("The public SQL Server sample must generate SQL Server source readers") execution_plan(resources,manifest,required_phases=manifest["lifecycle"]) for job in resources["jobs"].values():check_graph(job["tasks"]) - required_assets(output.parent) + required_assets(ROOT / "deployment/databricks", output) print("PASS generate-only "+name) if __name__=="__main__":main() diff --git a/scripts/check_public_sources.py b/scripts/check_public_sources.py index fc67f12..2de710e 100644 --- a/scripts/check_public_sources.py +++ b/scripts/check_public_sources.py @@ -7,7 +7,7 @@ def check_public_sources(): - for source in json.loads((ROOT / "Base/DataSources.json").read_text())["dataSources"]: + for source in json.loads((ROOT / "base/DataSources.json").read_text())["dataSources"]: fields = source.get("extendedProperties", {}) for key, value in fields.items(): if not isinstance(value, str): @@ -20,7 +20,7 @@ def check_public_sources(): raise ValueError(f"{source['name']}: public secret field must contain a sample reference") if key in {"username", "user"} and value != "sample_reader": raise ValueError(f"{source['name']}: public source user must be sample_reader") - for folder in (ROOT / "Base", ROOT / "Model"): + for folder in (ROOT / "base", ROOT / "model"): for path in folder.rglob("*.json"): text = path.read_text() if ".cloudapp.azure.com" in text or "aut0ora0dev" in text: diff --git a/scripts/databricks_jobs.py b/scripts/databricks_jobs.py index 18de7a6..b6c0c1c 100644 --- a/scripts/databricks_jobs.py +++ b/scripts/databricks_jobs.py @@ -13,9 +13,9 @@ JOB_REFERENCE = re.compile(r"\$\{resources\.jobs\.([^.]+)\.id\}") -def generated_resources(bundle) -> dict: +def generated_resources(output) -> dict: resources = {} - for path in sorted((bundle / "generated").rglob("*.yml")): + for path in sorted(output.rglob("*.yml")): for kind, values in (yaml.safe_load(path.read_text(encoding="utf-8")) or {}).get("resources", {}).items(): destination = resources.setdefault(kind, {}) if destination.keys() & values.keys(): diff --git a/tests/test_bundle_assets.py b/tests/test_bundle_assets.py new file mode 100644 index 0000000..3f42665 --- /dev/null +++ b/tests/test_bundle_assets.py @@ -0,0 +1,89 @@ +"""Verify configurable asset paths and Git-style sync filters without cloud access.""" +from pathlib import Path +import sys +import tempfile +import unittest + +import yaml + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) +from bundle_assets import required_assets # noqa: E402 + + +class BundleAssetsTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory(prefix="datam8-bundle-") + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.bundle = self.root / "delivery" + self.bundle.mkdir() + self.write("code/nested/entry.py", "# MAGIC %run ../../shared/helper\n") + self.write("shared/helper.py", "# Shared notebook\n") + self.write("extensions/extra.py", "# Additional authored notebook\n") + self.write(".gitignore", "code/\nshared/*.py\n") + resource = {"resources": {"jobs": {"example": {"tasks": [{"task_key": "entry", "notebook_task": { + "notebook_path": "${workspace.file_path}/${var.notebook_dir}/entry"}}]}}}} + self.write("artifacts/jobs/main.yaml", yaml.safe_dump(resource)) + extra = {"resources": {"pipelines": {"example": {"libraries": [{"file": { + "path": "${workspace.file_path}/extensions/extra.py"}}]}}}} + self.write("authored/pipeline.yml", yaml.safe_dump(extra)) + self.configuration = { + "bundle": {"name": "example"}, + "include": ["../artifacts/**/*.yaml", "../authored/*.yml"], + "variables": {"notebook_dir": {"default": "code/nested"}}, + "sync": {"paths": ["../code", "../shared", "../extensions"], + "include": ["../code/**/*.py", "../shared/[h]elper.py"]}} + + def write(self, name, text): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + def check(self): + (self.bundle / "databricks.yml").write_text(yaml.safe_dump(self.configuration), encoding="utf-8") + return required_assets(self.bundle, self.root / "artifacts") + + def test_custom_layout_recursive_includes_extra_resources_and_globs(self): + self.assertEqual(self.check(), ["code/nested/entry.py", "extensions/extra.py", "shared/helper.py"]) + + def test_unignored_assets_do_not_need_explicit_sync_include(self): + (self.root / ".gitignore").unlink() + self.configuration["sync"].pop("include") + self.assertIn("extensions/extra.py", self.check()) + + def test_ignored_assets_require_reinclusion(self): + self.configuration["sync"]["include"] = ["../shared/*.py"] + with self.assertRaisesRegex(ValueError, "excluded from bundle sync"): + self.check() + + def test_exclude_glob_rejects_referenced_dependency(self): + self.configuration["sync"]["exclude"] = ["../shared/*.py"] + with self.assertRaisesRegex(ValueError, "excluded from bundle sync: shared/helper.py"): + self.check() + + def test_include_negation_does_not_reinclude_ignored_asset(self): + self.configuration["sync"]["include"].append("!../code/nested/entry.py") + with self.assertRaisesRegex(ValueError, "excluded from bundle sync"): + self.check() + + def test_include_cannot_add_assets_outside_selected_sync_paths(self): + self.configuration["sync"]["paths"].remove("../shared") + with self.assertRaisesRegex(ValueError, "excluded from bundle sync: shared/helper.py"): + self.check() + + def test_missing_transitive_notebook_dependency_fails(self): + (self.root / "shared/helper.py").unlink() + with self.assertRaisesRegex(ValueError, "Missing %run dependency"): + self.check() + + def test_generated_resource_coverage_remains_required(self): + self.write("artifacts/omitted.yml", "resources: {}\n") + with self.assertRaisesRegex(ValueError, "cover every generated resource YAML"): + self.check() + + def test_inline_resources_and_default_sync_root(self): + self.write("delivery/notebook.py", "# Local notebook\n") + self.configuration = {"resources": {"jobs": {"local": {"tasks": [{"notebook_task": { + "notebook_path": "${workspace.file_path}/notebook"}}]}}}} + (self.bundle / "databricks.yml").write_text(yaml.safe_dump(self.configuration), encoding="utf-8") + self.assertEqual(required_assets(self.bundle), ["notebook.py"]) diff --git a/tests/test_ddl_generation.py b/tests/test_ddl_generation.py index 5e3086d..c140df1 100644 --- a/tests/test_ddl_generation.py +++ b/tests/test_ddl_generation.py @@ -2,7 +2,7 @@ import unittest from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "Generate/databricks-lake/__modules")) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "generate/databricks-lake/__modules")) from payload_helpers import build_create_table_sql diff --git a/tests/test_public_entrypoint.py b/tests/test_public_entrypoint.py index 6342a81..568803c 100644 --- a/tests/test_public_entrypoint.py +++ b/tests/test_public_entrypoint.py @@ -15,7 +15,7 @@ class PublicEntrypointTests(unittest.TestCase): @classmethod def setUpClass(cls): - source_types = json.loads((ROOT / "Base/DataSourceTypes.json").read_text()) + source_types = json.loads((ROOT / "base/DataSourceTypes.json").read_text()) cls.source_type = DataSourceType.model_validate(source_types["dataSourceTypes"][0]) init_builtin_plugins(plugin_id=cls.source_type.pluginId) cls.plugin_class = PluginManager().get_plugin(cls.source_type.pluginId) @@ -31,7 +31,7 @@ def test_saved_connection_forms_match_the_public_builtin(self): self.assertEqual(self.source_type.authModes, self.plugin_class.get_auth_modes()) def test_real_builtin_metadata_preserves_source_types_and_sample_mappings(self): - sources = json.loads((ROOT / "Base/DataSources.json").read_text()) + sources = json.loads((ROOT / "base/DataSources.json").read_text()) source = DataSource.model_validate(sources["dataSources"][0]) source.extendedProperties["password"] = "fixture:sql-password" with patch("datam8.plugins.base.SecretResolver") as secrets: diff --git a/tests/test_sample_model.py b/tests/test_sample_model.py index 7a41da1..5c0927c 100644 --- a/tests/test_sample_model.py +++ b/tests/test_sample_model.py @@ -7,11 +7,11 @@ class SampleModelTests(unittest.TestCase): def test_adventureworks_is_the_only_external_source(self): - sources = json.loads((ROOT / 'Base/DataSources.json').read_text())['dataSources'] + sources = json.loads((ROOT / 'base/DataSources.json').read_text())['dataSources'] self.assertEqual([s['name'] for s in sources], ['AdventureWorks']) def test_all_entity_references_resolve_inside_the_small_model(self): - entities = [json.loads(p.read_text()) for p in (ROOT / 'Model').rglob('*.json') if p.name != '.properties.json'] + entities = [json.loads(p.read_text()) for p in (ROOT / 'model').rglob('*.json') if p.name != '.properties.json'] identifiers = {e['id'] for e in entities} for entity in entities: for source in entity.get('sources', []): @@ -23,7 +23,7 @@ def test_all_entity_references_resolve_inside_the_small_model(self): self.assertIn(relation['targetLocation'], identifiers) def test_customer_uses_the_standard_table_without_custom_views(self): - path = ROOT / 'Model/010-Stage/AdventureWorks/SalesLT/Customer.json' + path = ROOT / 'model/010-Stage/AdventureWorks/SalesLT/Customer.json' sources = json.loads(path.read_text())['sources'] self.assertEqual(len(sources), 1) self.assertIn('[SalesLT].[Customer]', sources[0]['sourceLocation']) diff --git a/tests/test_solution_workflow.py b/tests/test_solution_workflow.py new file mode 100644 index 0000000..0bc94bc --- /dev/null +++ b/tests/test_solution_workflow.py @@ -0,0 +1,102 @@ +"""Check generation isolation and an internal Consumer extension fixture offline.""" +import hashlib +import json +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +from bundle_assets import bundle_configuration, required_assets # noqa: E402 + + +def snapshot(folder): + return {path.relative_to(folder).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() + for path in folder.rglob("*") if path.is_file()} + + +class SolutionWorkflowTests(unittest.TestCase): + def test_clean_isolation_bundle_assets_and_consumer_extension(self): + with tempfile.TemporaryDirectory(prefix="datam8-workflow-") as directory: + root = Path(directory) / "solution" + shutil.copytree(ROOT, root, ignore=shutil.ignore_patterns( + ".git", ".venv", "output", "__pycache__", ".ruff_cache", ".databricks")) + descriptor = root / "ORAYLISDatabricksSample.dm8s" + + def cli(*arguments): + result = subprocess.run([sys.executable, "-m", "datam8", *arguments, "-s", str(descriptor)], + cwd=root, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + cli("validate") + targets = json.loads(descriptor.read_text())["generatorTargets"] + for target in targets: + cli("generate", target["name"], "--clean-output") + baseline = {target["name"]: snapshot(root / target["outputPath"]) for target in targets} + authored = {name: snapshot(root / name) for name in ("base", "model", "utils", "deployment")} + # A clean must remove its own stale file, and preserve every other + # target's bytes, including unrelated files, plus authored notebooks. + for target in targets: + for other in targets: + (root / other["outputPath"] / "stale.txt").write_text("sentinel") + cli("generate", target["name"], "--clean-output") + self.assertFalse((root / target["outputPath"] / "stale.txt").exists()) + # Documentation deliberately records its generation timestamp. + baseline[target["name"]] = snapshot(root / target["outputPath"]) + for other in targets: + if other != target: + sentinel = root / other["outputPath"] / "stale.txt" + self.assertEqual(sentinel.read_text(), "sentinel") + sentinel.unlink() + self.assertEqual(snapshot(root / other["outputPath"]), baseline[other["name"]]) + for name, contents in authored.items(): + self.assertEqual(snapshot(root / name), contents) + + bundle = root / "deployment/databricks" + assets = required_assets(bundle, root / "output/databricks") + self.assertIn("utils/MigrationFramework.py", assets) + self.assertIn("output/databricks/runtime/uc_sources.py", assets) + configuration = bundle_configuration(bundle, root / "output/databricks") + self.assertEqual(configuration["targets"]["dev"]["mode"], "development") + self.assertEqual(configuration["targets"]["dev"]["cluster_id"], "${var.cluster_id}") + self.assertEqual(configuration["variables"]["unpause_schedules"]["default"], "PAUSED") + self.assertEqual(configuration["resources"]["jobs"]["Create_All_Tables"]["max_concurrent_runs"], 1) + date_tmdl = (root / "output/powerbi/tables/Date.tmdl").read_text() + self.assertNotIn("dataType: integer", date_tmdl) + self.assertIn("column DateID\n\t\tdataType: int64", date_tmdl) + for job in configuration["resources"]["jobs"].values(): + self.assertNotIn("permissions", job) + path = bundle / "databricks.yml" + original = path.read_text() + changed = yaml.safe_load(original) + changed["sync"]["exclude"] = ["../../utils/**"] + path.write_text(yaml.safe_dump(changed)) + with self.assertRaisesRegex(ValueError, "excluded"): + required_assets(bundle, root / "output/databricks") + path.write_text(original) + + entity_path = root / "model/040-Consumer/MasterData/Customer/Customer.json" + customer = json.loads(entity_path.read_text()) + customer["attributes"].append({ + "name": "SourceSystem", "ordinalNumber": len(customer["attributes"]) + 1, + "attributeType": "Text", "dataType": {"type": "string", "nullable": False}, + "dateAdded": "2026-01-01T00:00:00Z"}) + source = next(source for source in customer["sources"] if source["sourceLocation"] == 3000) + source["mapping"].append({"sourceName": "SourceSystem", "targetName": "SourceSystem"}) + entity_path.write_text(json.dumps(customer, indent=2)) + cli("validate") + for target in targets: + cli("generate", target["name"], "--clean-output") + self.assertEqual(snapshot(root / "output/databricks"), baseline["databricks"]) + self.assertIn("column SourceSystem", (root / "output/powerbi/tables/Customer.tmdl").read_text()) + pages = list((root / "output/docs/entities/040-consumer").rglob("customer-4000.md")) + self.assertEqual(len(pages), 1) + self.assertIn("SourceSystem", pages[0].read_text()) + self.assertNotEqual(snapshot(root / "output/docs"), baseline["docs"]) + self.assertNotEqual(snapshot(root / "output/powerbi"), baseline["powerbi"]) + self.assertFalse((root / "output/powerbi/powerbi-tabular").exists()) diff --git a/tests/test_uc_sources.py b/tests/test_uc_sources.py index e200c6a..81aecf1 100644 --- a/tests/test_uc_sources.py +++ b/tests/test_uc_sources.py @@ -4,7 +4,7 @@ import unittest from unittest.mock import Mock, patch -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "Generate/shared")) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "generate/shared")) from uc_sources import database_query, fetch_http_rows, read_database diff --git a/trivvy.yaml b/trivvy.yaml deleted file mode 100644 index f4b5dbb..0000000 --- a/trivvy.yaml +++ /dev/null @@ -1,29 +0,0 @@ -format: table -report: all - -severity: - - UNKNOWN - - LOW - - MEDIUM - - HIGH - - CRITICAL - -scan: - offline-scan: true - - skip-dirs: - - tests - - scanners: - - vuln - - misconfig - - secret - - license - -db: - skip-update: false - no-progress: true - -misconfiguration: - scanners: - - terraform diff --git a/Output/databricks-lake/utils/Complete_Load.py b/utils/Complete_Load.py similarity index 88% rename from Output/databricks-lake/utils/Complete_Load.py rename to utils/Complete_Load.py index 2a0d989..03af2b8 100644 --- a/Output/databricks-lake/utils/Complete_Load.py +++ b/utils/Complete_Load.py @@ -25,8 +25,8 @@ # COMMAND ---------- dbutils.widgets.text("env", "dev", "Environment") -dbutils.widgets.text("catalog_name", "example_catalog", "Catalog Name") -dbutils.widgets.text("owner", "example_owner", "Owner") +dbutils.widgets.text("catalog_name", "", "Catalog Name") +dbutils.widgets.text("owner", "", "Owner") # dbutils.widgets.dropdown( # "run_mode", @@ -42,6 +42,8 @@ env = dbutils.widgets.get("env") catalog_name = dbutils.widgets.get("catalog_name") owner = dbutils.widgets.get("owner") +if not catalog_name or not owner: + raise ValueError("Set catalog_name and owner explicitly before running this notebook") # run_mode = dbutils.widgets.get("run_mode") # sandbox = dbutils.widgets.get("sandbox") diff --git a/Output/databricks-lake/utils/ExtractionFramework.py b/utils/ExtractionFramework.py similarity index 100% rename from Output/databricks-lake/utils/ExtractionFramework.py rename to utils/ExtractionFramework.py diff --git a/Output/databricks-lake/utils/LoggingFramework.py b/utils/LoggingFramework.py similarity index 99% rename from Output/databricks-lake/utils/LoggingFramework.py rename to utils/LoggingFramework.py index de42431..d7963bf 100644 --- a/Output/databricks-lake/utils/LoggingFramework.py +++ b/utils/LoggingFramework.py @@ -33,7 +33,7 @@ def set_object_owner() -> None: if LoggingFramework.is_hive(catalog_name): return - ddl = "ALTER TABLE `%(catalog)s`.`%(schema)s`.%(table)s SET OWNER TO %(owner)s" + ddl = "ALTER TABLE `%(catalog)s`.`%(schema)s`.%(table)s SET OWNER TO `%(owner)s`" tables = ["loads", "extractions", "v_loads_jobs"] for t in tables: @@ -44,7 +44,7 @@ def set_object_owner() -> None: "owner": owner, }) - spark.sql("ALTER DATABASE `%(catalog)s`.`%(database)s` SET OWNER TO %(owner)s" % { + spark.sql("ALTER DATABASE `%(catalog)s`.`%(database)s` SET OWNER TO `%(owner)s`" % { "catalog": catalog_name, "database": zone, "owner": owner, diff --git a/Output/databricks-lake/utils/MigrationFramework.py b/utils/MigrationFramework.py similarity index 100% rename from Output/databricks-lake/utils/MigrationFramework.py rename to utils/MigrationFramework.py diff --git a/Output/databricks-lake/utils/Start_Load.py b/utils/Start_Load.py similarity index 88% rename from Output/databricks-lake/utils/Start_Load.py rename to utils/Start_Load.py index c90ff38..c45dfcd 100644 --- a/Output/databricks-lake/utils/Start_Load.py +++ b/utils/Start_Load.py @@ -25,8 +25,8 @@ # COMMAND ---------- dbutils.widgets.text("env", "dev", "Environment") -dbutils.widgets.text("catalog_name", "example_catalog", "Catalog Name") -dbutils.widgets.text("owner", "example_owner", "Owner") +dbutils.widgets.text("catalog_name", "", "Catalog Name") +dbutils.widgets.text("owner", "", "Owner") dbutils.widgets.text("job_name", "", "Job Name") dbutils.widgets.text("job_run_id", "", "Job Run ID") @@ -45,6 +45,8 @@ env = dbutils.widgets.get("env") catalog_name = dbutils.widgets.get("catalog_name") owner = dbutils.widgets.get("owner") +if not catalog_name or not owner: + raise ValueError("Set catalog_name and owner explicitly before running this notebook") job_name = dbutils.widgets.get("job_name") job_run_id = dbutils.widgets.get("job_run_id") diff --git a/uv.lock b/uv.lock index 72ad102..6cb77ab 100644 --- a/uv.lock +++ b/uv.lock @@ -252,12 +252,14 @@ version = "2.1.0" source = { virtual = "." } dependencies = [ { name = "datam8", extra = ["sql"] }, + { name = "pathspec" }, { name = "pyyaml" }, ] [package.metadata] requires-dist = [ { name = "datam8", extras = ["sql"], git = "https://github.com/oraylis/datam8-cli?rev=85ff50de3c7fc8e20abd4edd021d63df02c31b91" }, + { name = "pathspec", specifier = "==1.1.1" }, { name = "pyyaml", specifier = "==6.0.3" }, ] @@ -485,6 +487,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl", hash = "sha256:46f0b801948e98f427b412fcabb831677194c05c3b699b80de260374baa0b1e7", size = 13068, upload-time = "2025-07-10T20:10:54.377Z" }, ] +[[package]] +name = "pathspec" +version = "1.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, +] + [[package]] name = "polars" version = "1.39.3"