From 89eddbb3ec22b27b5b0d4ca78746d6e5a4905a81 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 26 Aug 2026 13:00:44 -0700 Subject: [PATCH 1/5] fix(hubs): never lower retention on redeploy Copy-FileToAzureBlob.ps1 unconditionally overwrote stored retention months with whatever the deploymentScript's Bicep parameters passed. Bicep always resolves a value for an optional parameter (defaulting to 13 if omitted), so the script couldn't tell an explicit redeploy value from a silently-defaulted one. A redeploy that dropped a previously-customized retention value silently reset it, and the next purge pipeline run aged out historical data older than the new cutoff -- oldest data first (#2206). Take the max of stored and incoming retention instead of overwriting: growing is always safe, shrinking is destructive and hard to reverse. Co-Authored-By: Claude Sonnet 5 --- .../Tests/Unit/HubsRetentionGuard.Tests.ps1 | 61 +++++++++++++++++++ .../Core/Copy-FileToAzureBlob.ps1 | 9 ++- 2 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 diff --git a/src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 b/src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 new file mode 100644 index 000000000..07713d185 --- /dev/null +++ b/src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 @@ -0,0 +1,61 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + Regression coverage for the settings.json retention guard (#2206): + Copy-FileToAzureBlob.ps1 unconditionally overwrote retention.ingestion.months / retention.final.months + with whatever the deploymentScript's Bicep parameters passed in. Bicep always resolves a value for an + optional parameter (defaulting to 13 if the caller didn't specify one), so the script cannot tell an + explicit redeploy value from a silently-defaulted one. A redeploy that omitted a previously-customized + retention value therefore silently reset it to the toolkit default, and the next purge pipeline run + aged out historical data older than the new (lower) cutoff -- oldest data first. + + The fix: never lower stored retention on redeploy. Growing retention is always safe; shrinking it has a + destructive, hard-to-reverse consequence (data purge), so the script now takes the max of the stored + value and the incoming one instead of overwriting unconditionally. +#> + +Describe 'HubsRetentionGuard' { + + BeforeAll { + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path + $scriptPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1' + $content = Get-Content -Path $scriptPath -Raw + } + + Context 'Never-shrink guard' { + + It 'Should take the max of stored and incoming ingestion retention' { + $content | Should -Match '\$json\.retention\.ingestion\.months\s*=\s*\[Math\]::Max\(\$json\.retention\.ingestion\.months,\s*\[Int32\]::Parse\(\$env:ingestionRetentionInMonths\)\)' ` + -Because 'a redeploy that omits an explicit retention value must not silently shrink stored retention and purge historical data (#2206)' + } + + It 'Should take the max of stored and incoming final retention' { + $content | Should -Match '\$json\.retention\.final\.months\s*=\s*\[Math\]::Max\(\$json\.retention\.final\.months,\s*\[Int32\]::Parse\(\$env:finalRetentionInMonths\)\)' ` + -Because 'a redeploy that omits an explicit retention value must not silently shrink stored retention and purge historical data (#2206)' + } + + It 'Should not unconditionally overwrite ingestion retention' { + $content | Should -Not -Match '\$json\.retention\.ingestion\.months\s*=\s*\[Int32\]::Parse\(\$env:ingestionRetentionInMonths\)\s*$' ` + -Because 'a direct assignment (rather than a max guard) was the source of the #2206 regression' + } + + It 'Should not unconditionally overwrite final retention' { + $content | Should -Not -Match '\$json\.retention\.final\.months\s*=\s*\[Int32\]::Parse\(\$env:finalRetentionInMonths\)\s*$' ` + -Because 'a direct assignment (rather than a max guard) was the source of the #2206 regression' + } + } + + Context 'First-run behavior unchanged' { + + It 'Should still seed ingestion retention from the parameter when no retention object exists yet' { + $content | Should -Match 'Add-Member -Name ingestion -Value \(ConvertFrom-Json "\{""months"":\$\(\$env:ingestionRetentionInMonths\)\}"\)' ` + -Because 'a brand-new settings.json has no stored value to protect, so the first deploy must still honor the requested retention' + } + + It 'Should still seed final retention from the parameter when no retention object exists yet' { + $content | Should -Match 'Add-Member -Name final -Value \(ConvertFrom-Json "\{""months"":\$\(\$env:finalRetentionInMonths\)\}"\)' ` + -Because 'a brand-new settings.json has no stored value to protect, so the first deploy must still honor the requested retention' + } + } +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1 index ffea9d454..195c7236a 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1 +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1 @@ -125,13 +125,16 @@ else } # Set or update ingestion retention +# NOTE: Bicep always passes a value here (defaulting to 13 if the caller didn't specify one), so this script +# cannot tell an explicit redeploy value from a silently-defaulted one. Never lower retention on redeploy -- +# shrinking silently ages out historical data the next time the purge pipeline runs (#2206); growing is safe. if (!($json.retention.ingestion)) { $json.retention | Add-Member -Name ingestion -Value (ConvertFrom-Json "{""months"":$($env:ingestionRetentionInMonths)}") -MemberType NoteProperty } else { - $json.retention.ingestion.months = [Int32]::Parse($env:ingestionRetentionInMonths) + $json.retention.ingestion.months = [Math]::Max($json.retention.ingestion.months, [Int32]::Parse($env:ingestionRetentionInMonths)) } # Set or update raw retention @@ -144,14 +147,14 @@ else $json.retention.raw.days = [Int32]::Parse($env:rawRetentionInDays) } -# Set or update final retention +# Set or update final retention (never lower on redeploy -- see note above) if (!($json.retention.final)) { $json.retention | Add-Member -Name final -Value (ConvertFrom-Json "{""months"":$($env:finalRetentionInMonths)}") -MemberType NoteProperty } else { - $json.retention.final.months = [Int32]::Parse($env:finalRetentionInMonths) + $json.retention.final.months = [Math]::Max($json.retention.final.months, [Int32]::Parse($env:finalRetentionInMonths)) } # Updating settings From 845afc83cb7d5e939aa3628c52cb5c4652691586 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Mon, 31 Aug 2026 23:53:52 -0700 Subject: [PATCH 2/5] docs: add changelog entry for retention never-shrink fix Co-Authored-By: Claude Sonnet 5 --- docs-mslearn/toolkit/changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 92501d68d..a2cf3f5a3 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -34,6 +34,7 @@ The following section lists features and enhancements that are currently in deve - Replaced redundant `tolower()` comparisons in hub KQL with case-insensitive operators (`has`, `=~`, `!~`) so the engine can use the term index instead of scanning every row ([#2213](https://github.com/microsoft/finops-toolkit/issues/2213)). - Replaced whole-term `contains` matches with `has` across hub KQL and the query catalog (resource ID paths, licensing phrases, SKU description terms) and added a per-row operator-equivalence regression harness with unit test coverage ([#2220](https://github.com/microsoft/finops-toolkit/pull/2220)). - **Fixed** + - Fixed hub redeploys silently resetting ingestion and Data Explorer retention to the 13-month default when a custom value wasn't explicitly re-specified, which caused the purge pipeline to age out historical data on the next run ([#2206](https://github.com/microsoft/finops-toolkit/issues/2206)). - Fixed private-network deployments that Azure Policy blocked when `defaultOutboundAccess` was omitted. Private mode subnets now set it to `false`, while an Azure Files private endpoint supports deployment-script storage and the NAT Gateway provides required container egress ([#2258](https://github.com/microsoft/finops-toolkit/issues/2258), [#2259](https://github.com/microsoft/finops-toolkit/pull/2259)). - Fixed the `ContractedCost` recompute guard to compare with a null-safe tolerance instead of exact float equality, eliminating millions of no-op rewrites that polluted the `x_SourceValues` audit trail while preserving the null-cost backfill and no longer overwriting an existing cost when the unit price is missing ([#2216](https://github.com/microsoft/finops-toolkit/issues/2216)). - Fixed the SQL VMs without Azure Hybrid Benefit recommendation query to join on the SQL VM `virtualMachineResourceId` instead of a case-sensitive VM name match that skipped VMs with uppercase names and dropped duplicate names, and made all Azure Resource Graph join kinds explicit so no query relies on the `innerunique` default ([#2225](https://github.com/microsoft/finops-toolkit/pull/2225)). From e952c898f909441362cf2d7bb475e94a3d1d5ce2 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 7 Oct 2026 11:31:41 -0700 Subject: [PATCH 3/5] fix(hubs): anchor regression guard regex in HubsRetentionGuard.Tests.ps1 The two 'should not unconditionally overwrite' assertions used a trailing \s*$ anchor. Without RegexOptions.Multiline, $ matches only the end of the whole file content, not end of line, so the pattern could only match if the offending assignment were the very last thing in the file. Both assertions passed whether or not the #2206 regression was present. Removed the anchor. Verified by temporarily reintroducing the old unconditional-overwrite line and re-running the suite: the test now fails red where it previously passed, then restored and confirmed green against the actual fix. Co-Authored-By: RolandKrummenacher Co-Authored-By: Claude --- src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 b/src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 index 07713d185..c29a5383f 100644 --- a/src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 +++ b/src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 @@ -36,12 +36,12 @@ Describe 'HubsRetentionGuard' { } It 'Should not unconditionally overwrite ingestion retention' { - $content | Should -Not -Match '\$json\.retention\.ingestion\.months\s*=\s*\[Int32\]::Parse\(\$env:ingestionRetentionInMonths\)\s*$' ` + $content | Should -Not -Match '\$json\.retention\.ingestion\.months\s*=\s*\[Int32\]::Parse\(\$env:ingestionRetentionInMonths\)' ` -Because 'a direct assignment (rather than a max guard) was the source of the #2206 regression' } It 'Should not unconditionally overwrite final retention' { - $content | Should -Not -Match '\$json\.retention\.final\.months\s*=\s*\[Int32\]::Parse\(\$env:finalRetentionInMonths\)\s*$' ` + $content | Should -Not -Match '\$json\.retention\.final\.months\s*=\s*\[Int32\]::Parse\(\$env:finalRetentionInMonths\)' ` -Because 'a direct assignment (rather than a max guard) was the source of the #2206 regression' } } From c3611def9fdd1f0d5cedb9c6e9a2b61e90c7c59d Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Thu, 15 Oct 2026 22:47:14 -0700 Subject: [PATCH 4/5] fix(hubs): keep retention on redeploy and upgrade unless specified Make all retention parameters nullable (exportRetentionInDays, ingestionRetentionInMonths, dataExplorerRawRetentionInDays, dataExplorerFinalRetentionInMonths). A deployment that omits one keeps the value stored in config/settings.json; explicit values, including lower ones, are applied. New hubs still get the documented defaults. - Replace the "never change a stored value" guard with the nullable pattern. - Settings script returns the effective raw retention, which Data Explorer applies; remove the softdelete = 0d step that ran on every deploy. - Deploy-FinOpsHub only passes retention parameters that are specified. - Portal keeps visible defaults and only sends values changed from the default, so a portal redeploy doesn't reset an existing hub. Fixes #2206 Fixes #2348 Co-Authored-By: Claude Opus 5.5 --- docs-mslearn/toolkit/changelog.md | 2 +- docs-mslearn/toolkit/hubs/data-processing.md | 5 +- docs-mslearn/toolkit/hubs/template.md | 10 +- src/powershell/Public/Deploy-FinOpsHub.ps1 | 38 +++-- .../Tests/Unit/Deploy-FinOpsHub.Tests.ps1 | 32 ++++ .../Tests/Unit/HubsRetentionGuard.Tests.ps1 | 92 ------------ .../Unit/HubsRetentionRedeploy.Tests.ps1 | 140 ++++++++++++++++++ .../finops-hub/createUiDefinition.json | 10 +- src/templates/finops-hub/main.bicep | 16 +- .../scripts/IngestionSetup_RawTables.kql | 21 --- .../Core/Copy-FileToAzureBlob.ps1 | 76 ++++------ .../Microsoft.FinOpsHubs/Core/app.bicep | 60 +++++--- .../modules/fx/hub-deploymentScript.bicep | 3 +- src/templates/finops-hub/modules/hub.bicep | 19 +-- 14 files changed, 294 insertions(+), 230 deletions(-) delete mode 100644 src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 create mode 100644 src/powershell/Tests/Unit/HubsRetentionRedeploy.Tests.ps1 diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index afcb81165..68818f565 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -35,7 +35,6 @@ The following section lists features and enhancements that are currently in deve - Replaced whole-term `contains` matches with `has` across hub KQL and the query catalog (resource ID paths, licensing phrases, SKU description terms) and added a per-row operator-equivalence regression harness with unit test coverage ([#2220](https://github.com/microsoft/finops-toolkit/pull/2220)). - Documented how the hub schema's columns nest in the hub database guide - the organization chain and the two parallel service chains - and brought the `Costs()` column reference back in line with the deployed schema by adding 27 missing columns, correcting three columns still listed under the names hubs v12 renamed away from, and fixing one column typed as `long` instead of `int` ([#2325](https://github.com/microsoft/finops-toolkit/issues/2325), [#2329](https://github.com/microsoft/finops-toolkit/issues/2329)). - **Fixed** - - Fixed hub redeploys silently resetting the stored ingestion and Data Explorer final retention to the 13-month default when a custom value wasn't explicitly re-specified, which caused the purge pipeline to age out historical data on the next run. Stored values are no longer changed by a redeploy; edit `config/settings.json` to change them ([#2206](https://github.com/microsoft/finops-toolkit/issues/2206)). - Fixed ADF schedule trigger deployment failures (`InvalidWorkflowTriggerRecurrence`) in regions not covered by the region-to-time-zone map (for example, Sweden Central), where the trigger's `startTime` was missing the required `Z` suffix for the `UTC` fallback time zone ([#2157](https://github.com/microsoft/finops-toolkit/issues/2157), [#2291](https://github.com/microsoft/finops-toolkit/pull/2291)). - Fixed managed exports failing with an `Unauthorized` error because the Role Based Access Control Administrator role was never assigned to the Data Factory identity. Roles requested by a hub app are now assigned on the publisher storage account even when the app doesn't create the storage account itself ([#2253](https://github.com/microsoft/finops-toolkit/issues/2253)). - Fixed private-network deployments that Azure Policy blocked when `defaultOutboundAccess` was omitted. Private mode subnets now set it to `false`, while an Azure Files private endpoint supports deployment-script storage and the NAT Gateway provides required container egress ([#2258](https://github.com/microsoft/finops-toolkit/issues/2258), [#2259](https://github.com/microsoft/finops-toolkit/pull/2259)). @@ -133,6 +132,7 @@ _Released June 2026_ - Fixed Data Factory ingestion memory pressure during emptiness filtering. - Replaced `isnotempty(strcat(x_SkuMeterId, x_SkuOfferId))` with separate `isnotempty()` checks in FinOps hub ingestion scripts to avoid temporary string allocation. - Hardened the reservation price backfill to select the highest on-demand price (`max()` instead of `min()`) when duplicate price rows collapse under a single reservation price lookup key, preventing understated commitment discount savings ([#2189](https://github.com/microsoft/finops-toolkit/pull/2189)). + - Fixed hub redeploys and upgrades resetting data retention to the defaults, which purged historical data in Data Explorer and changed export and ingestion retention. Retention parameters no longer have template defaults, so a deployment that doesn't specify a value keeps the current value in **config/settings.json** while explicit values, including lower ones, are still applied. New hubs still use the documented defaults, the Azure portal only applies retention values that were changed from the default, and redeploys no longer set Data Explorer raw table retention to 0 days before applying the requested value ([#2206](https://github.com/microsoft/finops-toolkit/issues/2206), [#2348](https://github.com/microsoft/finops-toolkit/issues/2348)). ### [Power BI reports](power-bi/reports.md) v15 diff --git a/docs-mslearn/toolkit/hubs/data-processing.md b/docs-mslearn/toolkit/hubs/data-processing.md index 86d9ed908..83ccb02d3 100644 --- a/docs-mslearn/toolkit/hubs/data-processing.md +++ b/docs-mslearn/toolkit/hubs/data-processing.md @@ -3,7 +3,7 @@ title: FinOps hubs data processing description: Learn how FinOps hubs process data, including scope setup, data normalization, and optimization, to enhance cost management and analysis. author: flanakin ms.author: micflan -ms.date: 10/14/2026 +ms.date: 10/15/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit @@ -53,7 +53,8 @@ The following diagram depicts the end-to-end data ingestion process within FinOp 3. When data is ingested into raw tables in Data Explorer, an update policy copies the data into the corresponding `{dataset}_final_v1_0` table using the `{dataset}_transform_v1_0()` function to normalize all data to align to FOCUS 1.0. 4. After ingestion, the **ingestion_ETL_dataExplorer** pipeline performs some cleanup, including purging data in the final table that is past the data retention period. - As of 0.7, Data Explorer applies data retention in raw tables while data retention in final tables is applied by the ingestion pipeline. If data ingestion stops, historical data isn't purged. - - Data retention can be configured during the template deployment or manually in the **config/settings.json** file in storage. Once a hub has stored ingestion or final retention values, redeploying with a different `ingestionRetentionInMonths` or `dataExplorerFinalRetentionInMonths` doesn't change them, so a lower value never purges data by accident. To change retention later, edit **config/settings.json** in the hub storage account. + - Data retention can be configured during the template deployment or manually in the **config/settings.json** file in storage. + - When you redeploy or upgrade a hub, retention parameters you don't specify keep their current values in **config/settings.json**. New hubs use the default values. In the Azure portal, values left at the default keep the current settings. To set a value back to the default on an existing hub, edit **config/settings.json** or specify the value with [Deploy-FinOpsHub](../powershell/hubs/deploy-finopshub.md). Data Explorer raw table retention from **config/settings.json** is applied on the next deployment. 6. (Optional) The **queries_DailySchedule** trigger runs the **queries_ExecuteETL** pipeline once per day to query Azure Resource Graph for additional recommendations and saves results to the **ingestion/Recommendations** folder. [Learn more](#azure-resource-graph-recommendations). 7. Reports and other tools like Power BI read data from Data Explorer or the **ingestion** container. - Data in Data Explorer can be read from the **Hub** database. diff --git a/docs-mslearn/toolkit/hubs/template.md b/docs-mslearn/toolkit/hubs/template.md index a17b37e91..bfbb04a2b 100644 --- a/docs-mslearn/toolkit/hubs/template.md +++ b/docs-mslearn/toolkit/hubs/template.md @@ -3,7 +3,7 @@ title: FinOps hub template description: Learn about what's included in the FinOps hub template including parameters, resources, and outputs. author: flanakin ms.author: micflan -ms.date: 08/24/2026 +ms.date: 10/15/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit @@ -91,10 +91,10 @@ Here are the parameters you can use to customize the deployment: | **tags** | Object | Optional. Tags to apply to all resources. We will also add the `cm-resource-parent` tag for improved cost roll-ups in Cost Management. | | | **tagsByResource** | Object | Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources. | | | **scopesToMonitor** | Array | Optional. List of scope IDs to monitor and ingest cost for. | | -| **exportRetentionInDays** | Int | Optional. Number of days of data to retain in the msexports container. | 0 | -| **ingestionRetentionInMonths** | Int | Optional. Number of months of data to retain in the ingestion container. | 13 | -| **dataExplorerRawRetentionInDays** | Int | Optional. Number of days of data to retain in the Data Explorer \*_raw tables. | 0 | -| **dataExplorerFinalRetentionInMonths** | Int | Optional. Number of months of data to retain in the Data Explorer \*_final_v\* tables. | 13 | +| **exportRetentionInDays** | Int | Optional. Number of days of data to retain in the msexports container. If not specified, a redeploy keeps the current value in **config/settings.json**. | 0 (new hubs) | +| **ingestionRetentionInMonths** | Int | Optional. Number of months of data to retain in the ingestion container. If not specified, a redeploy keeps the current value in **config/settings.json**. | 13 (new hubs) | +| **dataExplorerRawRetentionInDays** | Int | Optional. Number of days of data to retain in the Data Explorer \*_raw tables. If not specified, a redeploy keeps the current value in **config/settings.json**. | 0 (new hubs) | +| **dataExplorerFinalRetentionInMonths** | Int | Optional. Number of months of data to retain in the Data Explorer \*_final_v\* tables. If not specified, a redeploy keeps the current value in **config/settings.json**. | 13 (new hubs) | | **remoteHubStorageUri** | String | Optional. Data Lake storage endpoint from the remote (primary) hub storage account. Used for cross-tenant cost data collection where this hub sends processed data to a central hub. Example: `https://primaryhub.dfs.core.windows.net/` | | | **remoteHubStorageKey** | String | Optional. Storage account access key for the remote (primary) hub. Used with remoteHubStorageUri for cross-tenant scenarios. Must be kept secure as it provides full storage access. | | | **enableManagedExports** | Bool | Optional. Enable managed exports where your FinOps hub instance will create and run Cost Management exports on your behalf. Not supported for Microsoft Customer Agreement (MCA) billing profiles. Requires the ability to grant the Role Based Access Control Administrator role to FinOps hubs, which is required to create Cost Management exports. | True | diff --git a/src/powershell/Public/Deploy-FinOpsHub.ps1 b/src/powershell/Public/Deploy-FinOpsHub.ps1 index 6140e0af8..64bd83509 100644 --- a/src/powershell/Public/Deploy-FinOpsHub.ps1 +++ b/src/powershell/Public/Deploy-FinOpsHub.ps1 @@ -73,16 +73,16 @@ Example: @('/subscriptions/00000000-0000-0000-0000-000000000000', '/subscriptions/11111111-1111-1111-1111-111111111111') .PARAMETER ExportRetentionInDays - Optional. Number of days of data to retain in the msexports container. Default: 0. + Optional. Number of days of data to retain in the msexports container. If not specified, keeps the current value (0 for new hubs). .PARAMETER IngestionRetentionInMonths - Optional. Number of months of data to retain in the ingestion container. Default: 13. + Optional. Number of months of data to retain in the ingestion container. If not specified, keeps the current value (13 for new hubs). .PARAMETER DataExplorerRawRetentionInDays - Optional. Number of days of data to retain in the Data Explorer *_raw tables. Default: 0. + Optional. Number of days of data to retain in the Data Explorer *_raw tables. If not specified, keeps the current value (0 for new hubs). .PARAMETER DataExplorerFinalRetentionInMonths - Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. Default: 13. + Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. If not specified, keeps the current value (13 for new hubs). .PARAMETER NetworkMode Optional. Network mode for the hub: 'public' (default), 'vnet' (private endpoints, default outbound), or 'private' (private endpoints + NAT Gateway for controlled outbound access - required when the 'Subnets should be private' policy is enforced). @@ -183,12 +183,12 @@ function Deploy-FinOpsHub [Parameter()] [ValidateRange(0, 9999)] [int] - $DataExplorerRawRetentionInDays = 0, + $DataExplorerRawRetentionInDays, [Parameter()] [ValidateRange(0, 999)] [int] - $DataExplorerFinalRetentionInMonths = 13, + $DataExplorerFinalRetentionInMonths, [Parameter()] [ValidateSet('public', 'vnet', 'private')] @@ -218,12 +218,12 @@ function Deploy-FinOpsHub [Parameter()] [ValidateRange(0, 9999)] [int] - $ExportRetentionInDays = 0, + $ExportRetentionInDays, [Parameter()] [ValidateRange(0, 999)] [int] - $IngestionRetentionInMonths = 13 + $IngestionRetentionInMonths ) # Initialize toolkitPath before try block to ensure cleanup works even if early failure occurs @@ -297,13 +297,27 @@ function Deploy-FinOpsHub $parameterSplat.TemplateParameterObject.Add('dataExplorerName', $DataExplorerName) $parameterSplat.TemplateParameterObject.Add('dataExplorerSku', $DataExplorerSku) $parameterSplat.TemplateParameterObject.Add('dataExplorerCapacity', $DataExplorerCapacity) - $parameterSplat.TemplateParameterObject.Add('dataExplorerRawRetentionInDays', $DataExplorerRawRetentionInDays) - $parameterSplat.TemplateParameterObject.Add('dataExplorerFinalRetentionInMonths', $DataExplorerFinalRetentionInMonths) $parameterSplat.TemplateParameterObject.Add('enablePublicAccess', ($effectiveNetworkMode -eq 'public')) $parameterSplat.TemplateParameterObject.Add('virtualNetworkAddressPrefix', $VirtualNetworkAddressPrefix) - $parameterSplat.TemplateParameterObject.Add('exportRetentionInDays', $ExportRetentionInDays) - $parameterSplat.TemplateParameterObject.Add('ingestionRetentionInMonths', $IngestionRetentionInMonths) $parameterSplat.TemplateParameterObject.Add('scopesToMonitor', $ScopesToMonitor) + + # Only pass retention when specified so a redeploy keeps the current values (#2206, #2348) + if ($PSBoundParameters.ContainsKey('DataExplorerRawRetentionInDays')) + { + $parameterSplat.TemplateParameterObject.Add('dataExplorerRawRetentionInDays', $DataExplorerRawRetentionInDays) + } + if ($PSBoundParameters.ContainsKey('ExportRetentionInDays')) + { + $parameterSplat.TemplateParameterObject.Add('exportRetentionInDays', $ExportRetentionInDays) + } + if ($PSBoundParameters.ContainsKey('IngestionRetentionInMonths')) + { + $parameterSplat.TemplateParameterObject.Add('ingestionRetentionInMonths', $IngestionRetentionInMonths) + } + if ($PSBoundParameters.ContainsKey('DataExplorerFinalRetentionInMonths')) + { + $parameterSplat.TemplateParameterObject.Add('dataExplorerFinalRetentionInMonths', $DataExplorerFinalRetentionInMonths) + } } if ($Version -eq 'latest' -or [version]$Version -ge '0.10') diff --git a/src/powershell/Tests/Unit/Deploy-FinOpsHub.Tests.ps1 b/src/powershell/Tests/Unit/Deploy-FinOpsHub.Tests.ps1 index 1bb4512a0..124353578 100644 --- a/src/powershell/Tests/Unit/Deploy-FinOpsHub.Tests.ps1 +++ b/src/powershell/Tests/Unit/Deploy-FinOpsHub.Tests.ps1 @@ -268,5 +268,37 @@ InModuleScope 'FinOpsToolkit' { } -Times 1 } } + + Context 'Retention' { + BeforeAll { + Mock -CommandName 'Get-AzResourceGroup' -MockWith { return @{ ResourceGroupName = $rgName } } + Mock -CommandName 'New-AzResourceGroup' + Mock -CommandName 'Save-FinOpsHubTemplate' + Mock -CommandName 'Initialize-FinOpsHubDeployment' + $templateFile = Join-Path -Path ([System.IO.Path]::GetTempPath()) -ChildPath 'FinOps/finops-hub-v1.0.0/main.bicep' + Mock -CommandName 'Get-ChildItem' -MockWith { return @{ FullName = $templateFile } } + Mock -CommandName 'New-AzResourceGroupDeployment' + } + + It 'Should not pass retention when not specified so a redeploy keeps stored values (#2206, #2348)' { + { Deploy-FinOpsHub -Name $hubName -ResourceGroup $rgName -Location $location -Version 'latest' } | Should -Not -Throw + Should -Invoke -CommandName 'New-AzResourceGroupDeployment' -ParameterFilter { + -not $TemplateParameterObject.ContainsKey('exportRetentionInDays') ` + -and -not $TemplateParameterObject.ContainsKey('ingestionRetentionInMonths') ` + -and -not $TemplateParameterObject.ContainsKey('dataExplorerRawRetentionInDays') ` + -and -not $TemplateParameterObject.ContainsKey('dataExplorerFinalRetentionInMonths') + } -Times 1 + } + + It 'Should pass retention when specified, including 0' { + { Deploy-FinOpsHub -Name $hubName -ResourceGroup $rgName -Location $location -Version 'latest' -ExportRetentionInDays 0 -IngestionRetentionInMonths 36 -DataExplorerRawRetentionInDays 30 -DataExplorerFinalRetentionInMonths 60 } | Should -Not -Throw + Should -Invoke -CommandName 'New-AzResourceGroupDeployment' -ParameterFilter { + $TemplateParameterObject.ContainsKey('exportRetentionInDays') -and $TemplateParameterObject.exportRetentionInDays -eq 0 ` + -and $TemplateParameterObject.ingestionRetentionInMonths -eq 36 ` + -and $TemplateParameterObject.dataExplorerRawRetentionInDays -eq 30 ` + -and $TemplateParameterObject.dataExplorerFinalRetentionInMonths -eq 60 + } -Times 1 + } + } } } \ No newline at end of file diff --git a/src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 b/src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 deleted file mode 100644 index 04c59a5d4..000000000 --- a/src/powershell/Tests/Unit/HubsRetentionGuard.Tests.ps1 +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. - -<# - Regression coverage for the settings.json retention guard (#2206): - Copy-FileToAzureBlob.ps1 unconditionally overwrote retention.ingestion.months / retention.final.months - with whatever the deploymentScript's Bicep parameters passed in. Bicep always resolves a value for an - optional parameter (defaulting to 13 if the caller didn't specify one), so the script cannot tell an - explicit redeploy value from a silently-defaulted one. A redeploy that omitted a previously-customized - retention value therefore silently reset it to the toolkit default, and the next purge pipeline run - aged out historical data older than the new (lower) cutoff -- oldest data first. - - The fix: a value already stored in settings.json is kept on redeploy. The requested value only applies - when nothing is stored yet (new hub, or settings.json from before retention existed). - - These tests run the real script with stubbed Az cmdlets and assert on the JSON it uploads. -#> - -Describe 'HubsRetentionGuard' { - - BeforeAll { - $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path - $scriptPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1' - - function Invoke-CopyScript - { - param([string] $ExistingJson, [int] $Ingestion, [int] $Final) - - $workDir = Join-Path ([IO.Path]::GetTempPath()) ([guid]::NewGuid().ToString()) - New-Item -ItemType Directory -Path $workDir | Out-Null - $existingJson = $ExistingJson - - function New-AzStorageContext { param($StorageAccountName, [switch] $UseConnectedAccount) 'ctx' } - function Get-AzStorageBlobContent - { - param($Context, $Container, $Blob, $Destination, [switch] $Force) - if ($null -eq $existingJson) { return $null } - Set-Content -Path $Destination -Value $existingJson - return 'blob' - } - function Set-AzStorageBlobContent { param($Context, $Container, $File, [switch] $Force) } - - $env:storageAccountName = 'sa' - $env:containerName = 'config' - $env:scopes = '' - $env:ftkVersion = '0.0.0' - $env:msexportRetentionInDays = '0' - $env:rawRetentionInDays = '0' - $env:ingestionRetentionInMonths = "$Ingestion" - $env:finalRetentionInMonths = "$Final" - - Push-Location $workDir - try - { - & $scriptPath *> $null - # The script writes the final settings.json to the working directory before uploading it - return (Get-Content -Path (Join-Path $workDir settings.json) -Raw | ConvertFrom-Json) - } - finally { Pop-Location; Remove-Item $workDir -Recurse -Force -ErrorAction SilentlyContinue } - } - } - - Context 'Existing settings.json' { - - It 'Should keep a stored value that is higher than the requested one' { - $r = Invoke-CopyScript -ExistingJson '{"version":"","scopes":[],"retention":{"msexports":{"days":0},"ingestion":{"months":36},"raw":{"days":0},"final":{"months":60}}}' -Ingestion 13 -Final 13 - $r.retention.ingestion.months | Should -Be 36 -Because 'a redeploy without an explicit value must not reset customized retention (#2206)' - $r.retention.final.months | Should -Be 60 - } - - It 'Should keep a stored value that is lower than the requested one' { - $r = Invoke-CopyScript -ExistingJson '{"version":"","scopes":[],"retention":{"msexports":{"days":0},"ingestion":{"months":6},"raw":{"days":0},"final":{"months":6}}}' -Ingestion 13 -Final 13 - $r.retention.ingestion.months | Should -Be 6 -Because 'a stored value is never changed on redeploy' - $r.retention.final.months | Should -Be 6 - } - - It 'Should seed retention from the request when settings.json predates retention' { - $r = Invoke-CopyScript -ExistingJson '{"version":"","scopes":[]}' -Ingestion 24 -Final 30 - $r.retention.ingestion.months | Should -Be 24 - $r.retention.final.months | Should -Be 30 - } - } - - Context 'First deploy' { - - It 'Should honor the requested retention when no settings.json exists' { - $r = Invoke-CopyScript -ExistingJson $null -Ingestion 6 -Final 8 - $r.retention.ingestion.months | Should -Be 6 -Because 'there is no stored value to protect on a brand-new hub' - $r.retention.final.months | Should -Be 8 - } - } -} diff --git a/src/powershell/Tests/Unit/HubsRetentionRedeploy.Tests.ps1 b/src/powershell/Tests/Unit/HubsRetentionRedeploy.Tests.ps1 new file mode 100644 index 000000000..227ccd530 --- /dev/null +++ b/src/powershell/Tests/Unit/HubsRetentionRedeploy.Tests.ps1 @@ -0,0 +1,140 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + Regression coverage for retention on redeploy and upgrade (#2206, #2348): + - All retention parameters are nullable. The template only passes them to Copy-FileToAzureBlob.ps1 when the + deployment specifies them. When omitted, the script keeps the value stored in settings.json (defaults for new + hubs) and returns the effective raw value for the Data Explorer setup. Explicit values always apply. + - IngestionSetup_RawTables.kql no longer sets raw retention to 0 days before applying the requested value. + + The script tests run the real script with stubbed Az cmdlets and assert on the JSON it writes and its outputs. +#> + +Describe 'HubsRetentionRedeploy' { + + BeforeAll { + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path + $scriptPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1' + $kqlPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql' + $retentionVars = 'msexportRetentionInDays', 'ingestionRetentionInMonths', 'rawRetentionInDays', 'finalRetentionInMonths' + + function Invoke-CopyScript + { + param([string] $ExistingJson, [string] $MsexportDays, [string] $IngestionMonths, [string] $RawDays, [string] $FinalMonths) + + $workDir = Join-Path ([IO.Path]::GetTempPath()) ([guid]::NewGuid().ToString()) + New-Item -ItemType Directory -Path $workDir | Out-Null + $existingJson = $ExistingJson + + function New-AzStorageContext { param($StorageAccountName, [switch] $UseConnectedAccount) 'ctx' } + function Get-AzStorageBlobContent + { + param($Context, $Container, $Blob, $Destination, [switch] $Force) + if (-not $existingJson) { return $null } + Set-Content -Path $Destination -Value $existingJson + return 'blob' + } + function Set-AzStorageBlobContent { param($Context, $Container, $File, [switch] $Force) } + + $env:storageAccountName = 'sa' + $env:containerName = 'config' + $env:scopes = '' + $env:ftkVersion = '0.0.0' + # The template omits these environment variables when the deployment doesn't specify them + $env:msexportRetentionInDays = $MsexportDays + $env:ingestionRetentionInMonths = $IngestionMonths + $env:rawRetentionInDays = $RawDays + $env:finalRetentionInMonths = $FinalMonths + + Push-Location $workDir + try + { + # Dot-source so $DeploymentScriptOutputs is visible to the test + . $scriptPath *> $null + return [PSCustomObject]@{ + Settings = (Get-Content -Path (Join-Path $workDir settings.json) -Raw | ConvertFrom-Json) + Outputs = $DeploymentScriptOutputs + } + } + finally + { + Pop-Location + Remove-Item $workDir -Recurse -Force -ErrorAction SilentlyContinue + $retentionVars | ForEach-Object { Remove-Item "env:$_" -ErrorAction SilentlyContinue } + } + } + + $existing = '{"version":"","scopes":[],"retention":{"msexports":{"days":7},"ingestion":{"months":36},"raw":{"days":30},"final":{"months":60}}}' + } + + Context 'settings.json' { + + It 'Should keep stored values when retention is not specified' { + $r = Invoke-CopyScript -ExistingJson $existing + $r.Settings.retention.msexports.days | Should -Be 7 -Because 'a redeploy without exportRetentionInDays must not reset export retention (#2348)' + $r.Settings.retention.ingestion.months | Should -Be 36 -Because 'a redeploy without ingestionRetentionInMonths must not reset ingestion retention (#2206)' + $r.Settings.retention.raw.days | Should -Be 30 -Because 'a redeploy without dataExplorerRawRetentionInDays must not reset raw retention (#2348)' + $r.Settings.retention.final.months | Should -Be 60 -Because 'a redeploy without dataExplorerFinalRetentionInMonths must not reset final retention (#2206)' + $r.Outputs.rawRetentionInDays | Should -Be 30 -Because 'Data Explorer must reapply the stored raw retention' + } + + It 'Should apply explicitly specified values, including lower values and 0' { + $r = Invoke-CopyScript -ExistingJson $existing -MsexportDays '0' -IngestionMonths '6' -RawDays '0' -FinalMonths '12' + $r.Settings.retention.msexports.days | Should -Be 0 + $r.Settings.retention.ingestion.months | Should -Be 6 -Because 'an intentional value always applies' + $r.Settings.retention.raw.days | Should -Be 0 + $r.Settings.retention.final.months | Should -Be 12 + $r.Outputs.rawRetentionInDays | Should -Be 0 + } + + It 'Should use documented defaults for a new hub when retention is not specified' { + $r = Invoke-CopyScript -ExistingJson $null + $r.Settings.retention.msexports.days | Should -Be 0 + $r.Settings.retention.ingestion.months | Should -Be 13 + $r.Settings.retention.raw.days | Should -Be 0 + $r.Settings.retention.final.months | Should -Be 13 + $r.Outputs.rawRetentionInDays | Should -Be 0 + } + + It 'Should use requested values for a new hub' { + $r = Invoke-CopyScript -ExistingJson $null -MsexportDays '5' -IngestionMonths '24' -RawDays '14' -FinalMonths '30' + $r.Settings.retention.msexports.days | Should -Be 5 + $r.Settings.retention.ingestion.months | Should -Be 24 + $r.Settings.retention.raw.days | Should -Be 14 + $r.Settings.retention.final.months | Should -Be 30 + $r.Outputs.rawRetentionInDays | Should -Be 14 + } + + It 'Should seed defaults when settings.json predates retention' { + $r = Invoke-CopyScript -ExistingJson '{"version":"","scopes":[]}' + $r.Settings.retention.msexports.days | Should -Be 0 + $r.Settings.retention.ingestion.months | Should -Be 13 + $r.Settings.retention.raw.days | Should -Be 0 + $r.Settings.retention.final.months | Should -Be 13 + } + + It 'Should seed only missing values when settings.json is partial' { + $r = Invoke-CopyScript -ExistingJson '{"version":"","scopes":[],"retention":{"ingestion":{"months":24},"final":{"months":48}}}' + $r.Settings.retention.msexports.days | Should -Be 0 + $r.Settings.retention.ingestion.months | Should -Be 24 + $r.Settings.retention.raw.days | Should -Be 0 + $r.Settings.retention.final.months | Should -Be 48 + } + } + + Context 'Raw table retention script' { + + It 'Should never set raw retention to 0 days before applying the requested value' { + $kql = Get-Content -Path $kqlPath -Raw + $kql | Should -Not -Match 'softdelete\s*=\s*0d' -Because 'reapplying 0d on every deploy can purge raw data (#2348)' + } + + It 'Should set every raw table retention policy from the deployment value' { + $kql = Get-Content -Path $kqlPath -Raw + $policies = [regex]::Matches($kql, '(?m)^\.alter-merge table \w+_raw policy retention .*$') + $policies.Count | Should -BeGreaterThan 0 + $policies | ForEach-Object { $_.Value | Should -Match 'softdelete = \$\$rawRetentionInDays\$\$d' } + } + } +} diff --git a/src/templates/finops-hub/createUiDefinition.json b/src/templates/finops-hub/createUiDefinition.json index 27c8de8dc..77ba01053 100644 --- a/src/templates/finops-hub/createUiDefinition.json +++ b/src/templates/finops-hub/createUiDefinition.json @@ -668,7 +668,7 @@ "type": "Microsoft.Common.TextBlock", "visible": true, "options": { - "text": "Configure data retention settings for Azure Data Explorer." + "text": "Configure data retention settings for Azure Data Explorer. When you redeploy or upgrade a hub, values left at the default keep the current retention settings and changed values are applied. To set a value back to the default, edit config/settings.json in the hub storage account or deploy with PowerShell." } }, { @@ -1006,10 +1006,10 @@ "enableNatGateway": "[steps('advanced').networking.enableNatGateway]", "virtualNetworkAddressPrefix": "[steps('advanced').networking.virtualNetworkAddressPrefix]", "dataExplorerSku": "[steps('pricing').dataExplorer.dataExplorerSku]", - "exportRetentionInDays": "[steps('retention').storage.msexportsDays]", - "ingestionRetentionInMonths": "[steps('retention').storage.ingestionMonths]", - "dataExplorerRawRetentionInDays": "[steps('retention').dataExplorer.rawDays]", - "dataExplorerFinalRetentionInMonths": "[steps('retention').dataExplorer.finalMonths]", + "exportRetentionInDays": "[if(or(equals(steps('retention').storage.msexportsDays, ''), equals(steps('retention').storage.msexportsDays, '0')), parse('null'), steps('retention').storage.msexportsDays)]", + "ingestionRetentionInMonths": "[if(or(equals(steps('retention').storage.ingestionMonths, ''), equals(steps('retention').storage.ingestionMonths, '13')), parse('null'), steps('retention').storage.ingestionMonths)]", + "dataExplorerRawRetentionInDays": "[if(or(equals(steps('retention').dataExplorer.rawDays, ''), equals(steps('retention').dataExplorer.rawDays, '0')), parse('null'), steps('retention').dataExplorer.rawDays)]", + "dataExplorerFinalRetentionInMonths": "[if(or(equals(steps('retention').dataExplorer.finalMonths, ''), equals(steps('retention').dataExplorer.finalMonths, '13')), parse('null'), steps('retention').dataExplorer.finalMonths)]", "remoteHubStorageUri": "[if(equals(basics('analyticsBackend').analyticsEngine, 'remote'), basics('analyticsBackend').remoteHubStorageUri, '')]", "remoteHubStorageKey": "[if(equals(basics('analyticsBackend').analyticsEngine, 'remote'), basics('analyticsBackend').remoteHubStorageKey, '')]", "enablePurgeProtection": "[if(equals(basics('analyticsBackend').analyticsEngine, 'remote'), coalesce(basics('analyticsBackend').enablePurgeProtection, false), false)]", diff --git a/src/templates/finops-hub/main.bicep b/src/templates/finops-hub/main.bicep index d5b3a8850..03be52372 100644 --- a/src/templates/finops-hub/main.bicep +++ b/src/templates/finops-hub/main.bicep @@ -144,17 +144,17 @@ param tagsByResource object = {} @description('Optional. List of scope IDs to monitor and ingest cost for.') param scopesToMonitor array = [] -@description('Optional. Number of days of data to retain in the msexports container. Default: 0.') -param exportRetentionInDays int = 0 +@description('Optional. Number of days of data to retain in the msexports container. If not specified, keeps the current value from settings.json (0 for new hubs).') +param exportRetentionInDays int? -@description('Optional. Number of months of data to retain in the ingestion container. Default: 13.') -param ingestionRetentionInMonths int = 13 +@description('Optional. Number of months of data to retain in the ingestion container. If not specified, keeps the current value from settings.json (13 for new hubs).') +param ingestionRetentionInMonths int? -@description('Optional. Number of days of data to retain in the Data Explorer *_raw tables. Default: 0.') -param dataExplorerRawRetentionInDays int = 0 +@description('Optional. Number of days of data to retain in the Data Explorer *_raw tables. If not specified, keeps the current value from settings.json (0 for new hubs).') +param dataExplorerRawRetentionInDays int? -@description('Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. Default: 13.') -param dataExplorerFinalRetentionInMonths int = 13 +@description('Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. If not specified, keeps the current value from settings.json (13 for new hubs).') +param dataExplorerFinalRetentionInMonths int? @description('Optional. Enable public access to FinOps hubs resources. Default: true.') param enablePublicAccess bool = true diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql index 94fead987..3fbe87d92 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_RawTables.kql @@ -130,9 +130,6 @@ ] ``` -// ActualCosts_raw retention policy (clear historical data) -.alter-merge table ActualCosts_raw policy retention softdelete = 0d recoverability = disabled - // ActualCosts_raw retention policy (set the user-defined retention period) .alter-merge table ActualCosts_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled @@ -262,9 +259,6 @@ ] ``` -// AmortizedCosts_raw retention policy (clear historical data) -.alter-merge table AmortizedCosts_raw policy retention softdelete = 0d recoverability = disabled - // AmortizedCosts_raw retention policy (set the user-defined retention period) .alter-merge table AmortizedCosts_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled @@ -325,9 +319,6 @@ ] ``` -// CommitmentDiscountUsage_raw retention policy (clear historical data) -.alter-merge table CommitmentDiscountUsage_raw policy retention softdelete = 0d recoverability = disabled - // CommitmentDiscountUsage_raw retention policy (set the user-defined retention period) .alter-merge table CommitmentDiscountUsage_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled @@ -656,9 +647,6 @@ ] ``` -// Costs_raw retention policy (clear historical data) -.alter-merge table Costs_raw policy retention softdelete = 0d recoverability = disabled - // Costs_raw retention policy (set the user-defined retention period) .alter-merge table Costs_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled @@ -765,9 +753,6 @@ ] ``` -// Prices_raw retention policy (clear historical data) -.alter-merge table Prices_raw policy retention softdelete = 0d recoverability = disabled - // Prices_raw retention policy (set the user-defined retention period) .alter-merge table Prices_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled @@ -878,9 +863,6 @@ ] ``` -// Recommendations_raw retention policy (clear historical data) -.alter-merge table Recommendations_raw policy retention softdelete = 0d recoverability = disabled - // Recommendations_raw retention policy (set the user-defined retention period) .alter-merge table Recommendations_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled @@ -977,9 +959,6 @@ ] ``` -// Transactions_raw retention policy (clear historical data) -.alter-merge table Transactions_raw policy retention softdelete = 0d recoverability = disabled - // Transactions_raw retention policy (set the user-defined retention period) .alter-merge table Transactions_raw policy retention softdelete = $$rawRetentionInDays$$d recoverability = disabled diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1 index 05bd1b9d5..bc18068c6 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1 +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/Copy-FileToAzureBlob.ps1 @@ -59,15 +59,6 @@ if ($blob) } } -# Remember retention values already stored in settings.json before any defaults are seeded below -$storedIngestionMonths = $null -$storedFinalMonths = $null -if ($json -and $json.retention) -{ - if ($json.retention.ingestion -and $null -ne $json.retention.ingestion.months) { $storedIngestionMonths = [Int32]$json.retention.ingestion.months } - if ($json.retention.final -and $null -ne $json.retention.final.months) { $storedFinalMonths = [Int32]$json.retention.final.months } -} - # Set default if not found if (!$json) { @@ -123,49 +114,29 @@ if (!($json.retention)) $json | Add-Member -Name retention -Value (ConvertFrom-Json $retention) -MemberType NoteProperty } -# Set or update msexports retention -if (!($json.retention.msexports)) -{ - $json.retention | Add-Member -Name msexports -Value (ConvertFrom-Json "{""days"":$($env:msexportRetentionInDays)}") -MemberType NoteProperty -} -else -{ - $json.retention.msexports.days = [Int32]::Parse($env:msexportRetentionInDays) -} - -# Set ingestion retention -# NOTE: Bicep always passes a value here (defaulting to 13 if the caller didn't specify one), so this script -# cannot tell an explicit redeploy value from a silently-defaulted one. A value already stored in settings.json -# is therefore never changed on redeploy -- a silent change ages out data the purge pipeline acts on (#2206). -# The requested value only applies when nothing was stored yet (new hub or pre-0.4 settings). -if (!($json.retention.ingestion)) -{ - $json.retention | Add-Member -Name ingestion -Value (ConvertFrom-Json "{""months"":$($env:ingestionRetentionInMonths)}") -MemberType NoteProperty -} -else -{ - $json.retention.ingestion.months = if ($null -ne $storedIngestionMonths) { $storedIngestionMonths } else { [Int32]::Parse($env:ingestionRetentionInMonths) } -} - -# Set or update raw retention -if (!($json.retention.raw)) -{ - $json.retention | Add-Member -Name raw -Value (ConvertFrom-Json "{""days"":$($env:rawRetentionInDays)}") -MemberType NoteProperty -} -else -{ - $json.retention.raw.days = [Int32]::Parse($env:rawRetentionInDays) -} - -# Set final retention (keeps any stored value -- see note above) -if (!($json.retention.final)) +# Set retention +# NOTE: The template only passes retention values the deployment specifies. When a value isn't passed, keep the stored +# value (or the default for new hubs) so a redeploy or upgrade never resets retention (#2206, #2348). +function Set-Retention([string] $Name, [string] $Unit, [int] $Default, [string] $Requested) { - $json.retention | Add-Member -Name final -Value (ConvertFrom-Json "{""months"":$($env:finalRetentionInMonths)}") -MemberType NoteProperty -} -else -{ - $json.retention.final.months = if ($null -ne $storedFinalMonths) { $storedFinalMonths } else { [Int32]::Parse($env:finalRetentionInMonths) } + if (!($json.retention.$Name)) + { + $json.retention | Add-Member -Name $Name -Value ([PSCustomObject]@{ $Unit = $Default }) -MemberType NoteProperty -Force + } + if (-not [string]::IsNullOrWhiteSpace($Requested)) + { + Write-Output " Setting $Name retention to $Requested $Unit..." + $json.retention.$Name.$Unit = [Int32]::Parse($Requested) + } + else + { + Write-Output " Keeping $Name retention at $($json.retention.$Name.$Unit) $Unit..." + } } +Set-Retention -Name msexports -Unit days -Default 0 -Requested $env:msexportRetentionInDays +Set-Retention -Name ingestion -Unit months -Default 13 -Requested $env:ingestionRetentionInMonths +Set-Retention -Name raw -Unit days -Default 0 -Requested $env:rawRetentionInDays +Set-Retention -Name final -Unit months -Default 13 -Requested $env:finalRetentionInMonths # Updating settings Write-Output "Updating version to $env:ftkVersion..." @@ -181,3 +152,8 @@ $text | Out-File $filePath # Upload new/updated settings Write-Output "Uploading settings.json file..." Set-AzStorageBlobContent @storageContext -File $filePath -Force | Out-Null + +# Return the effective raw retention so the Data Explorer setup applies the stored value +$DeploymentScriptOutputs = @{ + rawRetentionInDays = [Int32]$json.retention.raw.days +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/app.bicep b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/app.bicep index 9e1a694c0..68e67d8b5 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/app.bicep +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Core/app.bicep @@ -24,17 +24,17 @@ param scopesToMonitor array // TODO: Move export retention to the CM Exports app? // cSpell:ignore msexport -@description('Optional. Number of days of data to retain in the msexports container. Default: 0.') -param msexportRetentionInDays int = 0 +@description('Optional. Number of days of data to retain in the msexports container. If not specified, keeps the current value from settings.json (0 for new hubs).') +param msexportRetentionInDays int? -@description('Optional. Number of months of data to retain in the ingestion container. Default: 13.') -param ingestionRetentionInMonths int = 13 +@description('Optional. Number of months of data to retain in the ingestion container. If not specified, keeps the current value from settings.json (13 for new hubs).') +param ingestionRetentionInMonths int? -@description('Optional. Number of days of data to retain in the Data Explorer *_raw tables. Default: 0.') -param rawRetentionInDays int = 0 +@description('Optional. Number of days of data to retain in the Data Explorer *_raw tables. If not specified, keeps the current value from settings.json (0 for new hubs).') +param rawRetentionInDays int? -@description('Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. Default: 13.') -param finalRetentionInMonths int = 13 +@description('Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. If not specified, keeps the current value from settings.json (13 for new hubs).') +param finalRetentionInMonths int? //============================================================================== @@ -124,22 +124,6 @@ module uploadSettings '../../fx/hub-deploymentScript.bicep' = { name: 'scopes' value: join(scopesToMonitor, '|') } - { - name: 'msexportRetentionInDays' - value: string(msexportRetentionInDays) - } - { - name: 'ingestionRetentionInMonths' - value: string(ingestionRetentionInMonths) - } - { - name: 'rawRetentionInDays' - value: string(rawRetentionInDays) - } - { - name: 'finalRetentionInMonths' - value: string(finalRetentionInMonths) - } { name: 'storageAccountName' value: app.storage @@ -148,6 +132,31 @@ module uploadSettings '../../fx/hub-deploymentScript.bicep' = { name: 'containerName' value: CONFIG } + // Only pass retention values that were explicitly specified so the script keeps stored values on redeploy (#2206, #2348) + ...(msexportRetentionInDays == null ? [] : [ + { + name: 'msexportRetentionInDays' + value: string(msexportRetentionInDays) + } + ]) + ...(rawRetentionInDays == null ? [] : [ + { + name: 'rawRetentionInDays' + value: string(rawRetentionInDays) + } + ]) + ...(ingestionRetentionInMonths == null ? [] : [ + { + name: 'ingestionRetentionInMonths' + value: string(ingestionRetentionInMonths) + } + ]) + ...(finalRetentionInMonths == null ? [] : [ + { + name: 'finalRetentionInMonths' + value: string(finalRetentionInMonths) + } + ]) ] } } @@ -336,6 +345,9 @@ output storageUrlForPowerBI string = 'https://${app.storage}.dfs.${environment() @description('Name of the managed identity used to create and stop ADF triggers.') output triggerManagerIdentityName string = appRegistration.outputs.triggerManagerIdentityName +@description('Effective number of days to retain data in the Data Explorer *_raw tables, as stored in settings.json after this deployment.') +output rawRetentionInDays int = int(uploadSettings.outputs.scriptOutputs.?rawRetentionInDays ?? (rawRetentionInDays ?? 0)) + @description('Metadata describing shared resources created by the Core app.') output metadata CoreMetadata = { id: 'Microsoft.FinOpsHubs.Core' diff --git a/src/templates/finops-hub/modules/fx/hub-deploymentScript.bicep b/src/templates/finops-hub/modules/fx/hub-deploymentScript.bicep index 95408c527..a50f3a652 100644 --- a/src/templates/finops-hub/modules/fx/hub-deploymentScript.bicep +++ b/src/templates/finops-hub/modules/fx/hub-deploymentScript.bicep @@ -125,4 +125,5 @@ resource script 'Microsoft.Resources/deploymentScripts@2023-08-01' = { // Outputs //============================================================================== -// None +@description('Values the script set in $DeploymentScriptOutputs. Empty if the script did not set any.') +output scriptOutputs object = script.properties.?outputs ?? {} diff --git a/src/templates/finops-hub/modules/hub.bicep b/src/templates/finops-hub/modules/hub.bicep index c5a563d63..d829a0bed 100644 --- a/src/templates/finops-hub/modules/hub.bicep +++ b/src/templates/finops-hub/modules/hub.bicep @@ -155,17 +155,17 @@ param tagsByResource object = {} @description('Optional. List of scope IDs to monitor and ingest cost for.') param scopesToMonitor array = [] -@description('Optional. Number of days of data to retain in the msexports container. Default: 0.') -param exportRetentionInDays int = 0 +@description('Optional. Number of days of data to retain in the msexports container. If not specified, keeps the current value from settings.json (0 for new hubs).') +param exportRetentionInDays int? -@description('Optional. Number of months of data to retain in the ingestion container. Default: 13.') -param ingestionRetentionInMonths int = 13 +@description('Optional. Number of months of data to retain in the ingestion container. If not specified, keeps the current value from settings.json (13 for new hubs).') +param ingestionRetentionInMonths int? -@description('Optional. Number of days of data to retain in the Data Explorer *_raw tables. Default: 0.') -param dataExplorerRawRetentionInDays int = 0 +@description('Optional. Number of days of data to retain in the Data Explorer *_raw tables. If not specified, keeps the current value from settings.json (0 for new hubs).') +param dataExplorerRawRetentionInDays int? -@description('Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. Default: 13.') -param dataExplorerFinalRetentionInMonths int = 13 +@description('Optional. Number of months of data to retain in the Data Explorer *_final_v* tables. If not specified, keeps the current value from settings.json (13 for new hubs).') +param dataExplorerFinalRetentionInMonths int? @description('Optional. Enable public access to the data lake. Default: true.') param enablePublicAccess bool = true @@ -309,7 +309,8 @@ module analytics 'Microsoft.FinOpsHubs/Analytics/app.bicep' = if (useFabric || u clusterName: dataExplorerName clusterSku: dataExplorerSku clusterCapacity: dataExplorerCapacity - rawRetentionInDays: dataExplorerRawRetentionInDays + // Use the effective value from settings.json so a redeploy that omits the parameter keeps the current raw retention + rawRetentionInDays: core.outputs.rawRetentionInDays // TODO: Figure out why this is breaking upgrades -- clusterTrustedExternalTenants: dataExplorerTrustedExternalTenants } } From 06aacf4d352ede0019a5a0fa9d88cbd207ff84d7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 16 Oct 2026 05:48:29 +0000 Subject: [PATCH 5/5] chore: Update ms.date in docs-mslearn files --- docs-mslearn/toolkit/changelog.md | 2 +- docs-mslearn/toolkit/hubs/data-processing.md | 2 +- docs-mslearn/toolkit/hubs/template.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 68818f565..534b75153 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 10/15/2026 +ms.date: 10/16/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/hubs/data-processing.md b/docs-mslearn/toolkit/hubs/data-processing.md index 83ccb02d3..037930498 100644 --- a/docs-mslearn/toolkit/hubs/data-processing.md +++ b/docs-mslearn/toolkit/hubs/data-processing.md @@ -3,7 +3,7 @@ title: FinOps hubs data processing description: Learn how FinOps hubs process data, including scope setup, data normalization, and optimization, to enhance cost management and analysis. author: flanakin ms.author: micflan -ms.date: 10/15/2026 +ms.date: 10/16/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit diff --git a/docs-mslearn/toolkit/hubs/template.md b/docs-mslearn/toolkit/hubs/template.md index bfbb04a2b..7f8241c27 100644 --- a/docs-mslearn/toolkit/hubs/template.md +++ b/docs-mslearn/toolkit/hubs/template.md @@ -3,7 +3,7 @@ title: FinOps hub template description: Learn about what's included in the FinOps hub template including parameters, resources, and outputs. author: flanakin ms.author: micflan -ms.date: 10/15/2026 +ms.date: 10/16/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit