From 69a60849cf1093b8a3366625859627090eb78578 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Sun, 20 Sep 2026 23:11:19 -0700 Subject: [PATCH 01/11] fix(power-bi): fix SkuMeterName column error in storage reports for FOCUS 1.2 exports Co-Authored-By: Claude Sonnet 5 --- docs-mslearn/toolkit/changelog.md | 4 +++- .../storage/Shared.Dataset/definition/tables/Costs.tmdl | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 6ca4dd7c3..18b927b83 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 09/11/2026 +ms.date: 09/20/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -58,6 +58,8 @@ The following section lists features and enhancements that are currently in deve - **Changed** - Switched the InstanceSizeFlexibility table in the storage and KQL shared datasets from the retired `ccmstorageprod` AutofitComboMeterData.csv to the FinOps toolkit [Instance size flexibility](open-data.md#instance-size-flexibility) open data file, joined to reservation recommendations on the unique ARM SKU name ([#2090](https://github.com/microsoft/finops-toolkit/issues/2090)). +- **Fixed** + - Fixed the storage reports failing to refresh with "The column 'SkuMeterName' of the table wasn't found" when the Cost Management export uses the FOCUS 1.2 schema ([#2332](https://github.com/microsoft/finops-toolkit/issues/2332)). ### [Optimization Engine](optimization-engine/overview.md) diff --git a/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl b/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl index 75880048f..04dadea32 100644 --- a/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl +++ b/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl @@ -1469,7 +1469,7 @@ table Costs then Table.ReplaceValue(InvoiceId, null, each [x_PricingCurrency], Replacer.ReplaceValue, {"PricingCurrency"}) else Table.AddColumn( InvoiceId, "PricingCurrency", each [x_PricingCurrency]), SkuMeter = if Has12 - then Table.ReplaceValue(PricingCurrency, null, each [x_SkuMeterName], Replacer.ReplaceValue, {"SkuMeterName"}) + then Table.ReplaceValue(PricingCurrency, null, each [x_SkuMeterName], Replacer.ReplaceValue, {"SkuMeter"}) else Table.AddColumn( PricingCurrency, "SkuMeter", each [x_SkuMeterName]), // Copy 1.2 values to 1.0 columns Align12 = Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(SkuMeter, From add6080bc7a58f4a473a18333035771d1de1fff4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 21 Sep 2026 06:11:54 +0000 Subject: [PATCH 02/11] chore: Update ms.date in docs-mslearn files --- docs-mslearn/toolkit/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 18b927b83..ff77643c7 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 09/20/2026 +ms.date: 09/21/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit From 42eea93f758af2bb112f3c9354536c173979e5bb Mon Sep 17 00:00:00 2001 From: Roland Krummenacher Date: Mon, 21 Sep 2026 08:49:37 +0200 Subject: [PATCH 03/11] test(power-bi): add a lint rule for Power Query column names Nothing in CI runs Power Query, so a mistyped column name in a .tmdl partition ships and surfaces only as a refresh failure on a customer's report. That is how issue #2332 happened: the FOCUS 1.2 branch of the Costs query searched {"SkuMeterName"}, a column no FOCUS version emits. Two static rules: - Every columnsToSearch literal in Table.ReplaceValue must name a column that exists - from a Cost Management export schema, created by the query itself, or declared on the model table. - References to columns FOCUS 1.2 drops (x_InvoiceId, x_PricingCurrency, x_SkuMeterName) need a Table.HasColumns or _exists guard, because ftk_Storage does no backfill for focuscost. The dropped set is derived from the export schemas rather than hard-coded, so a future FOCUS version updates the rule on its own. Scoped to focuscost partitions: the pricesheet and reservationtransactions datasets reuse some of these names legitimately. Rule 2 baselines the three existing unguarded references in Costs.tmdl as a ratchet. Verified the rule catches the original typo: reverting the one-line fix fails rule 1 with "got 'SkuMeterName'". Co-Authored-By: Claude Opus 5 --- docs-mslearn/toolkit/changelog.md | 2 +- .../Lint/PowerQueryColumnNames.Tests.ps1 | 169 ++++++++++++++++++ 2 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index ff77643c7..8e7efbf35 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -59,7 +59,7 @@ The following section lists features and enhancements that are currently in deve - **Changed** - Switched the InstanceSizeFlexibility table in the storage and KQL shared datasets from the retired `ccmstorageprod` AutofitComboMeterData.csv to the FinOps toolkit [Instance size flexibility](open-data.md#instance-size-flexibility) open data file, joined to reservation recommendations on the unique ARM SKU name ([#2090](https://github.com/microsoft/finops-toolkit/issues/2090)). - **Fixed** - - Fixed the storage reports failing to refresh with "The column 'SkuMeterName' of the table wasn't found" when the Cost Management export uses the FOCUS 1.2 schema ([#2332](https://github.com/microsoft/finops-toolkit/issues/2332)). + - Fixed the storage reports failing to refresh with "The column 'SkuMeterName' of the table wasn't found" when the Cost Management export uses the FOCUS 1.2 schema, and added a lint rule that fails the build when a Power Query column name doesn't exist ([#2332](https://github.com/microsoft/finops-toolkit/issues/2332)). ### [Optimization Engine](optimization-engine/overview.md) diff --git a/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 b/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 new file mode 100644 index 000000000..fdba8f0b0 --- /dev/null +++ b/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 @@ -0,0 +1,169 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +<# + Lint rule: Power Query column names in Power BI datasets must refer to columns that exist. + + Nothing in CI runs Power Query, so a mistyped column name in a .tmdl partition ships and only + surfaces as a refresh failure on a customer's report. That is exactly what happened with + issue #2332: the FOCUS 1.2 branch of the Costs query passed {"SkuMeterName"} as the + columnsToSearch of Table.ReplaceValue, but the column FOCUS 1.2 emits is SkuMeter. The typo + was introduced in fa19ad68 and survived because the 1.2 branch was never exercised. + + Two rules, both static: + + 1. Known columns - every columnsToSearch literal in Table.ReplaceValue must be a column that + exists: one of the Cost Management export schemas, one the query creates itself via + Table.AddColumn / Table.RenameColumns, or a column declared on the model table. + + 2. Version-dropped columns - FOCUS 1.2 removes three columns that earlier versions carry + (x_InvoiceId and x_PricingCurrency were promoted to InvoiceId / PricingCurrency, + x_SkuMeterName was renamed to SkuMeter). ftk_Storage does no backfill for focuscost + (CleanColumns passes ExtractColumns straight through), so the columns available are exactly + the union of what the export files contain. Referencing a dropped column without a + Table.HasColumns / _exists guard fails on a container that holds only 1.2 exports. + + Verified 2026-09-21 against a real FocusCost export (dataVersion 1.2-preview, 105 columns): + InvoiceId, PricingCurrency and SkuMeter are present; x_InvoiceId, x_PricingCurrency and + x_SkuMeterName are absent. Note that docs-mslearn/focus/metadata.md is not a reliable + source for this - it listed x_SkuMeterName under 1.2-preview until this change corrected it. + The ADF mappings under Exports/schemas are authoritative and are what this rule reads. + + Rule 2 is baselined per file as a ratchet: + - Guarding a reference REQUIRES lowering the count here (the test fails on stale entries). + - Adding a new unguarded reference is never allowed. +#> + +Describe 'PowerQueryColumnNames' { + + BeforeDiscovery { + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path + + # Power Query lives in the `source = ```...``` ` partition blocks of the dataset .tmdl files. + $partitionPattern = [regex]'(?s)source = \x60{3}(.*?)\x60{3}' + + $scanFiles = @( + Get-ChildItem -Path (Join-Path $repoRoot 'src/power-bi') -Filter '*.tmdl' -Recurse -File -ErrorAction SilentlyContinue | + Where-Object { $partitionPattern.IsMatch((Get-Content -Path $_.FullName -Raw)) } | + Sort-Object FullName | + ForEach-Object { + @{ FullName = $_.FullName; RelPath = $_.FullName.Substring($repoRoot.Length + 1).Replace('\', '/') } + } + ) + } + + BeforeAll { + $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path + $schemaDir = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.CostManagement/Exports/schemas' + + # Every column name any Cost Management export can deliver, across all dataset types and + # FOCUS versions. Both sides of the mapping count: `source` is the export column, `sink` is + # the name the hub stores it under, and queries legitimately reference either. + function Get-SchemaColumn([string] $path) + { + $mappings = (Get-Content -Path $path -Raw | ConvertFrom-Json).translator.mappings + return @($mappings | ForEach-Object { $_.source.name; $_.sink.name } | Where-Object { $_ } | Sort-Object -Unique) + } + + $schemaFiles = @(Get-ChildItem -Path $schemaDir -Filter '*.json' -File) + $knownColumns = [System.Collections.Generic.HashSet[string]]::new([string[]]@($schemaFiles | ForEach-Object { Get-SchemaColumn $_.FullName }), [StringComparer]::Ordinal) + + # Columns an earlier FOCUS version carries that 1.2 does not. Derived from the schemas + # rather than hard-coded so a future FOCUS version updates the rule automatically. + $focus10 = Get-SchemaColumn (Join-Path $schemaDir 'focuscost_1.0r2.json') + $focus12 = @( + (Get-SchemaColumn (Join-Path $schemaDir 'focuscost_1.2-preview.json')) + (Get-SchemaColumn (Join-Path $schemaDir 'focuscost_1.2.json')) + ) | Sort-Object -Unique + $droppedInFocus12 = @($focus10 | Where-Object { $_ -notin $focus12 }) + + $partitionPattern = [regex]'(?s)source = \x60{3}(.*?)\x60{3}' + # Last argument of Table.ReplaceValue: the columns the replacement searches. + $columnsToSearchPattern = [regex]'Replacer\.\w+\s*,\s*\{([^}]*)\}' + $quotedNamePattern = [regex]'"([A-Za-z_][A-Za-z0-9_]*)"' + $addColumnPattern = [regex]'Table\.AddColumn\(\s*[^,]+,\s*"([^"]+)"' + $renamePairPattern = [regex]'\{\s*"([^"]+)"\s*,\s*"([^"]+)"\s*\}' + $modelColumnPattern = [regex]'(?m)^\s*sourceColumn:\s*(\S+)' + + # Unguarded references to columns FOCUS 1.2 drops, counted per repo-relative path. + # Ratchet only: lower on fix, never raise. + # + # Costs.tmdl: the "Handle columns renamed in FOCUS 1.2 gracefully" block reads + # [x_InvoiceId], [x_PricingCurrency] and [x_SkuMeterName], and Align12 passes all three as + # columnsToSearch. On a container holding only 1.2 exports none of them exist. Guard the + # block with the _exists / _swapCol helpers already defined at the top of the query, then + # lower this entry. Tracked on issue #2332 / PR #2333. + $baseline = @{ + 'src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl' = 3 + } + } + + It 'Should find the export schemas and the dataset partitions' { + $knownColumns.Count | Should -BeGreaterThan 100 -Because 'the Cost Management export schemas define the column vocabulary this rule checks against' + $droppedInFocus12.Count | Should -BeGreaterThan 0 -Because 'FOCUS 1.2 renames or promotes columns that earlier versions carry; if this is empty the schema files moved or changed shape' + } + + It 'Should only search columns that exist: ' -ForEach $scanFiles { + $content = Get-Content -Path $FullName -Raw + $partitions = @($partitionPattern.Matches($content) | ForEach-Object { $_.Groups[1].Value }) + + # Columns the query builds for itself are just as valid as exported ones. + $created = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($partition in $partitions) + { + foreach ($match in $addColumnPattern.Matches($partition)) { [void] $created.Add($match.Groups[1].Value) } + foreach ($match in $renamePairPattern.Matches($partition)) { [void] $created.Add($match.Groups[2].Value) } + } + foreach ($match in $modelColumnPattern.Matches($content)) { [void] $created.Add($match.Groups[1].Value) } + + $searched = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($partition in $partitions) + { + foreach ($match in $columnsToSearchPattern.Matches($partition)) + { + foreach ($name in $quotedNamePattern.Matches($match.Groups[1].Value)) { [void] $searched.Add($name.Groups[1].Value) } + } + } + + $unknown = @($searched | Where-Object { -not $knownColumns.Contains($_) -and -not $created.Contains($_) } | Sort-Object) + + $unknown -join ', ' | Should -BeNullOrEmpty -Because 'Table.ReplaceValue fails with "The column '''' of the table wasn''t found" when columnsToSearch names a column that is not in the table. Every name must be an export column, a column the query adds itself, or a column on the model table (see issue #2332).' + } + + It 'Should not reference columns FOCUS 1.2 drops without a guard: ' -ForEach $scanFiles { + $content = Get-Content -Path $FullName -Raw + + # FOCUS versioning only applies to the focuscost dataset. The other datasets have their own + # schemas that happen to share column names: Prices reads "pricesheet" and renames MeterName + # to x_SkuMeterName, ReservationTransactions reads "reservationtransactions". Those uses are + # correct and must not be flagged. + $partitions = @( + $partitionPattern.Matches($content) | + ForEach-Object { $_.Groups[1].Value } | + Where-Object { $_ -match 'ftk_Storage\(\s*"focuscost"' } + ) + $allowed = if ($baseline.ContainsKey($RelPath)) { $baseline[$RelPath] } else { 0 } + + $unguarded = @( + foreach ($column in $droppedInFocus12) + { + $referenced = $false + $guarded = $false + foreach ($partition in $partitions) + { + if ($partition -match ('\[{0}\]|"{0}"' -f [regex]::Escape($column))) { $referenced = $true } + if ($partition -match ('(Table\.HasColumns|_exists)\([^)]*"{0}"' -f [regex]::Escape($column))) { $guarded = $true } + } + if ($referenced -and -not $guarded) { $column } + } + ) + + $unguarded.Count | Should -BeLessOrEqual $allowed -Because ("FOCUS 1.2 does not deliver these columns and ftk_Storage does not backfill them, so a container holding only 1.2 exports fails to refresh. Guard the reference with Table.HasColumns or the _exists helper. Unguarded here: $($unguarded -join ', ')") + + if ($unguarded.Count -le $allowed) + { + # Ratchet: if a reference was guarded, the baseline must be lowered so it cannot return. + $unguarded.Count | Should -Be $allowed -Because ("the unguarded reference count in this file dropped below the baseline ($allowed); lower the baseline entry for '$RelPath' in PowerQueryColumnNames.Tests.ps1 to $($unguarded.Count) (or remove it if 0) so the fix is locked in.") + } + } +} From 9a22d267aaa9c2a2310c263dcd667265563a49f2 Mon Sep 17 00:00:00 2001 From: Roland Krummenacher Date: Mon, 21 Sep 2026 08:49:38 +0200 Subject: [PATCH 04/11] docs(focus): drop x_SkuMeterName from the FOCUS 1.2 column list The FocusCost 1.2-preview table listed x_SkuMeterName alongside the SkuMeter column that replaced it. Verified against a real FocusCost export (dataVersion 1.2-preview, 105 columns): x_SkuMeterName is not delivered. The 1.0 and 1.0-preview(v1) tables keep the column. The ADF mappings in Exports/schemas already had this right. Co-Authored-By: Claude Opus 5 --- docs-mslearn/focus/metadata.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs-mslearn/focus/metadata.md b/docs-mslearn/focus/metadata.md index 93160875e..9a4d68d94 100644 --- a/docs-mslearn/focus/metadata.md +++ b/docs-mslearn/focus/metadata.md @@ -3,7 +3,7 @@ title: FOCUS metadata description: This article provides general information about the FOCUS dataset including the data generator, schema version, and columns included in the dataset. author: flanakin ms.author: micflan -ms.date: 04/01/2026 +ms.date: 09/21/2026 ms.topic: reference ms.service: finops ms.subservice: finops-learning-resources @@ -128,7 +128,6 @@ Columns include: | `x_SkuIsCreditEligible` | Boolean | Indicates if the charge is eligible for Azure credits. | | `x_SkuMeterCategory` | String | Name of the service the SKU falls within. | | `x_SkuMeterId` | String | Unique identifier (sometimes a GUID, but not always) for the usage meter. It usually maps to a specific SKU or range of SKUs that have a specific price. | -| `x_SkuMeterName` | String | Name of the usage meter. It usually maps to a specific SKU or range of SKUs that have a specific price. Not applicable for purchases. | | `x_SkuMeterSubcategory` | String | Group of SKU Classes that address the same core need within the SKU Group. | | `x_SkuOfferId` | String | Microsoft Cloud subscription type. | | `x_SkuOrderId` | String | Unique identifier of the entitlement product for this charge. Same as MCA ProductOrderId. Not applicable for EA. | From e7e11e01437201d9d9db5a464b490138115c078e Mon Sep 17 00:00:00 2001 From: Roland Krummenacher Date: Mon, 21 Sep 2026 09:06:35 +0200 Subject: [PATCH 05/11] fix(power-bi): handle FOCUS 1.2 exports that carry no 1.0 columns The "Handle columns renamed in FOCUS 1.2 gracefully" block read [x_InvoiceId], [x_PricingCurrency] and [x_SkuMeterName] unconditionally, and Align12 passed all three as columnsToSearch. FOCUS 1.2 does not deliver them - verified against a real export (dataVersion 1.2-preview, 105 columns): InvoiceId, PricingCurrency and SkuMeter are present, the three x_ columns are not. ftk_Storage does no backfill for focuscost, so a container holding only 1.2 exports has none of them and the refresh fails with "The column 'x_InvoiceId' of the table wasn't found" - the same class of error as #2332, just one step later. EnsureRenamedCols now adds whichever side is missing as a null column before Align12 copies values in both directions, so 1.0-only, 1.2-only and mixed containers all work. Has12 is gone: the per-column checks replace it and it had no other caller. Lowers the PowerQueryColumnNames rule 2 baseline to zero. Co-Authored-By: Claude Opus 5 --- docs-mslearn/toolkit/changelog.md | 2 +- .../definition/tables/Costs.tmdl | 31 ++++++++++++------- .../Lint/PowerQueryColumnNames.Tests.ps1 | 14 +++------ 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 8e7efbf35..364ec3ac4 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -59,7 +59,7 @@ The following section lists features and enhancements that are currently in deve - **Changed** - Switched the InstanceSizeFlexibility table in the storage and KQL shared datasets from the retired `ccmstorageprod` AutofitComboMeterData.csv to the FinOps toolkit [Instance size flexibility](open-data.md#instance-size-flexibility) open data file, joined to reservation recommendations on the unique ARM SKU name ([#2090](https://github.com/microsoft/finops-toolkit/issues/2090)). - **Fixed** - - Fixed the storage reports failing to refresh with "The column 'SkuMeterName' of the table wasn't found" when the Cost Management export uses the FOCUS 1.2 schema, and added a lint rule that fails the build when a Power Query column name doesn't exist ([#2332](https://github.com/microsoft/finops-toolkit/issues/2332)). + - Fixed the storage reports failing to refresh when the Cost Management export uses the FOCUS 1.2 schema. The FOCUS 1.2 branch searched for a column named `SkuMeterName` instead of `SkuMeter`, and the surrounding block assumed the FOCUS 1.0 columns (`x_InvoiceId`, `x_PricingCurrency`, `x_SkuMeterName`) were always present, which fails for a storage container that holds only FOCUS 1.2 exports. Added a lint rule that fails the build when a Power Query column name doesn't exist ([#2332](https://github.com/microsoft/finops-toolkit/issues/2332)). ### [Optimization Engine](optimization-engine/overview.md) diff --git a/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl b/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl index 04dadea32..27ce027af 100644 --- a/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl +++ b/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl @@ -1431,7 +1431,6 @@ table Costs PerformDeprecatedOptimizations = #"Deprecated: Perform Extra Query Optimizations", HasPreview = Table.HasColumns(RawData, {"ChargeSubcategory"}), Has10 = Table.HasColumns(RawData, {"ChargeClass"}), - Has12 = Table.HasColumns(RawData, {"InvoiceId"}), HandlePeview = if HasPreview = false then RawData else if PerformDeprecatedOptimizations = false then error "LegacyFocusVersion: Please enable the 'Perform Extra Query Optimizations' parameter" else ( @@ -1462,17 +1461,25 @@ table Costs RemovePreviewCols ), // Handle columns renamed in FOCUS 1.2 gracefully -- Copy values to/from 1.0 and 1.2 columns - InvoiceId = if Has12 - then Table.ReplaceValue(HandlePeview, null, each [x_InvoiceId], Replacer.ReplaceValue, {"InvoiceId"}) - else Table.AddColumn( HandlePeview, "InvoiceId", each [x_InvoiceId]), - PricingCurrency = if Has12 - then Table.ReplaceValue(InvoiceId, null, each [x_PricingCurrency], Replacer.ReplaceValue, {"PricingCurrency"}) - else Table.AddColumn( InvoiceId, "PricingCurrency", each [x_PricingCurrency]), - SkuMeter = if Has12 - then Table.ReplaceValue(PricingCurrency, null, each [x_SkuMeterName], Replacer.ReplaceValue, {"SkuMeter"}) - else Table.AddColumn( PricingCurrency, "SkuMeter", each [x_SkuMeterName]), - // Copy 1.2 values to 1.0 columns - Align12 = Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(SkuMeter, + // FOCUS 1.2 promotes x_InvoiceId/x_PricingCurrency to InvoiceId/PricingCurrency and renames + // x_SkuMeterName to SkuMeter. ftk_Storage does not backfill focuscost columns, so a container + // holding only 1.2 exports has none of the 1.0 columns (and a 1.0-only container none of the + // 1.2 ones). Add whichever side is missing as null first, then copy values in both directions. + EnsureRenamedCols = + let + a = if Table.HasColumns(HandlePeview, {"x_InvoiceId"}) then HandlePeview else Table.AddColumn(HandlePeview, "x_InvoiceId", each null), + b = if Table.HasColumns(a, {"x_PricingCurrency"}) then a else Table.AddColumn(a, "x_PricingCurrency", each null), + c = if Table.HasColumns(b, {"x_SkuMeterName"}) then b else Table.AddColumn(b, "x_SkuMeterName", each null), + d = if Table.HasColumns(c, {"InvoiceId"}) then c else Table.AddColumn(c, "InvoiceId", each null), + e = if Table.HasColumns(d, {"PricingCurrency"}) then d else Table.AddColumn(d, "PricingCurrency", each null), + f = if Table.HasColumns(e, {"SkuMeter"}) then e else Table.AddColumn(e, "SkuMeter", each null) + in + f, + // Copy 1.0 values to the 1.2 columns, then 1.2 values back to the 1.0 columns + Align12 = Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(EnsureRenamedCols, + null, each [x_InvoiceId], Replacer.ReplaceValue, {"InvoiceId"}), + null, each [x_PricingCurrency], Replacer.ReplaceValue, {"PricingCurrency"}), + null, each [x_SkuMeterName], Replacer.ReplaceValue, {"SkuMeter"}), null, each [InvoiceId], Replacer.ReplaceValue, {"x_InvoiceId"}), null, each [PricingCurrency], Replacer.ReplaceValue, {"x_PricingCurrency"}), null, each [SkuMeter], Replacer.ReplaceValue, {"x_SkuMeterName"}), diff --git a/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 b/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 index fdba8f0b0..fc3bf0408 100644 --- a/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 +++ b/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 @@ -86,16 +86,10 @@ Describe 'PowerQueryColumnNames' { $modelColumnPattern = [regex]'(?m)^\s*sourceColumn:\s*(\S+)' # Unguarded references to columns FOCUS 1.2 drops, counted per repo-relative path. - # Ratchet only: lower on fix, never raise. - # - # Costs.tmdl: the "Handle columns renamed in FOCUS 1.2 gracefully" block reads - # [x_InvoiceId], [x_PricingCurrency] and [x_SkuMeterName], and Align12 passes all three as - # columnsToSearch. On a container holding only 1.2 exports none of them exist. Guard the - # block with the _exists / _swapCol helpers already defined at the top of the query, then - # lower this entry. Tracked on issue #2332 / PR #2333. - $baseline = @{ - 'src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl' = 3 - } + # Ratchet only: lower on fix, never raise. Empty today - Costs.tmdl guards all three with + # Table.HasColumns in its EnsureRenamedCols step. Add an entry only for a reference that + # genuinely cannot be guarded, and say why. + $baseline = @{} } It 'Should find the export schemas and the dataset partitions' { From 49f1d117e1b46dff80a84d258a13adf93e837d21 Mon Sep 17 00:00:00 2001 From: Roland Krummenacher Date: Mon, 12 Oct 2026 08:55:49 +0200 Subject: [PATCH 06/11] refactor(power-bi): fold the FOCUS 1.2 column guard into a helper The previous version chained six let bindings named a through f, which reads nothing like the rest of the query - every other step there says what it does (HandlePeview, RemovePreviewCols, Align12). Adds an _ensureCol helper next to the existing _exists / _swapCol / _resetCol, and folds it over the six column names with List.Accumulate, the same idiom CleanColumns uses in expressions.tmdl. Nine lines become four and the placeholder names are gone. No behaviour change. The PowerQueryColumnNames rule now also accepts a column as guarded when it appears in a List.Accumulate name list that folds in _ensureCol. Verified it still bites: replacing EnsureRenamedCols with a plain pass-through makes it report all three columns. Co-Authored-By: Claude Opus 5 --- .../definition/tables/Costs.tmdl | 15 +++++---------- .../Tests/Lint/PowerQueryColumnNames.Tests.ps1 | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl b/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl index 27ce027af..5dd7d1c8a 100644 --- a/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl +++ b/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl @@ -1426,6 +1426,7 @@ table Costs _stringValue = (val) => if _isBlank(val) then null else val, _swapCol = (tbl, oldCol, newCol, newVal, optional newType) => let add = Table.AddColumn(tbl, newCol, newVal, newType), del = if Table.HasColumns(add, oldCol) then Table.RemoveColumns(add, {oldCol}) else add in del, _resetCol = (tbl, col, newVal, optional newType) => let tmpCol = Text.Combine({"tmp_", col}), tmp = _swapCol(tbl, col, tmpCol, newVal, newType), ren = Table.RenameColumns(tmp, {{tmpCol, col}}) in ren, + _ensureCol = (tbl, col) => if Table.HasColumns(tbl, {col}) then tbl else Table.AddColumn(tbl, col, each null), // Update previous FOCUS versions PerformDeprecatedOptimizations = #"Deprecated: Perform Extra Query Optimizations", @@ -1465,16 +1466,10 @@ table Costs // x_SkuMeterName to SkuMeter. ftk_Storage does not backfill focuscost columns, so a container // holding only 1.2 exports has none of the 1.0 columns (and a 1.0-only container none of the // 1.2 ones). Add whichever side is missing as null first, then copy values in both directions. - EnsureRenamedCols = - let - a = if Table.HasColumns(HandlePeview, {"x_InvoiceId"}) then HandlePeview else Table.AddColumn(HandlePeview, "x_InvoiceId", each null), - b = if Table.HasColumns(a, {"x_PricingCurrency"}) then a else Table.AddColumn(a, "x_PricingCurrency", each null), - c = if Table.HasColumns(b, {"x_SkuMeterName"}) then b else Table.AddColumn(b, "x_SkuMeterName", each null), - d = if Table.HasColumns(c, {"InvoiceId"}) then c else Table.AddColumn(c, "InvoiceId", each null), - e = if Table.HasColumns(d, {"PricingCurrency"}) then d else Table.AddColumn(d, "PricingCurrency", each null), - f = if Table.HasColumns(e, {"SkuMeter"}) then e else Table.AddColumn(e, "SkuMeter", each null) - in - f, + EnsureRenamedCols = List.Accumulate( + {"x_InvoiceId", "x_PricingCurrency", "x_SkuMeterName", "InvoiceId", "PricingCurrency", "SkuMeter"}, + HandlePeview, + (tbl, col) => _ensureCol(tbl, col)), // Copy 1.0 values to the 1.2 columns, then 1.2 values back to the 1.0 columns Align12 = Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(Table.ReplaceValue(EnsureRenamedCols, null, each [x_InvoiceId], Replacer.ReplaceValue, {"InvoiceId"}), diff --git a/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 b/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 index fc3bf0408..7c33f41d6 100644 --- a/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 +++ b/src/powershell/Tests/Lint/PowerQueryColumnNames.Tests.ps1 @@ -84,6 +84,8 @@ Describe 'PowerQueryColumnNames' { $addColumnPattern = [regex]'Table\.AddColumn\(\s*[^,]+,\s*"([^"]+)"' $renamePairPattern = [regex]'\{\s*"([^"]+)"\s*,\s*"([^"]+)"\s*\}' $modelColumnPattern = [regex]'(?m)^\s*sourceColumn:\s*(\S+)' + # List.Accumulate over a list of column names, folding in the _ensureCol helper. + $ensureListPattern = [regex]'(?s)List\.Accumulate\(\s*\{([^}]*)\}.{0,400}?_ensureCol' # Unguarded references to columns FOCUS 1.2 drops, counted per repo-relative path. # Ratchet only: lower on fix, never raise. Empty today - Costs.tmdl guards all three with @@ -138,11 +140,23 @@ Describe 'PowerQueryColumnNames' { ) $allowed = if ($baseline.ContainsKey($RelPath)) { $baseline[$RelPath] } else { 0 } + # A column counts as guarded either by a direct Table.HasColumns / _exists check, or by + # appearing in the name list of a List.Accumulate that folds the _ensureCol helper over the + # table - the idiom Costs.tmdl uses to add whichever side of a renamed pair is missing. + $ensured = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($partition in $partitions) + { + foreach ($match in $ensureListPattern.Matches($partition)) + { + foreach ($name in $quotedNamePattern.Matches($match.Groups[1].Value)) { [void] $ensured.Add($name.Groups[1].Value) } + } + } + $unguarded = @( foreach ($column in $droppedInFocus12) { $referenced = $false - $guarded = $false + $guarded = $ensured.Contains($column) foreach ($partition in $partitions) { if ($partition -match ('\[{0}\]|"{0}"' -f [regex]::Escape($column))) { $referenced = $true } @@ -152,7 +166,7 @@ Describe 'PowerQueryColumnNames' { } ) - $unguarded.Count | Should -BeLessOrEqual $allowed -Because ("FOCUS 1.2 does not deliver these columns and ftk_Storage does not backfill them, so a container holding only 1.2 exports fails to refresh. Guard the reference with Table.HasColumns or the _exists helper. Unguarded here: $($unguarded -join ', ')") + $unguarded.Count | Should -BeLessOrEqual $allowed -Because ("FOCUS 1.2 does not deliver these columns and ftk_Storage does not backfill them, so a container holding only 1.2 exports fails to refresh. Guard the reference with Table.HasColumns, or add the column through the _ensureCol helper. Unguarded here: $($unguarded -join ', ')") if ($unguarded.Count -le $allowed) { From 619bd1f1762fb9a46f52222de3b4f3dbe644120c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 12 Oct 2026 07:08:00 +0000 Subject: [PATCH 07/11] chore: Update ms.date in docs-mslearn files --- docs-mslearn/focus/metadata.md | 2 +- docs-mslearn/toolkit/changelog.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-mslearn/focus/metadata.md b/docs-mslearn/focus/metadata.md index 9a4d68d94..2aa622974 100644 --- a/docs-mslearn/focus/metadata.md +++ b/docs-mslearn/focus/metadata.md @@ -3,7 +3,7 @@ title: FOCUS metadata description: This article provides general information about the FOCUS dataset including the data generator, schema version, and columns included in the dataset. author: flanakin ms.author: micflan -ms.date: 09/21/2026 +ms.date: 10/12/2026 ms.topic: reference ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 364ec3ac4..277876f5e 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 09/21/2026 +ms.date: 10/12/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit From a477e11ec6b4a16a9065de43b9663689135bf079 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 14 Oct 2026 22:25:24 +0000 Subject: [PATCH 08/11] chore: Update ms.date in docs-mslearn files --- docs-mslearn/focus/metadata.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-mslearn/focus/metadata.md b/docs-mslearn/focus/metadata.md index 2aa622974..d4b2efb27 100644 --- a/docs-mslearn/focus/metadata.md +++ b/docs-mslearn/focus/metadata.md @@ -3,7 +3,7 @@ title: FOCUS metadata description: This article provides general information about the FOCUS dataset including the data generator, schema version, and columns included in the dataset. author: flanakin ms.author: micflan -ms.date: 10/12/2026 +ms.date: 10/14/2026 ms.topic: reference ms.service: finops ms.subservice: finops-learning-resources From a209cbc62e7d64ddb18195df120003c67dc8966a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 15 Oct 2026 08:06:50 +0000 Subject: [PATCH 09/11] chore: Update ms.date in docs-mslearn files --- docs-mslearn/focus/metadata.md | 2 +- docs-mslearn/toolkit/changelog.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs-mslearn/focus/metadata.md b/docs-mslearn/focus/metadata.md index d4b2efb27..7c3a2ef81 100644 --- a/docs-mslearn/focus/metadata.md +++ b/docs-mslearn/focus/metadata.md @@ -3,7 +3,7 @@ title: FOCUS metadata description: This article provides general information about the FOCUS dataset including the data generator, schema version, and columns included in the dataset. author: flanakin ms.author: micflan -ms.date: 10/14/2026 +ms.date: 10/15/2026 ms.topic: reference ms.service: finops ms.subservice: finops-learning-resources diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 4998e837e..c1f26294c 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 10/14/2026 +ms.date: 10/15/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit From ae950be8e306598e1b0640d587facbd45fceb305 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Thu, 15 Oct 2026 10:52:11 -0700 Subject: [PATCH 10/11] fix(hubs): detect FOCUS version before backfilling PricingCurrency Costs_transform_v1_2 filled PricingCurrency from x_PricingCurrency before deciding x_SourceVersion, so every FOCUS 1.0 row looked like 1.2. Those rows skipped the 1.0-specific fixes (for example, the savings plan purchase ChargeFrequency correction). Keep the delivered value in tmp_PricingCurrency and use it for version detection. Verified against ftk-dev: all Costs_final_v1_2 rows were labeled 1.2 or 1.2-preview with no InvoiceId or SkuMeter populated. Co-Authored-By: Claude Sonnet 5.5 --- docs-mslearn/toolkit/changelog.md | 1 + .../Analytics/scripts/IngestionSetup_v1_2.kql | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index c1f26294c..d5284b0f2 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -42,6 +42,7 @@ The following section lists features and enhancements that are currently in deve - Switched dimension enrichment in the v1_0/v1_2 ingestion transforms (`PricingUnits`, `Regions`, `ResourceTypes`, `Services`) from `join` to the broadcast-optimized `lookup` operator and deduplicated the `Services` mapping per resource type to prevent cost row fan-out ([#2225](https://github.com/microsoft/finops-toolkit/pull/2225)). - Fixed FinOps hub dashboard "Governance, policy, and risk" and "FinOps tools and services" links that landed on a 404 page by linking directly to the Microsoft Learn pages ([#2319](https://github.com/microsoft/finops-toolkit/issues/2319)). - Fixed the v1_0 and v1_2 price transforms setting `x_CommitmentDiscountSpendEligibility` from reservation meters and `x_CommitmentDiscountUsageEligibility` from savings plan meters, the opposite of the FOCUS `CommitmentDiscountCategory` value the same transform assigns. The spend column now reports savings plan pricing and the usage column reports reservation pricing. Prices ingested after upgrading carry the corrected values; rows ingested earlier keep the old values until they are reingested ([#2279](https://github.com/microsoft/finops-toolkit/issues/2279)). + - Fixed FinOps hubs labeling FOCUS 1.0 cost data as FOCUS 1.2 in `x_SourceVersion`. The cost transform backfilled `PricingCurrency` from `x_PricingCurrency` before detecting the FOCUS version, so every 1.0 row looked like 1.2 and skipped the 1.0-specific fixes (for example, the savings plan purchase `ChargeFrequency` correction). Version detection now uses the value delivered by the export ([#2333](https://github.com/microsoft/finops-toolkit/pull/2333)). ### [FinOps workbooks](workbooks/finops-workbooks-overview.md) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql index 7a87870dc..63c25dd22 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql @@ -365,6 +365,8 @@ Costs_transform_v1_2() )) // // Handle provider columns that moved to FOCUS + // Keep the original value so version detection below only sees what the export delivered (x_PricingCurrency backfill would make FOCUS 1.0 look like 1.2) + | extend tmp_PricingCurrency = PricingCurrency | extend PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency) // // Backup original prices/costs before the merge @@ -388,8 +390,8 @@ Costs_transform_v1_2() | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) | extend x_SourceType = coalesce(x_SourceType, iff(isnotempty(x_BillingProfileId), 'FocusCost', '')) | extend x_SourceVersion = coalesce(x_SourceVersion, case( - isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)) and isempty(SkuPriceDetails) and isnotempty(x_SkuDetails), '1.2-preview', - isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)), '1.2', + isnotempty(coalesce(InvoiceId, SkuMeter, tmp_PricingCurrency)) and isempty(SkuPriceDetails) and isnotempty(x_SkuDetails), '1.2-preview', + isnotempty(coalesce(InvoiceId, SkuMeter, tmp_PricingCurrency)), '1.2', isnotempty(coalesce(ChargeClass, CommitmentDiscountStatus, tostring(ConsumedQuantity), ConsumedUnit, tostring(ContractedCost), tostring(ContractedUnitPrice), RegionId, RegionName)), '1.0', isnotempty(coalesce(ChargeSubcategory, Region, tostring(UsageQuantity), UsageUnit)), iff(ProviderName == 'Microsoft', '1.0-preview(v1)', '1.0-preview'), '' From 83d88f0ba16a4cf78446762c2bf0510ddbf8bd23 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Thu, 15 Oct 2026 10:55:09 -0700 Subject: [PATCH 11/11] refactor(hubs): backfill PricingCurrency after version detection Move the PricingCurrency backfill below x_SourceVersion detection instead of cloning the delivered value into tmp_PricingCurrency. Nothing between the two positions reads PricingCurrency, so behavior is the same without the extra column. Co-Authored-By: Claude Sonnet 5.5 --- .../Analytics/scripts/IngestionSetup_v1_2.kql | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql index 63c25dd22..3488f8478 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql @@ -364,11 +364,6 @@ Costs_transform_v1_2() iff(x_SkuTerm == '1Year' or x_SkuTerm == '3Years' or x_SkuTerm == '5Years', 'SkuTermShouldBeAnInteger,', '') )) // - // Handle provider columns that moved to FOCUS - // Keep the original value so version detection below only sees what the export delivered (x_PricingCurrency backfill would make FOCUS 1.0 look like 1.2) - | extend tmp_PricingCurrency = PricingCurrency - | extend PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency) - // // Backup original prices/costs before the merge | extend old_ContractedCost = ContractedCost | extend old_ContractedUnitPrice = ContractedUnitPrice @@ -390,8 +385,8 @@ Costs_transform_v1_2() | extend x_SourceProvider = coalesce(x_SourceProvider, ProviderName) | extend x_SourceType = coalesce(x_SourceType, iff(isnotempty(x_BillingProfileId), 'FocusCost', '')) | extend x_SourceVersion = coalesce(x_SourceVersion, case( - isnotempty(coalesce(InvoiceId, SkuMeter, tmp_PricingCurrency)) and isempty(SkuPriceDetails) and isnotempty(x_SkuDetails), '1.2-preview', - isnotempty(coalesce(InvoiceId, SkuMeter, tmp_PricingCurrency)), '1.2', + isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)) and isempty(SkuPriceDetails) and isnotempty(x_SkuDetails), '1.2-preview', + isnotempty(coalesce(InvoiceId, SkuMeter, PricingCurrency)), '1.2', isnotempty(coalesce(ChargeClass, CommitmentDiscountStatus, tostring(ConsumedQuantity), ConsumedUnit, tostring(ContractedCost), tostring(ContractedUnitPrice), RegionId, RegionName)), '1.0', isnotempty(coalesce(ChargeSubcategory, Region, tostring(UsageQuantity), UsageUnit)), iff(ProviderName == 'Microsoft', '1.0-preview(v1)', '1.0-preview'), '' @@ -401,6 +396,9 @@ Costs_transform_v1_2() strcat(x_SourceChanges, iff(isempty(x_SourceChanges), '', ','), iff(x_SourceVersion == '', 'UnknownFocusVersion', 'LegacyFocusVersion')) ) // + // Handle provider columns that moved to FOCUS -- After version detection, which must only see what the export delivered (backfilling x_PricingCurrency first would make FOCUS 1.0 look like 1.2) + | extend PricingCurrency = coalesce(PricingCurrency, x_PricingCurrency) + // // Fix quantities | extend old_PricingQuantity = PricingQuantity, PricingQuantity = case( PricingQuantity != 0 or (EffectiveCost == 0 and BilledCost == 0), PricingQuantity,