diff --git a/CHANGELOG/v3.md b/CHANGELOG/v3.md index b623735..e008615 100644 --- a/CHANGELOG/v3.md +++ b/CHANGELOG/v3.md @@ -1,5 +1,19 @@ # v3 Changelog +## [v3.1.7] + +- Schema cache format 2. One JSON document per line with a keyword index in the header. The fast host + deserializes a keyword the first time a compile uses it instead of rehydrating the whole module. + Caches of the previous format are regenerated. +- The cache fingerprint is built from file names and sizes instead of write times, so the cache + shipped inside a package stays valid after installation. +- `Import-DscResource` modules are located by a manifest scan of `PSModulePath` instead of + `Get-Module -ListAvailable`. +- The text rewrite of the fast host runs as one pass over one parse instead of three parses with a + string copy per resource statement. +- Schema based resources (`*.schema.mof`) compile on the fast path. +- Added `Get-DscFastCompileTiming` and implemented `-Force` on `Invoke-DscFastCompile`. + ## [v3.1.6] - Fixed an issue where the MOF file encoding was set to UTF-8 without BOM instead of UTF-16LE required diff --git a/M365DSC.PSDesiredStateConfiguration/Compat/PSDesiredStateConfiguration/PSDesiredStateConfiguration.psd1 b/M365DSC.PSDesiredStateConfiguration/Compat/PSDesiredStateConfiguration/PSDesiredStateConfiguration.psd1 index a35827b..50c9183 100644 --- a/M365DSC.PSDesiredStateConfiguration/Compat/PSDesiredStateConfiguration/PSDesiredStateConfiguration.psd1 +++ b/M365DSC.PSDesiredStateConfiguration/Compat/PSDesiredStateConfiguration/PSDesiredStateConfiguration.psd1 @@ -6,7 +6,7 @@ RootModule = 'PSDesiredStateConfiguration.psm1' - ModuleVersion = '3.1.6' + ModuleVersion = '3.1.7' CompatiblePSEditions = @('Desktop', 'Core') @@ -30,6 +30,7 @@ 'Invoke-DscFastCompile' 'Export-DscSchemaCache' 'Test-DscSchemaCache' + 'Get-DscFastCompileTiming' ) CmdletsToExport = @() diff --git a/M365DSC.PSDesiredStateConfiguration/Compat/PSDesiredStateConfiguration/PSDesiredStateConfiguration.psm1 b/M365DSC.PSDesiredStateConfiguration/Compat/PSDesiredStateConfiguration/PSDesiredStateConfiguration.psm1 index b757549..a6fa2fe 100644 --- a/M365DSC.PSDesiredStateConfiguration/Compat/PSDesiredStateConfiguration/PSDesiredStateConfiguration.psm1 +++ b/M365DSC.PSDesiredStateConfiguration/Compat/PSDesiredStateConfiguration/PSDesiredStateConfiguration.psm1 @@ -13,6 +13,7 @@ $script:ForwardedCommands = @( 'Invoke-DscFastCompile' 'Export-DscSchemaCache' 'Test-DscSchemaCache' + 'Get-DscFastCompileTiming' ) function Resolve-EngineManifest diff --git a/M365DSC.PSDesiredStateConfiguration/FastHost.ps1 b/M365DSC.PSDesiredStateConfiguration/FastHost.ps1 index cd8b008..9d1ef33 100644 --- a/M365DSC.PSDesiredStateConfiguration/FastHost.ps1 +++ b/M365DSC.PSDesiredStateConfiguration/FastHost.ps1 @@ -21,23 +21,23 @@ $__fhBody = $args[$args.Count - 1] $__fhName = '' if ($__fhKeyword.NameMode -eq [System.Management.Automation.Language.DynamicKeywordNameMode]::NameRequired) { - if ($args.Count -lt 2 -or $args[0] -isnot [string]) + if ($args.Count -lt 2 -or $args[0] -isnot [System.String]) { throw "Resource '$__fhKeywordName': expected '$__fhKeywordName { ... }'." } $__fhName = $args[0] } -elseif ($args.Count -ge 2 -and $args[0] -is [string]) +elseif ($args.Count -ge 2 -and $args[0] -is [System.String]) { $__fhName = $args[0] } if ($__fhBody -isnot [hashtable]) { - throw "Resource '$__fhKeywordName': the property block reached the adapter as $($__fhBody.GetType().Name) instead of a hashtable. Convert-FastHostBodyToHashtable did not rewrite it." + throw "Resource '$__fhKeywordName': the property block reached the adapter as $($__fhBody.GetType().Name) instead of a hashtable. ConvertTo-FastHostCompileText did not rewrite it." } $__fhValue = $__fhBody $__fhFile = $MyInvocation.ScriptName -if ([string]::IsNullOrEmpty($__fhFile)) +if ([System.String]::IsNullOrEmpty($__fhFile)) { # The body runs from a scriptblock built out of the stripped text, so it carries no # file. Use the path the compile was started from, to keep SourceInfo identical to @@ -50,32 +50,78 @@ $__fhSource = "$__fhFile::$($MyInvocation.ScriptLineNumber)::$($MyInvocation.Off function Get-FastHostScriptPath { - [OutputType([string])] + [OutputType([System.String])] param() $script:FastHostScriptPath } +# The cache stays as text lines until a compile asks for a keyword. function Get-FastHostKeyword { param ( [Parameter(Mandatory)] - [string] + [System.String] $Name ) if ($script:FastHostKeywords) { - $script:FastHostKeywords[$Name] + $keyword = $script:FastHostKeywords[$Name] + if ($null -ne $keyword) + { + return $keyword + } } + + if ($script:FastHostKeywordSource) + { + foreach ($cache in $script:FastHostKeywordSource) + { + $keyword = Get-DscSchemaCacheKeyword -Cache $cache -Name $Name + if ($null -ne $keyword) + { + $script:FastHostKeywords[$keyword.Keyword] = $keyword + return $keyword + } + } + } + + $null +} + +function Get-FastHostKeywordName +{ + [OutputType([System.Collections.Generic.HashSet[System.String]])] + param() + + $names = New-Object -TypeName 'System.Collections.Generic.HashSet[System.String]' -ArgumentList ([System.StringComparer]::OrdinalIgnoreCase) + if ($script:FastHostKeywords) + { + foreach ($key in $script:FastHostKeywords.Keys) + { + $null = $names.Add($key) + } + } + if ($script:FastHostKeywordSource) + { + foreach ($cache in $script:FastHostKeywordSource) + { + foreach ($name in (Get-DscSchemaCacheKeywordName -Cache $cache)) + { + $null = $names.Add($name) + } + } + } + , $names } function Get-FastHostBodyScriptBlock { - [OutputType([scriptblock])] + [OutputType([ScriptBlock])] param ( [Parameter(Mandatory)] - [scriptblock] + [ScriptBlock] $Body ) @@ -84,13 +130,13 @@ function Get-FastHostBodyScriptBlock if ($null -eq $cached) { $inner = $text.Substring(1, $text.Length - 2) - $cached = [scriptblock]::Create('@{' + $inner + '}') + $cached = [ScriptBlock]::Create('@{' + $inner + '}') $script:FastHostBodyCache[$text] = $cached } $cached } -function Register-DscCachedKeywords +function Register-DscSchemaCache { param ( [Parameter(Mandatory)] @@ -105,16 +151,20 @@ function Register-DscCachedKeywords { $script:FastHostAdapters = New-Object -TypeName 'System.Collections.Generic.Dictionary[string,scriptblock]' -ArgumentList ([System.StringComparer]::OrdinalIgnoreCase) } - $adapter = [scriptblock]::Create($script:FastHostAdapterText) + if ($null -eq $script:FastHostKeywordSource) + { + $script:FastHostKeywordSource = New-Object -TypeName 'System.Collections.Generic.List[System.Object]' + } + $script:FastHostKeywordSource.Add($Cache) - foreach ($schemaObject in $Cache.keywords) + $adapter = [ScriptBlock]::Create($script:FastHostAdapterText) + $moduleName = [System.String]$Cache.module.name + foreach ($name in (Get-DscSchemaCacheKeywordName -Cache $Cache)) { - $keyword = ConvertFrom-DscKeywordSchemaObject -SchemaObject $schemaObject - $script:FastHostKeywords[$keyword.Keyword] = $keyword - $script:FastHostAdapters[$keyword.Keyword] = $adapter - if ($keyword.ImplementingModule) + $script:FastHostAdapters[$name] = $adapter + if ($moduleName) { - $script:FastHostAdapters["$($keyword.ImplementingModule)\$($keyword.Keyword)"] = $adapter + $script:FastHostAdapters["$moduleName\$name"] = $adapter } } } @@ -163,7 +213,7 @@ function Get-FastHostMergeCandidateKeyword $elements = $command.CommandElements if ($elements.Count -le 2) { - return [pscustomobject]@{ + return [PSCustomObject]@{ Name = $command.GetCommandName() EndOffset = $command.Extent.EndOffset } @@ -178,7 +228,7 @@ function Get-FastHostMergeCandidateKeyword return $null } - return [pscustomobject]@{ + return [PSCustomObject]@{ Name = $keywordElement.Value EndOffset = $command.Extent.EndOffset } @@ -187,147 +237,250 @@ function Get-FastHostMergeCandidateKeyword return $null } -# Joins 'KeywordName [InstanceName] { ... }' into one statement. With the -# keyword unknown to the parser, a next-line brace parses as a separate scriptblock -# statement and the engine rejects the resource as undefined. -function Merge-FastHostResourceStatements +function Get-FastHostBodyKeywordName { - [OutputType([string])] + [OutputType([System.String])] param ( [Parameter(Mandatory)] - [string] - $Text, + [System.Management.Automation.Language.CommandAst] + $Command + ) + + $elements = $Command.CommandElements + if ($elements.Count -ge 3 -and $elements[1].Extent.Text -eq '=') + { + $keyword = $elements[2] -as [System.Management.Automation.Language.StringConstantExpressionAst] + if ($null -eq $keyword) + { + return $null + } + return $keyword.Value + } + + return $Command.GetCommandName() +} + +# A parsed configuration statement imports its resource modules. Masking the word keeps the +# parser away from that, and the same-length replacement keeps every extent offset valid for +# the original text. +function Get-FastHostMaskedAst +{ + [OutputType([System.Management.Automation.Language.ScriptBlockAst])] + param ( [Parameter(Mandatory)] - $KeywordNames + [System.String] + $Text ) $masked = [regex]::Replace($Text, '(?i)\bConfiguration\b', 'C0nfiguration') $tokens = $null $parseErrors = $null - $ast = [System.Management.Automation.Language.Parser]::ParseInput($masked, [ref]$tokens, [ref]$parseErrors) + [System.Management.Automation.Language.Parser]::ParseInput($masked, [ref]$tokens, [ref]$parseErrors) +} + +function Invoke-FastHostTextEdit +{ + [OutputType([System.String])] + param ( + [Parameter(Mandatory)] + [AllowEmptyString()] + [System.String] + $Text, + + [Parameter(Mandatory)] + [AllowEmptyCollection()] + $Edit + ) + + if ($Edit.Count -eq 0) + { + return $Text + } - $splices = New-Object -TypeName 'System.Collections.Generic.List[object]' - $blocks = $ast.FindAll({ $args[0] -is [System.Management.Automation.Language.StatementBlockAst] -or $args[0] -is [System.Management.Automation.Language.NamedBlockAst] }, $true) - foreach ($block in $blocks) + $ordered = @($Edit | Sort-Object -Property Start, Length) + $builder = New-Object -TypeName System.Text.StringBuilder -ArgumentList ($Text.Length + $ordered.Count * 2) + $position = 0 + foreach ($item in $ordered) { - $statements = $block.Statements - for ($i = 0; $i -lt $statements.Count - 1; $i++) + if ($item.Start -lt $position) { - $first = $statements[$i] - $candidate = Get-FastHostMergeCandidateKeyword -Statement $first - if ($null -eq $candidate -or -not $candidate.Name -or -not $KeywordNames.Contains($candidate.Name)) + throw "Overlapping text edits at offset $($item.Start)." + } + $null = $builder.Append($Text.Substring($position, $item.Start - $position)) + $null = $builder.Append($item.Text) + $position = $item.Start + $item.Length + } + $null = $builder.Append($Text.Substring($position)) + $builder.ToString() +} + +function ConvertTo-FastHostCompileText +{ + [OutputType([System.String])] + param ( + [Parameter(Mandatory)] + [System.String] + $Text, + + [System.Management.Automation.Language.ScriptBlockAst] + $Ast, + + [AllowEmptyCollection()] + $ImportStatement = @(), + + [AllowEmptyCollection()] + $KeywordNames = @(), + + [switch] + $Merge, + + [switch] + $Convert + ) + + if ($null -eq $Ast) + { + $Ast = Get-FastHostMaskedAst -Text $Text + } + + $names = $KeywordNames -as [System.Collections.Generic.HashSet[System.String]] + if ($null -eq $names) + { + $names = New-Object -TypeName 'System.Collections.Generic.HashSet[System.String]' -ArgumentList ([System.StringComparer]::OrdinalIgnoreCase) + foreach ($name in @($KeywordNames)) { $null = $names.Add([System.String]$name) } + } + + $edits = New-Object -TypeName 'System.Collections.Generic.List[System.Object]' + foreach ($statement in @($ImportStatement)) + { + $edits.Add([PSCustomObject]@{ Start = $statement.Extent.StartOffset; Length = $statement.Extent.EndOffset - $statement.Extent.StartOffset; Text = '' }) + } + + if ($names.Count -eq 0 -or -not ($Merge -or $Convert)) + { + return Invoke-FastHostTextEdit -Text $Text -Edit $edits + } + + $bodyStarts = New-Object -TypeName 'System.Collections.Generic.HashSet[int]' + + # With the keyword unknown to the parser, a next-line brace parses as a separate scriptblock + # statement and the engine rejects the resource as undefined. + if ($Merge) + { + $blocks = $Ast.FindAll({ $args[0] -is [System.Management.Automation.Language.StatementBlockAst] -or $args[0] -is [System.Management.Automation.Language.NamedBlockAst] }, $true) + foreach ($block in $blocks) + { + $statements = $block.Statements + for ($i = 0; $i -lt $statements.Count - 1; $i++) { - continue + $candidate = Get-FastHostMergeCandidateKeyword -Statement $statements[$i] + if ($null -eq $candidate -or -not $candidate.Name -or -not $names.Contains($candidate.Name)) + { + continue + } + $second = $statements[$i + 1] + if ($second -isnot [System.Management.Automation.Language.PipelineAst] -or $second.PipelineElements.Count -ne 1) + { + continue + } + $expression = $second.PipelineElements[0] -as [System.Management.Automation.Language.CommandExpressionAst] + if ($null -eq $expression -or + ($expression.Expression -isnot [System.Management.Automation.Language.ScriptBlockExpressionAst] -and + $expression.Expression -isnot [System.Management.Automation.Language.HashtableAst])) + { + continue + } + $gapStart = $candidate.EndOffset + $gapLength = $second.Extent.StartOffset - $gapStart + if ($gapLength -gt 0) + { + $edits.Add([PSCustomObject]@{ Start = $gapStart; Length = $gapLength; Text = ' ' }) + } + if ($Convert -and $expression.Expression -is [System.Management.Automation.Language.ScriptBlockExpressionAst]) + { + $offset = $second.Extent.StartOffset + if ($bodyStarts.Add($offset)) + { + $prefix = if ($gapLength -gt 0 -or ($offset -gt 0 -and [char]::IsWhiteSpace($Text[$offset - 1]))) { '@' } else { ' @' } + $edits.Add([PSCustomObject]@{ Start = $offset; Length = 0; Text = $prefix }) + } + } } - $second = $statements[$i + 1] - if ($second -isnot [System.Management.Automation.Language.PipelineAst] -or $second.PipelineElements.Count -ne 1) + } + } + + if ($Convert) + { + $commands = $Ast.FindAll({ $args[0] -is [System.Management.Automation.Language.CommandAst] }, $true) + foreach ($command in $commands) + { + $elements = $command.CommandElements + $body = $elements[$elements.Count - 1] -as [System.Management.Automation.Language.ScriptBlockExpressionAst] + if ($null -eq $body) { continue } - $expression = $second.PipelineElements[0] -as [System.Management.Automation.Language.CommandExpressionAst] - if ($null -eq $expression -or - ($expression.Expression -isnot [System.Management.Automation.Language.ScriptBlockExpressionAst] -and - $expression.Expression -isnot [System.Management.Automation.Language.HashtableAst])) + + $name = Get-FastHostBodyKeywordName -Command $command + if (-not $name -or -not $names.Contains($name)) { continue } - $gapStart = $candidate.EndOffset - $gapLength = $second.Extent.StartOffset - $gapStart - if ($gapLength -gt 0) + + $offset = $body.Extent.StartOffset + if ($bodyStarts.Add($offset)) { - $splices.Add([pscustomobject]@{ Start = $gapStart; Length = $gapLength }) + # 'MSFT_Type @{' parses, 'MSFT_Type@{' does not + # Keep a separator when the source had none + $prefix = if ($offset -gt 0 -and -not [char]::IsWhiteSpace($Text[$offset - 1])) { ' @' } else { '@' } + $edits.Add([PSCustomObject]@{ Start = $offset; Length = 0; Text = $prefix }) } } } - foreach ($splice in ($splices | Sort-Object -Property Start -Descending)) - { - $Text = $Text.Remove($splice.Start, $splice.Length).Insert($splice.Start, ' ') - } - $Text + Invoke-FastHostTextEdit -Text $Text -Edit $edits } -function Get-FastHostBodyKeywordName +function Merge-FastHostResourceStatements { - [OutputType([string])] + [OutputType([System.String])] param ( [Parameter(Mandatory)] - [System.Management.Automation.Language.CommandAst] - $Command - ) - - $elements = $Command.CommandElements - if ($elements.Count -ge 3 -and $elements[1].Extent.Text -eq '=') - { - $keyword = $elements[2] -as [System.Management.Automation.Language.StringConstantExpressionAst] - if ($null -eq $keyword) - { - return $null - } + [System.String] + $Text, - return $keyword.Value - } + [Parameter(Mandatory)] + $KeywordNames + ) - return $Command.GetCommandName() + ConvertTo-FastHostCompileText -Text $Text -KeywordNames $KeywordNames -Merge } function Convert-FastHostBodyToHashtable { - [OutputType([string])] + [OutputType([System.String])] param ( [Parameter(Mandatory)] - [string] + [System.String] $Text, [Parameter(Mandatory)] $KeywordNames ) - $masked = [regex]::Replace($Text, '(?i)\bConfiguration\b', 'C0nfiguration') - $ast = [System.Management.Automation.Language.Parser]::ParseInput($masked, [ref]$null, [ref]$null) - - $offsets = New-Object -TypeName 'System.Collections.Generic.List[int]' - $commands = $ast.FindAll({ $args[0] -is [System.Management.Automation.Language.CommandAst] }, $true) - foreach ($command in $commands) - { - $elements = $command.CommandElements - $body = $elements[$elements.Count - 1] -as [System.Management.Automation.Language.ScriptBlockExpressionAst] - if ($null -eq $body) - { - continue - } - - $name = Get-FastHostBodyKeywordName -Command $command - if (-not $name -or -not $KeywordNames.Contains($name)) - { - continue - } - - $offsets.Add($body.Extent.StartOffset) - } - - foreach ($offset in ($offsets | Sort-Object -Descending)) - { - # 'MSFT_Type @{' parses, 'MSFT_Type@{' does not - # Keep a separator when the source had none - $prefix = if ($offset -gt 0 -and -not [char]::IsWhiteSpace($Text[$offset - 1])) { ' @' } else { '@' } - $Text = $Text.Insert($offset, $prefix) - } - $Text + ConvertTo-FastHostCompileText -Text $Text -KeywordNames $KeywordNames -Convert } + function Get-StrippedConfigurationText { param ( [Parameter(Mandatory)] - [string] + [System.String] $Text ) - $masked = [regex]::Replace($Text, '(?i)\bConfiguration\b', 'C0nfiguration') - $tokens = $null - $parseErrors = $null - $ast = [System.Management.Automation.Language.Parser]::ParseInput($masked, [ref]$tokens, [ref]$parseErrors) + $ast = Get-FastHostMaskedAst -Text $Text $configurationNames = @() $importStatements = @() @@ -429,6 +582,8 @@ function Get-StrippedConfigurationText Text = $null ModuleSpecs = @() ConfigurationNames = @() + Ast = $null + ImportStatements = @() } } @@ -440,6 +595,8 @@ function Get-StrippedConfigurationText Text = $null ModuleSpecs = @() ConfigurationNames = @() + Ast = $null + ImportStatements = @() } } @@ -453,46 +610,142 @@ function Get-StrippedConfigurationText } } - $stripped = $Text - foreach ($statement in ($importStatements | Sort-Object -Property { $_.Extent.StartOffset } -Descending)) - { - $stripped = $stripped.Remove($statement.Extent.StartOffset, $statement.Extent.EndOffset - $statement.Extent.StartOffset) - } - [PSCustomObject]@{ Supported = $true Reason = $null - Text = $stripped + Text = ConvertTo-FastHostCompileText -Text $Text -Ast $ast -ImportStatement $importStatements ModuleSpecs = $moduleSpecs ConfigurationNames = $configurationNames + Ast = $ast + ImportStatements = $importStatements } } +# Get-Module -ListAvailable analyzes every nested module of a manifest and costs seconds for +# a large resource module. A manifest scan is enough for the cache lookup. +function Resolve-FastHostModule +{ + [OutputType([System.Object])] + param ( + [Parameter(Mandatory)] + [System.String] + $ModuleName, + + [version] + $ModuleVersion + ) + + $candidates = New-Object -TypeName 'System.Collections.Generic.List[System.Object]' + foreach ($entry in ($env:PSModulePath -split [System.IO.Path]::PathSeparator)) + { + if ([System.String]::IsNullOrWhiteSpace($entry)) + { + continue + } + $moduleFolder = Join-Path -Path $entry -ChildPath $ModuleName + if (-not [System.IO.Directory]::Exists($moduleFolder)) + { + continue + } + $flat = Join-Path -Path $moduleFolder -ChildPath "$ModuleName.psd1" + if ([System.IO.File]::Exists($flat)) + { + $candidates.Add([PSCustomObject]@{ Path = $flat; FolderVersion = $null }) + } + foreach ($versionFolder in [System.IO.Directory]::EnumerateDirectories($moduleFolder)) + { + $manifest = Join-Path -Path $versionFolder -ChildPath "$ModuleName.psd1" + if (-not [System.IO.File]::Exists($manifest)) + { + continue + } + $folderVersion = $null + $null = [version]::TryParse((Split-Path $versionFolder -Leaf), [ref]$folderVersion) + $candidates.Add([PSCustomObject]@{ Path = $manifest; FolderVersion = $folderVersion }) + } + } + + $best = $null + foreach ($candidate in $candidates) + { + if ($ModuleVersion -and $candidate.FolderVersion -and $candidate.FolderVersion -ne $ModuleVersion) + { + continue + } + try + { + $data = Import-PowerShellDataFile -Path $candidate.Path -ErrorAction Stop + } + catch + { + continue + } + $version = $null + if (-not [version]::TryParse([System.String]$data.ModuleVersion, [ref]$version)) + { + continue + } + if ($ModuleVersion -and $version -ne $ModuleVersion) + { + continue + } + if ($null -eq $best -or $version -gt $best.Version) + { + $best = [PSCustomObject]@{ + Name = $ModuleName + Version = $version + ModuleBase = Split-Path $candidate.Path -Parent + Path = $candidate.Path + } + } + } + + $best +} + +function Get-DscFastCompileTiming +{ + [OutputType([System.Collections.Specialized.OrderedDictionary])] + param() + + if ($null -eq $script:FastHostTiming) + { + return $null + } + $copy = [ordered]@{} + foreach ($entry in $script:FastHostTiming.GetEnumerator()) + { + $copy[$entry.Key] = $entry.Value + } + $copy +} +Export-ModuleMember -Function Get-DscFastCompileTiming + function Invoke-DscFastCompile { [CmdletBinding(DefaultParameterSetName = 'Path')] param ( [Parameter(Mandatory, ParameterSetName = 'Path', Position = 0)] - [string] + [System.String] $Path, [Parameter(Mandatory, ParameterSetName = 'Text')] - [string] + [System.String] $ScriptText, - [string] + [System.String] $ConfigurationName, - [hashtable] + [System.Collections.Hashtable] $Parameters, - [object] + [System.Object] $ConfigurationData, - [string] + [System.String] $OutputPath, - [string[]] + [System.String[]] $SchemaCachePath, [switch] @@ -511,6 +764,11 @@ function Invoke-DscFastCompile $ScriptText = [System.IO.File]::ReadAllText($Path) } + $timing = [ordered]@{} + $script:FastHostTiming = $timing + $stopwatch = [System.Diagnostics.Stopwatch]::StartNew() + $total = [System.Diagnostics.Stopwatch]::StartNew() + Assert-DscConfigurationShim $fallbackReason = $null @@ -519,6 +777,8 @@ function Invoke-DscFastCompile { $fallbackReason = $stripResult.Reason } + $timing['parse'] = $stopwatch.ElapsedMilliseconds + $stopwatch.Restart() $resolvedModules = @() if (-not $fallbackReason) @@ -529,12 +789,7 @@ function Invoke-DscFastCompile $module = $script:FastHostResolvedModules[$specKey] if ($null -eq $module) { - $candidates = Get-Module -ListAvailable -Name $spec.ModuleName | Sort-Object -Property Version -Descending - if ($spec.ModuleVersion) - { - $candidates = $candidates | Where-Object { $_.Version -eq $spec.ModuleVersion } - } - $module = $candidates | Select-Object -First 1 + $module = Resolve-FastHostModule -ModuleName $spec.ModuleName -ModuleVersion $spec.ModuleVersion if ($null -ne $module) { $script:FastHostResolvedModules[$specKey] = $module @@ -549,17 +804,18 @@ function Invoke-DscFastCompile $dscResourcesPath = Join-Path $module.ModuleBase 'DscResources' if (Test-Path $dscResourcesPath) { - $schemaFiles = @([System.IO.Directory]::EnumerateFiles($dscResourcesPath, '*.schema.mof', [System.IO.SearchOption]::AllDirectories)) + - @([System.IO.Directory]::EnumerateFiles($dscResourcesPath, '*.schema.psm1', [System.IO.SearchOption]::AllDirectories)) - if ($schemaFiles.Count -gt 0) + $compositeFiles = @([System.IO.Directory]::EnumerateFiles($dscResourcesPath, '*.schema.psm1', [System.IO.SearchOption]::AllDirectories)) + if ($compositeFiles.Count -gt 0) { - $fallbackReason = "Module '$($module.Name)' contains script-based or composite resources, which the fast host does not support yet." + $fallbackReason = "Module '$($module.Name)' contains composite resources, which the fast host does not support yet." break } } $resolvedModules += $module } } + $timing['resolve'] = $stopwatch.ElapsedMilliseconds + $stopwatch.Restart() if ($fallbackReason) { @@ -574,11 +830,15 @@ function Invoke-DscFastCompile foreach ($module in $resolvedModules) { $registered = $script:FastHostRegisteredModules[$module.Name] - if ($registered -and $registered -eq $module.Version.ToString()) + if ($registered -and $registered -eq $module.Version.ToString() -and -not $Force) { continue } - $cache = Get-DscSchemaCache -Module $module -SchemaCachePath $SchemaCachePath + $cache = $null + if (-not $Force) + { + $cache = Get-DscSchemaCache -Module $module -SchemaCachePath $SchemaCachePath + } if (-not $cache) { $cache = New-DscSchemaCacheForModule -Module $module @@ -592,16 +852,16 @@ function Invoke-DscFastCompile Write-Warning -Message "Falling back to standard compilation: no usable schema cache for module '$($module.Name)' $($module.Version)." return Invoke-DscFastCompileBody -Text $ScriptText -ConfigurationName $ConfigurationName -Parameters $Parameters -ConfigurationData $ConfigurationData -OutputPath $OutputPath -ScriptPath $Path -ConfigurationNames $stripResult.ConfigurationNames } - Register-DscCachedKeywords -Cache $cache + Register-DscSchemaCache -Cache $cache $script:FastHostRegisteredModules[$module.Name] = $module.Version.ToString() } + $timing['cache'] = $stopwatch.ElapsedMilliseconds + $stopwatch.Restart() - $compileText = $stripResult.Text - if ($script:FastHostKeywords -and $script:FastHostKeywords.Count -gt 0) - { - $compileText = Merge-FastHostResourceStatements -Text $compileText -KeywordNames $script:FastHostKeywords.Keys - $compileText = Convert-FastHostBodyToHashtable -Text $compileText -KeywordNames $script:FastHostKeywords.Keys - } + $keywordNames = Get-FastHostKeywordName + $compileText = ConvertTo-FastHostCompileText -Text $ScriptText -Ast $stripResult.Ast -ImportStatement $stripResult.ImportStatements -KeywordNames $keywordNames -Merge -Convert + $timing['rewrite'] = $stopwatch.ElapsedMilliseconds + $stopwatch.Restart() $script:FastHostActive = $true $script:FastHostValidateMof = [bool]$ValidateMof @@ -616,6 +876,8 @@ function Invoke-DscFastCompile $script:FastHostValidateMof = $false $Global:PSDscFastCompileActive = $false [System.Management.Automation.Language.DynamicKeyword]::Reset() + $timing['compile'] = $stopwatch.ElapsedMilliseconds + $timing['total'] = $total.ElapsedMilliseconds } } Export-ModuleMember -Function Invoke-DscFastCompile @@ -624,33 +886,29 @@ function Invoke-DscFastCompileBody { param ( [Parameter(Mandatory)] - [string] + [System.String] $Text, - [string] + [System.String] $ConfigurationName, - [hashtable] + [System.Collections.Hashtable] $Parameters, - [object] + [System.Object] $ConfigurationData, - [string] + [System.String] $OutputPath, - [string] + [System.String] $ScriptPath, - [string[]] + [System.String[]] $ConfigurationNames ) - $scriptBlock = [scriptblock]::Create($Text) - - # A scriptblock built from text has no file, so the adapter cannot read the source - # path off $MyInvocation. Hand it the real one to keep SourceInfo aligned with the - # standard path. + $scriptBlock = [ScriptBlock]::Create($Text) $script:FastHostScriptPath = $ScriptPath # Parsing the configuration statement makes the engine load the inbox module by diff --git a/M365DSC.PSDesiredStateConfiguration/M365DSC.PSDesiredStateConfiguration.psd1 b/M365DSC.PSDesiredStateConfiguration/M365DSC.PSDesiredStateConfiguration.psd1 index e0070d4..45be285 100644 --- a/M365DSC.PSDesiredStateConfiguration/M365DSC.PSDesiredStateConfiguration.psd1 +++ b/M365DSC.PSDesiredStateConfiguration/M365DSC.PSDesiredStateConfiguration.psd1 @@ -9,7 +9,7 @@ RootModule = 'M365DSC.PSDesiredStateConfiguration.psm1' # Version number of this module. - ModuleVersion = '3.1.6' + ModuleVersion = '3.1.7' # Supported PSEditions CompatiblePSEditions = @('Desktop', 'Core') @@ -74,6 +74,7 @@ 'Invoke-DscFastCompile' 'Export-DscSchemaCache' 'Test-DscSchemaCache' + 'Get-DscFastCompileTiming' ) diff --git a/M365DSC.PSDesiredStateConfiguration/M365DSC.PSDesiredStateConfiguration.psm1 b/M365DSC.PSDesiredStateConfiguration/M365DSC.PSDesiredStateConfiguration.psm1 index 8ff4b4f..c034a88 100644 --- a/M365DSC.PSDesiredStateConfiguration/M365DSC.PSDesiredStateConfiguration.psm1 +++ b/M365DSC.PSDesiredStateConfiguration/M365DSC.PSDesiredStateConfiguration.psm1 @@ -93,6 +93,8 @@ $script:FastHostAdapters = $null $script:FastHostBodyCache = @{} $script:FastHostRegisteredModules = @{} $script:FastHostResolvedModules = @{} +$script:FastHostKeywordSource = $null +$script:FastHostTiming = $null function Get-CimKeywordImplementationFunction { [OutputType([scriptblock])] @@ -116,27 +118,11 @@ function Get-DscModuleFingerprint [OutputType([string])] param ( [Parameter(Mandatory)] - [System.Management.Automation.PSModuleInfo] $Module ) - $count = 0 - $maxTicks = [long]0 - foreach ($pattern in '*.psm1', '*.psd1', '*.mof') - { - foreach ($file in [System.IO.Directory]::EnumerateFiles($Module.ModuleBase, $pattern, [System.IO.SearchOption]::AllDirectories)) - { - $count++ - $ticks = [System.IO.File]::GetLastWriteTimeUtc($file).Ticks - if ($ticks -gt $maxTicks) - { - $maxTicks = $ticks - } - } - } - "${count}:$maxTicks" + ConvertTo-DscSourceFingerprint -Entry (Get-DscModuleSourceEntry -ModuleBase $Module.ModuleBase) } - function Reset-DscKeywordState { [OutputType([void])] @@ -514,7 +500,7 @@ function ConvertTo-MOFInstance $script:PsDscCompatibleVersion = "2.0.0" } # Look up the keyword definition; fast-host keywords take precedence over the parser table. - $keywordDefinition = if ($script:FastHostKeywords) { $script:FastHostKeywords[$Type] } else { $null } + $keywordDefinition = Get-FastHostKeyword -Name $Type if ($null -eq $keywordDefinition) { $keywordDefinition = [System.Management.Automation.Language.DynamicKeyword]::GetKeyword($Type) diff --git a/M365DSC.PSDesiredStateConfiguration/SchemaCache.ps1 b/M365DSC.PSDesiredStateConfiguration/SchemaCache.ps1 index 1a8441f..a15d4da 100644 --- a/M365DSC.PSDesiredStateConfiguration/SchemaCache.ps1 +++ b/M365DSC.PSDesiredStateConfiguration/SchemaCache.ps1 @@ -1,22 +1,22 @@ -# Persistent DSC schema cache: serializes DynamicKeyword definitions of a module -# to JSON so the fast host can register them without any module parsing. +# Line based cache. Line 1 holds the header with the keyword index, line 2 the source file +# list, every later line one keyword. -$script:SchemaCacheFormatVersion = 1 +$script:SchemaCacheFormatVersion = 2 function Get-DscSchemaCacheUserPath { - [OutputType([string])] + [OutputType([System.String])] param ( [Parameter(Mandatory)] - [string] + [System.String] $ModuleName, [Parameter(Mandatory)] - [version] + [System.Version] $ModuleVersion, [Parameter(Mandatory)] - [string] + [System.String] $Fingerprint ) @@ -83,7 +83,7 @@ function ConvertFrom-DscKeywordSchemaObject $keyword.ImplementingModule = $SchemaObject.implementingModule if ($SchemaObject.implementingModuleVersion) { - $keyword.ImplementingModuleVersion = [version]$SchemaObject.implementingModuleVersion + $keyword.ImplementingModuleVersion = [System.Version]$SchemaObject.implementingModuleVersion } $keyword.NameMode = [System.Management.Automation.Language.DynamicKeywordNameMode]$SchemaObject.nameMode $keyword.BodyMode = [System.Management.Automation.Language.DynamicKeywordBodyMode]$SchemaObject.bodyMode @@ -104,7 +104,7 @@ function ConvertFrom-DscKeywordSchemaObject { if ($null -ne $mapEntry) { - $property.ValueMap[[string]$mapEntry.key] = $mapEntry.value + $property.ValueMap[[System.String]$mapEntry.key] = $mapEntry.value } } $keyword.Properties.Add($item.Name, $property) @@ -113,23 +113,151 @@ function ConvertFrom-DscKeywordSchemaObject $keyword } +function Get-DscModuleSourceEntry +{ + [OutputType([System.Object[]])] + param ( + [Parameter(Mandatory)] + [System.String] + $ModuleBase + ) + + $entries = New-Object -TypeName 'System.Collections.Generic.List[System.Object]' + foreach ($pattern in '*.psm1', '*.psd1', '*.mof') + { + foreach ($file in [System.IO.Directory]::EnumerateFiles($ModuleBase, $pattern, [System.IO.SearchOption]::AllDirectories)) + { + $entries.Add([PSCustomObject]@{ + Path = $file.Substring($ModuleBase.Length).TrimStart('\', '/') + Length = [System.IO.FileInfo]::new($file).Length + }) + } + } + @($entries | Sort-Object -Property Path) +} + +function ConvertTo-DscSourceFingerprint +{ + [OutputType([System.String])] + param ( + [Parameter(Mandatory)] + [AllowEmptyCollection()] + [System.Object[]] + $Entry + ) + + $total = [long]0 + $builder = New-Object -TypeName System.Text.StringBuilder + foreach ($item in $Entry) + { + $total += [long]$item.Length + $null = $builder.Append($item.Path.ToLowerInvariant()).Append('|').Append([System.String]$item.Length).Append("`n") + } + $sha = [System.Security.Cryptography.SHA256]::Create() + try + { + $hash = $sha.ComputeHash([System.Text.Encoding]::UTF8.GetBytes($builder.ToString())) + } + finally + { + $sha.Dispose() + } + $hex = [System.BitConverter]::ToString($hash, 0, 8).Replace('-', '').ToLowerInvariant() + '{0}:{1}:{2}' -f $Entry.Count, $total, $hex +} + +function Read-DscSchemaCacheFile +{ + param ( + [Parameter(Mandatory)] + [System.String] + $Path + ) + + $lines = [System.IO.File]::ReadAllLines($Path) + if ($lines.Count -eq 0) + { + throw "Schema cache '$Path' is empty." + } + $header = ConvertFrom-Json -InputObject $lines[0] + if ($null -eq $header.formatVersion -or $null -eq $header.module) + { + throw "Schema cache '$Path' has no header." + } + if ($header.formatVersion -eq $script:SchemaCacheFormatVersion -and $lines.Count -lt 2) + { + throw "Schema cache '$Path' is truncated." + } + [PSCustomObject]@{ + Path = $Path + formatVersion = $header.formatVersion + generator = $header.generator + module = $header.module + index = $header.index + keywordCount = $header.keywordCount + resourceCount = $header.resourceCount + Lines = $lines + } +} + +function Get-DscSchemaCacheSourceList +{ + param ( + [Parameter(Mandatory)] + $Cache + ) + + ConvertFrom-Json -InputObject $Cache.Lines[1] +} + +function Get-DscSchemaCacheKeyword +{ + [OutputType([System.Management.Automation.Language.DynamicKeyword])] + param ( + [Parameter(Mandatory)] + $Cache, + + [Parameter(Mandatory)] + [System.String] + $Name + ) + + $entry = $Cache.index.PSObject.Properties[$Name] + if ($null -eq $entry) + { + return $null + } + ConvertFrom-DscKeywordSchemaObject -SchemaObject (ConvertFrom-Json -InputObject $Cache.Lines[[int]$entry.Value]) +} + +function Get-DscSchemaCacheKeywordName +{ + [OutputType([string[]])] + param ( + [Parameter(Mandatory)] + $Cache + ) + + @($Cache.index.PSObject.Properties | ForEach-Object { $_.Name }) +} + function Export-DscSchemaCache { [CmdletBinding()] param ( [Parameter(Mandatory, ParameterSetName = 'ByName', Position = 0)] - [string] + [System.String] $ModuleName, [Parameter(ParameterSetName = 'ByName')] - [version] + [System.Version] $RequiredVersion, [Parameter(Mandatory, ParameterSetName = 'ByModuleInfo')] [System.Management.Automation.PSModuleInfo] $Module, - [string] + [System.String] $OutputPath ) @@ -138,7 +266,7 @@ function Export-DscSchemaCache $candidates = Get-Module -ListAvailable -Name $ModuleName | Sort-Object -Property Version -Descending if ($RequiredVersion) { - $candidates = $candidates | Where-Object { $_.Version -eq $RequiredVersion } + $candidates = $candidates | Where-Object -Property Version -EQ $RequiredVersion } $Module = $candidates | Select-Object -First 1 if (-not $Module) @@ -148,40 +276,74 @@ function Export-DscSchemaCache } Reset-DscKeywordState - $defaultFunctions = New-Object -TypeName 'System.Collections.Generic.Dictionary[string,scriptblock]' -ArgumentList ([System.StringComparer]::OrdinalIgnoreCase) + $defaultFunctions = New-Object -TypeName 'System.Collections.Generic.Dictionary[System.String, ScriptBlock]' -ArgumentList ([System.StringComparer]::OrdinalIgnoreCase) [Microsoft.PowerShell.DesiredStateConfiguration.Internal.DscClassCache]::LoadDefaultCimKeywords($defaultFunctions) - $defaultKeywordNames = New-Object -TypeName 'System.Collections.Generic.HashSet[string]' -ArgumentList ([System.StringComparer]::OrdinalIgnoreCase) + $defaultKeywordNames = New-Object -TypeName 'System.Collections.Generic.HashSet[System.String]' -ArgumentList ([System.StringComparer]::OrdinalIgnoreCase) foreach ($keyword in [Microsoft.PowerShell.DesiredStateConfiguration.Internal.DscClassCache]::GetCachedKeywords()) { $null = $defaultKeywordNames.Add($keyword.Keyword) } - $resources = New-Object -TypeName 'System.Collections.Generic.List[string]' + $resources = New-Object -TypeName 'System.Collections.Generic.List[System.String]' $resources.Add('*') - $throwaway = New-Object -TypeName 'System.Collections.Generic.Dictionary[string,scriptblock]' -ArgumentList ([System.StringComparer]::OrdinalIgnoreCase) + $throwaway = New-Object -TypeName 'System.Collections.Generic.Dictionary[System.String, ScriptBlock]' -ArgumentList ([System.StringComparer]::OrdinalIgnoreCase) $null = [Microsoft.PowerShell.DesiredStateConfiguration.Internal.DscClassCache]::ImportClassResourcesFromModule($Module, $resources, $throwaway) - $keywords = @() + $dscResourcesPath = Join-Path $Module.ModuleBase 'DscResources' + if (Test-Path $dscResourcesPath) + { + foreach ($resourceDirectory in [System.IO.Directory]::EnumerateDirectories($dscResourcesPath)) + { + $resourceName = Split-Path $resourceDirectory -Leaf + if (-not (Test-Path (Join-Path $resourceDirectory "$resourceName.schema.mof"))) + { + continue + } + $schemaFilePath = $null + $keywordErrors = New-Object -TypeName 'System.Collections.ObjectModel.Collection[System.Exception]' + $null = [Microsoft.PowerShell.DesiredStateConfiguration.Internal.DscClassCache]::ImportCimKeywordsFromModule( + $Module, $resourceName, [ref] $schemaFilePath, $throwaway, $keywordErrors) + foreach ($keywordError in $keywordErrors) + { + Write-Warning -Message "Schema of resource '$resourceName' was not imported: $($keywordError.Message)" + } + } + } + + $keywords = New-Object -TypeName 'System.Collections.Generic.List[System.Object]' foreach ($keyword in [Microsoft.PowerShell.DesiredStateConfiguration.Internal.DscClassCache]::GetCachedKeywords()) { if (-not $defaultKeywordNames.Contains($keyword.Keyword)) { - $keywords += ConvertTo-DscKeywordSchemaObject -Keyword $keyword + $keywords.Add((ConvertTo-DscKeywordSchemaObject -Keyword $keyword)) } } - $fingerprint = Get-DscModuleFingerprint -Module $Module - $sourceHash = @{} - foreach ($pattern in '*.psm1', '*.psd1', '*.mof') + Reset-DscKeywordState + + $entries = Get-DscModuleSourceEntry -ModuleBase $Module.ModuleBase + $fingerprint = ConvertTo-DscSourceFingerprint -Entry $entries + $sources = [ordered]@{} + foreach ($item in $entries) + { + $sources[$item.Path] = @{ + length = $item.Length + sha256 = (Get-FileHash -Path (Join-Path $Module.ModuleBase $item.Path) -Algorithm SHA256).Hash + } + } + + $index = [ordered]@{} + $resourceCount = 0 + for ($i = 0; $i -lt $keywords.Count; $i++) { - foreach ($file in [System.IO.Directory]::EnumerateFiles($Module.ModuleBase, $pattern, [System.IO.SearchOption]::AllDirectories)) + $index[[System.String]$keywords[$i].keyword] = $i + 2 + if ($keywords[$i].nameMode -eq 'NameRequired') { - $relative = $file.Substring($Module.ModuleBase.Length).TrimStart('\', '/') - $sourceHash[$relative] = (Get-FileHash -Path $file -Algorithm SHA256).Hash + $resourceCount++ } } - $cache = @{ + $header = [ordered]@{ formatVersion = $script:SchemaCacheFormatVersion generator = @{ psdscVersion = $ExecutionContext.SessionState.Module.Version.ToString() @@ -191,13 +353,12 @@ function Export-DscSchemaCache name = $Module.Name version = $Module.Version.ToString() fingerprint = $fingerprint - sourceHash = $sourceHash } - keywords = $keywords + resourceCount = $resourceCount + keywordCount = $keywords.Count + index = $index } - Reset-DscKeywordState - if (-not $OutputPath) { $OutputPath = Join-Path $Module.ModuleBase 'DscSchemaCache.json' @@ -207,14 +368,21 @@ function Export-DscSchemaCache { $null = New-Item -ItemType Directory -Force -Path $directory } - $json = ConvertTo-Json -InputObject $cache -Depth 12 -Compress - [System.IO.File]::WriteAllText($OutputPath, $json, [System.Text.UTF8Encoding]::new($false)) + + $lines = New-Object -TypeName 'System.Collections.Generic.List[System.String]' + $lines.Add((ConvertTo-Json -InputObject $header -Depth 6 -Compress)) + $lines.Add((ConvertTo-Json -InputObject $sources -Depth 4 -Compress)) + foreach ($keyword in $keywords) + { + $lines.Add((ConvertTo-Json -InputObject $keyword -Depth 12 -Compress)) + } + [System.IO.File]::WriteAllLines($OutputPath, $lines, [System.Text.UTF8Encoding]::new($false)) Write-Verbose -Message "Schema cache for $($Module.Name) $($Module.Version) written to $OutputPath ($($keywords.Count) keywords)." [PSCustomObject]@{ ModuleName = $Module.Name ModuleVersion = $Module.Version - ResourceCount = @($keywords | Where-Object { $_.nameMode -eq 'NameRequired' }).Count + ResourceCount = $resourceCount KeywordCount = $keywords.Count Fingerprint = $fingerprint Path = $OutputPath @@ -228,11 +396,11 @@ function Test-DscSchemaCache [OutputType([bool])] param ( [Parameter(Mandatory)] - [string] + [System.String] $ModulePath, [Parameter(Mandatory)] - [string] + [System.String] $CachePath, [switch] @@ -248,13 +416,21 @@ function Test-DscSchemaCache $manifestPath = if ([System.IO.Path]::GetExtension($ModulePath) -eq '.psd1') { $ModulePath } else { Get-ChildItem -Path $ModulePath -Filter '*.psd1' | Select-Object -First 1 -ExpandProperty FullName } $moduleBase = Split-Path $manifestPath -Parent $manifest = Import-PowerShellDataFile -Path $manifestPath - $cache = ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllText($CachePath)) + try + { + $cache = Read-DscSchemaCacheFile -Path $CachePath + } + catch + { + Write-Warning -Message "Schema cache '$CachePath' could not be read: $($_.Exception.Message)" + return $false + } $valid = $true - if ($cache.formatVersion -gt $script:SchemaCacheFormatVersion) + if ($cache.formatVersion -ne $script:SchemaCacheFormatVersion) { - Write-Warning -Message "Cache format version $($cache.formatVersion) is newer than supported ($script:SchemaCacheFormatVersion)." - $valid = $false + Write-Warning -Message "Cache format version $($cache.formatVersion) is not the supported version $script:SchemaCacheFormatVersion." + return $false } if ($cache.module.version -ne $manifest.ModuleVersion) { @@ -262,7 +438,8 @@ function Test-DscSchemaCache $valid = $false } - foreach ($entry in $cache.module.sourceHash.PSObject.Properties) + $sources = Get-DscSchemaCacheSourceList -Cache $cache + foreach ($entry in $sources.PSObject.Properties) { $file = Join-Path $moduleBase $entry.Name if (-not (Test-Path $file)) @@ -270,7 +447,7 @@ function Test-DscSchemaCache Write-Warning -Message "File in cache no longer exists: $($entry.Name)" $valid = $false } - elseif ($Detailed -and (Get-FileHash -Path $file -Algorithm SHA256).Hash -ne $entry.Value) + elseif ($Detailed -and (Get-FileHash -Path $file -Algorithm SHA256).Hash -ne $entry.Value.sha256) { Write-Warning -Message "File changed since cache generation: $($entry.Name)" $valid = $false @@ -281,52 +458,42 @@ function Test-DscSchemaCache } Export-ModuleMember -Function Test-DscSchemaCache +# A file that appears next to the module later must not invalidate the cache. Only the +# recorded files count. function Get-DscSchemaCacheSourceFingerprint { - [OutputType([string])] + [OutputType([System.String])] param ( [Parameter(Mandatory)] - [string] + [System.String] $ModuleBase, [Parameter(Mandatory)] $Cache ) - if ($null -eq $Cache.module.sourceHash) - { - return $null - } - - $entries = @($Cache.module.sourceHash.PSObject.Properties) - if ($entries.Count -eq 0) - { - return $null - } - - $maxTicks = [long]0 - foreach ($entry in $entries) + $sources = Get-DscSchemaCacheSourceList -Cache $Cache + $entries = New-Object -TypeName 'System.Collections.Generic.List[System.Object]' + foreach ($entry in $sources.PSObject.Properties) { $file = Join-Path $ModuleBase $entry.Name if (-not [System.IO.File]::Exists($file)) { return $null } - $ticks = [System.IO.File]::GetLastWriteTimeUtc($file).Ticks - if ($ticks -gt $maxTicks) - { - $maxTicks = $ticks - } + $entries.Add([PSCustomObject]@{ Path = $entry.Name; Length = [System.IO.FileInfo]::new($file).Length }) } - - "$($entries.Count):$maxTicks" + if ($entries.Count -eq 0) + { + return $null + } + ConvertTo-DscSourceFingerprint -Entry @($entries | Sort-Object -Property Path) } function Get-DscSchemaCache { param ( [Parameter(Mandatory)] - [System.Management.Automation.PSModuleInfo] $Module, [string[]] @@ -350,16 +517,16 @@ function Get-DscSchemaCache } try { - $cache = ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllText($candidate)) + $cache = Read-DscSchemaCacheFile -Path $candidate } catch { - Write-Warning -Message "Schema cache '$candidate' could not be parsed: $($_.Exception.Message)" + Write-Warning -Message "Schema cache '$candidate' could not be read: $($_.Exception.Message)" continue } - if ($cache.formatVersion -gt $script:SchemaCacheFormatVersion) + if ($cache.formatVersion -ne $script:SchemaCacheFormatVersion) { - Write-Warning -Message "Schema cache '$candidate' has unsupported format version $($cache.formatVersion)." + Write-Verbose -Message "Schema cache '$candidate' has format version $($cache.formatVersion), expected $script:SchemaCacheFormatVersion." continue } if ($cache.module.name -ne $Module.Name -or $cache.module.version -ne $Module.Version.ToString()) @@ -367,15 +534,14 @@ function Get-DscSchemaCache Write-Warning -Message "Schema cache '$candidate' is for $($cache.module.name) $($cache.module.version), not $($Module.Name) $($Module.Version)." continue } - $recorded = Get-DscSchemaCacheSourceFingerprint -ModuleBase $Module.ModuleBase -Cache $cache - if ($null -eq $recorded) + if ($cache.module.fingerprint -ne $fingerprint) { - $recorded = $fingerprint - } - if ($cache.module.fingerprint -ne $recorded) - { - Write-Verbose -Message "Schema cache '$candidate' is stale (module files changed)." - continue + $recorded = Get-DscSchemaCacheSourceFingerprint -ModuleBase $Module.ModuleBase -Cache $cache + if ($cache.module.fingerprint -ne $recorded) + { + Write-Verbose -Message "Schema cache '$candidate' is stale (module files changed)." + continue + } } return $cache } @@ -387,13 +553,21 @@ function New-DscSchemaCacheForModule { param ( [Parameter(Mandatory)] - [System.Management.Automation.PSModuleInfo] $Module ) - $fingerprint = Get-DscModuleFingerprint -Module $Module - $userPath = Get-DscSchemaCacheUserPath -ModuleName $Module.Name -ModuleVersion $Module.Version -Fingerprint $fingerprint - Write-Verbose -Message "Generating schema cache for $($Module.Name) $($Module.Version) (one-time operation)." - $null = Export-DscSchemaCache -Module $Module -OutputPath $userPath - Get-DscSchemaCache -Module $Module -SchemaCachePath $userPath + $moduleInfo = $Module + if ($Module -isnot [System.Management.Automation.PSModuleInfo]) + { + $moduleInfo = Get-Module -ListAvailable -Name $Module.Path | Select-Object -First 1 + if ($null -eq $moduleInfo) + { + throw "Module manifest '$($Module.Path)' could not be loaded for schema discovery." + } + } + $fingerprint = Get-DscModuleFingerprint -Module $moduleInfo + $userPath = Get-DscSchemaCacheUserPath -ModuleName $moduleInfo.Name -ModuleVersion $moduleInfo.Version -Fingerprint $fingerprint + Write-Verbose -Message "Generating schema cache for $($moduleInfo.Name) $($moduleInfo.Version) (one-time operation)." + $null = Export-DscSchemaCache -Module $moduleInfo -OutputPath $userPath + Get-DscSchemaCache -Module $moduleInfo -SchemaCachePath $userPath } diff --git a/README.md b/README.md index 021fe44..ff02269 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ The fork is rebased onto the upstream v2.0.7 pure-script lineage, with a couple | Resource type | Standard path, Windows PowerShell 5.1 | Standard path, PowerShell 7 | Fast host | | --- | --- | --- | --- | | Class-based | Supported | Supported | Supported | -| Script-based (`*.schema.mof`) | Supported | Supported | Automatic fallback to the standard path | +| Script-based (`*.schema.mof`) | Supported | Supported | Supported | | Composite (`*.schema.psm1`) | Supported | Supported | Automatic fallback to the standard path | ## Quick start @@ -86,7 +86,7 @@ fast host work, with diagrams, is in [docs/Architecture.md](docs/Architecture.md 2. `%LOCALAPPDATA%\M365DSC.PSDesiredStateConfiguration\SchemaCache\__.json` - written after a live generation. 3. Live generation (then persisted to 2). -A cache is used only when its module name, version, and fingerprint match the resolved module. +A cache is used only when its format version, module name, version, and fingerprint match the resolved module. The fingerprint is built from file names and sizes, so an installed copy keeps the cache shipped with the package. ## Coexistence with other PSDesiredStateConfiguration versions diff --git a/docs/Architecture.md b/docs/Architecture.md index 1ef3e35..048d7ca 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -317,18 +317,17 @@ sequenceDiagram D-->>Caller: .mof files ``` -Before any of that can run, the text has to be rewritten four times, and every rewrite is steered by -an AST taken from a masked copy rather than from the text itself. +Before any of that can run, the text has to be rewritten, and every rewrite is steered by an AST +taken from a masked copy rather than from the text itself. One parse serves every edit, and +`ConvertTo-FastHostCompileText` applies all of them to the original text in a single pass. ```mermaid flowchart LR src["original text"] --> mask["mask the word Configuration
in a copy"] - mask --> parse["Parser.ParseInput
about 8 ms"] - parse --> collect["collect configuration names
and Import-DscResource ASTs"] - collect --> strip["remove import extents
from the original text"] - strip --> merge["Merge-FastHostResourceStatements"] - merge --> hash["Convert-FastHostBodyToHashtable"] - hash --> sb["scriptblock::Create"] + mask --> parse["Parser.ParseInput
once"] + parse --> collect["collect configuration names,
Import-DscResource ASTs,
next-line braces, keyword bodies"] + collect --> apply["ConvertTo-FastHostCompileText
one pass over the text"] + apply --> sb["scriptblock::Create"] ``` ### Masking instead of parsing @@ -390,8 +389,10 @@ argument happens to be a hashtable literal. ### Adapters -`Register-DscCachedKeywords` rehydrates every cache entry into a real `DynamicKeyword` object and -stores it in a case insensitive dictionary. Each keyword gets the same shared adapter scriptblock, +`Register-DscSchemaCache` keeps the cache as text lines plus the keyword index and registers the +adapters for every name in the index. `Get-FastHostKeyword` deserializes a keyword line into a real +`DynamicKeyword` the first time a compile asks for it and keeps it in a case insensitive dictionary, +so a compile pays for the keywords it uses and not for the whole module. Each keyword gets the same shared adapter scriptblock, registered twice, under the bare name and under the module qualified name, because a configuration body may call it either way. When the fast host is active the `Configuration` function merges those adapters plus four accessors (`Get-FastHostKeyword`, `Get-FastHostBodyScriptBlock`, @@ -425,7 +426,7 @@ reason. | `Import-DscResource` with non constant arguments | The module set cannot be determined without executing the script | | `-Name` without `-ModuleName` | Would require scanning every installed module | | The named module is not installed, or not in the requested version | Nothing to resolve a cache against | -| The module contains `*.schema.mof` or `*.schema.psm1` under `DscResources` | Script based and composite resources need the engine's own import | +| The module contains `*.schema.psm1` under `DscResources` | Composite resources need the engine's own import | | No usable schema cache and none can be generated | Nothing to register keywords from | `-NoFallback` turns each of these into a terminating error. Continuous integration should use it, @@ -438,8 +439,9 @@ passing job. `Export-DscSchemaCache` resets the keyword state, loads the default keywords and remembers their names, imports the module's class based resources once through -`DscClassCache::ImportClassResourcesFromModule`, and serializes every cached keyword that was not -already a default. It resets the keyword state afterwards as well, so generating a cache does not +`DscClassCache::ImportClassResourcesFromModule`, reads every `DscResources\\.schema.mof` +through `ImportCimKeywordsFromModule`, and serializes every cached keyword that was not already a +default. It resets the keyword state afterwards as well, so generating a cache does not leave the session holding the module's classes. Default output is `\DscSchemaCache.json`. The summary it returns carries the module name @@ -459,7 +461,7 @@ flowchart TB c -->|no| gen["Export-DscSchemaCache
one time, about 6 s"] gen --> persist["persist to the user cache"] --> use subgraph valid["valid means"] - v1["formatVersion not newer than known"] + v1["formatVersion is the known one"] v2["module name and version match"] v3["fingerprint matches"] end @@ -468,16 +470,20 @@ flowchart TB The user cache lives at `%LOCALAPPDATA%\M365DSC.PSDesiredStateConfiguration\SchemaCache\__.json`. -A cache has to earn its place before any of its keywords are used. Its format version may not be -newer than the reader knows, its module name and version have to match, and so does its fingerprint. -That fingerprint is `fileCount:maxLastWriteTimeUtcTicks` over `*.psm1`, `*.psd1` and `*.mof` below the -module base, which lets any added, removed or rewritten file invalidate the cache without hashing a -single byte at compile time. `Test-DscSchemaCache` checks the format and module version too, and with -`-Detailed` it re-hashes every file recorded in `sourceHash`, which is the accurate but slow variant -that belongs in a CI drift gate. - -Registration is remembered per module and version for the life of the session, so a second compile in -the same process reuses the keyword table and touches no JSON. +A cache has to earn its place before any of its keywords are used. Its format version has to be the +one the reader knows, its module name and version have to match, and so does its fingerprint. That +fingerprint is `fileCount:totalBytes:hash` over the sorted relative paths and sizes of every `*.psm1`, +`*.psd1` and `*.mof` file below the module base. It carries no write times, which is what lets the +cache shipped inside a package stay valid after `Install-Module` copied the files. When the file set +grew, the fingerprint is recomputed over the files the cache recorded, so an unrelated file placed +next to the module does not invalidate it. A file edited to the same size slips through, which is why +the build regenerates the cache and why `Test-DscSchemaCache -Detailed` re-hashes every recorded file +in a CI drift gate. + +The module named by `Import-DscResource` is located with a scan of `PSModulePath` for the manifest, +not with `Get-Module -ListAvailable`, which analyzes every nested module and costs seconds for a +large resource module. Registration is remembered per module and version for the life of the session, +so a second compile in the same process reuses the keyword table and touches no JSON. ### Format @@ -658,8 +664,8 @@ what makes the numbers in this document worth quoting. **The cache is the contract on the fast path.** A property that exists in the resource source but not in the cache does not exist as far as compilation is concerned. The fingerprint check is the only -thing keeping the two honest, and a cache has to be regenerated after any step that adds or trims -files in the module tree, because those files feed the fingerprint. +thing keeping the two honest, and a cache has to be regenerated after any step that adds, trims or +resizes files in the module tree, because those files feed the fingerprint. **Never call `LoadDefaultCimKeywords` to warm a cache.** It belongs in exactly one place, the top level compile that populates `functionsToDefine`. Calling it to inspect keyword state clears diff --git a/docs/FastHostContract.md b/docs/FastHostContract.md index 7dec849..5d1112b 100644 --- a/docs/FastHostContract.md +++ b/docs/FastHostContract.md @@ -29,7 +29,7 @@ Invoke-DscFastCompile [-ConfigurationData ] [-OutputPath ] [-SchemaCachePath ] # explicit cache files; otherwise resolution order below - [-Force] # regenerate a stale cache instead of treating it as missing + [-Force] # ignore every cache on disk, generate a fresh one and use it [-ValidateMof] # opt-in MI validation (requires live CIM classes, slower) [-NoFallback] # fail instead of degrading to the standard path ``` @@ -38,7 +38,17 @@ Returns the generated `.mof` files (`FileInfo[]`). Scripts that invoke their own While the script executes, `$Global:PSDscFastCompileActive` is `$true`. Generated export trailers use this as a recursion guard. -Falls back to standard compilation (with a warning) when: an `Import-DscResource` uses non-constant arguments or `-Name` without `-ModuleName`; a module contains script-based (`*.schema.mof`) or composite (`*.schema.psm1`) resources; or no usable schema cache can be obtained. +Modules named by `Import-DscResource` are located by scanning `$env:PSModulePath` for `\.psd1` and `\\.psd1` and reading `ModuleVersion` from the manifest. `Get-Module -ListAvailable` runs only when a cache has to be generated. + +Falls back to standard compilation (with a warning) when: an `Import-DscResource` uses non-constant arguments or `-Name` without `-ModuleName`; a module contains composite (`*.schema.psm1`) resources; or no usable schema cache can be obtained. Class-based resources and schema-based (`*.schema.mof`) resources compile on the fast path. + +### Get-DscFastCompileTiming + +```powershell +Get-DscFastCompileTiming +``` + +Returns an ordered dictionary with the milliseconds of the last `Invoke-DscFastCompile` in this session: `parse`, `resolve`, `cache`, `rewrite`, `compile` and `total`. `$null` before the first compile. ### Export-DscSchemaCache @@ -47,7 +57,7 @@ Export-DscSchemaCache -ModuleName [-RequiredVersion ] [-Output Export-DscSchemaCache -Module [-OutputPath ] ``` -Discovers all class-based resources of a module (one-time engine import, ~6 s for Microsoft365DSC) and writes the schema cache JSON. Default output: `\DscSchemaCache.json`. Returns a summary (`ModuleName`, `ModuleVersion`, `ResourceCount`, `KeywordCount`, `Fingerprint`, `Path`). +Discovers the class-based resources of a module through the engine (one-time import, ~6 s for Microsoft365DSC), reads every `DscResources\\.schema.mof`, and writes the schema cache. Default output: `\DscSchemaCache.json`. Returns a summary (`ModuleName`, `ModuleVersion`, `ResourceCount`, `KeywordCount`, `Fingerprint`, `Path`). ### Test-DscSchemaCache @@ -55,54 +65,68 @@ Discovers all class-based resources of a module (one-time engine import, ~6 s fo Test-DscSchemaCache -ModulePath -CachePath [-Detailed] ``` -Returns `$true`/`$false`. Validates format version and module version; `-Detailed` re-hashes every file recorded in `sourceHash` (CI drift gate). +Returns `$true`/`$false`. Validates format version and module version and that every recorded source file still exists; `-Detailed` re-hashes every recorded file (CI drift gate). ## Schema cache resolution order -1. `\DscSchemaCache.json` - shipped inside the resource module package. -2. `%LOCALAPPDATA%\M365DSC.PSDesiredStateConfiguration\SchemaCache\__.json` - written after a live generation. -3. Live generation (then persisted to 2). +1. Paths given through `-SchemaCachePath`. +2. `\DscSchemaCache.json` - shipped inside the resource module package. +3. `%LOCALAPPDATA%\M365DSC.PSDesiredStateConfiguration\SchemaCache\__.json` - written after a live generation. +4. Live generation (then persisted to 3). + +A cache is used only when its format version is the one the reader knows, its module name and version match the resolved module, and its fingerprint matches. The fingerprint is `fileCount:totalBytes:hash` over the sorted list of relative path and size of every `*.psm1`, `*.psd1` and `*.mof` file below the module base. It carries no write times, so a copy or an installation of the same files keeps its fingerprint. When the file set below the module base grew, the fingerprint is recomputed over the files the cache recorded, so an unrelated file placed next to the module does not invalidate it. + +## Cache format (formatVersion 2) + +The file is UTF-8 without BOM, one JSON document per line. + +Line 1, the header: -A cache is used only when its module name, version, and fingerprint (`fileCount:maxLastWriteTimeUtcTicks` over `*.psm1|*.psd1|*.mof`) match the resolved module. +```json +{ + "formatVersion": 2, + "generator": { "psdscVersion": "3.1.7", "psVersion": "7.6.5" }, + "module": { "name": "Microsoft365DSC", "version": "1.26.1007.1", "fingerprint": "595:10345678:bc7d0588069ae1cb" }, + "resourceCount": 531, + "keywordCount": 998, + "index": { "AADGroup": 2, "MSFT_AADGroupMember": 3 } +} +``` + +`index` maps every keyword name to its zero-based line number. + +Line 2, the recorded source files: + +```json +{ "Classes\\Part00.psm1": { "length": 713777, "sha256": "" } } +``` -## Cache format (formatVersion 1) +Every further line is one keyword: ```json { - "formatVersion": 1, - "generator": { "psdscVersion": "3.1.0", "psVersion": "7.4.0" }, - "module": { - "name": "Microsoft365DSC", - "version": "1.26.805.2", - "fingerprint": "594:639219468181493516", - "sourceHash": { "Classes\\Part00.psm1": "", "...": "..." } - }, - "keywords": [ - { - "keyword": "AADGroup", - "resourceName": "AADGroup", - "implementingModule": "Microsoft365DSC", - "implementingModuleVersion": "1.26.805.2", - "nameMode": "NameRequired", - "bodyMode": "Hashtable", - "directCall": false, - "metaStatement": false, - "properties": { - "DisplayName": { - "name": "DisplayName", "typeConstraint": "String", - "mandatory": true, "isKey": true, - "attributes": [], "values": [], - "valueMap": [ { "key": "...", "value": "..." } ] - } - } + "keyword": "AADGroup", + "resourceName": "AADGroup", + "implementingModule": "Microsoft365DSC", + "implementingModuleVersion": "1.26.1007.1", + "nameMode": "NameRequired", + "bodyMode": "Hashtable", + "directCall": false, + "metaStatement": false, + "properties": { + "DisplayName": { + "name": "DisplayName", "typeConstraint": "String", + "mandatory": true, "isKey": true, + "attributes": [], "values": [], + "valueMap": [ { "key": "...", "value": "..." } ] } - ] + } } ``` -Embedded complex types are ordinary `NoName` keywords in the same flat list. `valueMap` is an array of key/value pairs because DSC allows empty-string map keys, which JSON object properties cannot represent portably. +Embedded complex types are ordinary `NoName` keywords. `valueMap` is an array of key/value pairs because DSC allows empty-string map keys, which JSON object properties cannot represent portably. -Consumers must reject caches with a `formatVersion` greater than the one they know and regenerate. +The fast host reads the header at registration time and deserializes a keyword line the first time a compile asks for that keyword. Consumers must reject caches whose `formatVersion` differs from the one they know and regenerate. ## Module resolution requirement diff --git a/test/EngineCoverage.Tests.ps1 b/test/EngineCoverage.Tests.ps1 index 38e0b23..2809623 100644 --- a/test/EngineCoverage.Tests.ps1 +++ b/test/EngineCoverage.Tests.ps1 @@ -5,7 +5,7 @@ BeforeAll { function Invoke-PSDscStringifyValue { - param ($Value, [bool] $AsArray = $false, [type] $TargetType = [string]) + param ($Value, [bool] $AsArray = $false, [type] $TargetType = [System.String]) # stringify reads the alias table of the instance being rendered out of its caller # scope, which is ConvertTo-MOFInstance during a real compilation. @@ -169,10 +169,10 @@ Describe 'Get-MofInstanceText' { Describe 'Get-ComplexResourceQualifier' { It 'joins the qualifier chain including the current node' { $qualifier = Invoke-PSDscInEngineScope { - # The engine keeps the enclosing configuration names in a List[string] that + # The engine keeps the enclosing configuration names in a List[System.String] that # survives between configurations, so fill it in place and empty it again. $script:ConfigurationNestingStack.Clear() - $script:ConfigurationNestingStack.AddRange([string[]]@('a', 'b', 'c')) + $script:ConfigurationNestingStack.AddRange([System.String[]]@('a', 'b', 'c')) $qualifier = Get-ComplexResourceQualifier -IncludeCurrent $script:ConfigurationNestingStack.Clear() $qualifier @@ -184,7 +184,7 @@ Describe 'Get-ComplexResourceQualifier' { It 'joins the qualifier chain excluding the current node' { $qualifier = Invoke-PSDscInEngineScope { $script:ConfigurationNestingStack.Clear() - $script:ConfigurationNestingStack.AddRange([string[]]@('a', 'b', 'c')) + $script:ConfigurationNestingStack.AddRange([System.String[]]@('a', 'b', 'c')) $qualifier = Get-ComplexResourceQualifier $script:ConfigurationNestingStack.Clear() $qualifier @@ -512,7 +512,7 @@ configuration BadOutputCfg $null = New-Item -ItemType Directory -Path (Join-Path $moduleRoot 'DscResources\MSFT_LogResource') -Force Set-Content -Path (Join-Path $moduleRoot 'LogResModule.psd1') -Value '@{ RootModule = ''LogResModule.psm1''; ModuleVersion = ''1.0'' }' Set-Content -Path (Join-Path $moduleRoot 'LogResModule.psm1') -Value '' - Set-Content -Path (Join-Path $moduleRoot 'DscResources\MSFT_LogResource\MSFT_LogResource.schema.mof') -Value 'class MSFT_LogResource : OMI_BaseResource { [string] Message; };' + Set-Content -Path (Join-Path $moduleRoot 'DscResources\MSFT_LogResource\MSFT_LogResource.schema.mof') -Value 'class MSFT_LogResource : OMI_BaseResource { [System.String] Message; };' Import-Module -Name (Join-Path $moduleRoot 'LogResModule.psd1') -Force try @@ -708,11 +708,12 @@ Describe 'ValidateNodeResources' { Describe 'Coverage: CIM keyword implementation branches' { BeforeAll { function New-PSDscCimFakeCache { - param([string]$Destination) + param([System.String]$Destination) $module = Get-Module -ListAvailable -Name xTestClassResource | Sort-Object Version -Descending | Select-Object -First 1 $real = Join-Path $Destination 'real.json' Export-DscSchemaCache -Module $module -OutputPath $real | Out-Null - $cache = Get-Content -Raw $real | ConvertFrom-Json + $lines = [System.Collections.Generic.List[System.String]]::new([System.String[]] [System.IO.File]::ReadAllLines($real)) + $cache = [PSCustomObject]@{ Lines = $lines; Header = (ConvertFrom-Json -InputObject $lines[0]) } function Add-PSDscFakeKeyword($cache, $keyword, $resourceName, $nameMode, $props) { $properties = @{} foreach ($p in $props) { @@ -726,7 +727,7 @@ Describe 'Coverage: CIM keyword implementation branches' { valueMap = @($p.valueMap) } } - $cache.keywords += [PSCustomObject]@{ + $entry = [PSCustomObject]@{ keyword = $keyword resourceName = $resourceName implementingModule = $null @@ -737,6 +738,8 @@ Describe 'Coverage: CIM keyword implementation branches' { metaStatement = $false properties = $properties } + $cache.Lines.Add((ConvertTo-Json -InputObject $entry -Depth 12 -Compress)) + Add-Member -InputObject $cache.Header.index -MemberType NoteProperty -Name $keyword -Value ($cache.Lines.Count - 1) } Add-PSDscFakeKeyword $cache 'FakePlainKeyword' 'FakePlainClass' 'SimpleOptionalName' @( @{ name = 'NoDependsOn'; typeConstraint = 'string'; mandatory = $false; isKey = $false; values = @(); valueMap = @() } @@ -746,7 +749,9 @@ Describe 'Coverage: CIM keyword implementation branches' { @{ name = 'State'; typeConstraint = 'string'; mandatory = $false; isKey = $false; values = @('On', 'Off'); valueMap = @(@{ key = 'On'; value = 'Enabled' }, @{ key = 'Off'; value = 'Disabled' }) } ) $fake = Join-Path $Destination 'fake.json' - $cache | ConvertTo-Json -Depth 12 | Set-Content $fake + $cache.Header.keywordCount = $cache.Lines.Count - 2 + $cache.Lines[0] = ConvertTo-Json -InputObject $cache.Header -Depth 6 -Compress + [System.IO.File]::WriteAllLines($fake, $cache.Lines) $fake } } @@ -1056,13 +1061,13 @@ Describe 'Coverage: CIM keyword implementation via direct invocation' { BeforeAll { function Invoke-PSDscCimDirect { param( - [string]$Keyword, - [string]$ResourceName, - [string]$NameMode, - [string]$ImplementingModule, + [System.String]$Keyword, + [System.String]$ResourceName, + [System.String]$NameMode, + [System.String]$ImplementingModule, [hashtable]$Props, [hashtable]$Value, - [string]$Name = '', + [System.String]$Name = '', [hashtable]$Driver ) @@ -1076,7 +1081,7 @@ Describe 'Coverage: CIM keyword implementation via direct invocation' { $script:NodeResourceIdAliases[$kDriver.node] = [System.Collections.Generic.Dictionary[string,string]]::new([System.StringComparer]::OrdinalIgnoreCase) } $script:ConfigurationNestingStack.Clear() - foreach ($e in @($kDriver.nesting)) { $script:ConfigurationNestingStack.Add([string]$e) } + foreach ($e in @($kDriver.nesting)) { $script:ConfigurationNestingStack.Add([System.String]$e) } $script:FastHostKeywords = @{} $script:IsMetaConfig = [bool]$kDriver.isMetaConfig if ($kDriver.metaDocProcessed) { $script:PSMetaConfigurationProcessed = $true } @@ -1093,9 +1098,9 @@ Describe 'Coverage: CIM keyword implementation via direct invocation' { $p.TypeConstraint = $kProps[$pn].type $p.Mandatory = [bool]$kProps[$pn].mandatory $p.IsKey = [bool]$kProps[$pn].isKey - foreach ($v in @($kProps[$pn].values)) { $null = $p.Values.Add([string]$v) } + foreach ($v in @($kProps[$pn].values)) { $null = $p.Values.Add([System.String]$v) } if ($kProps[$pn].valueMap) { - foreach ($vk in $kProps[$pn].valueMap.Keys) { $p.ValueMap[[string]$vk] = [string]$kProps[$pn].valueMap[$vk] } + foreach ($vk in $kProps[$pn].valueMap.Keys) { $p.ValueMap[[System.String]$vk] = [System.String]$kProps[$pn].valueMap[$vk] } } if ($kProps[$pn].range) { $r = $kProps[$pn].range @@ -1287,8 +1292,8 @@ Describe 'Coverage: psm1 engine internals via direct invocation' { BeforeAll { function New-ProbeProperty { param( - [string] $Name, - [string] $TypeConstraint = 'string', + [System.String] $Name, + [System.String] $TypeConstraint = 'string', [bool] $IsKey = $false ) $p = New-Object System.Management.Automation.Language.DynamicKeywordProperty @@ -1301,7 +1306,7 @@ Describe 'Coverage: psm1 engine internals via direct invocation' { function New-ProbeKeyword { param( - [string] $Name, + [System.String] $Name, [hashtable] $PropertyTypes ) $k = New-Object System.Management.Automation.Language.DynamicKeyword @@ -1320,7 +1325,7 @@ Describe 'Coverage: psm1 engine internals via direct invocation' { param([hashtable] $Entries) $d = New-Object 'System.Collections.Generic.Dictionary[string,string]' ([System.StringComparer]::OrdinalIgnoreCase) foreach ($key in $Entries.Keys) { - $d[$key] = [string]$Entries[$key] + $d[$key] = [System.String]$Entries[$key] } $d } @@ -1458,9 +1463,9 @@ Describe 'Coverage: psm1 engine internals via direct invocation' { Invoke-PSDscInEngineScope { Initialize-ConfigurationRuntimeState $dict = New-Object 'System.Collections.Generic.Dictionary[string,string[]]' ([System.StringComparer]::OrdinalIgnoreCase) - $dict['r1'] = [string[]]@('foo') - $dict['barfoo'] = [string[]]@() - $dict['r2'] = [string[]]@('zork') + $dict['r1'] = [System.String[]]@('foo') + $dict['barfoo'] = [System.String[]]@() + $dict['r2'] = [System.String[]]@('zork') $script:NoNameNodesResources = $dict ValidateNoNameNodeResources $script:NoNameNodesResources = $null @@ -1480,9 +1485,9 @@ Describe 'Coverage: psm1 engine internals via direct invocation' { for ($i = 0; $i -lt 1026; $i++) { if ($i -lt 1025) { $dep = "r$($i + 1)" - $chain["r$i"] = [string[]]@($dep) + $chain["r$i"] = [System.String[]]@($dep) } else { - $chain["r$i"] = [string[]]@() + $chain["r$i"] = [System.String[]]@() } } $script:NodeResources = $chain @@ -1846,4 +1851,4 @@ configuration BadMkdirCfg $diag | Should -Match 'valid path segment' } } -} \ No newline at end of file +} diff --git a/test/FastHost.Tests.ps1 b/test/FastHost.Tests.ps1 index 05791fa..349e38f 100644 --- a/test/FastHost.Tests.ps1 +++ b/test/FastHost.Tests.ps1 @@ -487,7 +487,7 @@ SelfInvokeCfg -OutputPath '$outputDirectory' (Get-Content -Path $result.FullName -Raw) | Should -Match 'instance of ResourceForTests1' } - It 'falls back with a warning for a script-based resource module and still produces the MOF' { + It 'compiles a schema based resource module through the fast path' { $text = @' Configuration FallbackScriptCfg { @@ -504,10 +504,11 @@ Configuration FallbackScriptCfg } '@ $warnings = $null - $result = Invoke-DscFastCompile -ScriptText $text -OutputPath (Join-Path $TestDrive 'fallback-script') -WarningVariable warnings 3>$null - ($warnings -join ' ') | Should -Match 'Falling back to standard compilation' + $result = Invoke-DscFastCompile -ScriptText $text -OutputPath (Join-Path $TestDrive 'fallback-script') -NoFallback -WarningVariable warnings 3>$null + $warnings | Should -BeNullOrEmpty $result.Exists | Should -Be $true (Get-Content -Path $result.FullName -Raw) | Should -Match 'MSFT_xTestScriptResource' + (Get-Content -Path $result.FullName -Raw) | Should -Match 'Items = \{' } It 'falls back with a warning for a composite resource module and still produces the MOF' { @@ -971,3 +972,134 @@ Configuration FastMissingMandatoryCfg $diagnostics | Should -Match 'Value' } } + +Describe 'Fast host module resolution and rewrite' { + BeforeAll { + Reset-PSDscFastHostState + } + + AfterAll { + Reset-PSDscFastHostState + } + + It 'resolves an installed module from the manifest alone' { + $module = Invoke-PSDscInEngineScope { Resolve-FastHostModule -ModuleName 'xTestClassResource' } + + $module.Name | Should -Be 'xTestClassResource' + $module.Version | Should -Be ([version]'1.0') + $module.Path | Should -Be (Join-Path $module.ModuleBase 'xTestClassResource.psd1') + $module.Path | Should -Exist + } + + It 'returns nothing for a version that is not installed' { + Invoke-PSDscInEngineScope { Resolve-FastHostModule -ModuleName 'xTestClassResource' -ModuleVersion '99.0' } | Should -BeNullOrEmpty + } + + It 'prefers the highest version folder and honors a requested one' { + $root = Join-Path $TestDrive 'resolve-root' + foreach ($version in '1.0', '2.5') + { + $folder = Join-Path $root "xVersionedProbe\$version" + $null = New-Item -ItemType Directory -Path $folder -Force + New-ModuleManifest -Path (Join-Path $folder 'xVersionedProbe.psd1') -ModuleVersion $version + } + $originalPath = $env:PSModulePath + $env:PSModulePath = $root + [System.IO.Path]::PathSeparator + $env:PSModulePath + try + { + (Invoke-PSDscInEngineScope { Resolve-FastHostModule -ModuleName 'xVersionedProbe' }).Version | Should -Be ([version]'2.5') + (Invoke-PSDscInEngineScope { Resolve-FastHostModule -ModuleName 'xVersionedProbe' -ModuleVersion '1.0' }).Version | Should -Be ([version]'1.0') + } + finally + { + $env:PSModulePath = $originalPath + } + } + + It 'rewrites imports, next-line braces and bodies in one pass like the sequential passes' { + $text = @' +Configuration OnePassCfg +{ + Import-DscResource -ModuleName xTestClassResource + Node localhost + { + ResourceForTests1 a + { + Prop1 = 'x' + } + xTestClassResource b { Name = 'n'; Value = 'v'; Ensure = 'Present'; sArray = @('s') } + } +} +'@ + $input = @{ Text = $text; Names = @('ResourceForTests1', 'xTestClassResource') } + $onePass = Invoke-PSDscInEngineScope { + $strip = Get-StrippedConfigurationText -Text $args[0].Text + ConvertTo-FastHostCompileText -Text $args[0].Text -Ast $strip.Ast -ImportStatement $strip.ImportStatements -KeywordNames $args[0].Names -Merge -Convert + } $input + $sequential = Invoke-PSDscInEngineScope { + $strip = Get-StrippedConfigurationText -Text $args[0].Text + $merged = Merge-FastHostResourceStatements -Text $strip.Text -KeywordNames $args[0].Names + Convert-FastHostBodyToHashtable -Text $merged -KeywordNames $args[0].Names + } $input + + $onePass | Should -BeExactly $sequential + $onePass | Should -Not -Match 'Import-DscResource' + $onePass | Should -Match 'ResourceForTests1 a @\{' + $onePass | Should -Match 'xTestClassResource b @\{' + } + + It 'deserializes only the keywords a compile uses' { + Reset-PSDscFastHostState + $text = @' +Configuration LazyCfg +{ + Import-DscResource -ModuleName xTestClassResource + Node localhost + { + ResourceForTests1 a { Prop1 = 'lazy' } + } +} +'@ + $null = Invoke-DscFastCompile -ScriptText $text -OutputPath (Join-Path $TestDrive 'lazy-out') -NoFallback + + $materialized = @(Invoke-PSDscInEngineScope { @($script:FastHostKeywords.Keys) }) + $materialized | Should -Contain 'ResourceForTests1' + $materialized | Should -Not -Contain 'xTestClassResource' + (Invoke-PSDscInEngineScope { (Get-FastHostKeywordName).Count }) | Should -Be 5 + } + + It 'reports stage timings for the last compile' { + $timing = Get-DscFastCompileTiming + + @($timing.Keys) | Should -Be @('parse', 'resolve', 'cache', 'rewrite', 'compile', 'total') + $timing['total'] | Should -BeGreaterOrEqual $timing['compile'] + } + + It 'regenerates the cache with -Force' { + $module = Get-Module -ListAvailable -Name xTestClassResource | Select-Object -First 1 + $userPath = Invoke-PSDscInEngineScope { + Get-DscSchemaCacheUserPath -ModuleName $args[0].Name -ModuleVersion $args[0].Version -Fingerprint (Get-DscModuleFingerprint -Module $args[0]) + } $module + Remove-Item -Path $userPath -Force -ErrorAction Ignore + $text = @' +Configuration ForceCfg +{ + Import-DscResource -ModuleName xTestClassResource + Node localhost + { + ResourceForTests1 a { Prop1 = 'force' } + } +} +'@ + try + { + $null = Invoke-DscFastCompile -ScriptText $text -OutputPath (Join-Path $TestDrive 'force-out') -NoFallback -Force + + $userPath | Should -Exist + } + finally + { + Remove-Item -Path $userPath -Force -ErrorAction Ignore + } + } +} diff --git a/test/ModuleInternals.Tests.ps1 b/test/ModuleInternals.Tests.ps1 index b873a7a..b743b47 100644 --- a/test/ModuleInternals.Tests.ps1 +++ b/test/ModuleInternals.Tests.ps1 @@ -89,10 +89,10 @@ Describe 'Get-DscModuleFingerprint' { $script:FingerprintModule = Get-Module -ListAvailable (Join-Path $script:FingerprintRoot 'xTestClassResource\xTestClassResource.psd1') } - It 'counts the module files and takes the newest write time' { + It 'counts the module files, sums their sizes and hashes the size list' { $fingerprint = Invoke-PSDscInEngineScope { Get-DscModuleFingerprint -Module $args[0] } $script:FingerprintModule - $fingerprint | Should -Match '^2:[0-9]+$' + $fingerprint | Should -Match '^2:[0-9]+:[0-9a-f]{16}$' } It 'changes when a module file is touched' { diff --git a/test/PSDscTestHelper.psm1 b/test/PSDscTestHelper.psm1 index 48b8535..175580b 100644 --- a/test/PSDscTestHelper.psm1 +++ b/test/PSDscTestHelper.psm1 @@ -170,6 +170,9 @@ function Reset-PSDscFastHostState $script:FastHostRegisteredModules = @{} $script:FastHostKeywords = $null $script:FastHostAdapters = $null + $script:FastHostKeywordSource = $null + $script:FastHostResolvedModules = @{} + $script:FastHostTiming = $null } } diff --git a/test/SchemaCache.Tests.ps1 b/test/SchemaCache.Tests.ps1 index 4586b14..88a4d3f 100644 --- a/test/SchemaCache.Tests.ps1 +++ b/test/SchemaCache.Tests.ps1 @@ -18,6 +18,49 @@ BeforeAll { Copy-Item -Path (Join-Path $PSScriptRoot "TestModules\$Name") -Destination $Destination -Recurse -Force Join-Path $Destination $Name } + + function Read-PSDscCacheHeader + { + param ([string] $Path) + + ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllLines($Path)[0]) + } + + function Read-PSDscCacheSource + { + param ([string] $Path) + + ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllLines($Path)[1]) + } + + function Read-PSDscCacheKeyword + { + param ([string] $Path) + + $lines = [System.IO.File]::ReadAllLines($Path) + for ($i = 2; $i -lt $lines.Count; $i++) + { + ConvertFrom-Json -InputObject $lines[$i] + } + } + + function Copy-PSDscCacheWithLine + { + param ([string] $Source, [string] $Destination, [int] $Index, [scriptblock] $Mutate) + + $lines = [System.IO.File]::ReadAllLines($Source) + $object = ConvertFrom-Json -InputObject $lines[$Index] + & $Mutate $object + $lines[$Index] = ConvertTo-Json -InputObject $object -Depth 12 -Compress + [System.IO.File]::WriteAllLines($Destination, $lines) + } + + function Get-PSDscCacheKeywordCount + { + param ($Cache) + + @(Invoke-PSDscInEngineScope { Get-DscSchemaCacheKeywordName -Cache $args[0] } $Cache).Count + } } AfterAll { @@ -28,7 +71,8 @@ Describe 'Export-DscSchemaCache' { BeforeAll { $script:CachePath = Join-Path $TestDrive 'xTestClassResource-cache.json' $script:Summary = Export-DscSchemaCache -ModuleName xTestClassResource -OutputPath $script:CachePath - $script:Cache = ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllText($script:CachePath)) + $script:Header = Read-PSDscCacheHeader -Path $script:CachePath + $script:Keywords = @(Read-PSDscCacheKeyword -Path $script:CachePath) } It 'returns a summary with 4 resources and 5 keywords' { @@ -39,27 +83,42 @@ Describe 'Export-DscSchemaCache' { $script:Summary.Path | Should -Be $script:CachePath } - It 'writes parseable JSON with format version 1' { - $script:Cache | Should -Not -Be $null - $script:Cache.formatVersion | Should -Be 1 - $script:Cache.module.name | Should -Be 'xTestClassResource' - $script:Cache.module.version | Should -Be '1.0' - @($script:Cache.keywords).Count | Should -Be 5 - $embedded = $script:Cache.keywords | Where-Object { $_.keyword -eq 'EmbClass' } + It 'writes a header line with format version 2 and a keyword index' { + $script:Header.formatVersion | Should -Be 2 + $script:Header.module.name | Should -Be 'xTestClassResource' + $script:Header.module.version | Should -Be '1.0' + $script:Header.keywordCount | Should -Be 5 + $script:Header.resourceCount | Should -Be 4 + @($script:Header.index.PSObject.Properties).Count | Should -Be 5 + } + + It 'writes one keyword per line at the indexed position' { + $script:Keywords.Count | Should -Be 5 + $lines = [System.IO.File]::ReadAllLines($script:CachePath) + foreach ($entry in $script:Header.index.PSObject.Properties) + { + (ConvertFrom-Json -InputObject $lines[[int]$entry.Value]).keyword | Should -Be $entry.Name + } + $embedded = $script:Keywords | Where-Object { $_.keyword -eq 'EmbClass' } $embedded.nameMode | Should -Be 'NoName' } - It 'records a source hash for every module file' { - $files = @($script:Cache.module.sourceHash.PSObject.Properties) + It 'records size and hash for every module file on the source line' { + $files = @((Read-PSDscCacheSource -Path $script:CachePath).PSObject.Properties) ($files.Name | Sort-Object) -join ',' | Should -Be 'xTestClassResource.psd1,xTestClassResource.psm1' foreach ($file in $files) { - $file.Value | Should -Match '^[0-9A-F]{64}$' + $file.Value.sha256 | Should -Match '^[0-9A-F]{64}$' + $file.Value.length | Should -BeGreaterThan 0 } } + It 'fingerprints file count, total bytes and a size hash' { + $script:Header.module.fingerprint | Should -Match '^2:[0-9]+:[0-9a-f]{16}$' + } + It 'round-trips every keyword through ConvertFrom-DscKeywordSchemaObject' { - foreach ($schemaObject in $script:Cache.keywords) + foreach ($schemaObject in $script:Keywords) { $keyword = ConvertFrom-PSDscKeywordSchemaObject -SchemaObject $schemaObject $keyword.Keyword | Should -Be $schemaObject.keyword @@ -72,7 +131,7 @@ Describe 'Export-DscSchemaCache' { } It 'round-trips key, mandatory and value map details' { - $schemaObject = $script:Cache.keywords | Where-Object { $_.keyword -eq 'xTestClassResource' } + $schemaObject = $script:Keywords | Where-Object { $_.keyword -eq 'xTestClassResource' } $keyword = ConvertFrom-PSDscKeywordSchemaObject -SchemaObject $schemaObject $keyword.Properties['Name'].IsKey | Should -Be $true @@ -86,6 +145,15 @@ Describe 'Export-DscSchemaCache' { $keyword.Properties['Ensure'].ValueMap['Absent'] | Should -Be 'Absent' } + It 'includes the keywords of schema based resources' { + $summary = Export-DscSchemaCache -ModuleName xTestScriptResource -OutputPath (Join-Path $TestDrive 'script-cache.json') + + $summary.ResourceCount | Should -Be 1 + $summary.KeywordCount | Should -BeGreaterOrEqual 1 + $keywords = @(Read-PSDscCacheKeyword -Path $summary.Path) + ($keywords | Where-Object { $_.keyword -eq 'xTestScriptResource' }).resourceName | Should -Be 'MSFT_xTestScriptResource' + } + It 'writes next to the module when no output path is given' { $modulePath = Copy-PSDscTestModule -Name xTestClassResource -Destination (Join-Path $TestDrive 'default-output') $module = Get-Module -ListAvailable (Join-Path $modulePath 'xTestClassResource.psd1') @@ -145,24 +213,31 @@ Describe 'Test-DscSchemaCache' { ($warnings -join ' ') | Should -Match 'does not exist' } - It 'returns false with a warning for a newer cache format' { + It 'returns false with a warning for another cache format' { $futurePath = Join-Path $TestDrive 'future-format.json' - $cache = ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllText($script:CopyCachePath)) - $cache.formatVersion = 99 - [System.IO.File]::WriteAllText($futurePath, (ConvertTo-Json -InputObject $cache -Depth 12 -Compress)) + Copy-PSDscCacheWithLine -Source $script:CopyCachePath -Destination $futurePath -Index 0 -Mutate { $args[0].formatVersion = 99 } $warnings = $null $result = Test-DscSchemaCache -ModulePath $script:CopyDirectory -CachePath $futurePath -WarningVariable warnings 3>$null $result | Should -Be $false - ($warnings -join ' ') | Should -Match 'newer than supported' + ($warnings -join ' ') | Should -Match 'not the supported version' + } + + It 'returns false with a warning for a single line file' { + $singleLinePath = Join-Path $TestDrive 'single-line.json' + [System.IO.File]::WriteAllText($singleLinePath, '{"formatVersion":2}') + + $warnings = $null + $result = Test-DscSchemaCache -ModulePath $script:CopyDirectory -CachePath $singleLinePath -WarningVariable warnings 3>$null + + $result | Should -Be $false + ($warnings -join ' ') | Should -Match 'could not be read' } It 'returns false with a warning when the module version moved on' { $otherVersionPath = Join-Path $TestDrive 'other-version.json' - $cache = ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllText($script:CopyCachePath)) - $cache.module.version = '9.9' - [System.IO.File]::WriteAllText($otherVersionPath, (ConvertTo-Json -InputObject $cache -Depth 12 -Compress)) + Copy-PSDscCacheWithLine -Source $script:CopyCachePath -Destination $otherVersionPath -Index 0 -Mutate { $args[0].module.version = '9.9' } $warnings = $null $result = Test-DscSchemaCache -ModulePath $script:CopyDirectory -CachePath $otherVersionPath -WarningVariable warnings 3>$null @@ -173,9 +248,9 @@ Describe 'Test-DscSchemaCache' { It 'returns false with a warning when a cached file was removed' { $removedFilePath = Join-Path $TestDrive 'removed-file.json' - $cache = ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllText($script:CopyCachePath)) - Add-Member -InputObject $cache.module.sourceHash -MemberType NoteProperty -Name 'gone.psm1' -Value ('0' * 64) - [System.IO.File]::WriteAllText($removedFilePath, (ConvertTo-Json -InputObject $cache -Depth 12 -Compress)) + Copy-PSDscCacheWithLine -Source $script:CopyCachePath -Destination $removedFilePath -Index 1 -Mutate { + Add-Member -InputObject $args[0] -MemberType NoteProperty -Name 'gone.psm1' -Value ([pscustomobject]@{ length = 1; sha256 = ('0' * 64) }) + } $warnings = $null $result = Test-DscSchemaCache -ModulePath $script:CopyDirectory -CachePath $removedFilePath -WarningVariable warnings 3>$null @@ -207,12 +282,39 @@ Describe 'Get-DscSchemaCache' { } @{ Module = $script:Module; Candidates = @($script:ValidCachePath) } $cache.module.name | Should -Be 'xTestClassResource' - @($cache.keywords).Count | Should -Be 5 + $cache.keywordCount | Should -Be 5 + Get-PSDscCacheKeywordCount -Cache $cache | Should -Be 5 + } + + It 'accepts a module descriptor with Name, Version and ModuleBase' { + $descriptor = [pscustomobject]@{ + Name = $script:Module.Name + Version = $script:Module.Version + ModuleBase = $script:Module.ModuleBase + Path = $script:Module.Path + } + + $cache = Invoke-PSDscInEngineScope { + Get-DscSchemaCache -Module $args[0].Module -SchemaCachePath $args[0].Candidates + } @{ Module = $descriptor; Candidates = @($script:ValidCachePath) } + + $cache.module.name | Should -Be 'xTestClassResource' + } + + It 'deserializes a keyword on request and reports null for an unknown name' { + $cache = Invoke-PSDscInEngineScope { + Get-DscSchemaCache -Module $args[0].Module -SchemaCachePath $args[0].Candidates + } @{ Module = $script:Module; Candidates = @($script:ValidCachePath) } + + $keyword = Invoke-PSDscInEngineScope { Get-DscSchemaCacheKeyword -Cache $args[0] -Name 'xtestclassresource' } $cache + $keyword | Should -BeOfType ([System.Management.Automation.Language.DynamicKeyword]) + $keyword.Keyword | Should -Be 'xTestClassResource' + Invoke-PSDscInEngineScope { Get-DscSchemaCacheKeyword -Cache $args[0] -Name 'NoSuchKeyword' } $cache | Should -BeNullOrEmpty } It 'skips a candidate that is not valid JSON' { $brokenPath = Join-Path $TestDrive 'lookup-broken.json' - [System.IO.File]::WriteAllText($brokenPath, '{ not json') + [System.IO.File]::WriteAllText($brokenPath, "{ not json`n{") $cache = Invoke-PSDscInEngineScope { Get-DscSchemaCache -Module $args[0].Module -SchemaCachePath $args[0].Candidates -WarningAction SilentlyContinue @@ -221,24 +323,21 @@ Describe 'Get-DscSchemaCache' { $cache.module.name | Should -Be 'xTestClassResource' } - It 'skips a candidate with an unsupported format version' { + It 'skips a candidate with another format version' { $futurePath = Join-Path $TestDrive 'lookup-future.json' - $cache = ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllText($script:ValidCachePath)) - $cache.formatVersion = 99 - [System.IO.File]::WriteAllText($futurePath, (ConvertTo-Json -InputObject $cache -Depth 12 -Compress)) + Copy-PSDscCacheWithLine -Source $script:ValidCachePath -Destination $futurePath -Index 0 -Mutate { $args[0].formatVersion = 99 } $result = Invoke-PSDscInEngineScope { Get-DscSchemaCache -Module $args[0].Module -SchemaCachePath $args[0].Candidates -WarningAction SilentlyContinue } @{ Module = $script:Module; Candidates = @($futurePath, $script:ValidCachePath) } - $result.formatVersion | Should -Be 1 + $result.formatVersion | Should -Be 2 + $result.Path | Should -Be $script:ValidCachePath } It 'skips a candidate that belongs to another module' { $otherModulePath = Join-Path $TestDrive 'lookup-other-module.json' - $cache = ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllText($script:ValidCachePath)) - $cache.module.name = 'SomeOtherModule' - [System.IO.File]::WriteAllText($otherModulePath, (ConvertTo-Json -InputObject $cache -Depth 12 -Compress)) + Copy-PSDscCacheWithLine -Source $script:ValidCachePath -Destination $otherModulePath -Index 0 -Mutate { $args[0].module.name = 'SomeOtherModule' } $result = Invoke-PSDscInEngineScope { Get-DscSchemaCache -Module $args[0].Module -SchemaCachePath $args[0].Candidates -WarningAction SilentlyContinue @@ -248,17 +347,13 @@ Describe 'Get-DscSchemaCache' { } It 'returns nothing when every candidate is stale' { - # A version of its own, so no cache generated for the installed module can satisfy the - # lookup through the module base or the per user location. $isolated = New-PSDscIsolatedTestModule -Name xTestClassResource -Version '1.7' ` -Destination (Join-Path $TestDrive 'lookup-isolated') $isolatedModule = $isolated.Module $stalePath = Join-Path $TestDrive 'lookup-stale.json' $null = Export-DscSchemaCache -Module $isolatedModule -OutputPath $stalePath - $cache = ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllText($stalePath)) - $cache.module.fingerprint = '1:1' - [System.IO.File]::WriteAllText($stalePath, (ConvertTo-Json -InputObject $cache -Depth 12 -Compress)) + Copy-PSDscCacheWithLine -Source $stalePath -Destination $stalePath -Index 0 -Mutate { $args[0].module.fingerprint = '1:1:0000000000000000' } $result = Invoke-PSDscInEngineScope { Get-DscSchemaCache -Module $args[0].Module -SchemaCachePath $args[0].Candidates -WarningAction SilentlyContinue @@ -273,9 +368,7 @@ Describe 'Get-DscSchemaCache' { $stalePath = Join-Path $TestDrive 'lookup-no-warning.json' $null = Export-DscSchemaCache -Module $isolated.Module -OutputPath $stalePath - $cache = ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllText($stalePath)) - $cache.module.fingerprint = '1:1' - [System.IO.File]::WriteAllText($stalePath, (ConvertTo-Json -InputObject $cache -Depth 12 -Compress)) + Copy-PSDscCacheWithLine -Source $stalePath -Destination $stalePath -Index 0 -Mutate { $args[0].module.fingerprint = '1:1:0000000000000000' } $streams = Invoke-PSDscInEngineScope { Get-DscSchemaCache -Module $args[0].Module -SchemaCachePath $args[0].Candidates @@ -298,7 +391,26 @@ Describe 'Get-DscSchemaCache' { } @{ Module = $isolated.Module; Candidates = @($cachePath) } $result.module.name | Should -Be 'xTestClassResource' - @($result.keywords).Count | Should -Be 5 + $result.keywordCount | Should -Be 5 + } + + It 'returns the cache after the module files were copied with new write times' { + $isolated = New-PSDscIsolatedTestModule -Name xTestClassResource -Version '2.2' ` + -Destination (Join-Path $TestDrive 'lookup-copied') + + $cachePath = Join-Path $TestDrive 'lookup-copied.json' + $null = Export-DscSchemaCache -Module $isolated.Module -OutputPath $cachePath + + foreach ($file in Get-ChildItem -Path $isolated.Module.ModuleBase -File) + { + $file.LastWriteTimeUtc = [datetime]::UtcNow.AddDays(1) + } + + $result = Invoke-PSDscInEngineScope { + Get-DscSchemaCache -Module $args[0].Module -SchemaCachePath $args[0].Candidates -WarningAction SilentlyContinue + } @{ Module = $isolated.Module; Candidates = @($cachePath) } + + $result.module.name | Should -Be 'xTestClassResource' } It 'returns nothing when a recorded file no longer exists' { @@ -331,7 +443,30 @@ Describe 'New-DscSchemaCacheForModule' { $isolated.UserCachePath | Should -Exist $cache.module.name | Should -Be 'xTestClassResource' - @($cache.keywords).Count | Should -Be 5 + $cache.keywordCount | Should -Be 5 + } + finally + { + Remove-Item -Path $isolated.UserCachePath -Force -ErrorAction Ignore + } + } + + It 'resolves a module descriptor to the module for discovery' { + $isolated = New-PSDscIsolatedTestModule -Name xTestClassResource -Version '1.6' ` + -Destination (Join-Path $TestDrive 'generated-descriptor') + $descriptor = [pscustomobject]@{ + Name = $isolated.Module.Name + Version = $isolated.Module.Version + ModuleBase = $isolated.Module.ModuleBase + Path = $isolated.ManifestPath + } + + try + { + $cache = Invoke-PSDscInEngineScope { New-DscSchemaCacheForModule -Module $args[0] } @($descriptor) + + $isolated.UserCachePath | Should -Exist + $cache.keywordCount | Should -Be 5 } finally { @@ -358,10 +493,7 @@ Describe 'Stale schema cache handling in the fast host' { It 'compiles without warning when the cache is stale' { $doctoredPath = Join-Path $TestDrive 'doctored-cache.json' $null = Export-DscSchemaCache -Module $script:StaleModule.Module -OutputPath $doctoredPath - $cache = ConvertFrom-Json -InputObject ([System.IO.File]::ReadAllText($doctoredPath)) - $cache.module.fingerprint = '1:1' - $json = ConvertTo-Json -InputObject $cache -Depth 12 -Compress - [System.IO.File]::WriteAllText($doctoredPath, $json) + Copy-PSDscCacheWithLine -Source $doctoredPath -Destination $doctoredPath -Index 0 -Mutate { $args[0].module.fingerprint = '1:1:0000000000000000' } $script:StaleModule.UserCachePath | Should -Not -Exist $text = @'