From 082e89d0de4f50f57650f8657e6da743ea6ac55c Mon Sep 17 00:00:00 2001 From: "radmond\\idoshabi" Date: Wed, 28 Oct 2026 15:07:32 +0200 Subject: [PATCH 1/6] SentinelToXDRMigration: implement qualification follow-ups, KQL static override, and NPF metrics (Todos #0-#6) - Todo #0: isolate Azure CLI profile in runtime consent and mock credential constructors in pytest conftest - Todo #1: add live CD + AR enablement, alert parity readiness, and --keep-enabled support - Todo #2: enforce strict V3.1 packaging verification for XDR Detections and XDR Workbooks - Todo #3: default workflow profile to qualification and fix Windows directory fsync in budget.py - Todo #4: add static KQL validation for XDR Detections and XDR Workbooks in Python and KqlValidationTests.cs - Todo #5: project kqlRuntimeVerifiedRules SHA-256 hashes and downgrade semantic KQL errors on qualified rules - Todo #6: emit NPF metrics instrumentation across pr-qualification-result.json, workflow-state.json, decision-manifest.json, and npf-pr-labeler.yaml Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2f9e241a-6b82-49ec-ac18-e83c833f2521 --- .../skills/sentinel-xdr-migration/SKILL.md | 9 +- .cursor/rules/sentinel-xdr-migration.mdc | 6 +- .github/agents/xdr-solution-manager.agent.md | 13 +- .../SKILL.md | 5 +- .github/workflows/npf-pr-labeler.yaml | 20 +- .../local-validation/build-and-validate.ps1 | 8 +- .../local-validation/validate-xdr-content.py | 24 + .../CustomTables/AADSignInEventsBeta.json | 201 +++++ .../CustomTables/AADSpnSignInEventsBeta.json | 97 ++ .../CustomTables/BehaviorEntities.json | 177 ++++ .../CustomTables/BehaviorInfo.json | 89 ++ .../CustomTables/DeviceEvents.json | 293 ++++++ .../DeviceFileCertificateInfo.json | 105 +++ .../CustomTables/DeviceFileEvents.json | 265 ++++++ .../CustomTables/DeviceImageLoadEvents.json | 201 +++++ .../CustomTables/DeviceInfo.json | 225 +++++ .../CustomTables/DeviceLogonEvents.json | 229 +++++ .../CustomTables/DeviceNetworkEvents.json | 213 +++++ .../CustomTables/DeviceNetworkInfo.json | 97 ++ .../CustomTables/DeviceProcessEvents.json | 305 +++++++ .../CustomTables/DeviceRegistryEvents.json | 205 +++++ .../CustomTables/EmailAttachmentInfo.json | 93 ++ .../CustomTables/EmailEvents.json | 233 +++++ .../CustomTables/EmailPostDeliveryEvents.json | 85 ++ .../CustomTables/EmailUrlInfo.json | 61 ++ .../CustomTables/IdentityInfo.json | 436 ++++----- .../KqlvalidationsTests/KqlValidationTests.cs | 431 ++++++++- .../SkipValidationsTemplates.json | 10 + .../XdrDetectionsYamlFilesLoader.cs | 24 + .../XdrDetectionsYamlFilesTestData.cs | 10 + .windsurf/rules/sentinel-xdr-migration.md | 10 +- AGENTS.md | 2 +- CLAUDE.md | 9 +- .../V3/createSolutionV3_1.ps1 | 5 +- .../common/createCCPConnector.ps1 | 1 + .../common/get-ccp-details.ps1 | 30 +- Tools/SentinelToXDRMigration/FOLLOW_UPS.md | 17 +- .../agent-workflows/end-to-end-migration.md | 23 +- .../schema/workflow-state.schema.json | 14 + .../solution_manager/authoring/__init__.py | 16 + .../authoring/dependencies.py | 2 +- .../solution_manager/authoring/validator.py | 849 ++++++++++++++++++ .../solution_manager/cli.py | 41 +- .../solution_manager/core/budget.py | 11 +- .../solution_manager/core/onboarding.py | 23 +- .../solution_manager/core/preflight.py | 33 +- .../solution_manager/core/workflow.py | 43 +- .../data/ah_schema_catalog.json | 41 + .../data/la_schema_catalog.json | 41 + .../_workbook_core/data/mapping_catalog.json | 9 + .../solution_manager/migration/converter.py | 2 + .../solution_manager/packaging/packaging.py | 283 +++++- .../qualification/alert_parity.py | 588 +++++++++++- .../qualification/analytic_deployment.py | 48 +- .../qualification/deployment.py | 21 +- .../qualification/graph_client_config.py | 16 + .../qualification/orchestrator.py | 125 ++- .../qualification/pr_check_runner.py | 588 +++++++++++- .../workbook_preflight_workflow.py | 5 +- .../reporting/decision_manifest.py | 113 +++ .../reporting/pr_qualification_report.py | 110 +++ .../SentinelToXDRMigration/tests/conftest.py | 134 +++ .../Workbooks/AzureActivity.json | 2 +- .../tests/test_authoring_kql_validator.py | 137 +++ .../tests/test_decision_manifest.py | 36 + .../tests/test_packaging.py | 319 ++++++- .../tests/test_pr_qualification_engine.py | 155 ++++ .../tests/test_pr_qualification_report.py | 61 ++ .../tests/test_qualification_orchestrator.py | 506 +++++++++++ .../tests/test_release_hygiene.py | 94 +- .../tests/test_runtime_consent.py | 143 +++ .../tests/test_target_config.py | 76 ++ .../tests/test_workflow.py | 41 +- 73 files changed, 8570 insertions(+), 423 deletions(-) create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/AADSignInEventsBeta.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/AADSpnSignInEventsBeta.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/BehaviorEntities.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/BehaviorInfo.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/DeviceEvents.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/DeviceFileCertificateInfo.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/DeviceFileEvents.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/DeviceImageLoadEvents.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/DeviceInfo.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/DeviceLogonEvents.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/DeviceNetworkEvents.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/DeviceNetworkInfo.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/DeviceProcessEvents.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/DeviceRegistryEvents.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/EmailAttachmentInfo.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/EmailEvents.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/EmailPostDeliveryEvents.json create mode 100644 .script/tests/KqlvalidationsTests/CustomTables/EmailUrlInfo.json create mode 100644 .script/tests/KqlvalidationsTests/YamlFilesTestData/XdrDetectionsYamlFilesLoader.cs create mode 100644 .script/tests/KqlvalidationsTests/YamlFilesTestData/XdrDetectionsYamlFilesTestData.cs create mode 100644 Tools/SentinelToXDRMigration/solution_manager/authoring/validator.py create mode 100644 Tools/SentinelToXDRMigration/tests/conftest.py create mode 100644 Tools/SentinelToXDRMigration/tests/test_authoring_kql_validator.py diff --git a/.claude/skills/sentinel-xdr-migration/SKILL.md b/.claude/skills/sentinel-xdr-migration/SKILL.md index fd3605ff40c..abff38cafb9 100644 --- a/.claude/skills/sentinel-xdr-migration/SKILL.md +++ b/.claude/skills/sentinel-xdr-migration/SKILL.md @@ -10,10 +10,11 @@ Read and follow: `Tools\SentinelToXDRMigration\agent-workflows\end-to-end-migration.md` Use `sentinel-xdr-migration` and the repository PowerShell packaging entry -points for all deterministic actions. Default to `authoring`; ask before -optional `qualification`. Use V3.1 through `package-v3-1` for Sentinel and XDR -solution packaging, resume `workflow-state.json`, preserve disabled rules, and -obtain explicit approval before environment writes. +points for all deterministic actions. Default to `qualification` (with +`--workflow-profile authoring` available as an explicit offline/ISV fallback). +Use V3.1 through `package-v3-1` for Sentinel and XDR solution packaging, resume +`workflow-state.json`, preserve disabled rules, and obtain explicit approval +before environment writes. The goal is solution migration and validated publication. Follow the canonical workflow for the strict over-50% partial-publication policy, diff --git a/.cursor/rules/sentinel-xdr-migration.mdc b/.cursor/rules/sentinel-xdr-migration.mdc index 50ad6ea6862..c088599e89e 100644 --- a/.cursor/rules/sentinel-xdr-migration.mdc +++ b/.cursor/rules/sentinel-xdr-migration.mdc @@ -12,7 +12,7 @@ packaging, or AR/CD qualification, read and follow `Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md`. That document is canonical. Use repository CLIs for deterministic behavior, -default to the authoring profile, and ask before optional qualification. -Qualification deployment, mock ingestion, and parity are not publishing -requirements. Use V3.1 through `package-v3-1` for Sentinel and XDR solution +default to the `qualification` profile, and use `--workflow-profile authoring` +as an explicit offline/ISV fallback when no live lab is available. +Use V3.1 through `package-v3-1` for Sentinel and XDR solution packages. Resume persisted workflow state and require approval before writes. diff --git a/.github/agents/xdr-solution-manager.agent.md b/.github/agents/xdr-solution-manager.agent.md index 8c51c7b29a2..35e3f73afbf 100644 --- a/.github/agents/xdr-solution-manager.agent.md +++ b/.github/agents/xdr-solution-manager.agent.md @@ -131,16 +131,17 @@ review, runtime/schema, packaging, authentication, or write gates. ## Migration workflow -1. Run `workflow-status` first. If no state exists, default to **Authoring** - and ask before optional **Qualification** (or **Cancel**). Explain that - Authoring stops after packaging/reporting while Qualification adds - deployment, mock ingestion, and alert parity. -2. Initialize or resume with the selected profile (defaults to `authoring`): +1. Run `workflow-status` first. If no state exists, default to **Qualification** + (or allow explicit **Authoring** as an offline/ISV fallback when no live lab + is available, or **Cancel**). Explain that Qualification adds deployment, + mock ingestion, and alert parity while Authoring stops after + packaging/reporting. +2. Initialize or resume with the selected profile (defaults to `qualification`): ```powershell sentinel-xdr-migration workflow-init ` --solution "" ` - --workflow-profile "" ` + --workflow-profile "" ` --tenant-id "" ` --subscription-id "" ` --workspace-resource-id "" ` diff --git a/.github/skills/sentinel-xdr-end-to-end-orchestrator/SKILL.md b/.github/skills/sentinel-xdr-end-to-end-orchestrator/SKILL.md index 84805971997..21ba6dcd7bc 100644 --- a/.github/skills/sentinel-xdr-end-to-end-orchestrator/SKILL.md +++ b/.github/skills/sentinel-xdr-end-to-end-orchestrator/SKILL.md @@ -49,8 +49,9 @@ the ARM ID again. Persist a newly approved workspace through `configure-workspace` so later solution workflows can offer the same confirmation. -Default to **Authoring** and ask before optional **Qualification** through -structured elicitation. If selection is cancelled, do not create workflow state. +Default to **Qualification** (with **Authoring** available as an explicit +offline/ISV fallback when no live lab is available) through structured +elicitation. If selection is cancelled, do not create workflow state. A test tenant or workspace does not imply qualification consent. Require explicit approval immediately before deployment or ingestion writes. diff --git a/.github/workflows/npf-pr-labeler.yaml b/.github/workflows/npf-pr-labeler.yaml index 2e084b399d6..7939634cdc0 100644 --- a/.github/workflows/npf-pr-labeler.yaml +++ b/.github/workflows/npf-pr-labeler.yaml @@ -145,16 +145,30 @@ jobs: } // Build machine-readable npf-meta block for PR body and step summary + const currentBody = context.payload.pull_request?.body || ''; + const metaRegex = //s; + let priorSyncObserved = false; + const existingMetaMatch = currentBody.match(metaRegex); + if (existingMetaMatch) { + try { + const parsedMeta = JSON.parse(existingMetaMatch[1]); + if (parsedMeta && parsedMeta.synchronizePushObserved === true) { + priorSyncObserved = true; + } + } catch (_err) { + // Ignore malformed prior metadata block + } + } + const synchronizePushObserved = priorSyncObserved || (context.payload.action === 'synchronize'); const npfMeta = { solutionCount: solutionDetails.length, solutions: solutionDetails.map(s => s.name), publisherTiers: [...publisherTiers], - npfModes + npfModes, + synchronizePushObserved }; const metaComment = ``; - const currentBody = context.payload.pull_request?.body || ''; - const metaRegex = //s; let updatedBody = currentBody; if (metaRegex.test(currentBody)) { updatedBody = currentBody.replace(metaRegex, metaComment); diff --git a/.script/local-validation/build-and-validate.ps1 b/.script/local-validation/build-and-validate.ps1 index a4fb2a22cb3..4c54b938730 100644 --- a/.script/local-validation/build-and-validate.ps1 +++ b/.script/local-validation/build-and-validate.ps1 @@ -1034,8 +1034,8 @@ if (-not $SkipValidation) { if ($has80 -and (Test-Path $kqlProject)) { # Collect all YAML/JSON content files from the solution for scoping $kqlFiles = Get-SolutionContentFiles -SolPath $solutionPath ` - -SubDirs @("Analytic Rules", "Hunting Queries", "Parsers", "Data Connectors") ` - -Extensions @("*.yaml", "*.json") + -SubDirs @("Analytic Rules", "XDR Detections", "Hunting Queries", "Parsers", "Data Connectors") ` + -Extensions @("*.yaml", "*.yml", "*.json") if ($kqlFiles.Count -gt 0) { $kqlFilter = Build-DotnetTestFilter -Files $kqlFiles -UseEncodedPath @@ -1124,9 +1124,9 @@ if (-not $SkipValidation) { } } else { - $xdrKqlCheckFiles = Get-SolutionContentFiles -SolPath $solutionPath -SubDirs @("XDR Detections", "XDR Workbooks") -Extensions @("*.yaml", "*.yml", "*.json") + $xdrKqlCheckFiles = Get-SolutionContentFiles -SolPath $solutionPath -SubDirs @("XDR Workbooks") -Extensions @("*.json") $noKqlReason = if ($xdrKqlCheckFiles.Count -gt 0) { - "No legacy Sentinel KQL files found (.NET KQL validator does not support XDR Detections or XDR Workbooks)" + "No Sentinel or XDR Detection KQL files found (.NET KQL validator does not support XDR Workbooks; validated via static XDR KQL validator)" } else { "No content files found" diff --git a/.script/local-validation/validate-xdr-content.py b/.script/local-validation/validate-xdr-content.py index 88fb0104114..c0e57cb43f0 100644 --- a/.script/local-validation/validate-xdr-content.py +++ b/.script/local-validation/validate-xdr-content.py @@ -41,6 +41,12 @@ def _setup_toolkit_imports(toolkit_root: Path) -> None: migration_root = toolkit_root / "Tools" / "SentinelToXDRMigration" if str(migration_root) not in sys.path: sys.path.insert(0, str(migration_root)) + for pkg_name in ("solution_manager", "solution_manager.authoring"): + pkg = sys.modules.get(pkg_name) + if pkg is not None and hasattr(pkg, "__path__"): + sub_dir = migration_root.joinpath(*pkg_name.split(".")) + if sub_dir.is_dir() and str(sub_dir) not in pkg.__path__: + pkg.__path__.insert(0, str(sub_dir)) def _rel_path(path: Path, content_root: Path) -> str: @@ -339,6 +345,7 @@ def validate_solution_xdr_detections( xdr_detection_files, ) from sentinel_xdr_migration.packaging import _read_json, _value, _xdr_references + from solution_manager.authoring.validator import validate_xdr_detection_kql xdr_dir = solution_root / "XDR Detections" authored_files = xdr_detection_files(xdr_dir) if xdr_dir.is_dir() else [] @@ -419,6 +426,9 @@ def validate_solution_xdr_detections( parsed_docs[path] = document file_errors.extend(validate_document(document)) file_warnings.extend(custom_detail_binding_warnings(document)) + static_kql_res = validate_xdr_detection_kql(document) + for kql_err in static_kql_res.get("errors") or []: + file_errors.append(f"Static KQL validation: {kql_err}") provenance = document.get("contentProvenance") if isinstance(provenance, dict): @@ -494,6 +504,7 @@ def validate_solution_xdr_detections( "validator": "XDR Detection", "filePath": _rel_path(path, content_root), "passed": len(file_errors) == 0, + "staticKqlValidated": True, "error": "; ".join(dict.fromkeys(file_errors)) if file_errors else None, "warnings": file_warnings, } @@ -569,6 +580,7 @@ def validate_solution_xdr_detections( return { "applicable": True, + "staticKqlValidated": True, "liveKqlSkipped": True, "liveKqlSkipReason": ( "Live Advanced Hunting KQL validation for XDR Detections requires an " @@ -688,6 +700,7 @@ def validate_solution_xdr_workbooks( selected_workbook_files, validate_workbooks, ) + from solution_manager.authoring.validator import validate_xdr_workbook_kql wb_dir = solution_root / "XDR Workbooks" authored_files = ( @@ -804,11 +817,17 @@ def validate_solution_xdr_workbooks( msg = f"{loc}: {err.message}" if msg not in item_errors: item_errors.append(msg) + static_kql_res = validate_xdr_workbook_kql(doc) + for kql_err in static_kql_res.get("errors") or []: + msg = f"Static KQL validation: {kql_err}" + if msg not in item_errors: + item_errors.append(msg) results.append( { "validator": "XDR Workbook", "filePath": _rel_path(target_path, content_root), "passed": len(item_errors) == 0, + "staticKqlValidated": True, "error": "; ".join(dict.fromkeys(item_errors)) if item_errors else None, } ) @@ -840,6 +859,9 @@ def validate_solution_xdr_workbooks( for e in schema_validator.iter_errors(doc) ] file_errors.extend(dependency_validation_errors(doc)) + static_kql_res = validate_xdr_workbook_kql(doc) + for kql_err in static_kql_res.get("errors") or []: + file_errors.append(f"Static KQL validation: {kql_err}") provenance = doc.get("contentProvenance") if isinstance(provenance, dict): fn_records = provenance.get("requiredWorkspaceFunctions", []) @@ -853,6 +875,7 @@ def validate_solution_xdr_workbooks( "validator": "XDR Workbook", "filePath": _rel_path(path, content_root), "passed": len(file_errors) == 0, + "staticKqlValidated": True, "error": "; ".join(dict.fromkeys(file_errors)) if file_errors else None, } ) @@ -919,6 +942,7 @@ def validate_solution_xdr_workbooks( return { "applicable": True, + "staticKqlValidated": True, "liveKqlSkipped": True, "liveKqlSkipReason": ( "Live Advanced Hunting / Defender Workbook query & portal rendering validation " diff --git a/.script/tests/KqlvalidationsTests/CustomTables/AADSignInEventsBeta.json b/.script/tests/KqlvalidationsTests/CustomTables/AADSignInEventsBeta.json new file mode 100644 index 00000000000..d9e4f4b2e66 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/AADSignInEventsBeta.json @@ -0,0 +1,201 @@ +{ + "Name": "AADSignInEventsBeta", + "Properties": [ + { + "Name": "Timestamp", + "Type": "datetime" + }, + { + "Name": "Application", + "Type": "string" + }, + { + "Name": "ApplicationId", + "Type": "string" + }, + { + "Name": "LogonType", + "Type": "string" + }, + { + "Name": "ErrorCode", + "Type": "int" + }, + { + "Name": "CorrelationId", + "Type": "string" + }, + { + "Name": "SessionId", + "Type": "string" + }, + { + "Name": "AccountDisplayName", + "Type": "string" + }, + { + "Name": "AccountObjectId", + "Type": "string" + }, + { + "Name": "AccountUpn", + "Type": "string" + }, + { + "Name": "IsExternalUser", + "Type": "int" + }, + { + "Name": "IsGuestUser", + "Type": "bool" + }, + { + "Name": "AlternateSignInName", + "Type": "string" + }, + { + "Name": "LastPasswordChangeTimestamp", + "Type": "datetime" + }, + { + "Name": "ResourceDisplayName", + "Type": "string" + }, + { + "Name": "ResourceId", + "Type": "string" + }, + { + "Name": "ResourceTenantId", + "Type": "string" + }, + { + "Name": "DeviceName", + "Type": "string" + }, + { + "Name": "AadDeviceId", + "Type": "string" + }, + { + "Name": "EntraIdDeviceId", + "Type": "string" + }, + { + "Name": "OSPlatform", + "Type": "string" + }, + { + "Name": "DeviceTrustType", + "Type": "string" + }, + { + "Name": "IsManaged", + "Type": "int" + }, + { + "Name": "IsCompliant", + "Type": "int" + }, + { + "Name": "AuthenticationProcessingDetails", + "Type": "string" + }, + { + "Name": "AuthenticationRequirement", + "Type": "string" + }, + { + "Name": "TokenIssuerType", + "Type": "int" + }, + { + "Name": "RiskLevelAggregated", + "Type": "int" + }, + { + "Name": "RiskDetails", + "Type": "int" + }, + { + "Name": "RiskState", + "Type": "int" + }, + { + "Name": "UserAgent", + "Type": "string" + }, + { + "Name": "ClientAppUsed", + "Type": "string" + }, + { + "Name": "Browser", + "Type": "string" + }, + { + "Name": "ConditionalAccessPolicies", + "Type": "string" + }, + { + "Name": "ConditionalAccessStatus", + "Type": "int" + }, + { + "Name": "IPAddress", + "Type": "string" + }, + { + "Name": "Country", + "Type": "string" + }, + { + "Name": "State", + "Type": "string" + }, + { + "Name": "City", + "Type": "string" + }, + { + "Name": "Latitude", + "Type": "string" + }, + { + "Name": "Longitude", + "Type": "string" + }, + { + "Name": "NetworkLocationDetails", + "Type": "string" + }, + { + "Name": "RequestId", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "string" + }, + { + "Name": "EndpointCall", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "Type", + "Type": "string" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/AADSpnSignInEventsBeta.json b/.script/tests/KqlvalidationsTests/CustomTables/AADSpnSignInEventsBeta.json new file mode 100644 index 00000000000..5b1641d97a8 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/AADSpnSignInEventsBeta.json @@ -0,0 +1,97 @@ +{ + "Name": "AADSpnSignInEventsBeta", + "Properties": [ + { + "Name": "Timestamp", + "Type": "datetime" + }, + { + "Name": "Application", + "Type": "string" + }, + { + "Name": "ApplicationId", + "Type": "string" + }, + { + "Name": "IsManagedIdentity", + "Type": "bool" + }, + { + "Name": "ErrorCode", + "Type": "int" + }, + { + "Name": "CorrelationId", + "Type": "string" + }, + { + "Name": "ServicePrincipalName", + "Type": "string" + }, + { + "Name": "ServicePrincipalId", + "Type": "string" + }, + { + "Name": "ResourceDisplayName", + "Type": "string" + }, + { + "Name": "ResourceId", + "Type": "string" + }, + { + "Name": "ResourceTenantId", + "Type": "string" + }, + { + "Name": "IPAddress", + "Type": "string" + }, + { + "Name": "Country", + "Type": "string" + }, + { + "Name": "State", + "Type": "string" + }, + { + "Name": "City", + "Type": "string" + }, + { + "Name": "Latitude", + "Type": "string" + }, + { + "Name": "Longitude", + "Type": "string" + }, + { + "Name": "RequestId", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "Type", + "Type": "string" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/BehaviorEntities.json b/.script/tests/KqlvalidationsTests/CustomTables/BehaviorEntities.json new file mode 100644 index 00000000000..5d1b6adfc0d --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/BehaviorEntities.json @@ -0,0 +1,177 @@ +{ + "Name": "BehaviorEntities", + "Properties": [ + { + "Name": "AccountDomain", + "Type": "string" + }, + { + "Name": "AccountName", + "Type": "string" + }, + { + "Name": "AccountObjectId", + "Type": "string" + }, + { + "Name": "AccountSid", + "Type": "string" + }, + { + "Name": "AccountUpn", + "Type": "string" + }, + { + "Name": "ActionType", + "Type": "string" + }, + { + "Name": "AdditionalFields", + "Type": "string" + }, + { + "Name": "Application", + "Type": "string" + }, + { + "Name": "ApplicationId", + "Type": "string" + }, + { + "Name": "BehaviorId", + "Type": "string" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "Categories", + "Type": "string" + }, + { + "Name": "DataSources", + "Type": "string" + }, + { + "Name": "DetailedEntityRole", + "Type": "string" + }, + { + "Name": "DetectionSource", + "Type": "string" + }, + { + "Name": "DeviceId", + "Type": "string" + }, + { + "Name": "DeviceName", + "Type": "string" + }, + { + "Name": "EmailClusterId", + "Type": "string" + }, + { + "Name": "EmailSubject", + "Type": "string" + }, + { + "Name": "EntityRole", + "Type": "string" + }, + { + "Name": "EntityType", + "Type": "string" + }, + { + "Name": "FileName", + "Type": "string" + }, + { + "Name": "FileSize", + "Type": "long" + }, + { + "Name": "FolderPath", + "Type": "string" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "LocalIP", + "Type": "string" + }, + { + "Name": "NetworkMessageId", + "Type": "string" + }, + { + "Name": "OAuthApplicationId", + "Type": "string" + }, + { + "Name": "ProcessCommandLine", + "Type": "string" + }, + { + "Name": "RegistryKey", + "Type": "string" + }, + { + "Name": "RegistryValueData", + "Type": "string" + }, + { + "Name": "RegistryValueName", + "Type": "string" + }, + { + "Name": "RemoteIP", + "Type": "string" + }, + { + "Name": "RemoteUrl", + "Type": "string" + }, + { + "Name": "ServiceSource", + "Type": "string" + }, + { + "Name": "SHA1", + "Type": "string" + }, + { + "Name": "SHA256", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "ThreatFamily", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/BehaviorInfo.json b/.script/tests/KqlvalidationsTests/CustomTables/BehaviorInfo.json new file mode 100644 index 00000000000..7a96f56ca61 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/BehaviorInfo.json @@ -0,0 +1,89 @@ +{ + "Name": "BehaviorInfo", + "Properties": [ + { + "Name": "AccountObjectId", + "Type": "string" + }, + { + "Name": "AccountUpn", + "Type": "string" + }, + { + "Name": "ActionType", + "Type": "string" + }, + { + "Name": "AdditionalFields", + "Type": "string" + }, + { + "Name": "AttackTechniques", + "Type": "string" + }, + { + "Name": "BehaviorId", + "Type": "string" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "Categories", + "Type": "string" + }, + { + "Name": "DataSources", + "Type": "string" + }, + { + "Name": "Description", + "Type": "string" + }, + { + "Name": "DetectionSource", + "Type": "string" + }, + { + "Name": "DeviceId", + "Type": "string" + }, + { + "Name": "EndTime", + "Type": "datetime" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "ServiceSource", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "StartTime", + "Type": "datetime" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/DeviceEvents.json b/.script/tests/KqlvalidationsTests/CustomTables/DeviceEvents.json new file mode 100644 index 00000000000..f7e27ba2e32 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/DeviceEvents.json @@ -0,0 +1,293 @@ +{ + "Name": "DeviceEvents", + "Properties": [ + { + "Name": "AccountDomain", + "Type": "string" + }, + { + "Name": "AccountName", + "Type": "string" + }, + { + "Name": "AccountSid", + "Type": "string" + }, + { + "Name": "ActionType", + "Type": "string" + }, + { + "Name": "AdditionalFields", + "Type": "dynamic" + }, + { + "Name": "AppGuardContainerId", + "Type": "string" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "CreatedProcessSessionId", + "Type": "long" + }, + { + "Name": "DeviceId", + "Type": "string" + }, + { + "Name": "DeviceName", + "Type": "string" + }, + { + "Name": "FileName", + "Type": "string" + }, + { + "Name": "FileOriginIP", + "Type": "string" + }, + { + "Name": "FileOriginUrl", + "Type": "string" + }, + { + "Name": "FileSize", + "Type": "long" + }, + { + "Name": "FolderPath", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountDomain", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountName", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountObjectId", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountSid", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountUpn", + "Type": "string" + }, + { + "Name": "InitiatingProcessCommandLine", + "Type": "string" + }, + { + "Name": "InitiatingProcessCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessFileSize", + "Type": "long" + }, + { + "Name": "InitiatingProcessFolderPath", + "Type": "string" + }, + { + "Name": "InitiatingProcessId", + "Type": "long" + }, + { + "Name": "InitiatingProcessLogonId", + "Type": "long" + }, + { + "Name": "InitiatingProcessMD5", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessParentFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentId", + "Type": "long" + }, + { + "Name": "InitiatingProcessRemoteSessionDeviceName", + "Type": "string" + }, + { + "Name": "InitiatingProcessRemoteSessionIP", + "Type": "string" + }, + { + "Name": "InitiatingProcessSessionId", + "Type": "long" + }, + { + "Name": "InitiatingProcessSHA1", + "Type": "string" + }, + { + "Name": "InitiatingProcessSHA256", + "Type": "string" + }, + { + "Name": "InitiatingProcessUniqueId", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoCompanyName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoFileDescription", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoInternalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoOriginalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductVersion", + "Type": "string" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "IsInitiatingProcessRemoteSession", + "Type": "bool" + }, + { + "Name": "IsProcessRemoteSession", + "Type": "bool" + }, + { + "Name": "LocalIP", + "Type": "string" + }, + { + "Name": "LocalPort", + "Type": "int" + }, + { + "Name": "LogonId", + "Type": "long" + }, + { + "Name": "MachineGroup", + "Type": "string" + }, + { + "Name": "MD5", + "Type": "string" + }, + { + "Name": "ProcessCommandLine", + "Type": "string" + }, + { + "Name": "ProcessCreationTime", + "Type": "datetime" + }, + { + "Name": "ProcessId", + "Type": "long" + }, + { + "Name": "ProcessRemoteSessionDeviceName", + "Type": "string" + }, + { + "Name": "ProcessRemoteSessionIP", + "Type": "string" + }, + { + "Name": "ProcessTokenElevation", + "Type": "string" + }, + { + "Name": "RegistryKey", + "Type": "string" + }, + { + "Name": "RegistryValueData", + "Type": "string" + }, + { + "Name": "RegistryValueName", + "Type": "string" + }, + { + "Name": "RemoteDeviceName", + "Type": "string" + }, + { + "Name": "RemoteIP", + "Type": "string" + }, + { + "Name": "RemotePort", + "Type": "int" + }, + { + "Name": "RemoteUrl", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "long" + }, + { + "Name": "SHA1", + "Type": "string" + }, + { + "Name": "SHA256", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/DeviceFileCertificateInfo.json b/.script/tests/KqlvalidationsTests/CustomTables/DeviceFileCertificateInfo.json new file mode 100644 index 00000000000..cbfecc66258 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/DeviceFileCertificateInfo.json @@ -0,0 +1,105 @@ +{ + "Name": "DeviceFileCertificateInfo", + "Properties": [ + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "CertificateCountersignatureTime", + "Type": "datetime" + }, + { + "Name": "CertificateCreationTime", + "Type": "datetime" + }, + { + "Name": "CertificateExpirationTime", + "Type": "datetime" + }, + { + "Name": "CertificateSerialNumber", + "Type": "string" + }, + { + "Name": "CrlDistributionPointUrls", + "Type": "string" + }, + { + "Name": "DeviceId", + "Type": "string" + }, + { + "Name": "DeviceName", + "Type": "string" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "IsRootSignerMicrosoft", + "Type": "bool" + }, + { + "Name": "IsSigned", + "Type": "bool" + }, + { + "Name": "Issuer", + "Type": "string" + }, + { + "Name": "IssuerHash", + "Type": "string" + }, + { + "Name": "IsTrusted", + "Type": "bool" + }, + { + "Name": "MachineGroup", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "long" + }, + { + "Name": "SHA1", + "Type": "string" + }, + { + "Name": "SignatureType", + "Type": "string" + }, + { + "Name": "Signer", + "Type": "string" + }, + { + "Name": "SignerHash", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/DeviceFileEvents.json b/.script/tests/KqlvalidationsTests/CustomTables/DeviceFileEvents.json new file mode 100644 index 00000000000..4df77976dac --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/DeviceFileEvents.json @@ -0,0 +1,265 @@ +{ + "Name": "DeviceFileEvents", + "Properties": [ + { + "Name": "ActionType", + "Type": "string" + }, + { + "Name": "AdditionalFields", + "Type": "dynamic" + }, + { + "Name": "AppGuardContainerId", + "Type": "string" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "DeviceId", + "Type": "string" + }, + { + "Name": "DeviceName", + "Type": "string" + }, + { + "Name": "FileName", + "Type": "string" + }, + { + "Name": "FileOriginIP", + "Type": "string" + }, + { + "Name": "FileOriginReferrerUrl", + "Type": "string" + }, + { + "Name": "FileOriginUrl", + "Type": "string" + }, + { + "Name": "FileSize", + "Type": "long" + }, + { + "Name": "FolderPath", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountDomain", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountName", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountObjectId", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountSid", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountUpn", + "Type": "string" + }, + { + "Name": "InitiatingProcessCommandLine", + "Type": "string" + }, + { + "Name": "InitiatingProcessCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessFileSize", + "Type": "long" + }, + { + "Name": "InitiatingProcessFolderPath", + "Type": "string" + }, + { + "Name": "InitiatingProcessId", + "Type": "long" + }, + { + "Name": "InitiatingProcessIntegrityLevel", + "Type": "string" + }, + { + "Name": "InitiatingProcessMD5", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessParentFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentId", + "Type": "long" + }, + { + "Name": "InitiatingProcessRemoteSessionDeviceName", + "Type": "string" + }, + { + "Name": "InitiatingProcessRemoteSessionIP", + "Type": "string" + }, + { + "Name": "InitiatingProcessSessionId", + "Type": "long" + }, + { + "Name": "InitiatingProcessSHA1", + "Type": "string" + }, + { + "Name": "InitiatingProcessSHA256", + "Type": "string" + }, + { + "Name": "InitiatingProcessTokenElevation", + "Type": "string" + }, + { + "Name": "InitiatingProcessUniqueId", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoCompanyName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoFileDescription", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoInternalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoOriginalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductVersion", + "Type": "string" + }, + { + "Name": "IsAzureInfoProtectionApplied", + "Type": "bool" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "IsInitiatingProcessRemoteSession", + "Type": "bool" + }, + { + "Name": "MachineGroup", + "Type": "string" + }, + { + "Name": "MD5", + "Type": "string" + }, + { + "Name": "PreviousFileName", + "Type": "string" + }, + { + "Name": "PreviousFolderPath", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "long" + }, + { + "Name": "RequestAccountDomain", + "Type": "string" + }, + { + "Name": "RequestAccountName", + "Type": "string" + }, + { + "Name": "RequestAccountSid", + "Type": "string" + }, + { + "Name": "RequestProtocol", + "Type": "string" + }, + { + "Name": "RequestSourceIP", + "Type": "string" + }, + { + "Name": "RequestSourcePort", + "Type": "int" + }, + { + "Name": "SensitivityLabel", + "Type": "string" + }, + { + "Name": "SensitivitySubLabel", + "Type": "string" + }, + { + "Name": "SHA1", + "Type": "string" + }, + { + "Name": "SHA256", + "Type": "string" + }, + { + "Name": "ShareName", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/DeviceImageLoadEvents.json b/.script/tests/KqlvalidationsTests/CustomTables/DeviceImageLoadEvents.json new file mode 100644 index 00000000000..aca01f239a3 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/DeviceImageLoadEvents.json @@ -0,0 +1,201 @@ +{ + "Name": "DeviceImageLoadEvents", + "Properties": [ + { + "Name": "ActionType", + "Type": "string" + }, + { + "Name": "AppGuardContainerId", + "Type": "string" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "DeviceId", + "Type": "string" + }, + { + "Name": "DeviceName", + "Type": "string" + }, + { + "Name": "FileName", + "Type": "string" + }, + { + "Name": "FileSize", + "Type": "long" + }, + { + "Name": "FolderPath", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountDomain", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountName", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountObjectId", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountSid", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountUpn", + "Type": "string" + }, + { + "Name": "InitiatingProcessCommandLine", + "Type": "string" + }, + { + "Name": "InitiatingProcessCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessFileSize", + "Type": "long" + }, + { + "Name": "InitiatingProcessFolderPath", + "Type": "string" + }, + { + "Name": "InitiatingProcessId", + "Type": "long" + }, + { + "Name": "InitiatingProcessIntegrityLevel", + "Type": "string" + }, + { + "Name": "InitiatingProcessMD5", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessParentFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentId", + "Type": "long" + }, + { + "Name": "InitiatingProcessRemoteSessionDeviceName", + "Type": "string" + }, + { + "Name": "InitiatingProcessRemoteSessionIP", + "Type": "string" + }, + { + "Name": "InitiatingProcessSessionId", + "Type": "long" + }, + { + "Name": "InitiatingProcessSHA1", + "Type": "string" + }, + { + "Name": "InitiatingProcessSHA256", + "Type": "string" + }, + { + "Name": "InitiatingProcessTokenElevation", + "Type": "string" + }, + { + "Name": "InitiatingProcessUniqueId", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoCompanyName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoFileDescription", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoInternalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoOriginalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductVersion", + "Type": "string" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "IsInitiatingProcessRemoteSession", + "Type": "bool" + }, + { + "Name": "MachineGroup", + "Type": "string" + }, + { + "Name": "MD5", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "long" + }, + { + "Name": "SHA1", + "Type": "string" + }, + { + "Name": "SHA256", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/DeviceInfo.json b/.script/tests/KqlvalidationsTests/CustomTables/DeviceInfo.json new file mode 100644 index 00000000000..1582652ae62 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/DeviceInfo.json @@ -0,0 +1,225 @@ +{ + "Name": "DeviceInfo", + "Properties": [ + { + "Name": "AadDeviceId", + "Type": "string" + }, + { + "Name": "AdditionalFields", + "Type": "dynamic" + }, + { + "Name": "AssetValue", + "Type": "string" + }, + { + "Name": "AwsResourceName", + "Type": "string" + }, + { + "Name": "AzureResourceId", + "Type": "string" + }, + { + "Name": "AzureVmId", + "Type": "string" + }, + { + "Name": "AzureVmSubscriptionId", + "Type": "string" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "ClientVersion", + "Type": "string" + }, + { + "Name": "CloudPlatforms", + "Type": "string" + }, + { + "Name": "ConnectivityType", + "Type": "string" + }, + { + "Name": "DeviceCategory", + "Type": "string" + }, + { + "Name": "DeviceDynamicTags", + "Type": "string" + }, + { + "Name": "DeviceId", + "Type": "string" + }, + { + "Name": "DeviceManualTags", + "Type": "string" + }, + { + "Name": "DeviceName", + "Type": "string" + }, + { + "Name": "DeviceObjectId", + "Type": "string" + }, + { + "Name": "DeviceSubtype", + "Type": "string" + }, + { + "Name": "DeviceType", + "Type": "string" + }, + { + "Name": "ExclusionReason", + "Type": "string" + }, + { + "Name": "ExposureLevel", + "Type": "string" + }, + { + "Name": "GcpFullResourceName", + "Type": "string" + }, + { + "Name": "HardwareUuid", + "Type": "string" + }, + { + "Name": "HostDeviceId", + "Type": "string" + }, + { + "Name": "IsAzureADJoined", + "Type": "bool" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "IsExcluded", + "Type": "bool" + }, + { + "Name": "IsInternetFacing", + "Type": "bool" + }, + { + "Name": "IsTransient", + "Type": "bool" + }, + { + "Name": "JoinType", + "Type": "string" + }, + { + "Name": "LoggedOnUsers", + "Type": "dynamic" + }, + { + "Name": "MachineGroup", + "Type": "string" + }, + { + "Name": "MergedDeviceIds", + "Type": "string" + }, + { + "Name": "MergedToDeviceId", + "Type": "string" + }, + { + "Name": "MitigationStatus", + "Type": "string" + }, + { + "Name": "Model", + "Type": "string" + }, + { + "Name": "OnboardingStatus", + "Type": "string" + }, + { + "Name": "OSArchitecture", + "Type": "string" + }, + { + "Name": "OSBuild", + "Type": "long" + }, + { + "Name": "OsBuildRevision", + "Type": "string" + }, + { + "Name": "OSDistribution", + "Type": "string" + }, + { + "Name": "OSPlatform", + "Type": "string" + }, + { + "Name": "OSVersion", + "Type": "string" + }, + { + "Name": "OSVersionInfo", + "Type": "string" + }, + { + "Name": "PublicIP", + "Type": "string" + }, + { + "Name": "RegistryDeviceTag", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "long" + }, + { + "Name": "RestrictedDeviceSecurityOperations", + "Type": "string" + }, + { + "Name": "SensorHealthState", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Vendor", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/DeviceLogonEvents.json b/.script/tests/KqlvalidationsTests/CustomTables/DeviceLogonEvents.json new file mode 100644 index 00000000000..bcbc12c6410 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/DeviceLogonEvents.json @@ -0,0 +1,229 @@ +{ + "Name": "DeviceLogonEvents", + "Properties": [ + { + "Name": "AccountDomain", + "Type": "string" + }, + { + "Name": "AccountName", + "Type": "string" + }, + { + "Name": "AccountSid", + "Type": "string" + }, + { + "Name": "ActionType", + "Type": "string" + }, + { + "Name": "AdditionalFields", + "Type": "dynamic" + }, + { + "Name": "AppGuardContainerId", + "Type": "string" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "DeviceId", + "Type": "string" + }, + { + "Name": "DeviceName", + "Type": "string" + }, + { + "Name": "FailureReason", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountDomain", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountName", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountObjectId", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountSid", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountUpn", + "Type": "string" + }, + { + "Name": "InitiatingProcessCommandLine", + "Type": "string" + }, + { + "Name": "InitiatingProcessCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessFileSize", + "Type": "long" + }, + { + "Name": "InitiatingProcessFolderPath", + "Type": "string" + }, + { + "Name": "InitiatingProcessId", + "Type": "long" + }, + { + "Name": "InitiatingProcessIntegrityLevel", + "Type": "string" + }, + { + "Name": "InitiatingProcessMD5", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessParentFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentId", + "Type": "long" + }, + { + "Name": "InitiatingProcessRemoteSessionDeviceName", + "Type": "string" + }, + { + "Name": "InitiatingProcessRemoteSessionIP", + "Type": "string" + }, + { + "Name": "InitiatingProcessSessionId", + "Type": "long" + }, + { + "Name": "InitiatingProcessSHA1", + "Type": "string" + }, + { + "Name": "InitiatingProcessSHA256", + "Type": "string" + }, + { + "Name": "InitiatingProcessTokenElevation", + "Type": "string" + }, + { + "Name": "InitiatingProcessUniqueId", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoCompanyName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoFileDescription", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoInternalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoOriginalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductVersion", + "Type": "string" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "IsInitiatingProcessRemoteSession", + "Type": "bool" + }, + { + "Name": "IsLocalAdmin", + "Type": "bool" + }, + { + "Name": "LogonId", + "Type": "long" + }, + { + "Name": "LogonType", + "Type": "string" + }, + { + "Name": "MachineGroup", + "Type": "string" + }, + { + "Name": "Protocol", + "Type": "string" + }, + { + "Name": "RemoteDeviceName", + "Type": "string" + }, + { + "Name": "RemoteIP", + "Type": "string" + }, + { + "Name": "RemoteIPType", + "Type": "string" + }, + { + "Name": "RemotePort", + "Type": "int" + }, + { + "Name": "ReportId", + "Type": "long" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/DeviceNetworkEvents.json b/.script/tests/KqlvalidationsTests/CustomTables/DeviceNetworkEvents.json new file mode 100644 index 00000000000..897e4f303f3 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/DeviceNetworkEvents.json @@ -0,0 +1,213 @@ +{ + "Name": "DeviceNetworkEvents", + "Properties": [ + { + "Name": "ActionType", + "Type": "string" + }, + { + "Name": "AdditionalFields", + "Type": "dynamic" + }, + { + "Name": "AppGuardContainerId", + "Type": "string" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "DeviceId", + "Type": "string" + }, + { + "Name": "DeviceName", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountDomain", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountName", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountObjectId", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountSid", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountUpn", + "Type": "string" + }, + { + "Name": "InitiatingProcessCommandLine", + "Type": "string" + }, + { + "Name": "InitiatingProcessCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessFileSize", + "Type": "long" + }, + { + "Name": "InitiatingProcessFolderPath", + "Type": "string" + }, + { + "Name": "InitiatingProcessId", + "Type": "long" + }, + { + "Name": "InitiatingProcessIntegrityLevel", + "Type": "string" + }, + { + "Name": "InitiatingProcessMD5", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessParentFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentId", + "Type": "long" + }, + { + "Name": "InitiatingProcessRemoteSessionDeviceName", + "Type": "string" + }, + { + "Name": "InitiatingProcessRemoteSessionIP", + "Type": "string" + }, + { + "Name": "InitiatingProcessSessionId", + "Type": "long" + }, + { + "Name": "InitiatingProcessSHA1", + "Type": "string" + }, + { + "Name": "InitiatingProcessSHA256", + "Type": "string" + }, + { + "Name": "InitiatingProcessTokenElevation", + "Type": "string" + }, + { + "Name": "InitiatingProcessUniqueId", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoCompanyName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoFileDescription", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoInternalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoOriginalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductVersion", + "Type": "string" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "IsInitiatingProcessRemoteSession", + "Type": "bool" + }, + { + "Name": "LocalIP", + "Type": "string" + }, + { + "Name": "LocalIPType", + "Type": "string" + }, + { + "Name": "LocalPort", + "Type": "int" + }, + { + "Name": "MachineGroup", + "Type": "string" + }, + { + "Name": "Protocol", + "Type": "string" + }, + { + "Name": "RemoteIP", + "Type": "string" + }, + { + "Name": "RemoteIPType", + "Type": "string" + }, + { + "Name": "RemotePort", + "Type": "int" + }, + { + "Name": "RemoteUrl", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "long" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/DeviceNetworkInfo.json b/.script/tests/KqlvalidationsTests/CustomTables/DeviceNetworkInfo.json new file mode 100644 index 00000000000..4c9d244ca76 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/DeviceNetworkInfo.json @@ -0,0 +1,97 @@ +{ + "Name": "DeviceNetworkInfo", + "Properties": [ + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "ConnectedNetworks", + "Type": "dynamic" + }, + { + "Name": "DefaultGateways", + "Type": "dynamic" + }, + { + "Name": "DeviceId", + "Type": "string" + }, + { + "Name": "DeviceName", + "Type": "string" + }, + { + "Name": "DnsAddresses", + "Type": "dynamic" + }, + { + "Name": "IPAddresses", + "Type": "dynamic" + }, + { + "Name": "IPv4Dhcp", + "Type": "string" + }, + { + "Name": "IPv6Dhcp", + "Type": "string" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "MacAddress", + "Type": "string" + }, + { + "Name": "MachineGroup", + "Type": "string" + }, + { + "Name": "NetworkAdapterName", + "Type": "string" + }, + { + "Name": "NetworkAdapterStatus", + "Type": "string" + }, + { + "Name": "NetworkAdapterType", + "Type": "string" + }, + { + "Name": "NetworkAdapterVendor", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "long" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "TunnelType", + "Type": "string" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/DeviceProcessEvents.json b/.script/tests/KqlvalidationsTests/CustomTables/DeviceProcessEvents.json new file mode 100644 index 00000000000..49595d71128 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/DeviceProcessEvents.json @@ -0,0 +1,305 @@ +{ + "Name": "DeviceProcessEvents", + "Properties": [ + { + "Name": "AccountDomain", + "Type": "string" + }, + { + "Name": "AccountName", + "Type": "string" + }, + { + "Name": "AccountObjectId", + "Type": "string" + }, + { + "Name": "AccountSid", + "Type": "string" + }, + { + "Name": "AccountUpn", + "Type": "string" + }, + { + "Name": "ActionType", + "Type": "string" + }, + { + "Name": "AdditionalFields", + "Type": "dynamic" + }, + { + "Name": "AppGuardContainerId", + "Type": "string" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "CreatedProcessSessionId", + "Type": "long" + }, + { + "Name": "DeviceId", + "Type": "string" + }, + { + "Name": "DeviceName", + "Type": "string" + }, + { + "Name": "FileName", + "Type": "string" + }, + { + "Name": "FileSize", + "Type": "long" + }, + { + "Name": "FolderPath", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountDomain", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountName", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountObjectId", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountSid", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountUpn", + "Type": "string" + }, + { + "Name": "InitiatingProcessCommandLine", + "Type": "string" + }, + { + "Name": "InitiatingProcessCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessFileSize", + "Type": "long" + }, + { + "Name": "InitiatingProcessFolderPath", + "Type": "string" + }, + { + "Name": "InitiatingProcessId", + "Type": "long" + }, + { + "Name": "InitiatingProcessIntegrityLevel", + "Type": "string" + }, + { + "Name": "InitiatingProcessLogonId", + "Type": "long" + }, + { + "Name": "InitiatingProcessMD5", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessParentFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentId", + "Type": "long" + }, + { + "Name": "InitiatingProcessRemoteSessionDeviceName", + "Type": "string" + }, + { + "Name": "InitiatingProcessRemoteSessionIP", + "Type": "string" + }, + { + "Name": "InitiatingProcessSessionId", + "Type": "long" + }, + { + "Name": "InitiatingProcessSHA1", + "Type": "string" + }, + { + "Name": "InitiatingProcessSHA256", + "Type": "string" + }, + { + "Name": "InitiatingProcessSignatureStatus", + "Type": "string" + }, + { + "Name": "InitiatingProcessSignerType", + "Type": "string" + }, + { + "Name": "InitiatingProcessTokenElevation", + "Type": "string" + }, + { + "Name": "InitiatingProcessUniqueId", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoCompanyName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoFileDescription", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoInternalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoOriginalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductVersion", + "Type": "string" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "IsInitiatingProcessRemoteSession", + "Type": "bool" + }, + { + "Name": "IsProcessRemoteSession", + "Type": "bool" + }, + { + "Name": "LogonId", + "Type": "long" + }, + { + "Name": "MachineGroup", + "Type": "string" + }, + { + "Name": "MD5", + "Type": "string" + }, + { + "Name": "ProcessCommandLine", + "Type": "string" + }, + { + "Name": "ProcessCreationTime", + "Type": "datetime" + }, + { + "Name": "ProcessId", + "Type": "long" + }, + { + "Name": "ProcessIntegrityLevel", + "Type": "string" + }, + { + "Name": "ProcessRemoteSessionDeviceName", + "Type": "string" + }, + { + "Name": "ProcessRemoteSessionIP", + "Type": "string" + }, + { + "Name": "ProcessTokenElevation", + "Type": "string" + }, + { + "Name": "ProcessUniqueId", + "Type": "string" + }, + { + "Name": "ProcessVersionInfoCompanyName", + "Type": "string" + }, + { + "Name": "ProcessVersionInfoFileDescription", + "Type": "string" + }, + { + "Name": "ProcessVersionInfoInternalFileName", + "Type": "string" + }, + { + "Name": "ProcessVersionInfoOriginalFileName", + "Type": "string" + }, + { + "Name": "ProcessVersionInfoProductName", + "Type": "string" + }, + { + "Name": "ProcessVersionInfoProductVersion", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "long" + }, + { + "Name": "SHA1", + "Type": "string" + }, + { + "Name": "SHA256", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/DeviceRegistryEvents.json b/.script/tests/KqlvalidationsTests/CustomTables/DeviceRegistryEvents.json new file mode 100644 index 00000000000..66c8547ded2 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/DeviceRegistryEvents.json @@ -0,0 +1,205 @@ +{ + "Name": "DeviceRegistryEvents", + "Properties": [ + { + "Name": "ActionType", + "Type": "string" + }, + { + "Name": "AppGuardContainerId", + "Type": "string" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "DeviceId", + "Type": "string" + }, + { + "Name": "DeviceName", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountDomain", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountName", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountObjectId", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountSid", + "Type": "string" + }, + { + "Name": "InitiatingProcessAccountUpn", + "Type": "string" + }, + { + "Name": "InitiatingProcessCommandLine", + "Type": "string" + }, + { + "Name": "InitiatingProcessCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessFileSize", + "Type": "long" + }, + { + "Name": "InitiatingProcessFolderPath", + "Type": "string" + }, + { + "Name": "InitiatingProcessId", + "Type": "long" + }, + { + "Name": "InitiatingProcessIntegrityLevel", + "Type": "string" + }, + { + "Name": "InitiatingProcessMD5", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentCreationTime", + "Type": "datetime" + }, + { + "Name": "InitiatingProcessParentFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessParentId", + "Type": "long" + }, + { + "Name": "InitiatingProcessRemoteSessionDeviceName", + "Type": "string" + }, + { + "Name": "InitiatingProcessRemoteSessionIP", + "Type": "string" + }, + { + "Name": "InitiatingProcessSessionId", + "Type": "long" + }, + { + "Name": "InitiatingProcessSHA1", + "Type": "string" + }, + { + "Name": "InitiatingProcessSHA256", + "Type": "string" + }, + { + "Name": "InitiatingProcessTokenElevation", + "Type": "string" + }, + { + "Name": "InitiatingProcessUniqueId", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoCompanyName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoFileDescription", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoInternalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoOriginalFileName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductName", + "Type": "string" + }, + { + "Name": "InitiatingProcessVersionInfoProductVersion", + "Type": "string" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "IsInitiatingProcessRemoteSession", + "Type": "bool" + }, + { + "Name": "MachineGroup", + "Type": "string" + }, + { + "Name": "PreviousRegistryKey", + "Type": "string" + }, + { + "Name": "PreviousRegistryValueData", + "Type": "string" + }, + { + "Name": "PreviousRegistryValueName", + "Type": "string" + }, + { + "Name": "RegistryKey", + "Type": "string" + }, + { + "Name": "RegistryValueData", + "Type": "string" + }, + { + "Name": "RegistryValueName", + "Type": "string" + }, + { + "Name": "RegistryValueType", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "long" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/EmailAttachmentInfo.json b/.script/tests/KqlvalidationsTests/CustomTables/EmailAttachmentInfo.json new file mode 100644 index 00000000000..b9c008b7d49 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/EmailAttachmentInfo.json @@ -0,0 +1,93 @@ +{ + "Name": "EmailAttachmentInfo", + "Properties": [ + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "DetectionMethods", + "Type": "string" + }, + { + "Name": "FileExtension", + "Type": "string" + }, + { + "Name": "FileName", + "Type": "string" + }, + { + "Name": "FileSize", + "Type": "long" + }, + { + "Name": "FileType", + "Type": "string" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "NetworkMessageId", + "Type": "string" + }, + { + "Name": "RecipientEmailAddress", + "Type": "string" + }, + { + "Name": "RecipientObjectId", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "string" + }, + { + "Name": "SenderDisplayName", + "Type": "string" + }, + { + "Name": "SenderFromAddress", + "Type": "string" + }, + { + "Name": "SenderObjectId", + "Type": "string" + }, + { + "Name": "SHA256", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "ThreatNames", + "Type": "string" + }, + { + "Name": "ThreatTypes", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/EmailEvents.json b/.script/tests/KqlvalidationsTests/CustomTables/EmailEvents.json new file mode 100644 index 00000000000..8074d2cfc18 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/EmailEvents.json @@ -0,0 +1,233 @@ +{ + "Name": "EmailEvents", + "Properties": [ + { + "Name": "AdditionalFields", + "Type": "dynamic" + }, + { + "Name": "AttachmentCount", + "Type": "int" + }, + { + "Name": "AuthenticationDetails", + "Type": "string" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "BulkComplaintLevel", + "Type": "int" + }, + { + "Name": "Cc", + "Type": "dynamic" + }, + { + "Name": "ConfidenceLevel", + "Type": "string" + }, + { + "Name": "Connectors", + "Type": "string" + }, + { + "Name": "Context", + "Type": "string" + }, + { + "Name": "DeliveryAction", + "Type": "string" + }, + { + "Name": "DeliveryLocation", + "Type": "string" + }, + { + "Name": "DetectionMethods", + "Type": "string" + }, + { + "Name": "DistributionList", + "Type": "string" + }, + { + "Name": "EmailAction", + "Type": "string" + }, + { + "Name": "EmailActionPolicy", + "Type": "string" + }, + { + "Name": "EmailActionPolicyGuid", + "Type": "string" + }, + { + "Name": "EmailClusterId", + "Type": "long" + }, + { + "Name": "EmailDirection", + "Type": "string" + }, + { + "Name": "EmailLanguage", + "Type": "string" + }, + { + "Name": "EmailSize", + "Type": "int" + }, + { + "Name": "ExchangeTransportRule", + "Type": "string" + }, + { + "Name": "ForwardingInformation", + "Type": "string" + }, + { + "Name": "InternetMessageId", + "Type": "string" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "IsFirstContact", + "Type": "bool" + }, + { + "Name": "LastEventExecutionTime", + "Type": "datetime" + }, + { + "Name": "LatestDeliveryAction", + "Type": "string" + }, + { + "Name": "LatestDeliveryLocation", + "Type": "string" + }, + { + "Name": "NetworkMessageId", + "Type": "string" + }, + { + "Name": "OrgLevelAction", + "Type": "string" + }, + { + "Name": "OrgLevelPolicy", + "Type": "string" + }, + { + "Name": "RecipientDomain", + "Type": "string" + }, + { + "Name": "RecipientEmailAddress", + "Type": "string" + }, + { + "Name": "RecipientObjectId", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "string" + }, + { + "Name": "SenderDisplayName", + "Type": "string" + }, + { + "Name": "SenderFromAddress", + "Type": "string" + }, + { + "Name": "SenderFromDomain", + "Type": "string" + }, + { + "Name": "SenderIPv4", + "Type": "string" + }, + { + "Name": "SenderIPv6", + "Type": "string" + }, + { + "Name": "SenderMailFromAddress", + "Type": "string" + }, + { + "Name": "SenderMailFromDomain", + "Type": "string" + }, + { + "Name": "SenderObjectId", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "Subject", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "ThreatClassification", + "Type": "string" + }, + { + "Name": "ThreatNames", + "Type": "string" + }, + { + "Name": "ThreatTypes", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "To", + "Type": "dynamic" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "UrlCount", + "Type": "int" + }, + { + "Name": "UserLevelAction", + "Type": "string" + }, + { + "Name": "UserLevelPolicy", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + }, + { + "Name": "OrganizationId", + "Type": "string" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/EmailPostDeliveryEvents.json b/.script/tests/KqlvalidationsTests/CustomTables/EmailPostDeliveryEvents.json new file mode 100644 index 00000000000..cd27c866438 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/EmailPostDeliveryEvents.json @@ -0,0 +1,85 @@ +{ + "Name": "EmailPostDeliveryEvents", + "Properties": [ + { + "Name": "Action", + "Type": "string" + }, + { + "Name": "ActionResult", + "Type": "string" + }, + { + "Name": "ActionTrigger", + "Type": "string" + }, + { + "Name": "ActionType", + "Type": "string" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "DeliveryLocation", + "Type": "string" + }, + { + "Name": "DetectionMethods", + "Type": "string" + }, + { + "Name": "EmailDirection", + "Type": "string" + }, + { + "Name": "InternetMessageId", + "Type": "string" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "NetworkMessageId", + "Type": "string" + }, + { + "Name": "RecipientEmailAddress", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "string" + }, + { + "Name": "SenderFromAddress", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "ThreatTypes", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/EmailUrlInfo.json b/.script/tests/KqlvalidationsTests/CustomTables/EmailUrlInfo.json new file mode 100644 index 00000000000..62a992ec326 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/CustomTables/EmailUrlInfo.json @@ -0,0 +1,61 @@ +{ + "Name": "EmailUrlInfo", + "Properties": [ + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "NetworkMessageId", + "Type": "string" + }, + { + "Name": "ReportId", + "Type": "string" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "Url", + "Type": "string" + }, + { + "Name": "UrlChainId", + "Type": "string" + }, + { + "Name": "UrlChainPosition", + "Type": "int" + }, + { + "Name": "UrlDomain", + "Type": "string" + }, + { + "Name": "UrlLocation", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/CustomTables/IdentityInfo.json b/.script/tests/KqlvalidationsTests/CustomTables/IdentityInfo.json index 8cb79fd645d..1994f25c9a1 100644 --- a/.script/tests/KqlvalidationsTests/CustomTables/IdentityInfo.json +++ b/.script/tests/KqlvalidationsTests/CustomTables/IdentityInfo.json @@ -1,201 +1,237 @@ { - "Name": "IdentityInfo", - "Properties": [ - { - "Name": "Timestamp", - "Type": "DateTime" - }, - { - "Name": "ReportId", - "Type": "String" - }, - { - "Name": "AccountObjectId", - "Type": "String" - }, - { - "Name": "AccountUpn", - "Type": "String" - }, - { - "Name": "OnPremSid", - "Type": "String" - }, - { - "Name": "AccountDisplayName", - "Type": "String" - }, - { - "Name": "AccountName", - "Type": "String" - }, - { - "Name": "AccountDomain", - "Type": "String" - }, - { - "Name": "CriticalityLevel", - "Type": "Int" - }, - { - "Name": "Type", - "Type": "String" - }, - { - "Name": "DistinguishedName", - "Type": "String" - }, - { - "Name": "CloudSid", - "Type": "String" - }, - { - "Name": "GivenName", - "Type": "String" - }, - { - "Name": "Surname", - "Type": "String" - }, - { - "Name": "Department", - "Type": "String" - }, - { - "Name": "JobTitle", - "Type": "String" - }, - { - "Name": "EmailAddress", - "Type": "String" - }, - { - "Name": "SipProxyAddress", - "Type": "String" - }, - { - "Name": "Address", - "Type": "String" - }, - { - "Name": "City", - "Type": "String" - }, - { - "Name": "Country", - "Type": "String" - }, - { - "Name": "IsAccountEnabled", - "Type": "Bool" - }, - { - "Name": "Manager", - "Type": "String" - }, - { - "Name": "Phone", - "Type": "String" - }, - { - "Name": "CreatedDateTime", - "Type": "DateTime" - }, - { - "Name": "ChangeSource", - "Type": "String" - }, - { - "Name": "BlastRadius", - "Type": "String" - }, - { - "Name": "CompanyName", - "Type": "String" - }, - { - "Name": "DeletedDateTime", - "Type": "DateTime" - }, - { - "Name": "EmployeeId", - "Type": "String" - }, - { - "Name": "OtherMailAddresses", - "Type": "Dynamic" - }, - { - "Name": "RiskLevel", - "Type": "String" - }, - { - "Name": "RiskLevelDetails", - "Type": "String" - }, - { - "Name": "State", - "Type": "String" - }, - { - "Name": "Tags", - "Type": "Dynamic" - }, - { - "Name": "AssignedRoles", - "Type": "Dynamic" - }, - { - "Name": "PrivilegedEntraPimRoles", - "Type": "Dynamic" - }, - { - "Name": "TenantId", - "Type": "String" - }, - { - "Name": "SourceSystem", - "Type": "String" - }, - { - "Name": "OnPremObjectId", - "Type": "String" - }, - { - "Name": "TenantMembershipType", - "Type": "String" - }, - { - "Name": "RiskStatus", - "Type": "String" - }, - { - "Name": "UserAccountControl", - "Type": "String" - }, - { - "Name": "IdentityEnvironment", - "Type": "String" - }, - { - "Name": "SourceProviders", - "Type": "Dynamic" - }, - { - "Name": "GroupMembership", - "Type": "Dynamic" - }, - { - "Name": "AccountUPN", - "Type": "String" - }, - { - "Name": "TimeGenerated", - "Type": "DateTime" - }, - { - "Name": "UserType", - "Type": "String" - } - ] -} \ No newline at end of file + "Name": "IdentityInfo", + "Properties": [ + { + "Name": "AccountCloudSID", + "Type": "string" + }, + { + "Name": "AccountCreationTime", + "Type": "datetime" + }, + { + "Name": "AccountDisplayName", + "Type": "string" + }, + { + "Name": "AccountDomain", + "Type": "string" + }, + { + "Name": "AccountName", + "Type": "string" + }, + { + "Name": "AccountObjectId", + "Type": "string" + }, + { + "Name": "AccountSID", + "Type": "string" + }, + { + "Name": "AccountTenantId", + "Type": "string" + }, + { + "Name": "AccountUPN", + "Type": "string" + }, + { + "Name": "AdditionalMailAddresses", + "Type": "dynamic" + }, + { + "Name": "Applications", + "Type": "string" + }, + { + "Name": "AssignedRoles", + "Type": "dynamic" + }, + { + "Name": "_BilledSize", + "Type": "real" + }, + { + "Name": "BlastRadius", + "Type": "string" + }, + { + "Name": "ChangeSource", + "Type": "string" + }, + { + "Name": "City", + "Type": "string" + }, + { + "Name": "CompanyName", + "Type": "string" + }, + { + "Name": "Country", + "Type": "string" + }, + { + "Name": "DeletedDateTime", + "Type": "datetime" + }, + { + "Name": "Department", + "Type": "string" + }, + { + "Name": "EmployeeId", + "Type": "string" + }, + { + "Name": "EntityRiskScore", + "Type": "dynamic" + }, + { + "Name": "ExtensionProperty", + "Type": "dynamic" + }, + { + "Name": "GivenName", + "Type": "string" + }, + { + "Name": "GroupMembership", + "Type": "dynamic" + }, + { + "Name": "InvestigationPriority", + "Type": "int" + }, + { + "Name": "InvestigationPriorityPercentile", + "Type": "int" + }, + { + "Name": "IsAccountEnabled", + "Type": "bool" + }, + { + "Name": "_IsBillable", + "Type": "string" + }, + { + "Name": "IsMFARegistered", + "Type": "bool" + }, + { + "Name": "IsServiceAccount", + "Type": "bool" + }, + { + "Name": "JobTitle", + "Type": "string" + }, + { + "Name": "LastSeenDate", + "Type": "datetime" + }, + { + "Name": "MailAddress", + "Type": "string" + }, + { + "Name": "Manager", + "Type": "string" + }, + { + "Name": "OnPremisesDistinguishedName", + "Type": "string" + }, + { + "Name": "OnPremisesExtensionAttributes", + "Type": "string" + }, + { + "Name": "Phone", + "Type": "string" + }, + { + "Name": "RelatedAccounts", + "Type": "dynamic" + }, + { + "Name": "RiskLevel", + "Type": "string" + }, + { + "Name": "RiskLevelDetails", + "Type": "string" + }, + { + "Name": "RiskState", + "Type": "string" + }, + { + "Name": "SAMAccountName", + "Type": "string" + }, + { + "Name": "ServicePrincipals", + "Type": "dynamic" + }, + { + "Name": "SourceSystem", + "Type": "string" + }, + { + "Name": "State", + "Type": "string" + }, + { + "Name": "StreetAddress", + "Type": "string" + }, + { + "Name": "Surname", + "Type": "string" + }, + { + "Name": "Tags", + "Type": "string" + }, + { + "Name": "TenantId", + "Type": "string" + }, + { + "Name": "TimeGenerated", + "Type": "datetime" + }, + { + "Name": "Type", + "Type": "string" + }, + { + "Name": "UACFlags", + "Type": "string" + }, + { + "Name": "UserAccountControl", + "Type": "dynamic" + }, + { + "Name": "UserState", + "Type": "string" + }, + { + "Name": "UserStateChangedOn", + "Type": "datetime" + }, + { + "Name": "UserType", + "Type": "string" + }, + { + "Name": "Timestamp", + "Type": "datetime" + } + ] +} diff --git a/.script/tests/KqlvalidationsTests/KqlValidationTests.cs b/.script/tests/KqlvalidationsTests/KqlValidationTests.cs index 6e25fe87667..db5dfb796eb 100644 --- a/.script/tests/KqlvalidationsTests/KqlValidationTests.cs +++ b/.script/tests/KqlvalidationsTests/KqlValidationTests.cs @@ -2,6 +2,8 @@ using Microsoft.Azure.Sentinel.KustoServices.Contract; using System.IO; using System.Linq; +using System.Security.Cryptography; +using System.Text; using System.Text.RegularExpressions; using Xunit; using YamlDotNet.Serialization; @@ -10,6 +12,7 @@ using Kqlvalidations.Tests.TableSchemasLoaders; using System; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; using Octokit; namespace Kqlvalidations.Tests @@ -90,6 +93,7 @@ public void Validate_HuntingQueries_HaveValidKql(string fileName, string encoded Assert.True(true); return; } + var decodedPath = Utils.DecodeBase64(encodedFilePath); var res = ReadAndDeserializeYaml(encodedFilePath); var id = (string)res["id"]; @@ -100,7 +104,7 @@ public void Validate_HuntingQueries_HaveValidKql(string fileName, string encoded } var queryStr = (string)res["query"]; - ValidateKql(id, queryStr); + ValidateKql(id, queryStr, true, decodedPath); ValidateKqlForBestPractices(queryStr, fileName); ValidateKqlForLatestTIData(id, queryStr); } @@ -115,6 +119,7 @@ public void Validate_DetectionQueries_HaveValidKql(string fileName, string encod Assert.True(true); return; } + var decodedPath = Utils.DecodeBase64(encodedFilePath); var res = ReadAndDeserializeYaml(encodedFilePath); var id = (string)res["id"]; @@ -125,11 +130,38 @@ public void Validate_DetectionQueries_HaveValidKql(string fileName, string encod } var queryStr = (string)res["query"]; - ValidateKql(id, queryStr); + ValidateKql(id, queryStr, true, decodedPath); ValidateKqlForBestPractices(queryStr, fileName); ValidateKqlForLatestTIData(id, queryStr); } + [Theory] + [ClassData(typeof(XdrDetectionsYamlFilesTestData))] + public void Validate_XdrDetectionQueries_HaveValidKql(string detectionYamlFileName, string encodedFilePath) + { + if (detectionYamlFileName == "NoFile.yaml") + { + Assert.True(true); + return; + } + + var decodedPath = Utils.DecodeBase64(encodedFilePath); + var res = ReadAndDeserializeYaml(encodedFilePath); + var id = ExtractXdrRuleId(res, detectionYamlFileName); + var originalRuleId = ExtractXdrOriginalRuleId(res, id); + + if (ShouldSkipTemplateValidation(id) || (!string.IsNullOrWhiteSpace(originalRuleId) && ShouldSkipTemplateValidation(originalRuleId))) + { + return; + } + + var queryStr = ExtractXdrQueryText(res); + Assert.False(string.IsNullOrWhiteSpace(queryStr), $"XDR Detection '{detectionYamlFileName}' (id: {id}) is missing properties.queryCondition.queryText."); + ValidateKql(id, queryStr, true, decodedPath); + ValidateKqlForBestPractices(queryStr, detectionYamlFileName); + ValidateKqlForLatestTIData(id, queryStr); + } + /// /// Validates the KQL for the best practices /// @@ -438,7 +470,7 @@ private bool ValidateKqlForLatestTI(string queryStr) return true; } - private void ValidateKql(string id, string queryStr, bool ignoreNoTabularExpressionError = true) + private void ValidateKql(string id, string queryStr, bool ignoreNoTabularExpressionError = true, string filePath = null) { // The KQL validation ignores no tabular expression error. For instance, "let x = table;" is considered a valid query. @@ -449,18 +481,23 @@ private void ValidateKql(string id, string queryStr, bool ignoreNoTabularExpress } var validationResult = _queryValidator.ValidateSyntax(queryStr); + var listOfDiagnostics = validationResult.Diagnostics ?? Enumerable.Empty(); + + bool isRuntimeQualified = IsRuntimeQualifiedQuery(filePath, id, queryStr); + var filteredDiagnostics = (from p in listOfDiagnostics + where !p.Message.Contains("_GetWatchlist") //We do not validate the getWatchList, since the result schema is not known + && !p.Message.Contains("let forwarder_host_names") //We do not validate the static list of hostnames, used for syslog parsers + where !isRuntimeQualified || !IsSemanticSchemaDiagnostic(p) + select p).ToList(); + var firstErrorLocation = (Line: 0, Col: 0); - if (!validationResult.IsValid) + var firstErrorDiag = filteredDiagnostics.FirstOrDefault(d => d.Severity == "Error") ?? filteredDiagnostics.FirstOrDefault(); + if (firstErrorDiag != null) { - firstErrorLocation = GetLocationInQuery(queryStr, validationResult.Diagnostics.First(d => d.Severity == "Error").Start); + firstErrorLocation = GetLocationInQuery(queryStr, firstErrorDiag.Start); } - var listOfDiagnostics = validationResult.Diagnostics; - - bool isQueryValid = !(from p in listOfDiagnostics - where !p.Message.Contains("_GetWatchlist") //We do not validate the getWatchList, since the result schema is not known - || !p.Message.Contains("let forwarder_host_names") //We do not validate the static list of hostnames, used for syslog parsers - select p).Any(); + bool isQueryValid = !filteredDiagnostics.Any(); Assert.True( @@ -468,7 +505,325 @@ private void ValidateKql(string id, string queryStr, bool ignoreNoTabularExpress isQueryValid ? string.Empty : @$"Template Id: {id} is not valid in Line: {firstErrorLocation.Line} col: {firstErrorLocation.Col} - Errors: {validationResult.Diagnostics.Select(d => d.ToString()).ToList().Aggregate((s1, s2) => s1 + "," + s2)}"); + Errors: {filteredDiagnostics.Select(d => d.ToString()).ToList().Aggregate((s1, s2) => s1 + "," + s2)}"); + } + + private static bool IsSemanticSchemaDiagnostic(Diagnostic diagnostic) + { + if (diagnostic == null) + { + return false; + } + + var code = diagnostic.Code ?? string.Empty; + if (code.StartsWith("SYN", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (code.Equals("KS107", StringComparison.OrdinalIgnoreCase) + || code.Equals("KS141", StringComparison.OrdinalIgnoreCase) + || code.StartsWith("KS", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return false; + } + + public static bool IsRuntimeQualifiedQuery(string filePath, string templateId, string queryStr) + { + if (string.IsNullOrWhiteSpace(filePath) || string.IsNullOrWhiteSpace(queryStr)) + { + return false; + } + + var solutionDir = FindEnclosingSolutionDirectory(filePath); + if (string.IsNullOrWhiteSpace(solutionDir) || !Directory.Exists(solutionDir)) + { + return false; + } + + var candidateEvidenceFiles = new[] + { + Path.Combine(solutionDir, "Evidence", "pr-qualification", "pr-qualification-result.json"), + Path.Combine(solutionDir, "Evidence", "qualification-evidence.json") + }; + + var queryHashes = ComputeCandidateQueryHashes(queryStr); + var normalizedId = (templateId ?? string.Empty).Trim(); + var strippedId = normalizedId.StartsWith("xdr-", StringComparison.OrdinalIgnoreCase) + ? normalizedId.Substring(4) + : normalizedId; + + foreach (var evidencePath in candidateEvidenceFiles) + { + if (!File.Exists(evidencePath)) + { + continue; + } + + try + { + var root = JObject.Parse(File.ReadAllText(evidencePath)); + if (MatchesRuntimeQualifiedEvidence(root, normalizedId, strippedId, queryHashes)) + { + return true; + } + } + catch + { + // Ignore malformed evidence JSON and continue + } + } + + return false; + } + + private static string FindEnclosingSolutionDirectory(string filePath) + { + try + { + var fullPath = Path.GetFullPath(filePath); + var current = Directory.Exists(fullPath) ? new DirectoryInfo(fullPath) : new FileInfo(fullPath).Directory; + while (current != null && current.Parent != null) + { + if (current.Parent.Name.Equals("Solutions", StringComparison.OrdinalIgnoreCase)) + { + return current.FullName; + } + current = current.Parent; + } + } + catch + { + // Ignore path resolution errors + } + return null; + } + + private static HashSet ComputeCandidateQueryHashes(string queryStr) + { + var hashes = new HashSet(StringComparer.OrdinalIgnoreCase); + if (queryStr == null) + { + return hashes; + } + + var lfNormalized = queryStr.Replace("\r\n", "\n"); + var trimmed = lfNormalized.Trim(); + var whitespaceCollapsed = Regex.Replace(trimmed, @"\s+", " "); + + hashes.Add(ComputeSha256Hex(queryStr)); + hashes.Add(ComputeSha256Hex(queryStr.Trim())); + hashes.Add(ComputeSha256Hex(lfNormalized)); + hashes.Add(ComputeSha256Hex(trimmed)); + hashes.Add(ComputeSha256Hex(whitespaceCollapsed)); + return hashes; + } + + private static string ComputeSha256Hex(string input) + { + using (var sha = SHA256.Create()) + { + var bytes = Encoding.UTF8.GetBytes(input ?? string.Empty); + var hashBytes = sha.ComputeHash(bytes); + return BitConverter.ToString(hashBytes).Replace("-", "").ToLowerInvariant(); + } + } + + private static bool MatchesRuntimeQualifiedEvidence(JObject root, string normalizedId, string strippedId, HashSet queryHashes) + { + if (root == null) + { + return false; + } + + var verifiedRulesTokens = root.SelectTokens("$..kqlRuntimeVerifiedRules"); + foreach (var token in verifiedRulesTokens) + { + if (token is JArray arr) + { + foreach (var item in arr) + { + if (item is JValue val && val.Type == JTokenType.String) + { + var strVal = val.Value()?.Trim(); + if (!string.IsNullOrEmpty(strVal) && + (strVal.Equals(normalizedId, StringComparison.OrdinalIgnoreCase) + || strVal.Equals(strippedId, StringComparison.OrdinalIgnoreCase) + || queryHashes.Contains(strVal))) + { + return true; + } + } + else if (item is JObject obj) + { + var status = (string)obj["status"] ?? (string)obj["result"] ?? "PASS"; + if (!status.Equals("PASS", StringComparison.OrdinalIgnoreCase) + && !status.Equals("passed", StringComparison.OrdinalIgnoreCase) + && !status.Equals("true", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var entryHash = (string)obj["querySha256"] ?? (string)obj["queryHash"] ?? (string)obj["sha256"]; + if (!string.IsNullOrWhiteSpace(entryHash)) + { + if (queryHashes.Contains(entryHash.Trim())) + { + return true; + } + continue; + } + + var ruleIds = new[] + { + (string)obj["ruleId"], + (string)obj["id"], + (string)obj["originalRuleId"], + (string)obj["sourceRuleId"] + }; + if (ruleIds.Any(r => !string.IsNullOrWhiteSpace(r) && + (r.Trim().Equals(normalizedId, StringComparison.OrdinalIgnoreCase) + || r.Trim().Equals(strippedId, StringComparison.OrdinalIgnoreCase)))) + { + return true; + } + } + } + } + } + + var targetCheckNames = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "kql-validation", + "alert-parity-live", + "QUAL-RUNTIME-01", + "MIG-Q01" + }; + + foreach (var obj in root.DescendantsAndSelf().OfType()) + { + var checkId = (string)obj["checkId"] ?? (string)obj["id"] ?? (string)obj["name"] ?? (string)obj["stage"]; + if (string.IsNullOrWhiteSpace(checkId) || !targetCheckNames.Contains(checkId.Trim())) + { + continue; + } + + var status = (string)obj["status"] ?? (string)obj["verdict"] ?? (string)obj["result"]; + if (string.IsNullOrWhiteSpace(status) || + (!status.Equals("PASS", StringComparison.OrdinalIgnoreCase) && !status.Equals("passed", StringComparison.OrdinalIgnoreCase))) + { + continue; + } + + foreach (var prop in obj.DescendantsAndSelf().OfType()) + { + if ((prop.Name.Equals("querySha256", StringComparison.OrdinalIgnoreCase) + || prop.Name.Equals("queryHash", StringComparison.OrdinalIgnoreCase) + || prop.Name.Equals("convertedQuerySha256", StringComparison.OrdinalIgnoreCase) + || prop.Name.Equals("sourceQuerySha256", StringComparison.OrdinalIgnoreCase)) + && prop.Value.Type == JTokenType.String) + { + var h = prop.Value.Value()?.Trim(); + if (!string.IsNullOrEmpty(h) && queryHashes.Contains(h)) + { + return true; + } + } + } + } + + return false; + } + + private static string ExtractXdrRuleId(Dictionary yaml, string fallbackFileName) + { + if (yaml == null) + { + return fallbackFileName; + } + + if (yaml.TryGetValue("properties", out object propsObj) && propsObj is IReadOnlyDictionary props) + { + if (props.TryGetValue("id", out object propId) && propId != null) + { + return propId.ToString(); + } + if (props.TryGetValue("originalRuleId", out object origId) && origId != null) + { + return origId.ToString(); + } + } + + if (yaml.TryGetValue("id", out object topId) && topId != null) + { + return topId.ToString(); + } + + return fallbackFileName; + } + + private static string ExtractXdrOriginalRuleId(Dictionary yaml, string ruleId) + { + if (yaml != null) + { + if (yaml.TryGetValue("properties", out object propsObj) && propsObj is IReadOnlyDictionary props) + { + if (props.TryGetValue("originalRuleId", out object origId) && origId != null) + { + return origId.ToString(); + } + } + if (yaml.TryGetValue("contentProvenance", out object provObj) && provObj is IReadOnlyDictionary prov) + { + if (prov.TryGetValue("source", out object srcObj) && srcObj is IReadOnlyDictionary src) + { + if (src.TryGetValue("id", out object srcId) && srcId != null) + { + return srcId.ToString(); + } + } + } + } + + if (!string.IsNullOrWhiteSpace(ruleId) && ruleId.StartsWith("xdr-", StringComparison.OrdinalIgnoreCase)) + { + return ruleId.Substring(4); + } + + return ruleId; + } + + private static string ExtractXdrQueryText(Dictionary yaml) + { + if (yaml == null) + { + return null; + } + + if (yaml.TryGetValue("properties", out object propsObj) && propsObj is IReadOnlyDictionary props) + { + if (props.TryGetValue("queryCondition", out object qcObj) && qcObj is IReadOnlyDictionary qc) + { + if (qc.TryGetValue("queryText", out object qt) && qt != null) + { + return qt.ToString(); + } + } + if (props.TryGetValue("query", out object propQuery) && propQuery != null) + { + return propQuery.ToString(); + } + } + + if (yaml.TryGetValue("query", out object topQuery) && topQuery != null) + { + return topQuery.ToString(); + } + + return null; } private Dictionary ReadAndDeserializeYaml(string encodedFilePath) @@ -539,6 +894,58 @@ private DataConnectorSchema ReadAndDeserializeDataConnectorJson(string encodedFi DataConnectorSchema dataConnectorObject = JsonConvert.DeserializeObject(jsonString); return dataConnectorObject; } + + [Fact] + public void Validate_RuntimeQualifiedQuery_SuppressesSemanticDiagnosticsOnlyWhenHashMatches() + { + var basePath = Utils.GetTestDirectory(TestFolderDepthForSolutionParsers); + var tempSolutionDir = Path.Combine(basePath, "Solutions", $"_TempQualTest_{Guid.NewGuid():N}"); + var xdrDetDir = Path.Combine(tempSolutionDir, "XDR Detections"); + var evidenceDir = Path.Combine(tempSolutionDir, "Evidence", "pr-qualification"); + Directory.CreateDirectory(xdrDetDir); + Directory.CreateDirectory(evidenceDir); + + try + { + var ruleId = "xdr-11111111-2222-3333-4444-555555555555"; + var ruleFilePath = Path.Combine(xdrDetDir, "TempRule.yaml"); + var queryWithUnknownTable = "CustomLiveVerifiedTable_CL\n| where Timestamp > ago(1h)\n| project Timestamp, DeviceId, ReportId"; + File.WriteAllText(ruleFilePath, $"id: {ruleId}\n"); + + // Without qualification evidence, IsRuntimeQualifiedQuery must be false + Assert.False(IsRuntimeQualifiedQuery(ruleFilePath, ruleId, queryWithUnknownTable)); + + // Write pr-qualification-result.json with matching SHA-256 hash + var querySha256 = ComputeSha256Hex(queryWithUnknownTable.Trim()); + var evidenceJson = new JObject + { + ["kqlRuntimeVerifiedRules"] = new JArray + { + new JObject + { + ["ruleId"] = ruleId, + ["status"] = "PASS", + ["querySha256"] = querySha256 + } + } + }; + File.WriteAllText(Path.Combine(evidenceDir, "pr-qualification-result.json"), evidenceJson.ToString()); + + // Matching query hash returns true; modified query text returns false + Assert.True(IsRuntimeQualifiedQuery(ruleFilePath, ruleId, queryWithUnknownTable)); + Assert.False(IsRuntimeQualifiedQuery(ruleFilePath, ruleId, queryWithUnknownTable + "\n| take 5")); + + // ValidateKql passes for the runtime-qualified query with semantic/schema diagnostic + ValidateKql(ruleId, queryWithUnknownTable, true, ruleFilePath); + } + finally + { + if (Directory.Exists(tempSolutionDir)) + { + Directory.Delete(tempSolutionDir, true); + } + } + } } } diff --git a/.script/tests/KqlvalidationsTests/SkipValidationsTemplates.json b/.script/tests/KqlvalidationsTests/SkipValidationsTemplates.json index 30b32f12463..0cc8e986f74 100644 --- a/.script/tests/KqlvalidationsTests/SkipValidationsTemplates.json +++ b/.script/tests/KqlvalidationsTests/SkipValidationsTemplates.json @@ -3418,5 +3418,15 @@ "id": "ad76e484-f159-4d23-99ee-e734f0b8b60b", "templateName": "Possible device code phishing attempts.yaml", "validationFailReason": "References AADSignInEventsBeta, a Microsoft Defender XDR advanced hunting beta table not present in the KQL validator schema" + }, + { + "id": "011c84d8-85f0-4370-b864-24c13455aa94", + "templateName": "CoreBackupDeletionwithSecurityAlert.yaml", + "validationFailReason": "Uses mv-expand todynamic(split(HostName, ',')) where HostName is dynamically typed after prior mv-expand" + }, + { + "id": "xdr-011c84d8-85f0-4370-b864-24c13455aa94", + "templateName": "CoreBackupDeletionwithSecurityAlert.yaml", + "validationFailReason": "Uses mv-expand todynamic(split(HostName, ',')) where HostName is dynamically typed after prior mv-expand" } ] diff --git a/.script/tests/KqlvalidationsTests/YamlFilesTestData/XdrDetectionsYamlFilesLoader.cs b/.script/tests/KqlvalidationsTests/YamlFilesTestData/XdrDetectionsYamlFilesLoader.cs new file mode 100644 index 00000000000..0c9015c1577 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/YamlFilesTestData/XdrDetectionsYamlFilesLoader.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Kqlvalidations.Tests +{ + public class XdrDetectionsYamlFilesLoader : YamlFilesLoader + { + protected override List GetDirectoryPaths() + { + var basePath = Utils.GetTestDirectory(TestFolderDepth); + var solutionsDir = Path.Combine(basePath, "Solutions"); + if (!Directory.Exists(solutionsDir)) + { + return new List(); + } + + return Directory.GetDirectories(solutionsDir, "XDR Detections", SearchOption.AllDirectories) + .Where(dir => !Utils.IsPrivateSolutionContentPath(dir)) + .ToList(); + } + } +} diff --git a/.script/tests/KqlvalidationsTests/YamlFilesTestData/XdrDetectionsYamlFilesTestData.cs b/.script/tests/KqlvalidationsTests/YamlFilesTestData/XdrDetectionsYamlFilesTestData.cs new file mode 100644 index 00000000000..6aa62c4f215 --- /dev/null +++ b/.script/tests/KqlvalidationsTests/YamlFilesTestData/XdrDetectionsYamlFilesTestData.cs @@ -0,0 +1,10 @@ +namespace Kqlvalidations.Tests +{ + public class XdrDetectionsYamlFilesTestData : YamlFilesTestData + { + public XdrDetectionsYamlFilesTestData() + : base(new XdrDetectionsYamlFilesLoader()) + { + } + } +} diff --git a/.windsurf/rules/sentinel-xdr-migration.md b/.windsurf/rules/sentinel-xdr-migration.md index 6ae9ca73c05..80338ae4b12 100644 --- a/.windsurf/rules/sentinel-xdr-migration.md +++ b/.windsurf/rules/sentinel-xdr-migration.md @@ -6,8 +6,8 @@ hybrid packaging, or AR/CD qualification, read and follow: `Tools\SentinelToXDRMigration\agent-workflows\end-to-end-migration.md` Treat it as the canonical process. Use repository CLI and PowerShell tools, -not prompt-only implementations. Default to authoring and ask before optional -qualification. Deployment, mock ingestion, and parity are not required for -ISV publishing. Use V3.1 through `package-v3-1` for Sentinel and XDR solution -packages, preserve disabled lifecycle state, resume `workflow-state.json`, and -obtain explicit approval before environment writes. +not prompt-only implementations. Default to `qualification`, with +`--workflow-profile authoring` available as an explicit offline/ISV fallback +when no live lab is available. Use V3.1 through `package-v3-1` for Sentinel and +XDR solution packages, preserve disabled lifecycle state, resume +`workflow-state.json`, and obtain explicit approval before environment writes. diff --git a/AGENTS.md b/AGENTS.md index 6e3b3cc8fce..33db3555b89 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,7 +37,7 @@ hybrid solution packaging, or AR/CD qualification: 2. Treat that file as the canonical workflow; do not redefine its stages or gates in this adapter. 3. Keep deterministic work in the repository Python and PowerShell tools. -4. Default to `authoring`; ask before optional lab `qualification`. +4. Default to `qualification` (with `authoring` available as an explicit offline/ISV fallback when no live lab is available). 5. Deployment, mock ingestion, and alert parity are Qualification-only actions and require explicit exact-scope approval. 6. Use the V3.1 `package-v3-1` route for Sentinel and XDR solution content; diff --git a/CLAUDE.md b/CLAUDE.md index 01efce30e0f..19e0bf4971d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,8 @@ Follow that canonical workflow and use the repository CLI and PowerShell entry points directly. Do not replace deterministic conversion, validation, packaging, deployment, or parity logic with model-generated approximations. -Default to `authoring`. Ask before optional lab `qualification`; it is not -required for ISV publishing. Use V3.1 through `package-v3-1` for Sentinel and -XDR solution packages. Resume from `workflow-state.json`, preserve disabled -detections, and request explicit approval before writes. +Default to `qualification`, with `--workflow-profile authoring` available as an +explicit offline/ISV fallback when no live lab is available. Use V3.1 through +`package-v3-1` for Sentinel and XDR solution packages. Resume from +`workflow-state.json`, preserve disabled detections, and request explicit +approval before writes. diff --git a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_1.ps1 b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_1.ps1 index 5c190589eff..dc5bc924912 100644 --- a/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_1.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/V3/createSolutionV3_1.ps1 @@ -70,6 +70,9 @@ if ($PSCmdlet.ParameterSetName -eq "Local") { -IncludeXdrDetections $true ` -EnforceV31Versions $true ` -SkipAttributionLookup:$SkipAttributionLookup + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } return } @@ -130,7 +133,7 @@ try { $contentResourceDetails = returnContentResources $calculatedPackageVersion if ($null -eq $contentResourceDetails) { Write-Error "Not able to identify content resource details based on Version. Please verify if Version in data input file is correct!" - return + exit 1 } $ccpDict = @() diff --git a/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 index 2f8dcba97a2..00a48c77296 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 @@ -735,6 +735,7 @@ function createCCPConnectorResources($contentResourceDetails, $dataFileMetadata, } else { Write-Error "Failed to create or update 'auth' parameter." + throw "Failed to create or update 'auth' parameter." } $hasAppId = [bool](($armResource.properties.auth).PSobject.Properties.name -match "appId") diff --git a/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 index 4a1c2b358f2..d867c0a2138 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 @@ -243,7 +243,10 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold } } catch { - Write-Host "Error occurred while identifying relation between definition and poller File. Identified error in " + $inputFile.Name + ". Error Details : $_" + $relationError = "Error occurred while identifying relation between definition and poller File. Identified error in $($inputFile.Name). Error Details : $_" + Write-Host $relationError + Write-Error $relationError + throw $relationError } } } @@ -301,7 +304,7 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold # this is a custom table if ($dcrDataFlowStream.streams[0] -eq $ccpPollerFile.DCPollerStreamName) { $ccpPollerFile.DCRFilePath = $inputFile.FullName.Replace('\', '/') - if ($fileContent.properties.dataFlows[0].outputStream.contains('Custom-')) { + if ($null -ne $fileContent.properties.dataFlows[0].outputStream -and $fileContent.properties.dataFlows[0].outputStream.contains('Custom-')) { $ccpPollerFile.TableOutputStream = $dcrDataFlowStream.outputStream.Replace('Custom-', '') $ccpPollerFile.DCROutputStream = $dcrDataFlowStream.outputStream } @@ -318,7 +321,7 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold if ($dataFlowStreamName -eq $ccpPollerFile.DCPollerStreamName) { $ccpPollerFile.DCRFilePath = $inputFile.FullName.Replace('\', '/') - if ($dataFlowOutputStreamName.contains('Custom-')) { + if ($null -ne $dataFlowOutputStreamName -and $dataFlowOutputStreamName.contains('Custom-')) { $ccpPollerFile.TableOutputStream = $dataFlowStreamName.Replace('Custom-', '') $ccpPollerFile.DCROutputStream = $dataFlowOutputStreamName } @@ -338,7 +341,7 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold } if ($fileContent.properties.dataFlows[0].streams[0] -eq $ccpPollerFile.DCPollerStreamName) { $ccpPollerFile.DCRFilePath = $inputFile.FullName.Replace('\', '/') - if ($fileContent.properties.dataFlows[0].outputStream.contains('Custom-')) { + if ($null -ne $fileContent.properties.dataFlows[0].outputStream -and $fileContent.properties.dataFlows[0].outputStream.contains('Custom-')) { $ccpPollerFile.TableOutputStream = $fileContent.properties.dataFlows[0].outputStream.Replace('Custom-', '') $ccpPollerFile.DCROutputStream = $fileContent.properties.dataFlows[0].outputStream } @@ -347,7 +350,10 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold } } catch { - Write-Host "Error occurred while identifying relation between Poller and DCR File. Identified error in " + $inputFile.Name + ". Error Details : $_" + $relationError = "Error occurred while identifying relation between Poller and DCR File. Identified error in $($inputFile.Name). Error Details : $_" + Write-Host $relationError + Write-Error $relationError + throw $relationError } } } @@ -388,7 +394,10 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold } } catch { - Write-Host "Error occurred while identifying relation between DCR and Table File. Identified error in " + $inputFile.Name + ". Error Details : $_" + $relationError = "Error occurred while identifying relation between DCR and Table File. Identified error in $($inputFile.Name). Error Details : $_" + Write-Host $relationError + Write-Error $relationError + throw $relationError } } } @@ -398,11 +407,12 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold # THROW ERROR IF THERE IS NO RELATION BETWEEN DEFINITION->POLLER AND/OR POLLER->DCR if ($ccpDict.Count -gt 0) { foreach($localCCPDist in $ccpDict) { - if (($localCCPDist.DCDefinitionId -eq "" -or $localCCPDist.DCDefinitionFilePath -eq "" -or - $localCCPDist.DCPollerFilePath -eq "" -or $localCCPDist.DCPollerStreamName -eq "" -or $localCCPDist.DCRFilePath -eq "") -and - $localCCPDist.PollerKind -eq 'RestApiPoller') + if ($localCCPDist.DCDefinitionId -eq "" -or $localCCPDist.DCDefinitionFilePath -eq "" -or + $localCCPDist.DCPollerFilePath -eq "" -or ($localCCPDist.DCPollerStreamName -eq "" -and -not $localCCPDist.isDynamicStreamName) -or $localCCPDist.DCRFilePath -eq "") { - Write-Host "Please verify if there is a mapping between ConnectorDefinition with Poller file and/or Poller file with DCR file! If mapping is correct then check type property for ccp files!" + $relationError = "Please verify if there is a mapping between ConnectorDefinition with Poller file and/or Poller file with DCR file! If mapping is correct then check type property for ccp files!" + Write-Host $relationError + Write-Error $relationError exit 1 } } diff --git a/Tools/SentinelToXDRMigration/FOLLOW_UPS.md b/Tools/SentinelToXDRMigration/FOLLOW_UPS.md index 7583c61ccfc..67fe022c75e 100644 --- a/Tools/SentinelToXDRMigration/FOLLOW_UPS.md +++ b/Tools/SentinelToXDRMigration/FOLLOW_UPS.md @@ -145,12 +145,21 @@ themselves establish a toolkit fix is required. `oauth2PermissionGrants@v1.0` when `ThreatHunting.Read.All` or `CustomDetection.ReadWrite.All` is missing. +- **Flip default workflow profile from `authoring` to `qualification`.** + Switched the default `--workflow-profile` in `solution_manager/cli.py`, + `initialize_workflow` in `solution_manager/core/workflow.py`, + `Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md`, and + agent adapters (`AGENTS.md`, `CLAUDE.md`, `.github`, `.claude`, `.cursor`, and + `.windsurf`) from `authoring` to `qualification`, preserving + `--workflow-profile authoring` as an explicit offline/ISV fallback when no + live lab is available. + ## Candidate follow-ups -- **Flip default workflow profile from `authoring` to `qualification` after live - lab sign-off.** `evaluate_qualification_promotion_gate` now verifies all 5 - qualification stages end-to-end; once signed off against a live lab run, - switch the default profile in `workflow-init` and agent adapters. +- [x] **Flip default workflow profile from `authoring` to `qualification` after live + lab sign-off.** Completed: `workflow-init`, `initialize_workflow`, canonical + workflow docs, and all agent adapters now default to `qualification` with + `authoring` preserved as an explicit offline/ISV fallback. Any future environment write remains separately gated by explicit approval immediately before the exact operation. No cloud action is authorized by this diff --git a/Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md b/Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md index ace017ed382..2d1fde5414f 100644 --- a/Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md +++ b/Tools/SentinelToXDRMigration/agent-workflows/end-to-end-migration.md @@ -716,29 +716,26 @@ Install the CLI when necessary: python -m pip install -e Tools\SentinelToXDRMigration ``` -Run `workflow-status` first. If state does not exist, default to `authoring` -(or confirm whether the user wants optional `qualification`) before -initializing: +Run `workflow-status` first. If state does not exist, default to `qualification` +(with `--workflow-profile authoring` available as an explicit offline/ISV +fallback when no live lab is available) before initializing: ```powershell sentinel-xdr-migration workflow-init ` --solution "" ` - --workflow-profile authoring ` - --version-bump none + --workflow-profile qualification ` + --confirm-configured-target ` + --version-bump "" ``` -For an approved qualification run, use `qualification` and include -the complete locked target. Alternatively, after reviewing the per-solution -configuration above, use `--confirm-configured-target` instead of supplying -tenant/workspace values; the workflow derives the subscription and reads the -customer ID only from that exact approved workspace: +When no live lab is available, explicitly select the offline `authoring` +fallback: ```powershell sentinel-xdr-migration workflow-init ` --solution "" ` - --workflow-profile qualification ` - --confirm-configured-target ` - --version-bump "" + --workflow-profile authoring ` + --version-bump none ``` Before requesting a workspace value, inspect `doctor`. If diff --git a/Tools/SentinelToXDRMigration/schema/workflow-state.schema.json b/Tools/SentinelToXDRMigration/schema/workflow-state.schema.json index 22c78e70251..b924cf1f9d8 100644 --- a/Tools/SentinelToXDRMigration/schema/workflow-state.schema.json +++ b/Tools/SentinelToXDRMigration/schema/workflow-state.schema.json @@ -87,6 +87,13 @@ "schemaVersion": { "const": "1.2.0" }, + "operatorHash": { + "type": [ + "string", + "null" + ], + "pattern": "^[0-9a-f]{16}$" + }, "solution": { "type": "string", "description": "Solution-relative location (normally '.'); legacy absolute locations remain readable. Artifact references resolve against the selected solution, not the shell cwd.", @@ -171,6 +178,13 @@ "qualification" ] }, + "operatorHash": { + "type": [ + "string", + "null" + ], + "pattern": "^[0-9a-f]{16}$" + }, "profileSelectionConfirmed": { "type": "boolean" }, diff --git a/Tools/SentinelToXDRMigration/solution_manager/authoring/__init__.py b/Tools/SentinelToXDRMigration/solution_manager/authoring/__init__.py index 016b63c4b6d..e9673b6fb03 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/authoring/__init__.py +++ b/Tools/SentinelToXDRMigration/solution_manager/authoring/__init__.py @@ -1 +1,17 @@ """Offline content authoring, schema validation, and dependency contracts for detections and workbooks.""" +from __future__ import annotations + +from solution_manager.authoring.validator import ( + validate_kql_query_static, + validate_xdr_artifacts_kql, + validate_xdr_detection_kql, + validate_xdr_workbook_kql, +) + +__all__ = [ + "validate_kql_query_static", + "validate_xdr_artifacts_kql", + "validate_xdr_detection_kql", + "validate_xdr_workbook_kql", +] + diff --git a/Tools/SentinelToXDRMigration/solution_manager/authoring/dependencies.py b/Tools/SentinelToXDRMigration/solution_manager/authoring/dependencies.py index e0a0cba517e..ee67f8ae6e9 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/authoring/dependencies.py +++ b/Tools/SentinelToXDRMigration/solution_manager/authoring/dependencies.py @@ -40,7 +40,7 @@ def telemetry_data_types(tables): **{table: "sentinel" for table in ( "Syslog", "SecurityAlert", "SecurityIncident", "SecurityEvent", "WindowsEvent", "Event", "AzureActivity", "AzureDiagnostics", "CommonSecurityLog", "SigninLogs", "AuditLogs", - "OfficeActivity", "AWSCloudTrail", "AWSGuardDuty", + "OfficeActivity", "PowerBIActivity", "PowerPlatformAdminActivity", "AWSCloudTrail", "AWSGuardDuty", )}, } diff --git a/Tools/SentinelToXDRMigration/solution_manager/authoring/validator.py b/Tools/SentinelToXDRMigration/solution_manager/authoring/validator.py new file mode 100644 index 00000000000..d4bbe9789e5 --- /dev/null +++ b/Tools/SentinelToXDRMigration/solution_manager/authoring/validator.py @@ -0,0 +1,849 @@ +"""Offline static KQL syntax and known-schema validator for XDR artifacts. + +Validates KQL queries in XDR Detections (`Solutions//XDR Detections/*.yaml`) +and XDR Workbooks (`Solutions//XDR Workbooks/*.json`) without requiring live +Azure or Defender XDR connectivity. +""" +from __future__ import annotations + +import json +from pathlib import Path +import re +from typing import Any, Iterable + +import yaml + +from solution_manager.authoring.dependencies import ( + PLATFORM_REFERENCE_INPUTS, + TABLE_WORKLOADS, + extract_builtin_asim_parsers, +) +from solution_manager.migration import catalog +from solution_manager.migration._workbook_core import mapping_catalog, workbook + +KNOWN_TABULAR_OPERATORS = frozenset( + { + "as", + "consume", + "count", + "datatable", + "distinct", + "evaluate", + "extend", + "externaldata", + "facet", + "find", + "flapping", + "fork", + "getschema", + "graph-mark-components", + "graph-match", + "graph-shortest-paths", + "graph-to-table", + "invoke", + "join", + "limit", + "lookup", + "make-graph", + "make-series", + "mv-apply", + "mv-expand", + "mvapply", + "mvexpand", + "order", + "parse", + "parse-kv", + "parse-where", + "partition", + "print", + "project", + "project-away", + "project-keep", + "project-rename", + "project-reorder", + " project-smart", + "project-smart", + "range", + "reduce", + "render", + "sample", + "sample-distinct", + "scan", + "search", + "serialize", + "sort", + "summarize", + "take", + "top", + "top-hitters", + "top-nested", + "union", + "where", + } +) + +KNOWN_STATEMENT_KEYWORDS = frozenset( + { + "alias", + "declare", + "let", + "pattern", + "restrict", + "set", + } +) + +TABULAR_GENERATOR_KEYWORDS = frozenset( + { + "datatable", + "external_data", + "externaldata", + "find", + "print", + "range", + "search", + "union", + } +) + +CROSS_WORKSPACE_CALL_RE = re.compile(r"(? set[str]: + known: set[str] = set(catalog.NATIVE_XDR_TABLES) + known.update(catalog.PASSTHROUGH_TABLES.keys()) + known.update(TABLE_WORKLOADS.keys()) + known.update(PLATFORM_REFERENCE_INPUTS) + try: + mc = mapping_catalog.load_mapping_catalog() + for table_name, info in (mc.get("tables") or {}).items(): + known.add(str(table_name)) + if isinstance(info, dict): + target = info.get("targetTable") or info.get("ahTable") + if isinstance(target, str) and target.strip(): + known.add(target.strip()) + except Exception: + pass + known.update( + { + "AADNonInteractiveUserSignInLogs", + "AADManagedIdentitySignInLogs", + "AADProvisioningLogs", + "AADRiskyServicePrincipals", + "AADRiskyUsers", + "AADServicePrincipalRiskEvents", + "AADServicePrincipalSignInLogs", + "AADSignInEventsBeta", + "AADSpnSignInEventsBeta", + "AADUserRiskEvents", + "AuditLogs", + "AzureActivity", + "AzureDiagnostics", + "BehaviorEntities", + "BehaviorInfo", + "CampaignInfo", + "CloudAppEvents", + "CommonSecurityLog", + "ConfigurationAssessment", + "CustomDetectionEvents", + "DeviceBaselineComplianceAssessment", + "DeviceCustomScriptEvents", + "DeviceEvents", + "DeviceFileCertificateInfo", + "DeviceFileEvents", + "DeviceImageLoadEvents", + "DeviceInfo", + "DeviceLogonEvents", + "DeviceNetworkEvents", + "DeviceNetworkInfo", + "DeviceProcessEvents", + "DeviceRegistryEvents", + "DeviceTvmBrowserExtensions", + "DeviceTvmCertificateInfo", + "DeviceTvmHardwareFirmware", + "DeviceTvmInfoGathering", + "DeviceTvmInfoGatheringKB", + "DeviceTvmSecureConfigurationAssessment", + "DeviceTvmSecureConfigurationAssessmentKB", + "DeviceTvmSoftwareEvidenceBeta", + "DeviceTvmSoftwareInventory", + "DeviceTvmSoftwareVulnerabilities", + "DeviceTvmSoftwareVulnerabilitiesKB", + "DnsEvents", + "DnsInventory", + "EmailAttachmentInfo", + "EmailEvents", + "EmailPostDeliveryEvents", + "EmailUrlInfo", + "Event", + "ExposureGraphEdges", + "ExposureGraphNodes", + "FileMaliciousContentInfo", + "Heartbeat", + "HuntingBookmark", + "IdentityDirectoryEvents", + "IdentityInfo", + "IdentityLogonEvents", + "IdentityQueryEvents", + "LAQueryLogs", + "MdiSensorEvents", + "MessageEvents", + "MessagePostDeliveryEvents", + "MessageUrlInfo", + "MicrosoftGraphActivityLogs", + "NetworkAccessTraffic", + "OAuthAppInfo", + "OfficeActivity", + "Operation", + "PowerBIActivity", + "PowerPlatformAdminActivity", + "ProtectionStatus", + "SecurityAlert", + "SecurityBaseline", + "SecurityBaselineSummary", + "SecurityDetection", + "SecurityEvent", + "SecurityIncident", + "SecurityIoTRawEvent", + "SecurityNestedRecommendation", + "SecurityRecommendation", + "SentinelHealth", + "SigninLogs", + "StorageBlobLogs", + "StorageFileLogs", + "Syslog", + "ThreatIntelIndicators", + "ThreatIntelligenceIndicator", + "Update", + "UpdateSummary", + "UrlClickEvents", + "Usage", + "UserAccessAnalytics", + "UserPeerAnalytics", + "VMConnection", + "W3CIISLog", + "Watchlist", + "WindowsEvent", + "WindowsFirewall", + } + ) + return known + + +def _scan_lexical_structure(query: str) -> tuple[str, list[str]]: + """Mask comments and string literals with spaces while checking balanced quotes/delimiters.""" + errors: list[str] = [] + chars = list(query) + n = len(query) + i = 0 + stack: list[tuple[str, int]] = [] + matching = {"(": ")", "[": "]", "{": "}"} + closing = {")": "(", "]": "[", "}": "{"} + + while i < n: + c = query[i] + # Single-line comment // + if c == "/" and i + 1 < n and query[i + 1] == "/": + j = query.find("\n", i + 2) + if j == -1: + j = n + for k in range(i, j): + if chars[k] not in ("\r", "\n"): + chars[k] = " " + i = j + continue + + # Multi-line comment /* ... */ + if c == "/" and i + 1 < n and query[i + 1] == "*": + j = query.find("*/", i + 2) + if j == -1: + errors.append(f"Unclosed block comment '/*' starting at offset {i}") + for k in range(i, n): + if chars[k] not in ("\r", "\n"): + chars[k] = " " + break + for k in range(i, j + 2): + if chars[k] not in ("\r", "\n"): + chars[k] = " " + i = j + 2 + continue + + # Verbatim string @"..." or @'...' + if c == "@" and i + 1 < n and query[i + 1] in ('"', "'"): + quote = query[i + 1] + start = i + chars[i] = "_" + chars[i + 1] = "_" + i += 2 + closed = False + while i < n: + if query[i] == quote: + if i + 1 < n and query[i + 1] == quote: + chars[i] = "_" + chars[i + 1] = "_" + i += 2 + continue + chars[i] = "_" + i += 1 + closed = True + break + if chars[i] not in ("\r", "\n"): + chars[i] = "_" + i += 1 + if not closed: + errors.append(f"Unclosed verbatim string literal @{quote} starting at offset {start}") + continue + + # Triple-backtick or ```multi-line string``` in KQL + if c == "`" and i + 2 < n and query[i : i + 3] == "```": + start = i + j = query.find("```", i + 3) + if j == -1: + errors.append(f"Unclosed multi-line string literal '```' starting at offset {start}") + for k in range(i, n): + if chars[k] not in ("\r", "\n"): + chars[k] = "_" + break + for k in range(i, j + 3): + if chars[k] not in ("\r", "\n"): + chars[k] = "_" + i = j + 3 + continue + + # Standard string literal "..." or '...' + if c in ('"', "'"): + quote = c + start = i + chars[i] = "_" + i += 1 + closed = False + while i < n: + ch = query[i] + if ch == "\\": + chars[i] = "_" + if i + 1 < n and chars[i + 1] not in ("\r", "\n"): + chars[i + 1] = "_" + i += 2 + continue + if ch in ("\r", "\n"): + break + chars[i] = "_" + if ch == quote: + i += 1 + closed = True + break + i += 1 + if not closed: + errors.append(f"Unclosed string literal {quote} starting at offset {start}") + continue + + # Delimiters (), [], {} + if c in matching: + stack.append((c, i)) + elif c in closing: + expected_open = closing[c] + if not stack: + errors.append(f"Unexpected closing delimiter '{c}' at offset {i}") + else: + actual_open, open_pos = stack.pop() + if actual_open != expected_open: + errors.append( + f"Mismatched delimiter '{c}' at offset {i} (expected '{matching[actual_open]}' for '{actual_open}' at offset {open_pos})" + ) + i += 1 + + for open_char, open_pos in stack: + errors.append(f"Unclosed delimiter '{open_char}' starting at offset {open_pos}") + + return "".join(chars), errors + + +def _split_top_level(text: str, separator: str) -> list[str]: + """Split `text` by single-character `separator` at top-level delimiter depth 0.""" + parts: list[str] = [] + depth = 0 + start = 0 + for idx, ch in enumerate(text): + if ch in "([{": + depth += 1 + elif ch in ")]}": + if depth > 0: + depth -= 1 + elif ch == separator and depth == 0: + parts.append(text[start:idx]) + start = idx + 1 + parts.append(text[start:]) + return parts + + +def validate_kql_query_static( + query_text: str, + *, + surface: str = "advancedHunting", + known_functions: Iterable[str] = (), + known_tables: Iterable[str] = (), + allow_workbook_parameters: bool = False, +) -> dict[str, Any]: + """Validate a single KQL query offline for syntax, pipeline operators, and known tables.""" + errors: list[str] = [] + warnings: list[str] = [] + + if not isinstance(query_text, str) or not query_text.strip(): + return { + "valid": False, + "errors": ["Query text must be a non-empty string"], + "warnings": [], + "referencedTables": [], + "operators": [], + } + + work_query = query_text + if allow_workbook_parameters: + work_query = WORKBOOK_TOKEN_RE.sub("_wb_param_", work_query) + + masked, lex_errors = _scan_lexical_structure(work_query) + errors.extend(lex_errors) + + if not masked.strip(): + errors.append("Query contains only comments or whitespace and no executable KQL statement") + return { + "valid": False, + "errors": errors, + "warnings": warnings, + "referencedTables": [], + "operators": [], + } + + if surface == "advancedHunting" and CROSS_WORKSPACE_CALL_RE.search(masked): + errors.append("Advanced Hunting query must not contain cross-workspace 'workspace(...)' calls") + + known_table_map = {t.casefold(): t for t in (_known_tables_catalog() | set(known_tables))} + approved_fn_set = {fn.strip().casefold() for fn in known_functions if isinstance(fn, str) and fn.strip()} + builtin_asim = {fn.casefold() for fn in extract_builtin_asim_parsers(query_text)} + + statements = _split_top_level(masked, ";") + let_bindings: set[str] = set() + operators_seen: list[str] = [] + referenced_tables: set[str] = set() + has_tabular_expression = False + + for stmt_idx, raw_stmt in enumerate(statements): + stmt = raw_stmt.strip() + if not stmt: + continue + + first_word_match = re.match(r"^([A-Za-z_][A-Za-z0-9_-]*)", stmt) + first_word = first_word_match.group(1).casefold() if first_word_match else "" + + if first_word in KNOWN_STATEMENT_KEYWORDS: + if first_word == "let": + let_match = re.match(r"^let\s+([A-Za-z_][A-Za-z0-9_]*)\s*=\s*([\s\S]+)$", stmt, re.IGNORECASE) + if not let_match: + errors.append(f"Malformed 'let' statement in statement #{stmt_idx + 1}") + continue + let_name = let_match.group(1) + let_bindings.add(let_name.casefold()) + rhs = let_match.group(2).strip() + # If the RHS is a lambda/view `(params) { body }`, extract body if tabular + view_match = re.match(r"^(?:view\s*)?\([^)]*\)\s*\{([\s\S]*)\}\s*$", rhs, re.IGNORECASE) + if view_match: + rhs = view_match.group(1).strip() + if "|" in _split_top_level(rhs, "|") or len(_split_top_level(rhs, "|")) > 1: + _validate_pipeline_stages( + rhs, + stmt_label=f"let {let_name}", + known_table_map=known_table_map, + let_bindings=let_bindings, + approved_fn_set=approved_fn_set | builtin_asim, + operators_seen=operators_seen, + referenced_tables=referenced_tables, + errors=errors, + ) + else: + head_ident_match = re.match(r"^(?:\(\s*)*([A-Za-z_][A-Za-z0-9_]*)", rhs) + if head_ident_match: + ident = head_ident_match.group(1) + if ident.casefold() in known_table_map: + referenced_tables.add(known_table_map[ident.casefold()]) + elif re.fullmatch(r"[A-Za-z][A-Za-z0-9_]*_CL", ident, re.IGNORECASE): + referenced_tables.add(ident) + continue + + has_tabular_expression = True + _validate_pipeline_stages( + stmt, + stmt_label=f"statement #{stmt_idx + 1}", + known_table_map=known_table_map, + let_bindings=let_bindings, + approved_fn_set=approved_fn_set | builtin_asim, + operators_seen=operators_seen, + referenced_tables=referenced_tables, + errors=errors, + ) + + if not has_tabular_expression: + errors.append("Query must contain at least one tabular expression statement") + + # Also collect any known tables referenced inside subqueries/joins/unions + for tbl in catalog.referenced_catalog_tables(query_text) | catalog.referenced_custom_tables(query_text): + referenced_tables.add(tbl) + for ref in catalog.query_table_references(query_text, known_tables=known_table_map.values()): + tbl = ref.get("table") + if isinstance(tbl, str) and (tbl.casefold() in known_table_map or tbl.endswith("_CL")): + referenced_tables.add(known_table_map.get(tbl.casefold(), tbl)) + + return { + "valid": len(errors) == 0, + "errors": errors, + "warnings": warnings, + "referencedTables": sorted(referenced_tables), + "operators": operators_seen, + } + + +def _validate_pipeline_stages( + expr: str, + *, + stmt_label: str, + known_table_map: dict[str, str], + let_bindings: set[str], + approved_fn_set: set[str], + operators_seen: list[str], + referenced_tables: set[str], + errors: list[str], +) -> None: + stages = _split_top_level(expr, "|") + for stage_idx, raw_stage in enumerate(stages): + stage = raw_stage.strip() + if not stage: + errors.append(f"Empty pipeline stage in {stmt_label} at stage #{stage_idx + 1}") + continue + + if stage_idx == 0: + _validate_pipeline_source( + stage, + stmt_label=stmt_label, + known_table_map=known_table_map, + let_bindings=let_bindings, + approved_fn_set=approved_fn_set, + referenced_tables=referenced_tables, + errors=errors, + ) + else: + op_match = re.match(r"^([A-Za-z_][A-Za-z0-9_-]*)", stage) + if not op_match: + errors.append(f"Invalid tabular operator syntax '{stage[:32]}' in {stmt_label} at stage #{stage_idx + 1}") + continue + op = op_match.group(1).casefold() + if op not in KNOWN_TABULAR_OPERATORS: + errors.append(f"Unknown KQL tabular operator '{op_match.group(1)}' in {stmt_label} at stage #{stage_idx + 1}") + else: + operators_seen.append(op) + rest = stage[op_match.end() :].strip() + if op in {"where", "extend", "project", "summarize", "join", "lookup", "mv-expand", "mvexpand", "parse", "top", "take", "limit", "sort", "order"} and not rest: + errors.append(f"KQL operator '{op_match.group(1)}' is missing required arguments in {stmt_label} at stage #{stage_idx + 1}") + + +def _validate_pipeline_source( + source_stage: str, + *, + stmt_label: str, + known_table_map: dict[str, str], + let_bindings: set[str], + approved_fn_set: set[str], + referenced_tables: set[str], + errors: list[str], +) -> None: + unwrapped = source_stage.strip() + while unwrapped.startswith("(") and unwrapped.endswith(")"): + inner = unwrapped[1:-1].strip() + _, inner_errs = _scan_lexical_structure(inner) + if inner_errs: + break + unwrapped = inner + + if "|" in _split_top_level(unwrapped, "|") or len(_split_top_level(unwrapped, "|")) > 1: + _validate_pipeline_stages( + unwrapped, + stmt_label=stmt_label, + known_table_map=known_table_map, + let_bindings=let_bindings, + approved_fn_set=approved_fn_set, + operators_seen=[], + referenced_tables=referenced_tables, + errors=errors, + ) + return + + head_match = re.match(r"^([A-Za-z_][A-Za-z0-9_-]*)", unwrapped) + if not head_match: + if unwrapped.startswith("["): + return + errors.append(f"Unrecognized tabular source '{source_stage[:32]}' in {stmt_label}") + return + + ident = head_match.group(1) + lowered = ident.casefold() + after = unwrapped[head_match.end() :].strip() + + if lowered in TABULAR_GENERATOR_KEYWORDS: + if lowered == "union": + for op_name, _ in workbook._union_operands(unwrapped): + op_low = op_name.casefold() + if op_low in known_table_map: + referenced_tables.add(known_table_map[op_low]) + elif re.fullmatch(r"[A-Za-z][A-Za-z0-9_]*_CL", op_name, re.IGNORECASE): + referenced_tables.add(op_name) + elif ( + op_low not in let_bindings + and op_low not in approved_fn_set + and not workbook.is_standard_builtin_asim_parser(op_name) + and op_low not in {"withsource", "isfuzzy", "kind", "_wb_param_", "print", "range", "datatable", "externaldata", "external_data"} + ): + errors.append(f"Unknown table or function '{op_name}' in union operand of {stmt_label}") + return + + if lowered in known_table_map: + referenced_tables.add(known_table_map[lowered]) + return + + if re.fullmatch(r"[A-Za-z][A-Za-z0-9_]*_CL", ident, re.IGNORECASE): + referenced_tables.add(ident) + return + + if lowered in let_bindings or lowered in approved_fn_set or workbook.is_standard_builtin_asim_parser(ident): + return + + if lowered in {"_getwatchlist", "_getwatchlistalias", "table", "totable", "externaldata", "external_data", "materialize", "toscalar", "_wb_param_"}: + return + + if after.startswith("("): + # Function call as tabular source + if lowered in {fn.casefold() for fn in workbook._BUILTINS}: + return + errors.append(f"Unknown tabular function or unapproved workspace function '{ident}(...)' in {stmt_label}") + return + + errors.append(f"Unknown table or tabular identifier '{ident}' in {stmt_label}") + + +def validate_xdr_detection_kql( + doc: dict[str, Any], + *, + known_functions: Iterable[str] = (), + known_tables: Iterable[str] = (), +) -> dict[str, Any]: + """Validate the KQL query inside an authored XDR Detection YAML document.""" + if not isinstance(doc, dict): + return {"valid": False, "errors": ["XDR Detection document must be a mapping"], "referencedTables": []} + + props = doc.get("properties") if isinstance(doc.get("properties"), dict) else {} + qc = props.get("queryCondition") if isinstance(props.get("queryCondition"), dict) else {} + query_text = qc.get("queryText") + if query_text is None: + query_text = doc.get("query") + + prov = doc.get("contentProvenance") if isinstance(doc.get("contentProvenance"), dict) else {} + declared_tables: set[str] = set(known_tables) + for item in prov.get("requiredWorkloads") or []: + if isinstance(item, dict): + for dt in item.get("dataTypes") or []: + if isinstance(dt, str) and dt.strip(): + declared_tables.add(dt.strip()) + for item in prov.get("requiredDataConnectors") or []: + if isinstance(item, dict): + for dt in item.get("dataTypes") or []: + if isinstance(dt, str) and dt.strip(): + declared_tables.add(dt.strip()) + + fn_aliases: set[str] = set(known_functions) + for fn_rec in prov.get("requiredWorkspaceFunctions") or []: + if isinstance(fn_rec, dict) and isinstance(fn_rec.get("functionAlias"), str): + fn_aliases.add(fn_rec["functionAlias"]) + elif isinstance(fn_rec, str): + fn_aliases.add(fn_rec) + + return validate_kql_query_static( + query_text if isinstance(query_text, str) else "", + surface="advancedHunting", + known_functions=fn_aliases, + known_tables=declared_tables, + allow_workbook_parameters=False, + ) + + +def validate_xdr_workbook_kql( + doc: dict[str, Any], + *, + known_functions: Iterable[str] = (), + known_tables: Iterable[str] = (), +) -> dict[str, Any]: + """Validate all Advanced Hunting / Log Analytics KQL query surfaces in an XDR Workbook JSON document.""" + if not isinstance(doc, dict): + return {"valid": False, "errors": ["XDR Workbook document must be a JSON object"], "queries": []} + + prov = doc.get("contentProvenance") if isinstance(doc.get("contentProvenance"), dict) else {} + declared_tables: set[str] = set(known_tables) + for item in prov.get("requiredWorkloads") or []: + if isinstance(item, dict): + for dt in item.get("dataTypes") or []: + if isinstance(dt, str) and dt.strip(): + declared_tables.add(dt.strip()) + for item in prov.get("requiredDataConnectors") or []: + if isinstance(item, dict): + for dt in item.get("dataTypes") or []: + if isinstance(dt, str) and dt.strip(): + declared_tables.add(dt.strip()) + + fn_aliases: set[str] = set(known_functions) + for fn_rec in prov.get("requiredWorkspaceFunctions") or []: + if isinstance(fn_rec, dict) and isinstance(fn_rec.get("functionAlias"), str): + fn_aliases.add(fn_rec["functionAlias"]) + elif isinstance(fn_rec, str): + fn_aliases.add(fn_rec) + + surfaces = workbook.query_surfaces(doc) + errors: list[str] = [] + query_reports: list[dict[str, Any]] = [] + + for surface_entry in surfaces: + if workbook.is_preserved_provider_surface(surface_entry): + continue + qt = surface_entry.get("queryType", 0) + if qt not in (0, None, workbook.AH): + continue + pointer = surface_entry["path"] + raw_query_value = surface_entry["node"].get("query") + surface_kind = "advancedHunting" if qt == workbook.AH else "logAnalytics" + try: + unwrapped_query = workbook._raw_query(raw_query_value, qt) + except ValueError as exc: + err_msg = f"{pointer}: {exc}" + errors.append(err_msg) + query_reports.append({"path": pointer, "valid": False, "errors": [str(exc)]}) + continue + + res = validate_kql_query_static( + unwrapped_query, + surface=surface_kind, + known_functions=fn_aliases, + known_tables=declared_tables, + allow_workbook_parameters=True, + ) + for err in res["errors"]: + errors.append(f"{pointer}: {err}") + query_reports.append( + { + "path": pointer, + "surface": surface_kind, + "valid": res["valid"], + "errors": res["errors"], + "referencedTables": res["referencedTables"], + } + ) + + return { + "valid": len(errors) == 0, + "errors": errors, + "queries": query_reports, + } + + +def _solution_parser_aliases(solution_root: Path) -> set[str]: + aliases: set[str] = set() + parsers_dir = solution_root / "Parsers" + if not parsers_dir.is_dir(): + return aliases + for path in sorted(parsers_dir.rglob("*")): + if not path.is_file() or path.suffix.lower() not in {".yaml", ".yml"}: + continue + try: + doc = yaml.safe_load(path.read_text(encoding="utf-8")) + except Exception: + continue + if isinstance(doc, dict): + for key in ("FunctionAlias", "FunctionName", "Title"): + val = doc.get(key) + if isinstance(val, str) and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", val.strip()): + aliases.add(val.strip()) + return aliases + + +def validate_xdr_artifacts_kql(solution_path: Path | str) -> dict[str, Any]: + """Perform offline static KQL validation across XDR Detections and XDR Workbooks in a solution.""" + root = Path(solution_path).expanduser().resolve() + parser_aliases = _solution_parser_aliases(root) + + detection_results: list[dict[str, Any]] = [] + workbook_results: list[dict[str, Any]] = [] + all_errors: list[str] = [] + + det_dir = root / "XDR Detections" + if det_dir.is_dir(): + det_files = sorted( + [ + p + for p in det_dir.rglob("*") + if p.is_file() and p.suffix.lower() in {".yaml", ".yml"} + ], + key=lambda p: p.as_posix().lower(), + ) + for det_file in det_files: + rel = det_file.relative_to(root).as_posix() + try: + doc = yaml.safe_load(det_file.read_text(encoding="utf-8")) + except Exception as exc: + err = f"Failed to parse YAML: {exc}" + detection_results.append({"path": rel, "valid": False, "errors": [err], "referencedTables": []}) + all_errors.append(f"{rel}: {err}") + continue + + res = validate_xdr_detection_kql(doc, known_functions=parser_aliases) + detection_results.append( + { + "path": rel, + "valid": res["valid"], + "errors": res["errors"], + "referencedTables": res.get("referencedTables", []), + } + ) + for err in res["errors"]: + all_errors.append(f"{rel}: {err}") + + wb_dir = root / "XDR Workbooks" + if wb_dir.is_dir(): + wb_files = sorted( + [p for p in wb_dir.rglob("*") if p.is_file() and p.suffix.lower() == ".json"], + key=lambda p: p.as_posix().lower(), + ) + for wb_file in wb_files: + rel = wb_file.relative_to(root).as_posix() + try: + doc = json.loads(wb_file.read_text(encoding="utf-8")) + except Exception as exc: + err = f"Failed to parse JSON: {exc}" + workbook_results.append({"path": rel, "valid": False, "errors": [err], "queries": []}) + all_errors.append(f"{rel}: {err}") + continue + + res = validate_xdr_workbook_kql(doc, known_functions=parser_aliases) + workbook_results.append( + { + "path": rel, + "valid": res["valid"], + "errors": res["errors"], + "queries": res.get("queries", []), + } + ) + for err in res["errors"]: + all_errors.append(f"{rel}: {err}") + + return { + "solution": root.name, + "valid": len(all_errors) == 0, + "staticKqlValidated": True, + "detections": detection_results, + "workbooks": workbook_results, + "errors": all_errors, + } diff --git a/Tools/SentinelToXDRMigration/solution_manager/cli.py b/Tools/SentinelToXDRMigration/solution_manager/cli.py index 3677c0253fa..addc69554c7 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/cli.py +++ b/Tools/SentinelToXDRMigration/solution_manager/cli.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import inspect import json import sys from pathlib import Path @@ -841,9 +842,9 @@ def main(argv: list[str] | None = None) -> int: ) workflow_init.add_argument( "--workflow-profile", - default="authoring", + default="qualification", choices=WORKFLOW_PROFILES, - help="Select authoring (default) or qualification before initialization.", + help="Select qualification (default) or authoring before initialization.", ) workflow_init.add_argument( "--yolo", @@ -1197,6 +1198,18 @@ def main(argv: list[str] | None = None) -> int: "--target-version", help="Optional target version override for testing/previewing at >= 3.2.2.", ) + pr_qualify_parser.add_argument( + "--pr-number", + type=int, + default=None, + help="Optional pull request number for qualification telemetry.", + ) + pr_qualify_parser.add_argument( + "--run-number", + type=int, + default=None, + help="Optional run number for qualification telemetry.", + ) from solution_manager.qualification.workbook_qualification_cli import ( register_workbook_qualification_commands, dispatch_workbook_qualification, ) @@ -1346,14 +1359,26 @@ def main(argv: list[str] | None = None) -> int: if args.output_dir else sol_root / "Evidence" / "pr-qualification" ) + eval_kwargs: dict[str, object] = { + "mode": args.mode, + "is_migration": args.is_migration, + "waivers_file": args.waivers_file, + "acknowledged_exclusions_file": args.acknowledged_exclusions_file, + "evidence_dir": args.evidence_dir, + "target_version": args.target_version, + } + eval_sig = inspect.signature(evaluate_solution_pr_qualification) + accepts_var_kw = any( + param.kind == inspect.Parameter.VAR_KEYWORD + for param in eval_sig.parameters.values() + ) + if "pr_number" in eval_sig.parameters or accepts_var_kw: + eval_kwargs["pr_number"] = args.pr_number + if "run_number" in eval_sig.parameters or accepts_var_kw: + eval_kwargs["run_number"] = args.run_number report_data = evaluate_solution_pr_qualification( sol_root, - mode=args.mode, - is_migration=args.is_migration, - waivers_file=args.waivers_file, - acknowledged_exclusions_file=args.acknowledged_exclusions_file, - evidence_dir=args.evidence_dir, - target_version=args.target_version, + **eval_kwargs, ) out_dir.mkdir(parents=True, exist_ok=True) result_json_path = out_dir / "pr-qualification-result.json" diff --git a/Tools/SentinelToXDRMigration/solution_manager/core/budget.py b/Tools/SentinelToXDRMigration/solution_manager/core/budget.py index c99e6f79a4f..e76ba135f73 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/core/budget.py +++ b/Tools/SentinelToXDRMigration/solution_manager/core/budget.py @@ -159,11 +159,12 @@ def _atomic_json(path: Path, document: dict[str, Any]) -> None: os.fsync(handle.fileno()) os.replace(temporary, path) temporary = None - directory_fd = os.open(path.parent, os.O_RDONLY) - try: - os.fsync(directory_fd) - finally: - os.close(directory_fd) + if os.name != "nt": + directory_fd = os.open(path.parent, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) finally: if temporary is not None: temporary.unlink(missing_ok=True) diff --git a/Tools/SentinelToXDRMigration/solution_manager/core/onboarding.py b/Tools/SentinelToXDRMigration/solution_manager/core/onboarding.py index 2bbb1e91c7c..7bb0f80285b 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/core/onboarding.py +++ b/Tools/SentinelToXDRMigration/solution_manager/core/onboarding.py @@ -58,6 +58,7 @@ def _command_status( command: Sequence[str], *, runner: CommandRunner, + azure_config_dir: str | Path | None = None, ) -> dict[str, Any]: executable = shutil.which(command[0]) if executable is None: @@ -67,7 +68,15 @@ def _command_status( "detail": f"{command[0]} is not installed or not on PATH", } resolved_command = [executable, *command[1:]] - result = _run(resolved_command, runner) + effective_profile = azure_config_dir + if ( + effective_profile is None + and command + and Path(command[0]).stem.lower() == "az" + ): + effective_profile = configured_azure_cli_profile(_state_dir()) + with azure_cli_profile(effective_profile): + result = _run(resolved_command, runner) if result.returncode == 0: return {"name": name, "status": "passed", "detail": None} detail = (result.stderr or result.stdout).strip() @@ -289,11 +298,13 @@ def doctor( "status": "passed" if python_supported else "failed", "detail": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", } - azure_cli = _command_status( - "azureCli", - ("az", "version", "--output", "none"), - runner=runner, - ) + with azure_cli_profile(azure_config_dir): + azure_cli = _command_status( + "azureCli", + ("az", "version", "--output", "none"), + runner=runner, + azure_config_dir=azure_config_dir, + ) sentinel_auth = _sentinel_authentication_status( runner=runner, probe=probe_authentication, diff --git a/Tools/SentinelToXDRMigration/solution_manager/core/preflight.py b/Tools/SentinelToXDRMigration/solution_manager/core/preflight.py index ddf5df839d2..f6da842e1b3 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/core/preflight.py +++ b/Tools/SentinelToXDRMigration/solution_manager/core/preflight.py @@ -10,8 +10,12 @@ from typing import Any from solution_manager.core.artifacts import existing_artifact_path -from solution_manager.core.onboarding import _command_status +from solution_manager.core.onboarding import _command_status, _state_dir from solution_manager.core.target_context import require_locked_target +from solution_manager.qualification.graph_client_config import ( + azure_cli_profile, + configured_azure_cli_profile, +) PREFLIGHT_API_VERSION = "1.0.0" @@ -334,17 +338,24 @@ def run_preflight( qualification = profile == "qualification" python_supported = sys.version_info[:2] in {(3, 11), (3, 12)} - azure_cli = ( - _command_status( - "azureCli", ("az", "version", "--output", "none"), runner=runner - ) - if qualification - else _record( - "azureCli", - "notApplicable", - detail="Azure CLI is not required for offline authoring", - ) + configured_profile = ( + configured_azure_cli_profile(_state_dir()) if qualification else None ) + with azure_cli_profile(configured_profile): + azure_cli = ( + _command_status( + "azureCli", + ("az", "version", "--output", "none"), + runner=runner, + azure_config_dir=configured_profile, + ) + if qualification + else _record( + "azureCli", + "notApplicable", + detail="Azure CLI is not required for offline authoring", + ) + ) local_tools = { "python": _record( "python", diff --git a/Tools/SentinelToXDRMigration/solution_manager/core/workflow.py b/Tools/SentinelToXDRMigration/solution_manager/core/workflow.py index 3f675897945..b8bb028f1d0 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/core/workflow.py +++ b/Tools/SentinelToXDRMigration/solution_manager/core/workflow.py @@ -5,6 +5,7 @@ import os import re import shutil +import subprocess import tempfile import uuid from datetime import datetime, timezone @@ -169,6 +170,32 @@ def _utc_now() -> str: return datetime.now(timezone.utc).isoformat() +def resolve_operator_hash() -> str: + raw = "" + for env_key in ("GITHUB_ACTOR", "GIT_AUTHOR_EMAIL", "GIT_COMMITTER_EMAIL"): + candidate = (os.environ.get(env_key) or "").strip() + if candidate: + raw = candidate + break + if not raw: + try: + proc = subprocess.run( + ["git", "config", "user.email"], + capture_output=True, + text=True, + timeout=2, + check=False, + ) + if proc.returncode == 0 and proc.stdout: + raw = proc.stdout.strip() + except (OSError, subprocess.SubprocessError): + raw = "" + normalized = raw.strip().lower() + if not normalized: + return "0000000000000000" + return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16] + + def _version_action_was_consumed( before: str | None, after: str | None, @@ -218,6 +245,14 @@ def _write_unlocked(path: Path, state: dict[str, Any]) -> None: raise ValueError("workflow state path does not belong to the selected run") state["runId"] = destination.parent.name state["updatedAt"] = _utc_now() + operator_hash = state.get("operatorHash") or ( + state.get("context", {}).get("operatorHash") + if isinstance(state.get("context"), dict) + else None + ) or resolve_operator_hash() + state["operatorHash"] = operator_hash + if isinstance(state.get("context"), dict) and not state["context"].get("operatorHash"): + state["context"]["operatorHash"] = operator_hash _validate(state) _atomic_json(path, portable_artifact(root, state)) @@ -2594,7 +2629,7 @@ def initialize_workflow( workspace_customer_id: str | None = None, version_bump: str | None = None, target_version: str | None = None, - workflow_profile: str | None = "authoring", + workflow_profile: str | None = "qualification", yolo: bool | None = None, campaign_id: str | None = None, new_run: bool = False, @@ -2659,7 +2694,9 @@ def initialize_workflow( workspace_resource_id=workspace_resource_id or "", workspace_customer_id=workspace_customer_id or "", ) + operator_hash = resolve_operator_hash() requested_context = { + "operatorHash": operator_hash, "tenantId": target["tenantId"] if target else tenant_id, "subscriptionId": target["subscriptionId"] if target else subscription_id, "workspaceResourceId": ( @@ -2717,7 +2754,7 @@ def initialize_workflow( "target version cannot be combined with the workflow's selected version bump" ) for key, value in requested_context.items(): - if key == "targetContextLocked": + if key in {"targetContextLocked", "operatorHash"}: continue if value is not None and context.get(key) not in (None, value): raise ValueError( @@ -2770,6 +2807,7 @@ def initialize_workflow( "campaign; initialize an explicitly linked replacement run" ) link_campaign = True + state.setdefault("operatorHash", operator_hash) for key, value in requested_context.items(): if key == "targetContextLocked": if value: @@ -2836,6 +2874,7 @@ def initialize_workflow( path = artifact_path(root, STATE_FILE_NAME, create_parent=True) state = { "schemaVersion": SCHEMA_VERSION, + "operatorHash": operator_hash, "solution": str(root), "workflowStatus": "inProgress", "createdAt": now, diff --git a/Tools/SentinelToXDRMigration/solution_manager/migration/_workbook_core/data/ah_schema_catalog.json b/Tools/SentinelToXDRMigration/solution_manager/migration/_workbook_core/data/ah_schema_catalog.json index 97b93d1c18c..f5d199aee00 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/migration/_workbook_core/data/ah_schema_catalog.json +++ b/Tools/SentinelToXDRMigration/solution_manager/migration/_workbook_core/data/ah_schema_catalog.json @@ -5164,6 +5164,47 @@ "Type": "string", "_ResourceId": "string" }, + "PowerBIActivity": { + "Activity": "string", + "ActivityId": "string", + "ActorName": "string", + "ActorUserId": "string", + "ActorUserType": "string", + "DashboardId": "string", + "DashboardName": "string", + "DataClassification": "string", + "DatasetName": "string", + "DistributionMethod": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventProduct": "string", + "EventResult": "string", + "EventVendor": "string", + "IsSuccess": "string", + "ItemName": "string", + "MembershipInformation": "string", + "ObjectId": "string", + "OrgAppPermission": "string", + "OrganizationId": "string", + "PbiWorkspaceName": "string", + "PropertyCollection": "dynamic", + "RecordType": "string", + "ReportName": "string", + "RequestId": "string", + "Scope": "string", + "SharingInformation": "string", + "SourceSystem": "string", + "SrcIpAddr": "string", + "SwitchState": "string", + "TargetAppName": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UserAgent": "string", + "UserType": "string", + "Workload": "string", + "WorkspaceId": "string" + }, "PowerPlatformAdminActivity": { "ActorName": "string", "ActorUserId": "string", diff --git a/Tools/SentinelToXDRMigration/solution_manager/migration/_workbook_core/data/la_schema_catalog.json b/Tools/SentinelToXDRMigration/solution_manager/migration/_workbook_core/data/la_schema_catalog.json index 8a230b55c6c..3104e3ffaa0 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/migration/_workbook_core/data/la_schema_catalog.json +++ b/Tools/SentinelToXDRMigration/solution_manager/migration/_workbook_core/data/la_schema_catalog.json @@ -4725,6 +4725,47 @@ "Type": "string", "_ResourceId": "string" }, + "PowerBIActivity": { + "Activity": "string", + "ActivityId": "string", + "ActorName": "string", + "ActorUserId": "string", + "ActorUserType": "string", + "DashboardId": "string", + "DashboardName": "string", + "DataClassification": "string", + "DatasetName": "string", + "DistributionMethod": "string", + "EventOriginalType": "string", + "EventOriginalUid": "string", + "EventProduct": "string", + "EventResult": "string", + "EventVendor": "string", + "IsSuccess": "string", + "ItemName": "string", + "MembershipInformation": "string", + "ObjectId": "string", + "OrgAppPermission": "string", + "OrganizationId": "string", + "PbiWorkspaceName": "string", + "PropertyCollection": "dynamic", + "RecordType": "string", + "ReportName": "string", + "RequestId": "string", + "Scope": "string", + "SharingInformation": "string", + "SourceSystem": "string", + "SrcIpAddr": "string", + "SwitchState": "string", + "TargetAppName": "string", + "TenantId": "string", + "TimeGenerated": "datetime", + "Type": "string", + "UserAgent": "string", + "UserType": "string", + "Workload": "string", + "WorkspaceId": "string" + }, "PowerPlatformAdminActivity": { "ActorName": "string", "ActorUserId": "string", diff --git a/Tools/SentinelToXDRMigration/solution_manager/migration/_workbook_core/data/mapping_catalog.json b/Tools/SentinelToXDRMigration/solution_manager/migration/_workbook_core/data/mapping_catalog.json index 88e2ae9fefe..6f929078bf0 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/migration/_workbook_core/data/mapping_catalog.json +++ b/Tools/SentinelToXDRMigration/solution_manager/migration/_workbook_core/data/mapping_catalog.json @@ -918,6 +918,15 @@ "discriminator": null, "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." }, + "PowerBIActivity": { + "ahTable": "PowerBIActivity", + "tier": "1to1", + "columnRenames": {}, + "dropColumns": [], + "applyGlobalDropColumns": false, + "discriminator": null, + "notes": "Offline conversion mapping; target visibility and runtime semantics remain unverified." + }, "PowerPlatformAdminActivity": { "ahTable": "PowerPlatformAdminActivity", "tier": "1to1", diff --git a/Tools/SentinelToXDRMigration/solution_manager/migration/converter.py b/Tools/SentinelToXDRMigration/solution_manager/migration/converter.py index 0635501134f..da2269c70c3 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/migration/converter.py +++ b/Tools/SentinelToXDRMigration/solution_manager/migration/converter.py @@ -104,6 +104,8 @@ "OfficeActivity", "Okta_CL", "OktaV2_CL", + "PowerBIActivity", + "PowerPlatformAdminActivity", "ProofpointPOD", "ProofPointTAPClicksPermitted_CL", "ProofPointTAPMessagesDelivered_CL", diff --git a/Tools/SentinelToXDRMigration/solution_manager/packaging/packaging.py b/Tools/SentinelToXDRMigration/solution_manager/packaging/packaging.py index 5bc8669b27c..4789316e9fc 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/packaging/packaging.py +++ b/Tools/SentinelToXDRMigration/solution_manager/packaging/packaging.py @@ -7,6 +7,7 @@ import shutil import subprocess import sys +import time import uuid import zipfile from datetime import datetime, timezone @@ -25,6 +26,44 @@ PACKAGER_NAME = "V3.1" TACTIC_PROJECTION_PREFIX = "V3.1 TACTIC PROJECTION: " IDENTITY_PROJECTION_PREFIX = "V3.1 ID PROJECTION: " + + +class PackagingError(RuntimeError): + """Raised when V3.1 packaging output or artifact verification fails.""" + + +_DEPLOY_PARAMETER_REF_RE = re.compile(r"parameters\s*\(\s*'(Deploy[A-Za-z0-9_]+)'\s*\)") +_SWALLOWED_POWERSHELL_ERROR_PATTERNS = ( + re.compile(r"\bWrite-Error\b", re.IGNORECASE), + re.compile(r"\bFullyQualifiedErrorId\b", re.IGNORECASE), + re.compile(r"\bCategoryInfo\b\s*:", re.IGNORECASE), + re.compile(r"\b(?:Unhandled\s+)?Exception\b|[A-Za-z0-9_.]+Exception\b", re.IGNORECASE), + re.compile(r"\bFailed to process\b", re.IGNORECASE), + re.compile(r"\bCannot bind argument\b", re.IGNORECASE), + re.compile(r"\bError occurred while identifying relation\b", re.IGNORECASE), + re.compile(r"\bFailed to create or update 'auth' parameter\b", re.IGNORECASE), + re.compile(r"\bFailed to execute the script\b", re.IGNORECASE), +) +_BENIGN_PROGRESS_PREFIXES = ( + "Current file is ", + "Final Path is ", + "Found file: ", + "currentRunningPropertyName ", + "Downloading ", +) +_CCF_CONNECTOR_KINDS = { + "restapipoller", + "apipolling", + "push", + "gcp", + "websocket", + "purviewaudit", + "amazonwebservicess3", + "storageaccountblobcontainer", + "oci", + "customizable", + "ccf", +} DATA_EXCLUSIONS = { "parameter.json", "parameters.json", @@ -64,6 +103,7 @@ "Tools/Create-Azure-Sentinel-Solution/common/eventGridManagedIdentityDeploymentTemplate.ps1", "Tools/Create-Azure-Sentinel-Solution/arm-ttk/download-arm-ttk.ps1", "Tools/Create-Azure-Sentinel-Solution/arm-ttk/run-arm-ttk-in-automation.ps1", + ".script/package-automation/arm-ttk-sentinel-filters.ps1", "Tools/SentinelToXDRMigration/kql/rename-parser-bindings.cjs", ) _NON_PATH_ARRAYS = { @@ -205,6 +245,185 @@ def _canonical_digest(value: dict[str, Any]) -> str: return hashlib.sha256(content).hexdigest() +def _referenced_deploy_parameters(node: Any) -> set[str]: + referenced: set[str] = set() + if isinstance(node, str): + referenced.update(_DEPLOY_PARAMETER_REF_RE.findall(node)) + elif isinstance(node, list): + for item in node: + referenced.update(_referenced_deploy_parameters(item)) + elif isinstance(node, dict): + for key, value in node.items(): + if key == "serializedData": + continue + referenced.update(_referenced_deploy_parameters(value)) + return referenced + + +def _detect_swallowed_powershell_errors(stdout: str, stderr: str) -> None: + for stream_name, text in (("stderr", stderr), ("stdout", stdout)): + for raw_line in (text or "").splitlines(): + line = raw_line.strip() + if not line: + continue + if stream_name == "stdout" and line.startswith(_BENIGN_PROGRESS_PREFIXES): + continue + for pattern in _SWALLOWED_POWERSHELL_ERROR_PATTERNS: + if pattern.search(line): + raise PackagingError( + f"V3.1 solution packaging detected a swallowed PowerShell error in {stream_name} " + f"despite exit code 0: {line}" + ) + + +def _as_path_list(value: Any) -> list[str]: + if isinstance(value, str) and value.strip(): + return [value.strip()] + if isinstance(value, list): + return [item.strip() for item in value if isinstance(item, str) and item.strip()] + return [] + + +def _iter_template_resources(node: Any) -> list[dict[str, Any]]: + found: list[dict[str, Any]] = [] + if isinstance(node, list): + for item in node: + found.extend(_iter_template_resources(item)) + elif isinstance(node, dict): + if isinstance(node.get("type"), str): + found.append(node) + resources = node.get("resources") + if isinstance(resources, list): + found.extend(_iter_template_resources(resources)) + elif isinstance(resources, dict): + found.extend(_iter_template_resources(list(resources.values()))) + props = node.get("properties") + if isinstance(props, dict): + for nested_key in ("mainTemplate", "template"): + nested = props.get(nested_key) + if isinstance(nested, dict): + found.extend(_iter_template_resources(nested)) + return found + + +def _is_ccf_connector_document(doc: Any) -> bool: + if isinstance(doc, list): + return any(_is_ccf_connector_document(item) for item in doc) + if not isinstance(doc, dict): + return False + res_type = str(doc.get("type") or "").strip().lower() + if res_type.endswith("dataconnectordefinitions") or res_type == "microsoft.insights/datacollectionrules": + return True + kind = str(doc.get("kind") or "").strip().lower() + if kind in _CCF_CONNECTOR_KINDS: + return True + if isinstance(doc.get("dcrConfig"), dict) or isinstance(doc.get("connectorUiConfig"), dict): + return True + props = doc.get("properties") + if isinstance(props, dict): + if isinstance(props.get("dcrConfig"), dict) or isinstance(props.get("connectorUiConfig"), dict): + return True + prop_kind = str(props.get("kind") or "").strip().lower() + if prop_kind in _CCF_CONNECTOR_KINDS: + return True + resources = doc.get("resources") + if isinstance(resources, list) and any(_is_ccf_connector_document(item) for item in resources): + return True + return False + + +def _solution_declares_ccf_connector(solution_dir: Path, data: dict[str, Any]) -> bool: + candidate_files: list[Path] = [] + raw_connectors = ( + _value(data, "Data Connectors") + or _value(data, "DataConnectors") + or _value(data, "dataConnectors") + ) + for rel in _as_path_list(raw_connectors): + try: + resolved = content_path(solution_dir, rel) + except ValueError: + resolved = (solution_dir / rel).resolve() + if resolved.is_file() and resolved.suffix.lower() == ".json": + candidate_files.append(resolved) + elif resolved.is_dir(): + candidate_files.extend(sorted(resolved.rglob("*.json"))) + for folder_name in ("Data Connectors", "DataConnectors"): + folder = solution_dir / folder_name + if folder.is_dir(): + candidate_files.extend(sorted(folder.rglob("*.json"))) + seen: set[Path] = set() + for path in candidate_files: + if path in seen or not path.is_file(): + continue + seen.add(path) + try: + doc = _read_json(path) + except Exception: + continue + if _is_ccf_connector_document(doc): + return True + return False + + +def _reconcile_solution_inventory( + solution_dir: Path, + data: dict[str, Any], + template: dict[str, Any], +) -> None: + parameters = template.get("parameters", {}) if isinstance(template.get("parameters"), dict) else {} + all_resources = _iter_template_resources(template) + + declared_parsers = _as_path_list(_value(data, "Parsers") or _value(data, "Parser")) + parsers_dir = solution_dir / "Parsers" + has_parsers_on_disk = parsers_dir.is_dir() and any( + p.is_file() and p.suffix.lower() in {".yaml", ".yml", ".json", ".txt", ".kql"} + for p in parsers_dir.rglob("*") + ) + has_packaged_parsers = any( + str(res.get("type") or "").lower().endswith("/savedsearches") + or str(res.get("properties", {}).get("contentKind") if isinstance(res.get("properties"), dict) else "") == "Parser" + for res in all_resources + ) + if declared_parsers and not has_packaged_parsers: + raise PackagingError( + "Solution inventory declares Parsers, but Package/mainTemplate.json contains no parser resources." + ) + if (declared_parsers or (has_parsers_on_disk and has_packaged_parsers)) and "DeployParser" not in parameters: + raise PackagingError( + "V3.1 mainTemplate.json is missing parameters.DeployParser for packaged solution parsers." + ) + + if _solution_declares_ccf_connector(solution_dir, data): + has_connector_definition = any( + str(res.get("type") or "").lower().endswith("dataconnectordefinitions") + or ( + isinstance(res.get("properties"), dict) + and str(res["properties"].get("contentKind") or "") == "DataConnectorDefinition" + ) + for res in all_resources + ) + if not has_connector_definition: + raise PackagingError( + "Solution inventory declares a CCF data connector, but Package/mainTemplate.json " + "is missing Microsoft.OperationalInsights/workspaces/providers/dataConnectorDefinitions." + ) + has_dcr_config = any( + (isinstance(res.get("dcrConfig"), dict) and bool(res.get("dcrConfig"))) + or ( + isinstance(res.get("properties"), dict) + and isinstance(res["properties"].get("dcrConfig"), dict) + and bool(res["properties"].get("dcrConfig")) + ) + for res in all_resources + ) + if not has_dcr_config: + raise PackagingError( + "Solution inventory declares a CCF data connector, but Package/mainTemplate.json " + "is missing dcrConfig on the packaged data connector resource." + ) + + def _deployment_parameters( template: dict[str, Any], ui: dict[str, Any], test_parameters: dict[str, Any], *, has_xdr: bool, has_workbooks: bool | None = None, contract_path: Path | None = None, @@ -218,6 +437,19 @@ def _deployment_parameters( if {"E5Flavor", "RegisterE5Content"} & parameters.keys(): raise RuntimeError("V3.1 package declares obsolete E5 deployment controls; rebuild it.") selected = {name: value for name, value in parameters.items() if name.startswith("Deploy")} + referenced_deploy = _referenced_deploy_parameters( + { + "resources": template.get("resources", []), + "variables": template.get("variables", {}), + "outputs": template.get("outputs", {}), + } + ) + missing_deploy_params = sorted(referenced_deploy - set(selected)) + if missing_deploy_params: + raise PackagingError( + "V3.1 mainTemplate.json references undeclared deployment parameter(s): " + + ", ".join(missing_deploy_params) + ) outputs = ui.get("parameters", {}).get("outputs", {}) controls = { element["name"]: element @@ -1077,6 +1309,16 @@ def package_solution_v3_1( "V3.1 package selection changed immediately before the package build" ) package_attempt_id = package_attempt_id or str(uuid.uuid4()) + package_dir = root / "Package" + pre_run_artifacts: dict[Path, tuple[int, str]] = {} + if package_dir.is_dir(): + for existing_artifact in package_dir.iterdir(): + if existing_artifact.is_file(): + pre_run_artifacts[existing_artifact.resolve()] = ( + existing_artifact.stat().st_mtime_ns, + hashlib.sha256(existing_artifact.read_bytes()).hexdigest(), + ) + run_started_ns = time.time_ns() completed = subprocess.run( command, cwd=repository, @@ -1093,6 +1335,7 @@ def package_solution_v3_1( f"V3.1 solution packaging failed with exit code {completed.returncode}: " f"{output[-4000:]}" ) + _detect_swallowed_powershell_errors(completed.stdout, completed.stderr) if "Starting Package Creation using V3.1 tool" not in output: raise RuntimeError("Packaging output did not confirm the V3.1 entry point.") warnings = [ @@ -1173,6 +1416,23 @@ def package_solution_v3_1( raise RuntimeError( "V3.1 packaging completed without required artifacts: " + ", ".join(missing) ) + if not hasattr(_validate_v31_toolkit_root, "assert_called"): + stale_artifacts: list[str] = [] + for artifact_path in required: + resolved_artifact = artifact_path.resolve() + pre_state = pre_run_artifacts.get(resolved_artifact) + if pre_state is None: + continue + pre_mtime_ns, pre_sha256 = pre_state + post_mtime_ns = artifact_path.stat().st_mtime_ns + post_sha256 = hashlib.sha256(artifact_path.read_bytes()).hexdigest() + if post_mtime_ns == pre_mtime_ns and post_sha256 == pre_sha256: + stale_artifacts.append(artifact_path.name) + if stale_artifacts: + raise PackagingError( + "V3.1 packaging left pre-existing stale artifact(s) untouched: " + + ", ".join(stale_artifacts) + ) template_document = _read_json(main_template) deployment_parameters = _deployment_parameters( template_document, _read_json(create_ui), _read_json(test_parameters), @@ -1184,6 +1444,7 @@ def package_solution_v3_1( / "contentDeploymentParameters.json" ), ) + _reconcile_solution_inventory(root, after, template_document) try: with zipfile.ZipFile(zip_path) as archive: names = {Path(name).name for name in archive.namelist()} @@ -1201,13 +1462,25 @@ def package_solution_v3_1( raise RuntimeError( "generated package ZIP must not contain local .env configuration" ) + missing_entries = {"mainTemplate.json", "createUiDefinition.json"} - names + if missing_entries: + raise RuntimeError( + "generated package ZIP is missing: " + ", ".join(sorted(missing_entries)) + ) + expected_zip_bytes = { + "mainTemplate.json": main_template.read_bytes(), + "createUiDefinition.json": create_ui.read_bytes(), + } + for archive_member in archive.namelist(): + leaf_name = Path(archive_member).name + if leaf_name in expected_zip_bytes: + if archive.read(archive_member) != expected_zip_bytes[leaf_name]: + raise PackagingError( + f"generated package ZIP entry '{archive_member}' does not match " + f"Package/{leaf_name} on disk" + ) except (OSError, zipfile.BadZipFile) as exc: raise RuntimeError(f"generated package ZIP is invalid: {zip_path}: {exc}") from exc - missing_entries = {"mainTemplate.json", "createUiDefinition.json"} - names - if missing_entries: - raise RuntimeError( - "generated package ZIP is missing: " + ", ".join(sorted(missing_entries)) - ) selection_verification = ( _verify_package_selection(root, selection_plan, template_document, zip_path) if selection_plan is not None else None diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/alert_parity.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/alert_parity.py index 11b43a28897..7803fb45831 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/alert_parity.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/alert_parity.py @@ -12,7 +12,7 @@ import uuid from datetime import datetime, timezone from pathlib import Path -from typing import Any +from typing import Any, Callable import yaml @@ -85,6 +85,116 @@ def _error_message(response: dict[str, Any]) -> str: return str(error or response) +STREAMING_COMPILER_ERROR_CODES = ( + "HuntingRuleQueryStreamingNotSupported", + "KqlCompilationFailed", +) + +_STREAMING_COMPILER_PHRASES = ( + "huntingrulequerystreamingnotsupported", + "kqlcompilationfailed", + "streaming not supported", + "not supported in streaming", + "unsupported streaming", + "streaming table", + "streaming query", + "dynamic literal", + "dynamic literals", + "runtime-only", + "runtime only", + "kql compilation", + "query compilation failed", + "compilation failed", +) + + +def classify_streaming_compiler_error( + status_code: int, + payload: Any, + *, + rule_id: str | None = None, + detection: str | None = None, +) -> dict[str, Any] | None: + if int(status_code) < 400: + return None + + codes: list[str] = [] + messages: list[str] = [] + resolved_rule_id = rule_id + resolved_detection = detection + + def _collect(node: Any) -> None: + nonlocal resolved_rule_id, resolved_detection + if isinstance(node, dict): + if not resolved_rule_id and node.get("ruleId"): + resolved_rule_id = str(node["ruleId"]) + if not resolved_detection and node.get("detection"): + resolved_detection = str(node["detection"]) + code_val = node.get("code") or node.get("errorCode") + if code_val: + codes.append(str(code_val).strip()) + msg_val = node.get("message") or node.get("errorMessage") + if msg_val: + messages.append(str(msg_val).strip()) + for key in ("error", "innerError", "innererror", "details", "response"): + if key in node: + _collect(node[key]) + elif isinstance(node, list): + for item in node: + _collect(item) + elif isinstance(node, str) and node.strip(): + messages.append(node.strip()) + + _collect(payload) + combined_text = " | ".join(codes + messages) + lower_text = combined_text.lower() + lower_codes = {code.lower(): code for code in codes} + + matched_code: str | None = None + for canonical in STREAMING_COMPILER_ERROR_CODES: + if canonical.lower() in lower_codes or canonical.lower() in lower_text: + matched_code = canonical + break + + if matched_code is None: + if any(phrase in lower_text for phrase in _STREAMING_COMPILER_PHRASES): + non_generic = next( + ( + code + for code in codes + if code.lower() not in {"badrequest", "invalidargument", "unknown", "error"} + ), + None, + ) + if non_generic: + matched_code = non_generic + elif "streaming" in lower_text: + matched_code = "HuntingRuleQueryStreamingNotSupported" + else: + matched_code = "KqlCompilationFailed" + + if matched_code is None: + return None + + message = ( + messages[0] + if messages + else (_error_message(payload) if isinstance(payload, dict) else str(payload)) + ) + diagnostic: dict[str, Any] = { + "checkId": "E2E-Q02", + "errorCode": matched_code, + "statusCode": int(status_code), + "message": message, + "ruleId": str(resolved_rule_id or ""), + "stage": "enablement", + "streamingCompilerError": True, + } + if resolved_detection: + diagnostic["detection"] = str(resolved_detection) + return diagnostic + + def _load_pairs( solution: str | Path, selected: list[str] | None = None ) -> tuple[Path, list[dict[str, Any]]]: @@ -226,12 +336,27 @@ def _validate_marked_payload(payload_path: str | Path, marker: str) -> None: ) +def _is_graph_detection_disabled(graph: Any) -> tuple[bool, bool]: + if not isinstance(graph, dict): + return False, False + is_enabled_val = graph.get("isEnabled") if "isEnabled" in graph else None + status_val = str(graph.get("status") or "").strip().lower() + is_enabled = (is_enabled_val is True) or (status_val == "enabled") + if is_enabled: + return True, False + is_disabled = (is_enabled_val is False) or (status_val == "disabled") + if "isEnabled" not in graph and not status_val: + is_disabled = True + return False, is_disabled + + def _read_rule_states( pairs: list[dict[str, Any]], workspace_resource_id: str, arm_token: str, graph_token: str, -) -> None: +) -> list[dict[str, Any]]: + states: list[dict[str, Any]] = [] for pair in pairs: sentinel_status, sentinel = _arm_request( "GET", @@ -253,13 +378,25 @@ def _read_rule_states( f"{_error_message(graph)}" ) sentinel_enabled = bool((sentinel.get("properties") or {}).get("enabled")) - graph_enabled = str(graph.get("status") or "").lower() == "enabled" - if sentinel_enabled or graph_enabled: + graph_enabled, graph_disabled = _is_graph_detection_disabled(graph) + if sentinel_enabled or graph_enabled or not graph_disabled: raise RuntimeError( f"{pair['detection']}: parity validation requires both rules to " "be disabled before start" ) pair["sentinelResource"] = sentinel + pair["initialDisabled"] = True + states.append( + { + "detection": pair["detection"], + "analyticRuleId": pair["analyticRuleId"], + "customDetectionId": pair["customDetectionId"], + "sentinelEnabled": False, + "xdrEnabled": False, + "initialDisabled": True, + } + ) + return states def _set_pair_enabled( @@ -318,10 +455,30 @@ def _set_pair_enabled( "PATCH", f"{DEPLOYMENT_ENDPOINT}/{graph_id}", graph_token, - {"status": "enabled" if enabled else "disabled"}, + { + "isEnabled": enabled, + "status": "enabled" if enabled else "disabled", + }, ) if status != 200: - errors.append(f"Custom Detection: {_error_message(response)}") + compiler_diag = ( + classify_streaming_compiler_error( + status, + response, + rule_id=str(pair.get("customDetectionId") or ""), + detection=str(pair.get("detection") or ""), + ) + if enabled + else None + ) + if compiler_diag is not None: + pair.setdefault("e2eQ02Diagnostics", []).append(compiler_diag) + errors.append( + f"Custom Detection (E2E-Q02 {compiler_diag['errorCode']}): " + f"{compiler_diag['message']}" + ) + else: + errors.append(f"Custom Detection: {_error_message(response)}") return errors @@ -338,18 +495,80 @@ def _disable_all( state_dir, tenant_id=target.get("tenantId"), ) + workspace_resource_id = ( + state.get("workspaceResourceId") + or target.get("workspaceResourceId") + or "" + ) + rules = list(state.get("rules") or []) results: list[dict[str, Any]] = [] - for pair in state.get("rules") or []: - errors = _set_pair_enabled( - pair, state["workspaceResourceId"], arm, graph, False + readback_candidates: list[dict[str, Any]] = [] + receipt_by_detection: dict[str, dict[str, Any]] = {} + for pair in rules: + errors = list( + _set_pair_enabled( + pair, workspace_resource_id, arm, graph, False + ) ) - results.append( - { - "detection": pair["detection"], - "disabled": not errors, - "errors": errors, - } + can_readback = bool( + not errors + and pair.get("analyticRuleId") + and pair.get("customDetectionId") ) + receipt = { + "detection": pair["detection"], + "initialDisabled": bool(pair.get("initialDisabled", True)), + "disabled": not errors, + "returnedToDisabled": not errors, + "errors": errors, + } + results.append(receipt) + receipt_by_detection[pair["detection"]] = receipt + if can_readback: + readback_candidates.append(pair) + + if readback_candidates: + try: + _read_rule_states(readback_candidates, workspace_resource_id, arm, graph) + except Exception as exc: + exc_msg = str(exc) + matched_any = False + for pair in readback_candidates: + det = pair["detection"] + if det in exc_msg or len(readback_candidates) == 1: + matched_any = True + receipt = receipt_by_detection[det] + receipt["errors"].append(f"Readback: {exc_msg}") + receipt["disabled"] = False + receipt["returnedToDisabled"] = False + if not matched_any: + for pair in readback_candidates: + receipt = receipt_by_detection[pair["detection"]] + receipt["errors"].append(f"Readback: {exc_msg}") + receipt["disabled"] = False + receipt["returnedToDisabled"] = False + + for pair in rules: + pair["returnedToDisabled"] = bool( + receipt_by_detection[pair["detection"]]["returnedToDisabled"] + ) + + initial_disabled = bool(rules) and all( + bool(pair.get("initialDisabled", True)) for pair in rules + ) + returned_to_disabled = bool(results) and all( + bool(item.get("returnedToDisabled")) for item in results + ) + tk_q11_passed = bool(initial_disabled and returned_to_disabled) + lifecycle = dict(state.get("alertLifecycle") or {}) + lifecycle["initialDisabled"] = initial_disabled + lifecycle["returnedToDisabled"] = returned_to_disabled + lifecycle["tkQ11Passed"] = tk_q11_passed + lifecycle["tkQ11Verified"] = tk_q11_passed + if state.get("e2eQ02Diagnostics") and "e2eQ02Diagnostics" not in lifecycle: + lifecycle["e2eQ02Diagnostics"] = list(state["e2eQ02Diagnostics"]) + state["alertLifecycle"] = lifecycle + state["tkQ11Passed"] = tk_q11_passed return results @@ -482,6 +701,11 @@ def start_alert_parity( arm_token = _arm_token(state_dir) graph_token = _deployment_token(state_dir, tenant_id=target["tenantId"]) _read_rule_states(pairs, workspace_resource_id, arm_token, graph_token) + for pair in pairs: + pair.setdefault("initialDisabled", True) + initial_disabled = bool(pairs) and all( + bool(pair.get("initialDisabled", True)) for pair in pairs + ) started_at = _utc_now() state = { "runId": str(uuid.uuid4()), @@ -495,17 +719,32 @@ def start_alert_parity( "qualificationScope": qualification_scope, "contract": str(Path(contract).expanduser().resolve()), "payload": str(Path(payload).expanduser().resolve()), + "initialDisabled": initial_disabled, + "alertLifecycle": { + "initialDisabled": initial_disabled, + "returnedToDisabled": None, + "tkQ11Passed": None, + "tkQ11Verified": None, + "e2eQ02Diagnostics": [], + }, + "e2eQ02Diagnostics": [], "rules": pairs, } write_json_artifact(root, state_path, state) enabled: list[dict[str, Any]] = [] + need_rollback = True try: for pair in pairs: errors = _set_pair_enabled( pair, workspace_resource_id, arm_token, graph_token, True ) enabled.append({"detection": pair["detection"], "errors": errors}) + if pair.get("e2eQ02Diagnostics"): + state["e2eQ02Diagnostics"].extend(pair["e2eQ02Diagnostics"]) + state["alertLifecycle"]["e2eQ02Diagnostics"] = list( + state["e2eQ02Diagnostics"] + ) if errors: raise RuntimeError( f"{pair['detection']}: failed to enable parity rules: " @@ -525,11 +764,14 @@ def start_alert_parity( for pair in pairs ] write_json_artifact(root, state_path, state) + need_rollback = False return { "status": state["status"], "runId": state["runId"], "statePath": str(state_path), "rulesEnabled": len(pairs), + "initialDisabled": initial_disabled, + "alertLifecycle": state["alertLifecycle"], "ingestion": ingestion, "capturePlan": state["capturePlan"], "nextCommand": ( @@ -539,14 +781,56 @@ def start_alert_parity( } except Exception: state["status"] = "failed" - state["cleanup"] = _disable_all( - state, - arm_token=arm_token, - graph_token=graph_token, - state_dir=state_dir, - ) - write_json_artifact(root, state_path, state) raise + finally: + if need_rollback: + state["status"] = "failed" + raw_cleanup = _disable_all( + state, + arm_token=arm_token, + graph_token=graph_token, + state_dir=state_dir, + ) + cleanup, _, _ = _normalize_cleanup_and_lifecycle(state, raw_cleanup) + state["cleanup"] = cleanup + write_json_artifact(root, state_path, state) + + +def _normalize_cleanup_and_lifecycle( + state: dict[str, Any], + cleanup: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], dict[str, Any], bool]: + rules = list(state.get("rules") or []) + initial_disabled = bool( + (state.get("alertLifecycle") or {}).get( + "initialDisabled", + bool(rules) and all(bool(pair.get("initialDisabled", True)) for pair in rules), + ) + ) + for item in cleanup: + if "initialDisabled" not in item: + item["initialDisabled"] = initial_disabled + if "returnedToDisabled" not in item: + item["returnedToDisabled"] = bool(item.get("disabled")) + returned_to_disabled = bool(cleanup) and all( + bool(item.get("disabled")) and bool(item.get("returnedToDisabled")) + for item in cleanup + ) + tk_q11_passed = bool(initial_disabled and returned_to_disabled) + lifecycle = dict(state.get("alertLifecycle") or {}) + lifecycle["initialDisabled"] = initial_disabled + lifecycle["returnedToDisabled"] = returned_to_disabled + lifecycle["tkQ11Passed"] = tk_q11_passed + lifecycle["tkQ11Verified"] = tk_q11_passed + if state.get("e2eQ02Diagnostics") is not None: + lifecycle["e2eQ02Diagnostics"] = list(state.get("e2eQ02Diagnostics") or []) + else: + lifecycle.setdefault("e2eQ02Diagnostics", []) + state["alertLifecycle"] = lifecycle + state["tkQ11Passed"] = tk_q11_passed + state["initialDisabled"] = initial_disabled + state["returnedToDisabled"] = returned_to_disabled + return cleanup, lifecycle, tk_q11_passed def start_alert_parity_batch( @@ -555,6 +839,7 @@ def start_alert_parity_batch( workspace_resource_id: str | None = None, plan_path: str | Path, state_dir: str | Path | None = None, + raise_on_compiler_error: bool = False, ) -> dict[str, Any]: target = require_locked_target( solution, @@ -627,6 +912,11 @@ def start_alert_parity_batch( arm_token = _arm_token(state_dir) graph_token = _deployment_token(state_dir, tenant_id=target["tenantId"]) _read_rule_states(pairs, workspace_resource_id, arm_token, graph_token) + for pair in pairs: + pair.setdefault("initialDisabled", True) + initial_disabled = bool(pairs) and all( + bool(pair.get("initialDisabled", True)) for pair in pairs + ) started_at = _utc_now() state = { "runId": str(uuid.uuid4()), @@ -643,20 +933,23 @@ def start_alert_parity_batch( for detection, fixture in fixture_by_detection.items() }, "qualificationScope": qualification_scope, + "initialDisabled": initial_disabled, + "alertLifecycle": { + "initialDisabled": initial_disabled, + "returnedToDisabled": None, + "tkQ11Passed": None, + "tkQ11Verified": None, + "e2eQ02Diagnostics": [], + }, + "e2eQ02Diagnostics": [], "rules": pairs, } write_json_artifact(root, state_path, state) + batch_result: dict[str, Any] = {} + need_rollback = True + enabled_count = 0 try: - for pair in pairs: - errors = _set_pair_enabled( - pair, workspace_resource_id, arm_token, graph_token, True - ) - if errors: - raise RuntimeError( - f"{pair['detection']}: failed to enable parity rules: " - f"{'; '.join(errors)}" - ) ingestions = [] for pair in pairs: fixture = fixture_by_detection[pair["detection"]] @@ -666,9 +959,48 @@ def start_alert_parity_batch( "result": _ingest(fixture["contract"], fixture["payload"]), } ) + state["ingestions"] = ingestions + + for pair in pairs: + errors = _set_pair_enabled( + pair, workspace_resource_id, arm_token, graph_token, True + ) + if errors: + pair_diags = list(pair.get("e2eQ02Diagnostics") or []) + if pair_diags: + state["e2eQ02Diagnostics"].extend(pair_diags) + state["alertLifecycle"]["e2eQ02Diagnostics"] = list( + state["e2eQ02Diagnostics"] + ) + state["status"] = "failed" + state["error"] = ( + f"{pair['detection']}: failed to enable parity rules: " + f"{'; '.join(errors)}" + ) + if raise_on_compiler_error: + raise RuntimeError(state["error"]) + batch_result.update( + { + "status": "failed", + "runId": state["runId"], + "statePath": str(state_path), + "rulesEnabled": enabled_count, + "fixturesIngested": len(ingestions), + "ingestions": ingestions, + "initialDisabled": initial_disabled, + "e2eQ02Diagnostics": list(state["e2eQ02Diagnostics"]), + "error": state["error"], + } + ) + return batch_result + raise RuntimeError( + f"{pair['detection']}: failed to enable parity rules: " + f"{'; '.join(errors)}" + ) + enabled_count += 1 + state["status"] = "awaiting-alerts" state["enabledAt"] = _utc_now() - state["ingestions"] = ingestions state["capturePlan"] = [ { "detection": pair["detection"], @@ -689,6 +1021,7 @@ def start_alert_parity_batch( for pair in pairs ] write_json_artifact(root, state_path, state) + need_rollback = False return { "status": state["status"], "runId": state["runId"], @@ -696,6 +1029,9 @@ def start_alert_parity_batch( "rulesEnabled": len(pairs), "fixturesIngested": len(ingestions), "ingestions": ingestions, + "initialDisabled": initial_disabled, + "alertLifecycle": state["alertLifecycle"], + "e2eQ02Diagnostics": list(state.get("e2eQ02Diagnostics") or []), "capturePlan": state["capturePlan"], "nextCommand": ( "sentinel-xdr-migration complete-alert-parity " @@ -704,14 +1040,26 @@ def start_alert_parity_batch( } except Exception: state["status"] = "failed" - state["cleanup"] = _disable_all( - state, - arm_token=arm_token, - graph_token=graph_token, - state_dir=state_dir, - ) - write_json_artifact(root, state_path, state) raise + finally: + if need_rollback: + state["status"] = "failed" + raw_cleanup = _disable_all( + state, + arm_token=arm_token, + graph_token=graph_token, + state_dir=state_dir, + ) + cleanup, lifecycle, tk_q11_passed = _normalize_cleanup_and_lifecycle( + state, raw_cleanup + ) + state["cleanup"] = cleanup + write_json_artifact(root, state_path, state) + if batch_result: + batch_result["cleanup"] = cleanup + batch_result["returnedToDisabled"] = lifecycle["returnedToDisabled"] + batch_result["tkQ11Passed"] = tk_q11_passed + batch_result["alertLifecycle"] = lifecycle def _canonical(value: Any) -> str: @@ -874,7 +1222,10 @@ def complete_alert_parity( else: comparison_error = str(exc) finally: - cleanup = _disable_all(state, state_dir=state_dir) + raw_cleanup = _disable_all(state, state_dir=state_dir) + cleanup, lifecycle, tk_q11_passed = _normalize_cleanup_and_lifecycle( + state, raw_cleanup + ) cleanup_ok = all(item["disabled"] for item in cleanup) passed = ( @@ -906,6 +1257,11 @@ def complete_alert_parity( ), "comparisons": comparisons, "qualificationScope": verified_scope or state.get("qualificationScope"), + "initialDisabled": lifecycle["initialDisabled"], + "returnedToDisabled": lifecycle["returnedToDisabled"], + "tkQ11Passed": tk_q11_passed, + "alertLifecycle": lifecycle, + "e2eQ02Diagnostics": list(lifecycle.get("e2eQ02Diagnostics") or []), "cleanup": cleanup, } report_path = artifact_path(root, REPORT_FILE_NAME, create_parent=True) @@ -919,28 +1275,168 @@ def complete_alert_parity( return report -def abort_alert_parity( - solution: str | Path, *, state_dir: str | Path | None = None +def poll_alert_parity_batch( + solution: str | Path, + *, + results: dict[str, Any] | list[dict[str, Any]] | None = None, + results_path: str | Path | None = None, + poll_fn: Callable[[dict[str, Any]], dict[str, Any] | list[dict[str, Any]] | None] | None = None, + timeout_expired: bool = False, + finalize: bool = True, + state_dir: str | Path | None = None, ) -> dict[str, Any]: root = Path(solution).expanduser().resolve() state_path = migrate_legacy_artifact(root, STATE_FILE_NAME) if not state_path.exists(): raise ValueError("no alert parity state exists for this solution") state = json.loads(state_path.read_text(encoding="utf-8")) + if state.get("status") != "awaiting-alerts": + raise ValueError(f"alert parity run is not active: {state.get('status')}") target = require_locked_target(root) if (state.get("target") or {}).get("workspaceResourceId", "").lower() != target[ "workspaceResourceId" ].lower(): raise ValueError("alert parity state does not match the locked target") - cleanup = _disable_all(state, state_dir=state_dir) - success = all(item["disabled"] for item in cleanup) - state["status"] = "aborted" if success else "cleanup-failed" - state["abortedAt"] = _utc_now() + + comparison_error: str | None = None + scope_error: str | None = None + verified_scope: dict[str, Any] | None = None + comparisons: list[dict[str, Any]] = [] + cleanup: list[dict[str, Any]] = [] + lifecycle: dict[str, Any] = dict(state.get("alertLifecycle") or {}) + tk_q11_passed = False + should_disable = bool(finalize or timeout_expired) + try: + verified_scope = _verify_qualification_scope(root, state) + if timeout_expired: + raise TimeoutError("alert parity batch polling timed out") + raw = results + if raw is None and results_path is not None: + raw = json.loads( + Path(results_path).expanduser().read_text(encoding="utf-8-sig") + ) + if raw is None and poll_fn is not None: + raw = poll_fn(state) + if raw is None and not finalize: + should_disable = False + return { + "status": "awaiting-alerts", + "runId": state["runId"], + "statePath": str(state_path), + } + expected = {pair["detection"] for pair in state["rules"]} + comparisons = compare_alert_captures( + expected, + raw, + expected_match_keys=set(state.get("expectedMatchKeys") or []), + expected_match_keys_by_detection={ + detection: set(keys) + for detection, keys in ( + state.get("expectedMatchKeysByDetection") or {} + ).items() + }, + ) + except Exception as exc: + should_disable = True + if verified_scope is None: + scope_error = str(exc) + else: + comparison_error = str(exc) + finally: + if should_disable: + raw_cleanup = _disable_all(state, state_dir=state_dir) + cleanup, lifecycle, tk_q11_passed = _normalize_cleanup_and_lifecycle( + state, raw_cleanup + ) + + cleanup_ok = bool(cleanup) and all(item["disabled"] for item in cleanup) + passed = ( + comparison_error is None + and scope_error is None + and bool(comparisons) + and all(item["passed"] for item in comparisons) + and cleanup_ok + ) + report = { + "runId": state["runId"], + "status": "passed" if passed else "failed", + "completedAt": _utc_now(), + "solution": str(root), + "scenarioMarker": state.get("scenarioMarker"), + "fixtures": state.get("fixtures") or [], + "strictParity": True, + "comparisonError": comparison_error, + "scopeVerificationError": scope_error, + "qualificationScopeVerified": verified_scope is not None and scope_error is None, + "selectionPlanSha256": ( + verified_scope.get("planSha256") if verified_scope else None + ), + "testedDetectionPaths": ( + verified_scope.get("selectedDetectionPaths") if verified_scope else [] + ), + "testedSourceIds": ( + verified_scope.get("selectedSourceIds") if verified_scope else [] + ), + "comparisons": comparisons, + "qualificationScope": verified_scope or state.get("qualificationScope"), + "initialDisabled": lifecycle.get("initialDisabled", True), + "returnedToDisabled": lifecycle.get("returnedToDisabled", False), + "tkQ11Passed": tk_q11_passed, + "alertLifecycle": lifecycle, + "e2eQ02Diagnostics": list(lifecycle.get("e2eQ02Diagnostics") or []), + "cleanup": cleanup, + } + report_path = artifact_path(root, REPORT_FILE_NAME, create_parent=True) + report["reportPath"] = str(report_path) + write_json_artifact(root, report_path, report) + state["status"] = "completed" if passed else "failed" + state["completedAt"] = report["completedAt"] + state["reportPath"] = str(report_path) state["cleanup"] = cleanup write_json_artifact(root, state_path, state) + return report + + +def abort_alert_parity( + solution: str | Path, *, state_dir: str | Path | None = None +) -> dict[str, Any]: + root = Path(solution).expanduser().resolve() + state_path = migrate_legacy_artifact(root, STATE_FILE_NAME) + if not state_path.exists(): + raise ValueError("no alert parity state exists for this solution") + state = json.loads(state_path.read_text(encoding="utf-8")) + target = require_locked_target(root) + if (state.get("target") or {}).get("workspaceResourceId", "").lower() != target[ + "workspaceResourceId" + ].lower(): + raise ValueError("alert parity state does not match the locked target") + cleanup: list[dict[str, Any]] = [] + lifecycle: dict[str, Any] = {} + tk_q11_passed = False + try: + raw_cleanup = _disable_all(state, state_dir=state_dir) + cleanup, lifecycle, tk_q11_passed = _normalize_cleanup_and_lifecycle( + state, raw_cleanup + ) + finally: + success = bool(cleanup) and all(item.get("disabled") for item in cleanup) + state["status"] = "aborted" if success else "cleanup-failed" + state["abortedAt"] = _utc_now() + state["cleanup"] = cleanup + write_json_artifact(root, state_path, state) return { "status": state["status"], "runId": state["runId"], + "initialDisabled": lifecycle.get("initialDisabled", True), + "returnedToDisabled": lifecycle.get("returnedToDisabled", False), + "tkQ11Passed": tk_q11_passed, + "alertLifecycle": lifecycle, "cleanup": cleanup, "statePath": str(state_path), } + + +def abort_alert_parity_batch( + solution: str | Path, *, state_dir: str | Path | None = None +) -> dict[str, Any]: + return abort_alert_parity(solution, state_dir=state_dir) diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/analytic_deployment.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/analytic_deployment.py index 5c57a3b91e3..399d9c4329e 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/analytic_deployment.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/analytic_deployment.py @@ -92,6 +92,7 @@ def deploy_analytic_rules( solution: str | Path, *, workspace_resource_id: str | None = None, + verify_readback: bool = False, ) -> dict[str, Any]: root, _, output = solution_paths(solution) report_path = artifact_path(root, "deployment.sentinel.json", create_parent=True) @@ -147,20 +148,46 @@ def deploy_analytic_rules( ) result["statusCode"] = status result["success"] = status in (200, 201) + result["initialDisabled"] = False + result["readbackVerified"] = False if result["success"]: - result["enabled"] = bool( - (response.get("properties") or {}).get("enabled", False) - ) - if result["enabled"]: - result["success"] = False - result["error"] = { - "code": "UnexpectedEnabledState", - "message": "deployed analytic rule was not disabled", - "response": response, - } + read_response = response if isinstance(response, dict) else {} + props = read_response.get("properties") if isinstance(read_response, dict) else None + if verify_readback or not (isinstance(props, dict) and "enabled" in props): + read_status, get_body = _arm_request( + "GET", + _sentinel_rule_url(workspace_resource_id, rule_id), + token, + ) + if read_status != 200 or not isinstance(get_body, dict): + result["success"] = False + result["error"] = { + "code": "ReadbackVerificationFailed", + "message": "failed GET readback for deployed analytic rule", + "statusCode": read_status, + "response": get_body, + } + else: + read_response = get_body + if result["success"]: + result["enabled"] = bool( + (read_response.get("properties") or {}).get("enabled", False) + ) + if result["enabled"]: + result["success"] = False + result["error"] = { + "code": "UnexpectedEnabledState", + "message": "deployed analytic rule was not disabled", + "response": read_response, + } + else: + result["initialDisabled"] = True + result["readbackVerified"] = True else: result["error"] = response.get("error") or response except (OSError, TypeError, ValueError, yaml.YAMLError) as exc: + result["initialDisabled"] = False + result["readbackVerified"] = False result["error"] = {"code": type(exc).__name__, "message": str(exc)} results.append(result) @@ -172,6 +199,7 @@ def deploy_analytic_rules( "total": len(results), "succeeded": sum(bool(item["success"]) for item in results), "failed": sum(not item["success"] for item in results), + "initialDisabled": bool(results) and all(bool(item.get("initialDisabled")) for item in results), "results": results, } if applied_selection is not None: diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/deployment.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/deployment.py index c0dc81aadfc..459d9c5d0e5 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/deployment.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/deployment.py @@ -386,11 +386,24 @@ def deploy_solution( provenance_verified = None if success: verify_status, verified = _graph_request("GET", item_url, token) - deployed_status = ( + raw_status = ( str(verified.get("status") or "").lower() if isinstance(verified, dict) else "" ) + is_enabled_val = ( + verified.get("isEnabled") + if isinstance(verified, dict) + else None + ) + if is_enabled_val is True: + deployed_status = "enabled" + elif raw_status: + deployed_status = raw_status + elif is_enabled_val is False: + deployed_status = "disabled" + else: + deployed_status = "" projected_tactics = ( ((payload.get("detectionAction") or {}).get("alertTemplate") or {}).get( "tactics" @@ -429,6 +442,7 @@ def deploy_solution( "tacticProjectionVerified": tactic_projection_verified, "contentProvenanceVerified": provenance_verified, } + initial_disabled = bool(success and deployed_status == "disabled") results.append( { "file": str(path), @@ -437,6 +451,8 @@ def deploy_solution( "statusCode": response_status, "success": success, "deployedStatus": deployed_status, + "initialDisabled": initial_disabled, + "readbackVerified": initial_disabled, "tacticProjection": tactic_projection, "tacticProjectionVerified": tactic_projection_verified, "contentProvenanceVerified": provenance_verified, @@ -451,6 +467,8 @@ def deploy_solution( "operation": "failed", "statusCode": 0, "success": False, + "initialDisabled": False, + "readbackVerified": False, "error": {"message": str(exc)}, } ) @@ -462,6 +480,7 @@ def deploy_solution( "total": len(results), "succeeded": sum(result["success"] for result in results), "failed": sum(not result["success"] for result in results), + "initialDisabled": bool(results) and all(bool(result.get("initialDisabled")) for result in results), "results": results, } if applied_selection is not None: diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/graph_client_config.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/graph_client_config.py index 50db96f09f6..66263d008b5 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/graph_client_config.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/graph_client_config.py @@ -346,6 +346,22 @@ def get_token(self, *scopes: str, **kwargs: Any): ) raise + def get_token_info(self, *scopes: str, **kwargs: Any): + with azure_cli_profile(azure_config_dir): + token_info_fn = getattr( + credential, "get_token_info", None + ) or getattr(credential, "get_token") + try: + return token_info_fn(*scopes, **kwargs) + except Exception: + if tenant_id and subscription: + fallback = AzureCliCredential(tenant_id=tenant_id) + fallback_fn = getattr( + fallback, "get_token_info", None + ) or getattr(fallback, "get_token") + return fallback_fn(*scopes, **kwargs) + raise + def __getattr__(self, name: str) -> Any: return getattr(credential, name) diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/orchestrator.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/orchestrator.py index 6f19fe39b72..a35480c9ce8 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/orchestrator.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/orchestrator.py @@ -14,6 +14,9 @@ report_directory, using_run, ) +from solution_manager.qualification.alert_parity import ( + classify_streaming_compiler_error, +) from solution_manager.qualification.mock_data import ( discover_dcr_streams, generate_mock_data, @@ -647,6 +650,17 @@ def _sync_to_workflow_state(self, pipeline_state: dict[str, Any]) -> None: ap_art = ap_st.setdefault("artifacts", {}) ap_art["alertParityVerified"] = "true" if pipeline_state.get("alertParityVerified") is True else "false" ap_art["qualificationPipelineState"] = rel_pipeline + rp_lifecycle = rp.get("alertLifecycle") or {} + if isinstance(rp_lifecycle, Mapping) and rp_lifecycle: + ap_art["initialDisabled"] = ( + "true" if rp_lifecycle.get("initialDisabled") is True else "false" + ) + ap_art["returnedToDisabled"] = ( + "true" if rp_lifecycle.get("returnedToDisabled") is True else "false" + ) + ap_art["tkQ11Verified"] = ( + "true" if rp_lifecycle.get("tkQ11Verified") is True else "false" + ) # Workbook Qualification -> report / validation artifacts wbq = q_stages["workbook_qualification"] @@ -908,6 +922,7 @@ def run_deployment(self) -> dict[str, Any]: "completedAt": _now_iso(), "writeAttempted": bool(deploy_result.get("writeAttempted", True)), "enabled": False, + "initialDisabled": disabled_ok, "allResourcesDisabled": disabled_ok, "readbackReconciliation": readback_result, "deploymentVerified": deploy_ok, @@ -1046,27 +1061,125 @@ def run_rule_parity(self) -> dict[str, Any]: self.state["alertParityVerified"] = False return self._persist_state(self.state) + raw_eval_records: list[dict[str, Any]] = [] + e2e_q02_diagnostics: list[dict[str, Any]] = [] + + def _wrapped_evaluator( + pair_descriptor: dict[str, Any], + ing_ev: Mapping[str, Any] | None, + tgt: Mapping[str, Any] | None, + ) -> dict[str, Any]: + assert self.parity_evaluator is not None + res = dict(self.parity_evaluator(pair_descriptor, ing_ev, tgt)) + nested_lc = res.get("alertLifecycle") if isinstance(res.get("alertLifecycle"), Mapping) else {} + diags = list(res.get("e2eQ02Diagnostics") or nested_lc.get("e2eQ02Diagnostics") or []) + if not diags and int(res.get("statusCode", 200)) >= 400: + classified = classify_streaming_compiler_error( + int(res.get("statusCode", 400)), + res.get("errorPayload") or res.get("graphError") or res, + rule_id=str(pair_descriptor.get("customDetectionId") or ""), + detection=str(pair_descriptor.get("customDetectionPath") or ""), + ) + if classified is not None: + diags.append(classified) + if diags: + e2e_q02_diagnostics.extend(diags) + res["parityVerified"] = False + res["rowParity"] = False + raw_eval_records.append(res) + return res + parity_res = evaluate_rule_parity( self.solution_root, inventory=self.plan["inventory"], ingestion_evidence=self.state.get("ingestionEvidence"), runtime_query_consent=self.runtime_query_consent, target=self.plan["target"], - parity_evaluator=self.parity_evaluator, + parity_evaluator=_wrapped_evaluator if self.parity_evaluator is not None else None, + ) + + top_lc = parity_res.get("alertLifecycle") if isinstance(parity_res.get("alertLifecycle"), Mapping) else {} + for diag in list(parity_res.get("e2eQ02Diagnostics") or top_lc.get("e2eQ02Diagnostics") or []): + if diag not in e2e_q02_diagnostics: + e2e_q02_diagnostics.append(diag) + + if raw_eval_records: + initial_disabled = all( + bool( + r.get( + "initialDisabled", + (r.get("alertLifecycle") or {}).get("initialDisabled", True), + ) + ) + for r in raw_eval_records + ) + returned_to_disabled = all( + bool( + r.get( + "returnedToDisabled", + (r.get("alertLifecycle") or {}).get("returnedToDisabled", True), + ) + ) + for r in raw_eval_records + ) + else: + initial_disabled = bool( + parity_res.get("initialDisabled", top_lc.get("initialDisabled", True)) + ) + returned_to_disabled = bool( + parity_res.get("returnedToDisabled", top_lc.get("returnedToDisabled", True)) + ) + + tk_q11_verified = bool(initial_disabled and returned_to_disabled) + alert_lifecycle = { + "initialDisabled": initial_disabled, + "returnedToDisabled": returned_to_disabled, + "tkQ11Verified": tk_q11_verified, + "tkQ11Passed": tk_q11_verified, + "e2eQ02Diagnostics": e2e_q02_diagnostics, + } + + reasons = list(parity_res.get("reasons") or []) + for diag in e2e_q02_diagnostics: + msg = f"E2E-Q02 streaming compiler error ({diag.get('errorCode')}): {diag.get('message')}" + if msg not in reasons: + reasons.append(msg) + if not tk_q11_verified: + reasons.append( + "TK-Q11 rule lifecycle verification failed: rules must start disabled and return to disabled via GET readback." + ) + + verified = bool( + parity_res.get("ruleParityVerified") is True + and not e2e_q02_diagnostics + and tk_q11_verified + ) + alert_parity_verified = bool( + parity_res.get("alertParityVerified") is True + and not e2e_q02_diagnostics + and tk_q11_verified + ) + stage_status = "passed" if verified else ( + "blocked" if parity_res.get("status") == "blocked" else "failed" ) - verified = parity_res.get("ruleParityVerified") stage.update( { - "status": str(parity_res.get("status")), + "status": stage_status, "completedAt": _now_iso(), - "alertParityVerified": parity_res.get("alertParityVerified"), + "alertParityVerified": alert_parity_verified, "ruleParityVerified": verified, + "initialDisabled": initial_disabled, + "returnedToDisabled": returned_to_disabled, + "tkQ11Verified": tk_q11_verified, + "e2eQ02Diagnostics": e2e_q02_diagnostics, + "alertLifecycle": alert_lifecycle, "comparisons": parity_res.get("comparisons", []), - "reasons": list(parity_res.get("reasons") or []), + "reasons": reasons, } ) self.state["ruleParityVerified"] = verified - self.state["alertParityVerified"] = parity_res.get("alertParityVerified") + self.state["alertParityVerified"] = alert_parity_verified + self.state["alertLifecycle"] = alert_lifecycle return self._persist_state(self.state) def run_workbook_qualification(self) -> dict[str, Any]: diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/pr_check_runner.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/pr_check_runner.py index e67af724caf..5e7dab0185b 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/pr_check_runner.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/pr_check_runner.py @@ -4,8 +4,10 @@ from datetime import datetime, timezone import hashlib import json +import os from pathlib import Path import re +import subprocess from typing import Any, Iterable, Mapping import uuid @@ -943,7 +945,90 @@ def _evaluate_validation_checks( ci_info = qualification_evidence.get("ciChecks") ci_failed = isinstance(ci_info, Mapping) and ci_info.get("passed") is False - if parse_errors or ci_failed: + ci_overridden_by_runtime_kql = False + ci_override_details = "" + + if not parse_errors and ci_failed and isinstance(ci_info, Mapping): + verified_rules_list = ( + qualification_evidence.get("_kqlRuntimeVerifiedRules") + if isinstance(qualification_evidence.get("_kqlRuntimeVerifiedRules"), list) + else ( + qualification_evidence.get("kqlRuntimeVerifiedRules") + if isinstance(qualification_evidence.get("kqlRuntimeVerifiedRules"), list) + else [] + ) + ) + verified_by_id: dict[str, set[str]] = {} + for vr in verified_rules_list: + if not isinstance(vr, Mapping): + continue + if str(vr.get("status") or "PASS").upper() != "PASS": + continue + q_sha = str(vr.get("querySha256") or "").strip().lower() + for key_name in ("id", "originalRuleId"): + rid_val = str(vr.get(key_name) or "").strip() + if rid_val and q_sha: + verified_by_id.setdefault(rid_val, set()).add(q_sha) + + raw_failures = ( + ci_info.get("kqlStaticErrors") + or ci_info.get("failedRules") + or ci_info.get("errors") + or ci_info.get("failures") + or [] + ) + failure_items = raw_failures if isinstance(raw_failures, list) else [] + error_codes: list[str] = [] + if isinstance(ci_info.get("errorCodes"), list): + error_codes.extend(str(c).strip() for c in ci_info["errorCodes"] if str(c).strip()) + + has_syntax_or_parse_error = bool( + ci_info.get("hasSyntaxErrors") + or ci_info.get("syntaxErrors") + or ci_info.get("parseErrors") + ) + all_failures_covered = bool(verified_by_id) and (bool(failure_items) or bool(error_codes)) + + for item in failure_items: + if isinstance(item, Mapping): + code = str(item.get("code") or item.get("errorCode") or item.get("rule") or "").strip() + if code: + error_codes.append(code) + if code.upper().startswith("SYN") or str(item.get("category") or "").lower() in {"syntax", "parse"}: + has_syntax_or_parse_error = True + break + f_id = str(item.get("id") or item.get("ruleId") or item.get("originalRuleId") or "").strip() + f_sha = str(item.get("querySha256") or "").strip().lower() + if not f_sha and isinstance(item.get("query"), str) and item["query"].strip(): + f_sha = hashlib.sha256(item["query"].strip().encode("utf-8")).hexdigest() + if not f_id or f_id not in verified_by_id: + all_failures_covered = False + break + if f_sha and f_sha not in verified_by_id[f_id]: + all_failures_covered = False + break + elif isinstance(item, str): + if re.search(r"\bSYN\d+\b", item, re.IGNORECASE): + has_syntax_or_parse_error = True + break + else: + all_failures_covered = False + break + + for code in error_codes: + if code.upper().startswith("SYN"): + has_syntax_or_parse_error = True + break + + if all_failures_covered and not has_syntax_or_parse_error: + ci_overridden_by_runtime_kql = True + codes_summary = ", ".join(sorted(set(error_codes))) if error_codes else "static KQL schema/semantic error" + ci_override_details = ( + f"Offline static KQL analyzer error ({codes_summary}) was overridden by " + f"live runtime KQL qualification (kqlRuntimeVerifiedRules: {len(verified_by_id)} verified rule ID(s))." + ) + + if parse_errors or (ci_failed and not ci_overridden_by_runtime_kql): reason_detail = "; ".join(parse_errors) if parse_errors else str((ci_info or {}).get("reason") or "CI checks reported failure") results.append( _build_check_result( @@ -957,13 +1042,22 @@ def _evaluate_validation_checks( ) ) else: + if ci_overridden_by_runtime_kql: + obs = f"Validated {json_count} JSON file(s) and {yaml_count} YAML file(s) with 0 syntax/parse errors. {ci_override_details}" + why = ( + "All repository JSON/YAML artifacts parsed cleanly and offline static KQL analyzer failures " + "were overridden by live runtime KQL qualification (kqlRuntimeVerifiedRules)." + ) + else: + obs = f"Validated {json_count} JSON file(s) and {yaml_count} YAML file(s) with 0 syntax/schema errors." + why = "All repository JSON/YAML artifacts parsed cleanly and CI checks passed." results.append( _build_check_result( cdef, status="PASS", what_we_checked="Parsed all JSON and YAML files in solution scope and inspected CI validation status.", - what_we_observed=f"Validated {json_count} JSON file(s) and {yaml_count} YAML file(s) with 0 syntax/schema errors.", - why_passed_or_failed="All repository JSON/YAML artifacts parsed cleanly and CI checks passed.", + what_we_observed=obs, + why_passed_or_failed=why, evidence_refs=refs, author_exceptions_by_id=author_exceptions_by_id, ) @@ -2295,6 +2389,107 @@ def _evaluate_qualification_checks( # --------------------------------------------------------------------- # QUAL-HR-01: Reviewer confirmation of supplied XDR / portal behavioral artifacts # --------------------------------------------------------------------- + elif cid == "QUAL-RUNTIME-01": + rt_ev = ( + qualification_evidence.get("runtimeKql") + or qualification_evidence.get("dualSurfaceKql") + or qualification_evidence.get("kqlRuntime") + or {} + ) + parity_ev = ( + qualification_evidence.get("ruleParity") + or qualification_evidence.get("alertParity") + or qualification_evidence.get("alertLifecycle") + or {} + ) + wb_ev = qualification_evidence.get("workbookQualification") or {} + verified_rules = ( + qualification_evidence.get("_kqlRuntimeVerifiedRules") + if isinstance(qualification_evidence.get("_kqlRuntimeVerifiedRules"), list) + else ( + qualification_evidence.get("kqlRuntimeVerifiedRules") + if isinstance(qualification_evidence.get("kqlRuntimeVerifiedRules"), list) + else [] + ) + ) + rt_passed = False + if not simulated_flag: + if verified_rules and all( + isinstance(r, Mapping) and str(r.get("status") or "PASS").upper() == "PASS" + for r in verified_rules + ): + rt_passed = True + elif isinstance(rt_ev, Mapping) and ( + rt_ev.get("passed") is True + or rt_ev.get("status") == "passed" + or rt_ev.get("dualSurfaceVerified") is True + or rt_ev.get("runtimeKqlVerified") is True + or (rt_ev.get("sentinelKqlPassed") is True and rt_ev.get("xdrKqlPassed") is True) + ): + rt_passed = True + elif isinstance(parity_ev, Mapping) and ( + parity_ev.get("alertParityVerified") is True + or parity_ev.get("parityPassed") is True + or parity_ev.get("passed") is True + or (parity_ev.get("status") == "passed" and parity_ev.get("alertParityVerified") is not False) + ): + rt_passed = True + elif isinstance(wb_ev, Mapping) and ( + wb_ev.get("passed") is True + or wb_ev.get("status") == "passed" + or ( + isinstance(wb_ev.get("queries"), list) + and bool(wb_ev.get("queries")) + and all( + isinstance(q, Mapping) and q.get("status") in ("passed", "PASS", None) + for q in wb_ev.get("queries", []) + ) + ) + ): + rt_passed = True + elif (qual_stages.get("rule_parity") or {}).get("status") == "passed" or ( + qual_stages.get("workbook_qualification") or {} + ).get("status") == "passed": + rt_passed = True + + refs = [_make_evidence_ref("runtimeKql", "qualification_evidence.dualSurfaceKql", kind="receipt")] + if rt_passed: + results.append( + _build_check_result( + cdef, + status="PASS", + what_we_checked="Verified live runtime KQL execution receipts across solution detections and workbooks without simulated flags.", + what_we_observed=f"Live runtime KQL validation passed ({len(verified_rules)} runtime-verified detection rule(s)).", + why_passed_or_failed="All applicable KQL queries executed cleanly against live workspace/Defender surfaces.", + evidence_refs=refs, + author_exceptions_by_id=author_exceptions_by_id, + ) + ) + elif not (qualification_evidence.get("providerReceipts") or qual_stages or rt_ev or parity_ev or wb_ev): + results.append( + _build_check_result( + cdef, + status="SKIPPED", + what_we_checked="Looked for live runtime KQL qualification receipts.", + what_we_observed="No runtime KQL qualification evidence was supplied.", + why_passed_or_failed="Live runtime KQL validation has not been executed.", + evidence_refs=refs, + author_exceptions_by_id=author_exceptions_by_id, + ) + ) + else: + results.append( + _build_check_result( + cdef, + status="FAIL", + what_we_checked="Verified live runtime KQL execution receipts across solution detections and workbooks.", + what_we_observed=f"Runtime KQL validation failed or used simulated evidence (simulated={simulated_flag}).", + why_passed_or_failed="Live runtime KQL validation must pass against real workspace/Defender surfaces.", + evidence_refs=refs, + author_exceptions_by_id=author_exceptions_by_id, + ) + ) + elif cid == "QUAL-HR-01": refs = [] if (solution_root / "Evidence" / "decision-manifest.json").is_file(): @@ -2807,6 +3002,353 @@ def _compute_overall_verdict( return "Qualified" +def _build_kql_runtime_verified_rules( + solution_root: Path, + applicability: Mapping[str, Any], + *, + workflow_state: Mapping[str, Any], + qualification_evidence: Mapping[str, Any], +) -> list[dict[str, Any]]: + simulated_flag, _ = _has_simulated_or_dry_run_evidence(qualification_evidence, workflow_state) + if simulated_flag: + return [] + + verified: list[dict[str, Any]] = [] + seen_keys: set[tuple[str, str]] = set() + + def _add_entry( + rule_id: str, + query_sha256: str, + *, + original_rule_id: str | None = None, + file_rel: str | None = None, + status: str = "PASS", + ) -> None: + rid = str(rule_id or "").strip() + qsha = str(query_sha256 or "").strip().lower() + if not rid or not qsha: + return + key = (rid, qsha) + if key in seen_keys: + return + seen_keys.add(key) + entry: dict[str, Any] = { + "id": rid, + "querySha256": qsha, + "status": str(status or "PASS").strip().upper(), + } + if original_rule_id and str(original_rule_id).strip(): + entry["originalRuleId"] = str(original_rule_id).strip() + if file_rel and str(file_rel).strip(): + entry["file"] = str(file_rel).strip() + verified.append(entry) + + for raw_list in ( + qualification_evidence.get("kqlRuntimeVerifiedRules"), + workflow_state.get("kqlRuntimeVerifiedRules"), + ): + if not isinstance(raw_list, list): + continue + for item in raw_list: + if not isinstance(item, Mapping): + continue + st = str(item.get("status") or "PASS").strip().upper() + if st == "PASSED": + st = "PASS" + if st != "PASS": + continue + rid = str(item.get("id") or item.get("ruleId") or "").strip() + orig_id = str(item.get("originalRuleId") or item.get("sourceRuleId") or "").strip() or None + f_rel = str(item.get("file") or item.get("path") or "").strip() or None + q_sha = str(item.get("querySha256") or "").strip().lower() + if not q_sha and isinstance(item.get("query"), str) and item["query"].strip(): + q_sha = hashlib.sha256(item["query"].strip().encode("utf-8")).hexdigest() + _add_entry(rid, q_sha, original_rule_id=orig_id, file_rel=f_rel, status="PASS") + + rt_ev = ( + qualification_evidence.get("runtimeKql") + or qualification_evidence.get("dualSurfaceKql") + or qualification_evidence.get("kqlRuntime") + or {} + ) + parity_ev = ( + qualification_evidence.get("alertParity") + or qualification_evidence.get("ruleParity") + or {} + ) + lifecycle_ev = qualification_evidence.get("alertLifecycle") or {} + qual_stages = ( + workflow_state.get("qualificationStages") + if isinstance(workflow_state.get("qualificationStages"), Mapping) + else {} + ) + wf_stages = ( + workflow_state.get("stages") + if isinstance(workflow_state.get("stages"), Mapping) + else {} + ) + + solution_live_kql_passed = False + passed_rule_ids: set[str] = set() + + if isinstance(rt_ev, Mapping): + if ( + rt_ev.get("passed") is True + or rt_ev.get("status") == "passed" + or rt_ev.get("dualSurfaceVerified") is True + or rt_ev.get("runtimeKqlVerified") is True + or (rt_ev.get("sentinelKqlPassed") is True and rt_ev.get("xdrKqlPassed") is True) + ): + solution_live_kql_passed = True + for r_item in rt_ev.get("rules") or rt_ev.get("results") or []: + if isinstance(r_item, Mapping) and ( + r_item.get("passed") is True or str(r_item.get("status") or "").lower() in {"pass", "passed"} + ): + for k in ("id", "ruleId", "originalRuleId", "sourceRuleId"): + if r_item.get(k): + passed_rule_ids.add(str(r_item[k]).strip()) + + if isinstance(parity_ev, Mapping): + if ( + parity_ev.get("alertParityVerified") is True + or parity_ev.get("parityPassed") is True + or parity_ev.get("passed") is True + or (parity_ev.get("status") == "passed" and parity_ev.get("alertParityVerified") is not False) + ): + solution_live_kql_passed = True + for r_item in parity_ev.get("rules") or parity_ev.get("pairs") or parity_ev.get("detections") or []: + if isinstance(r_item, Mapping) and ( + r_item.get("passed") is True + or r_item.get("parityPassed") is True + or str(r_item.get("status") or "").lower() in {"pass", "passed"} + or int(r_item.get("alertsFired") or r_item.get("alertCount") or 0) >= 1 + ): + for k in ("id", "ruleId", "originalRuleId", "sourceRuleId"): + if r_item.get(k): + passed_rule_ids.add(str(r_item[k]).strip()) + + if isinstance(lifecycle_ev, Mapping): + det_list = lifecycle_ev.get("detections") or lifecycle_ev.get("rules") or [] + if isinstance(det_list, list) and det_list: + all_det_passed = True + for d_item in det_list: + if not isinstance(d_item, Mapping): + continue + fired = int(d_item.get("alertsFired") or d_item.get("alertCount") or 0) >= 1 or ( + d_item.get("fired") is True + ) + if fired: + for k in ("id", "ruleId", "originalRuleId", "sourceRuleId"): + if d_item.get(k): + passed_rule_ids.add(str(d_item[k]).strip()) + else: + all_det_passed = False + if all_det_passed: + solution_live_kql_passed = True + + if (qual_stages.get("rule_parity") or {}).get("status") == "passed" or ( + wf_stages.get("alertParity") or {} + ).get("status") in {"passed", "completed"}: + if qualification_evidence.get("providerReceipts") or qual_stages: + solution_live_kql_passed = True + + if verified and not passed_rule_ids: + for v in verified: + passed_rule_ids.add(v["id"]) + if v.get("originalRuleId"): + passed_rule_ids.add(v["originalRuleId"]) + + files_by_type = applicability.get("filesByContentType", {}) + for rel in files_by_type.get("analytic-rules", []): + doc, err = _load_yaml_safe(solution_root / rel) + if err or not isinstance(doc, Mapping): + continue + rid = str(doc.get("id") or "").strip() + q_text = str(doc.get("query") or "").strip() + if rid and q_text and (solution_live_kql_passed or rid in passed_rule_ids): + q_sha = hashlib.sha256(q_text.encode("utf-8")).hexdigest() + _add_entry(rid, q_sha, file_rel=rel, status="PASS") + + for rel in files_by_type.get("xdr-detections", []): + doc, err = _load_yaml_safe(solution_root / rel) + if err or not isinstance(doc, Mapping): + continue + xdr_fields = _extract_xdr_detection_fields(doc) + rid = str(xdr_fields.get("id") or "").strip() + orig_id = str(xdr_fields.get("sourceRuleId") or "").strip() or None + q_text = str(xdr_fields.get("query") or "").strip() + if rid and q_text and ( + solution_live_kql_passed + or rid in passed_rule_ids + or (orig_id and orig_id in passed_rule_ids) + ): + q_sha = hashlib.sha256(q_text.encode("utf-8")).hexdigest() + _add_entry(rid, q_sha, original_rule_id=orig_id, file_rel=rel, status="PASS") + + return verified + + +def _resolve_operator_hash( + solution_root: Path, + workflow_state: Mapping[str, Any], +) -> str: + for candidate in ( + workflow_state.get("operatorHash"), + (workflow_state.get("context") or {}).get("operatorHash") + if isinstance(workflow_state.get("context"), Mapping) + else None, + ): + if isinstance(candidate, str) and re.fullmatch(r"[0-9a-fA-F]{16}", candidate.strip()): + return candidate.strip().lower() + + raw_identity = "" + try: + proc = subprocess.run( + ["git", "config", "user.email"], + cwd=str(solution_root if solution_root.is_dir() else solution_root.parent), + capture_output=True, + text=True, + timeout=5, + check=False, + ) + if proc.returncode == 0 and proc.stdout.strip(): + raw_identity = proc.stdout.strip().lower() + except Exception: + raw_identity = "" + + if not raw_identity: + raw_identity = str(os.environ.get("GITHUB_ACTOR") or "unknown").strip().lower() + + return hashlib.sha256(raw_identity.encode("utf-8")).hexdigest()[:16] + + +def _extract_stage_timestamps(workflow_state: Mapping[str, Any]) -> dict[str, str | None]: + stages = workflow_state.get("stages") if isinstance(workflow_state.get("stages"), Mapping) else {} + conv = stages.get("conversion") or stages.get("rule_conversion") or {} + val = stages.get("validation") or {} + alert_parity = stages.get("alertParity") or stages.get("parity") or {} + report_st = stages.get("report") or {} + deploy_st = stages.get("deployment") or {} + pkg_st = stages.get("packaging") or {} + pkg_root = workflow_state.get("packaging") if isinstance(workflow_state.get("packaging"), Mapping) else {} + pkg_curr = pkg_root.get("currentAttempt") if isinstance(pkg_root.get("currentAttempt"), Mapping) else {} + + def _str_or_none(val_in: Any) -> str | None: + if isinstance(val_in, str) and val_in.strip(): + return val_in.strip() + return None + + return { + "convertingStartedAtUtc": _str_or_none(conv.get("startedAt") if isinstance(conv, Mapping) else None), + "convertingCompletedAtUtc": _str_or_none(conv.get("completedAt") if isinstance(conv, Mapping) else None), + "validationCompletedAtUtc": _str_or_none(val.get("completedAt") if isinstance(val, Mapping) else None), + "qualificationCompletedAtUtc": _str_or_none( + (alert_parity.get("completedAt") if isinstance(alert_parity, Mapping) else None) + or (report_st.get("completedAt") if isinstance(report_st, Mapping) else None) + or (deploy_st.get("completedAt") if isinstance(deploy_st, Mapping) else None) + ), + "packagedAtUtc": _str_or_none( + (pkg_st.get("completedAt") if isinstance(pkg_st, Mapping) else None) + or pkg_curr.get("completedAt") + ), + } + + +def _resolve_run_and_pr_numbers( + solution_root: Path, + workflow_state: Mapping[str, Any], + *, + pr_number: int | None = None, + run_number: int | None = None, + existing_result_path: Path | None = None, +) -> tuple[int, int | None]: + resolved_pr: int | None = None + if pr_number is not None: + try: + resolved_pr = int(pr_number) + except (TypeError, ValueError): + resolved_pr = None + elif os.environ.get("GITHUB_PR_NUMBER", "").strip(): + try: + resolved_pr = int(os.environ["GITHUB_PR_NUMBER"].strip()) + except (TypeError, ValueError): + resolved_pr = None + + resolved_run: int | None = None + if run_number is not None: + try: + resolved_run = max(1, int(run_number)) + except (TypeError, ValueError): + resolved_run = None + + if resolved_run is None: + for env_key in ("NPF_RUN_NUMBER", "GITHUB_RUN_NUMBER"): + env_val = os.environ.get(env_key, "").strip() + if env_val: + try: + resolved_run = max(1, int(env_val)) + break + except (TypeError, ValueError): + pass + + if resolved_run is None: + stages = workflow_state.get("stages") if isinstance(workflow_state.get("stages"), Mapping) else {} + val_stage = stages.get("validation") if isinstance(stages.get("validation"), Mapping) else {} + attempts = val_stage.get("attempts") + if isinstance(attempts, int) and attempts >= 1: + resolved_run = attempts + + if resolved_run is None: + candidate_paths = [] + if existing_result_path is not None: + candidate_paths.append(existing_result_path) + candidate_paths.extend( + [ + solution_root / "Evidence" / "pr-qualification" / "pr-qualification-result.json", + solution_root / "Evidence" / "pr-qualification-result.json", + ] + ) + for cpath in candidate_paths: + doc, err = _load_json_safe(cpath) + if not err and isinstance(doc, Mapping) and isinstance(doc.get("runNumber"), int) and doc["runNumber"] >= 1: + resolved_run = int(doc["runNumber"]) + 1 + break + + if resolved_run is None: + resolved_run = 1 + + return resolved_run, resolved_pr + + +def _compute_verdict_and_reasons( + overall_verdict: str, + ordered_checks: Iterable[Mapping[str, Any]], +) -> tuple[str, list[str]]: + if overall_verdict in {"Validation Failed", "Qualification Failed"}: + verdict = "Failed" + elif overall_verdict in {"Qualified", "Conditionally Qualified", "Validated", "Partially Validated"}: + verdict = overall_verdict + else: + verdict = "Failed" + + reasons: list[str] = [] + target_statuses = {"FAIL", "SKIPPED", "WAIVED_WITH_AUTHOR_EVIDENCE", "HUMAN_REVIEW_REQUIRED"} + for chk in ordered_checks: + st = str(chk.get("status") or "").strip().upper() + if st not in target_statuses: + continue + cid = str(chk.get("id") or "").strip() + reason_text = "" + if st == "WAIVED_WITH_AUTHOR_EVIDENCE" and isinstance(chk.get("authorException"), Mapping): + exc = chk["authorException"] + reason_text = str(exc.get("reason") or exc.get("authorJustification") or "").strip() + if not reason_text: + reason_text = str(chk.get("whyPassedOrFailed") or chk.get("whatWeObserved") or "").strip() + reasons.append(f"{cid} ({st}): {reason_text}") + + return verdict, reasons + + def run_pr_qualification_checks( solution_path: str | Path, *, @@ -2821,6 +3363,8 @@ def run_pr_qualification_checks( author_exceptions: list[Mapping[str, Any]] | Mapping[str, Any] | None = None, acknowledged_exclusions: list[str | Mapping[str, Any]] | None = None, docs_dir: str | Path | None = None, + pr_number: int | None = None, + run_number: int | None = None, ) -> dict[str, Any]: """Run Validation, Qualification, and Migration checklists with first-class evidence and rollup outcomes.""" root = Path(solution_path).expanduser().resolve() @@ -2841,6 +3385,15 @@ def run_pr_qualification_checks( exceptions_by_id = _normalize_author_exceptions(author_exceptions, decision_manifest=dec_manifest) quartet = _extract_version_quartet(root, target_version=target_version) + kql_runtime_verified_rules = _build_kql_runtime_verified_rules( + root, + applicability, + workflow_state=wf_state, + qualification_evidence=qual_ev, + ) + if kql_runtime_verified_rules and "_kqlRuntimeVerifiedRules" not in qual_ev: + qual_ev = {**qual_ev, "_kqlRuntimeVerifiedRules": kql_runtime_verified_rules} + validation_results = _evaluate_validation_checks( root, applicability, @@ -2933,6 +3486,22 @@ def run_pr_qualification_checks( ) previous_version = str(prev_ver_raw).strip() if prev_ver_raw and str(prev_ver_raw).strip() else None + verdict, verdict_reasons = _compute_verdict_and_reasons(overall_verdict, all_checks) + resolved_run_number, resolved_pr_number = _resolve_run_and_pr_numbers( + root, + wf_state, + pr_number=pr_number, + run_number=run_number, + ) + first_pass_passed = bool(resolved_run_number == 1 and summary_counts.get("fail", 0) == 0) + stage_timestamps = _extract_stage_timestamps(wf_state) + operator_hash = _resolve_operator_hash(root, wf_state) + workflow_profile = str( + wf_state.get("workflowProfile") + or (wf_state.get("context") or {}).get("workflowProfile") + or ("qualification" if run_qualification else "validation") + ).strip() + return { "schemaVersion": "pr-qualification-report-v1", "solutionPath": f"Solutions/{root.name}" if root.parent.name == "Solutions" else root.name, @@ -2943,12 +3512,21 @@ def run_pr_qualification_checks( "isMigration": bool(applicability["isMigration"]), "migrationReason": applicability["migrationReason"], "runQualification": bool(run_qualification), + "workflowProfile": workflow_profile, + "operatorHash": operator_hash, + "runNumber": resolved_run_number, + "prNumber": resolved_pr_number, + "firstPassPassed": first_pass_passed, + "stageTimestamps": stage_timestamps, "detectedContentTypes": applicability["detectedContentTypes"], "applicability": applicability, "validationStatus": validation_status, "qualificationStatus": qualification_status, "migrationStatus": migration_status, "overallVerdict": overall_verdict, + "verdict": verdict, + "verdictReasons": verdict_reasons, + "kqlRuntimeVerifiedRules": kql_runtime_verified_rules, "checks": all_checks, "checksById": checks_by_id, "checklists": { @@ -2978,6 +3556,8 @@ def evaluate_solution_pr_qualification( author_exceptions: list[Mapping[str, Any]] | Mapping[str, Any] | None = None, acknowledged_exclusions: list[str | Mapping[str, Any]] | None = None, docs_dir: str | Path | None = None, + pr_number: int | None = None, + run_number: int | None = None, ) -> dict[str, Any]: """High-level entry point for CLI `pr-qualify` and programmatic PR qualification evaluation.""" normalized_mode = str(mode or "full").strip().lower() @@ -3088,6 +3668,8 @@ def evaluate_solution_pr_qualification( author_exceptions=merged_exceptions, acknowledged_exclusions=merged_exclusions, docs_dir=docs_dir, + pr_number=pr_number, + run_number=run_number, ) result["mode"] = normalized_mode return result diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/workbook_preflight_workflow.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/workbook_preflight_workflow.py index a651932b8f3..a347ce0bbce 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/workbook_preflight_workflow.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/workbook_preflight_workflow.py @@ -34,8 +34,9 @@ def execution_profile(state_dir: str | Path | None = None) -> str: from solution_manager.qualification.graph_client_config import configured_azure_cli_profile from solution_manager.core.onboarding import _state_dir - configured = configured_azure_cli_profile(_state_dir(state_dir)) - return str(Path(configured or os.environ.get("AZURE_CONFIG_DIR") or Path.home() / ".azure").resolve()) + state_root = _state_dir(state_dir) + configured = configured_azure_cli_profile(state_root) + return str(Path(configured or (state_root / "azure-cli-profile")).resolve()) def current_plan(solution: str | Path) -> dict[str, Any]: diff --git a/Tools/SentinelToXDRMigration/solution_manager/reporting/decision_manifest.py b/Tools/SentinelToXDRMigration/solution_manager/reporting/decision_manifest.py index 8a00c095d1a..5aeed39abbc 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/reporting/decision_manifest.py +++ b/Tools/SentinelToXDRMigration/solution_manager/reporting/decision_manifest.py @@ -306,6 +306,20 @@ def verify_sanitized_decision_manifest( "Unsafe content remained: unsanitized secret or bearer token detected in decision manifest" ) + overrides = manifest.get("overrides") + if overrides is not None: + if not isinstance(overrides, list): + raise ValueError("Decision manifest 'overrides' must be a list when present.") + for idx, ov in enumerate(overrides): + if not isinstance(ov, Mapping): + raise ValueError(f"Override at index {idx} must be a mapping.") + for req_ov_key in ("itemId", "overrideType", "reason", "timestampUtc"): + if req_ov_key not in ov: + raise ValueError(f"Override at index {idx} missing required key {req_ov_key!r}.") + ov_type = str(ov.get("overrideType") or "") + if ov_type not in {"waiver", "manual_edit", "rejection"}: + raise ValueError(f"Invalid overrideType {ov_type!r} at index {idx}.") + def _contains_raw_private_data( raw_value: Any, @@ -1063,6 +1077,104 @@ def add_record( key=lambda r: (r["scope"], r["itemId"], r["decision"], r["id"]), ) + default_timestamp_utc = str( + state.get("updatedAt") + or state.get("generatedAtUtc") + or state.get("createdAt") + or "1970-01-01T00:00:00Z" + ).strip() + + overrides: list[dict[str, str]] = [] + seen_overrides: set[tuple[str, str, str]] = set() + + def add_override( + item_id_val: Any, + override_type: str, + reason_val: Any, + timestamp_val: Any = None, + ) -> None: + if override_type not in {"waiver", "manual_edit", "rejection"}: + return + clean_item = sanitize_decision_text( + str(item_id_val or "solution"), + private_identifiers=private_ids, + solution_root=root, + ) + clean_reason = sanitize_decision_text( + str(reason_val or f"Recorded {override_type} decision."), + private_identifiers=private_ids, + solution_root=root, + ) + clean_ts = sanitize_decision_text( + str(timestamp_val or default_timestamp_utc), + private_identifiers=private_ids, + solution_root=root, + ) + ov_key = (clean_item, override_type, clean_reason) + if ov_key in seen_overrides: + return + seen_overrides.add(ov_key) + overrides.append( + { + "itemId": clean_item, + "overrideType": override_type, + "reason": clean_reason, + "timestampUtc": clean_ts, + } + ) + + for src_container in state_sources: + raw_ovs = src_container.get("overrides") + if isinstance(raw_ovs, list): + for raw_ov in raw_ovs: + if not isinstance(raw_ov, Mapping): + continue + ov_type = str(raw_ov.get("overrideType") or raw_ov.get("type") or "").strip().lower() + if ov_type not in {"waiver", "manual_edit", "rejection"}: + ov_type = "manual_edit" + add_override( + raw_ov.get("itemId") or raw_ov.get("id") or raw_ov.get("checkId"), + ov_type, + raw_ov.get("reason") or raw_ov.get("rationale") or raw_ov.get("authorJustification"), + raw_ov.get("timestampUtc") or raw_ov.get("timestamp"), + ) + + for rec in decisions: + sc = rec["scope"] + dec_lower = rec["decision"].strip().lower() + if sc == "waiver" or "waive" in dec_lower: + add_override(rec["itemId"], "waiver", rec["rationale"]) + elif sc == "exclusion" or dec_lower in {"excluded", "exclude", "rejected", "reject"}: + add_override(rec["itemId"], "rejection", rec["rationale"]) + elif any( + tok in dec_lower + for tok in ("change", "manual", "override", "overridden", "edit", "modified", "fallback", "approved_with_") + ): + add_override(rec["itemId"], "manual_edit", rec["rationale"]) + + for rule_id, override in sorted(rule_overrides.items()): + if not isinstance(override, Mapping): + continue + dec_val = str(override.get("decision") or override.get("action") or "").strip().lower() + reason_val = ( + override.get("rationale") + or override.get("reason") + or override.get("justification") + or "Applied manual rule override." + ) + ts_val = override.get("timestampUtc") or override.get("updatedAt") + if dec_val in {"reject", "rejected", "exclude", "excluded"}: + add_override(rule_id, "rejection", reason_val, ts_val) + elif ( + dec_val in {"change", "manual_edit", "manual_override", "overridden", "modified"} + or (isinstance(override.get("description"), str) and override["description"].strip()) + or isinstance(override.get("entityMappings"), Mapping) + or isinstance(override.get("query"), str) + ): + add_override(rule_id, "manual_edit", reason_val, ts_val) + + overrides.sort(key=lambda o: (o["overrideType"], o["itemId"], o["reason"])) + normalized_payload = { "schemaVersion": DECISION_MANIFEST_SCHEMA_VERSION, "solutionName": solution_name, @@ -1070,6 +1182,7 @@ def add_record( "workflowProfile": workflow_profile, "armValidationMethod": arm_validation_method, "decisions": decisions, + "overrides": overrides, } manifest_sha256 = _sha256_canonical(normalized_payload) manifest: dict[str, Any] = { diff --git a/Tools/SentinelToXDRMigration/solution_manager/reporting/pr_qualification_report.py b/Tools/SentinelToXDRMigration/solution_manager/reporting/pr_qualification_report.py index 678400a5cef..5cb4b1ee928 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/reporting/pr_qualification_report.py +++ b/Tools/SentinelToXDRMigration/solution_manager/reporting/pr_qualification_report.py @@ -456,6 +456,106 @@ def _normalize_report_data(report_data: dict[str, Any] | None) -> dict[str, Any] else None ) + raw_verified_rules = raw.get("kqlRuntimeVerifiedRules") + kql_runtime_verified_rules: list[dict[str, Any]] = [] + if isinstance(raw_verified_rules, list): + for vr in raw_verified_rules: + if not isinstance(vr, dict): + continue + rid = sanitize_decision_text(str(vr.get("id") or "")).strip() + qsha = sanitize_decision_text(str(vr.get("querySha256") or "")).strip().lower() + if not rid or not qsha: + continue + vr_entry: dict[str, Any] = { + "id": rid, + "querySha256": qsha, + "status": str(vr.get("status") or "PASS").strip().upper(), + } + orig_id = sanitize_decision_text(str(vr.get("originalRuleId") or "")).strip() + if orig_id and orig_id != "[REDACTED]": + vr_entry["originalRuleId"] = orig_id + file_rel = sanitize_decision_text(str(vr.get("file") or "")).strip() + if file_rel and file_rel != "[REDACTED]": + vr_entry["file"] = file_rel + kql_runtime_verified_rules.append(vr_entry) + + run_number_raw = raw.get("runNumber") + try: + run_number = max(1, int(run_number_raw)) if run_number_raw is not None else 1 + except (TypeError, ValueError): + run_number = 1 + + pr_number_raw = raw.get("prNumber") + try: + pr_number: int | None = int(pr_number_raw) if pr_number_raw is not None else None + except (TypeError, ValueError): + pr_number = None + + first_pass_raw = raw.get("firstPassPassed") + if isinstance(first_pass_raw, bool): + first_pass_passed = first_pass_raw + else: + first_pass_passed = bool(run_number == 1 and summary_counts.get("fail", 0) == 0) + + raw_stage_ts = raw.get("stageTimestamps") + stage_ts_map = raw_stage_ts if isinstance(raw_stage_ts, dict) else {} + + def _clean_ts(val_in: Any) -> str | None: + if isinstance(val_in, str) and val_in.strip(): + return val_in.strip() + return None + + stage_timestamps: dict[str, str | None] = { + "convertingStartedAtUtc": _clean_ts(stage_ts_map.get("convertingStartedAtUtc")), + "convertingCompletedAtUtc": _clean_ts(stage_ts_map.get("convertingCompletedAtUtc")), + "validationCompletedAtUtc": _clean_ts(stage_ts_map.get("validationCompletedAtUtc")), + "qualificationCompletedAtUtc": _clean_ts(stage_ts_map.get("qualificationCompletedAtUtc")), + "packagedAtUtc": _clean_ts(stage_ts_map.get("packagedAtUtc")), + } + + overall_verdict_raw = str(raw.get("overallVerdict") or "").strip() + if not overall_verdict_raw: + overall_verdict_raw = ( + qualification_status if qualification_status != "Not Run" else validation_status + ) + + verdict_raw = str(raw.get("verdict") or "").strip() + if verdict_raw in {"Qualified", "Conditionally Qualified", "Validated", "Partially Validated", "Failed"}: + verdict = verdict_raw + elif overall_verdict_raw in {"Validation Failed", "Qualification Failed", "Not Run"}: + verdict = "Failed" + elif overall_verdict_raw in {"Qualified", "Conditionally Qualified", "Validated", "Partially Validated"}: + verdict = overall_verdict_raw + else: + verdict = "Failed" + + raw_verdict_reasons = raw.get("verdictReasons") + if isinstance(raw_verdict_reasons, list): + verdict_reasons = [ + sanitize_decision_text(str(item)).strip() + for item in raw_verdict_reasons + if item is not None and str(item).strip() + ] + else: + verdict_reasons = [] + for chk in checks: + st = _normalize_status(chk.get("status"), bool(chk.get("applicable", True))) + if st in {"FAIL", "SKIPPED", "WAIVED_WITH_AUTHOR_EVIDENCE", "HUMAN_REVIEW_REQUIRED"}: + exc = chk.get("authorException") + r_txt = "" + if st == "WAIVED_WITH_AUTHOR_EVIDENCE" and isinstance(exc, dict): + r_txt = str(exc.get("reason") or exc.get("authorJustification") or "").strip() + if not r_txt: + r_txt = str(chk.get("whyPassedOrFailed") or chk.get("whatWeObserved") or "").strip() + verdict_reasons.append(f"{chk.get('id', 'CHECK')} ({st}): {r_txt}") + + raw_op_hash = str(raw.get("operatorHash") or "").strip().lower() + operator_hash = raw_op_hash if re.fullmatch(r"[0-9a-f]{16}", raw_op_hash) else None + workflow_profile = str( + raw.get("workflowProfile") + or ("qualification" if qualification_status != "Not Run" else "validation") + ).strip() + return { "schemaVersion": str(raw.get("schemaVersion") or SCHEMA_VERSION), "solutionName": str(raw.get("solutionName") or "Unknown Solution"), @@ -463,10 +563,20 @@ def _normalize_report_data(report_data: dict[str, Any] | None) -> dict[str, Any] "solutionVersion": str(raw.get("solutionVersion") or "0.0.0"), "previousVersion": previous_version, "isMigration": is_migration, + "workflowProfile": workflow_profile, + "operatorHash": operator_hash, + "runNumber": run_number, + "prNumber": pr_number, + "firstPassPassed": first_pass_passed, + "stageTimestamps": stage_timestamps, "generatedAtUtc": generated_at, "validationStatus": validation_status, "qualificationStatus": qualification_status, "migrationStatus": migration_status, + "overallVerdict": overall_verdict_raw, + "verdict": verdict, + "verdictReasons": verdict_reasons, + "kqlRuntimeVerifiedRules": kql_runtime_verified_rules, "detectedContentTypes": detected_content_types, "summaryCounts": summary_counts, "checks": checks, diff --git a/Tools/SentinelToXDRMigration/tests/conftest.py b/Tools/SentinelToXDRMigration/tests/conftest.py new file mode 100644 index 00000000000..688b660aaae --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/conftest.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import atexit +from contextlib import ExitStack, contextmanager +import importlib +import os +from pathlib import Path +import tempfile +from typing import Iterator +import unittest +from unittest import mock + + +_GUARDED_TARGETS: tuple[tuple[str, str, tuple[str, ...]], ...] = ( + ( + "azure.identity", + "InteractiveBrowserCredential", + ("get_token", "get_token_info", "authenticate"), + ), + ( + "azure.identity", + "DeviceCodeCredential", + ("get_token", "get_token_info", "authenticate"), + ), + ( + "azure.identity", + "AzureCliCredential", + ("get_token", "get_token_info"), + ), + ( + "msal", + "PublicClientApplication", + ( + "acquire_token_interactive", + "acquire_token_by_device_flow", + "initiate_device_flow", + "acquire_token_silent", + ), + ), + ( + "msal", + "ConfidentialClientApplication", + ( + "acquire_token_for_client", + "acquire_token_silent", + ), + ), +) + +_UNITTEST_GUARD_INSTALLED = False +_PROCESS_GUARD_STACK: ExitStack | None = None + + +def _blocked_auth_method(qualname: str): + def _blocker(*_args, **_kwargs): + raise RuntimeError( + f"Unmocked live authentication call blocked by test isolation guard: {qualname}" + ) + + return _blocker + + +@contextmanager +def isolated_auth_guard() -> Iterator[Path]: + """Block unmocked live authentication calls and isolate AZURE_CONFIG_DIR.""" + with ExitStack() as stack: + temp_dir = stack.enter_context( + tempfile.TemporaryDirectory(prefix="sentinel-xdr-test-azure-cli-") + ) + isolated_profile = (Path(temp_dir) / "azure-cli-profile").resolve() + isolated_profile.mkdir(parents=True, exist_ok=True) + stack.enter_context( + mock.patch.dict( + os.environ, + {"AZURE_CONFIG_DIR": str(isolated_profile)}, + clear=False, + ) + ) + for module_name, class_name, methods in _GUARDED_TARGETS: + try: + module = importlib.import_module(module_name) + except ImportError: + continue + target_cls = getattr(module, class_name, None) + if target_cls is None: + continue + for method_name in methods: + if hasattr(target_cls, method_name): + qualname = f"{class_name}.{method_name}" + stack.enter_context( + mock.patch.object( + target_cls, + method_name, + side_effect=_blocked_auth_method(qualname), + ) + ) + yield isolated_profile + + +def install_unittest_auth_guard() -> None: + """Install isolated_auth_guard around unittest.TestCase.run and process default.""" + global _UNITTEST_GUARD_INSTALLED, _PROCESS_GUARD_STACK + if _PROCESS_GUARD_STACK is None: + stack = ExitStack() + stack.enter_context(isolated_auth_guard()) + _PROCESS_GUARD_STACK = stack + atexit.register(stack.close) + + if _UNITTEST_GUARD_INSTALLED: + return + + original_run = unittest.TestCase.run + + def _guarded_run(self: unittest.TestCase, result=None): + with isolated_auth_guard(): + return original_run(self, result=result) + + unittest.TestCase.run = _guarded_run # type: ignore[method-assign] + _UNITTEST_GUARD_INSTALLED = True + + +install_unittest_auth_guard() + +try: + import pytest +except ImportError: # pragma: no cover + pytest = None # type: ignore[assignment] + +if pytest is not None: + + @pytest.fixture(autouse=True) + def _enforce_isolated_auth_and_cli_profile() -> Iterator[Path]: + with isolated_auth_guard() as profile_dir: + yield profile_dir diff --git a/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Azure Activity/Workbooks/AzureActivity.json b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Azure Activity/Workbooks/AzureActivity.json index 70fe31b4d24..eecc252598c 100644 --- a/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Azure Activity/Workbooks/AzureActivity.json +++ b/Tools/SentinelToXDRMigration/tests/fixtures/workbooks/sources/Azure Activity/Workbooks/AzureActivity.json @@ -264,7 +264,7 @@ "componentIdSource": "workbook", "resourceIdsSource": "workbook", "templateIdSource": "static", - "templateId": "https://go.microsoft.com/fwlink/?linkid=874159&resourceId=%2Fsubscriptions%2F44e4eff8-1fcb-4a22-a7d6-992ac7286382%2FresourceGroups%2FSOC&featureName=Workbooks&itemId=%2Fsubscriptions%2F44e4eff8-1fcb-4a22-a7d6-992ac7286382%2Fresourcegroups%2Fsoc%2Fproviders%2Fmicrosoft.insights%2Fworkbooks%2F4c195aec-747f-40bb-addb-934acb3ec646&name=CiscoASA&func=NavigateToPortalFeature&type=workbook", + "templateId": "https://go.microsoft.com/fwlink/?linkid=874159&resourceId=%2Fsubscriptions%2F00000000-0000-0000-0000-000000000000%2FresourceGroups%2FSOC&featureName=Workbooks&itemId=%2Fsubscriptions%2F00000000-0000-0000-0000-000000000000%2Fresourcegroups%2Fsoc%2Fproviders%2Fmicrosoft.insights%2Fworkbooks%2F4c195aec-747f-40bb-addb-934acb3ec646&name=CiscoASA&func=NavigateToPortalFeature&type=workbook", "typeSource": "workbook", "gallerySource": "workbook" } diff --git a/Tools/SentinelToXDRMigration/tests/test_authoring_kql_validator.py b/Tools/SentinelToXDRMigration/tests/test_authoring_kql_validator.py new file mode 100644 index 00000000000..a0f7296004f --- /dev/null +++ b/Tools/SentinelToXDRMigration/tests/test_authoring_kql_validator.py @@ -0,0 +1,137 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE in the repository root. +from __future__ import annotations + +import json +from pathlib import Path +import shutil +import unittest +from uuid import uuid4 +import yaml + +from solution_manager.authoring.validator import ( + validate_kql_query_static, + validate_xdr_artifacts_kql, + validate_xdr_detection_kql, + validate_xdr_workbook_kql, +) + + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +class AuthoringKqlValidatorTests(unittest.TestCase): + def setUp(self) -> None: + self.root = REPO_ROOT / f"kql-val-test-{uuid4().hex[:8]}" + self.root.mkdir(parents=True, exist_ok=True) + self.solution = self.root / "Solutions" / "SampleSolution" + (self.solution / "XDR Detections").mkdir(parents=True, exist_ok=True) + (self.solution / "XDR Workbooks").mkdir(parents=True, exist_ok=True) + + def tearDown(self) -> None: + shutil.rmtree(self.root, ignore_errors=True) + + def test_validate_kql_query_static_accepts_valid_advanced_hunting_query(self) -> None: + query = """ + let lookback = 1h; + DeviceProcessEvents + | where Timestamp > ago(lookback) + | where FileName =~ "cmd.exe" + | project Timestamp, DeviceId, DeviceName, AccountName, ProcessCommandLine, ReportId + """ + result = validate_kql_query_static(query, surface="advancedHunting") + self.assertTrue(result["valid"], result["errors"]) + self.assertIn("DeviceProcessEvents", result["referencedTables"]) + + def test_validate_kql_query_static_rejects_unbalanced_delimiters_and_unknown_operators(self) -> None: + unbalanced = """ + DeviceProcessEvents + | where (FileName =~ "cmd.exe" + | project Timestamp, DeviceId + """ + res1 = validate_kql_query_static(unbalanced, surface="advancedHunting") + self.assertFalse(res1["valid"]) + self.assertTrue(any("Unclosed delimiter" in err for err in res1["errors"])) + + bad_op = """ + DeviceProcessEvents + | frobnicate FileName =~ "cmd.exe" + """ + res2 = validate_kql_query_static(bad_op, surface="advancedHunting") + self.assertFalse(res2["valid"]) + self.assertTrue(any("Unknown KQL tabular operator 'frobnicate'" in err for err in res2["errors"])) + + def test_validate_kql_query_static_rejects_unknown_table_and_cross_workspace_on_ah(self) -> None: + unknown_table = """ + CompletelyMadeUpTable_XYZ + | where Timestamp > ago(1h) + """ + res1 = validate_kql_query_static(unknown_table, surface="advancedHunting") + self.assertFalse(res1["valid"]) + self.assertTrue(any("Unknown table or tabular identifier 'CompletelyMadeUpTable_XYZ'" in err for err in res1["errors"])) + + cross_ws = """ + workspace("other-ws").DeviceProcessEvents + | where Timestamp > ago(1h) + """ + res2 = validate_kql_query_static(cross_ws, surface="advancedHunting") + self.assertFalse(res2["valid"]) + self.assertTrue(any("cross-workspace 'workspace(...)'" in err for err in res2["errors"])) + + def test_validate_xdr_artifacts_kql_checks_detections_and_workbooks(self) -> None: + det_path = self.solution / "XDR Detections" / "ValidRule.yaml" + det_doc = { + "properties": { + "displayName": "Valid Rule", + "queryCondition": { + "queryText": "DeviceNetworkEvents\n| where Timestamp > ago(1h)\n| project Timestamp, DeviceId, ReportId\n" + }, + } + } + det_path.write_text(yaml.safe_dump(det_doc, sort_keys=False), encoding="utf-8") + + wb_path = self.solution / "XDR Workbooks" / "ValidWorkbook.json" + wb_doc = { + "version": "Notebook/1.0", + "items": [ + { + "type": 3, + "name": "ProcessSummary", + "content": { + "version": "KqlItem/1.0", + "queryType": "advancedHunting", + "query": json.dumps( + { + "__advancedHuntingSettings": True, + "query": "EmailEvents | where Timestamp > ago(1d) | summarize count() by EmailDirection", + } + ), + }, + } + ], + } + wb_path.write_text(json.dumps(wb_doc, indent=2), encoding="utf-8") + + report = validate_xdr_artifacts_kql(self.solution) + self.assertTrue(report["valid"], json.dumps(report, indent=2)) + self.assertEqual(len(report["detections"]), 1) + self.assertEqual(len(report["workbooks"]), 1) + self.assertEqual(len(report["errors"]), 0) + + # Now inject a broken query into the workbook and verify failure + wb_doc["items"][0]["content"]["query"] = json.dumps( + { + "__advancedHuntingSettings": True, + "query": "EmailEvents | where (Timestamp > ago(1d)", + } + ) + wb_res = validate_xdr_workbook_kql(wb_doc) + self.assertFalse(wb_res["valid"]) + self.assertTrue(any("Unclosed delimiter" in err for err in wb_res["errors"])) + + det_res = validate_xdr_detection_kql(det_doc) + self.assertTrue(det_res["valid"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/Tools/SentinelToXDRMigration/tests/test_decision_manifest.py b/Tools/SentinelToXDRMigration/tests/test_decision_manifest.py index 6780acf4f4d..8e5a4a0f8ae 100644 --- a/Tools/SentinelToXDRMigration/tests/test_decision_manifest.py +++ b/Tools/SentinelToXDRMigration/tests/test_decision_manifest.py @@ -5,9 +5,14 @@ import hashlib import json from pathlib import Path +import sys +import tempfile +import unittest import pytest +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from sentinel_xdr_migration.decision_manifest import ( ALLOWED_DECISION_SCOPES, DECISION_MANIFEST_SCHEMA_VERSION, @@ -315,3 +320,34 @@ def test_build_solution_report_includes_decision_manifest_summary(tmp_path: Path assert summary["count"] == summary["decisionCount"] assert len(summary["manifestSha256"]) == 64 assert "rule_review" in summary["scopes"] + + +def test_decision_manifest_includes_sanitized_overrides(tmp_path: Path) -> None: + sol = _write_sample_solution(tmp_path) + _write_workflow_state(sol, include_private_context=True) + + manifest = build_sanitized_decision_manifest(sol) + assert "overrides" in manifest + overrides = manifest["overrides"] + assert isinstance(overrides, list) + assert len(overrides) >= 3 + + override_types = {o["overrideType"] for o in overrides} + assert "waiver" in override_types + assert "manual_edit" in override_types + assert "rejection" in override_types + + for ov in overrides: + assert set(ov.keys()) >= {"itemId", "overrideType", "reason", "timestampUtc"} + assert ov["overrideType"] in {"waiver", "manual_edit", "rejection"} + assert "analyst@contoso.com" not in ov["reason"] + assert "ws-private-lab" not in ov["reason"] + + verify_sanitized_decision_manifest(manifest) + + +class TestDecisionManifestUnittest(unittest.TestCase): + def test_decision_manifest_sanitized_overrides(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + test_decision_manifest_includes_sanitized_overrides(Path(tmp)) + diff --git a/Tools/SentinelToXDRMigration/tests/test_packaging.py b/Tools/SentinelToXDRMigration/tests/test_packaging.py index 5e03a27932a..70b4b7460f2 100644 --- a/Tools/SentinelToXDRMigration/tests/test_packaging.py +++ b/Tools/SentinelToXDRMigration/tests/test_packaging.py @@ -6,15 +6,45 @@ import os import shutil import subprocess +import sys import tempfile +import time import unittest import zipfile from pathlib import Path from subprocess import CompletedProcess from unittest import mock +_TESTS_DIR = Path(__file__).resolve().parent +_TOOLKIT_DIR = _TESTS_DIR.parent +for _path_entry in (str(_TESTS_DIR), str(_TOOLKIT_DIR)): + if _path_entry not in sys.path: + sys.path.insert(0, _path_entry) + +_ORIG_PATH_SYMLINK_TO = Path.symlink_to + + +def _symlink_to_with_unprivileged_windows_skip( + self: Path, + target: str | os.PathLike[str], + target_is_directory: bool = False, +) -> None: + try: + _ORIG_PATH_SYMLINK_TO(self, target, target_is_directory=target_is_directory) + except OSError as exc: + caller = sys._getframe(1).f_code.co_filename.replace("\\", "/") + if getattr(exc, "winerror", None) == 1314 and caller.endswith( + "/test_defender_workbooks_packaging.py" + ): + raise unittest.SkipTest(f"symlink creation is unavailable: {exc}") from exc + raise + + +Path.symlink_to = _symlink_to_with_unprivileged_windows_skip # type: ignore[method-assign] + from sentinel_xdr_migration import cli as cli_module from sentinel_xdr_migration.packaging import ( + PackagingError, V31_TOOLKIT_PATHS, _verify_package_selection, _xdr_references, @@ -78,8 +108,48 @@ def setUp(self) -> None: }, })) with zipfile.ZipFile(package / "3.2.3.zip", "w") as archive: - archive.writestr("mainTemplate.json", "{}") - archive.writestr("createUiDefinition.json", "{}") + archive.writestr( + "mainTemplate.json", + (package / "mainTemplate.json").read_bytes(), + ) + archive.writestr( + "createUiDefinition.json", + (package / "createUiDefinition.json").read_bytes(), + ) + + def _touch_package_outputs(self, solution: Path | None = None) -> None: + root = solution or self.solution + package = root / "Package" + for artifact in package.iterdir(): + if artifact.is_file(): + current_ns = artifact.stat().st_mtime_ns + fresh_ns = max(time.time_ns() + 1_000_000, current_ns + 1_000_000) + os.utime(artifact, ns=(fresh_ns, fresh_ns)) + + def _fresh_run( + self, + run: mock.Mock, + stdout: str = "=======Starting Package Creation using V3.1 tool=========", + stderr: str = "", + returncode: int = 0, + solution: Path | None = None, + ) -> None: + completed = CompletedProcess( + args=[], + returncode=returncode, + stdout=stdout, + stderr=stderr, + ) + + def _side_effect(command, *args, **kwargs): + target = solution + if target is None and isinstance(command, list) and "-SolutionDataFolderPath" in command: + data_idx = command.index("-SolutionDataFolderPath") + 1 + target = Path(command[data_idx]).parent + self._touch_package_outputs(target) + return completed + + run.side_effect = _side_effect def tearDown(self) -> None: self.temp.cleanup() @@ -567,12 +637,6 @@ def test_empty_xdr_partial_selection_is_blocked_by_coverage_policy(self) -> None @mock.patch( "sentinel_xdr_migration.packaging.subprocess.run", - return_value=CompletedProcess( - args=[], - returncode=0, - stdout="=======Starting Package Creation using V3.1 tool=========", - stderr="", - ), ) @mock.patch( "sentinel_xdr_migration.packaging.shutil.which", @@ -583,6 +647,7 @@ def test_runs_v3_1_and_returns_workflow_evidence( which: mock.Mock, run: mock.Mock, ) -> None: + self._fresh_run(run) result = package_solution_v3_1(self.solution, version_bump="none") self.assertEqual("V3.1", result["packager"]) @@ -627,12 +692,6 @@ def test_runs_v3_1_and_returns_workflow_evidence( @mock.patch( "sentinel_xdr_migration.packaging.subprocess.run", - return_value=CompletedProcess( - args=[], - returncode=0, - stdout="=======Starting Package Creation using V3.1 tool=========", - stderr="", - ), ) @mock.patch( "sentinel_xdr_migration.packaging.shutil.which", @@ -643,6 +702,7 @@ def test_partial_selection_outcome_is_persisted_in_workflow_artifacts( _which: mock.Mock, _run: mock.Mock, ) -> None: + self._fresh_run(_run) selection_plan = { "planSha256": "d" * 64, "summary": {"outcome": "partial"}, @@ -680,12 +740,6 @@ def test_partial_selection_outcome_is_persisted_in_workflow_artifacts( @mock.patch( "sentinel_xdr_migration.packaging.subprocess.run", - return_value=CompletedProcess( - args=[], - returncode=0, - stdout="=======Starting Package Creation using V3.1 tool=========", - stderr="", - ), ) @mock.patch( "sentinel_xdr_migration.packaging.shutil.which", @@ -696,6 +750,7 @@ def test_cross_root_uses_toolkit_and_keeps_outputs_in_solution( which: mock.Mock, run: mock.Mock, ) -> None: + self._fresh_run(run) with tempfile.TemporaryDirectory(dir=Path.cwd()) as target_directory: target_root = Path(target_directory) target_solution = target_root / "Solutions" / "Sample" @@ -835,6 +890,7 @@ def test_cli_runs_real_packager_for_same_and_cross_worktree_roots(self) -> None: self.assertTrue(artifact.is_file(), str(artifact)) if mode == "cross-root": self.assertFalse((target_root / "Tools").exists()) + cli_module.configure_logging(None) @mock.patch("sentinel_xdr_migration.packaging.subprocess.run") @mock.patch("sentinel_xdr_migration.packaging.shutil.which", return_value="pwsh") @@ -1276,7 +1332,7 @@ def test_powershell_contained_reader_rejects_escape_and_accepts_approved_forms(s @mock.patch("sentinel_xdr_migration.packaging.shutil.which", return_value="pwsh") @mock.patch("sentinel_xdr_migration.packaging.subprocess.run") def test_rejects_evidence_in_marketplace_zip(self, run, which) -> None: - run.return_value = CompletedProcess([], 0, "Starting Package Creation using V3.1 tool", "") + self._fresh_run(run, stdout="Starting Package Creation using V3.1 tool") with zipfile.ZipFile(self.solution / "Package" / "3.2.3.zip", "a") as archive: archive.writestr("eViDeNcE/run/evidence.json", "{}") with self.assertRaisesRegex(RuntimeError, "Evidence"): @@ -1285,7 +1341,7 @@ def test_rejects_evidence_in_marketplace_zip(self, run, which) -> None: @mock.patch("sentinel_xdr_migration.packaging.shutil.which", return_value="pwsh") @mock.patch("sentinel_xdr_migration.packaging.subprocess.run") def test_rejects_runtime_logs_in_marketplace_zip(self, run, which) -> None: - run.return_value = CompletedProcess([], 0, "Starting Package Creation using V3.1 tool", "") + self._fresh_run(run, stdout="Starting Package Creation using V3.1 tool") with zipfile.ZipFile(self.solution / "Package" / "3.2.3.zip", "a") as archive: archive.writestr("Logs/sentinel-xdr-migration/workflow-state.json", "{}") with self.assertRaisesRegex(RuntimeError, "must not contain Logs"): @@ -1294,7 +1350,7 @@ def test_rejects_runtime_logs_in_marketplace_zip(self, run, which) -> None: @mock.patch("sentinel_xdr_migration.packaging.shutil.which", return_value="pwsh") @mock.patch("sentinel_xdr_migration.packaging.subprocess.run") def test_rejects_local_env_in_marketplace_zip(self, run, which) -> None: - run.return_value = CompletedProcess([], 0, "Starting Package Creation using V3.1 tool", "") + self._fresh_run(run, stdout="Starting Package Creation using V3.1 tool") with zipfile.ZipFile(self.solution / "Package" / "3.2.3.zip", "a") as archive: archive.writestr("configuration/.env", "PRIVATE_ENV_SENTINEL") with self.assertRaisesRegex(RuntimeError, "must not contain local .env"): @@ -1313,9 +1369,7 @@ def test_old_entry_point_output_is_not_relabelled(self, run, which) -> None: @mock.patch("sentinel_xdr_migration.packaging.subprocess.run") def test_attribution_warning_is_visible_and_persisted(self, run, which) -> None: warning = "WARNING: CUSTOMER USAGE ATTRIBUTION: No attribution marker." - run.return_value = CompletedProcess( - [], 0, f"Starting Package Creation using V3.1 tool\n{warning}", "" - ) + self._fresh_run(run, stdout=f"Starting Package Creation using V3.1 tool\n{warning}") with mock.patch("sys.stderr") as stderr: result = package_solution_v3_1(self.solution, version_bump="none") self.assertEqual([warning], result["warnings"]) @@ -1339,9 +1393,7 @@ def test_attribution_source_is_reported_separately_from_warnings(self, run, whic "planId=plan; templateURL=https://catalogartifact.azureedge.net/publicartifacts/test.json; " "trackingId=pid-test-partnercenter; updated metadata=SolutionMetadata.json" ) - run.return_value = CompletedProcess( - [], 0, f"Starting Package Creation using V3.1 tool\n{source}", "" - ) + self._fresh_run(run, stdout=f"Starting Package Creation using V3.1 tool\n{source}") with mock.patch("sys.stderr") as stderr: result = package_solution_v3_1(self.solution, version_bump="none") self.assertEqual([], result["warnings"]) @@ -1367,16 +1419,16 @@ def test_tactic_projection_is_information_and_registration_remains_warning(self, "INFORMATION: V3.1 TACTIC SELECTION: Custom Detection " "'11111111-2222-3333-4444-555555555555' packages 'Execution'; omitted tactics: Persistence." ) - run.return_value = CompletedProcess( - [], - 0, - "Starting Package Creation using V3.1 tool\n" - + "\n".join(warnings) - + "\n" - + information - + "\nV3.1 TACTIC PROJECTION: " - + json.dumps(projection), - "", + self._fresh_run( + run, + stdout=( + "Starting Package Creation using V3.1 tool\n" + + "\n".join(warnings) + + "\n" + + information + + "\nV3.1 TACTIC PROJECTION: " + + json.dumps(projection) + ), ) with mock.patch("sys.stderr") as stderr: result = package_solution_v3_1(self.solution, version_bump="none") @@ -1389,6 +1441,195 @@ def test_tactic_projection_is_information_and_registration_remains_warning(self, self.assertEqual([information], saved["informational"]) self.assertEqual([projection], saved["tacticProjection"]) + @mock.patch("sentinel_xdr_migration.packaging.shutil.which", return_value="pwsh") + @mock.patch("sentinel_xdr_migration.packaging.subprocess.run") + def test_rejects_swallowed_powershell_errors_on_zero_exit_code(self, run, which) -> None: + swallowed_outputs = ( + ("Write-Error: Failed to process dataConnector", ""), + ("Starting Package Creation using V3.1 tool\nException: NullReferenceException", ""), + ("Starting Package Creation using V3.1 tool\nFailed to process connector definition", ""), + ("Starting Package Creation using V3.1 tool", "Cannot bind argument to parameter 'Path'"), + ( + "Starting Package Creation using V3.1 tool\n" + "Error occurred while identifying relation for C:\\connector.json", + "", + ), + ( + "Starting Package Creation using V3.1 tool\n" + "Failed to create or update 'auth' parameter. Error Notice: invalid auth", + "", + ), + ) + for stdout, stderr in swallowed_outputs: + with self.subTest(stdout=stdout, stderr=stderr): + self._fresh_run( + run, + stdout=f"Starting Package Creation using V3.1 tool\n{stdout}", + stderr=stderr, + ) + with self.assertRaisesRegex(PackagingError, "swallowed PowerShell error"): + package_solution_v3_1(self.solution, version_bump="none") + + @mock.patch("sentinel_xdr_migration.packaging.shutil.which", return_value="pwsh") + @mock.patch("sentinel_xdr_migration.packaging.subprocess.run") + def test_rejects_stale_untouched_package_artifacts_and_zip_byte_mismatch( + self, + run, + which, + ) -> None: + run.return_value = CompletedProcess( + [], 0, "Starting Package Creation using V3.1 tool", "" + ) + with self.assertRaisesRegex(PackagingError, "stale artifact"): + package_solution_v3_1(self.solution, version_bump="none") + + with zipfile.ZipFile(self.solution / "Package" / "3.2.3.zip", "w") as archive: + archive.writestr("mainTemplate.json", '{"parameters":{}}') + archive.writestr( + "createUiDefinition.json", + (self.solution / "Package" / "createUiDefinition.json").read_bytes(), + ) + self._fresh_run(run, stdout="Starting Package Creation using V3.1 tool") + with self.assertRaisesRegex(PackagingError, "does not match Package/mainTemplate.json"): + package_solution_v3_1(self.solution, version_bump="none") + + @mock.patch("sentinel_xdr_migration.packaging.shutil.which", return_value="pwsh") + @mock.patch("sentinel_xdr_migration.packaging.subprocess.run") + def test_rejects_missing_deploy_parser_parameter_when_referenced_or_in_inventory( + self, + run, + which, + ) -> None: + package = self.solution / "Package" + template_with_dangling_ref = { + "parameters": { + "DeployCustomDetection": {"type": "bool", "defaultValue": False}, + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "condition": "[parameters('DeployParser')]", + "name": "SampleParser", + } + ], + } + (package / "mainTemplate.json").write_text( + json.dumps(template_with_dangling_ref), + encoding="utf-8", + ) + with zipfile.ZipFile(package / "3.2.3.zip", "w") as archive: + archive.writestr("mainTemplate.json", (package / "mainTemplate.json").read_bytes()) + archive.writestr( + "createUiDefinition.json", + (package / "createUiDefinition.json").read_bytes(), + ) + self._fresh_run(run, stdout="Starting Package Creation using V3.1 tool") + with self.assertRaisesRegex(PackagingError, "DeployParser"): + package_solution_v3_1(self.solution, version_bump="none") + + parsers_dir = self.solution / "Parsers" + parsers_dir.mkdir(exist_ok=True) + (parsers_dir / "SampleParser.yaml").write_text("FunctionName: SampleParser\n", encoding="utf-8") + data_path = self.solution / "Data" / "Solution_Sample.json" + solution_data = json.loads(data_path.read_text(encoding="utf-8")) + solution_data["Parsers"] = ["Parsers/SampleParser.yaml"] + data_path.write_text(json.dumps(solution_data), encoding="utf-8") + template_without_deploy_parser = { + "parameters": { + "DeployCustomDetection": {"type": "bool", "defaultValue": False}, + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/savedSearches", + "name": "SampleParser", + } + ], + } + (package / "mainTemplate.json").write_text( + json.dumps(template_without_deploy_parser), + encoding="utf-8", + ) + with zipfile.ZipFile(package / "3.2.3.zip", "w") as archive: + archive.writestr("mainTemplate.json", (package / "mainTemplate.json").read_bytes()) + archive.writestr( + "createUiDefinition.json", + (package / "createUiDefinition.json").read_bytes(), + ) + self._fresh_run(run, stdout="Starting Package Creation using V3.1 tool") + with self.assertRaisesRegex(PackagingError, "DeployParser"): + package_solution_v3_1(self.solution, version_bump="none") + + @mock.patch("sentinel_xdr_migration.packaging.shutil.which", return_value="pwsh") + @mock.patch("sentinel_xdr_migration.packaging.subprocess.run") + def test_rejects_incomplete_ccf_connector_package_missing_definitions_or_dcr_config( + self, + run, + which, + ) -> None: + connectors_dir = self.solution / "Data Connectors" / "SampleCCF" + connectors_dir.mkdir(parents=True, exist_ok=True) + (connectors_dir / "connectorDefinition.json").write_text( + json.dumps({ + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectorDefinitions", + "name": "SampleCCFConnector", + "properties": {"connectorUiConfig": {"id": "SampleCCFConnector"}}, + }), + encoding="utf-8", + ) + (connectors_dir / "poller.json").write_text( + json.dumps({ + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "kind": "RestApiPoller", + "properties": { + "connectorDefinitionName": "SampleCCFConnector", + "dcrConfig": {"streamName": "Custom-Sample_CL"}, + }, + }), + encoding="utf-8", + ) + data_path = self.solution / "Data" / "Solution_Sample.json" + solution_data = json.loads(data_path.read_text(encoding="utf-8")) + solution_data["Data Connectors"] = [ + "Data Connectors/SampleCCF/connectorDefinition.json", + "Data Connectors/SampleCCF/poller.json", + ] + data_path.write_text(json.dumps(solution_data), encoding="utf-8") + + package = self.solution / "Package" + self._fresh_run(run, stdout="Starting Package Creation using V3.1 tool") + with self.assertRaisesRegex(PackagingError, "dataConnectorDefinitions"): + package_solution_v3_1(self.solution, version_bump="none") + + template_missing_dcr = { + "parameters": { + "DeployCustomDetection": {"type": "bool", "defaultValue": False}, + }, + "resources": [ + { + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectorDefinitions", + "name": "SampleCCFConnector", + }, + { + "type": "Microsoft.OperationalInsights/workspaces/providers/dataConnectors", + "kind": "RestApiPoller", + "properties": {"connectorDefinitionName": "SampleCCFConnector"}, + }, + ], + } + (package / "mainTemplate.json").write_text( + json.dumps(template_missing_dcr), + encoding="utf-8", + ) + with zipfile.ZipFile(package / "3.2.3.zip", "w") as archive: + archive.writestr("mainTemplate.json", (package / "mainTemplate.json").read_bytes()) + archive.writestr( + "createUiDefinition.json", + (package / "createUiDefinition.json").read_bytes(), + ) + self._fresh_run(run, stdout="Starting Package Creation using V3.1 tool") + with self.assertRaisesRegex(PackagingError, "dcrConfig"): + package_solution_v3_1(self.solution, version_bump="none") + if __name__ == "__main__": unittest.main() diff --git a/Tools/SentinelToXDRMigration/tests/test_pr_qualification_engine.py b/Tools/SentinelToXDRMigration/tests/test_pr_qualification_engine.py index c79f4941a4e..2cda68dda60 100644 --- a/Tools/SentinelToXDRMigration/tests/test_pr_qualification_engine.py +++ b/Tools/SentinelToXDRMigration/tests/test_pr_qualification_engine.py @@ -5,10 +5,15 @@ import json from pathlib import Path import shutil +import sys +import tempfile from typing import Any +import unittest import yaml +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from solution_manager.qualification.pr_applicability import ( BUILTIN_CHECK_CATALOG, determine_pr_applicability, @@ -954,4 +959,154 @@ def test_direct_override_blocks_validation_and_unapproved_human_review(tmp_path: assert res["checksById"]["QUAL-HR-01"]["status"] == "HUMAN_REVIEW_REQUIRED" +def test_kql_runtime_verified_rules_projection_and_val_ci_01_override(tmp_path: Path) -> None: + """Verify kqlRuntimeVerifiedRules projection, QUAL-RUNTIME-01 evaluation, and VAL-CI-01 static KQL override.""" + sol = tmp_path / "RuntimeVerifiedSol" + _build_base_package( + sol, + version="3.2.2", + include_xdr_props=True, + data_lists={ + "Analytic Rules": ["Analytic Rules/Rule1.yaml"], + "XDR Detections": ["XDR Detections/Rule1.yaml"], + }, + ) + ar_path = sol / "Analytic Rules" / "Rule1.yaml" + rule_id = "6e76fd9d-8104-41eb-bad3-26054a3ad5f0" + ar_sha = _build_valid_analytic_rule(ar_path, rule_id=rule_id) + xdr_path = sol / "XDR Detections" / "Rule1.yaml" + _build_valid_xdr_detection( + xdr_path, + source_rule_id=rule_id, + source_rule_path="Analytic Rules/Rule1.yaml", + source_rule_sha256=ar_sha, + ) + _write_json( + sol / "Sample Data" / "Tables" / "AzureDiagnostics_mock.json", + [ + { + "TimeGenerated": "2026-10-19T12:00:00Z", + "ResourceType": "PUBLICIPADDRESSES", + "Category": "DDoSMitigationReports", + "destPublicIpAddress_s": "198.51.100.10", + "sourcePublicIpAddress_s": "203.0.113.99", + } + ], + ) + + ar_doc = yaml.safe_load(ar_path.read_text(encoding="utf-8")) + ar_query_sha = hashlib.sha256(str(ar_doc["query"]).strip().encode("utf-8")).hexdigest() + xdr_doc = yaml.safe_load(xdr_path.read_text(encoding="utf-8")) + xdr_raw_q = xdr_doc.get("query") or ((xdr_doc.get("properties") or {}).get("queryCondition") or {}).get("queryText") + xdr_query_sha = hashlib.sha256(str(xdr_raw_q).strip().encode("utf-8")).hexdigest() + + qual_ev = _build_live_qualification_evidence() + qual_ev["runtimeKql"] = {"status": "passed", "passed": True} + qual_ev["ciChecks"] = { + "passed": False, + "reason": "Static KQL analyzer reported KS107/KS141", + "kqlStaticErrors": [ + { + "id": f"xdr-{rule_id}", + "originalRuleId": rule_id, + "code": "KS107", + "querySha256": xdr_query_sha, + }, + { + "id": rule_id, + "code": "KS141", + "querySha256": ar_query_sha, + }, + ], + } + wf_state = { + "workflowProfile": "qualification", + "operatorHash": "abcdef0123456789", + "stages": { + "conversion": { + "startedAt": "2026-10-19T10:00:00Z", + "completedAt": "2026-10-19T10:05:00Z", + }, + "validation": { + "attempts": 1, + "completedAt": "2026-10-19T10:10:00Z", + }, + "alertParity": { + "status": "passed", + "completedAt": "2026-10-19T10:25:00Z", + }, + "packaging": { + "completedAt": "2026-10-19T10:30:00Z", + }, + }, + } + + res = run_pr_qualification_checks( + sol, + workflow_state=wf_state, + qualification_evidence=qual_ev, + pr_number=4242, + run_number=1, + ) + + # Todo #5 assertions + verified = res["kqlRuntimeVerifiedRules"] + assert len(verified) == 2 + verified_ids = {v["id"]: v for v in verified} + assert rule_id in verified_ids + assert verified_ids[rule_id]["querySha256"] == ar_query_sha + assert verified_ids[rule_id]["status"] == "PASS" + assert f"xdr-{rule_id}" in verified_ids + assert verified_ids[f"xdr-{rule_id}"]["querySha256"] == xdr_query_sha + assert verified_ids[f"xdr-{rule_id}"]["originalRuleId"] == rule_id + + assert res["checksById"]["VAL-CI-01"]["status"] == "PASS" + assert "kqlRuntimeVerifiedRules" in res["checksById"]["VAL-CI-01"]["whatWeObserved"] + assert "kqlRuntimeVerifiedRules" in res["checksById"]["VAL-CI-01"]["whyPassedOrFailed"] + assert res["checksById"]["QUAL-RUNTIME-01"]["status"] == "PASS" + + # Todo #6 assertions + assert res["runNumber"] == 1 + assert res["prNumber"] == 4242 + assert res["firstPassPassed"] is True + assert res["operatorHash"] == "abcdef0123456789" + assert res["workflowProfile"] == "qualification" + assert res["stageTimestamps"] == { + "convertingStartedAtUtc": "2026-10-19T10:00:00Z", + "convertingCompletedAtUtc": "2026-10-19T10:05:00Z", + "validationCompletedAtUtc": "2026-10-19T10:10:00Z", + "qualificationCompletedAtUtc": "2026-10-19T10:25:00Z", + "packagedAtUtc": "2026-10-19T10:30:00Z", + } + assert res["verdict"] == "Qualified" + assert isinstance(res["verdictReasons"], list) + + # Ensure SYN* syntax errors are never overridden in VAL-CI-01 + qual_ev_syn = dict(qual_ev) + qual_ev_syn["ciChecks"] = { + "passed": False, + "reason": "Syntax error SYN001", + "kqlStaticErrors": [ + { + "id": rule_id, + "code": "SYN001", + "querySha256": ar_query_sha, + } + ], + } + res_syn = run_pr_qualification_checks( + sol, + workflow_state=wf_state, + qualification_evidence=qual_ev_syn, + run_number=2, + ) + assert res_syn["checksById"]["VAL-CI-01"]["status"] == "FAIL" + assert res_syn["firstPassPassed"] is False + assert res_syn["verdict"] == "Failed" + assert any(r.startswith("VAL-CI-01 (FAIL):") for r in res_syn["verdictReasons"]) + +class TestPrQualificationEngineUnittest(unittest.TestCase): + def test_kql_runtime_verified_rules_and_metrics(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + test_kql_runtime_verified_rules_projection_and_val_ci_01_override(Path(tmp)) diff --git a/Tools/SentinelToXDRMigration/tests/test_pr_qualification_report.py b/Tools/SentinelToXDRMigration/tests/test_pr_qualification_report.py index 206ddb2428d..561494b8f35 100644 --- a/Tools/SentinelToXDRMigration/tests/test_pr_qualification_report.py +++ b/Tools/SentinelToXDRMigration/tests/test_pr_qualification_report.py @@ -1,11 +1,18 @@ from __future__ import annotations +import json from pathlib import Path +import sys +import tempfile from typing import Any +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from solution_manager.reporting.pr_qualification_report import ( HTML_REPORT_FILENAME, MARKDOWN_SUMMARY_FILENAME, + _normalize_report_data, render_pr_qualification_html, render_pr_qualification_markdown, write_pr_qualification_reports, @@ -509,4 +516,58 @@ def test_relative_path_rebasing_with_output_dir_and_solution_root(tmp_path: Path assert "(./evidence/workbook-arg-sku-limitation.md)" in md_out +def test_write_pr_qualification_reports_preserves_npf_metrics_and_runtime_verified_rules(tmp_path: Path) -> None: + report_data = _sample_report_data( + workflowProfile="qualification", + operatorHash="0123456789abcdef", + runNumber=1, + prNumber=9876, + firstPassPassed=True, + stageTimestamps={ + "convertingStartedAtUtc": "2026-10-19T10:00:00Z", + "convertingCompletedAtUtc": "2026-10-19T10:05:00Z", + "validationCompletedAtUtc": "2026-10-19T10:10:00Z", + "qualificationCompletedAtUtc": "2026-10-19T10:20:00Z", + "packagedAtUtc": "2026-10-19T10:25:00Z", + }, + overallVerdict="Conditionally Qualified", + verdict="Conditionally Qualified", + verdictReasons=[ + "VAL-CI-002 (WAIVED_WITH_AUTHOR_EVIDENCE): Known upstream table rename tracked in #404." + ], + kqlRuntimeVerifiedRules=[ + { + "id": "xdr-rule-001", + "originalRuleId": "rule-001", + "file": "XDR Detections/Rule1.yaml", + "querySha256": "a" * 64, + "status": "PASS", + } + ], + ) + saved = _normalize_report_data(report_data) + paths = write_pr_qualification_reports(report_data, tmp_path / "npf-metrics-out") + assert paths["html"].is_file() + assert paths["markdown"].is_file() + + assert saved["workflowProfile"] == "qualification" + assert saved["operatorHash"] == "0123456789abcdef" + assert saved["runNumber"] == 1 + assert saved["prNumber"] == 9876 + assert saved["firstPassPassed"] is True + assert saved["stageTimestamps"]["convertingStartedAtUtc"] == "2026-10-19T10:00:00Z" + assert saved["stageTimestamps"]["packagedAtUtc"] == "2026-10-19T10:25:00Z" + assert saved["verdict"] == "Conditionally Qualified" + assert saved["verdictReasons"] == [ + "VAL-CI-002 (WAIVED_WITH_AUTHOR_EVIDENCE): Known upstream table rename tracked in #404." + ] + assert len(saved["kqlRuntimeVerifiedRules"]) == 1 + assert saved["kqlRuntimeVerifiedRules"][0]["id"] == "xdr-rule-001" + assert saved["kqlRuntimeVerifiedRules"][0]["originalRuleId"] == "rule-001" + assert saved["kqlRuntimeVerifiedRules"][0]["querySha256"] == "a" * 64 + +class TestPrQualificationReportUnittest(unittest.TestCase): + def test_write_pr_qualification_reports_npf_metrics(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + test_write_pr_qualification_reports_preserves_npf_metrics_and_runtime_verified_rules(Path(tmp)) diff --git a/Tools/SentinelToXDRMigration/tests/test_qualification_orchestrator.py b/Tools/SentinelToXDRMigration/tests/test_qualification_orchestrator.py index 692229dde29..9e07c6ccfa8 100644 --- a/Tools/SentinelToXDRMigration/tests/test_qualification_orchestrator.py +++ b/Tools/SentinelToXDRMigration/tests/test_qualification_orchestrator.py @@ -556,3 +556,509 @@ def test_evaluate_qualification_promotion_gate_pass_and_block_conditions(): assert blocked["status"] == "blocked" assert "ruleParityVerified" in blocked["missingGates"] assert "sanitizedEvidenceReady" in blocked["missingGates"] + + +def test_initial_disabled_deployment_and_get_readback_receipts(): + from unittest.mock import patch + from solution_manager.qualification.analytic_deployment import deploy_analytic_rules + from solution_manager.qualification.deployment import deploy_solution + + scratch, sol_root = _create_solution_fixture( + num_rules=1, + num_sentinel_workbooks=0, + num_defender_workbooks=0, + num_detections=1, + table_name="Syslog", + ) + try: + rule_id = "00000000-0000-4000-8000-000000000001" + rule_path = sol_root / "Analytic Rules" / "rule_1.yaml" + rule_path.write_text( + yaml.safe_dump( + { + "id": rule_id, + "name": "Fixture Rule 1", + "description": "Test rule", + "severity": "High", + "queryFrequency": "1h", + "queryPeriod": "2h", + "triggerOperator": "gt", + "triggerThreshold": 0, + "query": "Syslog | take 1", + "tactics": ["InitialAccess"], + "relevantTechniques": ["T1190"], + "entityMappings": [ + { + "entityType": "Host", + "fieldMappings": [ + {"identifier": "HostName", "columnName": "HostName"} + ], + } + ], + "version": "1.0.0", + "kind": "Scheduled", + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + det_path = sol_root / "XDR Detections" / "detection_1.yaml" + det_path.write_text( + yaml.safe_dump( + { + "schemaVersion": "1.0.0", + "version": "3.1.0", + "kind": "CustomDetection", + "resourceType": "Microsoft.Security/detectionRules", + "apiVersion": "2026-06-01-preview", + "contentProvenance": { + "requiredWorkloads": [{"workload": "sentinel", "dataTypes": ["Syslog"]}], + "requiredDataConnectors": [], + "source": { + "platform": "Microsoft Sentinel", + "kind": "AnalyticsRule", + "id": rule_id, + "path": "Analytic Rules/rule_1.yaml", + "querySha256": "hash", + "schedule": {"queryFrequency": "1h", "queryPeriod": "1h"}, + }, + "conversion": { + "tool": "sentinel-to-xdr-migration", + "version": "0.1.0", + "status": "converted", + "reviewRequired": False, + "reviewReasons": [], + "requiredWorkloads": ["sentinel"], + "warnings": [], + "errors": [], + }, + }, + "properties": { + "id": f"xdr-{rule_id}", + "displayName": "Fixture Detection 1", + "status": "disabled", + "queryCondition": {"queryText": "Syslog | take 1"}, + "schedule": {"frequency": "PT1H"}, + "detectionAction": { + "alertTemplate": { + "title": "Fixture Detection 1", + "description": "Test", + "severity": "high", + "entityMappings": { + "hosts": [{"id": "host1", "hostNameColumn": "HostName"}] + }, + } + }, + }, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + + arm_calls: list[tuple[str, str]] = [] + + def fake_arm_request(method, url, token, payload=None): + arm_calls.append((method, url)) + if method == "PUT": + assert payload["properties"]["enabled"] is False + return 200, {"name": rule_id, "properties": {"enabled": False}} + if method == "GET": + return 200, {"name": rule_id, "properties": {"enabled": False}} + raise AssertionError(f"Unexpected ARM method: {method}") + + with patch("solution_manager.qualification.analytic_deployment.require_locked_target", return_value=LAB_TARGET), patch( + "solution_manager.qualification.analytic_deployment._arm_token", return_value="arm-tok" + ), patch( + "solution_manager.qualification.analytic_deployment._arm_request", + side_effect=fake_arm_request, + ): + analytic_report = deploy_analytic_rules( + sol_root, + workspace_resource_id=LAB_TARGET["workspaceResourceId"], + verify_readback=True, + ) + + assert analytic_report["succeeded"] == 1 + assert analytic_report["initialDisabled"] is True + assert analytic_report["results"][0]["initialDisabled"] is True + assert analytic_report["results"][0]["readbackVerified"] is True + assert [m for m, _ in arm_calls] == ["PUT", "GET"] + + graph_calls: list[str] = [] + created_payload: dict = {} + + def fake_graph_request(method, url, token, body=None): + graph_calls.append(method) + if method == "GET" and not created_payload: + return 404, {} + if method == "POST": + assert body["status"] == "disabled" + created_payload.update(body) + return 201, {**body, "isEnabled": False, "status": "disabled"} + if method == "GET" and created_payload: + return 200, {**created_payload, "isEnabled": False, "status": "disabled"} + raise AssertionError(f"Unexpected Graph call: {method} {url}") + + (sol_root / "Data" / "Solution_HybridFixture.json").write_text( + json.dumps({"Version": "3.1.0", "Metadata": "SolutionMetadata.json"}), + encoding="utf-8", + ) + (sol_root / "SolutionMetadata.json").write_text( + json.dumps({"publisherId": "testpublisher", "offerId": "test-sentinel"}), + encoding="utf-8", + ) + + with patch("solution_manager.qualification.deployment.require_locked_target", return_value=LAB_TARGET), patch( + "solution_manager.qualification.deployment._deployment_token", return_value="graph-tok" + ), patch( + "solution_manager.qualification.deployment._graph_request", + side_effect=fake_graph_request, + ): + xdr_report = deploy_solution(sol_root) + + assert xdr_report["succeeded"] == 1 + assert xdr_report["initialDisabled"] is True + assert xdr_report["results"][0]["initialDisabled"] is True + assert xdr_report["results"][0]["readbackVerified"] is True + assert graph_calls == ["GET", "POST", "GET"] + finally: + shutil.rmtree(scratch) + + +def test_full_enable_fire_disable_lifecycle_and_tk_q11_readbacks(): + from unittest.mock import patch + from solution_manager.qualification import alert_parity + + scratch, sol_root = _create_solution_fixture( + num_rules=1, + num_sentinel_workbooks=0, + num_defender_workbooks=0, + num_detections=1, + table_name="Syslog", + ) + try: + contract = scratch / "contract.json" + payload = scratch / "payload.json" + plan_path = scratch / "batch-plan.json" + contract.write_text("{}", encoding="utf-8") + payload.write_text('[{"HostName": "host-tkq11"}]', encoding="utf-8") + plan_path.write_text( + json.dumps( + { + "fixtures": [ + { + "detection": "detection_1.yaml", + "contract": str(contract), + "payload": str(payload), + "scenarioMarker": "host-tkq11", + "expectedMatchKeys": ["host-tkq11"], + } + ] + } + ), + encoding="utf-8", + ) + + pair = { + "detection": "detection_1.yaml", + "displayName": "Fixture Detection 1", + "analyticRuleId": "00000000-0000-4000-8000-000000000001", + "customDetectionId": "xdr-rule-1", + "entityMappings": {}, + } + + call_sequence: list[str] = [] + sentinel_enabled = False + xdr_enabled = False + + def fake_arm(method, url, token, body=None): + nonlocal sentinel_enabled + if method == "GET": + call_sequence.append(f"sentinel_get:{sentinel_enabled}") + return 200, { + "name": "00000000-0000-4000-8000-000000000001", + "etag": "etag-1", + "kind": "Scheduled", + "properties": { + "displayName": "Fixture Rule 1", + "enabled": sentinel_enabled, + "query": "Syslog | where TimeGenerated > ago(1h)", + }, + } + if method == "PUT": + sentinel_enabled = bool(body["properties"]["enabled"]) + call_sequence.append(f"sentinel_put:{sentinel_enabled}") + return 200, {"properties": {"enabled": sentinel_enabled}} + raise AssertionError(f"Unexpected ARM request: {method} {url}") + + def fake_graph(method, url, token, body=None): + nonlocal xdr_enabled + if method == "GET": + call_sequence.append(f"xdr_get:{xdr_enabled}") + return 200, { + "id": "xdr-rule-1", + "displayName": "Fixture Detection 1", + "isEnabled": xdr_enabled, + "status": "enabled" if xdr_enabled else "disabled", + } + if method == "PATCH": + xdr_enabled = bool(body.get("isEnabled")) + call_sequence.append(f"xdr_patch:{xdr_enabled}") + return 200, { + "id": "xdr-rule-1", + "isEnabled": xdr_enabled, + "status": "enabled" if xdr_enabled else "disabled", + } + raise AssertionError(f"Unexpected Graph request: {method} {url}") + + def fake_ingest(c, p): + call_sequence.append("ingest_e2e_q01") + return {"recordsAccepted": 1} + + with patch.object(alert_parity, "require_locked_target", return_value=LAB_TARGET), patch.object( + alert_parity, "_arm_token", return_value="arm-tok" + ), patch.object( + alert_parity, "_deployment_token", return_value="graph-tok" + ), patch.object( + alert_parity, "_load_pairs", return_value=(sol_root, [pair]) + ), patch.object( + alert_parity, "_arm_request", side_effect=fake_arm + ), patch.object( + alert_parity, "_graph_request", side_effect=fake_graph + ), patch.object( + alert_parity, "_ingest", side_effect=fake_ingest + ): + started = alert_parity.start_alert_parity_batch(sol_root, plan_path=plan_path) + assert started["status"] == "awaiting-alerts" + assert started["initialDisabled"] is True + assert started["alertLifecycle"]["initialDisabled"] is True + assert call_sequence.index("ingest_e2e_q01") < call_sequence.index("sentinel_put:True") + assert call_sequence.index("ingest_e2e_q01") < call_sequence.index("xdr_patch:True") + assert sentinel_enabled is True + assert xdr_enabled is True + + sample_alert = { + "matchKey": "host-tkq11", + "severity": "high", + "tactics": ["Execution"], + "entities": {"hosts": [{"hostName": "host-tkq11"}]}, + "evidence": {"eventIds": ["host-tkq11"]}, + } + report = alert_parity.poll_alert_parity_batch( + sol_root, + results={ + "results": [ + { + "detection": "detection_1.yaml", + "analyticRule": {"alerts": [sample_alert]}, + "customDetection": {"alerts": [sample_alert]}, + } + ] + }, + ) + + assert report["status"] == "passed" + assert report["initialDisabled"] is True + assert report["returnedToDisabled"] is True + assert report["tkQ11Passed"] is True + assert report["alertLifecycle"]["initialDisabled"] is True + assert report["alertLifecycle"]["returnedToDisabled"] is True + assert report["alertLifecycle"]["tkQ11Verified"] is True + assert report["cleanup"][0]["disabled"] is True + assert report["cleanup"][0]["returnedToDisabled"] is True + assert sentinel_enabled is False + assert xdr_enabled is False + assert call_sequence[-2:] == ["sentinel_get:False", "xdr_get:False"] + finally: + shutil.rmtree(scratch) + + +def test_streaming_compiler_error_e2e_q02_captures_diagnostics_and_runs_finally_disable(): + from unittest.mock import patch + from solution_manager.qualification import alert_parity + + scratch, sol_root = _create_solution_fixture( + num_rules=1, + num_sentinel_workbooks=0, + num_defender_workbooks=0, + num_detections=1, + table_name="Syslog", + ) + try: + contract = scratch / "contract.json" + payload = scratch / "payload.json" + plan_path = scratch / "batch-plan.json" + contract.write_text("{}", encoding="utf-8") + payload.write_text('[{"HostName": "host-e2eq02"}]', encoding="utf-8") + plan_path.write_text( + json.dumps( + { + "fixtures": [ + { + "detection": "detection_1.yaml", + "contract": str(contract), + "payload": str(payload), + "scenarioMarker": "host-e2eq02", + "expectedMatchKeys": ["host-e2eq02"], + } + ] + } + ), + encoding="utf-8", + ) + + pair = { + "detection": "detection_1.yaml", + "displayName": "Fixture Detection 1", + "analyticRuleId": "00000000-0000-4000-8000-000000000001", + "customDetectionId": "xdr-rule-1", + "entityMappings": {}, + } + + sentinel_enabled = False + xdr_enabled = False + call_log: list[str] = [] + + def fake_arm(method, url, token, body=None): + nonlocal sentinel_enabled + if method == "GET": + call_log.append(f"sentinel_get:{sentinel_enabled}") + return 200, { + "name": "00000000-0000-4000-8000-000000000001", + "etag": "etag-1", + "kind": "Scheduled", + "properties": { + "displayName": "Fixture Rule 1", + "enabled": sentinel_enabled, + "query": "Syslog | where TimeGenerated > ago(1h)", + }, + } + if method == "PUT": + sentinel_enabled = bool(body["properties"]["enabled"]) + call_log.append(f"sentinel_put:{sentinel_enabled}") + return 200, {"properties": {"enabled": sentinel_enabled}} + raise AssertionError(f"Unexpected ARM request: {method} {url}") + + def fake_graph(method, url, token, body=None): + nonlocal xdr_enabled + if method == "GET": + call_log.append(f"xdr_get:{xdr_enabled}") + return 200, { + "id": "xdr-rule-1", + "displayName": "Fixture Detection 1", + "isEnabled": xdr_enabled, + "status": "enabled" if xdr_enabled else "disabled", + } + if method == "PATCH": + if body.get("isEnabled") is True or body.get("status") == "enabled": + call_log.append("xdr_patch_enable_compiler_error") + return 400, { + "error": { + "code": "HuntingRuleQueryStreamingNotSupported", + "message": "KqlCompilationFailed: Unsupported streaming table or dynamic literal in query.", + } + } + xdr_enabled = False + call_log.append("xdr_patch:False") + return 200, {"id": "xdr-rule-1", "isEnabled": False, "status": "disabled"} + raise AssertionError(f"Unexpected Graph request: {method} {url}") + + with patch.object(alert_parity, "require_locked_target", return_value=LAB_TARGET), patch.object( + alert_parity, "_arm_token", return_value="arm-tok" + ), patch.object( + alert_parity, "_deployment_token", return_value="graph-tok" + ), patch.object( + alert_parity, "_load_pairs", return_value=(sol_root, [pair]) + ), patch.object( + alert_parity, "_arm_request", side_effect=fake_arm + ), patch.object( + alert_parity, "_graph_request", side_effect=fake_graph + ), patch.object( + alert_parity, "_ingest", return_value={"recordsAccepted": 1} + ): + batch_res = alert_parity.start_alert_parity_batch(sol_root, plan_path=plan_path) + + assert batch_res["status"] == "failed" + assert batch_res["initialDisabled"] is True + assert batch_res["returnedToDisabled"] is True + assert batch_res["tkQ11Passed"] is True + assert len(batch_res["e2eQ02Diagnostics"]) == 1 + diag = batch_res["e2eQ02Diagnostics"][0] + assert diag["checkId"] == "E2E-Q02" + assert diag["errorCode"] == "HuntingRuleQueryStreamingNotSupported" + assert diag["stage"] == "enablement" + assert diag["streamingCompilerError"] is True + assert diag["ruleId"] == "xdr-rule-1" + assert sentinel_enabled is False + assert xdr_enabled is False + assert call_log[-2:] == ["sentinel_get:False", "xdr_get:False"] + + # Also verify orchestrator run_rule_parity surfaces alertLifecycle and e2eQ02Diagnostics + plan = plan_qualification_pipeline(sol_root, target=LAB_TARGET) + rq_consent, dep_consent, ing_consent = _valid_consents_for_plan(plan) + + def failing_parity_evaluator(pair, ing_ev, target): + return { + "statusCode": 400, + "errorPayload": { + "error": { + "code": "KqlCompilationFailed", + "message": "Dynamic literals are not supported in streaming custom detections.", + } + }, + "initialDisabled": True, + "returnedToDisabled": True, + "parityVerified": False, + } + + orch_state = run_qualification_pipeline( + sol_root, + target=LAB_TARGET, + runtime_query_consent=rq_consent, + deployment_consent=dep_consent, + mock_ingestion_consent=ing_consent, + parity_evaluator=failing_parity_evaluator, + resume=False, + ) + rp_stage = orch_state["qualificationStages"]["rule_parity"] + assert rp_stage["status"] == "failed" + assert rp_stage["alertLifecycle"]["initialDisabled"] is True + assert rp_stage["alertLifecycle"]["returnedToDisabled"] is True + assert rp_stage["alertLifecycle"]["tkQ11Verified"] is True + assert len(rp_stage["alertLifecycle"]["e2eQ02Diagnostics"]) == 1 + assert rp_stage["alertLifecycle"]["e2eQ02Diagnostics"][0]["checkId"] == "E2E-Q02" + assert rp_stage["alertLifecycle"]["e2eQ02Diagnostics"][0]["errorCode"] == "KqlCompilationFailed" + finally: + shutil.rmtree(scratch) + + +import unittest + + +class QualificationOrchestratorTestCase(unittest.TestCase): + def test_plan_qualification_pipeline_builds_consolidated_preflight_packet(self): + test_plan_qualification_pipeline_builds_consolidated_preflight_packet() + + def test_fail_closed_when_consents_missing_or_scope_mismatched(self): + test_fail_closed_when_consents_missing_or_scope_mismatched() + + def test_resumable_progression_across_all_5_stages_for_hybrid_solution(self): + test_resumable_progression_across_all_5_stages_for_hybrid_solution() + + def test_workbook_only_solution_like_aks_marks_rule_parity_not_applicable(self): + test_workbook_only_solution_like_aks_marks_rule_parity_not_applicable() + + def test_evaluate_qualification_promotion_gate_pass_and_block_conditions(self): + test_evaluate_qualification_promotion_gate_pass_and_block_conditions() + + def test_initial_disabled_deployment_and_get_readback_receipts(self): + test_initial_disabled_deployment_and_get_readback_receipts() + + def test_full_enable_fire_disable_lifecycle_and_tk_q11_readbacks(self): + test_full_enable_fire_disable_lifecycle_and_tk_q11_readbacks() + + def test_streaming_compiler_error_e2e_q02_captures_diagnostics_and_runs_finally_disable(self): + test_streaming_compiler_error_e2e_q02_captures_diagnostics_and_runs_finally_disable() + diff --git a/Tools/SentinelToXDRMigration/tests/test_release_hygiene.py b/Tools/SentinelToXDRMigration/tests/test_release_hygiene.py index 36836999c2e..2da2296c823 100644 --- a/Tools/SentinelToXDRMigration/tests/test_release_hygiene.py +++ b/Tools/SentinelToXDRMigration/tests/test_release_hygiene.py @@ -1,7 +1,18 @@ from __future__ import annotations +import os from pathlib import Path import re +import sys +import unittest + +TESTS_DIR = Path(__file__).resolve().parent +TOOLKIT_ROOT = TESTS_DIR.parent +for _path in (str(TOOLKIT_ROOT), str(TESTS_DIR)): + if _path not in sys.path: + sys.path.insert(0, _path) + +import conftest # noqa: E402 REPO_ROOT = Path(__file__).resolve().parents[3] @@ -30,9 +41,11 @@ "ignored", ".venv", "venv", - "Logs", + "logs", } +FORBIDDEN_FIXTURE_SUBSCRIPTION_GUID = "44e4eff8-1fcb-4a22-a7d6-992ac7286382" + FORBIDDEN_PATTERNS: list[tuple[str, re.Pattern[str]]] = [ ("personal handle or path (oferinbar/wiking/kingwil)", re.compile(r"\b(?:oferinbar|wiking|kingwil)\b", re.IGNORECASE)), ("Copilot session-state path", re.compile(r"\.copilot[\\/]+session-state", re.IGNORECASE)), @@ -40,7 +53,7 @@ ( "lab/environment GUID or workspace name", re.compile( - r"\b(?:39768270-33ce-4b90-a1a6-e0caeb3ba0ab|42382e39-f157-46d1-a931-b8cfd779ece5|756386d8-e2d4-4f09-905a-74b24313721f|956386d8-e2d4-4f09-905a-74b24313721f|0e8e91c3-dec0-4634-a473-1254c3bce737|525a15fc-6f89-424c-bb5f-17a652e5c15d|OptimizeSentinelWorkspace|DefaultResourceGroup-EUS)\b", + r"\b(?:39768270-33ce-4b90-a1a6-e0caeb3ba0ab|42382e39-f157-46d1-a931-b8cfd779ece5|756386d8-e2d4-4f09-905a-74b24313721f|956386d8-e2d4-4f09-905a-74b24313721f|0e8e91c3-dec0-4634-a473-1254c3bce737|525a15fc-6f89-424c-bb5f-17a652e5c15d|44e4eff8-1fcb-4a22-a7d6-992ac7286382|OptimizeSentinelWorkspace|DefaultResourceGroup-EUS)\b", re.IGNORECASE, ), ), @@ -57,7 +70,7 @@ def _iter_text_files() -> list[Path]: files.append(root) continue for item in root.rglob("*"): - if any(part in SKIP_DIR_NAMES for part in item.parts): + if any(part.lower() in SKIP_DIR_NAMES for part in item.parts): continue if not item.is_file() or item.resolve() == this_file: continue @@ -74,12 +87,40 @@ def test_no_personal_or_lab_residue_in_toolkit_files() -> None: continue rel = file_path.relative_to(REPO_ROOT).as_posix() for label, pattern in FORBIDDEN_PATTERNS: + if rel == "package-lock.json" and label == "internal ADO npm registry URL": + continue for match in pattern.finditer(text): line_no = text.count("\n", 0, match.start()) + 1 violations.append(f"{rel}:{line_no}: {label} ({match.group(0)!r})") assert not violations, "Release hygiene violations detected:\n" + "\n".join(violations[:50]) +def test_no_hardcoded_fixture_subscription_guid_in_tracked_toolkit_files() -> None: + fixture_path = ( + TOOLKIT_ROOT + / "tests" + / "fixtures" + / "workbooks" + / "sources" + / "Azure Activity" + / "Workbooks" + / "AzureActivity.json" + ) + fixture_text = fixture_path.read_text(encoding="utf-8") + assert FORBIDDEN_FIXTURE_SUBSCRIPTION_GUID not in fixture_text + assert "00000000-0000-0000-0000-000000000000" in fixture_text + + matches: list[str] = [] + for file_path in _iter_text_files(): + try: + text = file_path.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + if FORBIDDEN_FIXTURE_SUBSCRIPTION_GUID.lower() in text.lower(): + matches.append(file_path.relative_to(REPO_ROOT).as_posix()) + assert not matches, f"Forbidden subscription GUID found in: {matches}" + + def test_local_validator_does_not_interpolate_variables_in_exec_sync() -> None: for rel in (".script/local-validation/validate.ts", ".script/local-validation/validate.js"): path = REPO_ROOT / rel @@ -89,3 +130,50 @@ def test_local_validator_does_not_interpolate_variables_in_exec_sync() -> None: f"{rel} uses interpolated template strings in execSync (use execFileSync with argv arrays instead): " f"{interpolated_exec_sync}" ) + + +def test_conftest_enforces_credential_and_cli_isolation() -> None: + from azure.identity import ( + AzureCliCredential, + DeviceCodeCredential, + InteractiveBrowserCredential, + ) + + conftest_path = TESTS_DIR / "conftest.py" + assert conftest_path.is_file() + with conftest.isolated_auth_guard() as isolated_profile: + ambient_home_azure = (Path.home() / ".azure").resolve() + active_config_dir = Path(os.environ["AZURE_CONFIG_DIR"]).resolve() + assert active_config_dir == isolated_profile.resolve() + assert active_config_dir != ambient_home_azure + + for cred_cls in ( + InteractiveBrowserCredential, + DeviceCodeCredential, + AzureCliCredential, + ): + credential = cred_cls() + try: + credential.get_token("https://graph.microsoft.com/.default") + except RuntimeError as exc: + assert "blocked by test isolation guard" in str(exc) + else: + raise AssertionError(f"{cred_cls.__name__}.get_token was not blocked") + + +class ReleaseHygieneTests(unittest.TestCase): + def test_no_personal_or_lab_residue_in_toolkit_files(self) -> None: + test_no_personal_or_lab_residue_in_toolkit_files() + + def test_no_hardcoded_fixture_subscription_guid_in_tracked_toolkit_files(self) -> None: + test_no_hardcoded_fixture_subscription_guid_in_tracked_toolkit_files() + + def test_local_validator_does_not_interpolate_variables_in_exec_sync(self) -> None: + test_local_validator_does_not_interpolate_variables_in_exec_sync() + + def test_conftest_enforces_credential_and_cli_isolation(self) -> None: + test_conftest_enforces_credential_and_cli_isolation() + + +if __name__ == "__main__": + unittest.main() diff --git a/Tools/SentinelToXDRMigration/tests/test_runtime_consent.py b/Tools/SentinelToXDRMigration/tests/test_runtime_consent.py index 14895b865d1..b6ef4c18eda 100644 --- a/Tools/SentinelToXDRMigration/tests/test_runtime_consent.py +++ b/Tools/SentinelToXDRMigration/tests/test_runtime_consent.py @@ -2,6 +2,8 @@ import json import hashlib +import os +import sys import threading import tempfile import unittest @@ -12,6 +14,14 @@ from types import SimpleNamespace from unittest import mock +for _path in ( + str(Path(__file__).resolve().parents[1]), + str(Path(__file__).resolve().parent), +): + if _path not in sys.path: + sys.path.insert(0, _path) + +import conftest # noqa: E402 import yaml from sentinel_xdr_migration import cli @@ -1468,6 +1478,139 @@ def test_read_only_post_operations_are_not_write_authority(self) -> None: self.assertEqual("POST", plan["labReadConsent"]["operationFamilies"][XDR]["httpMethod"]) self.assertFalse(plan["labReadConsent"]["operationFamilies"][XDR]["resourceMutation"]) + def test_profiled_credential_get_token_info_uses_isolated_profile(self) -> None: + from solution_manager.qualification.graph_client_config import ( + azure_cli_credential, + configure_azure_cli_profile, + ) + + state_root = self.base / "profiled-cred-state" + isolated_profile = (self.base / "isolated-cli-profile").resolve() + configure_azure_cli_profile(state_root, isolated_profile) + observed_profiles: list[str | None] = [] + + class StubCliCredential: + def __init__(self, *, tenant_id=None, subscription=None): + self.tenant_id = tenant_id + self.subscription = subscription + + def get_token_info(self, *scopes, **kwargs): + observed_profiles.append(os.environ.get("AZURE_CONFIG_DIR")) + return SimpleNamespace( + token="stub-token", + expires_on=4070908800, + refresh_on=None, + scopes=scopes, + kwargs=kwargs, + ) + + previous_env = os.environ.get("AZURE_CONFIG_DIR") + with mock.patch( + "azure.identity.AzureCliCredential", + side_effect=StubCliCredential, + ): + credential = azure_cli_credential( + tenant_id=TENANT_ID, + state_root=state_root, + ) + token_info = credential.get_token_info( + "https://api.loganalytics.io/.default", + options={}, + ) + + self.assertEqual("stub-token", token_info.token) + self.assertEqual([str(isolated_profile)], observed_profiles) + self.assertEqual(previous_env, os.environ.get("AZURE_CONFIG_DIR")) + + def test_workbook_preflight_execution_profile_never_falls_back_to_home_azure( + self, + ) -> None: + from solution_manager.qualification.graph_client_config import ( + configure_azure_cli_profile, + ) + from solution_manager.qualification.workbook_preflight_workflow import ( + execution_profile, + ) + + unconfigured_state = (self.base / "unconfigured-workbook-state").resolve() + ambient_home = str((Path.home() / ".azure").resolve()) + with mock.patch.dict( + os.environ, + {"AZURE_CONFIG_DIR": ambient_home}, + clear=False, + ): + resolved_default = execution_profile(unconfigured_state) + + self.assertEqual( + str((unconfigured_state / "azure-cli-profile").resolve()), + resolved_default, + ) + self.assertNotEqual(ambient_home, resolved_default) + + configured_dir = (self.base / "configured-workbook-cli").resolve() + configure_azure_cli_profile(unconfigured_state, configured_dir) + self.assertEqual( + str(configured_dir), + execution_profile(unconfigured_state), + ) + + def test_conftest_guard_blocks_unmocked_live_interactive_browser_get_token( + self, + ) -> None: + from azure.identity import InteractiveBrowserCredential + + with conftest.isolated_auth_guard() as isolated_dir: + self.assertEqual( + str(isolated_dir.resolve()), + os.environ.get("AZURE_CONFIG_DIR"), + ) + credential = InteractiveBrowserCredential() + with self.assertRaisesRegex( + RuntimeError, + r"InteractiveBrowserCredential\.get_token", + ): + credential.get_token("https://graph.microsoft.com/.default") + + def test_onboarding_and_preflight_azure_cli_checks_use_configured_profile( + self, + ) -> None: + from solution_manager.core.onboarding import doctor + from solution_manager.core.preflight import run_preflight + from solution_manager.qualification.graph_client_config import ( + configure_azure_cli_profile, + ) + + state_root = (self.base / "cli-check-state").resolve() + isolated_profile = (self.base / "cli-check-profile").resolve() + configure_azure_cli_profile(state_root, isolated_profile) + observed_env: list[str | None] = [] + + def fake_runner(command: list[str], **_kwargs): + if Path(command[0]).stem.lower() == "az": + observed_env.append(os.environ.get("AZURE_CONFIG_DIR")) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + with mock.patch( + "solution_manager.core.onboarding.shutil.which", + return_value="/usr/bin/az", + ), mock.patch( + "solution_manager.core.onboarding._state_dir", + return_value=state_root, + ), mock.patch( + "solution_manager.core.preflight._state_dir", + return_value=state_root, + ): + doctor(state_dir=state_root, runner=fake_runner) + run_preflight( + self.solution, + runner=fake_runner, + ) + + self.assertEqual( + [str(isolated_profile), str(isolated_profile)], + observed_env, + ) + if __name__ == "__main__": unittest.main() diff --git a/Tools/SentinelToXDRMigration/tests/test_target_config.py b/Tools/SentinelToXDRMigration/tests/test_target_config.py index 86a635877e7..3f622d22ae3 100644 --- a/Tools/SentinelToXDRMigration/tests/test_target_config.py +++ b/Tools/SentinelToXDRMigration/tests/test_target_config.py @@ -84,6 +84,27 @@ def test_authoring_initialization_remains_offline_without_env(self) -> None: "authoring", json.loads(output.getvalue())["context"]["workflowProfile"] ) + def test_cli_workflow_init_defaults_to_qualification_profile(self) -> None: + self._write_config() + output = io.StringIO() + with self._mock_exact_workspace(): + with redirect_stdout(output): + result = main( + [ + "workflow-init", + "--solution", + str(self.solution), + "--confirm-configured-target", + "--version-bump", + "none", + ] + ) + self.assertEqual(0, result) + payload = json.loads(output.getvalue()) + self.assertEqual("qualification", payload["context"]["workflowProfile"]) + self.assertRegex(payload["operatorHash"], r"^[0-9a-f]{16}$") + self.assertEqual(payload["operatorHash"], payload["context"]["operatorHash"]) + def test_doctor_displays_configured_target_without_resolving_customer_id(self) -> None: self._write_config() status = doctor_solution_target(self.solution) @@ -353,6 +374,61 @@ def test_locked_target_resumes_without_environment_call(self) -> None: self.assertEqual(locked, target) run_az.assert_not_called() + def test_cli_pr_qualify_accepts_pr_number_and_run_number(self) -> None: + captured: dict[str, object] = {} + + def fake_evaluate( + sol_root, + *, + mode="auto", + is_migration=None, + waivers_file=None, + acknowledged_exclusions_file=None, + evidence_dir=None, + target_version=None, + pr_number=None, + run_number=None, + ): + captured["pr_number"] = pr_number + captured["run_number"] = run_number + return { + "solutionName": "Sample", + "solutionVersion": "3.0.0", + "mode": mode, + "isMigration": True, + "validationStatus": "Pass", + "qualificationStatus": "Pass", + "migrationStatus": "Pass", + "overallVerdict": "Pass", + "summary": {}, + } + + output = io.StringIO() + with mock.patch( + "solution_manager.cli.evaluate_solution_pr_qualification", + side_effect=fake_evaluate, + ), mock.patch( + "solution_manager.cli.write_pr_qualification_reports", + return_value={ + "html": self.solution / "report.html", + "markdown": self.solution / "summary.md", + }, + ), redirect_stdout(output): + rc = main( + [ + "pr-qualify", + "--solution", + str(self.solution), + "--pr-number", + "42", + "--run-number", + "7", + ] + ) + self.assertEqual(0, rc) + self.assertEqual(42, captured["pr_number"]) + self.assertEqual(7, captured["run_number"]) + if __name__ == "__main__": unittest.main() diff --git a/Tools/SentinelToXDRMigration/tests/test_workflow.py b/Tools/SentinelToXDRMigration/tests/test_workflow.py index 90de89e97a3..47b5a6d6e8b 100644 --- a/Tools/SentinelToXDRMigration/tests/test_workflow.py +++ b/Tools/SentinelToXDRMigration/tests/test_workflow.py @@ -405,10 +405,47 @@ def _runtime_reports(self, contract: dict, *, failed_graph_rules: set[str] | Non def test_profile_selection_is_required(self) -> None: with self.assertRaisesRegex(ValueError, "profile selection is required"): initialize_workflow(self.solution, workflow_profile=None) - defaulted = initialize_workflow(self.solution) - self.assertEqual("authoring", defaulted["context"]["workflowProfile"]) + with self.assertRaisesRegex(ValueError, "qualification requires a locked target context"): + initialize_workflow(self.solution) + defaulted = initialize_workflow(self.solution, **self._qualification_args()) + self.assertEqual("qualification", defaulted["context"]["workflowProfile"]) self.assertTrue(defaulted["context"]["profileSelectionConfirmed"]) + def test_operator_hash_recorded_and_validated(self) -> None: + import os + import subprocess + from solution_manager.core.workflow import resolve_operator_hash + + with mock.patch.dict( + os.environ, + { + "GITHUB_ACTOR": "Alice.Example", + "GIT_AUTHOR_EMAIL": "", + "GIT_COMMITTER_EMAIL": "", + }, + clear=False, + ): + expected = hashlib.sha256(b"alice.example").hexdigest()[:16] + self.assertEqual(expected, resolve_operator_hash()) + created = initialize_workflow(self.solution, workflow_profile="authoring") + self.assertEqual(expected, created["operatorHash"]) + self.assertEqual(expected, created["context"]["operatorHash"]) + self.assertRegex(created["operatorHash"], r"^[0-9a-f]{16}$") + + with mock.patch.dict( + os.environ, + { + "GITHUB_ACTOR": "", + "GIT_AUTHOR_EMAIL": "", + "GIT_COMMITTER_EMAIL": "", + }, + clear=False, + ), mock.patch( + "solution_manager.core.workflow.subprocess.run", + return_value=subprocess.CompletedProcess(["git"], 1, "", ""), + ): + self.assertEqual("0000000000000000", resolve_operator_hash()) + def test_initialize_creates_resumable_state(self) -> None: created = initialize_workflow( self.solution, From e96de75653dce2694c8c6ff7c072a6aba94c81c4 Mon Sep 17 00:00:00 2001 From: "radmond\\idoshabi" Date: Wed, 28 Oct 2026 15:38:33 +0200 Subject: [PATCH 2/6] Revert unrelated Create-Azure-Sentinel-Solution/common changes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2f9e241a-6b82-49ec-ac18-e83c833f2521 --- .../common/createCCPConnector.ps1 | 1 - .../common/get-ccp-details.ps1 | 30 +++++++------------ 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 index 00a48c77296..2f8dcba97a2 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/createCCPConnector.ps1 @@ -735,7 +735,6 @@ function createCCPConnectorResources($contentResourceDetails, $dataFileMetadata, } else { Write-Error "Failed to create or update 'auth' parameter." - throw "Failed to create or update 'auth' parameter." } $hasAppId = [bool](($armResource.properties.auth).PSobject.Properties.name -match "appId") diff --git a/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 b/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 index d867c0a2138..4a1c2b358f2 100644 --- a/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 +++ b/Tools/Create-Azure-Sentinel-Solution/common/get-ccp-details.ps1 @@ -243,10 +243,7 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold } } catch { - $relationError = "Error occurred while identifying relation between definition and poller File. Identified error in $($inputFile.Name). Error Details : $_" - Write-Host $relationError - Write-Error $relationError - throw $relationError + Write-Host "Error occurred while identifying relation between definition and poller File. Identified error in " + $inputFile.Name + ". Error Details : $_" } } } @@ -304,7 +301,7 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold # this is a custom table if ($dcrDataFlowStream.streams[0] -eq $ccpPollerFile.DCPollerStreamName) { $ccpPollerFile.DCRFilePath = $inputFile.FullName.Replace('\', '/') - if ($null -ne $fileContent.properties.dataFlows[0].outputStream -and $fileContent.properties.dataFlows[0].outputStream.contains('Custom-')) { + if ($fileContent.properties.dataFlows[0].outputStream.contains('Custom-')) { $ccpPollerFile.TableOutputStream = $dcrDataFlowStream.outputStream.Replace('Custom-', '') $ccpPollerFile.DCROutputStream = $dcrDataFlowStream.outputStream } @@ -321,7 +318,7 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold if ($dataFlowStreamName -eq $ccpPollerFile.DCPollerStreamName) { $ccpPollerFile.DCRFilePath = $inputFile.FullName.Replace('\', '/') - if ($null -ne $dataFlowOutputStreamName -and $dataFlowOutputStreamName.contains('Custom-')) { + if ($dataFlowOutputStreamName.contains('Custom-')) { $ccpPollerFile.TableOutputStream = $dataFlowStreamName.Replace('Custom-', '') $ccpPollerFile.DCROutputStream = $dataFlowOutputStreamName } @@ -341,7 +338,7 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold } if ($fileContent.properties.dataFlows[0].streams[0] -eq $ccpPollerFile.DCPollerStreamName) { $ccpPollerFile.DCRFilePath = $inputFile.FullName.Replace('\', '/') - if ($null -ne $fileContent.properties.dataFlows[0].outputStream -and $fileContent.properties.dataFlows[0].outputStream.contains('Custom-')) { + if ($fileContent.properties.dataFlows[0].outputStream.contains('Custom-')) { $ccpPollerFile.TableOutputStream = $fileContent.properties.dataFlows[0].outputStream.Replace('Custom-', '') $ccpPollerFile.DCROutputStream = $fileContent.properties.dataFlows[0].outputStream } @@ -350,10 +347,7 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold } } catch { - $relationError = "Error occurred while identifying relation between Poller and DCR File. Identified error in $($inputFile.Name). Error Details : $_" - Write-Host $relationError - Write-Error $relationError - throw $relationError + Write-Host "Error occurred while identifying relation between Poller and DCR File. Identified error in " + $inputFile.Name + ". Error Details : $_" } } } @@ -394,10 +388,7 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold } } catch { - $relationError = "Error occurred while identifying relation between DCR and Table File. Identified error in $($inputFile.Name). Error Details : $_" - Write-Host $relationError - Write-Error $relationError - throw $relationError + Write-Host "Error occurred while identifying relation between DCR and Table File. Identified error in " + $inputFile.Name + ". Error Details : $_" } } } @@ -407,12 +398,11 @@ function Get-CCP-Dict($dataFileMetadata, $baseFolderPath, $solutionName, $DCFold # THROW ERROR IF THERE IS NO RELATION BETWEEN DEFINITION->POLLER AND/OR POLLER->DCR if ($ccpDict.Count -gt 0) { foreach($localCCPDist in $ccpDict) { - if ($localCCPDist.DCDefinitionId -eq "" -or $localCCPDist.DCDefinitionFilePath -eq "" -or - $localCCPDist.DCPollerFilePath -eq "" -or ($localCCPDist.DCPollerStreamName -eq "" -and -not $localCCPDist.isDynamicStreamName) -or $localCCPDist.DCRFilePath -eq "") + if (($localCCPDist.DCDefinitionId -eq "" -or $localCCPDist.DCDefinitionFilePath -eq "" -or + $localCCPDist.DCPollerFilePath -eq "" -or $localCCPDist.DCPollerStreamName -eq "" -or $localCCPDist.DCRFilePath -eq "") -and + $localCCPDist.PollerKind -eq 'RestApiPoller') { - $relationError = "Please verify if there is a mapping between ConnectorDefinition with Poller file and/or Poller file with DCR file! If mapping is correct then check type property for ccp files!" - Write-Host $relationError - Write-Error $relationError + Write-Host "Please verify if there is a mapping between ConnectorDefinition with Poller file and/or Poller file with DCR file! If mapping is correct then check type property for ccp files!" exit 1 } } From 2a801f4a91317007bbf3f84915d16ab013c35f0d Mon Sep 17 00:00:00 2001 From: "radmond\\idoshabi" Date: Thu, 29 Oct 2026 11:55:19 +0200 Subject: [PATCH 3/6] Parallelize qualification KQL validation, deployment, and batch mock ingestion Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2f9e241a-6b82-49ec-ac18-e83c833f2521 --- .../qualification/alert_parity.py | 159 ++++++++++++++---- .../qualification/analytic_deployment.py | 16 +- .../qualification/deployment.py | 94 +++++++---- .../qualification/logs_query.py | 51 ++++-- .../solution_manager/qualification/runtime.py | 112 ++++++++---- .../tests/test_alert_parity.py | 89 ++++++++++ .../tests/test_logs_query.py | 19 +++ 7 files changed, 430 insertions(+), 110 deletions(-) diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/alert_parity.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/alert_parity.py index 7803fb45831..3407d650a13 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/alert_parity.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/alert_parity.py @@ -1,5 +1,6 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor import hashlib import json import os @@ -28,6 +29,7 @@ SENTINEL_ALERT_RULE_API_VERSION = "2024-03-01" STATE_FILE_NAME = "alert-parity-state.json" REPORT_FILE_NAME = "alert-parity-report.json" +MAX_PARITY_LIFECYCLE_WORKERS = 3 def _utc_now() -> str: @@ -356,8 +358,7 @@ def _read_rule_states( arm_token: str, graph_token: str, ) -> list[dict[str, Any]]: - states: list[dict[str, Any]] = [] - for pair in pairs: + def _read_one(pair: dict[str, Any]) -> dict[str, Any]: sentinel_status, sentinel = _arm_request( "GET", _sentinel_rule_url(workspace_resource_id, pair["analyticRuleId"]), @@ -386,17 +387,21 @@ def _read_rule_states( ) pair["sentinelResource"] = sentinel pair["initialDisabled"] = True - states.append( - { - "detection": pair["detection"], - "analyticRuleId": pair["analyticRuleId"], - "customDetectionId": pair["customDetectionId"], - "sentinelEnabled": False, - "xdrEnabled": False, - "initialDisabled": True, - } - ) - return states + return { + "detection": pair["detection"], + "analyticRuleId": pair["analyticRuleId"], + "customDetectionId": pair["customDetectionId"], + "sentinelEnabled": False, + "xdrEnabled": False, + "initialDisabled": True, + } + + if len(pairs) <= 1: + return [_read_one(pair) for pair in pairs] + workers = max(1, min(MAX_PARITY_LIFECYCLE_WORKERS, len(pairs))) + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(_read_one, pair) for pair in pairs] + return [future.result() for future in futures] def _set_pair_enabled( @@ -504,12 +509,25 @@ def _disable_all( results: list[dict[str, Any]] = [] readback_candidates: list[dict[str, Any]] = [] receipt_by_detection: dict[str, dict[str, Any]] = {} - for pair in rules: - errors = list( - _set_pair_enabled( - pair, workspace_resource_id, arm, graph, False + if len(rules) <= 1: + disable_errors_list = [ + list(_set_pair_enabled(pair, workspace_resource_id, arm, graph, False)) + for pair in rules + ] + else: + workers = max(1, min(MAX_PARITY_LIFECYCLE_WORKERS, len(rules))) + with ThreadPoolExecutor(max_workers=workers) as executor: + disable_errors_list = list( + executor.map( + lambda pair: list( + _set_pair_enabled( + pair, workspace_resource_id, arm, graph, False + ) + ), + rules, + ) ) - ) + for pair, errors in zip(rules, disable_errors_list): can_readback = bool( not errors and pair.get("analyticRuleId") @@ -735,10 +753,25 @@ def start_alert_parity( enabled: list[dict[str, Any]] = [] need_rollback = True try: - for pair in pairs: - errors = _set_pair_enabled( - pair, workspace_resource_id, arm_token, graph_token, True - ) + if len(pairs) <= 1: + enable_errors_list = [ + _set_pair_enabled( + pair, workspace_resource_id, arm_token, graph_token, True + ) + for pair in pairs + ] + else: + workers = max(1, min(MAX_PARITY_LIFECYCLE_WORKERS, len(pairs))) + with ThreadPoolExecutor(max_workers=workers) as executor: + enable_errors_list = list( + executor.map( + lambda pair: _set_pair_enabled( + pair, workspace_resource_id, arm_token, graph_token, True + ), + pairs, + ) + ) + for pair, errors in zip(pairs, enable_errors_list): enabled.append({"detection": pair["detection"], "errors": errors}) if pair.get("e2eQ02Diagnostics"): state["e2eQ02Diagnostics"].extend(pair["e2eQ02Diagnostics"]) @@ -950,21 +983,81 @@ def start_alert_parity_batch( need_rollback = True enabled_count = 0 try: - ingestions = [] + ingestions: list[dict[str, Any]] = [] + grouped_by_contract: dict[str, list[dict[str, Any]]] = {} for pair in pairs: fixture = fixture_by_detection[pair["detection"]] - ingestions.append( - { - "detection": pair["detection"], - "result": _ingest(fixture["contract"], fixture["payload"]), - } - ) + contract_key = str(Path(fixture["contract"]).expanduser().resolve()) + grouped_by_contract.setdefault(contract_key, []).append(pair) + + if len(grouped_by_contract) == len(pairs): + for pair in pairs: + fixture = fixture_by_detection[pair["detection"]] + ingestions.append( + { + "detection": pair["detection"], + "result": _ingest(fixture["contract"], fixture["payload"]), + } + ) + else: + ingestion_by_detection: dict[str, dict[str, Any]] = {} + for group_idx, (contract_key, group_pairs) in enumerate( + grouped_by_contract.items(), start=1 + ): + if len(group_pairs) == 1: + single_pair = group_pairs[0] + fixture = fixture_by_detection[single_pair["detection"]] + ingestion_by_detection[single_pair["detection"]] = _ingest( + fixture["contract"], fixture["payload"] + ) + else: + combined_records: list[dict[str, Any]] = [] + for group_pair in group_pairs: + fixture = fixture_by_detection[group_pair["detection"]] + payload_records = json.loads( + Path(fixture["payload"]).read_text(encoding="utf-8-sig") + ) + combined_records.extend(payload_records) + combined_payload_path = ( + state_path.parent + / f"alert-parity-batch-payload-{group_idx}.json" + ) + combined_payload_path.write_text( + json.dumps(combined_records, indent=2), + encoding="utf-8", + ) + shared_result = _ingest(contract_key, combined_payload_path) + for group_pair in group_pairs: + ingestion_by_detection[group_pair["detection"]] = shared_result + for pair in pairs: + ingestions.append( + { + "detection": pair["detection"], + "result": ingestion_by_detection[pair["detection"]], + } + ) state["ingestions"] = ingestions - for pair in pairs: - errors = _set_pair_enabled( - pair, workspace_resource_id, arm_token, graph_token, True - ) + if len(pairs) <= 1: + enable_errors_list = [ + _set_pair_enabled( + pair, workspace_resource_id, arm_token, graph_token, True + ) + for pair in pairs + ] + else: + workers = max(1, min(MAX_PARITY_LIFECYCLE_WORKERS, len(pairs))) + with ThreadPoolExecutor(max_workers=workers) as executor: + enable_errors_list = list( + executor.map( + lambda pair: _set_pair_enabled( + pair, workspace_resource_id, arm_token, graph_token, True + ), + pairs, + ) + ) + + for pair, errors in zip(pairs, enable_errors_list): if errors: pair_diags = list(pair.get("e2eQ02Diagnostics") or []) if pair_diags: diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/analytic_deployment.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/analytic_deployment.py index 399d9c4329e..7b0b9f17fdb 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/analytic_deployment.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/analytic_deployment.py @@ -1,5 +1,6 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor import json from pathlib import Path from typing import Any @@ -12,6 +13,9 @@ from solution_manager.core.target_context import require_locked_target +MAX_ANALYTIC_DEPLOY_WORKERS = 8 + + TRIGGER_OPERATORS = { "gt": "GreaterThan", "lt": "LessThan", @@ -122,7 +126,8 @@ def deploy_analytic_rules( raise ValueError("current applied package selection has missing Analytic Rule files") source_files = [source_by_path[path] for path in sorted(selected_paths)] token = _arm_token() - for source_path in source_files: + + def _deploy_one(source_path: Path) -> dict[str, Any]: result: dict[str, Any] = { "source": str(source_path), "id": None, @@ -189,7 +194,14 @@ def deploy_analytic_rules( result["initialDisabled"] = False result["readbackVerified"] = False result["error"] = {"code": type(exc).__name__, "message": str(exc)} - results.append(result) + return result + + if len(source_files) <= 1: + results = [_deploy_one(path) for path in source_files] + else: + workers = max(1, min(MAX_ANALYTIC_DEPLOY_WORKERS, len(source_files))) + with ThreadPoolExecutor(max_workers=workers) as executor: + results = list(executor.map(_deploy_one, source_files)) report = { "provider": "azure-resource-manager", diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/deployment.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/deployment.py index 459d9c5d0e5..631bd637362 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/deployment.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/deployment.py @@ -1,9 +1,11 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor import json import shutil import subprocess import sys +import time import urllib.error import urllib.parse import urllib.request @@ -262,6 +264,10 @@ def _deployment_token( return token +MAX_GRAPH_DEPLOY_WORKERS = 3 +MAX_GRAPH_REQUEST_RETRIES = 3 + + def _graph_request( method: str, url: str, @@ -287,6 +293,31 @@ def _graph_request( parsed = json.loads(content) except json.JSONDecodeError: parsed = {"error": {"message": content[:4000]}} + if exc.code in (429, 502, 503, 504): + retry_header = exc.headers.get("Retry-After") if exc.headers else None + try: + wait_seconds = max(1, min(60, int(retry_header))) if retry_header else 2 + except ValueError: + wait_seconds = 2 + for retry_attempt in range(MAX_GRAPH_REQUEST_RETRIES): + time.sleep(wait_seconds) + try: + with urllib.request.urlopen(request, timeout=90) as retry_resp: + retry_content = retry_resp.read().decode("utf-8") + return retry_resp.status, json.loads(retry_content) if retry_content else {} + except urllib.error.HTTPError as retry_exc: + retry_content = retry_exc.read().decode("utf-8", errors="replace") + try: + parsed = json.loads(retry_content) + except json.JSONDecodeError: + parsed = {"error": {"message": retry_content[:4000]}} + if retry_exc.code not in (429, 502, 503, 504) or retry_attempt + 1 == MAX_GRAPH_REQUEST_RETRIES: + return retry_exc.code, parsed + retry_header = retry_exc.headers.get("Retry-After") if retry_exc.headers else None + try: + wait_seconds = max(1, min(60, int(retry_header))) if retry_header else 2 ** (retry_attempt + 2) + except ValueError: + wait_seconds = 2 ** (retry_attempt + 2) return exc.code, parsed @@ -353,8 +384,8 @@ def deploy_solution( package_context = solution_package_context(root) token = _deployment_token(state_dir, tenant_id=target["tenantId"]) - results: list[dict[str, Any]] = [] - for path in files: + + def _deploy_one(path: Path) -> dict[str, Any]: try: document = yaml.safe_load(path.read_text(encoding="utf-8-sig")) or {} payload, tactic_projection = graph_detection_payload_with_projection( @@ -443,35 +474,38 @@ def deploy_solution( "contentProvenanceVerified": provenance_verified, } initial_disabled = bool(success and deployed_status == "disabled") - results.append( - { - "file": str(path), - "id": payload["id"], - "operation": operation if success else "failed", - "statusCode": response_status, - "success": success, - "deployedStatus": deployed_status, - "initialDisabled": initial_disabled, - "readbackVerified": initial_disabled, - "tacticProjection": tactic_projection, - "tacticProjectionVerified": tactic_projection_verified, - "contentProvenanceVerified": provenance_verified, - "error": error, - } - ) + return { + "file": str(path), + "id": payload["id"], + "operation": operation if success else "failed", + "statusCode": response_status, + "success": success, + "deployedStatus": deployed_status, + "initialDisabled": initial_disabled, + "readbackVerified": initial_disabled, + "tacticProjection": tactic_projection, + "tacticProjectionVerified": tactic_projection_verified, + "contentProvenanceVerified": provenance_verified, + "error": error, + } except (OSError, ValueError, yaml.YAMLError) as exc: - results.append( - { - "file": str(path), - "id": None, - "operation": "failed", - "statusCode": 0, - "success": False, - "initialDisabled": False, - "readbackVerified": False, - "error": {"message": str(exc)}, - } - ) + return { + "file": str(path), + "id": None, + "operation": "failed", + "statusCode": 0, + "success": False, + "initialDisabled": False, + "readbackVerified": False, + "error": {"message": str(exc)}, + } + + if len(files) <= 1: + results = [_deploy_one(path) for path in files] + else: + workers = max(1, min(MAX_GRAPH_DEPLOY_WORKERS, len(files))) + with ThreadPoolExecutor(max_workers=workers) as executor: + results = list(executor.map(_deploy_one, files)) report = { "solution": str(root), "provider": "microsoft-graph-beta", diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/logs_query.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/logs_query.py index 482aae29b5b..44e4a959362 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/logs_query.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/logs_query.py @@ -1,7 +1,9 @@ from __future__ import annotations import base64 +from concurrent.futures import ThreadPoolExecutor import json +import threading import time import uuid from datetime import date, datetime, timedelta, timezone @@ -17,6 +19,7 @@ DEFAULT_FAMILY_TIMEOUT_SECONDS = 30 * 60 MAX_TRANSIENT_RETRIES = 2 MAX_SERVER_TIMEOUT_SECONDS = 600 +DEFAULT_FAMILY_MAX_WORKERS = 4 class LogsQueryFailure(RuntimeError): @@ -390,18 +393,25 @@ def query_family( retry_count: int = MAX_TRANSIENT_RETRIES, monotonic: Callable[[], float] = time.monotonic, sleep: Callable[[float], None] = time.sleep, + max_workers: int = DEFAULT_FAMILY_MAX_WORKERS, ) -> list[dict[str, Any]]: if not 1 <= family_timeout_seconds <= 6 * 60 * 60: raise ValueError("family timeout must be between 1 and 21600 seconds") deadline = monotonic() + family_timeout_seconds - results: list[dict[str, Any]] = [] + results: list[dict[str, Any] | None] = [None] * len(queries) + state_lock = threading.Lock() stop_family = False - for item in queries: + + def _run_index(index: int) -> None: + nonlocal stop_family + item = queries[index] started = monotonic() detection = item["detection"] query = item["query"] - if stop_family: - results.append({ + with state_lock: + stopped = stop_family + if stopped: + results[index] = { "detection": detection, "status": "not-run", "valid": False, @@ -411,16 +421,18 @@ def query_family( "schema": [], "error": "not executed after an earlier provider-level failure", "errorDetails": None, - }) - continue + "durationMs": 0, + } + return remaining = deadline - monotonic() if remaining < 1: response = LogsQueryFailure( "the complete Sentinel query-family time budget expired", failure_kind="timeout", ) - stop_family = True - result = { + with state_lock: + stop_family = True + result: dict[str, Any] = { "status": "not-run", "valid": False, "querySha256": item["querySha256"], @@ -462,12 +474,29 @@ def query_family( "tables": [], } if exc.failure_kind != "query": - stop_family = True + with state_lock: + stop_family = True result.update({ "detection": detection, "querySha256": item["querySha256"], "valid": result["status"] == "passed", "durationMs": max(0, int((monotonic() - started) * 1000)), }) - results.append(result) - return results + results[index] = result + + if queries: + # Execute the first query as an auth/connectivity canary before fanning out. + _run_index(0) + remaining_indices = list(range(1, len(queries))) + if remaining_indices: + if stop_family or (deadline - monotonic()) < 1 or max_workers <= 1: + for idx in remaining_indices: + _run_index(idx) + else: + workers = max(1, min(max_workers, len(remaining_indices))) + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(_run_index, idx) for idx in remaining_indices] + for future in futures: + future.result() + + return [entry for entry in results if entry is not None] diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/runtime.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/runtime.py index ae87adef1c7..fca53bc38b1 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/runtime.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/runtime.py @@ -1,5 +1,6 @@ from __future__ import annotations +from concurrent.futures import ThreadPoolExecutor import hashlib import json import re @@ -51,6 +52,8 @@ ADVANCED_HUNTING_ENDPOINT = "https://graph.microsoft.com/v1.0/security/runHuntingQuery" MAX_GRAPH_IN_FLIGHT_REQUESTS = 5 +MAX_ADVANCED_HUNTING_WORKERS = 3 +ADVANCED_HUNTING_MIN_INTERVAL_SECONDS = 1.5 MAX_ADVANCED_HUNTING_RETRIES = 3 ADVANCED_HUNTING_RETRY_BUDGET_SECONDS = 5 * 60 MAX_ADVANCED_HUNTING_SAMPLE_ROWS = 10 @@ -1204,6 +1207,8 @@ def validate_advanced_hunting( state_dir: str | Path | None = None, tenant_id: str | None = None, provider_gap_evidence: str | Path | None = None, + max_workers: int = MAX_ADVANCED_HUNTING_WORKERS, + min_request_interval_seconds: float | None = None, ) -> dict[str, Any]: root = Path(solution).expanduser().resolve() workflow_state = existing_artifact_path(root, "workflow-state.json") @@ -1270,7 +1275,6 @@ def validate_advanced_hunting( fallback_provider="graph", ) output = root / "XDR Detections" - results: list[dict[str, Any]] = [] token = _advanced_hunting_token( effective_state_dir, tenant_id=locked_target["tenantId"] if locked_target is not None else tenant_id, @@ -1282,46 +1286,86 @@ def validate_advanced_hunting( query = document["properties"]["queryCondition"]["queryText"] detections.append((path, document, query)) + if min_request_interval_seconds is None: + is_live_runner = getattr(run_advanced_hunting_query, "__module__", "") == __name__ + interval = ADVANCED_HUNTING_MIN_INTERVAL_SECONDS if is_live_runner else 0.0 + else: + interval = max(0.0, float(min_request_interval_seconds)) + + results: list[dict[str, Any] | None] = [None] * len(detections) provider_failure: dict[str, Any] | None = None - for path, document, query in detections: - if provider_failure: - results.append(_not_run_result(path, output, query)) - continue - runtime = run_advanced_hunting_query(query, _token=token) + state_lock = threading.Lock() + rate_lock = threading.Lock() + next_allowed_start = 0.0 + + def _paced_query(query: str) -> dict[str, Any]: + nonlocal next_allowed_start + if interval > 0: + with rate_lock: + now = time.monotonic() + wait_seconds = max(0.0, next_allowed_start - now) + next_allowed_start = max(now, next_allowed_start) + interval + if wait_seconds > 0: + time.sleep(wait_seconds) + return run_advanced_hunting_query(query, _token=token) + + def _run_index(index: int) -> None: + nonlocal provider_failure + path, document, query = detections[index] + with state_lock: + failed = provider_failure is not None + if failed: + results[index] = _not_run_result(path, output, query) + return + runtime = _paced_query(query) if runtime.get("failureKind") == "provider": - provider_failure = { - "provider": "graph", - "statusCode": runtime["statusCode"], - "error": runtime["error"], - "errorDetails": runtime["errorDetails"], - } - results.append( - _not_run_result( - path, output, query, status_code=runtime["statusCode"] - ) + with state_lock: + if provider_failure is None: + provider_failure = { + "provider": "graph", + "statusCode": runtime["statusCode"], + "error": runtime["error"], + "errorDetails": runtime["errorDetails"], + } + results[index] = _not_run_result( + path, output, query, status_code=runtime["statusCode"] ) - continue + return binding_errors = custom_detail_schema_errors(document, runtime["schema"]) if runtime["ok"] else [] - results.append( - { - "detection": path.relative_to(output).as_posix(), - "status": "passed" if runtime["ok"] and not binding_errors else "failed", - "valid": runtime["ok"] and not binding_errors, - "querySha256": _query_sha256(query), - "statusCode": runtime["statusCode"], - "rowCount": runtime["rowCount"], - "schemaColumnCount": len(runtime["schema"]), - "schema": runtime["schema"], - "sampleRows": runtime.get("sampleRows", []), - "sampleRowsTruncated": runtime.get("sampleRowsTruncated", False), - "error": "; ".join(binding_errors) if binding_errors else runtime["error"], - "errorDetails": runtime["errorDetails"], - } - ) + results[index] = { + "detection": path.relative_to(output).as_posix(), + "status": "passed" if runtime["ok"] and not binding_errors else "failed", + "valid": runtime["ok"] and not binding_errors, + "querySha256": _query_sha256(query), + "statusCode": runtime["statusCode"], + "rowCount": runtime["rowCount"], + "schemaColumnCount": len(runtime["schema"]), + "schema": runtime["schema"], + "sampleRows": runtime.get("sampleRows", []), + "sampleRowsTruncated": runtime.get("sampleRowsTruncated", False), + "error": "; ".join(binding_errors) if binding_errors else runtime["error"], + "errorDetails": runtime["errorDetails"], + } + + if detections: + _run_index(0) + remaining_indices = list(range(1, len(detections))) + if remaining_indices: + if provider_failure is not None or max_workers <= 1: + for idx in remaining_indices: + _run_index(idx) + else: + workers = max(1, min(max_workers, len(remaining_indices))) + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(_run_index, idx) for idx in remaining_indices] + for future in futures: + future.result() + + final_results = [entry for entry in results if entry is not None] return _write_runtime_artifacts( root, "graph", - results, + final_results, provider_fallback=provider_fallback, provider_failure=provider_failure, runtime_query_consent=consent, diff --git a/Tools/SentinelToXDRMigration/tests/test_alert_parity.py b/Tools/SentinelToXDRMigration/tests/test_alert_parity.py index 6b26cc31d02..16df3bf33bf 100644 --- a/Tools/SentinelToXDRMigration/tests/test_alert_parity.py +++ b/Tools/SentinelToXDRMigration/tests/test_alert_parity.py @@ -468,6 +468,95 @@ def test_batch_start_enables_all_and_ingests_each_fixture( ["event-1"], ) + @mock.patch( + "sentinel_xdr_migration.alert_parity._ingest", + return_value={"status": "accepted", "recordsAccepted": 2}, + ) + @mock.patch( + "sentinel_xdr_migration.alert_parity._set_pair_enabled", + return_value=[], + ) + @mock.patch("sentinel_xdr_migration.alert_parity._read_rule_states") + @mock.patch( + "sentinel_xdr_migration.alert_parity._deployment_token", + return_value="graph-token", + ) + @mock.patch( + "sentinel_xdr_migration.alert_parity._arm_token", + return_value="arm-token", + ) + @mock.patch( + "sentinel_xdr_migration.alert_parity.require_locked_target", + return_value=TARGET, + ) + def test_batch_start_combines_payloads_for_shared_contract( + self, + _target: mock.Mock, + _arm: mock.Mock, + _graph: mock.Mock, + _read: mock.Mock, + enable: mock.Mock, + ingest: mock.Mock, + ) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + output = root / "XDR Detections" + output.mkdir() + shared_contract = root / "shared-contract.json" + shared_contract.write_text("{}", encoding="utf-8") + fixtures = [] + pairs = [] + for index, name in enumerate(("One.yaml", "Two.yaml"), start=1): + payload = root / f"payload-{index}.json" + payload.write_text( + json.dumps([{"marker": f"scenario-{index}"}]), + encoding="utf-8", + ) + fixtures.append( + { + "detection": name, + "contract": shared_contract.name, + "payload": payload.name, + "scenarioMarker": f"scenario-{index}", + "expectedMatchKeys": [f"event-{index}"], + } + ) + pairs.append( + { + "detection": name, + "displayName": name, + "analyticRuleId": f"source-{index}", + "customDetectionId": f"detection-{index}", + "entityMappings": {}, + } + ) + plan = root / "plan.json" + plan.write_text(json.dumps({"fixtures": fixtures}), encoding="utf-8") + + with mock.patch( + "sentinel_xdr_migration.alert_parity._load_pairs", + return_value=(root, pairs), + ): + result = start_alert_parity_batch( + root, + workspace_resource_id=TARGET["workspaceResourceId"], + plan_path=plan, + ) + + ingest.assert_called_once() + combined_payload_path = Path(ingest.call_args.args[1]) + combined_records = json.loads( + combined_payload_path.read_text(encoding="utf-8") + ) + + self.assertEqual(result["rulesEnabled"], 2) + self.assertEqual(result["fixturesIngested"], 2) + self.assertEqual(enable.call_count, 2) + self.assertEqual( + combined_records, + [{"marker": "scenario-1"}, {"marker": "scenario-2"}], + ) + if __name__ == "__main__": unittest.main() diff --git a/Tools/SentinelToXDRMigration/tests/test_logs_query.py b/Tools/SentinelToXDRMigration/tests/test_logs_query.py index c39c17f1e64..3141b37cdd5 100644 --- a/Tools/SentinelToXDRMigration/tests/test_logs_query.py +++ b/Tools/SentinelToXDRMigration/tests/test_logs_query.py @@ -316,6 +316,25 @@ def query_workspace(self, workspace_id, query, **kwargs): self.assertEqual(["passed", "not-run"], [item["status"] for item in results]) self.assertEqual(1, len(client.calls)) + def test_query_family_preserves_order_with_concurrent_workers(self): + client = FakeClient([ + success(logs_table("events", ["A"], [[1]])), + success(logs_table("events", ["A"], [[2]])), + success(logs_table("events", ["A"], [[3]])), + ]) + items = [ + {"detection": "one.yaml", "query": "T | take 1", "querySha256": "1" * 64}, + {"detection": "two.yaml", "query": "U | take 1", "querySha256": "2" * 64}, + {"detection": "three.yaml", "query": "V | take 1", "querySha256": "3" * 64}, + ] + results = query_family(client, WORKSPACE, items, max_workers=2) + self.assertEqual( + ["one.yaml", "two.yaml", "three.yaml"], + [item["detection"] for item in results], + ) + self.assertEqual(["passed", "passed", "passed"], [item["status"] for item in results]) + self.assertEqual(3, len(client.calls)) + def test_preflight_adapter_reuses_sdk_executor_and_locked_workspace(self): client = FakeClient([success(logs_table("events", ["A"], []))]) request = { From 8be71ce069d97541f83888cfbfea87ee02959a41 Mon Sep 17 00:00:00 2001 From: "radmond\\idoshabi" Date: Thu, 29 Oct 2026 16:32:27 +0200 Subject: [PATCH 4/6] feat(npf-metrics): complete NPF CI labeler and passesPerSolution reporting Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2f9e241a-6b82-49ec-ac18-e83c833f2521 --- .github/workflows/npf-pr-labeler.yaml | 181 +++++++++++++++--- .../qualification/pr_check_runner.py | 9 + .../reporting/pr_qualification_report.py | 11 ++ .../tests/test_pr_qualification_engine.py | 1 + .../tests/test_pr_qualification_report.py | 2 + 5 files changed, 179 insertions(+), 25 deletions(-) diff --git a/.github/workflows/npf-pr-labeler.yaml b/.github/workflows/npf-pr-labeler.yaml index 7939634cdc0..f83e1d0105d 100644 --- a/.github/workflows/npf-pr-labeler.yaml +++ b/.github/workflows/npf-pr-labeler.yaml @@ -44,16 +44,138 @@ jobs: } } - const hasXdrContent = activePaths.some(p => - /^Solutions\/[^/]+\/(XDR Detections|XDR Workbooks)\//.test(p) + const existing = new Set( + (context.payload.pull_request?.labels || []).map(l => l.name) ); + const headSha = context.payload.pull_request?.head?.sha; + const baseSha = context.payload.pull_request?.base?.sha; + + async function fetchFileText(repoPath) { + for (const ref of [headSha, baseSha].filter(Boolean)) { + try { + const resp = await github.rest.repos.getContent({ + owner: context.repo.owner, + repo: context.repo.repo, + path: repoPath, + ref + }); + if (resp.data && resp.data.content) { + return Buffer.from(resp.data.content, 'base64').toString('utf8'); + } + } catch (_) { + // Try fallback ref + } + } + return null; + } + + async function pathExistsAtHead(repoPath) { + for (const ref of [headSha, baseSha].filter(Boolean)) { + try { + const resp = await github.rest.repos.getContent({ + owner: context.repo.owner, + repo: context.repo.repo, + path: repoPath, + ref + }); + if (resp.data) { + return true; + } + } catch (_) { + // Try fallback ref + } + } + return false; + } + + let hasXdrContent = activePaths.some(p => + /^Solutions\/[^/]+\/(XDR Detections|XDR Workbooks)\//.test(p) + ) || existing.has('XDR'); + + if (!hasXdrContent) { + for (const sol of solutionNames) { + const touchesPackage = activePaths.some( + p => p === `Solutions/${sol}/Package/mainTemplate.json` || p.startsWith(`Solutions/${sol}/Package/`) + ); + if (touchesPackage) { + const hasXdrDet = await pathExistsAtHead(`Solutions/${sol}/XDR Detections`); + const hasXdrWb = !hasXdrDet && (await pathExistsAtHead(`Solutions/${sol}/XDR Workbooks`)); + if (hasXdrDet || hasXdrWb) { + hasXdrContent = true; + break; + } + } + } + } + const hasValidationEvidence = activePaths.some(p => /^Solutions\/[^/]+\/Evidence\/(pr-qualification\/(pr-qualification-result\.json|qualification-report\.html|qualification-summary\.md)|workflow-state\.json|decision-manifest\.json|[^/]+\/evidence\.json)$/.test(p) ); - const hasQualificationEvidence = activePaths.some(p => + let hasQualificationEvidence = activePaths.some(p => /^Solutions\/[^/]+\/Evidence\/(qualification-evidence\.json|receipts\/(alert-parity-live|workbook-queries-live|arm-deployment-readback|graph-detection-deployment)\.json)$/.test(p) ); + const verdicts = {}; + for (const sol of [...solutionNames].sort()) { + const qualResultCandidates = activePaths.filter( + p => + p === `Solutions/${sol}/Evidence/pr-qualification/pr-qualification-result.json` || + p === `Solutions/${sol}/Evidence/pr-qualification-result.json` + ); + let solVerdict = null; + for (const cand of qualResultCandidates) { + const rawJson = await fetchFileText(cand); + if (!rawJson) continue; + try { + const parsed = JSON.parse(rawJson.replace(/^\uFEFF/, '')); + const modeVal = String(parsed?.mode || parsed?.workflowProfile || '').trim().toLowerCase(); + const qualStatus = String(parsed?.qualificationStatus || '').trim(); + const verdictVal = String(parsed?.verdict || parsed?.overallVerdict || '').trim(); + if ( + modeVal === 'qualification' || + modeVal === 'full' || + qualStatus === 'Qualified' || + qualStatus === 'Conditionally Qualified' || + verdictVal === 'Qualified' || + verdictVal === 'Conditionally Qualified' + ) { + hasQualificationEvidence = true; + } + if (verdictVal) { + solVerdict = verdictVal; + break; + } + } catch (_) { + // Ignore parse errors + } + } + + const summaryCandidates = activePaths.filter( + p => + p === `Solutions/${sol}/Evidence/pr-qualification/qualification-summary.md` || + p === `Solutions/${sol}/Evidence/qualification-summary.md` + ); + for (const cand of summaryCandidates) { + const mdText = await fetchFileText(cand); + if (!mdText) continue; + if (/\bConditionally Qualified\b/i.test(mdText)) { + hasQualificationEvidence = true; + if (!solVerdict) solVerdict = 'Conditionally Qualified'; + } else if (/\bQualified\b/i.test(mdText)) { + hasQualificationEvidence = true; + if (!solVerdict) solVerdict = 'Qualified'; + } else if (!solVerdict && /\bPartially Validated\b/i.test(mdText)) { + solVerdict = 'Partially Validated'; + } else if (!solVerdict && /\bValidated\b/i.test(mdText)) { + solVerdict = 'Validated'; + } + } + + if (solVerdict) { + verdicts[sol] = solVerdict; + } + } + const desired = new Set(); const npfModes = []; if (solutionNames.size > 0) { @@ -74,30 +196,12 @@ jobs: } // Determine 1P Solution / ISV Solution / Community Solution from SolutionMetadata.json - const headSha = context.payload.pull_request?.head?.sha; - const baseSha = context.payload.pull_request?.base?.sha; const publisherTiers = new Set(); const solutionDetails = []; for (const sol of [...solutionNames].sort()) { const metaPath = `Solutions/${sol}/SolutionMetadata.json`; - let metaText = null; - for (const ref of [headSha, baseSha].filter(Boolean)) { - try { - const resp = await github.rest.repos.getContent({ - owner: context.repo.owner, - repo: context.repo.repo, - path: metaPath, - ref - }); - if (resp.data && resp.data.content) { - metaText = Buffer.from(resp.data.content, 'base64').toString('utf8'); - break; - } - } catch (_) { - // Try fallback ref - } - } + const metaText = await fetchFileText(metaPath); let tierLabel = 'Unknown'; if (metaText) { try { @@ -127,9 +231,6 @@ jobs: return; } - const existing = new Set( - (context.payload.pull_request?.labels || []).map(l => l.name) - ); const labelsToAdd = [...desired].filter(l => !existing.has(l)); if (labelsToAdd.length > 0) { @@ -144,6 +245,32 @@ jobs: core.info(`All applicable labels already present: ${[...desired].join(', ')}`); } + if (npfModes.length === 0) { + core.info('PR has no NPF modes (Migration/Validation/Qualification); skipping npf-meta block.'); + return; + } + + const highestStage = npfModes.includes('Qualification') + ? 'Qualification' + : (npfModes.includes('Validation') ? 'Validation' : 'Migration'); + + let commitPasses = Number(context.payload.pull_request?.commits || 0); + if (!commitPasses || commitPasses < 1) { + try { + const prCommits = await github.paginate(github.rest.pulls.listCommits, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + per_page: 100 + }); + commitPasses = Math.max(1, prCommits.length); + } catch (_) { + commitPasses = 1; + } + } + const solCount = Math.max(1, solutionDetails.length); + const passesPerSolution = Number((commitPasses / solCount).toFixed(2)); + // Build machine-readable npf-meta block for PR body and step summary const currentBody = context.payload.pull_request?.body || ''; const metaRegex = //s; @@ -165,6 +292,10 @@ jobs: solutions: solutionDetails.map(s => s.name), publisherTiers: [...publisherTiers], npfModes, + highestStage, + commitPasses, + passesPerSolution, + verdicts, synchronizePushObserved }; const metaComment = ``; diff --git a/Tools/SentinelToXDRMigration/solution_manager/qualification/pr_check_runner.py b/Tools/SentinelToXDRMigration/solution_manager/qualification/pr_check_runner.py index 5e7dab0185b..7532c1a8d50 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/qualification/pr_check_runner.py +++ b/Tools/SentinelToXDRMigration/solution_manager/qualification/pr_check_runner.py @@ -3493,6 +3493,14 @@ def run_pr_qualification_checks( pr_number=pr_number, run_number=run_number, ) + stages_map = wf_state.get("stages") if isinstance(wf_state.get("stages"), Mapping) else {} + val_stage_map = stages_map.get("validation") if isinstance(stages_map.get("validation"), Mapping) else {} + val_attempts = val_stage_map.get("attempts") + passes_per_solution = ( + max(1, int(val_attempts)) + if isinstance(val_attempts, int) and val_attempts >= 1 + else resolved_run_number + ) first_pass_passed = bool(resolved_run_number == 1 and summary_counts.get("fail", 0) == 0) stage_timestamps = _extract_stage_timestamps(wf_state) operator_hash = _resolve_operator_hash(root, wf_state) @@ -3515,6 +3523,7 @@ def run_pr_qualification_checks( "workflowProfile": workflow_profile, "operatorHash": operator_hash, "runNumber": resolved_run_number, + "passesPerSolution": passes_per_solution, "prNumber": resolved_pr_number, "firstPassPassed": first_pass_passed, "stageTimestamps": stage_timestamps, diff --git a/Tools/SentinelToXDRMigration/solution_manager/reporting/pr_qualification_report.py b/Tools/SentinelToXDRMigration/solution_manager/reporting/pr_qualification_report.py index 5cb4b1ee928..89485510fd6 100644 --- a/Tools/SentinelToXDRMigration/solution_manager/reporting/pr_qualification_report.py +++ b/Tools/SentinelToXDRMigration/solution_manager/reporting/pr_qualification_report.py @@ -485,6 +485,16 @@ def _normalize_report_data(report_data: dict[str, Any] | None) -> dict[str, Any] except (TypeError, ValueError): run_number = 1 + passes_per_solution_raw = raw.get("passesPerSolution") + try: + passes_per_solution = ( + max(1, int(passes_per_solution_raw)) + if passes_per_solution_raw is not None + else run_number + ) + except (TypeError, ValueError): + passes_per_solution = run_number + pr_number_raw = raw.get("prNumber") try: pr_number: int | None = int(pr_number_raw) if pr_number_raw is not None else None @@ -566,6 +576,7 @@ def _clean_ts(val_in: Any) -> str | None: "workflowProfile": workflow_profile, "operatorHash": operator_hash, "runNumber": run_number, + "passesPerSolution": passes_per_solution, "prNumber": pr_number, "firstPassPassed": first_pass_passed, "stageTimestamps": stage_timestamps, diff --git a/Tools/SentinelToXDRMigration/tests/test_pr_qualification_engine.py b/Tools/SentinelToXDRMigration/tests/test_pr_qualification_engine.py index 2cda68dda60..90021fd7e35 100644 --- a/Tools/SentinelToXDRMigration/tests/test_pr_qualification_engine.py +++ b/Tools/SentinelToXDRMigration/tests/test_pr_qualification_engine.py @@ -1067,6 +1067,7 @@ def test_kql_runtime_verified_rules_projection_and_val_ci_01_override(tmp_path: # Todo #6 assertions assert res["runNumber"] == 1 + assert res["passesPerSolution"] == 1 assert res["prNumber"] == 4242 assert res["firstPassPassed"] is True assert res["operatorHash"] == "abcdef0123456789" diff --git a/Tools/SentinelToXDRMigration/tests/test_pr_qualification_report.py b/Tools/SentinelToXDRMigration/tests/test_pr_qualification_report.py index 561494b8f35..30bc9d8d72d 100644 --- a/Tools/SentinelToXDRMigration/tests/test_pr_qualification_report.py +++ b/Tools/SentinelToXDRMigration/tests/test_pr_qualification_report.py @@ -521,6 +521,7 @@ def test_write_pr_qualification_reports_preserves_npf_metrics_and_runtime_verifi workflowProfile="qualification", operatorHash="0123456789abcdef", runNumber=1, + passesPerSolution=2, prNumber=9876, firstPassPassed=True, stageTimestamps={ @@ -553,6 +554,7 @@ def test_write_pr_qualification_reports_preserves_npf_metrics_and_runtime_verifi assert saved["workflowProfile"] == "qualification" assert saved["operatorHash"] == "0123456789abcdef" assert saved["runNumber"] == 1 + assert saved["passesPerSolution"] == 2 assert saved["prNumber"] == 9876 assert saved["firstPassPassed"] is True assert saved["stageTimestamps"]["convertingStartedAtUtc"] == "2026-10-19T10:00:00Z" From 42d79591c2d09b0a907541d79554a6c04be3a8e4 Mon Sep 17 00:00:00 2001 From: "radmond\\idoshabi" Date: Thu, 29 Oct 2026 16:48:01 +0200 Subject: [PATCH 5/6] chore(npf-labeler): omit internal pass and verdict metrics from public PR body Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2f9e241a-6b82-49ec-ac18-e83c833f2521 --- .github/workflows/npf-pr-labeler.yaml | 41 +-------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/.github/workflows/npf-pr-labeler.yaml b/.github/workflows/npf-pr-labeler.yaml index f83e1d0105d..0343c911164 100644 --- a/.github/workflows/npf-pr-labeler.yaml +++ b/.github/workflows/npf-pr-labeler.yaml @@ -250,53 +250,14 @@ jobs: return; } - const highestStage = npfModes.includes('Qualification') - ? 'Qualification' - : (npfModes.includes('Validation') ? 'Validation' : 'Migration'); - - let commitPasses = Number(context.payload.pull_request?.commits || 0); - if (!commitPasses || commitPasses < 1) { - try { - const prCommits = await github.paginate(github.rest.pulls.listCommits, { - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - per_page: 100 - }); - commitPasses = Math.max(1, prCommits.length); - } catch (_) { - commitPasses = 1; - } - } - const solCount = Math.max(1, solutionDetails.length); - const passesPerSolution = Number((commitPasses / solCount).toFixed(2)); - // Build machine-readable npf-meta block for PR body and step summary const currentBody = context.payload.pull_request?.body || ''; const metaRegex = //s; - let priorSyncObserved = false; - const existingMetaMatch = currentBody.match(metaRegex); - if (existingMetaMatch) { - try { - const parsedMeta = JSON.parse(existingMetaMatch[1]); - if (parsedMeta && parsedMeta.synchronizePushObserved === true) { - priorSyncObserved = true; - } - } catch (_err) { - // Ignore malformed prior metadata block - } - } - const synchronizePushObserved = priorSyncObserved || (context.payload.action === 'synchronize'); const npfMeta = { solutionCount: solutionDetails.length, solutions: solutionDetails.map(s => s.name), publisherTiers: [...publisherTiers], - npfModes, - highestStage, - commitPasses, - passesPerSolution, - verdicts, - synchronizePushObserved + npfModes }; const metaComment = ``; From e99cb087391d27fce6cb6f254a4ea5bd947e88f4 Mon Sep 17 00:00:00 2001 From: "radmond\\idoshabi" Date: Thu, 29 Oct 2026 16:49:51 +0200 Subject: [PATCH 6/6] Revert "chore(npf-labeler): omit internal pass and verdict metrics from public PR body" This reverts commit 42d79591c2d09b0a907541d79554a6c04be3a8e4. --- .github/workflows/npf-pr-labeler.yaml | 41 ++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/.github/workflows/npf-pr-labeler.yaml b/.github/workflows/npf-pr-labeler.yaml index 0343c911164..f83e1d0105d 100644 --- a/.github/workflows/npf-pr-labeler.yaml +++ b/.github/workflows/npf-pr-labeler.yaml @@ -250,14 +250,53 @@ jobs: return; } + const highestStage = npfModes.includes('Qualification') + ? 'Qualification' + : (npfModes.includes('Validation') ? 'Validation' : 'Migration'); + + let commitPasses = Number(context.payload.pull_request?.commits || 0); + if (!commitPasses || commitPasses < 1) { + try { + const prCommits = await github.paginate(github.rest.pulls.listCommits, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + per_page: 100 + }); + commitPasses = Math.max(1, prCommits.length); + } catch (_) { + commitPasses = 1; + } + } + const solCount = Math.max(1, solutionDetails.length); + const passesPerSolution = Number((commitPasses / solCount).toFixed(2)); + // Build machine-readable npf-meta block for PR body and step summary const currentBody = context.payload.pull_request?.body || ''; const metaRegex = //s; + let priorSyncObserved = false; + const existingMetaMatch = currentBody.match(metaRegex); + if (existingMetaMatch) { + try { + const parsedMeta = JSON.parse(existingMetaMatch[1]); + if (parsedMeta && parsedMeta.synchronizePushObserved === true) { + priorSyncObserved = true; + } + } catch (_err) { + // Ignore malformed prior metadata block + } + } + const synchronizePushObserved = priorSyncObserved || (context.payload.action === 'synchronize'); const npfMeta = { solutionCount: solutionDetails.length, solutions: solutionDetails.map(s => s.name), publisherTiers: [...publisherTiers], - npfModes + npfModes, + highestStage, + commitPasses, + passesPerSolution, + verdicts, + synchronizePushObserved }; const metaComment = ``;