diff --git a/.github/workflows/publish-artifacts.yml b/.github/workflows/publish-artifacts.yml index 84cb681..823c1c6 100644 --- a/.github/workflows/publish-artifacts.yml +++ b/.github/workflows/publish-artifacts.yml @@ -6,6 +6,9 @@ on: signing_enabled: description: "Whether package signing was enabled for this run." value: ${{ jobs.package.outputs.signing_enabled }} + signature_verification_enabled: + description: "Whether signed package verification was enabled for this run." + value: ${{ jobs.package.outputs.signature_verification_enabled }} inputs: package_version: description: "Optional package version, usually the release tag without the leading v." @@ -47,6 +50,8 @@ on: required: false NUGET_SIGN_TIMESTAMP_URL: required: false + NUGET_SIGN_SKIP_VERIFICATION: + required: false permissions: contents: read @@ -57,6 +62,7 @@ jobs: runs-on: ubuntu-latest outputs: signing_enabled: ${{ steps.signing.outputs.enabled }} + signature_verification_enabled: ${{ steps.signing.outputs.verify_enabled }} steps: - name: Checkout @@ -150,6 +156,7 @@ jobs: NUGET_SIGN_CERTIFICATE_PASSWORD: ${{ secrets.NUGET_SIGN_CERTIFICATE_PASSWORD }} NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }} NUGET_SIGN_TIMESTAMP_URL: ${{ secrets.NUGET_SIGN_TIMESTAMP_URL }} + NUGET_SIGN_SKIP_VERIFICATION: ${{ secrets.NUGET_SIGN_SKIP_VERIFICATION }} run: | if [ -n "$NUGET_SIGN_CERTIFICATE_BASE64" ] \ && [ -n "$NUGET_SIGN_CERTIFICATE_PASSWORD" ] \ @@ -162,6 +169,14 @@ jobs: echo "Package signing disabled. Continuing with unsigned packages." fi + if [ "${NUGET_SIGN_SKIP_VERIFICATION,,}" = "true" ]; then + echo "verify_enabled=false" >> "$GITHUB_OUTPUT" + echo "Signed package verification disabled by NUGET_SIGN_SKIP_VERIFICATION." + else + echo "verify_enabled=true" >> "$GITHUB_OUTPUT" + echo "Signed package verification enabled." + fi + - name: Import signing certificate if: ${{ steps.signing.outputs.enabled == 'true' }} env: @@ -186,7 +201,7 @@ jobs: done - name: Verify signed packages - if: ${{ steps.signing.outputs.enabled == 'true' }} + if: ${{ steps.signing.outputs.enabled == 'true' && steps.signing.outputs.verify_enabled == 'true' }} env: NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }} run: | diff --git a/.github/workflows/publish-attested.yml b/.github/workflows/publish-attested.yml index 1c012b3..8ceb7f4 100644 --- a/.github/workflows/publish-attested.yml +++ b/.github/workflows/publish-attested.yml @@ -22,6 +22,8 @@ on: required: false NUGET_SIGN_TIMESTAMP_URL: required: false + NUGET_SIGN_SKIP_VERIFICATION: + required: false permissions: contents: write @@ -39,6 +41,7 @@ jobs: NUGET_SIGN_CERTIFICATE_PASSWORD: ${{ secrets.NUGET_SIGN_CERTIFICATE_PASSWORD }} NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }} NUGET_SIGN_TIMESTAMP_URL: ${{ secrets.NUGET_SIGN_TIMESTAMP_URL }} + NUGET_SIGN_SKIP_VERIFICATION: ${{ secrets.NUGET_SIGN_SKIP_VERIFICATION }} release: name: Upload artifacts to draft release diff --git a/.github/workflows/publish-packages.yml b/.github/workflows/publish-packages.yml index 3185d94..3fea20a 100644 --- a/.github/workflows/publish-packages.yml +++ b/.github/workflows/publish-packages.yml @@ -106,6 +106,7 @@ jobs: NUGET_SIGN_CERTIFICATE_PASSWORD: ${{ secrets.NUGET_SIGN_CERTIFICATE_PASSWORD }} NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }} NUGET_SIGN_TIMESTAMP_URL: ${{ secrets.NUGET_SIGN_TIMESTAMP_URL }} + NUGET_SIGN_SKIP_VERIFICATION: ${{ secrets.NUGET_SIGN_SKIP_VERIFICATION }} publish-nuget: name: Publish to NuGet.org @@ -130,7 +131,7 @@ jobs: merge-multiple: true - name: Verify signed packages - if: ${{ needs.package.outputs.signing_enabled == 'true' }} + if: ${{ needs.package.outputs.signing_enabled == 'true' && needs.package.outputs.signature_verification_enabled == 'true' }} env: NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }} run: | @@ -180,7 +181,7 @@ jobs: merge-multiple: true - name: Verify signed packages - if: ${{ needs.package.outputs.signing_enabled == 'true' }} + if: ${{ needs.package.outputs.signing_enabled == 'true' && needs.package.outputs.signature_verification_enabled == 'true' }} env: NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }} run: | diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index c105fba..3ead79b 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -43,3 +43,4 @@ jobs: NUGET_SIGN_CERTIFICATE_PASSWORD: ${{ secrets.NUGET_SIGN_CERTIFICATE_PASSWORD }} NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT: ${{ secrets.NUGET_SIGN_CERTIFICATE_SHA256_FINGERPRINT }} NUGET_SIGN_TIMESTAMP_URL: ${{ secrets.NUGET_SIGN_TIMESTAMP_URL }} + NUGET_SIGN_SKIP_VERIFICATION: ${{ secrets.NUGET_SIGN_SKIP_VERIFICATION }} diff --git a/Benchmarks/Concurrency/BatchSchedulingBenchmarks.cs b/Benchmarks/Concurrency/BatchSchedulingBenchmarks.cs new file mode 100644 index 0000000..3b6199a --- /dev/null +++ b/Benchmarks/Concurrency/BatchSchedulingBenchmarks.cs @@ -0,0 +1,88 @@ +using BenchmarkDotNet.Attributes; +using Microsoft.Extensions.DependencyInjection; +using ModularityKit.Mutator.Abstractions.Audit; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Abstractions.History; +using ModularityKit.Mutator.Benchmarks.Concurrency.Support; +using ModularityKit.Mutator.Benchmarks.Diagnostics.Support; + +namespace ModularityKit.Mutator.Benchmarks.Concurrency; + +/// +/// Benchmarks concurrent batch executions competing for limited runtime availability. +/// +[BenchmarkCategory("Concurrency")] +[MemoryDiagnoser] +[InProcess] +public class BatchSchedulingBenchmarks +{ + private const int RuntimeSlots = 2; + + private IMutationEngine _engine = null!; + private BatchScenario[] _scenarios = null!; + + /// + /// Controls how many batch executions compete for runtime slots during a single benchmark iteration. + /// + [Params(2, 4)] + public int ConcurrentBatches { get; set; } + + /// + /// Controls how many mutations each competing batch executes. + /// + [Params(4, 16)] + public int BatchSize { get; set; } + + /// + /// Prepares the engine and precomputed batch scenarios for the selected parameters. + /// + [GlobalSetup] + public void Setup() + { + _engine = ConcurrencyBenchmarkScenario.BuildEngine( + RuntimeSlots, + services => + { + services.AddSingleton(); + services.AddSingleton(); + }); + + _scenarios = Enumerable + .Range(0, ConcurrentBatches) + .Select(CreateBatchScenario) + .ToArray(); + } + + /// + /// Measures scheduler pressure when several ordered batches compete for a limited number of engine slots. + /// + [Benchmark] + public async Task ConcurrentBatches_LimitedRuntimeAvailability() + { + var tasks = new Task[ConcurrentBatches]; + + for (var index = 0; index < ConcurrentBatches; index++) + { + var scenario = _scenarios[index]; + tasks[index] = _engine.ExecuteBatchAsync(scenario.Mutations, scenario.State); + } + + await Task.WhenAll(tasks); + GC.KeepAlive(tasks); + } + + private BatchScenario CreateBatchScenario(int batchIndex) + { + var stateId = $"batch-state-{batchIndex}"; + var mutations = Enumerable + .Range(0, BatchSize) + .Select(step => (IMutation)ConcurrencyBenchmarkScenario.CreateCommitMutation(stateId, $"batch-{batchIndex}-{step}")) + .ToArray(); + + return new BatchScenario(new ConcurrencyState(batchIndex, 0), mutations); + } + + private sealed record BatchScenario( + ConcurrencyState State, + IReadOnlyList> Mutations); +} diff --git a/Benchmarks/Concurrency/GateContentionBenchmarks.cs b/Benchmarks/Concurrency/GateContentionBenchmarks.cs new file mode 100644 index 0000000..4cccccb --- /dev/null +++ b/Benchmarks/Concurrency/GateContentionBenchmarks.cs @@ -0,0 +1,64 @@ +using BenchmarkDotNet.Attributes; +using Microsoft.Extensions.DependencyInjection; +using ModularityKit.Mutator.Abstractions.Audit; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Abstractions.History; +using ModularityKit.Mutator.Benchmarks.Concurrency.Support; +using ModularityKit.Mutator.Benchmarks.Diagnostics.Support; + +namespace ModularityKit.Mutator.Benchmarks.Concurrency; + +/// +/// Benchmarks state-level gate contention in the core mutation runtime. +/// +[BenchmarkCategory("Concurrency")] +[MemoryDiagnoser] +[InProcess] +public class GateContentionBenchmarks +{ + private const string SharedStateId = "shared-concurrency-state"; + + private IMutationEngine _engine = null!; + private ConcurrencyState _state = null!; + + /// + /// Prepares an engine with a concurrency limit high enough to isolate state-gate contention. + /// + [GlobalSetup] + public void Setup() + { + _engine = ConcurrencyBenchmarkScenario.BuildEngine( + maxConcurrentMutations: 4, + configureServices: services => + { + services.AddSingleton(); + services.AddSingleton(); + }); + + _state = new ConcurrencyState(42, 0); + } + + /// + /// Measures two concurrent executions targeting the same state identifier while one execution blocks the gate. + /// + [Benchmark] + public async Task SharedStateGate_TwoConcurrentExecutions() + { + using var gate = new BlockingMutationGate(); + var firstMutation = ConcurrencyBenchmarkScenario.CreateBlockingMutation(gate, SharedStateId, "first"); + var secondMutation = ConcurrencyBenchmarkScenario.CreateCommitMutation(SharedStateId, "second"); + + var firstTask = _engine.ExecuteAsync(firstMutation, _state); + + if (!gate.WaitForEntries(expectedEntries: 1, timeout: TimeSpan.FromSeconds(5))) + throw new InvalidOperationException("Blocking benchmark mutation did not enter the gate in time."); + + var secondTask = _engine.ExecuteAsync(secondMutation, _state); + + Thread.SpinWait(100_000); + gate.Release(); + + var results = await Task.WhenAll(firstTask, secondTask); + GC.KeepAlive(results); + } +} diff --git a/Benchmarks/Concurrency/ParallelExecutionBenchmarks.cs b/Benchmarks/Concurrency/ParallelExecutionBenchmarks.cs new file mode 100644 index 0000000..f9f5ecd --- /dev/null +++ b/Benchmarks/Concurrency/ParallelExecutionBenchmarks.cs @@ -0,0 +1,68 @@ +using BenchmarkDotNet.Attributes; +using Microsoft.Extensions.DependencyInjection; +using ModularityKit.Mutator.Abstractions.Audit; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Abstractions.History; +using ModularityKit.Mutator.Benchmarks.Concurrency.Support; +using ModularityKit.Mutator.Benchmarks.Diagnostics.Support; + +namespace ModularityKit.Mutator.Benchmarks.Concurrency; + +/// +/// Benchmarks parallel execution throughput across distinct runtime state identifiers. +/// +[BenchmarkCategory("Concurrency")] +[MemoryDiagnoser] +[InProcess] +public class ParallelExecutionBenchmarks +{ + private IMutationEngine _engine = null!; + private ConcurrencyState[] _states = null!; + private IncrementConcurrencyMutation[] _mutations = null!; + + /// + /// Controls how many distinct mutation executions run in parallel during a single benchmark iteration. + /// + [Params(2, 8)] + public int Parallelism { get; set; } + + /// + /// Prepares the engine, state snapshots, and mutation list for the selected parallelism level. + /// + [GlobalSetup] + public void Setup() + { + _engine = ConcurrencyBenchmarkScenario.BuildEngine( + Parallelism, + services => + { + services.AddSingleton(); + services.AddSingleton(); + }); + + _states = Enumerable + .Range(0, Parallelism) + .Select(index => new ConcurrencyState(index, 0)) + .ToArray(); + + _mutations = Enumerable + .Range(0, Parallelism) + .Select(index => ConcurrencyBenchmarkScenario.CreateCommitMutation($"parallel-state-{index}", $"parallel-{index}")) + .ToArray(); + } + + /// + /// Measures concurrent execution across distinct state identifiers without diagnostics storage noise. + /// + [Benchmark(Baseline = true)] + public async Task ParallelDistinctStates_ExecuteAsync() + { + var tasks = new Task[Parallelism]; + + for (var index = 0; index < Parallelism; index++) + tasks[index] = _engine.ExecuteAsync(_mutations[index], _states[index]); + + await Task.WhenAll(tasks); + GC.KeepAlive(tasks); + } +} diff --git a/Benchmarks/Concurrency/Support/BlockingGateMutation.cs b/Benchmarks/Concurrency/Support/BlockingGateMutation.cs new file mode 100644 index 0000000..6bc84d9 --- /dev/null +++ b/Benchmarks/Concurrency/Support/BlockingGateMutation.cs @@ -0,0 +1,68 @@ +using ModularityKit.Mutator.Abstractions; +using ModularityKit.Mutator.Abstractions.Changes; +using ModularityKit.Mutator.Abstractions.Context; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Abstractions.Intent; +using ModularityKit.Mutator.Abstractions.Results; + +namespace ModularityKit.Mutator.Benchmarks.Concurrency.Support; + +/// +/// Minimal commit mutation that blocks inside the execution pipeline to expose gate contention. +/// +internal sealed class BlockingGateMutation( + MutationContext context, + BlockingMutationGate gate) : IMutation +{ + /// + /// Gets the benchmark mutation intent metadata. + /// + public MutationIntent Intent { get; } = new() + { + OperationName = "BlockingGateMutation", + Category = "Benchmark", + Description = "Block inside mutation execution to expose core runtime gate contention.", + RiskLevel = MutationRiskLevel.Low, + IsReversible = true + }; + + /// + /// Gets the execution context bound to the benchmark mutation instance. + /// + public MutationContext Context { get; } = context; + + /// + /// Applies the benchmark mutation after waiting on the shared gate. + /// + /// The input state. + /// The successful mutation result containing the updated state and changes. + public MutationResult Apply(ConcurrencyState state) + { + gate.Enter(); + + var nextState = state with + { + Counter = state.Counter + 1, + Revision = state.Revision + 1 + }; + + return MutationResult.Success( + nextState, + ChangeSet.Single( + StateChange.Modified(nameof(ConcurrencyState.Counter), state.Counter, nextState.Counter))); + } + + /// + /// Validates the provided state before mutation execution. + /// + /// The input state. + /// A successful validation result. + public ValidationResult Validate(ConcurrencyState state) => ValidationResult.Success(); + + /// + /// Simulates the benchmark mutation using the same state transition as commit execution. + /// + /// The input state. + /// The simulated mutation result. + public MutationResult Simulate(ConcurrencyState state) => Apply(state); +} diff --git a/Benchmarks/Concurrency/Support/BlockingMutationGate.cs b/Benchmarks/Concurrency/Support/BlockingMutationGate.cs new file mode 100644 index 0000000..1194342 --- /dev/null +++ b/Benchmarks/Concurrency/Support/BlockingMutationGate.cs @@ -0,0 +1,43 @@ +namespace ModularityKit.Mutator.Benchmarks.Concurrency.Support; + +/// +/// Coordinates blocked benchmark mutations and tracks observed entry counts. +/// +internal sealed class BlockingMutationGate : IDisposable +{ + private readonly ManualResetEventSlim _release = new(false); + private int _entered; + + /// + /// Gets the number of mutations that have entered the blocking region. + /// + public int EnteredCount => Volatile.Read(ref _entered); + + /// + /// Waits until the expected number of mutations have entered the gate. + /// + /// Number of entries required before returning success. + /// Maximum time to wait. + /// when the expected number of entries arrived before timeout; otherwise . + public bool WaitForEntries(int expectedEntries, TimeSpan timeout) + => SpinWait.SpinUntil(() => Volatile.Read(ref _entered) >= expectedEntries, timeout); + + /// + /// Enters the blocking region and waits until released. + /// + public void Enter() + { + Interlocked.Increment(ref _entered); + _release.Wait(); + } + + /// + /// Releases all blocked benchmark mutations. + /// + public void Release() => _release.Set(); + + /// + /// Disposes the underlying synchronization primitive. + /// + public void Dispose() => _release.Dispose(); +} diff --git a/Benchmarks/Concurrency/Support/ConcurrencyBenchmarkScenario.cs b/Benchmarks/Concurrency/Support/ConcurrencyBenchmarkScenario.cs new file mode 100644 index 0000000..a3141f5 --- /dev/null +++ b/Benchmarks/Concurrency/Support/ConcurrencyBenchmarkScenario.cs @@ -0,0 +1,81 @@ +using Microsoft.Extensions.DependencyInjection; +using ModularityKit.Mutator.Abstractions; +using ModularityKit.Mutator.Abstractions.Context; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Runtime; + +namespace ModularityKit.Mutator.Benchmarks.Concurrency.Support; + +/// +/// Creates repeatable concurrency benchmark scenarios and engine instances. +/// +internal static class ConcurrencyBenchmarkScenario +{ + /// + /// Builds a performance-oriented mutation engine for concurrency benchmark scenarios. + /// + /// The engine-wide concurrency limit. + /// Optional service registrations overriding default runtime services. + /// A configured mutation engine instance. + public static IMutationEngine BuildEngine( + int maxConcurrentMutations, + Action? configureServices = null) + { + var services = new ServiceCollection(); + services.AddMutators(configure: options => + { + options.AlwaysValidate = MutationEngineOptions.Performance.AlwaysValidate; + options.EnableDetailedMetrics = MutationEngineOptions.Performance.EnableDetailedMetrics; + options.StopBatchOnFirstFailure = MutationEngineOptions.Performance.StopBatchOnFirstFailure; + options.MaxConcurrentMutations = maxConcurrentMutations; + }); + + configureServices?.Invoke(services); + + return services + .BuildServiceProvider() + .GetRequiredService(); + } + + /// + /// Creates a minimal commit mutation bound to the supplied benchmark state identifier. + /// + /// The state identifier used by the runtime gate. + /// The operation suffix used to build a stable correlation identifier. + /// A concurrency benchmark mutation configured for commit execution. + public static IncrementConcurrencyMutation CreateCommitMutation(string stateId, string operationSuffix) + { + var context = MutationContext.System("benchmark-concurrency") + with + { + StateId = stateId, + Mode = MutationMode.Commit, + CorrelationId = $"{stateId}:{operationSuffix}" + }; + + return new IncrementConcurrencyMutation(context); + } + + /// + /// Creates a blocking commit mutation bound to the supplied benchmark state identifier. + /// + /// The shared gate coordinating blocked execution. + /// The state identifier used by the runtime gate. + /// The operation suffix used to build a stable correlation identifier. + /// A blocking concurrency benchmark mutation configured for commit execution. + public static BlockingGateMutation CreateBlockingMutation( + BlockingMutationGate gate, + string stateId, + string operationSuffix) + { + var context = MutationContext.System("benchmark-concurrency") + with + { + StateId = stateId, + Mode = MutationMode.Commit, + CorrelationId = $"{stateId}:{operationSuffix}" + }; + + return new BlockingGateMutation(context, gate); + } +} diff --git a/Benchmarks/Concurrency/Support/ConcurrencyState.cs b/Benchmarks/Concurrency/Support/ConcurrencyState.cs new file mode 100644 index 0000000..b532f8e --- /dev/null +++ b/Benchmarks/Concurrency/Support/ConcurrencyState.cs @@ -0,0 +1,8 @@ +namespace ModularityKit.Mutator.Benchmarks.Concurrency.Support; + +/// +/// Minimal state used by concurrency benchmark scenarios. +/// +/// The mutable numeric field exercised by the benchmark mutation. +/// The revision counter advanced on each benchmark mutation. +public sealed record ConcurrencyState(int Counter, int Revision); diff --git a/Benchmarks/Concurrency/Support/IncrementConcurrencyMutation.cs b/Benchmarks/Concurrency/Support/IncrementConcurrencyMutation.cs new file mode 100644 index 0000000..64f34e3 --- /dev/null +++ b/Benchmarks/Concurrency/Support/IncrementConcurrencyMutation.cs @@ -0,0 +1,66 @@ +using ModularityKit.Mutator.Abstractions; +using ModularityKit.Mutator.Abstractions.Changes; +using ModularityKit.Mutator.Abstractions.Context; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Abstractions.Intent; +using ModularityKit.Mutator.Abstractions.Results; + +namespace ModularityKit.Mutator.Benchmarks.Concurrency.Support; + +/// +/// Minimal commit mutation used to measure core runtime concurrency overhead. +/// +internal sealed class IncrementConcurrencyMutation(MutationContext context) : IMutation +{ + /// + /// Gets the benchmark mutation intent metadata. + /// + public MutationIntent Intent { get; } = new() + { + OperationName = "IncrementConcurrencyState", + Category = "Benchmark", + Description = "Increment benchmark state to measure core runtime concurrency overhead.", + RiskLevel = MutationRiskLevel.Low, + IsReversible = true + }; + + /// + /// Gets the execution context bound to the benchmark mutation instance. + /// + public MutationContext Context { get; } = context; + + /// + /// Applies the benchmark mutation to the provided state. + /// + /// The input state. + /// The successful mutation result containing the updated state and changes. + public MutationResult Apply(ConcurrencyState state) + { + var nextState = state with + { + Counter = state.Counter + 1, + Revision = state.Revision + 1 + }; + + return MutationResult.Success( + nextState, + ChangeSet.FromChanges( + StateChange.Modified(nameof(ConcurrencyState.Counter), state.Counter, nextState.Counter), + StateChange.Modified(nameof(ConcurrencyState.Revision), state.Revision, nextState.Revision) + )); + } + + /// + /// Validates the provided state before mutation execution. + /// + /// The input state. + /// A successful validation result. + public ValidationResult Validate(ConcurrencyState state) => ValidationResult.Success(); + + /// + /// Simulates the benchmark mutation using the same state transition as commit execution. + /// + /// The input state. + /// The simulated mutation result. + public MutationResult Simulate(ConcurrencyState state) => Apply(state); +} diff --git a/Benchmarks/Diagnostics/DiagnosticsOverheadBenchmarks.cs b/Benchmarks/Diagnostics/DiagnosticsOverheadBenchmarks.cs new file mode 100644 index 0000000..9b532e4 --- /dev/null +++ b/Benchmarks/Diagnostics/DiagnosticsOverheadBenchmarks.cs @@ -0,0 +1,79 @@ +using BenchmarkDotNet.Attributes; +using Microsoft.Extensions.DependencyInjection; +using ModularityKit.Mutator.Abstractions.Audit; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Abstractions.History; +using ModularityKit.Mutator.Benchmarks.Diagnostics.Support; + +namespace ModularityKit.Mutator.Benchmarks.Diagnostics; + +/// +/// Benchmarks audit, history, and logging-style overhead in the core mutation pipeline. +/// +[BenchmarkCategory("Diagnostics")] +[MemoryDiagnoser] +[InProcess] +public class DiagnosticsOverheadBenchmarks +{ + private IMutationEngine _noDiagnosticsEngine = null!; + private IMutationEngine _auditHistoryEngine = null!; + private IMutationEngine _combinedDiagnosticsEngine = null!; + private DiagnosticsState _state = null!; + private DiagnosticsMutation _mutation = null!; + + /// + /// Prepares baseline, audit-history, and combined observability benchmark engines. + /// + [GlobalSetup] + public void Setup() + { + _noDiagnosticsEngine = DiagnosticsBenchmarkScenario.BuildEngine( + services => + { + services.AddSingleton(); + services.AddSingleton(); + }); + + _auditHistoryEngine = DiagnosticsBenchmarkScenario.BuildEngine(); + + _combinedDiagnosticsEngine = DiagnosticsBenchmarkScenario.BuildEngine( + configureEngine: engine => + { + engine.RegisterInterceptor(new PassiveBenchmarkInterceptor()); + engine.RegisterInterceptor(new FormattingLoggingInterceptor()); + }); + + _state = new DiagnosticsState(42, "baseline"); + _mutation = DiagnosticsBenchmarkScenario.CreateCommitMutation("diagnostics"); + } + + /// + /// Measures commit execution with observability paths disabled via no-op audit and history services. + /// + [Benchmark(Baseline = true)] + public async Task NoDiagnostics_Baseline() + { + var result = await _noDiagnosticsEngine.ExecuteAsync(_mutation, _state); + GC.KeepAlive(result); + } + + /// + /// Measures commit execution with the default audit and history capture path enabled. + /// + [Benchmark] + public async Task AuditHistory_Enabled() + { + var result = await _auditHistoryEngine.ExecuteAsync(_mutation, _state); + GC.KeepAlive(result); + } + + /// + /// Measures commit execution with audit/history capture plus interceptor and logging-style formatting enabled. + /// + [Benchmark] + public async Task CombinedInterceptionAndDiagnostics_Enabled() + { + var result = await _combinedDiagnosticsEngine.ExecuteAsync(_mutation, _state); + GC.KeepAlive(result); + } +} diff --git a/Benchmarks/Diagnostics/InterceptorOverheadBenchmarks.cs b/Benchmarks/Diagnostics/InterceptorOverheadBenchmarks.cs new file mode 100644 index 0000000..eebf52a --- /dev/null +++ b/Benchmarks/Diagnostics/InterceptorOverheadBenchmarks.cs @@ -0,0 +1,67 @@ +using BenchmarkDotNet.Attributes; +using Microsoft.Extensions.DependencyInjection; +using ModularityKit.Mutator.Abstractions.Audit; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Abstractions.History; +using ModularityKit.Mutator.Benchmarks.Diagnostics.Support; + +namespace ModularityKit.Mutator.Benchmarks.Diagnostics; + +/// +/// Benchmarks interceptor overhead in the core mutation pipeline independently from audit and history storage. +/// +[BenchmarkCategory("Diagnostics")] +[MemoryDiagnoser] +[InProcess] +public class InterceptorOverheadBenchmarks +{ + private IMutationEngine _baselineEngine = null!; + private IMutationEngine _interceptorEngine = null!; + private DiagnosticsState _state = null!; + private DiagnosticsMutation _mutation = null!; + + /// + /// Prepares engines with and without a passive interceptor while disabling audit and history storage noise. + /// + [GlobalSetup] + public void Setup() + { + _baselineEngine = DiagnosticsBenchmarkScenario.BuildEngine( + services => + { + services.AddSingleton(); + services.AddSingleton(); + }); + + _interceptorEngine = DiagnosticsBenchmarkScenario.BuildEngine( + services => + { + services.AddSingleton(); + services.AddSingleton(); + }, + engine => engine.RegisterInterceptor(new PassiveBenchmarkInterceptor())); + + _state = new DiagnosticsState(42, "baseline"); + _mutation = DiagnosticsBenchmarkScenario.CreateCommitMutation("interceptor"); + } + + /// + /// Measures the commit pipeline without interceptors, audit persistence, or history persistence. + /// + [Benchmark(Baseline = true)] + public async Task NoInterceptor_Baseline() + { + var result = await _baselineEngine.ExecuteAsync(_mutation, _state); + GC.KeepAlive(result); + } + + /// + /// Measures the same commit pipeline with one passive interceptor enabled. + /// + [Benchmark] + public async Task PassiveInterceptor_Enabled() + { + var result = await _interceptorEngine.ExecuteAsync(_mutation, _state); + GC.KeepAlive(result); + } +} diff --git a/Benchmarks/Diagnostics/Support/DiagnosticsBenchmarkScenario.cs b/Benchmarks/Diagnostics/Support/DiagnosticsBenchmarkScenario.cs new file mode 100644 index 0000000..f522f6d --- /dev/null +++ b/Benchmarks/Diagnostics/Support/DiagnosticsBenchmarkScenario.cs @@ -0,0 +1,58 @@ +using Microsoft.Extensions.DependencyInjection; +using ModularityKit.Mutator.Abstractions; +using ModularityKit.Mutator.Abstractions.Context; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Runtime; + +namespace ModularityKit.Mutator.Benchmarks.Diagnostics.Support; + +/// +/// Creates repeatable diagnostics benchmark scenarios and engine instances. +/// +internal static class DiagnosticsBenchmarkScenario +{ + /// + /// Gets the shared state identifier used by diagnostics benchmark cases. + /// + public const string StateId = "diagnostics-benchmark-state"; + + /// + /// Builds performance oriented mutation engine for diagnostics benchmark scenarios. + /// + /// Optional service registrations overriding default runtime services. + /// Optional engine configuration executed after engine resolution. + /// A configured mutation engine instance. + public static IMutationEngine BuildEngine( + Action? configureServices = null, + Action? configureEngine = null) + { + var services = new ServiceCollection(); + services.AddMutators(MutationEngineOptions.Performance); + + configureServices?.Invoke(services); + + var engine = services + .BuildServiceProvider() + .GetRequiredService(); + + configureEngine?.Invoke(engine); + return engine; + } + + /// + /// Creates a minimal commit mutation instance bound to the shared diagnostics benchmark state. + /// + /// The operation suffix used to build a stable correlation identifier. + /// A diagnostics mutation configured for commit execution. + public static DiagnosticsMutation CreateCommitMutation(string operationSuffix) + { + var context = MutationContext.System("diagnostics-benchmark") with + { + StateId = StateId, + Mode = MutationMode.Commit, + CorrelationId = $"{StateId}:{operationSuffix}" + }; + + return new DiagnosticsMutation(context); + } +} diff --git a/Benchmarks/Diagnostics/Support/DiagnosticsMutation.cs b/Benchmarks/Diagnostics/Support/DiagnosticsMutation.cs new file mode 100644 index 0000000..fec5cc4 --- /dev/null +++ b/Benchmarks/Diagnostics/Support/DiagnosticsMutation.cs @@ -0,0 +1,66 @@ +using ModularityKit.Mutator.Abstractions; +using ModularityKit.Mutator.Abstractions.Changes; +using ModularityKit.Mutator.Abstractions.Context; +using ModularityKit.Mutator.Abstractions.Engine; +using ModularityKit.Mutator.Abstractions.Intent; +using ModularityKit.Mutator.Abstractions.Results; + +namespace ModularityKit.Mutator.Benchmarks.Diagnostics.Support; + +/// +/// Minimal commit mutation used to measure diagnostics and interception overhead. +/// +internal sealed class DiagnosticsMutation(MutationContext context) : IMutation +{ + /// + /// Gets the benchmark mutation intent metadata. + /// + public MutationIntent Intent { get; } = new() + { + OperationName = "DiagnosticsMutation", + Category = "Benchmark", + Description = "Minimal commit mutation used to measure interception and diagnostics overhead.", + RiskLevel = MutationRiskLevel.Low, + IsReversible = true + }; + + /// + /// Gets the execution context bound to the benchmark mutation instance. + /// + public MutationContext Context { get; } = context; + + /// + /// Applies the benchmark mutation to the provided state. + /// + /// The input state. + /// The successful mutation result containing the updated state and changes. + public MutationResult Apply(DiagnosticsState state) + { + var nextState = state with + { + Counter = state.Counter + 1, + LastOperation = Context.CorrelationId ?? string.Empty + }; + + return MutationResult.Success( + nextState, + ChangeSet.FromChanges( + StateChange.Modified(nameof(DiagnosticsState.Counter), state.Counter, nextState.Counter), + StateChange.Modified(nameof(DiagnosticsState.LastOperation), state.LastOperation, nextState.LastOperation) + )); + } + + /// + /// Validates the provided state before mutation execution. + /// + /// The input state. + /// A successful validation result. + public ValidationResult Validate(DiagnosticsState state) => ValidationResult.Success(); + + /// + /// Simulates the benchmark mutation using the same state transition as commit execution. + /// + /// The input state. + /// The simulated mutation result. + public MutationResult Simulate(DiagnosticsState state) => Apply(state); +} diff --git a/Benchmarks/Diagnostics/Support/DiagnosticsState.cs b/Benchmarks/Diagnostics/Support/DiagnosticsState.cs new file mode 100644 index 0000000..ae053b6 --- /dev/null +++ b/Benchmarks/Diagnostics/Support/DiagnosticsState.cs @@ -0,0 +1,8 @@ +namespace ModularityKit.Mutator.Benchmarks.Diagnostics.Support; + +/// +/// Minimal state used by diagnostics benchmark scenarios. +/// +/// The mutable numeric field exercised by the benchmark mutation. +/// The last logical operation label written by the mutation. +public sealed record DiagnosticsState(int Counter, string LastOperation); diff --git a/Benchmarks/Diagnostics/Support/FormattingLoggingInterceptor.cs b/Benchmarks/Diagnostics/Support/FormattingLoggingInterceptor.cs new file mode 100644 index 0000000..9c8fb9d --- /dev/null +++ b/Benchmarks/Diagnostics/Support/FormattingLoggingInterceptor.cs @@ -0,0 +1,67 @@ +using ModularityKit.Mutator.Abstractions.Changes; +using ModularityKit.Mutator.Abstractions.Context; +using ModularityKit.Mutator.Abstractions.Intent; +using ModularityKit.Mutator.Runtime.Interception; + +namespace ModularityKit.Mutator.Benchmarks.Diagnostics.Support; + +/// +/// Logging style interceptor that formats lifecycle messages and writes them to a sink. +/// +internal sealed class FormattingLoggingInterceptor : MutationInterceptorBase +{ + private readonly TextWriter _sink = TextWriter.Null; + + /// + /// Gets the interceptor name used in benchmark scenarios. + /// + public override string Name => nameof(FormattingLoggingInterceptor); + + /// + /// Formats and writes pre mutation log line to the configured sink. + /// + /// The mutation intent. + /// The mutation context. + /// The current state. + /// The execution identifier. + /// The cancellation token. + /// A completed task. + public override Task OnBeforeMutationAsync( + MutationIntent intent, + MutationContext context, + object state, + string executionId, + CancellationToken cancellationToken = default) + { + _sink.WriteLine($"[Before] {intent.OperationName} by {context.ActorId} (ExecutionId: {executionId})"); + return Task.CompletedTask; + } + + /// + /// Formats and writes post mutation log lines, including the generated change set, to the configured sink. + /// + /// The mutation intent. + /// The mutation context. + /// The state before mutation execution. + /// The state after mutation execution. + /// The generated change set. + /// The execution identifier. + /// The cancellation token. + /// A completed task. + public override Task OnAfterMutationAsync( + MutationIntent intent, + MutationContext context, + object? oldState, + object? newState, + ChangeSet changes, + string executionId, + CancellationToken cancellationToken = default) + { + _sink.WriteLine($"[After] {intent.OperationName}, changes: {changes.Changes.Count} (ExecutionId: {executionId})"); + + foreach (var change in changes.Changes) + _sink.WriteLine($" - {change.Path}: {change.OldValue} -> {change.NewValue}"); + + return Task.CompletedTask; + } +} diff --git a/Benchmarks/Diagnostics/Support/NoOpAuditor.cs b/Benchmarks/Diagnostics/Support/NoOpAuditor.cs new file mode 100644 index 0000000..596034c --- /dev/null +++ b/Benchmarks/Diagnostics/Support/NoOpAuditor.cs @@ -0,0 +1,60 @@ +using ModularityKit.Mutator.Abstractions.Audit; + +namespace ModularityKit.Mutator.Benchmarks.Diagnostics.Support; + +/// +/// No-op auditor used to remove audit persistence noise from selected benchmark cases. +/// +internal sealed class NoOpAuditor : IMutationAuditor +{ + /// + /// Ignores the supplied audit entry. + /// + /// The audit entry. + /// A completed task. + public Task AuditAsync(MutationAuditEntry entry) => Task.CompletedTask; + + /// + /// Ignores the supplied audit entry. + /// + /// The audit entry. + /// The cancellation token. + /// A completed task. + public Task AuditAsync(MutationAuditEntry entry, CancellationToken cancellationToken) => Task.CompletedTask; + + /// + /// Returns an empty audit log for the requested state. + /// + /// The state identifier. + /// An empty audit log. + public Task> GetAuditLogAsync(string stateId) + => Task.FromResult>([]); + + /// + /// Returns an empty audit log for the requested state and time range. + /// + /// The state identifier. + /// The lower bound of the time range. + /// The upper bound of the time range. + /// An empty audit log. + public Task> GetAuditLogAsync( + string stateId, + DateTimeOffset? from, + DateTimeOffset? to) + => Task.FromResult>([]); + + /// + /// Returns an empty audit log for the requested state and time range. + /// + /// The state identifier. + /// The lower bound of the time range. + /// The upper bound of the time range. + /// The cancellation token. + /// An empty audit log. + public Task> GetAuditLogAsync( + string stateId, + DateTimeOffset? from, + DateTimeOffset? to, + CancellationToken cancellationToken) + => Task.FromResult>([]); +} diff --git a/Benchmarks/Diagnostics/Support/NoOpHistoryStore.cs b/Benchmarks/Diagnostics/Support/NoOpHistoryStore.cs new file mode 100644 index 0000000..887fcb4 --- /dev/null +++ b/Benchmarks/Diagnostics/Support/NoOpHistoryStore.cs @@ -0,0 +1,54 @@ +using ModularityKit.Mutator.Abstractions.History; + +namespace ModularityKit.Mutator.Benchmarks.Diagnostics.Support; + +/// +/// Noop history store used to remove history persistence noise from selected benchmark cases. +/// +internal sealed class NoOpHistoryStore : IMutationHistoryStore +{ + /// + /// Ignores the supplied history entry. + /// + /// The history entry. + /// The cancellation token. + /// A completed task. + public Task StoreAsync(MutationHistoryEntry entry, CancellationToken cancellationToken = default) => Task.CompletedTask; + + /// + /// Returns an empty mutation history for the requested state. + /// + /// The state identifier. + /// The cancellation token. + /// An empty mutation history. + public Task GetHistoryAsync(string stateId, CancellationToken cancellationToken = default) + => Task.FromResult(new MutationHistory { StateId = stateId, Entries = [] }); + + /// + /// Returns an empty mutation history for the requested state and time range. + /// + /// The state identifier. + /// The lower bound of the time range. + /// The upper bound of the time range. + /// The cancellation token. + /// An empty mutation history. + public Task GetHistoryRangeAsync( + string stateId, + DateTimeOffset from, + DateTimeOffset to, + CancellationToken cancellationToken = default) + => Task.FromResult(new MutationHistory { StateId = stateId, Entries = [] }); + + /// + /// Returns an empty list of recent mutation history entries. + /// + /// The state identifier. + /// The requested number of entries. + /// The cancellation token. + /// An empty list of mutation history entries. + public Task> GetRecentAsync( + string stateId, + int count, + CancellationToken cancellationToken = default) + => Task.FromResult>([]); +} diff --git a/Benchmarks/Diagnostics/Support/PassiveBenchmarkInterceptor.cs b/Benchmarks/Diagnostics/Support/PassiveBenchmarkInterceptor.cs new file mode 100644 index 0000000..1e13506 --- /dev/null +++ b/Benchmarks/Diagnostics/Support/PassiveBenchmarkInterceptor.cs @@ -0,0 +1,55 @@ +using ModularityKit.Mutator.Abstractions.Changes; +using ModularityKit.Mutator.Abstractions.Context; +using ModularityKit.Mutator.Abstractions.Intent; +using ModularityKit.Mutator.Runtime.Interception; + +namespace ModularityKit.Mutator.Benchmarks.Diagnostics.Support; + +/// +/// Minimal interceptor that participates in the full mutation lifecycle without side effects. +/// +internal sealed class PassiveBenchmarkInterceptor : MutationInterceptorBase +{ + /// + /// Gets the interceptor name used in benchmark scenarios. + /// + public override string Name => nameof(PassiveBenchmarkInterceptor); + + /// + /// Handles the pre mutation hook without introducing observable side effects. + /// + /// The mutation intent. + /// The mutation context. + /// The current state. + /// The execution identifier. + /// The cancellation token. + /// A completed task. + public override Task OnBeforeMutationAsync( + MutationIntent intent, + MutationContext context, + object state, + string executionId, + CancellationToken cancellationToken = default) + => Task.CompletedTask; + + /// + /// Handles the post mutation hook without introducing observable side effects. + /// + /// The mutation intent. + /// The mutation context. + /// The state before mutation execution. + /// The state after mutation execution. + /// The generated change set. + /// The execution identifier. + /// The cancellation token. + /// A completed task. + public override Task OnAfterMutationAsync( + MutationIntent intent, + MutationContext context, + object? oldState, + object? newState, + ChangeSet changes, + string executionId, + CancellationToken cancellationToken = default) + => Task.CompletedTask; +} diff --git a/Benchmarks/README.md b/Benchmarks/README.md index 2f079b2..d98df69 100644 --- a/Benchmarks/README.md +++ b/Benchmarks/README.md @@ -10,9 +10,11 @@ This folder contains BenchmarkDotNet measurements for `ModularityKit.Mutator`. - batch execution overhead - throughput for single mutation execution across multiple state sizes - throughput for batch mutation execution across multiple state sizes and batch sizes -- policy evaluation overhead for no-policy, synchronous policy, asynchronous policy, and mixed multi-policy runs +- policy evaluation overhead for no policy, synchronous policy, asynchronous policy, and mixed multi policy runs +- interception, audit/history, and logging diagnostics overhead in the core runtime +- parallel execution, state gate contention, and concurrent batch scheduling pressure in the core runtime -The throughput benchmarks use a cloned array-backed state so state-size effects remain visible in the +The throughput benchmarks use cloned array backed state so state size effects remain visible in the actual mutation path rather than being hidden behind an artificial inner loop. ## Run @@ -42,6 +44,18 @@ dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --fil dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --filter '*PolicyEvaluationMultiBenchmarks*' ``` +Run the diagnostics suite: + +```bash +dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --anyCategories Diagnostics +``` + +Run the concurrency suite: + +```bash +dotnet Benchmarks/bin/Release/net10.0/ModularityKit.Mutator.Benchmarks.dll --anyCategories Concurrency +``` + Key parameters reported by BenchmarkDotNet: - `StateSize` controls the size of the cloned mutation state