diff --git a/.gitignore b/.gitignore index 4d65f48..dbf90a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,20 @@ -.idea/ +# DocFX generated HTML site +_site/ + +# DocFX metadata output +obj/api/ + +# .NET build artifacts bin/ obj/ -_site/ -__pycache__/ -*.pyc +*.dll +*.exe +*.pdb +*.deps.json +*.runtimeconfig.json + +# Dumps +*.dmp +*.dump +*.mdmp +*.core diff --git a/ModularityKit.Mutator.slnx b/ModularityKit.Mutator.slnx index 7e1be38..58d53f4 100644 --- a/ModularityKit.Mutator.slnx +++ b/ModularityKit.Mutator.slnx @@ -18,6 +18,7 @@ + diff --git a/Tests/.gitignore b/Tests/.gitignore new file mode 100644 index 0000000..4d65f48 --- /dev/null +++ b/Tests/.gitignore @@ -0,0 +1,6 @@ +.idea/ +bin/ +obj/ +_site/ +__pycache__/ +*.pyc diff --git a/Tests/ModularityKit.Mutator.Examples.SmokeTests/AssemblyInfo.cs b/Tests/ModularityKit.Mutator.Examples.SmokeTests/AssemblyInfo.cs new file mode 100644 index 0000000..2171200 --- /dev/null +++ b/Tests/ModularityKit.Mutator.Examples.SmokeTests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/Tests/ModularityKit.Mutator.Examples.SmokeTests/Examples/Core/CoreExamplesSmokeTests.cs b/Tests/ModularityKit.Mutator.Examples.SmokeTests/Examples/Core/CoreExamplesSmokeTests.cs new file mode 100644 index 0000000..bd4990b --- /dev/null +++ b/Tests/ModularityKit.Mutator.Examples.SmokeTests/Examples/Core/CoreExamplesSmokeTests.cs @@ -0,0 +1,49 @@ +using ModularityKit.Mutator.Examples.SmokeTests.Support; + +namespace ModularityKit.Mutator.Examples.SmokeTests.Examples.Core; + +/// +/// Smoke coverage for the executable samples shipped under Examples/Core. +/// +public sealed class CoreExamplesSmokeTests +{ + [Fact] + public Task BillingQuotas_runs_successfully() + => ExampleSmokeRunner.RunAndAssertAsync(Create("BillingQuotas", "Examples/Core/BillingQuotas/BillingQuotas.csproj")); + + [Fact] + public Task FeatureFlags_runs_successfully() + => ExampleSmokeRunner.RunAndAssertAsync(Create("FeatureFlags", "Examples/Core/FeatureFlags/FeatureFlags.csproj")); + + [Fact] + public Task IamRoles_runs_successfully() + => ExampleSmokeRunner.RunAndAssertAsync(Create("IamRoles", "Examples/Core/IamRoles/IamRoles.csproj")); + + [Fact] + public Task WorkflowApprovals_runs_successfully() + => ExampleSmokeRunner.RunAndAssertAsync(Create("WorkflowApprovals", "Examples/Core/WorkflowApprovals/WorkflowApprovals.csproj")); + + private static ExampleSmokeCase Create(string name, string projectPath) + => new( + name, + projectPath, + result => + { + if (result.TimedOut) + return "process timed out"; + + if (result.ExitCode != 0) + return $"expected exit code 0 but got {result.ExitCode}"; + + if (string.IsNullOrWhiteSpace(result.StandardOutput)) + return "example did not produce any stdout"; + + if (!result.StandardOutput.Contains("METRICS & STATISTICS", StringComparison.Ordinal)) + return "expected metrics section in stdout"; + + if (result.StandardError.Contains("Unhandled exception", StringComparison.OrdinalIgnoreCase)) + return "stderr contains unhandled exception output"; + + return null; + }); +} diff --git a/Tests/ModularityKit.Mutator.Examples.SmokeTests/Examples/Governance/GovernanceExamplesSmokeTests.cs b/Tests/ModularityKit.Mutator.Examples.SmokeTests/Examples/Governance/GovernanceExamplesSmokeTests.cs new file mode 100644 index 0000000..0834223 --- /dev/null +++ b/Tests/ModularityKit.Mutator.Examples.SmokeTests/Examples/Governance/GovernanceExamplesSmokeTests.cs @@ -0,0 +1,90 @@ +using ModularityKit.Mutator.Examples.SmokeTests.Support; + +namespace ModularityKit.Mutator.Examples.SmokeTests.Examples.Governance; + +/// +/// Smoke coverage for the executable samples shipped under Examples/Governance. +/// +public sealed class GovernanceExamplesSmokeTests +{ + [Fact] + public Task ApprovalWorkflow_runs_successfully() + => ExampleSmokeRunner.RunAndAssertAsync(Create("ApprovalWorkflow", "Examples/Governance/ApprovalWorkflow/ApprovalWorkflow.csproj")); + + [Fact] + public Task GovernedExecution_runs_successfully() + => ExampleSmokeRunner.RunAndAssertAsync(Create("GovernedExecution", "Examples/Governance/GovernedExecution/GovernedExecution.csproj")); + + [Fact] + public Task DecisionTaxonomy_runs_successfully() + => ExampleSmokeRunner.RunAndAssertAsync(Create("DecisionTaxonomy", "Examples/Governance/DecisionTaxonomy/DecisionTaxonomy.csproj")); + + [Fact] + public Task Queries_runs_successfully() + => ExampleSmokeRunner.RunAndAssertAsync(Create("Queries", "Examples/Governance/Queries/Queries.csproj")); + + [Fact] + public Task RedisQueries_runs_successfully() + => ExampleSmokeRunner.RunAndAssertAsync(CreateRedis()); + + [Fact] + public Task RequestLifecycle_runs_successfully() + => ExampleSmokeRunner.RunAndAssertAsync(Create("RequestLifecycle", "Examples/Governance/RequestLifecycle/RequestLifecycle.csproj")); + + [Fact] + public Task VersionedResolution_runs_successfully() + => ExampleSmokeRunner.RunAndAssertAsync(Create("VersionedResolution", "Examples/Governance/VersionedResolution/VersionedResolution.csproj")); + + private static ExampleSmokeCase Create(string name, string projectPath) + => new( + name, + projectPath, + result => + { + if (result.TimedOut) + return "process timed out"; + + if (result.ExitCode != 0) + return $"expected exit code 0 but got {result.ExitCode}"; + + if (string.IsNullOrWhiteSpace(result.StandardOutput) && string.IsNullOrWhiteSpace(result.StandardError)) + return "example did not produce any output"; + + if (result.StandardError.Contains("Unhandled exception", StringComparison.OrdinalIgnoreCase)) + return "stderr contains unhandled exception output"; + + return null; + }); + + private static ExampleSmokeCase CreateRedis() + => new( + "RedisQueries", + "Examples/Governance/RedisQueries/RedisQueries.csproj", + result => + { + if (result.TimedOut) + return "process timed out"; + + if (result.ExitCode != 0) + return $"expected exit code 0 but got {result.ExitCode}"; + + var hasRedisOutput = result.StandardOutput.Contains("Pending Approval Queue", StringComparison.Ordinal) + && result.StandardOutput.Contains("Recent Execution Outcomes", StringComparison.Ordinal); + + var hasExpectedDependencyWarning = + result.StandardError.Contains("Could not connect to Redis", StringComparison.Ordinal) + && result.StandardError.Contains("Start Redis locally or set MODULARITYKIT_REDIS to a reachable endpoint.", StringComparison.Ordinal); + + if (!hasRedisOutput && !hasExpectedDependencyWarning) + return "expected either Redis query output or the documented Redis prerequisite warning"; + + if (result.StandardError.Contains("Unhandled exception", StringComparison.OrdinalIgnoreCase)) + return "stderr contains unhandled exception output"; + + return null; + }, + new Dictionary + { + ["MODULARITYKIT_REDIS"] = "localhost:6379,connectTimeout=1000,abortConnect=false,syncTimeout=1000" + }); +} diff --git a/Tests/ModularityKit.Mutator.Examples.SmokeTests/GlobalUsings.cs b/Tests/ModularityKit.Mutator.Examples.SmokeTests/GlobalUsings.cs new file mode 100644 index 0000000..c802f44 --- /dev/null +++ b/Tests/ModularityKit.Mutator.Examples.SmokeTests/GlobalUsings.cs @@ -0,0 +1 @@ +global using Xunit; diff --git a/Tests/ModularityKit.Mutator.Examples.SmokeTests/ModularityKit.Mutator.Examples.SmokeTests.csproj b/Tests/ModularityKit.Mutator.Examples.SmokeTests/ModularityKit.Mutator.Examples.SmokeTests.csproj new file mode 100644 index 0000000..0bba76e --- /dev/null +++ b/Tests/ModularityKit.Mutator.Examples.SmokeTests/ModularityKit.Mutator.Examples.SmokeTests.csproj @@ -0,0 +1,21 @@ + + + + net10.0 + enable + enable + false + true + $(NoWarn);1591 + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + diff --git a/Tests/ModularityKit.Mutator.Examples.SmokeTests/README.md b/Tests/ModularityKit.Mutator.Examples.SmokeTests/README.md new file mode 100644 index 0000000..1edfc81 --- /dev/null +++ b/Tests/ModularityKit.Mutator.Examples.SmokeTests/README.md @@ -0,0 +1,62 @@ +# Example Smoke Tests + +This project provides lightweight smoke coverage for the executable samples under: + +- `Examples/Core` +- `Examples/Governance` + +The goal is to catch sample drift against the current public API and runtime behavior without turning the examples into a second full test suite. + +## Structure + +- `Examples/Core` contains smoke tests for core runtime samples. +- `Examples/Governance` contains smoke tests for governance and Redis-backed samples. +- `Support` contains the shared runner and result models used by the smoke layer. + +## How it works + +Each smoke test: + +1. builds the target example project in `Release` +2. executes the built assembly with `dotnet exec` +3. captures `stdout` and `stderr` +4. validates a small set of expected signals for that sample + +The runner keeps the checks focused on sample viability: + +- non-zero exit codes fail the test +- hung processes are terminated by timeout +- failures include captured output for fast diagnosis + +## Redis example behavior + +`RedisQueries` accepts either: + +- normal Redis-backed query output, or +- the documented "could not connect to Redis" message + +That keeps the sample smoke testable even when Redis is not running locally. + +## Run + +Build: + +```bash +dotnet build Tests/ModularityKit.Mutator.Examples.SmokeTests/ModularityKit.Mutator.Examples.SmokeTests.csproj -c Debug +``` + +Run: + +```bash +dotnet test Tests/ModularityKit.Mutator.Examples.SmokeTests/ModularityKit.Mutator.Examples.SmokeTests.csproj -c Debug --no-build +``` + +Run a single smoke test: + +```bash +dotnet test Tests/ModularityKit.Mutator.Examples.SmokeTests/ModularityKit.Mutator.Examples.SmokeTests.csproj -c Debug --no-build --filter FullyQualifiedName=ModularityKit.Mutator.Examples.SmokeTests.GovernanceExamplesSmokeTests.RedisQueries_runs_successfully +``` + +## Environment note + +`vstest` uses a local socket. In restricted sandboxes that block local bind operations, the test host may need to run outside the sandbox even though the examples themselves are local processes. diff --git a/Tests/ModularityKit.Mutator.Examples.SmokeTests/Support/ExampleRunResult.cs b/Tests/ModularityKit.Mutator.Examples.SmokeTests/Support/ExampleRunResult.cs new file mode 100644 index 0000000..ecad59e --- /dev/null +++ b/Tests/ModularityKit.Mutator.Examples.SmokeTests/Support/ExampleRunResult.cs @@ -0,0 +1,10 @@ +namespace ModularityKit.Mutator.Examples.SmokeTests.Support; + +/// +/// Captures the observable outcome of running an example process. +/// +public sealed record ExampleRunResult( + int ExitCode, + string StandardOutput, + string StandardError, + bool TimedOut); diff --git a/Tests/ModularityKit.Mutator.Examples.SmokeTests/Support/ExampleSmokeCase.cs b/Tests/ModularityKit.Mutator.Examples.SmokeTests/Support/ExampleSmokeCase.cs new file mode 100644 index 0000000..2ee0f03 --- /dev/null +++ b/Tests/ModularityKit.Mutator.Examples.SmokeTests/Support/ExampleSmokeCase.cs @@ -0,0 +1,13 @@ +namespace ModularityKit.Mutator.Examples.SmokeTests.Support; + +/// +/// Describes a single executable example covered by the smoke test suite. +/// +public sealed record ExampleSmokeCase( + string Name, + string ProjectPath, + Func Validate, + IReadOnlyDictionary? EnvironmentVariables = null) +{ + public override string ToString() => Name; +} diff --git a/Tests/ModularityKit.Mutator.Examples.SmokeTests/Support/ExampleSmokeRunner.cs b/Tests/ModularityKit.Mutator.Examples.SmokeTests/Support/ExampleSmokeRunner.cs new file mode 100644 index 0000000..e8f7998 --- /dev/null +++ b/Tests/ModularityKit.Mutator.Examples.SmokeTests/Support/ExampleSmokeRunner.cs @@ -0,0 +1,215 @@ +using System.Diagnostics; +using System.Text; + +namespace ModularityKit.Mutator.Examples.SmokeTests.Support; + +internal static class ExampleSmokeRunner +{ + private static readonly TimeSpan Timeout = TimeSpan.FromSeconds(45); + private const string TargetFramework = "net10.0"; + private static readonly Lock BuildSync = new(); + private static readonly HashSet BuiltProjects = []; + + public static async Task RunAndAssertAsync(ExampleSmokeCase example, CancellationToken cancellationToken = default) + { + await EnsureBuiltAsync(example.ProjectPath, cancellationToken).ConfigureAwait(false); + var result = await RunAsync(example, cancellationToken).ConfigureAwait(false); + var validationError = example.Validate(result); + + Assert.True( + validationError is null, + $"{example.Name} smoke check failed: {validationError}{Environment.NewLine}{FormatResult(result)}"); + } + + private static async Task RunAsync(ExampleSmokeCase example, CancellationToken cancellationToken) + { + var repositoryRoot = ResolveRepositoryRoot(); + var projectDirectory = Path.GetDirectoryName(Path.Combine(repositoryRoot, example.ProjectPath)) + ?? throw new DirectoryNotFoundException($"Could not resolve project directory for '{example.ProjectPath}'."); + var assemblyName = Path.GetFileNameWithoutExtension(example.ProjectPath); + var assemblyPath = Path.Combine(projectDirectory, "bin", "Release", TargetFramework, $"{assemblyName}.dll"); + + return await RunProcessAsync( + repositoryRoot, + arguments => + { + arguments.Add("exec"); + arguments.Add(assemblyPath); + }, + example.EnvironmentVariables, + cancellationToken).ConfigureAwait(false); + } + + private static async Task EnsureBuiltAsync(string projectPath, CancellationToken cancellationToken) + { + lock (BuildSync) + { + if (BuiltProjects.Contains(projectPath)) + return; + } + + var repositoryRoot = ResolveRepositoryRoot(); + var buildResult = await RunProcessAsync( + repositoryRoot, + arguments => + { + arguments.Add("build"); + arguments.Add(projectPath); + arguments.Add("--configuration"); + arguments.Add("Release"); + }, + environmentVariables: null, + cancellationToken).ConfigureAwait(false); + + if (buildResult.ExitCode != 0 || buildResult.TimedOut) + { + Assert.Fail($"{projectPath} build failed before smoke execution.{Environment.NewLine}{FormatResult(buildResult)}"); + } + + lock (BuildSync) + { + BuiltProjects.Add(projectPath); + } + } + + private static string ResolveRepositoryRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + + while (directory is not null) + { + if (File.Exists(Path.Combine(directory.FullName, "ModularityKit.Mutator.slnx"))) + return directory.FullName; + + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not locate repository root for smoke tests."); + } + + private static void TryKill(Process process) + { + try + { + if (!process.HasExited) + process.Kill(entireProcessTree: true); + } + catch + { + } + } + + private static string FormatResult(ExampleRunResult result) + { + var builder = new StringBuilder(); + builder.AppendLine($"Timed out: {result.TimedOut}"); + builder.AppendLine($"Exit code: {result.ExitCode}"); + builder.AppendLine("stdout:"); + builder.AppendLine(string.IsNullOrWhiteSpace(result.StandardOutput) ? "" : result.StandardOutput.Trim()); + builder.AppendLine("stderr:"); + builder.AppendLine(string.IsNullOrWhiteSpace(result.StandardError) ? "" : result.StandardError.Trim()); + return builder.ToString(); + } + + private static async Task RunProcessAsync( + string workingDirectory, + Action> configureArguments, + IReadOnlyDictionary? environmentVariables, + CancellationToken cancellationToken) + { + var startInfo = new ProcessStartInfo("dotnet") + { + WorkingDirectory = workingDirectory, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + var arguments = new List(); + configureArguments(arguments); + + foreach (var argument in arguments) + startInfo.ArgumentList.Add(argument); + + if (environmentVariables is not null) + { + foreach (var environmentVariable in environmentVariables) + { + if (environmentVariable.Value is null) + startInfo.Environment.Remove(environmentVariable.Key); + else + startInfo.Environment[environmentVariable.Key] = environmentVariable.Value; + } + } + + using var process = new Process + { + StartInfo = startInfo, + EnableRaisingEvents = true + }; + + var stdout = new StringBuilder(); + var stderr = new StringBuilder(); + var stdoutCompleted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var stderrCompleted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var exited = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + process.OutputDataReceived += (_, args) => + { + if (args.Data is null) + { + stdoutCompleted.TrySetResult(); + return; + } + + stdout.AppendLine(args.Data); + }; + + process.ErrorDataReceived += (_, args) => + { + if (args.Data is null) + { + stderrCompleted.TrySetResult(); + return; + } + + stderr.AppendLine(args.Data); + }; + + process.Exited += (_, _) => exited.TrySetResult(); + + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var allCompleted = exited.Task; + var timeoutTask = Task.Delay(Timeout, timeoutCts.Token); + var completedTask = await Task.WhenAny(allCompleted, timeoutTask).ConfigureAwait(false); + + if (completedTask != allCompleted) + { + TryKill(process); + await AwaitSafely(exited.Task).ConfigureAwait(false); + return new ExampleRunResult(-1, stdout.ToString(), stderr.ToString(), TimedOut: true); + } + + timeoutCts.Cancel(); + + var drainTask = Task.WhenAll(stdoutCompleted.Task, stderrCompleted.Task); + await Task.WhenAny(drainTask, Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken)).ConfigureAwait(false); + + return new ExampleRunResult(process.ExitCode, stdout.ToString(), stderr.ToString(), TimedOut: false); + } + + private static async Task AwaitSafely(Task task) + { + try + { + await task.ConfigureAwait(false); + } + catch + { + } + } +}