diff --git a/ModularPipelines.All.sln b/ModularPipelines.All.sln index 8d6715b4211..d6cd982cc43 100644 --- a/ModularPipelines.All.sln +++ b/ModularPipelines.All.sln @@ -251,6 +251,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModularPipelines.Yq.UnitTes EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModularPipelines.Templates", "src\ModularPipelines.Templates\ModularPipelines.Templates.csproj", "{89576148-F56C-463D-BCEB-6C3C9E75550B}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModularPipelines.Testing", "src\ModularPipelines.Testing\ModularPipelines.Testing.csproj", "{0150323C-1B27-42F8-918D-D646F0B1ED77}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModularPipelines.Testing.UnitTests", "test\ModularPipelines.Testing.UnitTests\ModularPipelines.Testing.UnitTests.csproj", "{A4899287-D537-4F8C-98EC-C3E9CB7F069E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1665,6 +1669,30 @@ Global {89576148-F56C-463D-BCEB-6C3C9E75550B}.Release|x64.Build.0 = Release|Any CPU {89576148-F56C-463D-BCEB-6C3C9E75550B}.Release|x86.ActiveCfg = Release|Any CPU {89576148-F56C-463D-BCEB-6C3C9E75550B}.Release|x86.Build.0 = Release|Any CPU + {0150323C-1B27-42F8-918D-D646F0B1ED77}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0150323C-1B27-42F8-918D-D646F0B1ED77}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0150323C-1B27-42F8-918D-D646F0B1ED77}.Debug|x64.ActiveCfg = Debug|Any CPU + {0150323C-1B27-42F8-918D-D646F0B1ED77}.Debug|x64.Build.0 = Debug|Any CPU + {0150323C-1B27-42F8-918D-D646F0B1ED77}.Debug|x86.ActiveCfg = Debug|Any CPU + {0150323C-1B27-42F8-918D-D646F0B1ED77}.Debug|x86.Build.0 = Debug|Any CPU + {0150323C-1B27-42F8-918D-D646F0B1ED77}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0150323C-1B27-42F8-918D-D646F0B1ED77}.Release|Any CPU.Build.0 = Release|Any CPU + {0150323C-1B27-42F8-918D-D646F0B1ED77}.Release|x64.ActiveCfg = Release|Any CPU + {0150323C-1B27-42F8-918D-D646F0B1ED77}.Release|x64.Build.0 = Release|Any CPU + {0150323C-1B27-42F8-918D-D646F0B1ED77}.Release|x86.ActiveCfg = Release|Any CPU + {0150323C-1B27-42F8-918D-D646F0B1ED77}.Release|x86.Build.0 = Release|Any CPU + {A4899287-D537-4F8C-98EC-C3E9CB7F069E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A4899287-D537-4F8C-98EC-C3E9CB7F069E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A4899287-D537-4F8C-98EC-C3E9CB7F069E}.Debug|x64.ActiveCfg = Debug|Any CPU + {A4899287-D537-4F8C-98EC-C3E9CB7F069E}.Debug|x64.Build.0 = Debug|Any CPU + {A4899287-D537-4F8C-98EC-C3E9CB7F069E}.Debug|x86.ActiveCfg = Debug|Any CPU + {A4899287-D537-4F8C-98EC-C3E9CB7F069E}.Debug|x86.Build.0 = Debug|Any CPU + {A4899287-D537-4F8C-98EC-C3E9CB7F069E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A4899287-D537-4F8C-98EC-C3E9CB7F069E}.Release|Any CPU.Build.0 = Release|Any CPU + {A4899287-D537-4F8C-98EC-C3E9CB7F069E}.Release|x64.ActiveCfg = Release|Any CPU + {A4899287-D537-4F8C-98EC-C3E9CB7F069E}.Release|x64.Build.0 = Release|Any CPU + {A4899287-D537-4F8C-98EC-C3E9CB7F069E}.Release|x86.ActiveCfg = Release|Any CPU + {A4899287-D537-4F8C-98EC-C3E9CB7F069E}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1788,6 +1816,8 @@ Global {6D367646-6CD9-47F8-8C74-5124FE36F61A} = {F213898F-1E32-48F1-AB8C-83D2BD01A93B} {40294462-4153-4DC3-9C6D-4E997BF83309} = {F213898F-1E32-48F1-AB8C-83D2BD01A93B} {89576148-F56C-463D-BCEB-6C3C9E75550B} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {0150323C-1B27-42F8-918D-D646F0B1ED77} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {A4899287-D537-4F8C-98EC-C3E9CB7F069E} = {F213898F-1E32-48F1-AB8C-83D2BD01A93B} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A5905A5D-B4E1-4A7A-9279-0283D86A9F7F} diff --git a/README.md b/README.md index d61a5560245..cf5819a7e0d 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,7 @@ ModularPipelines has strongly-typed wrappers for the tools you already use: | ModularPipelines.SonarScanner | Helpers for interacting with SonarScanner CLI for SonarQube and SonarCloud. | [![nuget](https://img.shields.io/nuget/v/ModularPipelines.SonarScanner.svg)](https://www.nuget.org/packages/ModularPipelines.SonarScanner/) | | ModularPipelines.TeamCity | Helpers for interacting with TeamCity build agents. | [![nuget](https://img.shields.io/nuget/v/ModularPipelines.TeamCity.svg)](https://www.nuget.org/packages/ModularPipelines.TeamCity/) | | ModularPipelines.Templates | Templates for creating realistic ModularPipelines build, test, and publish pipelines. | [![nuget](https://img.shields.io/nuget/v/ModularPipelines.Templates.svg)](https://www.nuget.org/packages/ModularPipelines.Templates/) | +| ModularPipelines.Testing | Supported test harness for isolated ModularPipelines module tests. | [![nuget](https://img.shields.io/nuget/v/ModularPipelines.Testing.svg)](https://www.nuget.org/packages/ModularPipelines.Testing/) | | ModularPipelines.Terraform | Helpers for interacting with Terraform CLI. | [![nuget](https://img.shields.io/nuget/v/ModularPipelines.Terraform.svg)](https://www.nuget.org/packages/ModularPipelines.Terraform/) | | ModularPipelines.Trivy | Helpers for interacting with Trivy security scanner CLI. | [![nuget](https://img.shields.io/nuget/v/ModularPipelines.Trivy.svg)](https://www.nuget.org/packages/ModularPipelines.Trivy/) | | ModularPipelines.WinGet | Helpers for interacting with the Windows Package Manager. | [![nuget](https://img.shields.io/nuget/v/ModularPipelines.WinGet.svg)](https://www.nuget.org/packages/ModularPipelines.WinGet/) | diff --git a/docs/docs/how-to/testing.md b/docs/docs/how-to/testing.md index e3abf064faa..2a7edb3d8ea 100644 --- a/docs/docs/how-to/testing.md +++ b/docs/docs/how-to/testing.md @@ -3,101 +3,151 @@ title: Testing sidebar_position: 9 --- -## Testing with Mocked File System +Install `ModularPipelines.Testing` to execute one module without starting the full +pipeline scheduler: -ModularPipelines supports mocking file system operations for unit testing. All file I/O goes through `IFileSystemProvider`, which can be replaced with a mock implementation. +```bash +dotnet add package ModularPipelines.Testing +``` + +The test harness uses the normal module execution pipeline, so skip conditions, +timeouts, retries, and direct module hooks behave as they do in a pipeline. It +provides test-safe defaults: -### Why Mock the File System? +- external commands are intercepted and return a successful result; +- file and directory operations use an isolated in-memory filesystem; +- progress, logos, dependency chains, and result printing are disabled; +- module failures are returned for assertions instead of escaping from the harness. -- **Speed**: Tests run faster without actual disk I/O -- **Isolation**: Tests don't depend on file system state -- **Predictability**: No flaky tests due to file permissions or disk space -- **CI-friendly**: Works in any environment without file system setup +## Execute a module -### Example: Mocking File Reads +Specify the module and result types for strongly typed value access: ```csharp -using Moq; -using ModularPipelines; -using ModularPipelines.Enums; -using ModularPipelines.FileSystem; -using ModularPipelines.Extensions; +using ModularPipelines.Testing; [Test] -public async Task MyModule_ReadsConfigFile() +public async Task Build_returns_the_artifact() { - // Create a mock provider - var mockProvider = new Mock(); - mockProvider.Setup(p => p.ReadAllTextAsync( - It.IsAny(), - It.IsAny())) - .ReturnsAsync("{\"setting\": \"value\"}"); + var run = await ModuleTester.For() + .ExecuteAsync(); - // Run pipeline with mock - using var builder = Pipeline.CreateBuilder(args); + await Assert.That(run.Value!.Name).IsEqualTo("application.zip"); + await Assert.That(run.Exception).IsNull(); +} +``` - builder.ConfigurePipelineOptions(options => options with - { - ThrowOnPipelineFailure = false, - }); +If only the module type is convenient, use the type-erased overload. `Value` is +then `object?`, while `Result` still contains the full module metadata: - builder.Services.AddSingleton(mockProvider.Object); - builder.AddModule(); +```csharp +var run = await ModuleTester.For().ExecuteAsync(); - var result = await builder.ExecutePipelineAsync(); +var artifact = (BuildArtifact)run.Value!; +``` - // Assert results - Assert.That(result.Status, Is.EqualTo(Status.Successful)); -} +## Seed dependency results + +Register a dependency result without executing that dependency: + +```csharp +var restoredPackages = CommandResult.Ok("Restore succeeded."); + +var run = await ModuleTester.For() + .WithDependencyResult(restoredPackages) + .ExecuteAsync(); ``` -### Example: Verifying File Writes +The dependency module is registered normally, then its successful result is +completed before the target module starts. Calls such as +`await context.GetModule()` therefore receive the seeded value. +If a required dependency has no seeded result, `ExecuteAsync` fails immediately +and names the missing dependency instead of waiting for the module timeout. + +## Intercept and inspect commands + +Commands never start real processes unless you explicitly replace the test +harness behavior. The default interceptor returns `CommandResult.Ok()`. + +Provide a handler when a module needs command output: ```csharp -[Test] -public async Task MyModule_WritesOutputFile() -{ - var mockProvider = new Mock(); +var run = await ModuleTester.For() + .InterceptCommands(invocation => + { + if (invocation.CommandLine.Tool == "dotnet") + { + return CommandResult.Ok("Build succeeded."); + } + + return CommandResult.Ok(); + }) + .ExecuteAsync(); + +await Assert.That(run.Commands).Count().IsEqualTo(1); +await Assert.That(run.Commands[0].CommandLine.Arguments) + .IsEquivalentTo(["build", "--configuration", "Release"]); +``` - using var builder = Pipeline.CreateBuilder(args); +Each `RecordedCommand` contains the parsed `CommandInvocation` and the simulated +`CommandResult`. This avoids assertions against a quoted display string. +Intercepted nonzero exit codes follow `CommandExecutionOptions` normally and +throw `CommandException` when `ThrowOnNonZeroExitCode` is enabled. - builder.Services.AddSingleton(mockProvider.Object); - builder.AddModule(); +`ICommandInterceptor` is also a public framework seam. Register an implementation +in a normal pipeline when command interception is needed outside +`ModularPipelines.Testing`. Return `null` to let the next interceptor or the real +process executor handle the command. - await builder.ExecutePipelineAsync(); +## Use the in-memory filesystem - // Verify the write occurred with expected content - mockProvider.Verify(p => p.WriteAllTextAsync( - It.Is(path => path.Contains("output")), - It.Is(content => content.Contains("result")), - It.IsAny())); -} +Files obtained through `context.Files` automatically use the harness filesystem: + +```csharp +var run = await ModuleTester.For() + .ExecuteAsync(); + +var manifest = await run.FileSystem.ReadAllTextAsync("/output/manifest.json"); ``` -### Important Notes +`InMemoryFileSystemProvider` implements `IFileSystemProvider`, including file and +directory creation, reads, writes, streams, copies, moves, deletion, enumeration, +and path helpers. You can also construct and register it directly in other tests. +Physical metadata such as attributes, timestamps, and file length is not part of +`IFileSystemProvider`; accessing it through an in-memory-backed `File` or `Folder` +throws `NotSupportedException` rather than reading the real filesystem. -- **Always use `context.Files`**: Files created via `context.Files.GetFile()` will use the injected provider. Files created directly via `new File("path")` use the real file system. +Code under test must obtain `File` and `Folder` instances from `context.Files`. +Direct construction such as `new File("path")` intentionally uses the physical +`SystemFileSystemProvider`. -- **Provider Registration**: The mock provider must be registered before the pipeline runs. Using `services.AddSingleton()` overrides the default `SystemFileSystemProvider`. +## Register constructor services -- **Mock ALL methods your code uses**: The mock provider only intercepts methods you explicitly set up. If your module calls `ReadAllTextAsync`, `FileExists`, and `Combine`, you must mock all three. Unmocked methods may throw or return default values depending on your mocking framework. +Use `WithService` for module constructor dependencies: -- **Implicit operators bypass mocking**: Implicit conversions like `File file = "/path/to/file"` create instances using the default `SystemFileSystemProvider`, not your mock. For full testability, always use `context.Files.GetFile()`. +```csharp +var settings = new BuildSettings { Configuration = "Release" }; + +var run = await ModuleTester.For() + .WithService(settings) + .ExecuteAsync(); +``` + +## Assert skipped and failed runs -- **Static methods are not mockable**: Methods like `File.GetNewTemporaryFilePath()` and `Folder.CreateTemporaryFolder()` use the real file system. Design your modules to receive paths via constructor or use `context.Files.CreateTemporaryFolder()` instead. +The harness configures `ThrowOnPipelineFailure = false`, so failed modules are +returned for assertions. Successful runs expose `Status.Successful`. -- **Mocking Path Operations**: If your code uses path operations, mock them too: - ```csharp - mockProvider.Setup(p => p.Combine(It.IsAny())) - .Returns((string[] paths) => Path.Combine(paths)); - ``` +The run object exposes safe outcome properties: -### What Gets Mocked +```csharp +var skipped = await ModuleTester.For().ExecuteAsync(); +await Assert.That(skipped.SkipDecision!.Reason).IsEqualTo("Feature disabled"); + +var failed = await ModuleTester.For().ExecuteAsync(); +await Assert.That(failed.Exception).IsTypeOf(); +await Assert.That(failed.Result).IsTypeOf.Failure>(); +``` -The `IFileSystemProvider` interface covers: -- File reads: `ReadAllTextAsync`, `ReadLinesAsync`, `ReadAllBytesAsync` -- File writes: `WriteAllTextAsync`, `WriteAllBytesAsync`, `WriteAllLinesAsync`, `AppendAllTextAsync` -- File management: `DeleteFile`, `CopyFile`, `MoveFile`, `FileExists` -- Directory operations: `CreateDirectory`, `DeleteDirectory`, `MoveDirectory`, `DirectoryExists` -- Enumeration: `EnumerateFiles`, `EnumerateDirectories` -- Path utilities: `GetTempPath`, `GetRandomFileName`, `Combine`, `GetRelativePath` +Use `Result` when assertions need timing, status, or the discriminated result +variant. Use `Value`, `Exception`, and `SkipDecision` for concise safe access. diff --git a/src/ModularPipelines.Analyzers/ModularPipelines.Analyzers.Test/ModularPipelinesAnalyzersUnitTests.cs b/src/ModularPipelines.Analyzers/ModularPipelines.Analyzers.Test/ModularPipelinesAnalyzersUnitTests.cs index 2067dc54073..0a8951a4c76 100644 --- a/src/ModularPipelines.Analyzers/ModularPipelines.Analyzers.Test/ModularPipelinesAnalyzersUnitTests.cs +++ b/src/ModularPipelines.Analyzers/ModularPipelines.Analyzers.Test/ModularPipelinesAnalyzersUnitTests.cs @@ -292,9 +292,11 @@ public async Task UseAsync(IModuleContext context) using ModularPipelines.Attributes; namespace Example; + public class Dependency : Module { - protected override Task ExecuteAsync(IModuleContext context, CancellationToken cancellationToken) => Task.FromResult(null); + protected override Task ExecuteAsync(IModuleContext context, CancellationToken cancellationToken) + => Task.FromResult(null); } [DependsOn] @@ -308,7 +310,8 @@ public async Task UseAsync(IModuleContext context) } } - protected override Task ExecuteAsync(IModuleContext context, CancellationToken cancellationToken) => Task.FromResult(null); + protected override Task ExecuteAsync(IModuleContext context, CancellationToken cancellationToken) + => Task.FromResult(null); } """.ReplaceLineEndings("\n"); var expected = VerifyCS.Diagnostic(MissingDependsOnAttributeAnalyzer.DiagnosticId) diff --git a/src/ModularPipelines.Build/Modules/FindProjectsModule.cs b/src/ModularPipelines.Build/Modules/FindProjectsModule.cs index 48e540e6eef..185c87a978f 100644 --- a/src/ModularPipelines.Build/Modules/FindProjectsModule.cs +++ b/src/ModularPipelines.Build/Modules/FindProjectsModule.cs @@ -51,6 +51,7 @@ protected override ModuleConfiguration Configure() => ModuleConfiguration.Create Sourcy.DotNet.Projects.ModularPipelines_SonarScanner, Sourcy.DotNet.Projects.ModularPipelines_TeamCity, Sourcy.DotNet.Projects.ModularPipelines_Templates, + Sourcy.DotNet.Projects.ModularPipelines_Testing, Sourcy.DotNet.Projects.ModularPipelines_Terraform, Sourcy.DotNet.Projects.ModularPipelines_Trivy, Sourcy.DotNet.Projects.ModularPipelines_WinGet, diff --git a/src/ModularPipelines.Testing/InMemoryFileSystemProvider.cs b/src/ModularPipelines.Testing/InMemoryFileSystemProvider.cs new file mode 100644 index 00000000000..e720904b492 --- /dev/null +++ b/src/ModularPipelines.Testing/InMemoryFileSystemProvider.cs @@ -0,0 +1,979 @@ +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; +using System.Text; +using ModularPipelines.FileSystem; + +namespace ModularPipelines.Testing; + +/// +/// Thread-safe in-memory implementation of . +/// +public sealed class InMemoryFileSystemProvider : IFileSystemProvider +{ + private readonly ConcurrentDictionary _files; + private readonly ConcurrentDictionary _directories; + private readonly HashSet _exclusiveOpenFiles; + private readonly Dictionary _openReaders; + private readonly Lock _sync = new(); + private readonly StringComparer _pathComparer; + private readonly StringComparison _pathComparison; + + /// + /// Initializes an empty in-memory filesystem. + /// + public InMemoryFileSystemProvider() + { + _pathComparer = OperatingSystem.IsWindows() + ? StringComparer.OrdinalIgnoreCase + : StringComparer.Ordinal; + _pathComparison = OperatingSystem.IsWindows() + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal; + _files = new ConcurrentDictionary(_pathComparer); + _directories = new ConcurrentDictionary(_pathComparer); +#pragma warning disable IDE0028 // Collection expressions cannot retain the platform path comparer. + _exclusiveOpenFiles = new HashSet(_pathComparer); + _openReaders = new Dictionary(_pathComparer); +#pragma warning restore IDE0028 + CreateDirectory(Environment.CurrentDirectory); + CreateDirectory(GetTempPath()); + } + + /// + public Task ReadAllTextAsync( + string path, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(DecodeText(GetFile(path))); + } + + /// + public async IAsyncEnumerable ReadLinesAsync( + string path, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + using var stream = OpenRead(path); + using var reader = new StreamReader( + stream, + Encoding.UTF8, + detectEncodingFromByteOrderMarks: true); + while (await reader.ReadLineAsync(cancellationToken).ConfigureAwait(false) is { } line) + { + yield return line; + } + } + + /// + public Task ReadAllBytesAsync( + string path, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult(GetFile(path).ToArray()); + } + + /// + public Task WriteAllTextAsync( + string path, + string contents, + CancellationToken cancellationToken = default) => + WriteAllBytesAsync(path, Encoding.UTF8.GetBytes(contents), cancellationToken); + + /// + public Task WriteAllBytesAsync( + string path, + byte[] contents, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + SetFile(path, contents); + return Task.CompletedTask; + } + + /// + public async Task WriteAllLinesAsync( + string path, + IEnumerable contents, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + await using var stream = Open(path, FileMode.Create, FileAccess.Write); + await WriteLinesAsync(stream, contents, cancellationToken).ConfigureAwait(false); + } + + /// + public Task AppendAllTextAsync( + string path, + string contents, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + lock (_sync) + { + var normalized = NormalizeFilePath(path); + var existing = _files.TryGetValue(normalized, out var bytes) ? bytes : []; + var appended = new byte[existing.Length + Encoding.UTF8.GetByteCount(contents)]; + existing.CopyTo(appended, 0); + Encoding.UTF8.GetBytes(contents, appended.AsSpan(existing.Length)); + SetFile(normalized, appended); + } + + return Task.CompletedTask; + } + + /// + public async Task AppendAllLinesAsync( + string path, + IEnumerable contents, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + await using var stream = Open(path, FileMode.Append, FileAccess.Write); + await WriteLinesAsync(stream, contents, cancellationToken).ConfigureAwait(false); + } + + /// + public Stream OpenRead(string path) + { + lock (_sync) + { + var normalized = NormalizeFilePath(path); + var initial = GetFile(normalized); + if (_exclusiveOpenFiles.Contains(normalized)) + { + throw new IOException( + $"The in-memory file '{normalized}' is already open."); + } + + _openReaders.TryGetValue(normalized, out var readerCount); + _openReaders[normalized] = readerCount + 1; + try + { + return CreateStream( + normalized, + FileMode.Open, + FileAccess.Read, + initial, + () => ReleaseOpenReader(normalized)); + } + catch + { + ReleaseOpenReader(normalized); + throw; + } + } + } + + /// + public Stream Create(string path) => Open(path, FileMode.Create, FileAccess.ReadWrite); + + /// + public Stream Open(string path, FileMode mode, FileAccess access) + { + lock (_sync) + { + var normalized = NormalizeFilePath(path); + var exists = _files.TryGetValue(normalized, out var existing); + + ValidateOpenArguments(mode, access, exists, normalized); + if (IsOpen(normalized)) + { + throw new IOException( + $"The in-memory file '{normalized}' is already open."); + } + + var initial = GetInitialContents(mode, existing); + if (ShouldInitializeFile(mode, exists)) + { + SetFile(normalized, initial); + } + + _exclusiveOpenFiles.Add(normalized); + try + { + return CreateStream( + normalized, + mode, + access, + initial, + () => ReleaseExclusiveOpenFile(normalized)); + } + catch + { + _exclusiveOpenFiles.Remove(normalized); + throw; + } + } + } + + private static void ValidateOpenArguments( + FileMode mode, + FileAccess access, + bool exists, + string normalizedPath) + { + if (mode == FileMode.Append && access != FileAccess.Write) + { + throw new ArgumentException("Append mode requires write-only access.", nameof(access)); + } + + if (mode is FileMode.Create or FileMode.CreateNew or FileMode.Truncate + && access == FileAccess.Read) + { + throw new ArgumentException($"{mode} mode requires write access.", nameof(access)); + } + + if (mode is FileMode.Open or FileMode.Truncate && !exists) + { + throw new FileNotFoundException("The in-memory file does not exist.", normalizedPath); + } + + if (mode == FileMode.CreateNew && exists) + { + throw new IOException($"The in-memory file '{normalizedPath}' already exists."); + } + } + + private static byte[] GetInitialContents(FileMode mode, byte[]? existing) => + mode is FileMode.Create or FileMode.CreateNew or FileMode.Truncate + ? [] + : existing ?? []; + + private static bool ShouldInitializeFile(FileMode mode, bool exists) => + mode is FileMode.Create or FileMode.CreateNew or FileMode.Truncate + || (!exists && mode is (FileMode.OpenOrCreate or FileMode.Append)); + + private static string DecodeText(byte[] contents) + { + using var stream = new MemoryStream(contents, writable: false); + using var reader = new StreamReader( + stream, + Encoding.UTF8, + detectEncodingFromByteOrderMarks: true); + return reader.ReadToEnd(); + } + + private CommittingMemoryStream CreateStream( + string normalizedPath, + FileMode mode, + FileAccess access, + byte[] initial, + Action release) + { + var stream = new CommittingMemoryStream( + initial, + access, + mode == FileMode.Append, + bytes => SetFile(normalizedPath, bytes, allowOpenFile: true), + release); + + if (mode == FileMode.Append) + { + stream.Position = stream.Length; + } + + return stream; + } + + private void ReleaseExclusiveOpenFile(string normalizedPath) + { + lock (_sync) + { + _exclusiveOpenFiles.Remove(normalizedPath); + } + } + + private void ReleaseOpenReader(string normalizedPath) + { + lock (_sync) + { + var readerCount = _openReaders[normalizedPath]; + if (readerCount == 1) + { + _openReaders.Remove(normalizedPath); + } + else + { + _openReaders[normalizedPath] = readerCount - 1; + } + } + } + + /// + public void DeleteFile(string path) + { + lock (_sync) + { + var normalized = NormalizeFilePath(path); + var directoryForm = NormalizeDirectoryPath(normalized); + if (_directories.ContainsKey(directoryForm)) + { + throw new UnauthorizedAccessException( + $"The in-memory path '{directoryForm}' is a directory."); + } + + if (IsOpen(normalized)) + { + throw new IOException( + $"The in-memory file '{normalized}' is already open."); + } + + _files.TryRemove(normalized, out _); + } + } + + /// + public void CopyFile(string sourcePath, string destinationPath, bool overwrite) + { + lock (_sync) + { + var source = NormalizeFilePath(sourcePath); + var destination = NormalizeFilePath(destinationPath); + if (_pathComparer.Equals(source, destination)) + { + throw new IOException( + $"The source and destination paths are the same: '{source}'."); + } + + if (!overwrite && _files.ContainsKey(destination)) + { + throw new IOException($"The in-memory file '{destination}' already exists."); + } + + SetFile(destination, GetFile(source)); + } + } + + /// + public void MoveFile(string sourcePath, string destinationPath) + { + lock (_sync) + { + var source = NormalizeFilePath(sourcePath); + var destination = NormalizeFilePath(destinationPath); + if (IsOpen(source)) + { + throw new IOException( + $"The in-memory file '{source}' is already open."); + } + + if (_files.ContainsKey(destination)) + { + throw new IOException($"The in-memory file '{destination}' already exists."); + } + + ValidateFileDestination(destination); + + if (!_files.TryRemove(source, out var contents)) + { + throw new FileNotFoundException("The in-memory file does not exist.", source); + } + + SetFile(destination, contents); + } + } + + /// + public bool FileExists(string path) + { + try + { + return !string.IsNullOrWhiteSpace(path) + && _files.ContainsKey(NormalizeFilePath(path)); + } + catch (Exception exception) when (IsPathProbeException(exception)) + { + return false; + } + } + + /// + public void CreateDirectory(string path) + { + lock (_sync) + { + var current = NormalizeDirectoryPath(path); + var directoriesToCreate = new List(); + while (!string.IsNullOrEmpty(current)) + { + if (_files.ContainsKey(current)) + { + throw new IOException( + $"The in-memory path '{current}' is already a file."); + } + + directoriesToCreate.Add(current); + var parent = Path.GetDirectoryName(current); + if (string.IsNullOrEmpty(parent) || _pathComparer.Equals(parent, current)) + { + break; + } + + current = parent; + } + + foreach (var directory in directoriesToCreate) + { + _directories.TryAdd(directory, 0); + } + } + } + + /// + public void DeleteDirectory(string path, bool recursive) + { + lock (_sync) + { + var normalized = NormalizeDirectoryPath(path); + if (!_directories.ContainsKey(normalized)) + { + throw new DirectoryNotFoundException(normalized); + } + + var descendants = GetDescendantDirectories(normalized).ToArray(); + var files = GetDescendantFiles(normalized).ToArray(); + if (!recursive && (descendants.Length > 0 || files.Length > 0)) + { + throw new IOException($"The in-memory directory '{normalized}' is not empty."); + } + + ValidateFilesAreClosed(files); + + foreach (var file in files) + { + _files.TryRemove(file, out _); + } + + foreach (var directory in descendants.Append(normalized)) + { + _directories.TryRemove(directory, out _); + } + } + } + + /// + public void MoveDirectory(string sourcePath, string destinationPath) + { + lock (_sync) + { + var source = NormalizeDirectoryPath(sourcePath); + var destination = NormalizeDirectoryPath(destinationPath); + if (!_directories.ContainsKey(source)) + { + throw new DirectoryNotFoundException(source); + } + + if (_directories.ContainsKey(destination)) + { + throw new IOException($"The in-memory directory '{destination}' already exists."); + } + + if (_files.ContainsKey(destination)) + { + throw new IOException($"The in-memory path '{destination}' is already a file."); + } + + if (IsDescendant(destination, source)) + { + throw new IOException("A directory cannot be moved inside itself."); + } + + var destinationParent = Path.GetDirectoryName(destination); + if (string.IsNullOrEmpty(destinationParent) + || !_directories.ContainsKey(destinationParent)) + { + throw new DirectoryNotFoundException(destinationParent); + } + + var descendants = GetDescendantDirectories(source).ToArray(); + var files = GetDescendantFiles(source).ToArray(); + ValidateFilesAreClosed(files); + + CreateDirectory(destination); + foreach (var directory in descendants) + { + CreateDirectory(ReplacePrefix(directory, source, destination)); + } + + foreach (var file in files) + { + MoveFile(file, ReplacePrefix(file, source, destination)); + } + + foreach (var directory in descendants.Append(source)) + { + _directories.TryRemove(directory, out _); + } + } + } + + /// + public bool DirectoryExists(string path) + { + try + { + return !string.IsNullOrWhiteSpace(path) + && _directories.ContainsKey(NormalizeDirectoryPath(path)); + } + catch (Exception exception) when (IsPathProbeException(exception)) + { + return false; + } + } + + /// + public IEnumerable EnumerateFiles( + string path, + string searchPattern, + SearchOption searchOption) => + EnumerateEntries(_files.Keys, path, searchPattern, searchOption); + + /// + public IEnumerable EnumerateDirectories( + string path, + string searchPattern, + SearchOption searchOption) => + EnumerateEntries(_directories.Keys, path, searchPattern, searchOption); + + /// + public string GetTempPath() => + Path.Combine(Path.GetTempPath(), "ModularPipelines.Testing") + + Path.DirectorySeparatorChar; + + /// + public string GetRandomFileName() => Path.GetRandomFileName(); + + /// + public string Combine(params string[] paths) => Path.Combine(paths); + + /// + public string GetRelativePath(string relativeTo, string path) => + Path.GetRelativePath(relativeTo, path); + + private static async Task WriteLinesAsync( + Stream stream, + IEnumerable contents, + CancellationToken cancellationToken) + { + await using var writer = new StreamWriter( + stream, + new UTF8Encoding(encoderShouldEmitUTF8Identifier: false), + leaveOpen: true); + using var enumerator = contents.GetEnumerator(); + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + var hasNext = enumerator.MoveNext(); + cancellationToken.ThrowIfCancellationRequested(); + if (!hasNext) + { + return; + } + + await writer.WriteLineAsync( + (enumerator.Current ?? string.Empty).AsMemory(), + cancellationToken) + .ConfigureAwait(false); + } + } + + private void ValidateFilesAreClosed(IEnumerable files) + { + foreach (var file in files) + { + if (IsOpen(file)) + { + throw new IOException( + $"The in-memory file '{file}' is already open."); + } + } + } + + private byte[] GetFile(string path) + { + lock (_sync) + { + var normalized = NormalizeFilePath(path); + if (_exclusiveOpenFiles.Contains(normalized)) + { + throw new IOException( + $"The in-memory file '{normalized}' is already open."); + } + + return _files.TryGetValue(normalized, out var contents) + ? contents + : throw new FileNotFoundException( + "The in-memory file does not exist.", + normalized); + } + } + + private void SetFile(string path, byte[] contents, bool allowOpenFile = false) + { + lock (_sync) + { + var normalized = NormalizeFilePath(path); + if (!allowOpenFile && IsOpen(normalized)) + { + throw new IOException( + $"The in-memory file '{normalized}' is already open."); + } + + ValidateFileDestination(normalized); + _files[normalized] = [.. contents]; + } + } + + private void ValidateFileDestination(string normalizedPath) + { + var directoryForm = NormalizeDirectoryPath(normalizedPath); + if (Path.EndsInDirectorySeparator(normalizedPath) + || _directories.ContainsKey(directoryForm)) + { + throw new IOException( + $"The in-memory path '{directoryForm}' is a directory or directory-form path."); + } + + var parent = Path.GetDirectoryName(normalizedPath); + if (string.IsNullOrEmpty(parent) || !_directories.ContainsKey(parent)) + { + throw new DirectoryNotFoundException(parent); + } + } + + private bool IsOpen(string normalizedPath) => + _exclusiveOpenFiles.Contains(normalizedPath) + || _openReaders.ContainsKey(normalizedPath); + + private static bool IsPathProbeException(Exception exception) => + exception is ArgumentException + or NotSupportedException + or PathTooLongException + or DirectoryNotFoundException; + + private string NormalizeDirectoryPath(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + ValidateFileNameSegments(path); + ValidatePathTraversal(path); + var fullPath = Path.GetFullPath(path); + var root = Path.GetPathRoot(fullPath); + return root is not null && _pathComparer.Equals(root, fullPath) + ? fullPath + : fullPath.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + } + + private string NormalizeFilePath(string path) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + if (path.Length >= 2 + && path[^1] == '.' + && IsDirectorySeparator(path[^2])) + { + throw new ArgumentException( + "A file path cannot end in a current-directory segment.", + nameof(path)); + } + + ValidateFileNameSegments(path); + ValidatePathTraversal(path); + return Path.GetFullPath(path); + } + + private static void ValidateFileNameSegments(string path) + { + var unresolvedPath = Path.IsPathFullyQualified(path) + ? path + : Path.Combine(Environment.CurrentDirectory, path); + var root = Path.GetPathRoot(unresolvedPath); + var pathWithoutRoot = string.IsNullOrEmpty(root) + ? unresolvedPath + : unresolvedPath[root.Length..]; + var invalidCharacters = Path.GetInvalidFileNameChars(); + + foreach (var segment in pathWithoutRoot.Split( + [Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar], + StringSplitOptions.RemoveEmptyEntries)) + { + if (segment.IndexOfAny(invalidCharacters) >= 0) + { + throw new ArgumentException( + $"The path contains an invalid file-name character: '{segment}'.", + nameof(path)); + } + } + } + + private void ValidatePathTraversal(string path) + { + var unresolvedPath = Path.IsPathFullyQualified(path) + ? path + : Path.Combine(Environment.CurrentDirectory, path); + var root = Path.GetPathRoot(unresolvedPath); + if (string.IsNullOrEmpty(root)) + { + return; + } + + var current = root; + foreach (var segment in unresolvedPath[root.Length..].Split( + [Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar], + StringSplitOptions.RemoveEmptyEntries)) + { + if (_files.ContainsKey(current)) + { + throw new DirectoryNotFoundException(current); + } + + current = segment switch + { + "." => current, + ".." => Path.GetDirectoryName(current) ?? current, + _ => Path.Combine(current, segment), + }; + } + } + + private IEnumerable EnumerateEntries( + IEnumerable entries, + string path, + string searchPattern, + SearchOption searchOption) + { + var root = NormalizeDirectoryPath(path); + var separatorIndex = searchPattern.LastIndexOfAny( + [Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar]); + var expression = searchPattern; + if (separatorIndex >= 0) + { + root = NormalizeDirectoryPath(Path.Combine(root, searchPattern[..separatorIndex])); + expression = searchPattern[(separatorIndex + 1)..]; + } + + if (!_directories.ContainsKey(root)) + { + throw new DirectoryNotFoundException(root); + } + + return [.. entries + .Where(entry => IsDescendant(entry, root)) + .Where(entry => searchOption == SearchOption.AllDirectories + || !Path.GetRelativePath(root, entry) + .Contains(Path.DirectorySeparatorChar, StringComparison.Ordinal)) + .Where(entry => System.IO.Enumeration.FileSystemName.MatchesSimpleExpression( + expression, + Path.GetFileName(entry), + _pathComparison == StringComparison.OrdinalIgnoreCase)) + .OrderBy(static entry => entry, _pathComparer)]; + } + + private IEnumerable GetDescendantFiles(string path) => + _files.Keys.Where(entry => IsDescendant(entry, path)); + + private IEnumerable GetDescendantDirectories(string path) => + _directories.Keys.Where(entry => IsDescendant(entry, path)); + + private bool IsDescendant(string candidate, string parent) + { + if (!candidate.StartsWith(parent, _pathComparison) || candidate.Length <= parent.Length) + { + return false; + } + + return Path.EndsInDirectorySeparator(parent) + || IsDirectorySeparator(candidate[parent.Length]); + } + + private static bool IsDirectorySeparator(char value) => + value == Path.DirectorySeparatorChar || value == Path.AltDirectorySeparatorChar; + + private static string ReplacePrefix(string path, string source, string destination) => + destination + path[source.Length..]; + + private sealed class CommittingMemoryStream : MemoryStream + { + private readonly Action _commit; + private readonly bool _append; + private readonly bool _readable; + private readonly bool _writable; + private readonly long _appendStart; + private readonly bool _initializing = true; + private bool _committed; + private readonly Action _release; + private bool _released; + + public CommittingMemoryStream( + byte[] contents, + FileAccess access, + bool append, + Action commit, + Action release) + : base(Math.Max(contents.Length, 256)) + { + _append = append; + _readable = access != FileAccess.Write; + _writable = access != FileAccess.Read; + _appendStart = append ? contents.Length : 0; + _commit = commit; + _release = release; + base.Write(contents, 0, contents.Length); + Position = 0; + _initializing = false; + } + + public override bool CanRead => _readable && base.CanRead; + + public override bool CanWrite => (_initializing || _writable) && base.CanWrite; + + public override long Position + { + get => base.Position; + set + { + EnsureValidPosition(value); + base.Position = value; + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + EnsureReadable(); + return base.Read(buffer, offset, count); + } + + public override int Read(Span buffer) + { + EnsureReadable(); + return base.Read(buffer); + } + + public override ValueTask ReadAsync( + Memory buffer, + CancellationToken cancellationToken = default) + { + EnsureReadable(); + return base.ReadAsync(buffer, cancellationToken); + } + + public override Task ReadAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken) + { + EnsureReadable(); + return base.ReadAsync(buffer, offset, count, cancellationToken); + } + + public override int ReadByte() + { + EnsureReadable(); + return base.ReadByte(); + } + + public override long Seek(long offset, SeekOrigin loc) + { + var position = loc switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => Position + offset, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentOutOfRangeException(nameof(loc)), + }; + EnsureValidPosition(position); + return base.Seek(offset, loc); + } + + public override void Write(byte[] buffer, int offset, int count) + { + EnsureWritable(); + base.Write(buffer, offset, count); + } + + public override void Write(ReadOnlySpan buffer) + { + EnsureWritable(); + base.Write(buffer); + } + + public override ValueTask WriteAsync( + ReadOnlyMemory buffer, + CancellationToken cancellationToken = default) + { + EnsureWritable(); + return base.WriteAsync(buffer, cancellationToken); + } + + public override Task WriteAsync( + byte[] buffer, + int offset, + int count, + CancellationToken cancellationToken) + { + EnsureWritable(); + return base.WriteAsync(buffer, offset, count, cancellationToken); + } + + public override void WriteByte(byte value) + { + EnsureWritable(); + base.WriteByte(value); + } + + public override void SetLength(long value) + { + EnsureWritable(); + EnsureValidPosition(value); + base.SetLength(value); + } + + protected override void Dispose(bool disposing) + { + try + { + if (disposing && _writable && !_committed) + { + _committed = true; + _commit(ToArray()); + } + } + finally + { + if (disposing && !_released) + { + _released = true; + _release(); + } + + base.Dispose(disposing); + } + } + + private void EnsureWritable() + { + if (!_writable) + { + throw new NotSupportedException("The in-memory stream does not support writing."); + } + + EnsureValidPosition(Position); + } + + private void EnsureReadable() + { + if (!_readable) + { + throw new NotSupportedException("The in-memory stream does not support reading."); + } + } + + private void EnsureValidPosition(long position) + { + if (!_initializing && _append && position < _appendStart) + { + throw new IOException("Cannot seek before the append boundary."); + } + } + } +} diff --git a/src/ModularPipelines.Testing/ModularPipelines.Testing.csproj b/src/ModularPipelines.Testing/ModularPipelines.Testing.csproj new file mode 100644 index 00000000000..de2a201b987 --- /dev/null +++ b/src/ModularPipelines.Testing/ModularPipelines.Testing.csproj @@ -0,0 +1,12 @@ + + + A supported test harness for isolated module execution, command recording, and in-memory file operations. + Supported test harness for isolated ModularPipelines module tests. + true + + $(NoWarn);CS1591;MPG0013 + + + + + diff --git a/src/ModularPipelines.Testing/ModularPipelines.Testing.sln b/src/ModularPipelines.Testing/ModularPipelines.Testing.sln new file mode 100644 index 00000000000..896d2376ebf --- /dev/null +++ b/src/ModularPipelines.Testing/ModularPipelines.Testing.sln @@ -0,0 +1,118 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModularPipelines.Testing", "ModularPipelines.Testing.csproj", "{46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModularPipelines.Development.Analyzers", "..\ModularPipelines.Development.Analyzers\ModularPipelines.Development.Analyzers.csproj", "{B766C742-2653-4F6E-AC4F-8932FA080A54}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModularPipelines.SourceGenerator", "..\ModularPipelines.SourceGenerator\ModularPipelines.SourceGenerator.csproj", "{8A91D9C5-3966-42EC-A389-2D0C06C1CB68}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModularPipelines", "..\ModularPipelines\ModularPipelines.csproj", "{0B32F4EB-6A26-4D23-A584-367D26EFE913}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModularPipelines.Analyzers", "..\ModularPipelines.Analyzers\ModularPipelines.Analyzers\ModularPipelines.Analyzers.csproj", "{D0EF2827-D7B4-41B4-B60D-160783C64618}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModularPipelines.Analyzers.CodeFixes", "..\ModularPipelines.Analyzers\ModularPipelines.Analyzers.CodeFixes\ModularPipelines.Analyzers.CodeFixes.csproj", "{661A0D65-6117-41DA-8F5E-F6A02FB02134}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModularPipelines.Testing.UnitTests", "..\..\test\ModularPipelines.Testing.UnitTests\ModularPipelines.Testing.UnitTests.csproj", "{C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}.Debug|x64.ActiveCfg = Debug|Any CPU + {46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}.Debug|x64.Build.0 = Debug|Any CPU + {46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}.Debug|x86.ActiveCfg = Debug|Any CPU + {46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}.Debug|x86.Build.0 = Debug|Any CPU + {46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}.Release|Any CPU.Build.0 = Release|Any CPU + {46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}.Release|x64.ActiveCfg = Release|Any CPU + {46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}.Release|x64.Build.0 = Release|Any CPU + {46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}.Release|x86.ActiveCfg = Release|Any CPU + {46D7D33F-97AA-40FF-9EEF-AC21D8A9D0CC}.Release|x86.Build.0 = Release|Any CPU + {B766C742-2653-4F6E-AC4F-8932FA080A54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B766C742-2653-4F6E-AC4F-8932FA080A54}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B766C742-2653-4F6E-AC4F-8932FA080A54}.Debug|x64.ActiveCfg = Debug|Any CPU + {B766C742-2653-4F6E-AC4F-8932FA080A54}.Debug|x64.Build.0 = Debug|Any CPU + {B766C742-2653-4F6E-AC4F-8932FA080A54}.Debug|x86.ActiveCfg = Debug|Any CPU + {B766C742-2653-4F6E-AC4F-8932FA080A54}.Debug|x86.Build.0 = Debug|Any CPU + {B766C742-2653-4F6E-AC4F-8932FA080A54}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B766C742-2653-4F6E-AC4F-8932FA080A54}.Release|Any CPU.Build.0 = Release|Any CPU + {B766C742-2653-4F6E-AC4F-8932FA080A54}.Release|x64.ActiveCfg = Release|Any CPU + {B766C742-2653-4F6E-AC4F-8932FA080A54}.Release|x64.Build.0 = Release|Any CPU + {B766C742-2653-4F6E-AC4F-8932FA080A54}.Release|x86.ActiveCfg = Release|Any CPU + {B766C742-2653-4F6E-AC4F-8932FA080A54}.Release|x86.Build.0 = Release|Any CPU + {8A91D9C5-3966-42EC-A389-2D0C06C1CB68}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8A91D9C5-3966-42EC-A389-2D0C06C1CB68}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A91D9C5-3966-42EC-A389-2D0C06C1CB68}.Debug|x64.ActiveCfg = Debug|Any CPU + {8A91D9C5-3966-42EC-A389-2D0C06C1CB68}.Debug|x64.Build.0 = Debug|Any CPU + {8A91D9C5-3966-42EC-A389-2D0C06C1CB68}.Debug|x86.ActiveCfg = Debug|Any CPU + {8A91D9C5-3966-42EC-A389-2D0C06C1CB68}.Debug|x86.Build.0 = Debug|Any CPU + {8A91D9C5-3966-42EC-A389-2D0C06C1CB68}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8A91D9C5-3966-42EC-A389-2D0C06C1CB68}.Release|Any CPU.Build.0 = Release|Any CPU + {8A91D9C5-3966-42EC-A389-2D0C06C1CB68}.Release|x64.ActiveCfg = Release|Any CPU + {8A91D9C5-3966-42EC-A389-2D0C06C1CB68}.Release|x64.Build.0 = Release|Any CPU + {8A91D9C5-3966-42EC-A389-2D0C06C1CB68}.Release|x86.ActiveCfg = Release|Any CPU + {8A91D9C5-3966-42EC-A389-2D0C06C1CB68}.Release|x86.Build.0 = Release|Any CPU + {0B32F4EB-6A26-4D23-A584-367D26EFE913}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0B32F4EB-6A26-4D23-A584-367D26EFE913}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0B32F4EB-6A26-4D23-A584-367D26EFE913}.Debug|x64.ActiveCfg = Debug|Any CPU + {0B32F4EB-6A26-4D23-A584-367D26EFE913}.Debug|x64.Build.0 = Debug|Any CPU + {0B32F4EB-6A26-4D23-A584-367D26EFE913}.Debug|x86.ActiveCfg = Debug|Any CPU + {0B32F4EB-6A26-4D23-A584-367D26EFE913}.Debug|x86.Build.0 = Debug|Any CPU + {0B32F4EB-6A26-4D23-A584-367D26EFE913}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0B32F4EB-6A26-4D23-A584-367D26EFE913}.Release|Any CPU.Build.0 = Release|Any CPU + {0B32F4EB-6A26-4D23-A584-367D26EFE913}.Release|x64.ActiveCfg = Release|Any CPU + {0B32F4EB-6A26-4D23-A584-367D26EFE913}.Release|x64.Build.0 = Release|Any CPU + {0B32F4EB-6A26-4D23-A584-367D26EFE913}.Release|x86.ActiveCfg = Release|Any CPU + {0B32F4EB-6A26-4D23-A584-367D26EFE913}.Release|x86.Build.0 = Release|Any CPU + {D0EF2827-D7B4-41B4-B60D-160783C64618}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D0EF2827-D7B4-41B4-B60D-160783C64618}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D0EF2827-D7B4-41B4-B60D-160783C64618}.Debug|x64.ActiveCfg = Debug|Any CPU + {D0EF2827-D7B4-41B4-B60D-160783C64618}.Debug|x64.Build.0 = Debug|Any CPU + {D0EF2827-D7B4-41B4-B60D-160783C64618}.Debug|x86.ActiveCfg = Debug|Any CPU + {D0EF2827-D7B4-41B4-B60D-160783C64618}.Debug|x86.Build.0 = Debug|Any CPU + {D0EF2827-D7B4-41B4-B60D-160783C64618}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D0EF2827-D7B4-41B4-B60D-160783C64618}.Release|Any CPU.Build.0 = Release|Any CPU + {D0EF2827-D7B4-41B4-B60D-160783C64618}.Release|x64.ActiveCfg = Release|Any CPU + {D0EF2827-D7B4-41B4-B60D-160783C64618}.Release|x64.Build.0 = Release|Any CPU + {D0EF2827-D7B4-41B4-B60D-160783C64618}.Release|x86.ActiveCfg = Release|Any CPU + {D0EF2827-D7B4-41B4-B60D-160783C64618}.Release|x86.Build.0 = Release|Any CPU + {661A0D65-6117-41DA-8F5E-F6A02FB02134}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {661A0D65-6117-41DA-8F5E-F6A02FB02134}.Debug|Any CPU.Build.0 = Debug|Any CPU + {661A0D65-6117-41DA-8F5E-F6A02FB02134}.Debug|x64.ActiveCfg = Debug|Any CPU + {661A0D65-6117-41DA-8F5E-F6A02FB02134}.Debug|x64.Build.0 = Debug|Any CPU + {661A0D65-6117-41DA-8F5E-F6A02FB02134}.Debug|x86.ActiveCfg = Debug|Any CPU + {661A0D65-6117-41DA-8F5E-F6A02FB02134}.Debug|x86.Build.0 = Debug|Any CPU + {661A0D65-6117-41DA-8F5E-F6A02FB02134}.Release|Any CPU.ActiveCfg = Release|Any CPU + {661A0D65-6117-41DA-8F5E-F6A02FB02134}.Release|Any CPU.Build.0 = Release|Any CPU + {661A0D65-6117-41DA-8F5E-F6A02FB02134}.Release|x64.ActiveCfg = Release|Any CPU + {661A0D65-6117-41DA-8F5E-F6A02FB02134}.Release|x64.Build.0 = Release|Any CPU + {661A0D65-6117-41DA-8F5E-F6A02FB02134}.Release|x86.ActiveCfg = Release|Any CPU + {661A0D65-6117-41DA-8F5E-F6A02FB02134}.Release|x86.Build.0 = Release|Any CPU + {C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}.Debug|x64.ActiveCfg = Debug|Any CPU + {C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}.Debug|x64.Build.0 = Debug|Any CPU + {C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}.Debug|x86.ActiveCfg = Debug|Any CPU + {C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}.Debug|x86.Build.0 = Debug|Any CPU + {C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}.Release|Any CPU.Build.0 = Release|Any CPU + {C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}.Release|x64.ActiveCfg = Release|Any CPU + {C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}.Release|x64.Build.0 = Release|Any CPU + {C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}.Release|x86.ActiveCfg = Release|Any CPU + {C6477AA3-9CA9-44E4-9B0B-757CB7BDC35C}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/ModularPipelines.Testing/ModuleTestRun.cs b/src/ModularPipelines.Testing/ModuleTestRun.cs new file mode 100644 index 00000000000..e456ae62ba8 --- /dev/null +++ b/src/ModularPipelines.Testing/ModuleTestRun.cs @@ -0,0 +1,76 @@ +using ModularPipelines.FileSystem; +using ModularPipelines.Models; + +namespace ModularPipelines.Testing; + +/// +/// Captures the observable outcome of an isolated module execution. +/// +public class ModuleTestRun +{ + internal ModuleTestRun( + IModuleResult result, + IReadOnlyList commands, + IFileSystemProvider fileSystem) + { + Result = result; + Commands = commands; + FileSystem = fileSystem; + } + + /// + /// Gets the complete module result. + /// + public IModuleResult Result { get; } + + /// + /// Gets the module value when execution succeeded. + /// + public object? Value => Result.ValueOrDefault; + + /// + /// Gets the failure exception, if any. + /// + public Exception? Exception => Result.ExceptionOrDefault; + + /// + /// Gets the skip decision, if any. + /// + public SkipDecision? SkipDecision => Result.SkipDecisionOrDefault; + + /// + /// Gets commands intercepted during the run. + /// + public IReadOnlyList Commands { get; } + + /// + /// Gets the isolated filesystem used during the run. + /// + public IFileSystemProvider FileSystem { get; } +} + +/// +/// Captures the strongly typed outcome of an isolated module execution. +/// +/// The module result value type. +public sealed class ModuleTestRun : ModuleTestRun +{ + internal ModuleTestRun( + ModuleResult result, + IReadOnlyList commands, + IFileSystemProvider fileSystem) + : base(result, commands, fileSystem) + { + Result = result; + } + + /// + /// Gets the strongly typed module result. + /// + public new ModuleResult Result { get; } + + /// + /// Gets the successful value, or the default value otherwise. + /// + public new T? Value => Result.ValueOrDefault; +} diff --git a/src/ModularPipelines.Testing/ModuleTester.cs b/src/ModularPipelines.Testing/ModuleTester.cs new file mode 100644 index 00000000000..c0506449356 --- /dev/null +++ b/src/ModularPipelines.Testing/ModuleTester.cs @@ -0,0 +1,343 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Logging; +using ModularPipelines.Context; +using ModularPipelines.Context.Domains.Shell; +using ModularPipelines.Console; +using ModularPipelines.Engine; +using ModularPipelines.Engine.Dependencies; +using ModularPipelines.Engine.Execution; +using ModularPipelines.Enums; +using ModularPipelines.Extensions; +using ModularPipelines.FileSystem; +using ModularPipelines.Helpers; +using ModularPipelines.Logging; +using ModularPipelines.Models; +using ModularPipelines.Modules; + +namespace ModularPipelines.Testing; + +/// +/// Creates isolated module test builders. +/// +public static class ModuleTester +{ + /// + /// Creates a type-erased test builder. + /// + /// The module to execute. + /// A module test builder. + public static ModuleTestBuilder For() + where TModule : class, IModule => + new(); + + /// + /// Creates a strongly typed test builder. + /// + /// The module to execute. + /// The module value type. + /// A strongly typed module test builder. + public static ModuleTestBuilder For() + where TModule : Module => + new(); +} + +/// +/// Configures isolated execution of one module. +/// +/// The module to execute. +public class ModuleTestBuilder + where TModule : class, IModule +{ + private readonly List> _registrations = []; + private readonly List _dependencySeeds = []; + private Func>? _commandHandler; + + /// + /// Registers a service instance for the module. + /// + /// The service type. + /// The service instance. + /// This builder. + public ModuleTestBuilder WithService(TService service) + where TService : class + { + _registrations.Add(builder => builder.Services.AddSingleton(service)); + return this; + } + + /// + /// Seeds a successful dependency result without executing the dependency. + /// + /// The dependency module. + /// The dependency value type. + /// The dependency value. + /// This builder. + public ModuleTestBuilder WithDependencyResult(TResult? value) + where TDependency : Module + { + _registrations.Add(static builder => builder.AddModule()); + _dependencySeeds.Add(new DependencySeed(value)); + return this; + } + + /// + /// Intercepts every command and returns a caller-provided result. + /// + /// The command handler. + /// This builder. + public ModuleTestBuilder InterceptCommands(Func handler) + { + ArgumentNullException.ThrowIfNull(handler); + _commandHandler = (invocation, _) => ValueTask.FromResult(handler(invocation)); + return this; + } + + /// + /// Intercepts every command and returns a caller-provided asynchronous result. + /// + /// The asynchronous command handler. + /// This builder. + public ModuleTestBuilder InterceptCommands( + Func> handler) + { + ArgumentNullException.ThrowIfNull(handler); + _commandHandler = handler; + return this; + } + + /// + /// Executes the module in an isolated test pipeline. + /// + /// The cancellation token. + /// The captured module run. + public async Task ExecuteAsync(CancellationToken cancellationToken = default) + { + var execution = await ExecuteCoreAsync(cancellationToken).ConfigureAwait(false); + return new ModuleTestRun(execution.Result, execution.Commands, execution.FileSystem); + } + + internal async Task ExecuteCoreAsync(CancellationToken cancellationToken) + { + var fileSystem = new InMemoryFileSystemProvider(); + var recorder = new RecordingCommandInterceptor(); + if (_commandHandler is not null) + { + recorder.SetHandler(_commandHandler); + } + + using var builder = Pipeline.CreateBuilder(new PipelineBuilderOptions + { + EnableCommandLineOptions = false, + }); + + builder.ConfigurePipelineOptions(options => options with + { + ShowProgressInConsole = false, + PrintResults = false, + PrintLogo = false, + PrintDependencyChains = false, + ThrowOnPipelineFailure = false, + }); + builder.Services.AddLogging(logging => logging.ClearProviders()); + builder.Services.Replace(ServiceDescriptor.Singleton(fileSystem)); + builder.Services.AddSingleton(recorder); + builder.Services.AddSingleton(recorder); + builder.Services.AddSingleton(NoOpConsoleServices.Instance); + builder.Services.AddSingleton(NoOpConsoleServices.Instance); + builder.Services.AddSingleton(NoOpConsoleServices.Instance); + builder.AddModule(); + + foreach (var registration in _registrations) + { + registration(builder); + } + + await using var pipeline = await builder.BuildAsync().ConfigureAwait(false); + + foreach (var dependencySeed in _dependencySeeds) + { + dependencySeed.Apply(pipeline.Services); + } + + var module = pipeline.Services.GetServices() + .OfType() + .Single(); + ValidateRequiredDependencyResults(module, pipeline.Services); + var executionContext = ExecutionContextFactory.Create(module, typeof(TModule)); + if (cancellationToken.CanBeCanceled) + { + var originalCancellationTokenSource = + executionContext.ModuleCancellationTokenSource; + executionContext.ModuleCancellationTokenSource = + CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + originalCancellationTokenSource.Dispose(); + } + + var pipelineContext = pipeline.Services.GetRequiredService(); + var logger = GetModuleLogger(pipeline.Services); + var moduleContext = new ModuleContext(pipelineContext, module, executionContext, logger); + var executionPipeline = pipeline.Services.GetRequiredService(); + var executor = ModuleExecutionDelegateFactory.GetExecutor(module.ResultType); + + await using var loggerScope = new ModuleLoggerScope(logger, typeof(TModule)); + + IModuleResult result; + try + { + result = await executor( + executionPipeline, + module, + executionContext, + moduleContext, + CancellationToken.None) + .ConfigureAwait(false); + } + catch when (executionContext.ExecutionTask.IsCompletedSuccessfully) + { + result = await executionContext.ExecutionTask.ConfigureAwait(false); + } + + pipeline.Services.GetRequiredService() + .RegisterResult(typeof(TModule), result); + + var effectiveFileSystem = pipeline.Services.GetRequiredService(); + return new ExecutionOutcome(result, recorder.Commands, effectiveFileSystem); + } + + private static IModuleLogger GetModuleLogger(IServiceProvider services) + { + if (GeneratedModuleMetadata.TryGetRuntime(typeof(TModule), out var runtime)) + { + return runtime.GetLogger(services); + } + + return services.GetRequiredService>(); + } + + private static void ValidateRequiredDependencyResults( + IModule module, + IServiceProvider services) + { + var registeredModuleTypes = services.GetServices() + .Select(static registeredModule => registeredModule.GetType()) + .ToArray(); + var dependencyRegistry = services.GetRequiredService(); + var metadataRegistry = services.GetRequiredService(); + var resultRegistry = services.GetRequiredService(); + var missingDependencies = ModuleDependencyResolver + .GetAllDependencies( + module, + registeredModuleTypes, + dependencyRegistry, + metadataRegistry) + .Where(static dependency => !dependency.Optional) + .Select(static dependency => dependency.DependencyType) + .Distinct() + .Where(dependencyType => resultRegistry.GetResult(dependencyType) is null) + .Select(static dependencyType => dependencyType.Name) + .Order(StringComparer.Ordinal) + .ToArray(); + + if (missingDependencies.Length > 0) + { + throw new InvalidOperationException( + $"Required dependency results must be seeded before testing {typeof(TModule).Name}: " + + string.Join(", ", missingDependencies) + + ". Call WithDependencyResult for each dependency."); + } + } + + internal sealed record ExecutionOutcome( + IModuleResult Result, + IReadOnlyList Commands, + IFileSystemProvider FileSystem); + + private interface IDependencySeed + { + void Apply(IServiceProvider services); + } + + private sealed class DependencySeed(TResult? value) : IDependencySeed + where TDependency : Module + { + public void Apply(IServiceProvider services) + { + var module = services.GetServices() + .OfType() + .Single(); + var now = DateTimeOffset.UtcNow; + var result = new ModuleResult.Success(value) + { + ModuleName = typeof(TDependency).Name, + ModuleTypeName = typeof(TDependency).FullName, + ModuleDuration = TimeSpan.Zero, + ModuleStart = now, + ModuleEnd = now, + ModuleStatus = Status.Successful, + ModuleType = typeof(TDependency), + }; + + ((IModule) module).TrySetDistributedResult(result); + services.GetRequiredService() + .RegisterResult(typeof(TDependency), result); + } + } +} + +/// +/// Configures strongly typed isolated execution of one module. +/// +/// The module to execute. +/// The module value type. +public sealed class ModuleTestBuilder : ModuleTestBuilder + where TModule : Module +{ + /// + public new ModuleTestBuilder WithService(TService service) + where TService : class + { + base.WithService(service); + return this; + } + + /// + public new ModuleTestBuilder WithDependencyResult( + TDependencyResult? value) + where TDependency : Module + { + base.WithDependencyResult(value); + return this; + } + + /// + public new ModuleTestBuilder InterceptCommands( + Func handler) + { + base.InterceptCommands(handler); + return this; + } + + /// + public new ModuleTestBuilder InterceptCommands( + Func> handler) + { + base.InterceptCommands(handler); + return this; + } + + /// + /// Executes the module in an isolated test pipeline. + /// + /// The cancellation token. + /// The captured strongly typed module run. + public new async Task> ExecuteAsync( + CancellationToken cancellationToken = default) + { + var execution = await ExecuteCoreAsync(cancellationToken).ConfigureAwait(false); + return new ModuleTestRun( + (ModuleResult) execution.Result, + execution.Commands, + execution.FileSystem); + } +} diff --git a/src/ModularPipelines.Testing/NoOpConsoleServices.cs b/src/ModularPipelines.Testing/NoOpConsoleServices.cs new file mode 100644 index 00000000000..419daf52d02 --- /dev/null +++ b/src/ModularPipelines.Testing/NoOpConsoleServices.cs @@ -0,0 +1,158 @@ +using MEL.Spectre; +using Microsoft.Extensions.Logging; +using ModularPipelines.Console; +using ModularPipelines.Engine; +using ModularPipelines.Helpers; +using ModularPipelines.Models; +using ModularPipelines.Modules; + +namespace ModularPipelines.Testing; + +internal sealed class NoOpConsoleServices : + IConsoleCoordinator, + IOutputCoordinator, + IProgressDisplay, + IProgressSession +{ + public static NoOpConsoleServices Instance { get; } = new(); + + private static NoOpModuleOutputBuffer Buffer { get; } = new(); + + public void Install() + { + } + + public Task BeginProgressAsync( + OrganizedModules modules, + CancellationToken cancellationToken) => + Task.FromResult(this); + + public IModuleOutputBuffer GetModuleBuffer(Type moduleType) => Buffer; + + public IModuleOutputBuffer GetUnattributedBuffer() => Buffer; + + public Task> FlushPendingWritesAsync() => + Task.FromResult>([]); + + public Task FlushInProgressModuleOutputAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task FlushModuleOutputAsync() => Task.CompletedTask; + + public void WriteResults(PipelineSummary summary) + { + } + + public void AddDeferredException(string message) + { + } + + public void WriteExceptions() + { + } + + public void Uninstall() + { + } + + public void EnableOutputBuffering() + { + } + + public void SetProgressController(IProgressController controller) + { + } + + public Task EnqueueAndFlushAsync( + IModuleOutputBuffer buffer, + OutputFlushKind flushKind, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task WaitForPendingFlushesAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public void SetProgressActive(bool isActive) + { + } + + public Task OnModuleCompletedAsync( + IModuleOutputBuffer buffer, + Type moduleType, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task FlushDeferredAsync(CancellationToken cancellationToken = default) => + Task.CompletedTask; + + public Task RunAsync( + OrganizedModules organizedModules, + CancellationToken cancellationToken) => + Task.CompletedTask; + + public void OnModuleStarted(ModuleState moduleState, TimeSpan estimatedDuration) + { + } + + public void OnModuleCompleted(ModuleState moduleState, bool isSuccessful) + { + } + + public void OnModuleSkipped(ModuleState moduleState) + { + } + + public void OnSubModuleCreated( + IModule parentModule, + SubModuleBase subModule, + TimeSpan estimatedDuration) + { + } + + public void OnSubModuleCompleted(SubModuleBase subModule, bool isSuccessful) + { + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + private sealed class NoOpModuleOutputBuffer : IModuleOutputBuffer + { + public Type ModuleType => typeof(void); + + public bool HasOutput => false; + + public bool IsComplete => true; + + public bool NeedsCompletionFlush => false; + + public void WriteLine(string message) + { + } + + public void WriteGroupCommand(IBuildSystemFormatter formatter, string? command) + { + } + + public void AddLogEvent(IBufferedLogEvent logEvent) + { + } + + public void SetException(Exception exception) + { + } + + public void MarkComplete() + { + } + + public Task FlushToAsync( + TextWriter console, + IBuildSystemFormatter formatter, + ILogger logger, + ISpectreConsoleLoggerControl loggerControl, + OutputFlushKind flushKind, + IReadOnlyList? fallbackLoggers = null, + CancellationToken cancellationToken = default) => + Task.CompletedTask; + } +} diff --git a/src/ModularPipelines.Testing/RecordedCommand.cs b/src/ModularPipelines.Testing/RecordedCommand.cs new file mode 100644 index 00000000000..e51204c103b --- /dev/null +++ b/src/ModularPipelines.Testing/RecordedCommand.cs @@ -0,0 +1,17 @@ +using ModularPipelines.Context.Domains.Shell; +using ModularPipelines.Models; + +namespace ModularPipelines.Testing; + +/// +/// A parsed command invocation and its intercepted result. +/// +/// The intercepted invocation. +/// The result returned to the module. +public sealed record RecordedCommand(CommandInvocation Invocation, CommandResult Result) +{ + /// + /// Gets the parsed command line. + /// + public CommandLine CommandLine => Invocation.CommandLine; +} diff --git a/src/ModularPipelines.Testing/RecordingCommandInterceptor.cs b/src/ModularPipelines.Testing/RecordingCommandInterceptor.cs new file mode 100644 index 00000000000..b29a22ff917 --- /dev/null +++ b/src/ModularPipelines.Testing/RecordingCommandInterceptor.cs @@ -0,0 +1,37 @@ +using System.Collections.Concurrent; +using ModularPipelines.Context.Domains.Shell; +using ModularPipelines.Models; + +namespace ModularPipelines.Testing; + +internal sealed class RecordingCommandInterceptor : ICommandInterceptor +{ + private readonly ConcurrentQueue<(long Sequence, RecordedCommand Command)> _commands = new(); + private Func> _handler = + static (_, _) => ValueTask.FromResult(CommandResult.Ok()); + private long _nextSequence; + + public IReadOnlyList Commands => + [.. _commands.OrderBy(record => record.Sequence).Select(record => record.Command)]; + + public void SetHandler(Func> handler) + { + _handler = handler; + } + + public async ValueTask InterceptAsync( + CommandInvocation invocation, + CancellationToken cancellationToken = default) + { + var sequence = Interlocked.Increment(ref _nextSequence); + var result = await _handler(invocation, cancellationToken).ConfigureAwait(false); + var effectiveResult = result with + { + CommandInput = invocation.CommandInput, + WorkingDirectory = invocation.WorkingDirectory, + EnvironmentVariables = invocation.EnvironmentVariables, + }; + _commands.Enqueue((sequence, new RecordedCommand(invocation, effectiveResult))); + return result; + } +} diff --git a/src/ModularPipelines/Context/Checksum.cs b/src/ModularPipelines/Context/Checksum.cs index 410bef76046..0484aaac6b4 100644 --- a/src/ModularPipelines/Context/Checksum.cs +++ b/src/ModularPipelines/Context/Checksum.cs @@ -1,20 +1,21 @@ using System.Security.Cryptography; using ModularPipelines.Context.Domains.Files; +using ModularPipelines.FileSystem; namespace ModularPipelines.Context; -internal class Checksum : IChecksumContext +internal class Checksum(IFileSystemProvider fileSystemProvider) : IChecksumContext { public string Md5(string filePath) { - if (!File.Exists(filePath)) + if (!fileSystemProvider.FileExists(filePath)) { throw new FileNotFoundException($"Cannot calculate MD5 checksum: file not found at '{filePath}'", filePath); } using var md5 = MD5.Create(); - using var stream = File.OpenRead(filePath); + using var stream = fileSystemProvider.OpenRead(filePath); var hash = md5.ComputeHash(stream); - return BitConverter.ToString(hash).Replace("-", string.Empty); + return Convert.ToHexString(hash); } } diff --git a/src/ModularPipelines/Context/Command.cs b/src/ModularPipelines/Context/Command.cs index 3c8b9ca02ec..cdc6c1ce427 100644 --- a/src/ModularPipelines/Context/Command.cs +++ b/src/ModularPipelines/Context/Command.cs @@ -11,6 +11,7 @@ using ModularPipelines.Exceptions; using ModularPipelines.Helpers.Internal; using ModularPipelines.Logging; +using ModularPipelines.Models; using ModularPipelines.Options; using CommandResult = ModularPipelines.Models.CommandResult; @@ -24,6 +25,7 @@ internal sealed class Command : ICommandContext { private readonly ICommandLogger _commandLogger; private readonly ICommandLineBuilder _commandLineBuilder; + private readonly IEnumerable _commandInterceptors; private readonly ISecretProvider _secretProvider; private readonly ISecretRegistry _secretRegistry; private readonly ISecretObfuscator _secretObfuscator; @@ -31,12 +33,14 @@ internal sealed class Command : ICommandContext public Command( ICommandLogger commandLogger, ICommandLineBuilder commandLineBuilder, + IEnumerable commandInterceptors, ISecretProvider secretProvider, ISecretRegistry secretRegistry, ISecretObfuscator secretObfuscator) { _commandLogger = commandLogger; _commandLineBuilder = commandLineBuilder; + _commandInterceptors = commandInterceptors; _secretProvider = secretProvider; _secretRegistry = secretRegistry; _secretObfuscator = secretObfuscator; @@ -79,30 +83,152 @@ public async Task ExecuteCommandLineToolAsync( command = command.WithCredentials(execOpts.CommandLineCredentials.ToCliWrapCredentials()); } - if (execOpts.InternalDryRun) + cancellationToken.ThrowIfCancellationRequested(); + + var commandMetadata = new CommandResult(command); + var invocation = new CommandInvocation( + new CommandLine(tool, parsedArgs), + options, + execOpts, + commandMetadata.CommandInput, + commandMetadata.WorkingDirectory, + commandMetadata.EnvironmentVariables); + + using var timeoutCancellationToken = CreateTimeoutCancellationToken(execOpts); + using var linkedCancellationToken = + CreateLinkedCancellationToken(timeoutCancellationToken, cancellationToken); + + try { - _commandLogger.Log( - options: options, - execOpts: execOpts, - inputToLog: execOpts.InputLoggingManipulator == null ? command.ToString() : execOpts.InputLoggingManipulator(command.ToString()), - exitCode: 0, - runTime: TimeSpan.Zero, - standardOutput: "Dummy Output Response", - standardError: "Dummy Error Response", - commandWorkingDirPath: command.WorkingDirPath - ); - - return new CommandResult(command); + linkedCancellationToken.Token.ThrowIfCancellationRequested(); + + var intercepted = await TryInterceptAsync( + invocation, + command, + options, + execOpts, + linkedCancellationToken.Token) + .ConfigureAwait(false); + if (intercepted is not null) + { + return intercepted; + } + + if (execOpts.InternalDryRun) + { + return ExecuteDryRun(command, options, execOpts); + } + + return await Of( + command, + options, + execOpts, + linkedCancellationToken.Token, + cancellationToken, + timeoutCancellationToken) + .ConfigureAwait(false); + } + catch (OperationCanceledException exception) + when (!cancellationToken.IsCancellationRequested + && timeoutCancellationToken?.IsCancellationRequested is true) + { + throw CreateTimeoutException(execOpts, exception); } + } - return await Of(command, options, execOpts, cancellationToken).ConfigureAwait(false); + private async Task TryInterceptAsync( + CommandInvocation invocation, + CliWrap.Command command, + CommandLineToolOptions options, + CommandExecutionOptions executionOptions, + CancellationToken cancellationToken) + { + foreach (var interceptor in _commandInterceptors) + { + var intercepted = await interceptor + .InterceptAsync(invocation, cancellationToken) + .ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + if (intercepted is null) + { + continue; + } + + var result = ApplyCommandMetadata(intercepted, command); + LogInterceptedCommand(options, executionOptions, result); + if (result.ExitCode != 0 && executionOptions.ThrowOnNonZeroExitCode) + { + throw new CommandException(CreateFailureResult( + command, + executionOptions, + result.CommandInput, + result.ExitCode, + result.Duration, + result.StandardOutput, + result.StandardError, + result.StartTime, + result.EndTime)); + } + + return result; + } + + return null; + } + + private CommandResult ExecuteDryRun( + CliWrap.Command command, + CommandLineToolOptions options, + CommandExecutionOptions executionOptions) + { + var commandText = command.ToString(); + _commandLogger.Log( + options: options, + execOpts: executionOptions, + inputToLog: executionOptions.InputLoggingManipulator?.Invoke(commandText) ?? commandText, + exitCode: 0, + runTime: TimeSpan.Zero, + standardOutput: "Dummy Output Response", + standardError: "Dummy Error Response", + commandWorkingDirPath: command.WorkingDirPath); + + return new CommandResult(command); + } + + private static CommandResult ApplyCommandMetadata(CommandResult result, CliWrap.Command command) + { + var metadata = new CommandResult(command); + return result with + { + CommandInput = metadata.CommandInput, + WorkingDirectory = metadata.WorkingDirectory, + EnvironmentVariables = metadata.EnvironmentVariables, + }; + } + + private void LogInterceptedCommand( + CommandLineToolOptions options, + CommandExecutionOptions executionOptions, + CommandResult result) + { + _commandLogger.Log( + options, + executionOptions, + executionOptions.InputLoggingManipulator?.Invoke(result.CommandInput) ?? result.CommandInput, + result.ExitCode, + result.Duration, + result.StandardOutput, + result.StandardError, + result.WorkingDirectory); } private async Task Of( CliWrap.Command command, CommandLineToolOptions options, CommandExecutionOptions execOpts, - CancellationToken cancellationToken = default) + CancellationToken executionCancellationToken, + CancellationToken callerCancellationToken, + CancellationTokenSource? timeoutCancellationToken) { var standardOutputBuffer = new BoundedCommandOutputBuffer(execOpts.MaxCapturedOutputLength); var standardErrorBuffer = new BoundedCommandOutputBuffer(execOpts.MaxCapturedOutputLength); @@ -121,18 +247,12 @@ private async Task Of( var inputToLog = GetInputToLog(command, execOpts); var loggingFailures = new DeferredCommandLoggingFailures(); - // Only create timeout token if ExecutionTimeout is specified to avoid unnecessary allocations - using var timeoutCancellationToken = CreateTimeoutCancellationToken(execOpts); - - // Link the timeout token with the passed cancellation token, or just wrap the original if no timeout - using var linkedCancellationToken = CreateLinkedCancellationToken(timeoutCancellationToken, cancellationToken); - using var forcefulCancellationToken = new CancellationTokenSource(); using var processTreeTerminator = new ProcessTreeTerminator(); using var processTreeCancellationRegistration = forcefulCancellationToken.Token.Register(processTreeTerminator.Kill); - var registration = linkedCancellationToken.Token.Register( + var registration = executionCancellationToken.Register( () => ScheduleForcefulCancellation(forcefulCancellationToken, execOpts.GracefulShutdownTimeout)); loggingFailures.Capture( () => _commandLogger.LogCommandStart( @@ -161,14 +281,14 @@ private async Task Of( configureStartInfo: ConfigureStartInfo, configureProcess: processTreeTerminator.Attach, forcefulCancellationToken: CancellationToken.None, - gracefulCancellationToken: linkedCancellationToken.Token); + gracefulCancellationToken: executionCancellationToken); using var descendantCaptureRegistration = - linkedCancellationToken.Token.Register(processTreeTerminator.BeginGracefulShutdown); + executionCancellationToken.Register(processTreeTerminator.BeginGracefulShutdown); result = await executionTask.ConfigureAwait(false); await WaitForForcefulCancellationAsync( processTreeTerminator, - linkedCancellationToken.Token, + executionCancellationToken, forcefulCancellationToken.Token).ConfigureAwait(false); standardOutput = standardOutputBuffer.ToString(); @@ -178,7 +298,7 @@ await WaitForForcefulCancellationAsync( { await WaitForForcefulCancellationAsync( processTreeTerminator, - linkedCancellationToken.Token, + executionCancellationToken, forcefulCancellationToken.Token).ConfigureAwait(false); standardOutput = standardOutputBuffer.ToString(); @@ -214,7 +334,7 @@ await WaitForForcefulCancellationAsync( { await WaitForForcefulCancellationAsync( processTreeTerminator, - linkedCancellationToken.Token, + executionCancellationToken, forcefulCancellationToken.Token).ConfigureAwait(false); standardOutput = standardOutputBuffer.ToString(); @@ -235,7 +355,7 @@ await WaitForForcefulCancellationAsync( command.WorkingDirPath)); var failure = loggingFailures.CombineWith(e); - if (ShouldPreserveCallerCancellation(e, failure, cancellationToken)) + if (ShouldPreserveCallerCancellation(e, failure, callerCancellationToken)) { throw; } @@ -249,7 +369,7 @@ await WaitForForcefulCancellationAsync( stopwatch.Elapsed, standardOutput, standardError, - cancellationToken, + callerCancellationToken, timeoutCancellationToken); } @@ -317,9 +437,7 @@ private Exception CreateExecutionFailure( && !cancellationToken.IsCancellationRequested && timeoutCancellationToken?.IsCancellationRequested is true) { - return new TimeoutException( - $"Command execution timed out after {execOpts.ExecutionTimeout!.Value}.", - combinedFailure); + return CreateTimeoutException(execOpts, combinedFailure); } return new CommandException( @@ -334,6 +452,13 @@ private Exception CreateExecutionFailure( combinedFailure); } + private static TimeoutException CreateTimeoutException( + CommandExecutionOptions executionOptions, + Exception innerException) => + new( + $"Command execution timed out after {executionOptions.ExecutionTimeout!.Value}.", + innerException); + private CommandResult CreateFailureResult( CliWrap.Command command, CommandExecutionOptions execOpts, diff --git a/src/ModularPipelines/Context/Domains/Implementations/FilesContext.cs b/src/ModularPipelines/Context/Domains/Implementations/FilesContext.cs index 868262ac137..79481e1cf8c 100644 --- a/src/ModularPipelines/Context/Domains/Implementations/FilesContext.cs +++ b/src/ModularPipelines/Context/Domains/Implementations/FilesContext.cs @@ -1,5 +1,4 @@ using Microsoft.Extensions.FileSystemGlobbing; -using Microsoft.Extensions.FileSystemGlobbing.Abstractions; using ModularPipelines.Context.Domains.Files; using ModularPipelines.FileSystem; using File = ModularPipelines.FileSystem.File; @@ -9,25 +8,14 @@ namespace ModularPipelines.Context.Domains.Implementations; /// /// Provides file system operations with rich File and Folder return types. /// -internal class FilesContext : IFilesContext +internal class FilesContext( + IFileSystemContext fileSystemContext, + IFileSystemProvider fileSystemProvider, + IZipContext zip, + IChecksumContext checksum) : IFilesContext { - private readonly IFileSystemContext _fileSystemContext; - - /// - /// Initializes a new instance of the class. - /// - /// The file system context for basic file operations. - /// The zip context for compression operations. - /// The checksum context for file checksum operations. - public FilesContext( - IFileSystemContext fileSystemContext, - IZipContext zip, - IChecksumContext checksum) - { - _fileSystemContext = fileSystemContext; - Zip = zip; - Checksum = checksum; - } + private readonly IFileSystemContext _fileSystemContext = fileSystemContext; + private readonly IFileSystemProvider _fileSystemProvider = fileSystemProvider; /// public File GetFile(string path) => _fileSystemContext.GetFile(path); @@ -39,108 +27,44 @@ public FilesContext( public Folder GetFolder(System.Environment.SpecialFolder specialFolder) => _fileSystemContext.GetFolder(specialFolder); /// - public IEnumerable Glob(string pattern) - { - // Use the current directory as the root for globbing - var currentDirectory = Directory.GetCurrentDirectory(); - var directoryInfo = new DirectoryInfo(currentDirectory); - - return new Matcher(StringComparison.OrdinalIgnoreCase) - .AddInclude(pattern) - .Execute(new DirectoryInfoWrapper(directoryInfo)) - .Files - .Select(x => new File(Path.Combine(currentDirectory, x.Path))) - .Distinct(); - } + public IEnumerable Glob(string pattern) => + GetFolder(System.Environment.CurrentDirectory).GetFiles(pattern); /// public IEnumerable GlobFolders(string pattern) { - // Use the current directory as the root for globbing - var currentDirectory = Directory.GetCurrentDirectory(); - var directoryInfo = new DirectoryInfo(currentDirectory); + var currentDirectory = System.Environment.CurrentDirectory; + var matcher = new Matcher(StringComparison.OrdinalIgnoreCase) + .AddInclude(pattern); - // For folder globbing, we need to handle patterns that match directories - // The Matcher is designed for files, so we match files and then extract unique parent directories - // Alternatively, we enumerate directories and filter by pattern - return EnumerateFoldersMatchingPattern(directoryInfo, pattern) - .Select(x => new Folder(x.FullName)) + return _fileSystemProvider + .EnumerateDirectories(currentDirectory, "*", SearchOption.AllDirectories) + .Where(path => matcher.Match( + _fileSystemProvider.GetRelativePath(currentDirectory, path)).HasMatches) + .Select(_fileSystemContext.GetFolder) .Distinct(); } /// public Task ReadAsync(string path, CancellationToken cancellationToken = default) - => System.IO.File.ReadAllTextAsync(path, cancellationToken); + => _fileSystemProvider.ReadAllTextAsync(path, cancellationToken); /// public Task WriteAsync(string path, string content, CancellationToken cancellationToken = default) - => System.IO.File.WriteAllTextAsync(path, content, cancellationToken); + => _fileSystemProvider.WriteAllTextAsync(path, content, cancellationToken); /// public Task ExistsAsync(string path, CancellationToken cancellationToken = default) - => Task.FromResult(System.IO.File.Exists(path) || Directory.Exists(path)); + { + cancellationToken.ThrowIfCancellationRequested(); + return Task.FromResult( + _fileSystemProvider.FileExists(path) + || _fileSystemProvider.DirectoryExists(path)); + } /// - public IZipContext Zip { get; } + public IZipContext Zip { get; } = zip; /// - public IChecksumContext Checksum { get; } - - /// - /// Enumerates directories matching a glob pattern. - /// - /// The root directory to search from. - /// The glob pattern to match. - /// An enumerable of matching directories. - private static IEnumerable EnumerateFoldersMatchingPattern(DirectoryInfo rootDirectory, string pattern) - { - // Use Matcher to find files matching the pattern with a wildcard appended - // This will match any file in directories that match the pattern - var matcher = new Matcher(StringComparison.OrdinalIgnoreCase) - .AddInclude(pattern.TrimEnd('/') + "/**/*"); - - var result = matcher.Execute(new DirectoryInfoWrapper(rootDirectory)); - - // Extract unique parent directories from matched files - var matchedDirs = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (var file in result.Files) - { - // Get the directory part of the matched file relative to root - var relativePath = file.Path; - var directoryPath = Path.GetDirectoryName(relativePath); - - if (!string.IsNullOrEmpty(directoryPath)) - { - // Walk up the directory path and check if it matches the original pattern - var fullPath = Path.Combine(rootDirectory.FullName, directoryPath); - if (Directory.Exists(fullPath)) - { - matchedDirs.Add(fullPath); - } - } - } - - // Also try direct directory enumeration for patterns that might not have files - try - { - // Convert glob pattern to search pattern (simplified) - var searchPattern = pattern.Replace("**", "*").TrimEnd('/'); - if (searchPattern.Contains('/')) - { - searchPattern = Path.GetFileName(searchPattern); - } - - foreach (var dir in rootDirectory.EnumerateDirectories(searchPattern, SearchOption.AllDirectories)) - { - matchedDirs.Add(dir.FullName); - } - } - catch (ArgumentException) - { - // Invalid search pattern characters - ignore and use only matcher results - } - - return matchedDirs.Select(x => new DirectoryInfo(x)); - } + public IChecksumContext Checksum { get; } = checksum; } diff --git a/src/ModularPipelines/Context/Domains/Shell/CommandInvocation.cs b/src/ModularPipelines/Context/Domains/Shell/CommandInvocation.cs new file mode 100644 index 00000000000..1c78dfd72e7 --- /dev/null +++ b/src/ModularPipelines/Context/Domains/Shell/CommandInvocation.cs @@ -0,0 +1,21 @@ +using ModularPipelines.Models; +using ModularPipelines.Options; + +namespace ModularPipelines.Context.Domains.Shell; + +/// +/// Describes a command after its tool and arguments have been parsed, but before execution. +/// +/// The parsed command line. +/// The original strongly typed tool options, when available. +/// The command execution options. +/// The effective command input. +/// The effective working directory. +/// The effective public environment variables. +public sealed record CommandInvocation( + CommandLine CommandLine, + CommandLineToolOptions? ToolOptions, + CommandExecutionOptions ExecutionOptions, + string CommandInput, + string WorkingDirectory, + IReadOnlyDictionary EnvironmentVariables); diff --git a/src/ModularPipelines/Context/Domains/Shell/ICommandInterceptor.cs b/src/ModularPipelines/Context/Domains/Shell/ICommandInterceptor.cs new file mode 100644 index 00000000000..bc7b20c50b4 --- /dev/null +++ b/src/ModularPipelines/Context/Domains/Shell/ICommandInterceptor.cs @@ -0,0 +1,22 @@ +using ModularPipelines.Models; + +namespace ModularPipelines.Context.Domains.Shell; + +/// +/// Intercepts commands after parsing and before process creation. +/// +public interface ICommandInterceptor +{ + /// + /// Attempts to handle a command. + /// + /// The parsed command invocation. + /// The cancellation token. + /// + /// A replacement result when the command was handled; otherwise, + /// to continue to the next interceptor or execute the process. + /// + ValueTask InterceptAsync( + CommandInvocation invocation, + CancellationToken cancellationToken = default); +} diff --git a/src/ModularPipelines/Context/Zip.cs b/src/ModularPipelines/Context/Zip.cs index 6f539058e05..794c9d76b91 100644 --- a/src/ModularPipelines/Context/Zip.cs +++ b/src/ModularPipelines/Context/Zip.cs @@ -6,25 +6,61 @@ namespace ModularPipelines.Context; -internal class Zip : IZipContext +internal class Zip(IFileSystemProvider fileSystemProvider) : IZipContext { - private readonly IFileSystemProvider _fileSystemProvider; - - public Zip(IFileSystemProvider fileSystemProvider) - { - _fileSystemProvider = fileSystemProvider; - } + private readonly IFileSystemProvider _fileSystemProvider = fileSystemProvider; public File ZipFolder(Folder folder, string outputPath, CompressionLevel compressionLevel) { - _fileSystemProvider.CreateDirectory(outputPath.GetDirectory()!); - - if (outputPath.GetPathType() == PathType.Directory) + var outputIsDirectory = _fileSystemProvider.DirectoryExists(outputPath) + || (!_fileSystemProvider.FileExists(outputPath) + && IsDirectoryPath(outputPath)); + if (outputIsDirectory) { outputPath = _fileSystemProvider.Combine(outputPath, Guid.NewGuid().ToString("N") + ".zip"); } - ZipFile.CreateFromDirectory(folder.Path, outputPath, compressionLevel, false); + if (_fileSystemProvider.FileExists(outputPath)) + { + throw new IOException($"The file '{outputPath}' already exists."); + } + + _fileSystemProvider.CreateDirectory(outputPath.GetDirectory()!); + var directories = _fileSystemProvider + .EnumerateDirectories(folder.Path, "*", SearchOption.AllDirectories) + .ToArray(); + var files = _fileSystemProvider + .EnumerateFiles(folder.Path, "*", SearchOption.AllDirectories) + .ToArray(); + + using (var output = _fileSystemProvider.Open( + outputPath, + FileMode.CreateNew, + FileAccess.ReadWrite)) + using (var archive = new ZipArchive(output, ZipArchiveMode.Create)) + { + foreach (var directory in directories) + { + var entryName = NormalizeEntryName( + _fileSystemProvider.GetRelativePath(folder.Path, directory)) + "/"; + archive.CreateEntry(entryName); + } + + foreach (var file in files) + { + var entryName = NormalizeEntryName( + _fileSystemProvider.GetRelativePath(folder.Path, file)); + var entry = archive.CreateEntry(entryName, compressionLevel); + if (_fileSystemProvider is SystemFileSystemProvider) + { + entry.LastWriteTime = System.IO.File.GetLastWriteTime(file); + } + + using var source = _fileSystemProvider.OpenRead(file); + using var destination = entry.Open(); + source.CopyTo(destination); + } + } if (!_fileSystemProvider.FileExists(outputPath)) { @@ -49,34 +85,11 @@ public Folder UnZipToFolder(string zipPath, string outputFolderPath, bool overwr try { - using var archive = ZipFile.OpenRead(zipPath); + using var zipStream = _fileSystemProvider.OpenRead(zipPath); + using var archive = new ZipArchive(zipStream, ZipArchiveMode.Read); foreach (var entry in archive.Entries) { - var destinationPath = Path.GetFullPath(Path.Combine(destinationDir, entry.FullName)); - - // Validate against Zip Slip attack (path traversal) - if (!destinationPath.StartsWith(destinationDir + Path.DirectorySeparatorChar, StringComparison.Ordinal) && - !destinationPath.Equals(destinationDir, StringComparison.Ordinal)) - { - throw new InvalidOperationException($"Zip entry '{entry.FullName}' would extract outside the target directory."); - } - - // Handle directory entries - if (string.IsNullOrEmpty(entry.Name)) - { - _fileSystemProvider.CreateDirectory(destinationPath); - continue; - } - - // Ensure parent directory exists - var parentDir = Path.GetDirectoryName(destinationPath); - if (!string.IsNullOrEmpty(parentDir)) - { - _fileSystemProvider.CreateDirectory(parentDir); - } - - // Extract file - entry.ExtractToFile(destinationPath, overwriteFiles); + ExtractEntry(entry, destinationDir, overwriteFiles); } } catch (InvalidDataException ex) @@ -94,4 +107,68 @@ public Folder UnZipToFolder(string zipPath, string outputFolderPath, bool overwr return new Folder(outputFolderPath, _fileSystemProvider); } + + private void ExtractEntry( + ZipArchiveEntry entry, + string destinationDirectory, + bool overwriteFiles) + { + var destinationPath = GetValidatedDestinationPath(entry, destinationDirectory); + if (string.IsNullOrEmpty(entry.Name)) + { + _fileSystemProvider.CreateDirectory(destinationPath); + return; + } + + var parentDirectory = Path.GetDirectoryName(destinationPath); + if (!string.IsNullOrEmpty(parentDirectory)) + { + _fileSystemProvider.CreateDirectory(parentDirectory); + } + + if (_fileSystemProvider.FileExists(destinationPath) && !overwriteFiles) + { + throw new IOException($"The file '{destinationPath}' already exists."); + } + + using (var source = entry.Open()) + using (var destination = _fileSystemProvider.Open( + destinationPath, + overwriteFiles ? FileMode.Create : FileMode.CreateNew, + FileAccess.Write)) + { + source.CopyTo(destination); + } + + if (_fileSystemProvider is SystemFileSystemProvider) + { + System.IO.File.SetLastWriteTime( + destinationPath, + entry.LastWriteTime.LocalDateTime); + } + } + + private static string GetValidatedDestinationPath( + ZipArchiveEntry entry, + string destinationDirectory) + { + var destinationPath = Path.GetFullPath( + Path.Combine(destinationDirectory, entry.FullName)); + if (!destinationPath.StartsWith( + destinationDirectory + Path.DirectorySeparatorChar, + StringComparison.Ordinal) + && !destinationPath.Equals(destinationDirectory, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Zip entry '{entry.FullName}' would extract outside the target directory."); + } + + return destinationPath; + } + + private static string NormalizeEntryName(string path) => + path.Replace(Path.DirectorySeparatorChar, '/'); + + private static bool IsDirectoryPath(string path) => + PathHelpers.EndsWithDirectorySeparator(path) || !Path.HasExtension(path); } diff --git a/src/ModularPipelines/Engine/ModuleExecutionPipeline.cs b/src/ModularPipelines/Engine/ModuleExecutionPipeline.cs index e3fbf2abcbd..84a28ac2399 100644 --- a/src/ModularPipelines/Engine/ModuleExecutionPipeline.cs +++ b/src/ModularPipelines/Engine/ModuleExecutionPipeline.cs @@ -199,11 +199,14 @@ private void SetupCancellation( { // Create a linked token source that cancels when: // - The engine singleton is cancelled (module failures, external cancellation via Ctrl+C or test timeout) + // - The execution caller is cancelled // - The original module token is cancelled (preserves any existing cancellation on the module) - // All external cancellation flows through _engineCancellationToken (see ExecutionOrchestrator line 108) + // Pipeline-wide external cancellation flows through _engineCancellationToken + // (see ExecutionOrchestrator line 108). var originalToken = executionContext.ModuleCancellationTokenSource.Token; executionContext.ModuleCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource( _engineCancellationToken.Token, + engineCancellationToken, originalToken); } diff --git a/src/ModularPipelines/FileSystem/File.cs b/src/ModularPipelines/FileSystem/File.cs index 7983ae68ace..f319c782356 100644 --- a/src/ModularPipelines/FileSystem/File.cs +++ b/src/ModularPipelines/FileSystem/File.cs @@ -80,7 +80,7 @@ public Task ReadBytesAsync(CancellationToken cancellationToken = default /// Failure to dispose the stream will result in resource leaks and may prevent other operations on the file. /// /// - /// Recommended usage with await using (async) or using (sync): + /// Recommended usage with await using (async) or using (sync). /// /// /// @@ -162,21 +162,23 @@ public Task AppendAsync(IEnumerable contents, CancellationToken cancella } /// > - public bool Exists => FileInfo.Exists; + public bool Exists => _provider.FileExists(Path); - public bool Hidden => (FileInfo.Attributes & FileAttributes.Hidden) == FileAttributes.Hidden; + public bool Hidden => (GetPhysicalFileInfo().Attributes & FileAttributes.Hidden) == FileAttributes.Hidden; /// > - public string Name => FileInfo.Name; + public string Name => System.IO.Path.GetFileName(Path); /// > public string NameWithoutExtension => System.IO.Path.GetFileNameWithoutExtension(this); /// > - public Folder? Folder => FileInfo.Directory; + public Folder? Folder => System.IO.Path.GetDirectoryName(Path) is { } directory + ? new Folder(directory, _provider) + : null; /// > - public string Path => FileInfo.FullName; + public string Path => _fileInfo.FullName; /// /// Gets the original path string that was used to construct this File instance. @@ -212,23 +214,23 @@ public async Task CreateAsync() /// > public FileAttributes Attributes { - get { return FileInfo.Attributes; } - set { FileInfo.Attributes = value; } + get { return GetPhysicalFileInfo().Attributes; } + set { GetPhysicalFileInfo().Attributes = value; } } /// > - public bool IsReadOnly => FileInfo.IsReadOnly; + public bool IsReadOnly => GetPhysicalFileInfo().IsReadOnly; /// > - public DateTimeOffset CreationTime => FileInfo.CreationTime; + public DateTimeOffset CreationTime => GetPhysicalFileInfo().CreationTime; - public DateTimeOffset LastWriteTimeUtc => FileInfo.LastWriteTimeUtc; + public DateTimeOffset LastWriteTimeUtc => GetPhysicalFileInfo().LastWriteTimeUtc; /// > - public string Extension => FileInfo.Extension; + public string Extension => System.IO.Path.GetExtension(Path); /// > - public long Length => FileInfo.Length; + public long Length => GetPhysicalFileInfo().Length; /// > public void Delete() @@ -246,6 +248,7 @@ public void Delete() /// For true async I/O, consider using stream-based operations where available. /// /// Cancellation token. + /// A task that completes when the file has been deleted. public Task DeleteAsync(CancellationToken cancellationToken = default) { LogFileOperation("Deleting File: {Path}", this); @@ -408,7 +411,7 @@ public override string ToString() /// public bool Equals(File? other) { - if (ReferenceEquals(null, other)) + if (other is null) { return false; } @@ -458,6 +461,17 @@ public override int GetHashCode() return !Equals(left, right); } + private FileInfo GetPhysicalFileInfo() + { + if (!ReferenceEquals(_provider, SystemFileSystemProvider.Instance)) + { + throw new NotSupportedException( + "File metadata is unavailable through the configured IFileSystemProvider."); + } + + return FileInfo; + } + /// /// Logs a file operation. /// diff --git a/src/ModularPipelines/FileSystem/Folder.cs b/src/ModularPipelines/FileSystem/Folder.cs index 9aeb41735cd..cd13837f74f 100644 --- a/src/ModularPipelines/FileSystem/Folder.cs +++ b/src/ModularPipelines/FileSystem/Folder.cs @@ -49,16 +49,21 @@ private Folder(DirectoryInfo directoryInfo, string originalPath, IFileSystemProv _provider = provider; } - public bool Exists => DirectoryInfo.Exists; + public bool Exists => _provider.DirectoryExists(Path); - public bool Hidden => (DirectoryInfo.Attributes & FileAttributes.Hidden) == FileAttributes.Hidden; + public bool Hidden => (GetPhysicalDirectoryInfo().Attributes & FileAttributes.Hidden) == FileAttributes.Hidden; - public string Name => DirectoryInfo.Name; + public string Name => _directoryInfo.Name; [JsonConverter(typeof(FolderPathJsonConverter))] - public Folder? Parent => DirectoryInfo.Parent; + public Folder? Parent => System.IO.Path.GetDirectoryName( + Path.TrimEnd( + System.IO.Path.DirectorySeparatorChar, + System.IO.Path.AltDirectorySeparatorChar)) is { } parent + ? new Folder(parent, _provider) + : null; - public string Path => DirectoryInfo.FullName; + public string Path => _directoryInfo.FullName; /// /// Gets the original path string that was used to construct this Folder instance. @@ -71,8 +76,8 @@ private Folder(DirectoryInfo directoryInfo, string originalPath, IFileSystemProv public FileAttributes Attributes { - get => DirectoryInfo.Attributes; - set => DirectoryInfo.Attributes = value; + get => GetPhysicalDirectoryInfo().Attributes; + set => GetPhysicalDirectoryInfo().Attributes = value; } [JsonConverter(typeof(FolderPathJsonConverter))] @@ -80,20 +85,21 @@ public Folder Root { get { - if (DirectoryInfo.Root.FullName == Path) + var rootPath = System.IO.Path.GetPathRoot(Path)!; + if (rootPath == Path) { return this; } - return DirectoryInfo.Root; + return new Folder(rootPath, _provider); } } - public DateTimeOffset CreationTime => DirectoryInfo.CreationTime; + public DateTimeOffset CreationTime => GetPhysicalDirectoryInfo().CreationTime; - public DateTimeOffset LastWriteTimeUtc => DirectoryInfo.LastWriteTimeUtc; + public DateTimeOffset LastWriteTimeUtc => GetPhysicalDirectoryInfo().LastWriteTimeUtc; - public string Extension => DirectoryInfo.Extension; + public string Extension => System.IO.Path.GetExtension(Path); public Folder Create() { @@ -136,6 +142,7 @@ public void Delete() /// Uses thread pool offloading as no native async delete API exists in .NET. /// /// Cancellation token. + /// A task that completes when the folder has been deleted. public Task DeleteAsync(CancellationToken cancellationToken = default) { LogFolderOperation("Deleting Folder: {Path}", this); @@ -187,40 +194,50 @@ public void Clean(bool removeReadOnlyAttribute, bool continueOnError) { LogFolderOperation("Cleaning Folder: {Path}", this); + if (removeReadOnlyAttribute) + { + EnsurePhysicalMetadataSupported(); + } + var errors = new List(); - foreach (var directory in DirectoryInfo.EnumerateDirectories("*", SearchOption.TopDirectoryOnly)) + foreach (var directoryPath in _provider + .EnumerateDirectories(Path, "*", SearchOption.TopDirectoryOnly) + .ToArray()) { try { if (removeReadOnlyAttribute) { - RemoveReadOnlyAttributeRecursively(directory); + RemoveReadOnlyAttributeRecursively(new DirectoryInfo(directoryPath)); } - directory.Delete(true); + _provider.DeleteDirectory(directoryPath, recursive: true); } catch (Exception ex) when (continueOnError) { - LogFolderWarning(ex, "Failed to delete directory: {Path}", directory.FullName); + LogFolderWarning(ex, "Failed to delete directory: {Path}", directoryPath); errors.Add(ex); } } - foreach (var file in DirectoryInfo.EnumerateFiles("*", SearchOption.TopDirectoryOnly)) + foreach (var filePath in _provider + .EnumerateFiles(Path, "*", SearchOption.TopDirectoryOnly) + .ToArray()) { try { + var file = new FileInfo(filePath); if (removeReadOnlyAttribute && (file.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) { file.Attributes &= ~FileAttributes.ReadOnly; } - file.Delete(); + _provider.DeleteFile(filePath); } catch (Exception ex) when (continueOnError) { - LogFolderWarning(ex, "Failed to delete file: {Path}", file.FullName); + LogFolderWarning(ex, "Failed to delete file: {Path}", filePath); errors.Add(ex); } } @@ -253,60 +270,43 @@ public Folder CopyTo(string targetPath) public Folder CopyTo(string targetPath, bool preserveTimestamps) { LogFolderOperationWithDestination("Copying Folder: {Source} > {Destination}", this, targetPath); + var copyPhysicalMetadata = ReferenceEquals(_provider, SystemFileSystemProvider.Instance); + if (preserveTimestamps && !copyPhysicalMetadata) + { + EnsurePhysicalMetadataSupported(); + } _provider.CreateDirectory(targetPath); // Copy all subdirectories first foreach (var dirPath in _provider.EnumerateDirectories(this, "*", SearchOption.AllDirectories)) { - var sourceDir = new DirectoryInfo(dirPath); var relativePath = _provider.GetRelativePath(this, dirPath); var newPath = _provider.Combine(targetPath, relativePath); _provider.CreateDirectory(newPath); - var targetDir = new DirectoryInfo(newPath); - - // Preserve directory attributes - targetDir.Attributes = sourceDir.Attributes; - - if (preserveTimestamps) + if (copyPhysicalMetadata) { - targetDir.CreationTimeUtc = sourceDir.CreationTimeUtc; - targetDir.LastWriteTimeUtc = sourceDir.LastWriteTimeUtc; - targetDir.LastAccessTimeUtc = sourceDir.LastAccessTimeUtc; + CopyDirectoryMetadata(dirPath, newPath, preserveTimestamps); } } // Copy all files foreach (var filePath in _provider.EnumerateFiles(this, "*", SearchOption.AllDirectories)) { - var sourceFile = new FileInfo(filePath); var relativePath = _provider.GetRelativePath(this, filePath); var newPath = _provider.Combine(targetPath, relativePath); _provider.CopyFile(filePath, newPath, overwrite: true); - var targetFile = new FileInfo(newPath); - - // Preserve file attributes - targetFile.Attributes = sourceFile.Attributes; - - if (preserveTimestamps) + if (copyPhysicalMetadata) { - targetFile.CreationTimeUtc = sourceFile.CreationTimeUtc; - targetFile.LastWriteTimeUtc = sourceFile.LastWriteTimeUtc; - targetFile.LastAccessTimeUtc = sourceFile.LastAccessTimeUtc; + CopyFileMetadata(filePath, newPath, preserveTimestamps); } } - // Preserve root directory attributes and timestamps after all content is copied - var targetRootDir = new DirectoryInfo(targetPath); - targetRootDir.Attributes = DirectoryInfo.Attributes; - - if (preserveTimestamps) + if (copyPhysicalMetadata) { - targetRootDir.CreationTimeUtc = DirectoryInfo.CreationTimeUtc; - targetRootDir.LastWriteTimeUtc = DirectoryInfo.LastWriteTimeUtc; - targetRootDir.LastAccessTimeUtc = DirectoryInfo.LastAccessTimeUtc; + CopyDirectoryMetadata(Path, targetPath, preserveTimestamps); } return new Folder(targetPath, _provider); @@ -336,6 +336,11 @@ public Task CopyToAsync(string targetPath, CancellationToken cancellatio public async Task CopyToAsync(string targetPath, bool preserveTimestamps, CancellationToken cancellationToken = default) { LogFolderOperationWithDestination("Copying Folder: {Source} > {Destination}", this, targetPath); + var copyPhysicalMetadata = ReferenceEquals(_provider, SystemFileSystemProvider.Instance); + if (preserveTimestamps && !copyPhysicalMetadata) + { + EnsurePhysicalMetadataSupported(); + } _provider.CreateDirectory(targetPath); @@ -344,19 +349,13 @@ public async Task CopyToAsync(string targetPath, bool preserveTimestamps { cancellationToken.ThrowIfCancellationRequested(); - var sourceDir = new DirectoryInfo(dirPath); var relativePath = _provider.GetRelativePath(this, dirPath); var newPath = _provider.Combine(targetPath, relativePath); _provider.CreateDirectory(newPath); - var targetDir = new DirectoryInfo(newPath); - targetDir.Attributes = sourceDir.Attributes; - - if (preserveTimestamps) + if (copyPhysicalMetadata) { - targetDir.CreationTimeUtc = sourceDir.CreationTimeUtc; - targetDir.LastWriteTimeUtc = sourceDir.LastWriteTimeUtc; - targetDir.LastAccessTimeUtc = sourceDir.LastAccessTimeUtc; + CopyDirectoryMetadata(dirPath, newPath, preserveTimestamps); } } @@ -365,7 +364,6 @@ public async Task CopyToAsync(string targetPath, bool preserveTimestamps { cancellationToken.ThrowIfCancellationRequested(); - var sourceFile = new FileInfo(filePath); var relativePath = _provider.GetRelativePath(this, filePath); var newPath = _provider.Combine(targetPath, relativePath); @@ -379,26 +377,15 @@ public async Task CopyToAsync(string targetPath, bool preserveTimestamps } } - var targetFile = new FileInfo(newPath); - targetFile.Attributes = sourceFile.Attributes; - - if (preserveTimestamps) + if (copyPhysicalMetadata) { - targetFile.CreationTimeUtc = sourceFile.CreationTimeUtc; - targetFile.LastWriteTimeUtc = sourceFile.LastWriteTimeUtc; - targetFile.LastAccessTimeUtc = sourceFile.LastAccessTimeUtc; + CopyFileMetadata(filePath, newPath, preserveTimestamps); } } - // Preserve root directory attributes and timestamps - var targetRootDir = new DirectoryInfo(targetPath); - targetRootDir.Attributes = DirectoryInfo.Attributes; - - if (preserveTimestamps) + if (copyPhysicalMetadata) { - targetRootDir.CreationTimeUtc = DirectoryInfo.CreationTimeUtc; - targetRootDir.LastWriteTimeUtc = DirectoryInfo.LastWriteTimeUtc; - targetRootDir.LastAccessTimeUtc = DirectoryInfo.LastAccessTimeUtc; + CopyDirectoryMetadata(Path, targetPath, preserveTimestamps); } return new Folder(targetPath, _provider); @@ -472,8 +459,7 @@ public IEnumerable GetFolders(Func predicate, Func {Expression}", this, predicateExpression); - return SafeWalk.EnumerateFolders(this, exclusionFilters) - .Select(x => new Folder(x)) + return EnumerateFolders(exclusionFilters) .Distinct() .Where(predicate); } @@ -482,8 +468,7 @@ public IEnumerable GetFiles(Func predicate, Func { LogFolderOperationWithExpression("Searching Files in: {Path} > {Expression}", this, predicateExpression); - return SafeWalk.EnumerateFiles(this, directoryExclusionFilters) - .Select(x => new File(x)) + return EnumerateFiles(directoryExclusionFilters) .Distinct() .Where(predicate); } @@ -492,11 +477,11 @@ public IEnumerable GetFiles(string globPattern) { LogFolderOperationWithExpression("Searching Files in: {Path} > {Glob}", this, globPattern); - return new Matcher(StringComparison.OrdinalIgnoreCase) - .AddInclude(globPattern) - .Execute(new DirectoryInfoWrapper(DirectoryInfo)) - .Files - .Select(x => new File(System.IO.Path.Combine(this, x.Path))) + var matcher = new Matcher(StringComparison.OrdinalIgnoreCase) + .AddInclude(globPattern); + return _provider.EnumerateFiles(Path, "*", SearchOption.AllDirectories) + .Where(path => matcher.Match(_provider.GetRelativePath(Path, path)).HasMatches) + .Select(path => new File(path, _provider)) .Distinct(); } @@ -510,15 +495,15 @@ public IEnumerable GetFiles(string globPattern) public IEnumerable ListFiles() { - return DirectoryInfo.EnumerateFiles("*", SearchOption.TopDirectoryOnly) - .Select(x => new File(x)) + return _provider.EnumerateFiles(Path, "*", SearchOption.TopDirectoryOnly) + .Select(path => new File(path, _provider)) .Distinct(); } public IEnumerable ListFolders() { - return DirectoryInfo.EnumerateDirectories("*", SearchOption.TopDirectoryOnly) - .Select(x => new Folder(x)) + return _provider.EnumerateDirectories(Path, "*", SearchOption.TopDirectoryOnly) + .Select(path => new Folder(path, _provider)) .Distinct(); } @@ -569,7 +554,7 @@ public override string ToString() /// public bool Equals(Folder? other) { - if (ReferenceEquals(null, other)) + if (other is null) { return false; } @@ -619,6 +604,114 @@ public override int GetHashCode() return !Equals(left, right); } + private IEnumerable EnumerateFolders(Func exclusionFilter) + { + if (ReferenceEquals(_provider, SystemFileSystemProvider.Instance)) + { + return SafeWalk.EnumerateFolders(this, exclusionFilter) + .Select(path => new Folder(path, _provider)); + } + + return _provider.EnumerateDirectories(Path, "*", SearchOption.AllDirectories) + .Where(path => !IsExcludedByDirectoryFilter(path, includeEntry: true, exclusionFilter)) + .Select(path => new Folder(path, _provider)); + } + + private IEnumerable EnumerateFiles(Func exclusionFilter) + { + if (ReferenceEquals(_provider, SystemFileSystemProvider.Instance)) + { + return SafeWalk.EnumerateFiles(this, exclusionFilter) + .Select(path => new File(path, _provider)); + } + + return _provider.EnumerateFiles(Path, "*", SearchOption.AllDirectories) + .Where(path => !IsExcludedByDirectoryFilter(path, includeEntry: false, exclusionFilter)) + .Select(path => new File(path, _provider)); + } + + private static void CopyDirectoryMetadata( + string sourcePath, + string targetPath, + bool preserveTimestamps) + { + var source = new DirectoryInfo(sourcePath); + var target = new DirectoryInfo(targetPath) + { + Attributes = source.Attributes, + }; + + if (preserveTimestamps) + { + target.CreationTimeUtc = source.CreationTimeUtc; + target.LastWriteTimeUtc = source.LastWriteTimeUtc; + target.LastAccessTimeUtc = source.LastAccessTimeUtc; + } + } + + private static void CopyFileMetadata( + string sourcePath, + string targetPath, + bool preserveTimestamps) + { + var source = new FileInfo(sourcePath); + var target = new FileInfo(targetPath) + { + Attributes = source.Attributes, + }; + + if (preserveTimestamps) + { + target.CreationTimeUtc = source.CreationTimeUtc; + target.LastWriteTimeUtc = source.LastWriteTimeUtc; + target.LastAccessTimeUtc = source.LastAccessTimeUtc; + } + } + + private bool IsExcludedByDirectoryFilter( + string entryPath, + bool includeEntry, + Func exclusionFilter) + { + var current = includeEntry + ? entryPath + : System.IO.Path.GetDirectoryName(entryPath); + while (current is not null) + { + if (_provider.GetRelativePath(Path, current) == ".") + { + break; + } + + if (exclusionFilter(new Folder(current, _provider))) + { + return true; + } + + current = System.IO.Path.GetDirectoryName( + current.TrimEnd( + System.IO.Path.DirectorySeparatorChar, + System.IO.Path.AltDirectorySeparatorChar)); + } + + return false; + } + + private DirectoryInfo GetPhysicalDirectoryInfo() + { + EnsurePhysicalMetadataSupported(); + return DirectoryInfo; + } + + private void EnsurePhysicalMetadataSupported() + { + if (!ReferenceEquals(_provider, SystemFileSystemProvider.Instance)) + { + throw new NotSupportedException( + "Folder metadata is unavailable through the configured IFileSystemProvider."); + } + } + private static void RemoveReadOnlyAttributeRecursively(DirectoryInfo directory) { if ((directory.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) diff --git a/src/ModularPipelines/Models/CommandResult.cs b/src/ModularPipelines/Models/CommandResult.cs index 6969646d71a..ead6a2751eb 100644 --- a/src/ModularPipelines/Models/CommandResult.cs +++ b/src/ModularPipelines/Models/CommandResult.cs @@ -1,3 +1,4 @@ +using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; using CliWrap; using ModularPipelines.Helpers.Internal; @@ -6,6 +7,30 @@ namespace ModularPipelines.Models; public record CommandResult { + private static readonly IReadOnlyDictionary EmptyEnvironmentVariables = + new ReadOnlyDictionary(new Dictionary()); + + /// + /// Creates a successful command result for command interceptors and tests. + /// + /// The simulated standard output. + /// The simulated standard error. + /// A successful command result. + public static CommandResult Ok(string standardOutput = "", string standardError = "") + { + var completedAt = DateTimeOffset.UtcNow; + return new CommandResult( + commandInput: string.Empty, + workingDirectory: Environment.CurrentDirectory, + standardOutput: standardOutput, + standardError: standardError, + environmentVariables: EmptyEnvironmentVariables, + startTime: completedAt, + endTime: completedAt, + duration: TimeSpan.Zero, + exitCode: 0); + } + /// /// Gets the command that was executed. /// diff --git a/src/ModularPipelines/ModularPipelines.csproj b/src/ModularPipelines/ModularPipelines.csproj index 59ffadf0bba..31a71221d11 100644 --- a/src/ModularPipelines/ModularPipelines.csproj +++ b/src/ModularPipelines/ModularPipelines.csproj @@ -96,6 +96,9 @@ <_Parameter1>ModularPipelines.TestHelpers + + <_Parameter1>ModularPipelines.Testing + <_Parameter1>ModularPipelines.Azure.UnitTests diff --git a/test/ModularPipelines.Testing.UnitTests/InMemoryFileSystemProviderTests.cs b/test/ModularPipelines.Testing.UnitTests/InMemoryFileSystemProviderTests.cs new file mode 100644 index 00000000000..cf2d33babfd --- /dev/null +++ b/test/ModularPipelines.Testing.UnitTests/InMemoryFileSystemProviderTests.cs @@ -0,0 +1,801 @@ +using System.Text; + +namespace ModularPipelines.Testing.UnitTests; + +public class InMemoryFileSystemProviderTests +{ + [Test] + public async Task SupportsFileLifecycleOperations() + { + var provider = new InMemoryFileSystemProvider(); + var root = Path.Combine(provider.GetTempPath(), "lifecycle"); + var source = Path.Combine(root, "source.txt"); + var copy = Path.Combine(root, "copy.txt"); + var moved = Path.Combine(root, "moved.txt"); + + provider.CreateDirectory(root); + await provider.WriteAllTextAsync(source, "first"); + await provider.AppendAllTextAsync(source, " second"); + provider.CopyFile(source, copy, overwrite: false); + provider.MoveFile(copy, moved); + + using (Assert.Multiple()) + { + await Assert.That(await provider.ReadAllTextAsync(source)).IsEqualTo("first second"); + await Assert.That(await provider.ReadAllTextAsync(moved)).IsEqualTo("first second"); + await Assert.That(provider.FileExists(copy)).IsFalse(); + await Assert.That(provider.EnumerateFiles(root, "*.txt", SearchOption.TopDirectoryOnly)) + .Count() + .IsEqualTo(2); + } + + provider.DeleteFile(source); + provider.DeleteDirectory(root, recursive: true); + + await Assert.That(provider.DirectoryExists(root)).IsFalse(); + } + + [Test] + public async Task AppendAllTextPreservesExistingBytes() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "binary-prefix.txt"); + await provider.WriteAllBytesAsync(path, [0xFF, 0xFE]); + + await provider.AppendAllTextAsync(path, "x"); + + var bytes = await provider.ReadAllBytesAsync(path); + byte[] expected = [0xFF, 0xFE, (byte) 'x']; + await Assert.That(bytes.SequenceEqual(expected)).IsTrue(); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task LineWritesCheckCancellationBeforeEnumeration(bool append) + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "cancel-lines-before.txt"); + await provider.WriteAllTextAsync(path, "original"); + var enumerated = false; + using var cancellationTokenSource = new CancellationTokenSource(); + await cancellationTokenSource.CancelAsync(); + + IEnumerable GetLines() + { + enumerated = true; + yield return "replacement"; + } + + Task WriteAsync() => append + ? provider.AppendAllLinesAsync(path, GetLines(), cancellationTokenSource.Token) + : provider.WriteAllLinesAsync(path, GetLines(), cancellationTokenSource.Token); + + await Assert.That(WriteAsync).Throws(); + await Assert.That(enumerated).IsFalse(); + await Assert.That(await provider.ReadAllTextAsync(path)).IsEqualTo("original"); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task LineWritesCheckCancellationDuringEnumeration(bool append) + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "cancel-lines-during.txt"); + await provider.WriteAllTextAsync(path, "original"); + var continuedAfterCancellation = false; + using var cancellationTokenSource = new CancellationTokenSource(); + + IEnumerable GetLines() + { + yield return "first"; + cancellationTokenSource.Cancel(); + yield return "should-not-be-written"; + continuedAfterCancellation = true; + yield return "should-not-be-enumerated"; + } + + Task WriteAsync() => append + ? provider.AppendAllLinesAsync(path, GetLines(), cancellationTokenSource.Token) + : provider.WriteAllLinesAsync(path, GetLines(), cancellationTokenSource.Token); + + await Assert.That(WriteAsync).Throws(); + await Assert.That(continuedAfterCancellation).IsFalse(); + await Assert.That(await provider.ReadAllTextAsync(path)) + .IsEqualTo($"{(append ? "original" : string.Empty)}first{Environment.NewLine}"); + } + + [Test] + [Arguments(false)] + [Arguments(true)] + public async Task LineWritesPreserveLinesWrittenBeforeEnumerationFails(bool append) + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "partial-lines.txt"); + await provider.WriteAllTextAsync(path, "original"); + + static IEnumerable GetLines() + { + yield return "first"; + throw new InvalidOperationException("enumeration failed"); + } + + Task WriteAsync() => append + ? provider.AppendAllLinesAsync(path, GetLines()) + : provider.WriteAllLinesAsync(path, GetLines()); + + await Assert.That(WriteAsync).Throws(); + await Assert.That(await provider.ReadAllTextAsync(path)) + .IsEqualTo($"{(append ? "original" : string.Empty)}first{Environment.NewLine}"); + } + + [Test] + public async Task ReadAllTextDetectsByteOrderMarks() + { + Encoding[] encodings = + [ + new UTF8Encoding(encoderShouldEmitUTF8Identifier: true), + Encoding.Unicode, + Encoding.BigEndianUnicode, + new UTF32Encoding(bigEndian: false, byteOrderMark: true), + new UTF32Encoding(bigEndian: true, byteOrderMark: true), + ]; + + var provider = new InMemoryFileSystemProvider(); + foreach (var encoding in encodings) + { + var path = Path.Combine(provider.GetTempPath(), $"{encoding.CodePage}.txt"); + var contents = encoding.GetPreamble() + .Concat(encoding.GetBytes("contents")) + .ToArray(); + await provider.WriteAllBytesAsync(path, contents); + + await Assert.That(await provider.ReadAllTextAsync(path)).IsEqualTo("contents"); + } + } + + [Test] + public async Task DeleteFileRejectsDirectoryPaths() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "directory-as-file"); + provider.CreateDirectory(path); + + await Assert.That(() => provider.DeleteFile(path)) + .Throws(); + await Assert.That(provider.DirectoryExists(path)).IsTrue(); + } + + [Test] + public async Task DeleteFileRejectsDirectoryPathsWithTrailingSeparators() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "directory-form-delete"); + provider.CreateDirectory(path); + + await Assert.That(() => provider.DeleteFile(path + Path.DirectorySeparatorChar)) + .Throws(); + await Assert.That(provider.DirectoryExists(path)).IsTrue(); + } + + [Test] + public async Task RejectsCopyingFileOntoItself() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "same-file.txt"); + await provider.WriteAllTextAsync(path, "contents"); + + await Assert.That(() => provider.CopyFile(path, path, overwrite: true)) + .Throws(); + await Assert.That(await provider.ReadAllTextAsync(path)).IsEqualTo("contents"); + } + + [Test] + public async Task CommitsWritableStreamsOnDispose() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "stream.bin"); + + await using (var stream = provider.Create(path)) + { + await stream.WriteAsync(Encoding.UTF8.GetBytes("stream contents")); + } + + await Assert.That(Encoding.UTF8.GetString(await provider.ReadAllBytesAsync(path))) + .IsEqualTo("stream contents"); + } + + [Test] + public async Task OpenReadStreamsRejectWrites() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "readonly.txt"); + await provider.WriteAllTextAsync(path, "contents"); + + using var stream = provider.OpenRead(path); + + await Assert.That(() => stream.WriteByte(1)).Throws(); +#pragma warning disable CA1835 // The legacy overload is the behavior under test. + await Assert.That(async () => + await stream.WriteAsync([1], 0, 1, CancellationToken.None)) + .Throws(); +#pragma warning restore CA1835 + } + + [Test] + public async Task SeedsCurrentDirectoryHierarchy() + { + var provider = new InMemoryFileSystemProvider(); + + await Assert.That(provider.DirectoryExists(Environment.CurrentDirectory)).IsTrue(); + } + + [Test] + public async Task TempPathEndsWithDirectorySeparator() + { + var provider = new InMemoryFileSystemProvider(); + var tempPath = provider.GetTempPath(); + var childPath = tempPath + "artifact.txt"; + + await provider.WriteAllTextAsync(childPath, "contents"); + + await Assert.That(Path.EndsInDirectorySeparator(tempPath)).IsTrue(); + await Assert.That(Path.GetDirectoryName(childPath)) + .IsEqualTo(tempPath.TrimEnd(Path.DirectorySeparatorChar)); + await Assert.That(provider.FileExists(childPath)).IsTrue(); + } + + [Test] + [Arguments("")] + [Arguments(" ")] + [Arguments("\0")] + public async Task InvalidPathsDoNotExist(string path) + { + var provider = new InMemoryFileSystemProvider(); + + using (Assert.Multiple()) + { + await Assert.That(provider.FileExists(path)).IsFalse(); + await Assert.That(provider.DirectoryExists(path)).IsFalse(); + } + } + + [Test] + public async Task TrailingSeparatorDoesNotResolveAsFile() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "trailing-separator.txt"); + var directoryPath = path + Path.DirectorySeparatorChar; + await provider.WriteAllTextAsync(path, "original"); + + async Task ReadAsync() => _ = await provider.ReadAllTextAsync(directoryPath); + async Task WriteAsync() => await provider.WriteAllTextAsync(directoryPath, "replacement"); + + await Assert.That(provider.FileExists(directoryPath)).IsFalse(); + await Assert.That(ReadAsync).ThrowsException(); + await Assert.That(WriteAsync).ThrowsException(); + await Assert.That(await provider.ReadAllTextAsync(path)).IsEqualTo("original"); + } + + [Test] + public async Task TerminalCurrentDirectorySegmentDoesNotResolveAsFile() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "terminal-dot.txt"); + var directoryForm = Path.Combine(path, "."); + await provider.WriteAllTextAsync(path, "original"); + + async Task ReadAsync() => _ = await provider.ReadAllTextAsync(directoryForm); + async Task WriteAsync() => await provider.WriteAllTextAsync(directoryForm, "replacement"); + + await Assert.That(provider.FileExists(directoryForm)).IsFalse(); + await Assert.That(ReadAsync).ThrowsException(); + await Assert.That(WriteAsync).ThrowsException(); + await Assert.That(await provider.ReadAllTextAsync(path)).IsEqualTo("original"); + } + + [Test] + public async Task ParentSegmentsCannotTraverseFiles() + { + var provider = new InMemoryFileSystemProvider(); + var root = Path.Combine(provider.GetTempPath(), "file-parent-segment"); + var file = Path.Combine(root, "file"); + var target = Path.Combine(root, "target.txt"); + var malformedPath = Path.Combine(file, "..", "target.txt"); + provider.CreateDirectory(root); + await provider.WriteAllTextAsync(file, "not a directory"); + await provider.WriteAllTextAsync(target, "original"); + + async Task ReadAsync() => _ = await provider.ReadAllTextAsync(malformedPath); + async Task WriteAsync() => await provider.WriteAllTextAsync(malformedPath, "replacement"); + + await Assert.That(provider.FileExists(malformedPath)).IsFalse(); + await Assert.That(ReadAsync).Throws(); + await Assert.That(WriteAsync).Throws(); + await Assert.That(await provider.ReadAllTextAsync(target)).IsEqualTo("original"); + } + + [Test] + public async Task ReadLinesKeepsSharedHandleOpenUntilEnumeratorIsDisposed() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "read-lines-handle.txt"); + await provider.WriteAllLinesAsync(path, ["first", "second"]); + var enumerator = provider.ReadLinesAsync(path).GetAsyncEnumerator(); + + try + { + await Assert.That(await enumerator.MoveNextAsync()).IsTrue(); + await Assert.That(() => provider.DeleteFile(path)).Throws(); + await Assert.That(() => provider.WriteAllTextAsync(path, "replacement")) + .Throws(); + } + finally + { + await enumerator.DisposeAsync(); + } + + await provider.WriteAllTextAsync(path, "replacement"); + await Assert.That(await provider.ReadAllTextAsync(path)).IsEqualTo("replacement"); + } + + [Test] + public async Task OpenReadAllowsSharedReadHandles() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "shared-read.txt"); + await provider.WriteAllTextAsync(path, "contents"); + + using var first = provider.OpenRead(path); + using var second = provider.OpenRead(path); + + using (Assert.Multiple()) + { + await Assert.That(first.CanRead).IsTrue(); + await Assert.That(second.CanRead).IsTrue(); + } + } + + [Test] + public async Task GenericOpenEnforcesExclusiveHandles() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "exclusive.txt"); + await provider.WriteAllTextAsync(path, "contents"); + + using var stream = provider.Open(path, FileMode.Open, FileAccess.Read); + await Assert.That(() => provider.OpenRead(path)).Throws(); + } + + [Test] + public async Task GenericOpenRejectsDirectReadsAndCopies() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "exclusive-direct-read.txt"); + var copy = Path.Combine(provider.GetTempPath(), "exclusive-direct-read-copy.txt"); + await provider.WriteAllTextAsync(path, "contents"); + + using var stream = provider.Open(path, FileMode.Open, FileAccess.Read); + + async Task ReadTextAsync() => _ = await provider.ReadAllTextAsync(path); + async Task ReadBytesAsync() => _ = await provider.ReadAllBytesAsync(path); + + await Assert.That(ReadTextAsync).Throws(); + await Assert.That(ReadBytesAsync).Throws(); + await Assert.That(() => provider.CopyFile(path, copy, overwrite: false)) + .Throws(); + } + + [Test] + public async Task MoveFileRejectsOpenSourceHandles() + { + var provider = new InMemoryFileSystemProvider(); + var source = Path.Combine(provider.GetTempPath(), "open-move-source.txt"); + var destination = Path.Combine(provider.GetTempPath(), "open-move-destination.txt"); + await provider.WriteAllTextAsync(source, "original"); + + await using (var stream = provider.Open(source, FileMode.Open, FileAccess.ReadWrite)) + { + stream.SetLength(0); + await stream.WriteAsync(Encoding.UTF8.GetBytes("updated")); + + await Assert.That(() => provider.MoveFile(source, destination)) + .Throws(); + using (Assert.Multiple()) + { + await Assert.That(provider.FileExists(source)).IsTrue(); + await Assert.That(provider.FileExists(destination)).IsFalse(); + } + } + + provider.MoveFile(source, destination); + using (Assert.Multiple()) + { + await Assert.That(provider.FileExists(source)).IsFalse(); + await Assert.That(await provider.ReadAllTextAsync(destination)).IsEqualTo("updated"); + } + } + + [Test] + public async Task DeleteFileRejectsOpenWritableHandles() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "open-delete.txt"); + await provider.WriteAllTextAsync(path, "original"); + + await using (var stream = provider.Open(path, FileMode.Open, FileAccess.ReadWrite)) + { + stream.SetLength(0); + await stream.WriteAsync(Encoding.UTF8.GetBytes("updated")); + + await Assert.That(() => provider.DeleteFile(path)).Throws(); + await Assert.That(provider.FileExists(path)).IsTrue(); + } + + await Assert.That(await provider.ReadAllTextAsync(path)).IsEqualTo("updated"); + + provider.DeleteFile(path); + await Assert.That(provider.FileExists(path)).IsFalse(); + } + + [Test] + public async Task DeleteDirectoryRejectsOpenDescendantHandles() + { + var provider = new InMemoryFileSystemProvider(); + var root = Path.Combine(provider.GetTempPath(), "open-delete-directory"); + var path = Path.Combine(root, "child", "artifact.txt"); + provider.CreateDirectory(Path.GetDirectoryName(path)!); + await provider.WriteAllTextAsync(path, "original"); + + await using (var stream = provider.Open(path, FileMode.Open, FileAccess.ReadWrite)) + { + stream.SetLength(0); + await stream.WriteAsync(Encoding.UTF8.GetBytes("updated")); + + await Assert.That(() => provider.DeleteDirectory(root, recursive: true)) + .Throws(); + using (Assert.Multiple()) + { + await Assert.That(provider.DirectoryExists(root)).IsTrue(); + await Assert.That(provider.FileExists(path)).IsTrue(); + } + } + + await Assert.That(await provider.ReadAllTextAsync(path)).IsEqualTo("updated"); + provider.DeleteDirectory(root, recursive: true); + await Assert.That(provider.DirectoryExists(root)).IsFalse(); + } + + [Test] + public async Task OpenHandlesRejectDirectWrites() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "direct-write.txt"); + await provider.WriteAllTextAsync(path, "original"); + + using (provider.OpenRead(path)) + { + await Assert.That(() => provider.WriteAllTextAsync(path, "replacement")) + .Throws(); + await Assert.That(() => provider.AppendAllTextAsync(path, " appended")) + .Throws(); + } + + await Assert.That(await provider.ReadAllTextAsync(path)).IsEqualTo("original"); + } + + [Test] + public async Task MovesDirectoryTrees() + { + var provider = new InMemoryFileSystemProvider(); + var source = Path.Combine(provider.GetTempPath(), "source"); + var child = Path.Combine(source, "child"); + var destination = Path.Combine(provider.GetTempPath(), "destination"); + var sourceFile = Path.Combine(child, "artifact.txt"); + var destinationFile = Path.Combine(destination, "child", "artifact.txt"); + + provider.CreateDirectory(child); + await provider.WriteAllTextAsync(sourceFile, "artifact"); + provider.MoveDirectory(source, destination); + + using (Assert.Multiple()) + { + await Assert.That(provider.DirectoryExists(source)).IsFalse(); + await Assert.That(provider.DirectoryExists(Path.Combine(destination, "child"))).IsTrue(); + await Assert.That(await provider.ReadAllTextAsync(destinationFile)).IsEqualTo("artifact"); + } + } + + [Test] + public async Task MoveDirectoryRejectsOpenDescendantHandlesAtomically() + { + var provider = new InMemoryFileSystemProvider(); + var source = Path.Combine(provider.GetTempPath(), "open-move-directory"); + var destination = Path.Combine(provider.GetTempPath(), "moved-directory"); + var firstFile = Path.Combine(source, "first.txt"); + var openFile = Path.Combine(source, "second.txt"); + provider.CreateDirectory(source); + await provider.WriteAllTextAsync(firstFile, "first"); + await provider.WriteAllTextAsync(openFile, "second"); + + await using (provider.Open(openFile, FileMode.Open, FileAccess.ReadWrite)) + { + await Assert.That(() => provider.MoveDirectory(source, destination)) + .Throws(); + using (Assert.Multiple()) + { + await Assert.That(provider.DirectoryExists(source)).IsTrue(); + await Assert.That(provider.DirectoryExists(destination)).IsFalse(); + await Assert.That(provider.FileExists(firstFile)).IsTrue(); + await Assert.That(provider.FileExists(openFile)).IsTrue(); + } + } + + provider.MoveDirectory(source, destination); + using (Assert.Multiple()) + { + await Assert.That(provider.DirectoryExists(source)).IsFalse(); + await Assert.That(provider.FileExists(Path.Combine(destination, "first.txt"))).IsTrue(); + await Assert.That(provider.FileExists(Path.Combine(destination, "second.txt"))).IsTrue(); + } + } + + [Test] + public async Task OpenOrCreateWithReadAccessCreatesAnEmptyFile() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "created-on-open.txt"); + + using var stream = provider.Open(path, FileMode.OpenOrCreate, FileAccess.Read); + + using (Assert.Multiple()) + { + await Assert.That(stream.Length).IsEqualTo(0); + await Assert.That(provider.FileExists(path)).IsTrue(); + } + } + + [Test] + public async Task RejectsMovingDirectoryInsideItself() + { + var provider = new InMemoryFileSystemProvider(); + var source = Path.Combine(provider.GetTempPath(), "source"); + var destination = Path.Combine(source, "nested"); + provider.CreateDirectory(source); + + await Assert.That(() => provider.MoveDirectory(source, destination)) + .Throws(); + } + + [Test] + public async Task EnumeratesFilesUnderFileSystemRoot() + { + var provider = new InMemoryFileSystemProvider(); + var root = Path.GetPathRoot(Environment.CurrentDirectory)!; + var path = Path.Combine(root, $"root-{Guid.NewGuid():N}.txt"); + await provider.WriteAllTextAsync(path, "contents"); + + await Assert.That(provider.EnumerateFiles(root, "*.txt", SearchOption.AllDirectories)) + .Contains(path); + } + + [Test] + [Arguments(SearchOption.TopDirectoryOnly, 1)] + [Arguments(SearchOption.AllDirectories, 2)] + public async Task FileEnumerationHonorsDirectoriesInSearchPatterns( + SearchOption searchOption, + int expectedCount) + { + var provider = new InMemoryFileSystemProvider(); + var root = Path.Combine(provider.GetTempPath(), "file-pattern-directory"); + var subdirectory = Path.Combine(root, "sub"); + var nested = Path.Combine(subdirectory, "nested"); + provider.CreateDirectory(nested); + await provider.WriteAllTextAsync(Path.Combine(subdirectory, "first.json"), "first"); + await provider.WriteAllTextAsync(Path.Combine(nested, "second.json"), "second"); + + var entries = provider.EnumerateFiles( + root, + Path.Combine("sub", "*.json"), + searchOption); + + await Assert.That(entries.Count()).IsEqualTo(expectedCount); + } + + [Test] + [Arguments(SearchOption.TopDirectoryOnly, 1)] + [Arguments(SearchOption.AllDirectories, 2)] + public async Task DirectoryEnumerationHonorsDirectoriesInSearchPatterns( + SearchOption searchOption, + int expectedCount) + { + var provider = new InMemoryFileSystemProvider(); + var root = Path.Combine(provider.GetTempPath(), "directory-pattern-directory"); + var subdirectory = Path.Combine(root, "sub"); + provider.CreateDirectory(Path.Combine(subdirectory, "target-one")); + provider.CreateDirectory(Path.Combine(subdirectory, "nested", "target-two")); + + var entries = provider.EnumerateDirectories( + root, + Path.Combine("sub", "target-*"), + searchOption); + + await Assert.That(entries.Count()).IsEqualTo(expectedCount); + } + + [Test] + public async Task UnixBackslashesDoNotCreateDescendantRelationships() + { + if (OperatingSystem.IsWindows()) + { + return; + } + + var provider = new InMemoryFileSystemProvider(); + var root = Path.Combine(provider.GetTempPath(), "root"); + var sibling = $"{root}\\outside"; + var siblingFile = Path.Combine(sibling, "artifact.txt"); + provider.CreateDirectory(root); + provider.CreateDirectory(sibling); + await provider.WriteAllTextAsync(siblingFile, "contents"); + + await Assert.That(provider.EnumerateFiles(root, "*", SearchOption.AllDirectories)) + .DoesNotContain(siblingFile); + } + + [Test] + public async Task ConcurrentAppendsDoNotLoseWrites() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "concurrent.txt"); + + await Task.WhenAll(Enumerable.Range(0, 100) + .Select(_ => provider.AppendAllTextAsync(path, "x"))); + + await Assert.That((await provider.ReadAllTextAsync(path)).Length).IsEqualTo(100); + } + + [Test] + public async Task RejectsFilesWithoutExistingParent() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "missing", "artifact.txt"); + + await Assert.That(() => provider.WriteAllTextAsync(path, "contents")) + .Throws(); + } + + [Test] + public async Task RejectsPlatformInvalidFileNameCharacters() + { + if (!OperatingSystem.IsWindows()) + { + return; + } + + var provider = new InMemoryFileSystemProvider(); + var invalidFile = Path.Combine(provider.GetTempPath(), "artifact?.txt"); + var invalidDirectory = Path.Combine(provider.GetTempPath(), "artifacts*"); + + using (Assert.Multiple()) + { + await Assert.That(() => provider.WriteAllTextAsync(invalidFile, "contents")) + .Throws(); + await Assert.That(() => provider.CreateDirectory(invalidDirectory)) + .Throws(); + } + } + + [Test] + public async Task RejectsFileAndDirectoryPathCollisions() + { + var provider = new InMemoryFileSystemProvider(); + var filePath = Path.Combine(provider.GetTempPath(), "artifact"); + await provider.WriteAllTextAsync(filePath, "contents"); + var nestedDirectoryPath = Path.Combine(filePath, "child"); + + using (Assert.Multiple()) + { + await Assert.That(() => provider.CreateDirectory(nestedDirectoryPath)) + .Throws(); + await Assert.That(provider.DirectoryExists(nestedDirectoryPath)).IsFalse(); + } + + var directoryPath = Path.Combine(provider.GetTempPath(), "directory"); + provider.CreateDirectory(directoryPath); + + await Assert.That(() => provider.WriteAllTextAsync(directoryPath, "contents")) + .Throws(); + } + + [Test] + public async Task RejectsDirectoryFormFileDestinations() + { + var provider = new InMemoryFileSystemProvider(); + var directoryPath = Path.Combine(provider.GetTempPath(), "directory-form"); + var directoryForm = directoryPath + Path.DirectorySeparatorChar; + var sourcePath = Path.Combine(provider.GetTempPath(), "source.txt"); + provider.CreateDirectory(directoryPath); + await provider.WriteAllTextAsync(sourcePath, "contents"); + + using (Assert.Multiple()) + { + await Assert.That(() => provider.WriteAllTextAsync(directoryForm, "contents")) + .Throws(); + await Assert.That(() => provider.Create(directoryForm)).Throws(); + await Assert.That(() => provider.CopyFile(sourcePath, directoryForm, overwrite: true)) + .Throws(); + await Assert.That(provider.DirectoryExists(directoryPath)).IsTrue(); + await Assert.That(provider.FileExists(directoryForm)).IsFalse(); + } + } + + [Test] + public async Task MoveDirectoryRequiresExistingDestinationParent() + { + var provider = new InMemoryFileSystemProvider(); + var source = Path.Combine(provider.GetTempPath(), "source"); + var destination = Path.Combine(provider.GetTempPath(), "missing", "destination"); + provider.CreateDirectory(source); + + await Assert.That(() => provider.MoveDirectory(source, destination)) + .Throws(); + } + + [Test] + public async Task EmptyLineSequenceWritesEmptyFile() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "empty.txt"); + + await provider.WriteAllLinesAsync(path, []); + + await Assert.That(await provider.ReadAllTextAsync(path)).IsEmpty(); + } + + [Test] + public async Task WriteOnlyStreamsRejectReads() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "write-only.txt"); + + using var stream = provider.Open(path, FileMode.Create, FileAccess.Write); + + using (Assert.Multiple()) + { + await Assert.That(() => stream.ReadByte()).Throws(); + await Assert.That(async () => await stream.ReadAsync(new byte[1])) + .Throws(); + } + } + + [Test] + public async Task AppendStreamsRejectSeekingBeforeOriginalEnd() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "append.txt"); + await provider.WriteAllTextAsync(path, "original"); + + using var stream = provider.Open(path, FileMode.Append, FileAccess.Write); + + await Assert.That(() => stream.Position = 0).Throws(); + } + + [Test] + public async Task RegularStreamsPreserveNegativeArgumentExceptions() + { + var provider = new InMemoryFileSystemProvider(); + var path = Path.Combine(provider.GetTempPath(), "regular.txt"); + + using var stream = provider.Open(path, FileMode.Create, FileAccess.ReadWrite); + + using (Assert.Multiple()) + { + await Assert.That(() => stream.Position = -1) + .Throws(); + await Assert.That(() => stream.SetLength(-1)) + .Throws(); + } + } +} diff --git a/test/ModularPipelines.Testing.UnitTests/ModularPipelines.Testing.UnitTests.csproj b/test/ModularPipelines.Testing.UnitTests/ModularPipelines.Testing.UnitTests.csproj new file mode 100644 index 00000000000..afb80b2235a --- /dev/null +++ b/test/ModularPipelines.Testing.UnitTests/ModularPipelines.Testing.UnitTests.csproj @@ -0,0 +1,13 @@ + + + net10.0 + false + Exe + + + + + + + + diff --git a/test/ModularPipelines.Testing.UnitTests/ModuleTesterTests.cs b/test/ModularPipelines.Testing.UnitTests/ModuleTesterTests.cs new file mode 100644 index 00000000000..e7a6724e506 --- /dev/null +++ b/test/ModularPipelines.Testing.UnitTests/ModuleTesterTests.cs @@ -0,0 +1,723 @@ +using ModularPipelines.Configuration; +using ModularPipelines.Context; +using ModularPipelines.Engine; +using ModularPipelines.Exceptions; +using ModularPipelines.FileSystem; +using ModularPipelines.Models; +using ModularPipelines.Modules; +using ModularPipelines.Options; + +namespace ModularPipelines.Testing.UnitTests; + +public class ModuleTesterTests +{ + private const string InterceptedSecret = "intercepted-command-secret"; + + [Test] + public async Task ExecutesModuleAndReturnsTypedValue() + { + var run = await ModuleTester.For().ExecuteAsync(); + + using (Assert.Multiple()) + { + await Assert.That(run.Value).IsEqualTo("value"); + await Assert.That(run.Result).IsTypeOf.Success>(); + await Assert.That(run.Exception).IsNull(); + await Assert.That(run.SkipDecision).IsNull(); + } + } + + [Test] + public async Task ProposalCompatibleApiSeedsDependencyResult() + { + var run = await ModuleTester.For() + .WithDependencyResult("seeded") + .ExecuteAsync(); + + await Assert.That(run.Value).IsEqualTo("seeded consumed"); + } + + [Test] + public async Task InterceptsAndRecordsParsedCommands() + { + var run = await ModuleTester.For() + .InterceptCommands(invocation => + CommandResult.Ok($"stubbed {invocation.CommandLine.Tool}")) + .ExecuteAsync(); + + using (Assert.Multiple()) + { + await Assert.That(run.Value).IsEqualTo("stubbed imaginary-tool"); + await Assert.That(run.Commands).Count().IsEqualTo(1); + await Assert.That(run.Commands[0].CommandLine.Tool).IsEqualTo("imaginary-tool"); + await Assert.That(run.Commands[0].CommandLine.Arguments) + .IsEquivalentTo(["build", "--configuration", "Release"]); + await Assert.That(run.Commands[0].Result.StandardOutput) + .IsEqualTo("stubbed imaginary-tool"); + await Assert.That(run.Commands[0].Result.WorkingDirectory) + .IsEqualTo(Path.GetTempPath()); + await Assert.That(run.Commands[0].Result.EnvironmentVariables["RECORDED_VALUE"]) + .IsEqualTo("effective"); + } + } + + [Test] + public async Task UsesSafeDefaultCommandStub() + { + var run = await ModuleTester.For().ExecuteAsync(); + + using (Assert.Multiple()) + { + await Assert.That(run.Commands).Count().IsEqualTo(1); + await Assert.That(run.Commands[0].Result.ExitCode).IsEqualTo(0); + await Assert.That(run.Exception).IsNull(); + } + } + + [Test] + public async Task InterceptedCommandHonorsThrowOnNonZeroExitCode() + { + var run = await ModuleTester.For() + .InterceptCommands(_ => CommandResult.Ok(standardError: "failed") with + { + ExitCode = 1, + }) + .ExecuteAsync(); + + await Assert.That(run.Exception).IsTypeOf(); + } + + [Test] + public async Task InterceptedCommandFailureObfuscatesExceptionResult() + { + var run = await ModuleTester.For() + .WithService(new TestSecretObfuscator()) + .InterceptCommands(_ => CommandResult.Ok(InterceptedSecret, InterceptedSecret) with + { + ExitCode = 1, + }) + .ExecuteAsync(); + + var exception = (CommandException) run.Exception!; + using (Assert.Multiple()) + { + await Assert.That(exception.Result.CommandInput).DoesNotContain(InterceptedSecret); + await Assert.That(exception.Result.StandardOutput).DoesNotContain(InterceptedSecret); + await Assert.That(exception.Result.StandardError).DoesNotContain(InterceptedSecret); + await Assert.That(exception.Result.EnvironmentVariables["SECRET_VALUE"]) + .DoesNotContain(InterceptedSecret); + } + } + + [Test] + [Timeout(5_000)] + public async Task InterceptedCommandHonorsExecutionTimeout( + CancellationToken cancellationToken) + { + var run = await ModuleTester.For() + .InterceptCommands(async (_, token) => + { + await Task.Delay(Timeout.InfiniteTimeSpan, token); + return CommandResult.Ok(); + }) + .ExecuteAsync(cancellationToken); + + await Assert.That(run.Exception).IsTypeOf(); + } + + [Test] + [Timeout(5_000)] + public async Task SynchronousInterceptorCannotReturnAfterExecutionTimeout( + CancellationToken cancellationToken) + { + var run = await ModuleTester.For() + .InterceptCommands(_ => + { + Thread.Sleep(200); + return CommandResult.Ok(); + }) + .ExecuteAsync(cancellationToken); + + await Assert.That(run.Exception).IsTypeOf(); + } + + [Test] + [Timeout(5_000)] + public async Task MissingRequiredDependencyFailsFast(CancellationToken cancellationToken) + { + async Task Act() => + _ = await ModuleTester.For() + .ExecuteAsync(cancellationToken); + + var exception = await Assert.That(Act) + .Throws(); + + await Assert.That(exception!.Message).Contains(nameof(DependencyModule)); + } + + [Test] + public async Task ReportsSkipDecisionWithoutExecutingModule() + { + var run = await ModuleTester.For().ExecuteAsync(); + + using (Assert.Multiple()) + { + await Assert.That(run.SkipDecision).IsNotNull(); + await Assert.That(run.SkipDecision!.Reason).IsEqualTo("not needed"); + await Assert.That(run.Result).IsTypeOf.Skipped>(); + } + } + + [Test] + public async Task CapturesModuleFailure() + { + var run = await ModuleTester.For().ExecuteAsync(); + + using (Assert.Multiple()) + { + await Assert.That(run.Exception).IsTypeOf(); + await Assert.That(run.Result).IsTypeOf.Failure>(); + } + } + + [Test] + public async Task ProvidesIsolatedInMemoryFileSystem() + { + var physicalPath = Path.Combine( + Path.GetTempPath(), + $"modular-pipelines-test-{Guid.NewGuid():N}", + "artifact.txt"); + + var run = await ModuleTester.For() + .WithService(new FilePath(physicalPath)) + .ExecuteAsync(); + + using (Assert.Multiple()) + { + await Assert.That(run.Value).IsEqualTo("contents"); + await Assert.That(run.FileSystem.FileExists(physicalPath)).IsTrue(); + await Assert.That(System.IO.File.Exists(physicalPath)).IsFalse(); + } + } + + [Test] + public async Task VirtualizesFolderEnumerationAndCleaning() + { + var root = Path.Combine( + Path.GetTempPath(), + $"modular-pipelines-test-{Guid.NewGuid():N}"); + + var run = await ModuleTester.For() + .WithService(new FilePath(root)) + .ExecuteAsync(); + + using (Assert.Multiple()) + { + await Assert.That(run.Value).IsEqualTo("1:2:0"); + await Assert.That(Directory.Exists(root)).IsFalse(); + } + } + + [Test] + public async Task UnsupportedVirtualMetadataFailsLoudly() + { + var root = Path.Combine( + Path.GetTempPath(), + $"modular-pipelines-test-{Guid.NewGuid():N}"); + + var run = await ModuleTester.For() + .WithService(new FilePath(Path.Combine(root, "artifact.txt"))) + .ExecuteAsync(); + + await Assert.That(run.Exception).IsTypeOf(); + } + + [Test] + public async Task CopiesFoldersInVirtualFileSystem() + { + var root = Path.Combine( + Path.GetTempPath(), + $"modular-pipelines-test-{Guid.NewGuid():N}"); + + var run = await ModuleTester.For() + .WithService(new FilePath(root)) + .ExecuteAsync(); + + using (Assert.Multiple()) + { + await Assert.That(run.Value).IsEqualTo("contents"); + await Assert.That(run.Exception).IsNull(); + await Assert.That(Directory.Exists(root)).IsFalse(); + } + } + + [Test] + public async Task ConcurrentCommandsPreserveInvocationOrder() + { + var releaseFirstCommand = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var run = await ModuleTester.For() + .InterceptCommands(async (invocation, cancellationToken) => + { + if (invocation.CommandLine.Tool == "first-tool") + { + await releaseFirstCommand.Task.WaitAsync(cancellationToken); + } + else + { + releaseFirstCommand.TrySetResult(); + } + + return CommandResult.Ok(invocation.CommandLine.Tool); + }) + .ExecuteAsync(); + + using (Assert.Multiple()) + { + await Assert.That(run.Commands).Count().IsEqualTo(2); + await Assert.That(run.Commands[0].CommandLine.Tool).IsEqualTo("first-tool"); + await Assert.That(run.Commands[1].CommandLine.Tool).IsEqualTo("second-tool"); + } + } + + [Test] + public async Task ReturnsOverriddenFileSystemProvider() + { + var fileSystem = new InMemoryFileSystemProvider(); + var path = Path.Combine(fileSystem.GetTempPath(), "overridden-provider.txt"); + + var run = await ModuleTester.For() + .WithService(fileSystem) + .WithService(new FilePath(path)) + .ExecuteAsync(); + + using (Assert.Multiple()) + { + await Assert.That(run.Value).IsEqualTo("contents"); + await Assert.That(run.FileSystem).IsSameReferenceAs(fileSystem); + await Assert.That(fileSystem.FileExists(path)).IsTrue(); + } + } + + [Test] + public async Task VirtualizesCompleteFilesContext() + { + var relativeRoot = $"module-tester-{Guid.NewGuid():N}"; + var physicalRoot = Path.Combine(Environment.CurrentDirectory, relativeRoot); + + var run = await ModuleTester.For() + .WithService(new FilePath(relativeRoot)) + .ExecuteAsync(); + + using (Assert.Multiple()) + { + await Assert.That(run.Value).IsEqualTo("contents:1:1:1:1"); + await Assert.That(run.Exception).IsNull(); + await Assert.That(run.FileSystem.FileExists( + Path.Combine(physicalRoot, "artifact.txt"))).IsTrue(); + await Assert.That(run.FileSystem.FileExists($"{physicalRoot}.zip")).IsTrue(); + await Assert.That(run.FileSystem.FileExists( + Path.Combine($"{physicalRoot}-unzipped", "artifact.txt"))).IsTrue(); + await Assert.That(Directory.Exists(physicalRoot)).IsFalse(); + await Assert.That(System.IO.File.Exists($"{physicalRoot}.zip")).IsFalse(); + } + } + + [Test] + public async Task RootMatchingExclusionDoesNotHideDescendants() + { + var root = Path.Combine( + Path.GetTempPath(), + $"root-exclusion-{Guid.NewGuid():N}"); + + var run = await ModuleTester.For() + .WithService(new FilePath(root)) + .ExecuteAsync(); + + await Assert.That(run.Value).IsEqualTo(2); + } + + [Test] + [Timeout(5_000)] + public async Task CallerCancellationStopsRunningModule(CancellationToken cancellationToken) + { + var started = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + using var cancellationTokenSource = + CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var executionTask = ModuleTester.For() + .WithService(started) + .ExecuteAsync(cancellationTokenSource.Token); + + await started.Task.WaitAsync(cancellationToken); + await cancellationTokenSource.CancelAsync(); + var run = await executionTask.WaitAsync(cancellationToken); + + await Assert.That(run.Exception).IsTypeOf(); + } + + [Test] + [Timeout(5_000)] + public async Task CallerCancellationStopsAlwaysRunModule(CancellationToken cancellationToken) + { + var started = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + using var cancellationTokenSource = + CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + var executionTask = ModuleTester.For() + .WithService(started) + .ExecuteAsync(cancellationTokenSource.Token); + + await started.Task.WaitAsync(cancellationToken); + await cancellationTokenSource.CancelAsync(); + var run = await executionTask.WaitAsync(cancellationToken); + + await Assert.That(run.Exception).IsTypeOf(); + } + + [Test] + public async Task ZipOutputHandlingUsesVirtualFileSystem() + { + var root = Path.Combine( + Path.GetTempPath(), + $"zip-output-{Guid.NewGuid():N}"); + var hostCollision = Path.Combine(root, "host-collision.zip"); + Directory.CreateDirectory(hostCollision); + + try + { + var run = await ModuleTester.For() + .WithService(new FilePath(root)) + .ExecuteAsync(); + + await Assert.That(run.Value).IsEqualTo("True:True:True"); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + public sealed class ValueModule : Module + { + protected override Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + => Task.FromResult("value"); + } + + public sealed class DependencyModule : Module + { + protected override Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + => throw new InvalidOperationException("Seeded dependency must not execute."); + } + + public sealed class DependentModule : Module + { + protected override ModuleConfiguration Configure() => ModuleConfiguration.Create() + .DependsOn() + .Build(); + + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var dependency = await context.GetModule(); + return $"{dependency.ValueOrDefault} consumed"; + } + } + + [ModularPipelines.Attributes.DependsOn] + public sealed class AttributedDependentModule : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var dependency = await context.GetModule(); + return $"{dependency.ValueOrDefault} consumed"; + } + } + + public sealed class CommandModule : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var result = await context.Shell.Command.ExecuteCommandLineToolAsync( + new GenericCommandLineToolOptions("imaginary-tool") + { + Arguments = ["build", "--configuration", "Release"], + }, + new CommandExecutionOptions + { + WorkingDirectory = Path.GetTempPath(), + EnvironmentVariables = new Dictionary + { + ["RECORDED_VALUE"] = "effective", + }, + }, + cancellationToken: cancellationToken); + + return result.StandardOutput; + } + } + + public sealed class SkippedModule : Module + { + protected override ModuleConfiguration Configure() => ModuleConfiguration.Create() + .WithSkipWhen(_ => SkipDecision.Skip("not needed")) + .Build(); + + protected override Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + => throw new InvalidOperationException("Skipped module must not execute."); + } + + public sealed class FailingModule : Module + { + protected override Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + => throw new InvalidOperationException("expected"); + } + + public sealed class FileModule(FilePath filePath) : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var file = context.Files.GetFile(filePath.Value); + file.Folder!.Create(); + await file.WriteAsync("contents", cancellationToken); + return file.Exists ? await file.ReadAsync(cancellationToken) : null; + } + } + + public sealed class FolderModule(FilePath rootPath) : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var folder = context.Files.GetFolder(rootPath.Value).Create(); + await folder.GetFile("first.txt").WriteAsync("first", cancellationToken); + var nested = folder.CreateFolder("nested"); + await nested.GetFile("second.txt").WriteAsync("second", cancellationToken); + + var listedCount = folder.ListFiles().Count(); + var recursiveCount = folder.GetFiles(file => file.Extension == ".txt").Count(); + folder.Clean(); + return $"{listedCount}:{recursiveCount}:{folder.ListFiles().Count()}"; + } + } + + public sealed class FileMetadataModule(FilePath filePath) : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var file = context.Files.GetFile(filePath.Value); + file.Folder!.Create(); + await file.WriteAsync("contents", cancellationToken); + return file.Length; + } + } + + public sealed class FolderCopyModule(FilePath rootPath) : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var root = context.Files.GetFolder(rootPath.Value).Create(); + var source = root.CreateFolder("source"); + await source.GetFile("artifact.txt").WriteAsync("contents", cancellationToken); + + var copy = source.CopyTo(root.GetFolder("copy").Path); + return await copy.GetFile("artifact.txt").ReadAsync(cancellationToken); + } + } + + public sealed class SecretCommandModule : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var result = await context.Shell.Command.ExecuteCommandLineToolAsync( + new GenericCommandLineToolOptions("imaginary-tool") + { + Arguments = [InterceptedSecret], + }, + new CommandExecutionOptions + { + EnvironmentVariables = new Dictionary + { + ["SECRET_VALUE"] = InterceptedSecret, + }, + }, + cancellationToken: cancellationToken); + + return result.StandardOutput; + } + } + + private sealed class TestSecretObfuscator : ISecretObfuscator + { + public string Obfuscate(string? input, object? optionsObject) => + input?.Replace(InterceptedSecret, "********", StringComparison.Ordinal) + ?? string.Empty; + } + + public sealed class ConcurrentCommandModule : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var first = context.Shell.Command.ExecuteCommandLineToolAsync( + new GenericCommandLineToolOptions("first-tool"), + cancellationToken: cancellationToken); + var second = context.Shell.Command.ExecuteCommandLineToolAsync( + new GenericCommandLineToolOptions("second-tool"), + cancellationToken: cancellationToken); + + var results = await Task.WhenAll(first, second); + return string.Join(',', results.Select(result => result.StandardOutput)); + } + } + + public sealed class TimedCommandModule : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var result = await context.Shell.Command.ExecuteCommandLineToolAsync( + new GenericCommandLineToolOptions("imaginary-tool"), + new CommandExecutionOptions + { + ExecutionTimeout = TimeSpan.FromMilliseconds(100), + }, + cancellationToken: cancellationToken); + + return result.StandardOutput; + } + } + + public sealed class FilesContextModule(FilePath rootPath) : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var root = context.Files.GetFolder(rootPath.Value).Create(); + var filePath = Path.Combine(rootPath.Value, "artifact.txt"); + await context.Files.WriteAsync(filePath, "contents", cancellationToken); + var contents = await context.Files.ReadAsync(filePath, cancellationToken); + var exists = await context.Files.ExistsAsync(filePath, cancellationToken); + var files = context.Files.Glob($"{rootPath.Value}/**/*.txt").Count(); + var folders = context.Files.GlobFolders(rootPath.Value).Count(); + var checksum = context.Files.Checksum.Md5(filePath); + + var zipPath = Path.GetFullPath($"{rootPath.Value}.zip"); + context.Files.Zip.ZipFolder(root, zipPath); + context.Files.Zip.UnZipToFolder( + zipPath, + Path.GetFullPath($"{rootPath.Value}-unzipped")); + + return $"{contents}:{(exists ? 1 : 0)}:{files}:{folders}:{checksum.Length / 32}"; + } + } + + public sealed class FolderExclusionModule(FilePath rootPath) : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var root = context.Files.GetFolder(rootPath.Value).Create(); + await root.GetFile("root.txt").WriteAsync("root", cancellationToken); + await root.CreateFolder("nested") + .GetFile("nested.txt") + .WriteAsync("nested", cancellationToken); + + return root.GetFiles( + _ => true, + candidate => candidate.Path == root.Path).Count(); + } + } + + public sealed class CancellableModule(TaskCompletionSource started) : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + started.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return "unreachable"; + } + } + + public sealed class CancellableAlwaysRunModule(TaskCompletionSource started) : Module + { + protected override ModuleConfiguration Configure() => ModuleConfiguration.Create() + .WithAlwaysRun() + .Build(); + + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + started.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return "unreachable"; + } + } + + public sealed class ZipOutputModule(FilePath rootPath) : Module + { + protected override async Task ExecuteAsync( + IModuleContext context, + CancellationToken cancellationToken) + { + var root = context.Files.GetFolder(rootPath.Value).Create(); + var source = root.CreateFolder("source"); + await source.GetFile("artifact.txt").WriteAsync("contents", cancellationToken); + + var dottedDirectory = root.CreateFolder("archives.v1"); + var directoryZip = context.Files.Zip.ZipFolder(source, dottedDirectory.Path); + + var extensionlessFile = root.GetFile("archive"); + await extensionlessFile.WriteAsync("existing", cancellationToken); + var rejectedExistingFile = false; + try + { + context.Files.Zip.ZipFolder(source, extensionlessFile.Path); + } + catch (IOException) + { + rejectedExistingFile = true; + } + + var preservedExistingFile = + await extensionlessFile.ReadAsync(cancellationToken) == "existing"; + var collidingOutput = root.GetFile("host-collision.zip"); + var isolatedZip = context.Files.Zip.ZipFolder(source, collidingOutput.Path); + + return $"{directoryZip.Folder?.Path == dottedDirectory.Path}:" + + $"{rejectedExistingFile && preservedExistingFile}:" + + $"{isolatedZip.Path == collidingOutput.Path}"; + } + } + + public sealed record FilePath(string Value); +} diff --git a/test/ModularPipelines.UnitTests/Helpers/ZipTests.cs b/test/ModularPipelines.UnitTests/Helpers/ZipTests.cs index 4bfa7714af9..17062bc6a8f 100644 --- a/test/ModularPipelines.UnitTests/Helpers/ZipTests.cs +++ b/test/ModularPipelines.UnitTests/Helpers/ZipTests.cs @@ -1,9 +1,12 @@ +using System.IO.Compression; using ModularPipelines.Context; +using ModularPipelines.FileSystem; using ModularPipelines.Git.Extensions; using ModularPipelines.Models; using ModularPipelines.Modules; using ModularPipelines.TestHelpers; using ModularPipelines.TestHelpers.Assertions; +using Moq; namespace ModularPipelines.UnitTests.Helpers; @@ -157,4 +160,129 @@ public async Task UnZipped_Folder_Exists() await Assert.That(expectedFolder.GetFiles("*", SearchOption.AllDirectories)).Count().IsEqualTo(1); } } + + [Test] + public async Task PhysicalProviderPreservesFileLastWriteTimes() + { + var root = Path.Combine( + Path.GetTempPath(), + $"ModularPipelines-zip-timestamp-{Guid.NewGuid():N}"); + var sourceDirectory = Path.Combine(root, "source"); + var sourceFile = Path.Combine(sourceDirectory, "artifact.txt"); + var zipPath = Path.Combine(root, "artifact.zip"); + var destinationDirectory = Path.Combine(root, "destination"); + var extractedFile = Path.Combine(destinationDirectory, "artifact.txt"); + var expectedTimestamp = new DateTime(2020, 1, 2, 3, 4, 6); + Directory.CreateDirectory(sourceDirectory); + + try + { + await System.IO.File.WriteAllTextAsync(sourceFile, "contents"); + System.IO.File.SetLastWriteTime(sourceFile, expectedTimestamp); + var zip = new Zip(SystemFileSystemProvider.Instance); + + zip.ZipFolder(new Folder(sourceDirectory), zipPath, CompressionLevel.Optimal); + zip.UnZipToFolder(zipPath, destinationDirectory, overwriteFiles: true); + + await Assert.That(System.IO.File.GetLastWriteTime(extractedFile)) + .IsEqualTo(expectedTimestamp); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [Test] + public async Task ZipFolderRejectsExistingOutputWithoutChangingIt() + { + var root = Path.Combine( + Path.GetTempPath(), + $"ModularPipelines-existing-zip-{Guid.NewGuid():N}"); + var sourceDirectory = Path.Combine(root, "source"); + var sourceFile = Path.Combine(sourceDirectory, "artifact.txt"); + var zipPath = Path.Combine(root, "artifact.zip"); + Directory.CreateDirectory(sourceDirectory); + + try + { + await System.IO.File.WriteAllTextAsync(sourceFile, "contents"); + await System.IO.File.WriteAllTextAsync(zipPath, "existing"); + var zip = new Zip(SystemFileSystemProvider.Instance); + + await Assert.That(() => + zip.ZipFolder( + new Folder(sourceDirectory), + zipPath, + CompressionLevel.Optimal)) + .Throws(); + await Assert.That(await System.IO.File.ReadAllTextAsync(zipPath)) + .IsEqualTo("existing"); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + + [Test] + public async Task UnZipDoesNotOverwriteFileCreatedAfterExistenceCheck() + { + var root = Path.Combine( + Path.GetTempPath(), + $"ModularPipelines-unzip-race-{Guid.NewGuid():N}"); + var zipPath = Path.Combine(root, "artifact.zip"); + var destinationDirectory = Path.Combine(root, "destination"); + var destinationPath = Path.Combine(destinationDirectory, "artifact.txt"); + Directory.CreateDirectory(root); + + try + { + using (var archive = ZipFile.Open(zipPath, ZipArchiveMode.Create)) + { + await using var writer = new StreamWriter(archive.CreateEntry("artifact.txt").Open()); + await writer.WriteAsync("archive contents"); + } + + var competingFileCreated = false; + var fileSystemProvider = new Mock(MockBehavior.Strict); + fileSystemProvider + .Setup(provider => provider.FileExists(It.IsAny())) + .Returns((string path) => + { + if (path == destinationPath && !competingFileCreated) + { + System.IO.File.WriteAllText(destinationPath, "competing contents"); + competingFileCreated = true; + return false; + } + + return System.IO.File.Exists(path); + }); + fileSystemProvider + .Setup(provider => provider.CreateDirectory(It.IsAny())) + .Callback((string path) => Directory.CreateDirectory(path)); + fileSystemProvider + .Setup(provider => provider.OpenRead(zipPath)) + .Returns(() => System.IO.File.OpenRead(zipPath)); + fileSystemProvider + .Setup(provider => provider.Open( + It.IsAny(), + It.IsAny(), + It.IsAny())) + .Returns((string path, FileMode mode, FileAccess access) => + System.IO.File.Open(path, mode, access, FileShare.None)); + var zip = new Zip(fileSystemProvider.Object); + + await Assert.That(() => + zip.UnZipToFolder(zipPath, destinationDirectory, overwriteFiles: false)) + .Throws(); + await Assert.That(await System.IO.File.ReadAllTextAsync(destinationPath)) + .IsEqualTo("competing contents"); + } + finally + { + Directory.Delete(root, recursive: true); + } + } }