Add fingerprint-based module caching

docs/docs/how-to/module-caching.md

@@ -0,0 +1,104 @@
+---
+title: Cache Module Results
+---
+
+# Cache Module Results
+
+Fingerprint-based module caching skips work when its declared inputs have not changed. A cache entry contains the typed module result and files declared with `ProducesArtifact`, so a hit can restore outputs before dependent modules start.
+
+## Enable a local cache
+
+Register the filesystem backend once:
+
+```csharp
+using ModularPipelines.Caching;
+using ModularPipelines.Extensions;
+
+builder.AddModuleCache<FileSystemModuleCache>();
+```
+
+Then declare every file input that can affect a module:
+
+```csharp
+[CacheInputs("src/**/*.cs", "*.csproj")]
+[ProducesArtifact("application", "artifacts/publish/**/*")]
+public sealed class BuildModule : Module<BuildOutput>
+{
+    protected override Task<BuildOutput?> ExecuteAsync(
+        IModuleContext context,
+        CancellationToken cancellationToken)
+    {
+        // Build the application.
+    }
+}
+```
+
+Patterns are relative to `ModuleCacheOptions.WorkingDirectory`. They support `*`, `?`, and recursive `**` wildcards.
+
+The fingerprint contains:
+
+- the module type and assembly version;
+- the content and relative path of every declared input;
+- explicit key parts and declared environment variable values;
+- normalized results from all direct, selector-based, and dynamic dependencies.
+
+A hit has `Status.UsedHistory`. Cache lookup or storage failures are logged and the module executes normally.
+
+## Include configuration
+
+File contents do not represent every input. Add configuration or tool versions explicitly:
+
+```csharp
+protected override ModuleConfiguration Configure() =>
+    ModuleConfiguration.Create()
+        .WithCacheKeyPart($"configuration={configurationName}")
+        .WithCacheKeyPart($"sdk={sdkVersion}")
+        .WithCacheEnvironmentVariable("TARGET_RUNTIME")
+        .Build();
+```
+
+Changing any key part or declared environment value invalidates the entry.
+
+## Configure limits and locations
+
+```csharp
+builder.AddModuleCache<FileSystemModuleCache>(options =>
+{
+    options.WorkingDirectory = repositoryRoot;
+    options.CacheDirectory = Path.Combine(repositoryRoot, ".cache", "modules");
+    options.MaximumInputFiles = 50_000;
+    options.MaximumHashConcurrency = 8;
+});
+```
+
+The file limit prevents unexpectedly broad globs. File hashes use a persistent modification-time and size index; changed files are hashed concurrently.
+
+## Share entries through S3 or Redis
+
+The S3 and Redis packages provide cross-run cache backends. Their cache namespaces do not use distributed pipeline run identifiers, so concurrent runs can safely share entries.
+
+```csharp
+builder.AddS3ModuleCache(options =>
+{
+    options.BucketName = "pipeline-artifacts";
+    options.Region = "eu-west-2";
+});
+```
+
+Or use Redis:
+
+```csharp
+builder.AddRedisModuleCache(
+    redis => redis.ConnectionString = "localhost:6379",
+    cacheEntries => cacheEntries.TimeToLiveSeconds = 86_400);
+```
+
+## Correctness rules
+
+- Declare all file inputs. An undeclared input cannot invalidate a cache entry.
+- Add key parts for arguments, configuration objects, external service versions, and other non-file inputs.
+- Return dependency values that change whenever relevant upstream state changes. Only direct dependency results are fingerprinted, so constant or `Unit` results do not propagate transitive invalidation.
+- Declare environment variables individually; the framework does not fingerprint the entire process environment.
+- Input files are content-hashed on every fingerprint calculation. File size and timestamps are not treated as proof that content is unchanged.
+- Use `ProducesArtifact` for files a cache hit must recreate.
+- Do not cache modules whose result cannot be serialized to JSON.

docs/docs/mp-packages/distributed-artifacts-s3.md

@@ -31,3 +31,17 @@ builder.AddS3DistributedArtifactStore(options =>
 ```
 
 Credentials use the AWS SDK credential chain. Configure the optional service URL when targeting an S3-compatible provider.
+
+## Module caching
+
+Use the same package as a shareable, cross-run module cache:
+
+```csharp
+builder.AddS3ModuleCache(options =>
+{
+    options.BucketName = "pipeline-cache";
+    options.Region = "eu-west-2";
+});
+```
+
+See [Cache Module Results](../how-to/module-caching.md) for input declarations, artifact restoration, and fingerprint configuration.

docs/docs/mp-packages/distributed-redis.md

@@ -29,3 +29,15 @@ builder.AddRedisDistributed(
 ```
 
 `AddRedisDistributedCoordinator` and `AddRedisDistributedArtifactStore` are also available when only one Redis service is required.
+
+## Module caching
+
+Redis can provide a shareable, cross-run module cache without enabling distributed execution:
+
+```csharp
+builder.AddRedisModuleCache(
+    redis => redis.ConnectionString = "localhost:6379",
+    cacheEntries => cacheEntries.TimeToLiveSeconds = 86_400);
+```
+
+See [Cache Module Results](../how-to/module-caching.md) for input declarations, artifact restoration, and fingerprint configuration.

src/ModularPipelines.Distributed.Artifacts.S3/Artifacts/S3ClientFactory.cs

@@ -0,0 +1,26 @@
+using Amazon;
+using Amazon.S3;
+using ModularPipelines.Distributed.Artifacts.S3.Configuration;
+
+namespace ModularPipelines.Distributed.Artifacts.S3.Artifacts;
+
+internal static class S3ClientFactory
+{
+    public static IAmazonS3 Create(S3ArtifactOptions options)
+    {
+        var config = new AmazonS3Config
+        {
+            RegionEndpoint = RegionEndpoint.GetBySystemName(options.Region),
+            ForcePathStyle = options.ForcePathStyle,
+        };
+
+        if (!string.IsNullOrEmpty(options.ServiceUrl))
+        {
+            config.ServiceURL = options.ServiceUrl;
+        }
+
+        return !string.IsNullOrEmpty(options.AccessKey) && !string.IsNullOrEmpty(options.SecretKey)
+            ? new AmazonS3Client(options.AccessKey, options.SecretKey, config)
+            : new AmazonS3Client(config);
+    }
+}

src/ModularPipelines.Distributed.Artifacts.S3/Artifacts/S3DistributedArtifactStoreFactory.cs

@@ -1,4 +1,3 @@
-using Amazon;
 using Amazon.S3;
 using Amazon.S3.Model;
 using ModularPipelines.Distributed.Artifacts.S3.Configuration;
@@ -24,26 +23,7 @@ public S3DistributedArtifactStoreFactory(
 
     public async Task<IDistributedArtifactStore> CreateAsync(CancellationToken cancellationToken)
     {
-        var config = new AmazonS3Config
-        {
-            RegionEndpoint = RegionEndpoint.GetBySystemName(_s3Options.Region),
-            ForcePathStyle = _s3Options.ForcePathStyle,
-        };
-
-        if (!string.IsNullOrEmpty(_s3Options.ServiceUrl))
-        {
-            config.ServiceURL = _s3Options.ServiceUrl;
-        }
-
-        IAmazonS3 s3;
-        if (!string.IsNullOrEmpty(_s3Options.AccessKey) && !string.IsNullOrEmpty(_s3Options.SecretKey))
-        {
-            s3 = new AmazonS3Client(_s3Options.AccessKey, _s3Options.SecretKey, config);
-        }
-        else
-        {
-            s3 = new AmazonS3Client(config);
-        }
+        var s3 = S3ClientFactory.Create(_s3Options);
 
         var runId = RunIdentifierResolver.Resolve(_s3Options.RunIdentifier);
 

src/ModularPipelines.Distributed.Artifacts.S3/Caching/S3ModuleCache.cs

@@ -0,0 +1,114 @@
+using System.Net;
+using Amazon.S3;
+using Amazon.S3.Model;
+using ModularPipelines.Caching;
+using ModularPipelines.Distributed.Artifacts.S3.Artifacts;
+using ModularPipelines.Distributed.Artifacts.S3.Configuration;
+
+namespace ModularPipelines.Distributed.Artifacts.S3.Caching;
+
+/// <summary>
+/// Stores shareable module cache entries in S3 or an S3-compatible service.
+/// Cache keys are independent of distributed pipeline run identifiers.
+/// </summary>
+public sealed class S3ModuleCache : IModuleCacheStore, IDisposable
+{
+    private readonly S3ArtifactOptions _options;
+    private readonly Lazy<IAmazonS3> _client;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="S3ModuleCache"/> class.
+    /// </summary>
+    public S3ModuleCache(S3ArtifactOptions options)
+    {
+        ValidateOptions(options);
+        _options = options;
+        _client = new Lazy<IAmazonS3>(() => S3ClientFactory.Create(options));
+    }
+
+    internal S3ModuleCache(S3ArtifactOptions options, IAmazonS3 client)
+    {
+        ValidateOptions(options);
+        ArgumentNullException.ThrowIfNull(client);
+        _options = options;
+        _client = new Lazy<IAmazonS3>(() => client);
+    }
+
+    /// <inheritdoc />
+    public async Task<Stream?> OpenReadAsync(string fingerprint, CancellationToken cancellationToken)
+    {
+        ModuleCacheFingerprint.Validate(fingerprint);
+
+        GetObjectResponse response;
+        try
+        {
+            response = await _client.Value
+                .GetObjectAsync(_options.BucketName, BuildObjectKey(fingerprint), cancellationToken)
+                .ConfigureAwait(false);
+        }
+        catch (AmazonS3Exception exception) when (exception.StatusCode == HttpStatusCode.NotFound)
+        {
+            return null;
+        }
+
+        using (response)
+        {
+            var temporary = Path.GetTempFileName();
+            var stream = new FileStream(
+                temporary,
+                FileMode.Create,
+                FileAccess.ReadWrite,
+                FileShare.None,
+                64 * 1024,
+                FileOptions.Asynchronous | FileOptions.DeleteOnClose);
+            try
+            {
+                await response.ResponseStream.CopyToAsync(stream, cancellationToken).ConfigureAwait(false);
+                stream.Position = 0;
+                return stream;
+            }
+            catch
+            {
+                await stream.DisposeAsync().ConfigureAwait(false);
+                throw;
+            }
+        }
+    }
+
+    /// <inheritdoc />
+    public async Task WriteAsync(string fingerprint, Stream content, CancellationToken cancellationToken)
+    {
+        ModuleCacheFingerprint.Validate(fingerprint);
+        ArgumentNullException.ThrowIfNull(content);
+
+        var request = new PutObjectRequest
+        {
+            BucketName = _options.BucketName,
+            Key = BuildObjectKey(fingerprint),
+            InputStream = content,
+            ContentType = "application/zip",
+            DisablePayloadSigning = true,
+        };
+
+        await _client.Value.PutObjectAsync(request, cancellationToken).ConfigureAwait(false);
+    }
+
+    /// <inheritdoc />
+    public void Dispose()
+    {
+        if (_client.IsValueCreated)
+        {
+            _client.Value.Dispose();
+        }
+    }
+
+    private string BuildObjectKey(string fingerprint) =>
+        $"{_options.KeyPrefix.TrimEnd('/')}/module-cache/v1/{fingerprint.ToLowerInvariant()}.zip";
+
+    private static void ValidateOptions(S3ArtifactOptions options)
+    {
+        ArgumentNullException.ThrowIfNull(options);
+        ArgumentException.ThrowIfNullOrWhiteSpace(options.BucketName);
+        ArgumentException.ThrowIfNullOrWhiteSpace(options.KeyPrefix);
+    }
+}

src/ModularPipelines.Distributed.Artifacts.S3/Extensions/S3DistributedExtensions.cs

@@ -1,6 +1,9 @@
 using Microsoft.Extensions.DependencyInjection;
+using ModularPipelines.Caching;
 using ModularPipelines.Distributed.Artifacts.S3.Artifacts;
+using ModularPipelines.Distributed.Artifacts.S3.Caching;
 using ModularPipelines.Distributed.Artifacts.S3.Configuration;
+using ModularPipelines.Extensions;
 
 namespace ModularPipelines.Distributed.Artifacts.S3.Extensions;
 
@@ -9,6 +12,24 @@ namespace ModularPipelines.Distributed.Artifacts.S3.Extensions;
 /// </summary>
 public static class S3DistributedExtensions
 {
+    /// <summary>
+    /// Enables a shareable S3-backed module cache without enabling distributed execution.
+    /// </summary>
+    /// <param name="builder">The pipeline builder.</param>
+    /// <param name="configureS3">Configures the S3-compatible service.</param>
+    /// <param name="configureCache">Optionally configures fingerprinting behavior.</param>
+    /// <returns>The same builder instance for chaining.</returns>
+    public static PipelineBuilder AddS3ModuleCache(
+        this PipelineBuilder builder,
+        Action<S3ArtifactOptions> configureS3,
+        Action<ModuleCacheOptions>? configureCache = null)
+    {
+        var options = new S3ArtifactOptions();
+        configureS3(options);
+        builder.Services.AddSingleton(_ => new S3ModuleCache(options));
+        return builder.AddModuleCache<S3ModuleCache>(configureCache);
+    }
+
     /// <summary>
     /// Registers the S3-compatible distributed artifact store factory.
     /// Works with AWS S3, Cloudflare R2, Backblaze B2, and MinIO.