From 470c17926b0f75f41c008a498852820beab83054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Mon, 16 Nov 2026 10:58:48 -0800 Subject: [PATCH 1/3] Serve the site with a per-request file middleware instead of MapStaticAssets MapStaticAssets needs a manifest of every file of the web root. With the site's ~70,500 files (8.4 GB), building it dominates every StaticHost build and publish, and each instance loads it at startup into one route per file. StaticHost now turns static web assets off and serves the web root with SiteFileMiddleware: - It looks the file up per request, prefers the brotli copy next to it when the client accepts br, and answers single byte ranges. - It computes the ETag (base64 SHA-256, the same value MapStaticAssets sent) the first time a file is served and caches it in memory. - It keeps the existing caching: immutable for /_astro/, the SDK's one-hour default for images, video and fonts, no-cache otherwise. - It steps aside when an endpoint matched, so API, health and download endpoints keep precedence. Site.targets publishes the frontend's output to wwwroot, as the static web assets pipeline did: it asks frontend.esproj for JavaScriptOutputGroup (which runs its BuildCommand) and writes a brotli copy (Optimal, the level used before) of each text file, reusing copies newer than their source. It replaces Caching.targets. The Frontend Build job no longer copies dist into wwwroot and rebuilds StaticHost to regenerate the manifest. Measured on a 4-core VM against the real production build of the site, current setup vs this change: - dotnet publish (Astro build excluded): 213 s -> 55 s (22 s when the brotli copies are reused) - CI-style build with dist in wwwroot: 86 s -> 9 s - Endpoints manifest: 142 MB -> none - First response after start: 13-18 s -> 2.3-5.8 s (the rest is startup both share) - Memory after warm load: 2.1-2.2 GB -> 280 MB RSS - Each of the 33k pages once: 1,450 -> 3,900 req/s, p99 56 -> 9 ms - Warm load: 5,700-5,850 -> 10,100-10,800 req/s, p99 21 -> 9 ms Comparing both servers on every page and file, with and without br (207,786 responses), Content-Type, Cache-Control, ETag, Vary and Content-Length match. The differences: Last-Modified on brotli responses is now the file's own, 4 tiny files no longer get a brotli copy that is not smaller, and StaticHost's own wwwroot (.gitignore, the dev-only Scalar theme) is no longer published. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01871HTuV3jb825jyAUQ3L7d --- .github/workflows/frontend-build.yml | 27 +- src/frontend/playwright.config.mjs | 6 +- .../MarkdownNegotiationMiddleware.cs | 7 +- src/statichost/StaticHost/Caching.targets | 37 -- src/statichost/StaticHost/GlobalUsings.cs | 1 + src/statichost/StaticHost/Program.cs | 10 +- src/statichost/StaticHost/Site.targets | 107 ++++++ .../StaticHost/SiteFiles/SiteFileETags.cs | 45 +++ .../SiteFiles/SiteFileExtensions.cs | 14 + .../SiteFiles/SiteFileMiddleware.cs | 273 +++++++++++++++ src/statichost/StaticHost/StaticHost.csproj | 15 +- .../AgentReadinessTestServer.cs | 7 +- tests/StaticHost.Tests/SiteFileTests.cs | 327 ++++++++++++++++++ 13 files changed, 801 insertions(+), 75 deletions(-) delete mode 100644 src/statichost/StaticHost/Caching.targets create mode 100644 src/statichost/StaticHost/Site.targets create mode 100644 src/statichost/StaticHost/SiteFiles/SiteFileETags.cs create mode 100644 src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs create mode 100644 src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs create mode 100644 tests/StaticHost.Tests/SiteFileTests.cs diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml index b5b811da58..9a5512e7c3 100644 --- a/.github/workflows/frontend-build.yml +++ b/.github/workflows/frontend-build.yml @@ -84,8 +84,9 @@ jobs: node-version: ${{ inputs.node_version }} # The E2E shards serve the site with StaticHost, as production does, - # from the build output this job uploads (statichost-bin). Its restore - # and compile need no site, so they run alongside the Astro build. + # from the build output this job uploads (statichost-bin). StaticHost + # reads the site from its web root per request, so its build needs no + # site and runs alongside the Astro build. - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: global-json-file: global.json @@ -138,16 +139,6 @@ jobs: bun run build:production fi - # StaticHost serves its wwwroot, which holds a copy of dist (hard links) - # as in production. MapStaticAssets lists every file of the web root: - # this incremental build redoes only that, while the site archive is packed - # and uploaded. - - name: Start the StaticHost build - run: | - cp -alf dist/. ../statichost/StaticHost/wwwroot/ - "$SETUP" wait statichost-compile - "$SETUP" start statichost-build dotnet build -c Release --no-restore ../statichost/StaticHost - - name: Save Astro build state if: ${{ success() && (github.event_name == 'push' || steps.astro_cache.outputs.cache-matched-key == '') }} uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -218,7 +209,7 @@ jobs: - name: Wait for the StaticHost build run: | - "$SETUP" wait statichost-build + "$SETUP" wait statichost-compile "$SETUP" report - name: Upload the StaticHost build @@ -449,17 +440,17 @@ jobs: "$SETUP" start playwright-deps bun run playwright install-deps chromium "$SETUP" start playwright-browsers bun run playwright install chromium - # The Frontend Build job built StaticHost against this run's dist. - # StaticHost reads the web root's files per request, so it starts - # before the site is there, and boots during its download. + # The Frontend Build job built StaticHost. It reads the web root's files + # per request, so it starts before the site is there, and boots during + # its download. - name: Download the StaticHost build uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: statichost-bin path: src/statichost/StaticHost/bin/Release/net10.0 - # Any answer, a 404 included, means StaticHost built its routing table, - # which the first request does. + # Any answer, a 404 included, means StaticHost is up and has built its + # routing table, which the first request does. - name: Start StaticHost run: | mkdir -p ../statichost/StaticHost/wwwroot diff --git a/src/frontend/playwright.config.mjs b/src/frontend/playwright.config.mjs index 6d1b1e0133..827d21acfc 100644 --- a/src/frontend/playwright.config.mjs +++ b/src/frontend/playwright.config.mjs @@ -66,9 +66,9 @@ export default defineConfig({ webServer: isCI ? { // CI serves the built site with StaticHost, as production does (Markdown negotiation, canonical - // redirects, the 404 page, headers). The Frontend Build job builds it against the site's files - // (wwwroot holds a copy of dist), so the tests start its assembly directly: `dotnet run --no-build` - // still evaluates the project, which takes seconds. The E2E jobs start it before dist arrives + // redirects, the 404 page, headers) from its wwwroot, where the E2E jobs unpack dist. The Frontend + // Build job builds it, so the tests start its assembly directly: `dotnet run --no-build` still + // evaluates the project, which takes seconds. The E2E jobs start it before dist arrives // (.github/scripts/start-web-server.mjs), hence reuseExistingServer. command: `dotnet bin/Release/net10.0/StaticHost.dll --urls ${e2eBaseUrl}`, cwd: fileURLToPath(new URL('../statichost/StaticHost/', import.meta.url)), diff --git a/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs b/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs index 8dc52cde71..1c0eed87c4 100644 --- a/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs +++ b/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs @@ -14,9 +14,8 @@ namespace StaticHost.AgentReadiness; /// HTML page (via starlight-page-actions). E.g. /get-started/quickstart/ /// has companion /get-started/quickstart.md. This middleware short-circuits /// the pipeline by streaming that file directly when markdown is preferred — -/// it does NOT rewrite Request.Path because rewriting after -/// UseRouting() would not re-trigger endpoint selection in -/// MapStaticAssets(). +/// it does NOT rewrite Request.Path, because the negotiated response needs +/// its own caching headers (below), which the site file middleware would replace. /// /// /// Therefore the middleware MUST run before UseDefaultFiles and @@ -44,7 +43,7 @@ public MarkdownNegotiationMiddleware( { _next = next; _logger = logger; - // Use the same provider MapStaticAssets reads from so wwwroot is the source of truth. + // Use the same provider SiteFileMiddleware reads from so wwwroot is the source of truth. _fileProvider = env.WebRootFileProvider; } diff --git a/src/statichost/StaticHost/Caching.targets b/src/statichost/StaticHost/Caching.targets deleted file mode 100644 index 4d76f64913..0000000000 --- a/src/statichost/StaticHost/Caching.targets +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - <_AstroEndpoints Include="@(StaticWebAssetEndpoint)" - Condition="$([System.String]::Copy('%(Identity)').StartsWith('_astro/'))" /> - - - - - <_CacheHeaderOperation Include="Replace"> - Header - Cache-Control - max-age=31536000, public, immutable - - - - - - - - - - - - - - - diff --git a/src/statichost/StaticHost/GlobalUsings.cs b/src/statichost/StaticHost/GlobalUsings.cs index 9d5d095639..d7de8cf05c 100644 --- a/src/statichost/StaticHost/GlobalUsings.cs +++ b/src/statichost/StaticHost/GlobalUsings.cs @@ -4,6 +4,7 @@ global using StaticHost.AgentReadiness; global using StaticHost.Live; global using StaticHost.Routing; +global using StaticHost.SiteFiles; global using StaticHost.Telemetry; global using OpenTelemetry; diff --git a/src/statichost/StaticHost/Program.cs b/src/statichost/StaticHost/Program.cs index 5e4d3c60d6..3ccf5dc5a0 100644 --- a/src/statichost/StaticHost/Program.cs +++ b/src/statichost/StaticHost/Program.cs @@ -12,6 +12,7 @@ } builder.AddLiveStatus(); +builder.Services.AddSiteFiles(); builder.Services.AddHsts(options => { @@ -43,8 +44,7 @@ // Agent-readiness middlewares MUST run before UseDefaultFiles + UseRouting: // * UseDefaultFiles rewrites /foo/ -> /foo/index.html, breaking .md-companion mapping. -// * MapStaticAssets registers endpoints during UseRouting, so a path rewrite -// after UseRouting wouldn't re-trigger endpoint selection. +// * A path rewrite after UseRouting wouldn't re-trigger endpoint selection. // MarkdownNegotiationMiddleware short-circuits to serve the .md body directly; // LinkHeaderMiddleware attaches a Link header on HTML 2xx responses via OnStarting. app.UseAgentReadiness(); @@ -92,6 +92,10 @@ await next(); }); +// The site's files. Requests that matched an endpoint below (health, downloads, live status) +// pass through to it; a file that does not exist falls through to the 404 page above. +app.UseSiteFiles(); + app.MapGet("/healthz", (HttpContext context) => { // A health probe must always reach the origin, never a cached copy. @@ -139,6 +143,4 @@ return Results.Redirect("https://aka.ms/aspire/get/install.sh"); }); -app.MapStaticAssets(); - await app.RunAsync(); diff --git a/src/statichost/StaticHost/Site.targets b/src/statichost/StaticHost/Site.targets new file mode 100644 index 0000000000..2cba6ca7a7 --- /dev/null +++ b/src/statichost/StaticHost/Site.targets @@ -0,0 +1,107 @@ + + + + + .html;.htm;.md;.markdown;.txt;.xml;.json;.webmanifest;.js;.mjs;.jsx;.css;.map;.svg;.ico;.bmp;.otf;.wasm;.csv;.tsv;.ics;.xsl;.xslt + + + + + + + + + + + + + + + + (Extensions.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase); + var publishFiles = new ConcurrentBag(); + var compressed = 0; + + Func publish = (path, relativePath) => + { + var item = new TaskItem(path); + item.SetMetadata("RelativePath", "wwwroot/" + relativePath.Replace('\\', '/')); + item.SetMetadata("CopyToPublishDirectory", "PreserveNewest"); + item.SetMetadata("ExcludeFromSingleFile", "true"); + return item; + }; + + Parallel.ForEach(Files, file => + { + var source = file.GetMetadata("FullPath"); + var relativePath = file.GetMetadata("TemplateSubPath") + Path.GetFileName(source); + publishFiles.Add(publish(source, relativePath)); + + if (!extensions.Contains(Path.GetExtension(source))) + { + return; + } + + // Reuse a copy that is newer than its source, so an unchanged site compresses once. + var target = Path.Combine(OutputDirectory, relativePath + ".br"); + if (!File.Exists(target) || File.GetLastWriteTimeUtc(target) < File.GetLastWriteTimeUtc(source)) + { + Directory.CreateDirectory(Path.GetDirectoryName(target)); + var temporary = target + ".tmp"; + using (var input = File.OpenRead(source)) + using (var output = File.Create(temporary)) + using (var brotli = (Stream)Activator.CreateInstance(brotliStream, output, level)) + { + input.CopyTo(brotli); + } + + File.Delete(target); + File.Move(temporary, target); + Interlocked.Increment(ref compressed); + } + + // A copy that saves nothing (tiny files) is not worth serving. + if (new FileInfo(target).Length < new FileInfo(source).Length) + { + publishFiles.Add(publish(target, relativePath + ".br")); + } + }); + + PublishFiles = publishFiles.ToArray(); + Log.LogMessage(MessageImportance.High, "Site: " + Files.Length + " files, " + compressed + " compressed now, " + (PublishFiles.Length - Files.Length) + " brotli copies published."); + ]]> + + + + + + + $(MSBuildProjectDirectory)/$(IntermediateOutputPath)site-br/ + + + + + + + + + + + diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs new file mode 100644 index 0000000000..ef5c00828c --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -0,0 +1,45 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using Microsoft.Extensions.FileProviders; + +namespace StaticHost.SiteFiles; + +/// +/// Computes a file's ETag from its SHA-256 the first time it is served, then keeps it in memory. +/// +/// +/// The value is the base64 SHA-256 in quotes, which is what MapStaticAssets sent, so the +/// switch does not invalidate the copies browsers hold. A content hash, unlike the length and +/// last-modified time UseStaticFiles uses, stays the same across deployments for files +/// that did not change. An entry is recomputed when the file's length or last-modified time +/// changes, which covers a web root that is replaced while StaticHost runs (CI unpacks the site +/// after StaticHost starts). +/// +internal sealed class SiteFileETags +{ + private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); + + public int Count => _entries.Count; + + public async ValueTask GetAsync(IFileInfo file, CancellationToken cancellationToken) + { + var key = file.PhysicalPath ?? file.Name; + var length = file.Length; + var lastModified = file.LastModified; + + if (_entries.TryGetValue(key, out var entry) && entry.Length == length && entry.LastModified == lastModified) + { + return entry.ETag; + } + + // Two first requests for the same file may both hash it; the result is the same. + await using var stream = file.CreateReadStream(); + var hash = await SHA256.HashDataAsync(stream, cancellationToken); + var etag = $"\"{Convert.ToBase64String(hash)}\""; + + _entries[key] = new Entry(length, lastModified, etag); + return etag; + } + + private sealed record Entry(long Length, DateTimeOffset LastModified, string ETag); +} diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs b/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs new file mode 100644 index 0000000000..6be7e35cab --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs @@ -0,0 +1,14 @@ +namespace StaticHost.SiteFiles; + +internal static class SiteFileExtensions +{ + public static IServiceCollection AddSiteFiles(this IServiceCollection services) => + services.AddSingleton(); + + /// + /// Serves the web root's files, preferring the precompressed brotli copy. Call it after + /// UseRouting: it leaves requests that matched an endpoint to that endpoint. + /// + public static IApplicationBuilder UseSiteFiles(this IApplicationBuilder app) => + app.UseMiddleware(); +} diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs new file mode 100644 index 0000000000..9ba5bb9fd1 --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -0,0 +1,273 @@ +using Microsoft.AspNetCore.StaticFiles; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Primitives; +using Microsoft.Net.Http.Headers; + +namespace StaticHost.SiteFiles; + +/// +/// Serves the site's files from the web root, with the brotli copy the publish wrote next to a +/// file (page.html.br) when the client accepts it. +/// +/// +/// +/// This replaces MapStaticAssets, which reads a manifest of every file at startup and builds +/// a route for each one. With about 70,000 files, that manifest took a minute of every build and +/// publish, and each instance seconds and gigabytes of memory to start. This middleware keeps no +/// per-file state up front: it looks the file up on each request, as UseStaticFiles does, +/// and computes the ETag the first time a file is served (). +/// +/// +/// It runs after UseRouting and steps aside when an endpoint matched, so the API, health +/// and download endpoints keep precedence over a file of the same path, as they had over the +/// static asset endpoints. +/// +/// +/// A request for one byte range gets it (video players need ranges); other range requests get the +/// whole file. +/// +/// +internal sealed class SiteFileMiddleware +{ + // Images, video and fonts get the SDK's default media caching, as with MapStaticAssets. + internal const string MediaCacheControl = "max-age=3600, must-revalidate"; + + // Astro hashes the names of the files it emits under /_astro/, so they can be cached forever. + // https://starlight.astro.build/environmental-impact/#caching + internal const string ImmutableCacheControl = "max-age=31536000, public, immutable"; + + internal const string DefaultCacheControl = "no-cache"; + + private const string BrotliExtension = ".br"; + + private const string UnknownContentType = "application/octet-stream"; + + private static readonly PathString s_immutablePrefix = "/_astro"; + + private readonly RequestDelegate _next; + private readonly IFileProvider _fileProvider; + private readonly SiteFileETags _etags; + private readonly FileExtensionContentTypeProvider _contentTypes; + + public SiteFileMiddleware(RequestDelegate next, IWebHostEnvironment environment, SiteFileETags etags) + { + _next = next; + _fileProvider = environment.WebRootFileProvider; + _etags = etags; + _contentTypes = CreateContentTypeProvider(); + } + + public async Task InvokeAsync(HttpContext context) + { + var request = context.Request; + if (context.GetEndpoint() is not null + || !(HttpMethods.IsGet(request.Method) || HttpMethods.IsHead(request.Method)) + || !request.Path.HasValue + || request.Path.Value.EndsWith('/')) + { + await _next(context); + return; + } + + var file = _fileProvider.GetFileInfo(request.Path.Value); + if (!file.Exists || file.IsDirectory) + { + await _next(context); + return; + } + + // Every file the site emits is meant to be served, including types this provider does + // not know (Pagefind's index, asciinema casts), as MapStaticAssets did. + if (!_contentTypes.TryGetContentType(request.Path.Value, out var contentType)) + { + contentType = UnknownContentType; + } + + var compressed = _fileProvider.GetFileInfo(request.Path.Value + BrotliExtension); + var hasCompressed = compressed.Exists && !compressed.IsDirectory; + var useCompressed = hasCompressed && AcceptsBrotli(request.Headers.AcceptEncoding); + var served = useCompressed ? compressed : file; + + var response = context.Response; + var etag = await _etags.GetAsync(served, context.RequestAborted); + var lastModified = TruncateToSeconds(file.LastModified); + + var headers = response.Headers; + headers.CacheControl = GetCacheControl(request.Path, contentType); + headers.ETag = etag; + headers.LastModified = HeaderUtilities.FormatDate(lastModified); + if (hasCompressed) + { + headers.Vary = HeaderNames.AcceptEncoding; + } + + if (IsNotModified(request, etag, lastModified)) + { + response.StatusCode = StatusCodes.Status304NotModified; + return; + } + + headers.AcceptRanges = "bytes"; + var range = GetRange(request, etag, lastModified, served.Length, out var unsatisfiable); + if (unsatisfiable) + { + response.StatusCode = StatusCodes.Status416RangeNotSatisfiable; + headers.ContentRange = $"bytes */{served.Length}"; + return; + } + + response.ContentType = contentType; + if (useCompressed) + { + headers.ContentEncoding = "br"; + } + + var (offset, count) = range ?? (0, served.Length); + if (range is not null) + { + response.StatusCode = StatusCodes.Status206PartialContent; + headers.ContentRange = $"bytes {offset}-{offset + count - 1}/{served.Length}"; + } + else + { + response.StatusCode = StatusCodes.Status200OK; + } + + response.ContentLength = count; + if (HttpMethods.IsHead(request.Method)) + { + return; + } + + await response.SendFileAsync(served, offset, count, context.RequestAborted); + } + + internal static string GetCacheControl(PathString path, string contentType) + { + if (path.StartsWithSegments(s_immutablePrefix, StringComparison.Ordinal)) + { + return ImmutableCacheControl; + } + + return contentType.StartsWith("image/", StringComparison.Ordinal) + || contentType.StartsWith("video/", StringComparison.Ordinal) + || contentType.StartsWith("font/", StringComparison.Ordinal) + ? MediaCacheControl + : DefaultCacheControl; + } + + internal static bool AcceptsBrotli(StringValues acceptEncoding) + { + if (!StringWithQualityHeaderValue.TryParseList(acceptEncoding, out var encodings)) + { + return false; + } + + double? brotli = null; + double? wildcard = null; + foreach (var encoding in encodings) + { + var quality = encoding.Quality ?? 1; + if (StringSegment.Equals(encoding.Value, "br", StringComparison.OrdinalIgnoreCase)) + { + brotli = quality; + } + else if (StringSegment.Equals(encoding.Value, "*", StringComparison.Ordinal)) + { + wildcard = quality; + } + } + + // An explicit "br;q=0" refuses brotli even when "*" accepts everything else. + return (brotli ?? wildcard ?? 0) > 0; + } + + private static bool IsNotModified(HttpRequest request, string etag, DateTimeOffset lastModified) + { + var requestHeaders = request.GetTypedHeaders(); + + // If-None-Match takes precedence over If-Modified-Since (RFC 9110, 13.1.3). + var ifNoneMatch = requestHeaders.IfNoneMatch; + if (ifNoneMatch.Count > 0) + { + var current = EntityTagHeaderValue.Parse(etag); + foreach (var candidate in ifNoneMatch) + { + if (candidate.Equals(EntityTagHeaderValue.Any) || candidate.Compare(current, useStrongComparison: false)) + { + return true; + } + } + + return false; + } + + return requestHeaders.IfModifiedSince is { } ifModifiedSince && lastModified <= ifModifiedSince; + } + + /// + /// Returns the single byte range to send, or to send the whole file, + /// which a server may do for any range request it does not support, such as several ranges. + /// Safari only plays video from servers that answer ranges. + /// + private static (long Offset, long Count)? GetRange( + HttpRequest request, string etag, DateTimeOffset lastModified, long length, out bool unsatisfiable) + { + unsatisfiable = false; + var requestHeaders = request.GetTypedHeaders(); + var range = requestHeaders.Range; + if (range is null + || !StringSegment.Equals(range.Unit, "bytes", StringComparison.OrdinalIgnoreCase) + || range.Ranges.Count != 1) + { + return null; + } + + // If-Range applies the range only to the representation the client has (RFC 9110, 13.1.5). + if (requestHeaders.IfRange is { } ifRange) + { + var matches = ifRange.EntityTag is { } tag + ? tag.Compare(EntityTagHeaderValue.Parse(etag), useStrongComparison: true) + : ifRange.LastModified == lastModified; + if (!matches) + { + return null; + } + } + + var item = range.Ranges.First(); + long start, end; + if (item.From is { } from) + { + start = from; + end = Math.Min(item.To ?? long.MaxValue, length - 1); + } + else + { + // A suffix range: the last N bytes. + start = Math.Max(0, length - (item.To ?? 0)); + end = length - 1; + } + + if (start >= length || start > end) + { + unsatisfiable = true; + return null; + } + + return (start, end - start + 1); + } + + private static DateTimeOffset TruncateToSeconds(DateTimeOffset value) => + new(value.Year, value.Month, value.Day, value.Hour, value.Minute, value.Second, value.Offset); + + private static FileExtensionContentTypeProvider CreateContentTypeProvider() + { + var provider = new FileExtensionContentTypeProvider(); + + // The SDK's static web assets mapping, which production sent until now, differs here. + provider.Mappings[".md"] = "text/markdown"; + + return provider; + } +} diff --git a/src/statichost/StaticHost/StaticHost.csproj b/src/statichost/StaticHost/StaticHost.csproj index a836de6f0c..8550c3aace 100644 --- a/src/statichost/StaticHost/StaticHost.csproj +++ b/src/statichost/StaticHost/StaticHost.csproj @@ -8,11 +8,14 @@ true noble-chiseled - - false - brotli - - <_BlazorBrotliCompressionLevel>Optimal + + false + + $(DefaultItemExcludes);wwwroot/** @@ -50,5 +53,5 @@ - + diff --git a/tests/StaticHost.Tests/AgentReadinessTestServer.cs b/tests/StaticHost.Tests/AgentReadinessTestServer.cs index 1750a6d4e9..b395f548d4 100644 --- a/tests/StaticHost.Tests/AgentReadinessTestServer.cs +++ b/tests/StaticHost.Tests/AgentReadinessTestServer.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using StaticHost.AgentReadiness; using StaticHost.Routing; +using StaticHost.SiteFiles; namespace StaticHost.Tests; @@ -48,6 +49,7 @@ public static async Task StartAsync( { web.UseTestServer(); web.UseWebRoot(wwwroot.Path); + web.ConfigureServices(static services => services.AddSiteFiles()); web.Configure(app => { if (pathBase.HasValue) @@ -60,10 +62,9 @@ public static async Task StartAsync( app.UseCanonicalPathRedirects(); app.UseAgentReadiness(); app.UseDefaultFiles(); - app.UseStaticFiles(); + app.UseSiteFiles(); - // Endpoint that mimics MapStaticAssets fallback: 404 if - // no static asset matched. + // 404 when no file matched, as in production. app.Run(static ctx => { ctx.Response.StatusCode = StatusCodes.Status404NotFound; diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs new file mode 100644 index 0000000000..83812ee018 --- /dev/null +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -0,0 +1,327 @@ +using System.IO.Compression; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; +using StaticHost.SiteFiles; + +namespace StaticHost.Tests; + +public sealed class SiteFileTests +{ + private const string Page = "" + "Hello, brotli. " + ""; + + [Fact] + public async Task Serves_the_brotli_copy_when_the_client_accepts_it() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + var compressed = await File.ReadAllBytesAsync(Path.Combine(server.WebRoot, "docs", "page", "index.html.br")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.AcceptEncoding.ParseAdd("gzip, deflate, br"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(["br"], response.Content.Headers.ContentEncoding); + Assert.Equal("text/html", response.Content.Headers.ContentType?.MediaType); + Assert.Equal(compressed.Length, response.Content.Headers.ContentLength); + Assert.Contains("Accept-Encoding", response.Headers.Vary); + Assert.Equal(compressed, await response.Content.ReadAsByteArrayAsync()); + Assert.Equal(ETagOf(compressed), response.Headers.ETag?.Tag); + } + + [Fact] + public async Task Serves_the_file_itself_when_the_client_does_not_accept_brotli() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + + using var response = await server.Client.GetAsync("/docs/page/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Content.Headers.ContentEncoding); + Assert.Contains("Accept-Encoding", response.Headers.Vary); + Assert.Equal(Page, await response.Content.ReadAsStringAsync()); + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes(Page)), response.Headers.ETag?.Tag); + } + + [Fact] + public async Task File_without_a_brotli_copy_has_no_vary_header() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("robots.txt", "User-agent: *\n")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/robots.txt"); + request.Headers.AcceptEncoding.ParseAdd("br"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Content.Headers.ContentEncoding); + Assert.Empty(response.Headers.Vary); + Assert.Equal("User-agent: *\n", await response.Content.ReadAsStringAsync()); + } + + [Theory] + [InlineData("", false)] + [InlineData("br", true)] + [InlineData("gzip, deflate, br, zstd", true)] + [InlineData("BR;q=0.5", true)] + [InlineData("gzip", false)] + [InlineData("br;q=0", false)] + [InlineData("*", true)] + [InlineData("*;q=0", false)] + [InlineData("br;q=0, *", false)] + public void Accepts_brotli_per_the_accept_encoding_header(string acceptEncoding, bool expected) => + Assert.Equal(expected, SiteFileMiddleware.AcceptsBrotli(new StringValues(acceptEncoding))); + + [Fact] + public async Task Matching_if_none_match_returns_not_modified() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var first = await server.Client.GetAsync("/docs/page/"); + var etag = first.Headers.ETag; + Assert.NotNull(etag); + + foreach (var ifNoneMatch in new[] { etag.Tag, $"W/{etag.Tag}", "\"other\", " + etag.Tag, "*" }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("If-None-Match", ifNoneMatch); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.NotModified, response.StatusCode); + Assert.Equal(etag.Tag, response.Headers.ETag?.Tag); + Assert.Equal("no-cache", response.Headers.CacheControl?.ToString()); + Assert.Empty(await response.Content.ReadAsByteArrayAsync()); + } + } + + [Fact] + public async Task Other_if_none_match_returns_the_file() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("If-None-Match", "\"stale\""); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(Page, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task If_modified_since_applies_only_without_if_none_match() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var first = await server.Client.GetAsync("/docs/page/"); + var lastModified = first.Content.Headers.LastModified; + Assert.NotNull(lastModified); + + using var conditional = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + conditional.Headers.IfModifiedSince = lastModified; + using var notModified = await server.Client.SendAsync(conditional); + Assert.Equal(HttpStatusCode.NotModified, notModified.StatusCode); + + using var both = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + both.Headers.IfModifiedSince = lastModified; + both.Headers.TryAddWithoutValidation("If-None-Match", "\"stale\""); + using var modified = await server.Client.SendAsync(both); + Assert.Equal(HttpStatusCode.OK, modified.StatusCode); + } + + [Fact] + public async Task ETag_follows_a_file_that_changes() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("data.json", "{\"v\":1}")); + + using var before = await server.Client.GetAsync("/data.json"); + await File.WriteAllTextAsync(Path.Combine(server.WebRoot, "data.json"), "{\"v\":22}"); + using var after = await server.Client.GetAsync("/data.json"); + + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes("{\"v\":1}")), before.Headers.ETag?.Tag); + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes("{\"v\":22}")), after.Headers.ETag?.Tag); + Assert.Equal("{\"v\":22}", await after.Content.ReadAsStringAsync()); + } + + [Theory] + [InlineData("/_astro/index.Bx1y2z3.js", SiteFileMiddleware.ImmutableCacheControl)] + [InlineData("/_astro/logo.Bx1y2z3.png", SiteFileMiddleware.ImmutableCacheControl)] + [InlineData("/favicon.svg", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/images/hero.webp", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/fonts/site.woff2", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/casts/demo.mp4", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/docs/page/index.html", SiteFileMiddleware.DefaultCacheControl)] + [InlineData("/docs/page.md", SiteFileMiddleware.DefaultCacheControl)] + [InlineData("/pagefind/pagefind.js", SiteFileMiddleware.DefaultCacheControl)] + public async Task Cache_control_depends_on_the_path_and_the_type(string path, string expected) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile(path, "x")); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + // HttpClient reorders the directives it parses; compare what the server sent. + Assert.Equal(expected, response.Headers.NonValidated["Cache-Control"].ToString()); + } + + [Fact] + public async Task Markdown_is_served_as_text_markdown() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SamplePages.SeedFolderPage(root, "docs/page")); + + using var response = await server.Client.GetAsync("/docs/page.md"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("text/markdown", response.Content.Headers.ContentType?.MediaType); + } + + [Fact] + public async Task Head_sends_the_headers_without_the_body() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var response = await server.Client.SendAsync(new HttpRequestMessage(HttpMethod.Head, "/docs/page/")); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(Encoding.UTF8.GetByteCount(Page), response.Content.Headers.ContentLength); + Assert.NotNull(response.Headers.ETag); + Assert.Empty(await response.Content.ReadAsByteArrayAsync()); + } + + [Theory] + [InlineData("bytes=2-5", HttpStatusCode.PartialContent, "2345", "bytes 2-5/10")] + [InlineData("bytes=7-", HttpStatusCode.PartialContent, "789", "bytes 7-9/10")] + [InlineData("bytes=-3", HttpStatusCode.PartialContent, "789", "bytes 7-9/10")] + [InlineData("bytes=8-100", HttpStatusCode.PartialContent, "89", "bytes 8-9/10")] + [InlineData("bytes=0-1, 4-5", HttpStatusCode.OK, "0123456789", null)] + [InlineData("items=0-1", HttpStatusCode.OK, "0123456789", null)] + public async Task Answers_a_single_byte_range(string range, HttpStatusCode status, string body, string? contentRange) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", range); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + Assert.Equal(body, await response.Content.ReadAsStringAsync()); + Assert.Equal(contentRange, response.Content.Headers.ContentRange?.ToString()); + Assert.Equal(["bytes"], response.Headers.AcceptRanges); + } + + [Fact] + public async Task Range_past_the_end_is_not_satisfiable() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", "bytes=10-"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.RequestedRangeNotSatisfiable, response.StatusCode); + Assert.Equal("bytes */10", response.Content.Headers.ContentRange?.ToString()); + } + + [Fact] + public async Task If_range_applies_the_range_only_to_the_current_file() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + using var first = await server.Client.GetAsync("/casts/demo.mp4"); + var etag = first.Headers.ETag!.Tag; + + foreach (var (ifRange, status) in new[] { (etag, HttpStatusCode.PartialContent), ("\"stale\"", HttpStatusCode.OK) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", "bytes=0-1"); + request.Headers.TryAddWithoutValidation("If-Range", ifRange); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + } + + [Theory] + [InlineData("/missing.html")] + [InlineData("/missing/")] + [InlineData("/docs")] + public async Task Missing_files_fall_through(string path) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + + [Theory] + [InlineData("/pagefind/fragment/en_1a2b3c4.pf_fragment")] + [InlineData("/pagefind/wasm.en.pagefind")] + [InlineData("/casts/demo.cast")] + public async Task Files_of_unknown_types_are_served_as_octet_stream(string path) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile(path, "x")); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("application/octet-stream", response.Content.Headers.ContentType?.MediaType); + Assert.Equal("x", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task Post_falls_through() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("data.json", "{}")); + + using var response = await server.Client.PostAsync("/data.json", new StringContent("{}")); + + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + + [Fact] + public async Task Matched_endpoint_takes_precedence_over_a_file() + { + using var wwwroot = new TempWebRoot(); + wwwroot.WriteFile("api/data.json", "{\"from\":\"file\"}"); + wwwroot.WriteFile("other.json", "{\"from\":\"file\"}"); + + using var host = new HostBuilder() + .ConfigureLogging(static logging => logging.ClearProviders()) + .ConfigureWebHost(web => + { + web.UseTestServer(); + web.UseWebRoot(wwwroot.Path); + web.ConfigureServices(static services => services.AddRouting().AddSiteFiles()); + web.Configure(static app => + { + app.UseRouting(); + app.UseSiteFiles(); + app.UseEndpoints(static endpoints => endpoints.MapGet("/api/data.json", () => Results.Text("endpoint"))); + }); + }) + .Build(); + await host.StartAsync(); + using var client = host.GetTestClient(); + + Assert.Equal("endpoint", await client.GetStringAsync("/api/data.json")); + Assert.Equal("{\"from\":\"file\"}", await client.GetStringAsync("/other.json")); + + await host.StopAsync(); + } + + private static void SeedCompressedPage(TempWebRoot root, string slug) + { + root.WriteFile($"{slug}/index.html", Page); + + using var output = new MemoryStream(); + using (var brotli = new BrotliStream(output, CompressionLevel.SmallestSize, leaveOpen: true)) + { + brotli.Write(Encoding.UTF8.GetBytes(Page)); + } + + File.WriteAllBytes(Path.Combine(root.Path, slug.Replace('/', Path.DirectorySeparatorChar), "index.html.br"), output.ToArray()); + } + + private static string ETagOf(byte[] content) => $"\"{Convert.ToBase64String(SHA256.HashData(content))}\""; +} From d4efc16ad7140cbe21820807c42c42120fb4996c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Tue, 17 Nov 2026 08:43:26 -0800 Subject: [PATCH 2/3] Compress the site with a file-based app and tighten file preconditions - Write the brotli copies with src/statichost/compress-site.cs, a file-based app that Site.targets runs, instead of an inline MSBuild task that loaded BrotliStream by reflection (which fails under .NET Framework MSBuild). - Reuse a brotli copy only while it carries its source's exact timestamp. A source replaced by new content with an older timestamp (a cache restore or an archive extraction) kept the copy of the old content. - Answer If-Match and If-Unmodified-Since with 412, as MapStaticAssets did. - Apply a date in If-Range only to uncompressed responses: Last-Modified is the original file's, so it does not identify the bytes of its brotli copy. - Key cached ETags by the file's path in the web root, not its physical path or bare name, and parse the ETag once per request. --- src/statichost/StaticHost/Site.targets | 107 +++++------------- .../StaticHost/SiteFiles/SiteFileETags.cs | 12 +- .../SiteFiles/SiteFileMiddleware.cs | 72 ++++++++---- src/statichost/compress-site.cs | 67 +++++++++++ tests/StaticHost.Tests/SiteFileTests.cs | 47 ++++++++ 5 files changed, 200 insertions(+), 105 deletions(-) create mode 100644 src/statichost/compress-site.cs diff --git a/src/statichost/StaticHost/Site.targets b/src/statichost/StaticHost/Site.targets index 2cba6ca7a7..3eadbdc080 100644 --- a/src/statichost/StaticHost/Site.targets +++ b/src/statichost/StaticHost/Site.targets @@ -9,99 +9,48 @@ output, and compress it once at publish. Caching and ETags are decided per request. Asking frontend.esproj for JavaScriptOutputGroup runs its BuildCommand and returns dist's - files, as the static web assets pipeline did. + files, as the static web assets pipeline did. The brotli copies are written by a file-based + app (../compress-site.cs), which reuses the copies of files that did not change. --> .html;.htm;.md;.markdown;.txt;.xml;.json;.webmanifest;.js;.mjs;.jsx;.css;.map;.svg;.ico;.bmp;.otf;.wasm;.csv;.tsv;.ics;.xsl;.xslt + $(MSBuildThisFileDirectory)../compress-site.cs + + $(DOTNET_HOST_PATH) + dotnet - - - - - - - - - - - - - - (Extensions.Split(new[] { ';' }, StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase); - var publishFiles = new ConcurrentBag(); - var compressed = 0; - - Func publish = (path, relativePath) => - { - var item = new TaskItem(path); - item.SetMetadata("RelativePath", "wwwroot/" + relativePath.Replace('\\', '/')); - item.SetMetadata("CopyToPublishDirectory", "PreserveNewest"); - item.SetMetadata("ExcludeFromSingleFile", "true"); - return item; - }; - - Parallel.ForEach(Files, file => - { - var source = file.GetMetadata("FullPath"); - var relativePath = file.GetMetadata("TemplateSubPath") + Path.GetFileName(source); - publishFiles.Add(publish(source, relativePath)); - - if (!extensions.Contains(Path.GetExtension(source))) - { - return; - } - - // Reuse a copy that is newer than its source, so an unchanged site compresses once. - var target = Path.Combine(OutputDirectory, relativePath + ".br"); - if (!File.Exists(target) || File.GetLastWriteTimeUtc(target) < File.GetLastWriteTimeUtc(source)) - { - Directory.CreateDirectory(Path.GetDirectoryName(target)); - var temporary = target + ".tmp"; - using (var input = File.OpenRead(source)) - using (var output = File.Create(temporary)) - using (var brotli = (Stream)Activator.CreateInstance(brotliStream, output, level)) - { - input.CopyTo(brotli); - } - - File.Delete(target); - File.Move(temporary, target); - Interlocked.Increment(ref compressed); - } - - // A copy that saves nothing (tiny files) is not worth serving. - if (new FileInfo(target).Length < new FileInfo(source).Length) - { - publishFiles.Add(publish(target, relativePath + ".br")); - } - }); - - PublishFiles = publishFiles.ToArray(); - Log.LogMessage(MessageImportance.High, "Site: " + Files.Length + " files, " + compressed + " compressed now, " + (PublishFiles.Length - Files.Length) + " brotli copies published."); - ]]> - - - - $(MSBuildProjectDirectory)/$(IntermediateOutputPath)site-br/ + <_SiteIntermediateDirectory>$(MSBuildProjectDirectory)/$(IntermediateOutputPath) + <_SiteCompressedDirectory>$(_SiteIntermediateDirectory)site-br/ + <_SiteFileList>$(_SiteIntermediateDirectory)site-files.txt + <_SiteBrotliList>$(_SiteIntermediateDirectory)site-br.txt - - - + + + + + + + + + + + diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs index ef5c00828c..9c9d887911 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -19,15 +19,15 @@ internal sealed class SiteFileETags { private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); - public int Count => _entries.Count; - - public async ValueTask GetAsync(IFileInfo file, CancellationToken cancellationToken) + /// The file's path in the web root, which identifies it whatever the provider. + /// The file to hash. + /// Cancels the hashing. + public async ValueTask GetAsync(string path, IFileInfo file, CancellationToken cancellationToken) { - var key = file.PhysicalPath ?? file.Name; var length = file.Length; var lastModified = file.LastModified; - if (_entries.TryGetValue(key, out var entry) && entry.Length == length && entry.LastModified == lastModified) + if (_entries.TryGetValue(path, out var entry) && entry.Length == length && entry.LastModified == lastModified) { return entry.ETag; } @@ -37,7 +37,7 @@ public async ValueTask GetAsync(IFileInfo file, CancellationToken cancel var hash = await SHA256.HashDataAsync(stream, cancellationToken); var etag = $"\"{Convert.ToBase64String(hash)}\""; - _entries[key] = new Entry(length, lastModified, etag); + _entries[path] = new Entry(length, lastModified, etag); return etag; } diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs index 9ba5bb9fd1..9c39bb203a 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -1,3 +1,4 @@ +using Microsoft.AspNetCore.Http.Headers; using Microsoft.AspNetCore.StaticFiles; using Microsoft.Extensions.FileProviders; using Microsoft.Extensions.Primitives; @@ -83,14 +84,17 @@ public async Task InvokeAsync(HttpContext context) contentType = UnknownContentType; } - var compressed = _fileProvider.GetFileInfo(request.Path.Value + BrotliExtension); + var compressedPath = request.Path.Value + BrotliExtension; + var compressed = _fileProvider.GetFileInfo(compressedPath); var hasCompressed = compressed.Exists && !compressed.IsDirectory; var useCompressed = hasCompressed && AcceptsBrotli(request.Headers.AcceptEncoding); var served = useCompressed ? compressed : file; var response = context.Response; - var etag = await _etags.GetAsync(served, context.RequestAborted); + var etag = await _etags.GetAsync(useCompressed ? compressedPath : request.Path.Value, served, context.RequestAborted); + var entityTag = new EntityTagHeaderValue(etag); var lastModified = TruncateToSeconds(file.LastModified); + var requestHeaders = request.GetTypedHeaders(); var headers = response.Headers; headers.CacheControl = GetCacheControl(request.Path, contentType); @@ -101,14 +105,16 @@ public async Task InvokeAsync(HttpContext context) headers.Vary = HeaderNames.AcceptEncoding; } - if (IsNotModified(request, etag, lastModified)) + if (EvaluatePreconditions(requestHeaders, entityTag, lastModified) is { } status) { - response.StatusCode = StatusCodes.Status304NotModified; + response.StatusCode = status; return; } headers.AcceptRanges = "bytes"; - var range = GetRange(request, etag, lastModified, served.Length, out var unsatisfiable); + // Last-Modified is the original file's, so a date in If-Range does not vouch for the bytes + // of its brotli copy, which can be rewritten while the original keeps its time. + var range = GetRange(requestHeaders, entityTag, useCompressed ? null : lastModified, served.Length, out var unsatisfiable); if (unsatisfiable) { response.StatusCode = StatusCodes.Status416RangeNotSatisfiable; @@ -182,27 +188,50 @@ internal static bool AcceptsBrotli(StringValues acceptEncoding) return (brotli ?? wildcard ?? 0) > 0; } - private static bool IsNotModified(HttpRequest request, string etag, DateTimeOffset lastModified) + /// + /// Evaluates the request's preconditions in the order of RFC 9110, 13.2.2, as + /// MapStaticAssets did. Returns 412 or 304 when one decides the response, otherwise + /// . + /// + private static int? EvaluatePreconditions(RequestHeaders requestHeaders, EntityTagHeaderValue etag, DateTimeOffset lastModified) { - var requestHeaders = request.GetTypedHeaders(); + // If-Match takes precedence over If-Unmodified-Since, and uses the strong comparison. + var ifMatch = requestHeaders.IfMatch; + if (ifMatch.Count > 0) + { + if (!Matches(ifMatch, etag, useStrongComparison: true)) + { + return StatusCodes.Status412PreconditionFailed; + } + } + else if (requestHeaders.IfUnmodifiedSince is { } ifUnmodifiedSince && lastModified > ifUnmodifiedSince) + { + return StatusCodes.Status412PreconditionFailed; + } // If-None-Match takes precedence over If-Modified-Since (RFC 9110, 13.1.3). var ifNoneMatch = requestHeaders.IfNoneMatch; if (ifNoneMatch.Count > 0) { - var current = EntityTagHeaderValue.Parse(etag); - foreach (var candidate in ifNoneMatch) + return Matches(ifNoneMatch, etag, useStrongComparison: false) ? StatusCodes.Status304NotModified : null; + } + + return requestHeaders.IfModifiedSince is { } ifModifiedSince && lastModified <= ifModifiedSince + ? StatusCodes.Status304NotModified + : null; + } + + private static bool Matches(IList candidates, EntityTagHeaderValue etag, bool useStrongComparison) + { + foreach (var candidate in candidates) + { + if (candidate.Equals(EntityTagHeaderValue.Any) || candidate.Compare(etag, useStrongComparison)) { - if (candidate.Equals(EntityTagHeaderValue.Any) || candidate.Compare(current, useStrongComparison: false)) - { - return true; - } + return true; } - - return false; } - return requestHeaders.IfModifiedSince is { } ifModifiedSince && lastModified <= ifModifiedSince; + return false; } /// @@ -210,11 +239,14 @@ private static bool IsNotModified(HttpRequest request, string etag, DateTimeOffs /// which a server may do for any range request it does not support, such as several ranges. /// Safari only plays video from servers that answer ranges. /// + /// + /// The time a date in If-Range must match, or when a date does + /// not identify the bytes sent, so only an entity tag can. + /// private static (long Offset, long Count)? GetRange( - HttpRequest request, string etag, DateTimeOffset lastModified, long length, out bool unsatisfiable) + RequestHeaders requestHeaders, EntityTagHeaderValue etag, DateTimeOffset? lastModified, long length, out bool unsatisfiable) { unsatisfiable = false; - var requestHeaders = request.GetTypedHeaders(); var range = requestHeaders.Range; if (range is null || !StringSegment.Equals(range.Unit, "bytes", StringComparison.OrdinalIgnoreCase) @@ -227,8 +259,8 @@ private static (long Offset, long Count)? GetRange( if (requestHeaders.IfRange is { } ifRange) { var matches = ifRange.EntityTag is { } tag - ? tag.Compare(EntityTagHeaderValue.Parse(etag), useStrongComparison: true) - : ifRange.LastModified == lastModified; + ? tag.Compare(etag, useStrongComparison: true) + : lastModified is not null && ifRange.LastModified == lastModified; if (!matches) { return null; diff --git a/src/statichost/compress-site.cs b/src/statichost/compress-site.cs new file mode 100644 index 0000000000..88cec4f419 --- /dev/null +++ b/src/statichost/compress-site.cs @@ -0,0 +1,67 @@ +// Writes a brotli copy of each text file of the site, for StaticHost to serve to clients that +// accept it. StaticHost's publish runs it (StaticHost/Site.targets); it also runs alone: +// +// dotnet run --file compress-site.cs -- +// +// One line per site file: its full path, a tab, its path in the site. +// Where the copies are written, at their path in the site plus ".br". +// The extensions to compress, separated by semicolons. +// Written with the path in the site of each copy worth publishing. + +using System.Collections.Concurrent; +using System.IO.Compression; + +if (args.Length != 4) +{ + Console.Error.WriteLine("Usage: compress-site.cs "); + return 1; +} + +var (filesPath, outputDirectory, extensionList, publishedPath) = (args[0], Path.GetFullPath(args[1]), args[2], args[3]); +var extensions = new HashSet(extensionList.Split(';', StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase); + +var files = File.ReadLines(filesPath) + .Where(static line => line.Length > 0) + .Select(static line => line.Split('\t', 2)) + .Select(static parts => (Source: parts[0], SitePath: parts[1].Replace('\\', '/'))) + .Where(file => extensions.Contains(Path.GetExtension(file.Source))) + .ToList(); + +var published = new ConcurrentBag(); +var compressed = 0; + +Parallel.ForEach(files, file => +{ + // Reuse a copy stamped with its source's exact time, so an unchanged site compresses once. + // "Newer than the source" is not enough: a cache restore or an archive extraction gives + // new content an older time, and the copy of the old content would be published. + var target = Path.Combine(outputDirectory, file.SitePath + ".br"); + var sourceTime = File.GetLastWriteTimeUtc(file.Source); + if (!File.Exists(target) || File.GetLastWriteTimeUtc(target) != sourceTime) + { + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + var temporary = target + ".tmp"; + using (var input = File.OpenRead(file.Source)) + using (var output = File.Create(temporary)) + // Optimal is brotli quality 4, which the SDK used at publish (_BlazorBrotliCompressionLevel). + // SmallestSize (11) saves about a sixth more but compresses about 100 times slower. + using (var brotli = new BrotliStream(output, CompressionLevel.Optimal)) + { + input.CopyTo(brotli); + } + + File.SetLastWriteTimeUtc(temporary, sourceTime); + File.Move(temporary, target, overwrite: true); + Interlocked.Increment(ref compressed); + } + + // A copy that saves nothing (tiny files) is not worth serving. + if (new FileInfo(target).Length < new FileInfo(file.Source).Length) + { + published.Add(file.SitePath + ".br"); + } +}); + +File.WriteAllLines(publishedPath, published.Order(StringComparer.Ordinal)); +Console.WriteLine($"Site: {files.Count} files to compress, {compressed} compressed now, {published.Count} brotli copies published."); +return 0; diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index 83812ee018..f207a37af1 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -130,6 +130,53 @@ public async Task If_modified_since_applies_only_without_if_none_match() Assert.Equal(HttpStatusCode.OK, modified.StatusCode); } + [Fact] + public async Task If_match_and_if_unmodified_since_fail_for_another_version() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var first = await server.Client.GetAsync("/docs/page/"); + var etag = first.Headers.ETag!.Tag; + var lastModified = first.Content.Headers.LastModified!.Value; + + foreach (var (ifMatch, status) in new[] { (etag, HttpStatusCode.OK), ("*", HttpStatusCode.OK), ("\"stale\"", HttpStatusCode.PreconditionFailed), ($"W/{etag}", HttpStatusCode.PreconditionFailed) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("If-Match", ifMatch); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + + foreach (var (ifUnmodifiedSince, status) in new[] { (lastModified, HttpStatusCode.OK), (lastModified.AddDays(-1), HttpStatusCode.PreconditionFailed) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.IfUnmodifiedSince = ifUnmodifiedSince; + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + } + + [Fact] + public async Task If_range_date_does_not_apply_a_range_to_the_brotli_copy() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + using var first = await server.Client.GetAsync("/docs/page/"); + var lastModified = first.Content.Headers.LastModified!.Value; + + foreach (var (acceptEncoding, status) in new[] { ("identity", HttpStatusCode.PartialContent), ("br", HttpStatusCode.OK) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.AcceptEncoding.ParseAdd(acceptEncoding); + request.Headers.TryAddWithoutValidation("Range", "bytes=0-1"); + request.Headers.IfRange = new System.Net.Http.Headers.RangeConditionHeaderValue(lastModified); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + } + [Fact] public async Task ETag_follows_a_file_that_changes() { From 12329a6513e63110ce3dcf54d0d43938f3ca18bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Ros?= <1165805+sebastienros@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:00:42 -0800 Subject: [PATCH 3/3] Hash ETags with XxHash128 and explain the brotli lookup and key casing - Compute ETags with XxHash128 instead of SHA-256: an ETag only has to tell versions of a file apart, which needs no cryptographic hash. It still hashes the content, so unchanged files keep their ETag across deployments. - Say why the brotli copy is looked up even for clients that do not accept brotli (the Vary header both versions need), and why cached ETags are keyed by ordinal path. --- .../StaticHost/SiteFiles/SiteFileETags.cs | 42 ++++++++++++++----- .../SiteFiles/SiteFileMiddleware.cs | 3 ++ src/statichost/StaticHost/StaticHost.csproj | 1 + tests/StaticHost.Tests/SiteFileTests.cs | 2 +- 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs index 9c9d887911..772475b9cc 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -1,22 +1,29 @@ +using System.Buffers; using System.Collections.Concurrent; -using System.Security.Cryptography; +using System.IO.Hashing; using Microsoft.Extensions.FileProviders; namespace StaticHost.SiteFiles; /// -/// Computes a file's ETag from its SHA-256 the first time it is served, then keeps it in memory. +/// Computes a file's ETag from a hash of its content the first time it is served, then keeps it +/// in memory. /// /// -/// The value is the base64 SHA-256 in quotes, which is what MapStaticAssets sent, so the -/// switch does not invalidate the copies browsers hold. A content hash, unlike the length and -/// last-modified time UseStaticFiles uses, stays the same across deployments for files -/// that did not change. An entry is recomputed when the file's length or last-modified time -/// changes, which covers a web root that is replaced while StaticHost runs (CI unpacks the site -/// after StaticHost starts). +/// The hash is XxHash128: an ETag only has to tell versions of a file apart, which needs no +/// cryptographic hash. A content hash, unlike the length and last-modified time +/// UseStaticFiles uses, stays the same across deployments for files that did not change, +/// though CI writes every file again. An entry is recomputed when the file's length or +/// last-modified time changes, which covers a web root that is replaced while StaticHost runs +/// (CI unpacks the site after StaticHost starts). /// internal sealed class SiteFileETags { + private const int BufferSize = 64 * 1024; + + // Ordinal, as the file system is in production (Linux): two paths that differ only by case are + // two files there, and must not share an entry. On a case-insensitive disk they are one file + // under two keys, which costs one more hash. Canonical redirects send most requests to one casing. private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); /// The file's path in the web root, which identifies it whatever the provider. @@ -33,10 +40,23 @@ public async ValueTask GetAsync(string path, IFileInfo file, Cancellatio } // Two first requests for the same file may both hash it; the result is the same. - await using var stream = file.CreateReadStream(); - var hash = await SHA256.HashDataAsync(stream, cancellationToken); - var etag = $"\"{Convert.ToBase64String(hash)}\""; + var hash = new XxHash128(); + var buffer = ArrayPool.Shared.Rent(BufferSize); + try + { + await using var stream = file.CreateReadStream(); + int read; + while ((read = await stream.ReadAsync(buffer, cancellationToken)) > 0) + { + hash.Append(buffer.AsSpan(0, read)); + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + var etag = $"\"{Convert.ToBase64String(hash.GetCurrentHash())}\""; _entries[path] = new Entry(length, lastModified, etag); return etag; } diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs index 9c39bb203a..25d78b3952 100644 --- a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -84,6 +84,9 @@ public async Task InvokeAsync(HttpContext context) contentType = UnknownContentType; } + // Look for the brotli copy even when the client does not accept brotli: a file that has one + // must send Vary: Accept-Encoding with both versions, so that a cache does not hand the + // uncompressed one to every client. Nearly every browser accepts brotli anyway. var compressedPath = request.Path.Value + BrotliExtension; var compressed = _fileProvider.GetFileInfo(compressedPath); var hasCompressed = compressed.Exists && !compressed.IsDirectory; diff --git a/src/statichost/StaticHost/StaticHost.csproj b/src/statichost/StaticHost/StaticHost.csproj index 8550c3aace..791917b5e4 100644 --- a/src/statichost/StaticHost/StaticHost.csproj +++ b/src/statichost/StaticHost/StaticHost.csproj @@ -30,6 +30,7 @@ + diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs index f207a37af1..ede50a08f8 100644 --- a/tests/StaticHost.Tests/SiteFileTests.cs +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -370,5 +370,5 @@ private static void SeedCompressedPage(TempWebRoot root, string slug) File.WriteAllBytes(Path.Combine(root.Path, slug.Replace('/', Path.DirectorySeparatorChar), "index.html.br"), output.ToArray()); } - private static string ETagOf(byte[] content) => $"\"{Convert.ToBase64String(SHA256.HashData(content))}\""; + private static string ETagOf(byte[] content) => $"\"{Convert.ToBase64String(System.IO.Hashing.XxHash128.Hash(content))}\""; }