diff --git a/.github/workflows/frontend-build.yml b/.github/workflows/frontend-build.yml index b5b811da58..9a5512e7c3 100644 --- a/.github/workflows/frontend-build.yml +++ b/.github/workflows/frontend-build.yml @@ -84,8 +84,9 @@ jobs: node-version: ${{ inputs.node_version }} # The E2E shards serve the site with StaticHost, as production does, - # from the build output this job uploads (statichost-bin). Its restore - # and compile need no site, so they run alongside the Astro build. + # from the build output this job uploads (statichost-bin). StaticHost + # reads the site from its web root per request, so its build needs no + # site and runs alongside the Astro build. - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 with: global-json-file: global.json @@ -138,16 +139,6 @@ jobs: bun run build:production fi - # StaticHost serves its wwwroot, which holds a copy of dist (hard links) - # as in production. MapStaticAssets lists every file of the web root: - # this incremental build redoes only that, while the site archive is packed - # and uploaded. - - name: Start the StaticHost build - run: | - cp -alf dist/. ../statichost/StaticHost/wwwroot/ - "$SETUP" wait statichost-compile - "$SETUP" start statichost-build dotnet build -c Release --no-restore ../statichost/StaticHost - - name: Save Astro build state if: ${{ success() && (github.event_name == 'push' || steps.astro_cache.outputs.cache-matched-key == '') }} uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -218,7 +209,7 @@ jobs: - name: Wait for the StaticHost build run: | - "$SETUP" wait statichost-build + "$SETUP" wait statichost-compile "$SETUP" report - name: Upload the StaticHost build @@ -449,17 +440,17 @@ jobs: "$SETUP" start playwright-deps bun run playwright install-deps chromium "$SETUP" start playwright-browsers bun run playwright install chromium - # The Frontend Build job built StaticHost against this run's dist. - # StaticHost reads the web root's files per request, so it starts - # before the site is there, and boots during its download. + # The Frontend Build job built StaticHost. It reads the web root's files + # per request, so it starts before the site is there, and boots during + # its download. - name: Download the StaticHost build uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: statichost-bin path: src/statichost/StaticHost/bin/Release/net10.0 - # Any answer, a 404 included, means StaticHost built its routing table, - # which the first request does. + # Any answer, a 404 included, means StaticHost is up and has built its + # routing table, which the first request does. - name: Start StaticHost run: | mkdir -p ../statichost/StaticHost/wwwroot diff --git a/src/frontend/playwright.config.mjs b/src/frontend/playwright.config.mjs index 6d1b1e0133..827d21acfc 100644 --- a/src/frontend/playwright.config.mjs +++ b/src/frontend/playwright.config.mjs @@ -66,9 +66,9 @@ export default defineConfig({ webServer: isCI ? { // CI serves the built site with StaticHost, as production does (Markdown negotiation, canonical - // redirects, the 404 page, headers). The Frontend Build job builds it against the site's files - // (wwwroot holds a copy of dist), so the tests start its assembly directly: `dotnet run --no-build` - // still evaluates the project, which takes seconds. The E2E jobs start it before dist arrives + // redirects, the 404 page, headers) from its wwwroot, where the E2E jobs unpack dist. The Frontend + // Build job builds it, so the tests start its assembly directly: `dotnet run --no-build` still + // evaluates the project, which takes seconds. The E2E jobs start it before dist arrives // (.github/scripts/start-web-server.mjs), hence reuseExistingServer. command: `dotnet bin/Release/net10.0/StaticHost.dll --urls ${e2eBaseUrl}`, cwd: fileURLToPath(new URL('../statichost/StaticHost/', import.meta.url)), diff --git a/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs b/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs index 8dc52cde71..1c0eed87c4 100644 --- a/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs +++ b/src/statichost/StaticHost/AgentReadiness/MarkdownNegotiationMiddleware.cs @@ -14,9 +14,8 @@ namespace StaticHost.AgentReadiness; /// HTML page (via starlight-page-actions). E.g. /get-started/quickstart/ /// has companion /get-started/quickstart.md. This middleware short-circuits /// the pipeline by streaming that file directly when markdown is preferred — -/// it does NOT rewrite Request.Path because rewriting after -/// UseRouting() would not re-trigger endpoint selection in -/// MapStaticAssets(). +/// it does NOT rewrite Request.Path, because the negotiated response needs +/// its own caching headers (below), which the site file middleware would replace. /// /// /// Therefore the middleware MUST run before UseDefaultFiles and @@ -44,7 +43,7 @@ public MarkdownNegotiationMiddleware( { _next = next; _logger = logger; - // Use the same provider MapStaticAssets reads from so wwwroot is the source of truth. + // Use the same provider SiteFileMiddleware reads from so wwwroot is the source of truth. _fileProvider = env.WebRootFileProvider; } diff --git a/src/statichost/StaticHost/Caching.targets b/src/statichost/StaticHost/Caching.targets deleted file mode 100644 index 4d76f64913..0000000000 --- a/src/statichost/StaticHost/Caching.targets +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - <_AstroEndpoints Include="@(StaticWebAssetEndpoint)" - Condition="$([System.String]::Copy('%(Identity)').StartsWith('_astro/'))" /> - - - - - <_CacheHeaderOperation Include="Replace"> - Header - Cache-Control - max-age=31536000, public, immutable - - - - - - - - - - - - - - - diff --git a/src/statichost/StaticHost/GlobalUsings.cs b/src/statichost/StaticHost/GlobalUsings.cs index 9d5d095639..d7de8cf05c 100644 --- a/src/statichost/StaticHost/GlobalUsings.cs +++ b/src/statichost/StaticHost/GlobalUsings.cs @@ -4,6 +4,7 @@ global using StaticHost.AgentReadiness; global using StaticHost.Live; global using StaticHost.Routing; +global using StaticHost.SiteFiles; global using StaticHost.Telemetry; global using OpenTelemetry; diff --git a/src/statichost/StaticHost/Program.cs b/src/statichost/StaticHost/Program.cs index 5e4d3c60d6..3ccf5dc5a0 100644 --- a/src/statichost/StaticHost/Program.cs +++ b/src/statichost/StaticHost/Program.cs @@ -12,6 +12,7 @@ } builder.AddLiveStatus(); +builder.Services.AddSiteFiles(); builder.Services.AddHsts(options => { @@ -43,8 +44,7 @@ // Agent-readiness middlewares MUST run before UseDefaultFiles + UseRouting: // * UseDefaultFiles rewrites /foo/ -> /foo/index.html, breaking .md-companion mapping. -// * MapStaticAssets registers endpoints during UseRouting, so a path rewrite -// after UseRouting wouldn't re-trigger endpoint selection. +// * A path rewrite after UseRouting wouldn't re-trigger endpoint selection. // MarkdownNegotiationMiddleware short-circuits to serve the .md body directly; // LinkHeaderMiddleware attaches a Link header on HTML 2xx responses via OnStarting. app.UseAgentReadiness(); @@ -92,6 +92,10 @@ await next(); }); +// The site's files. Requests that matched an endpoint below (health, downloads, live status) +// pass through to it; a file that does not exist falls through to the 404 page above. +app.UseSiteFiles(); + app.MapGet("/healthz", (HttpContext context) => { // A health probe must always reach the origin, never a cached copy. @@ -139,6 +143,4 @@ return Results.Redirect("https://aka.ms/aspire/get/install.sh"); }); -app.MapStaticAssets(); - await app.RunAsync(); diff --git a/src/statichost/StaticHost/Site.targets b/src/statichost/StaticHost/Site.targets new file mode 100644 index 0000000000..3eadbdc080 --- /dev/null +++ b/src/statichost/StaticHost/Site.targets @@ -0,0 +1,56 @@ + + + + + .html;.htm;.md;.markdown;.txt;.xml;.json;.webmanifest;.js;.mjs;.jsx;.css;.map;.svg;.ico;.bmp;.otf;.wasm;.csv;.tsv;.ics;.xsl;.xslt + $(MSBuildThisFileDirectory)../compress-site.cs + + $(DOTNET_HOST_PATH) + dotnet + + + + + + <_SiteIntermediateDirectory>$(MSBuildProjectDirectory)/$(IntermediateOutputPath) + <_SiteCompressedDirectory>$(_SiteIntermediateDirectory)site-br/ + <_SiteFileList>$(_SiteIntermediateDirectory)site-files.txt + <_SiteBrotliList>$(_SiteIntermediateDirectory)site-br.txt + + + + + + + + + + + + + + + + + + + diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs new file mode 100644 index 0000000000..772475b9cc --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileETags.cs @@ -0,0 +1,65 @@ +using System.Buffers; +using System.Collections.Concurrent; +using System.IO.Hashing; +using Microsoft.Extensions.FileProviders; + +namespace StaticHost.SiteFiles; + +/// +/// Computes a file's ETag from a hash of its content the first time it is served, then keeps it +/// in memory. +/// +/// +/// The hash is XxHash128: an ETag only has to tell versions of a file apart, which needs no +/// cryptographic hash. A content hash, unlike the length and last-modified time +/// UseStaticFiles uses, stays the same across deployments for files that did not change, +/// though CI writes every file again. An entry is recomputed when the file's length or +/// last-modified time changes, which covers a web root that is replaced while StaticHost runs +/// (CI unpacks the site after StaticHost starts). +/// +internal sealed class SiteFileETags +{ + private const int BufferSize = 64 * 1024; + + // Ordinal, as the file system is in production (Linux): two paths that differ only by case are + // two files there, and must not share an entry. On a case-insensitive disk they are one file + // under two keys, which costs one more hash. Canonical redirects send most requests to one casing. + private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); + + /// The file's path in the web root, which identifies it whatever the provider. + /// The file to hash. + /// Cancels the hashing. + public async ValueTask GetAsync(string path, IFileInfo file, CancellationToken cancellationToken) + { + var length = file.Length; + var lastModified = file.LastModified; + + if (_entries.TryGetValue(path, out var entry) && entry.Length == length && entry.LastModified == lastModified) + { + return entry.ETag; + } + + // Two first requests for the same file may both hash it; the result is the same. + var hash = new XxHash128(); + var buffer = ArrayPool.Shared.Rent(BufferSize); + try + { + await using var stream = file.CreateReadStream(); + int read; + while ((read = await stream.ReadAsync(buffer, cancellationToken)) > 0) + { + hash.Append(buffer.AsSpan(0, read)); + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + + var etag = $"\"{Convert.ToBase64String(hash.GetCurrentHash())}\""; + _entries[path] = new Entry(length, lastModified, etag); + return etag; + } + + private sealed record Entry(long Length, DateTimeOffset LastModified, string ETag); +} diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs b/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs new file mode 100644 index 0000000000..6be7e35cab --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileExtensions.cs @@ -0,0 +1,14 @@ +namespace StaticHost.SiteFiles; + +internal static class SiteFileExtensions +{ + public static IServiceCollection AddSiteFiles(this IServiceCollection services) => + services.AddSingleton(); + + /// + /// Serves the web root's files, preferring the precompressed brotli copy. Call it after + /// UseRouting: it leaves requests that matched an endpoint to that endpoint. + /// + public static IApplicationBuilder UseSiteFiles(this IApplicationBuilder app) => + app.UseMiddleware(); +} diff --git a/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs new file mode 100644 index 0000000000..25d78b3952 --- /dev/null +++ b/src/statichost/StaticHost/SiteFiles/SiteFileMiddleware.cs @@ -0,0 +1,308 @@ +using Microsoft.AspNetCore.Http.Headers; +using Microsoft.AspNetCore.StaticFiles; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Primitives; +using Microsoft.Net.Http.Headers; + +namespace StaticHost.SiteFiles; + +/// +/// Serves the site's files from the web root, with the brotli copy the publish wrote next to a +/// file (page.html.br) when the client accepts it. +/// +/// +/// +/// This replaces MapStaticAssets, which reads a manifest of every file at startup and builds +/// a route for each one. With about 70,000 files, that manifest took a minute of every build and +/// publish, and each instance seconds and gigabytes of memory to start. This middleware keeps no +/// per-file state up front: it looks the file up on each request, as UseStaticFiles does, +/// and computes the ETag the first time a file is served (). +/// +/// +/// It runs after UseRouting and steps aside when an endpoint matched, so the API, health +/// and download endpoints keep precedence over a file of the same path, as they had over the +/// static asset endpoints. +/// +/// +/// A request for one byte range gets it (video players need ranges); other range requests get the +/// whole file. +/// +/// +internal sealed class SiteFileMiddleware +{ + // Images, video and fonts get the SDK's default media caching, as with MapStaticAssets. + internal const string MediaCacheControl = "max-age=3600, must-revalidate"; + + // Astro hashes the names of the files it emits under /_astro/, so they can be cached forever. + // https://starlight.astro.build/environmental-impact/#caching + internal const string ImmutableCacheControl = "max-age=31536000, public, immutable"; + + internal const string DefaultCacheControl = "no-cache"; + + private const string BrotliExtension = ".br"; + + private const string UnknownContentType = "application/octet-stream"; + + private static readonly PathString s_immutablePrefix = "/_astro"; + + private readonly RequestDelegate _next; + private readonly IFileProvider _fileProvider; + private readonly SiteFileETags _etags; + private readonly FileExtensionContentTypeProvider _contentTypes; + + public SiteFileMiddleware(RequestDelegate next, IWebHostEnvironment environment, SiteFileETags etags) + { + _next = next; + _fileProvider = environment.WebRootFileProvider; + _etags = etags; + _contentTypes = CreateContentTypeProvider(); + } + + public async Task InvokeAsync(HttpContext context) + { + var request = context.Request; + if (context.GetEndpoint() is not null + || !(HttpMethods.IsGet(request.Method) || HttpMethods.IsHead(request.Method)) + || !request.Path.HasValue + || request.Path.Value.EndsWith('/')) + { + await _next(context); + return; + } + + var file = _fileProvider.GetFileInfo(request.Path.Value); + if (!file.Exists || file.IsDirectory) + { + await _next(context); + return; + } + + // Every file the site emits is meant to be served, including types this provider does + // not know (Pagefind's index, asciinema casts), as MapStaticAssets did. + if (!_contentTypes.TryGetContentType(request.Path.Value, out var contentType)) + { + contentType = UnknownContentType; + } + + // Look for the brotli copy even when the client does not accept brotli: a file that has one + // must send Vary: Accept-Encoding with both versions, so that a cache does not hand the + // uncompressed one to every client. Nearly every browser accepts brotli anyway. + var compressedPath = request.Path.Value + BrotliExtension; + var compressed = _fileProvider.GetFileInfo(compressedPath); + var hasCompressed = compressed.Exists && !compressed.IsDirectory; + var useCompressed = hasCompressed && AcceptsBrotli(request.Headers.AcceptEncoding); + var served = useCompressed ? compressed : file; + + var response = context.Response; + var etag = await _etags.GetAsync(useCompressed ? compressedPath : request.Path.Value, served, context.RequestAborted); + var entityTag = new EntityTagHeaderValue(etag); + var lastModified = TruncateToSeconds(file.LastModified); + var requestHeaders = request.GetTypedHeaders(); + + var headers = response.Headers; + headers.CacheControl = GetCacheControl(request.Path, contentType); + headers.ETag = etag; + headers.LastModified = HeaderUtilities.FormatDate(lastModified); + if (hasCompressed) + { + headers.Vary = HeaderNames.AcceptEncoding; + } + + if (EvaluatePreconditions(requestHeaders, entityTag, lastModified) is { } status) + { + response.StatusCode = status; + return; + } + + headers.AcceptRanges = "bytes"; + // Last-Modified is the original file's, so a date in If-Range does not vouch for the bytes + // of its brotli copy, which can be rewritten while the original keeps its time. + var range = GetRange(requestHeaders, entityTag, useCompressed ? null : lastModified, served.Length, out var unsatisfiable); + if (unsatisfiable) + { + response.StatusCode = StatusCodes.Status416RangeNotSatisfiable; + headers.ContentRange = $"bytes */{served.Length}"; + return; + } + + response.ContentType = contentType; + if (useCompressed) + { + headers.ContentEncoding = "br"; + } + + var (offset, count) = range ?? (0, served.Length); + if (range is not null) + { + response.StatusCode = StatusCodes.Status206PartialContent; + headers.ContentRange = $"bytes {offset}-{offset + count - 1}/{served.Length}"; + } + else + { + response.StatusCode = StatusCodes.Status200OK; + } + + response.ContentLength = count; + if (HttpMethods.IsHead(request.Method)) + { + return; + } + + await response.SendFileAsync(served, offset, count, context.RequestAborted); + } + + internal static string GetCacheControl(PathString path, string contentType) + { + if (path.StartsWithSegments(s_immutablePrefix, StringComparison.Ordinal)) + { + return ImmutableCacheControl; + } + + return contentType.StartsWith("image/", StringComparison.Ordinal) + || contentType.StartsWith("video/", StringComparison.Ordinal) + || contentType.StartsWith("font/", StringComparison.Ordinal) + ? MediaCacheControl + : DefaultCacheControl; + } + + internal static bool AcceptsBrotli(StringValues acceptEncoding) + { + if (!StringWithQualityHeaderValue.TryParseList(acceptEncoding, out var encodings)) + { + return false; + } + + double? brotli = null; + double? wildcard = null; + foreach (var encoding in encodings) + { + var quality = encoding.Quality ?? 1; + if (StringSegment.Equals(encoding.Value, "br", StringComparison.OrdinalIgnoreCase)) + { + brotli = quality; + } + else if (StringSegment.Equals(encoding.Value, "*", StringComparison.Ordinal)) + { + wildcard = quality; + } + } + + // An explicit "br;q=0" refuses brotli even when "*" accepts everything else. + return (brotli ?? wildcard ?? 0) > 0; + } + + /// + /// Evaluates the request's preconditions in the order of RFC 9110, 13.2.2, as + /// MapStaticAssets did. Returns 412 or 304 when one decides the response, otherwise + /// . + /// + private static int? EvaluatePreconditions(RequestHeaders requestHeaders, EntityTagHeaderValue etag, DateTimeOffset lastModified) + { + // If-Match takes precedence over If-Unmodified-Since, and uses the strong comparison. + var ifMatch = requestHeaders.IfMatch; + if (ifMatch.Count > 0) + { + if (!Matches(ifMatch, etag, useStrongComparison: true)) + { + return StatusCodes.Status412PreconditionFailed; + } + } + else if (requestHeaders.IfUnmodifiedSince is { } ifUnmodifiedSince && lastModified > ifUnmodifiedSince) + { + return StatusCodes.Status412PreconditionFailed; + } + + // If-None-Match takes precedence over If-Modified-Since (RFC 9110, 13.1.3). + var ifNoneMatch = requestHeaders.IfNoneMatch; + if (ifNoneMatch.Count > 0) + { + return Matches(ifNoneMatch, etag, useStrongComparison: false) ? StatusCodes.Status304NotModified : null; + } + + return requestHeaders.IfModifiedSince is { } ifModifiedSince && lastModified <= ifModifiedSince + ? StatusCodes.Status304NotModified + : null; + } + + private static bool Matches(IList candidates, EntityTagHeaderValue etag, bool useStrongComparison) + { + foreach (var candidate in candidates) + { + if (candidate.Equals(EntityTagHeaderValue.Any) || candidate.Compare(etag, useStrongComparison)) + { + return true; + } + } + + return false; + } + + /// + /// Returns the single byte range to send, or to send the whole file, + /// which a server may do for any range request it does not support, such as several ranges. + /// Safari only plays video from servers that answer ranges. + /// + /// + /// The time a date in If-Range must match, or when a date does + /// not identify the bytes sent, so only an entity tag can. + /// + private static (long Offset, long Count)? GetRange( + RequestHeaders requestHeaders, EntityTagHeaderValue etag, DateTimeOffset? lastModified, long length, out bool unsatisfiable) + { + unsatisfiable = false; + var range = requestHeaders.Range; + if (range is null + || !StringSegment.Equals(range.Unit, "bytes", StringComparison.OrdinalIgnoreCase) + || range.Ranges.Count != 1) + { + return null; + } + + // If-Range applies the range only to the representation the client has (RFC 9110, 13.1.5). + if (requestHeaders.IfRange is { } ifRange) + { + var matches = ifRange.EntityTag is { } tag + ? tag.Compare(etag, useStrongComparison: true) + : lastModified is not null && ifRange.LastModified == lastModified; + if (!matches) + { + return null; + } + } + + var item = range.Ranges.First(); + long start, end; + if (item.From is { } from) + { + start = from; + end = Math.Min(item.To ?? long.MaxValue, length - 1); + } + else + { + // A suffix range: the last N bytes. + start = Math.Max(0, length - (item.To ?? 0)); + end = length - 1; + } + + if (start >= length || start > end) + { + unsatisfiable = true; + return null; + } + + return (start, end - start + 1); + } + + private static DateTimeOffset TruncateToSeconds(DateTimeOffset value) => + new(value.Year, value.Month, value.Day, value.Hour, value.Minute, value.Second, value.Offset); + + private static FileExtensionContentTypeProvider CreateContentTypeProvider() + { + var provider = new FileExtensionContentTypeProvider(); + + // The SDK's static web assets mapping, which production sent until now, differs here. + provider.Mappings[".md"] = "text/markdown"; + + return provider; + } +} diff --git a/src/statichost/StaticHost/StaticHost.csproj b/src/statichost/StaticHost/StaticHost.csproj index a836de6f0c..791917b5e4 100644 --- a/src/statichost/StaticHost/StaticHost.csproj +++ b/src/statichost/StaticHost/StaticHost.csproj @@ -8,11 +8,14 @@ true noble-chiseled - - false - brotli - - <_BlazorBrotliCompressionLevel>Optimal + + false + + $(DefaultItemExcludes);wwwroot/** @@ -27,6 +30,7 @@ + @@ -50,5 +54,5 @@ - + diff --git a/src/statichost/compress-site.cs b/src/statichost/compress-site.cs new file mode 100644 index 0000000000..88cec4f419 --- /dev/null +++ b/src/statichost/compress-site.cs @@ -0,0 +1,67 @@ +// Writes a brotli copy of each text file of the site, for StaticHost to serve to clients that +// accept it. StaticHost's publish runs it (StaticHost/Site.targets); it also runs alone: +// +// dotnet run --file compress-site.cs -- +// +// One line per site file: its full path, a tab, its path in the site. +// Where the copies are written, at their path in the site plus ".br". +// The extensions to compress, separated by semicolons. +// Written with the path in the site of each copy worth publishing. + +using System.Collections.Concurrent; +using System.IO.Compression; + +if (args.Length != 4) +{ + Console.Error.WriteLine("Usage: compress-site.cs "); + return 1; +} + +var (filesPath, outputDirectory, extensionList, publishedPath) = (args[0], Path.GetFullPath(args[1]), args[2], args[3]); +var extensions = new HashSet(extensionList.Split(';', StringSplitOptions.RemoveEmptyEntries), StringComparer.OrdinalIgnoreCase); + +var files = File.ReadLines(filesPath) + .Where(static line => line.Length > 0) + .Select(static line => line.Split('\t', 2)) + .Select(static parts => (Source: parts[0], SitePath: parts[1].Replace('\\', '/'))) + .Where(file => extensions.Contains(Path.GetExtension(file.Source))) + .ToList(); + +var published = new ConcurrentBag(); +var compressed = 0; + +Parallel.ForEach(files, file => +{ + // Reuse a copy stamped with its source's exact time, so an unchanged site compresses once. + // "Newer than the source" is not enough: a cache restore or an archive extraction gives + // new content an older time, and the copy of the old content would be published. + var target = Path.Combine(outputDirectory, file.SitePath + ".br"); + var sourceTime = File.GetLastWriteTimeUtc(file.Source); + if (!File.Exists(target) || File.GetLastWriteTimeUtc(target) != sourceTime) + { + Directory.CreateDirectory(Path.GetDirectoryName(target)!); + var temporary = target + ".tmp"; + using (var input = File.OpenRead(file.Source)) + using (var output = File.Create(temporary)) + // Optimal is brotli quality 4, which the SDK used at publish (_BlazorBrotliCompressionLevel). + // SmallestSize (11) saves about a sixth more but compresses about 100 times slower. + using (var brotli = new BrotliStream(output, CompressionLevel.Optimal)) + { + input.CopyTo(brotli); + } + + File.SetLastWriteTimeUtc(temporary, sourceTime); + File.Move(temporary, target, overwrite: true); + Interlocked.Increment(ref compressed); + } + + // A copy that saves nothing (tiny files) is not worth serving. + if (new FileInfo(target).Length < new FileInfo(file.Source).Length) + { + published.Add(file.SitePath + ".br"); + } +}); + +File.WriteAllLines(publishedPath, published.Order(StringComparer.Ordinal)); +Console.WriteLine($"Site: {files.Count} files to compress, {compressed} compressed now, {published.Count} brotli copies published."); +return 0; diff --git a/tests/StaticHost.Tests/AgentReadinessTestServer.cs b/tests/StaticHost.Tests/AgentReadinessTestServer.cs index 1750a6d4e9..b395f548d4 100644 --- a/tests/StaticHost.Tests/AgentReadinessTestServer.cs +++ b/tests/StaticHost.Tests/AgentReadinessTestServer.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using StaticHost.AgentReadiness; using StaticHost.Routing; +using StaticHost.SiteFiles; namespace StaticHost.Tests; @@ -48,6 +49,7 @@ public static async Task StartAsync( { web.UseTestServer(); web.UseWebRoot(wwwroot.Path); + web.ConfigureServices(static services => services.AddSiteFiles()); web.Configure(app => { if (pathBase.HasValue) @@ -60,10 +62,9 @@ public static async Task StartAsync( app.UseCanonicalPathRedirects(); app.UseAgentReadiness(); app.UseDefaultFiles(); - app.UseStaticFiles(); + app.UseSiteFiles(); - // Endpoint that mimics MapStaticAssets fallback: 404 if - // no static asset matched. + // 404 when no file matched, as in production. app.Run(static ctx => { ctx.Response.StatusCode = StatusCodes.Status404NotFound; diff --git a/tests/StaticHost.Tests/SiteFileTests.cs b/tests/StaticHost.Tests/SiteFileTests.cs new file mode 100644 index 0000000000..ede50a08f8 --- /dev/null +++ b/tests/StaticHost.Tests/SiteFileTests.cs @@ -0,0 +1,374 @@ +using System.IO.Compression; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; +using StaticHost.SiteFiles; + +namespace StaticHost.Tests; + +public sealed class SiteFileTests +{ + private const string Page = "" + "Hello, brotli. " + ""; + + [Fact] + public async Task Serves_the_brotli_copy_when_the_client_accepts_it() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + var compressed = await File.ReadAllBytesAsync(Path.Combine(server.WebRoot, "docs", "page", "index.html.br")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.AcceptEncoding.ParseAdd("gzip, deflate, br"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(["br"], response.Content.Headers.ContentEncoding); + Assert.Equal("text/html", response.Content.Headers.ContentType?.MediaType); + Assert.Equal(compressed.Length, response.Content.Headers.ContentLength); + Assert.Contains("Accept-Encoding", response.Headers.Vary); + Assert.Equal(compressed, await response.Content.ReadAsByteArrayAsync()); + Assert.Equal(ETagOf(compressed), response.Headers.ETag?.Tag); + } + + [Fact] + public async Task Serves_the_file_itself_when_the_client_does_not_accept_brotli() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + + using var response = await server.Client.GetAsync("/docs/page/"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Content.Headers.ContentEncoding); + Assert.Contains("Accept-Encoding", response.Headers.Vary); + Assert.Equal(Page, await response.Content.ReadAsStringAsync()); + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes(Page)), response.Headers.ETag?.Tag); + } + + [Fact] + public async Task File_without_a_brotli_copy_has_no_vary_header() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("robots.txt", "User-agent: *\n")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/robots.txt"); + request.Headers.AcceptEncoding.ParseAdd("br"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Empty(response.Content.Headers.ContentEncoding); + Assert.Empty(response.Headers.Vary); + Assert.Equal("User-agent: *\n", await response.Content.ReadAsStringAsync()); + } + + [Theory] + [InlineData("", false)] + [InlineData("br", true)] + [InlineData("gzip, deflate, br, zstd", true)] + [InlineData("BR;q=0.5", true)] + [InlineData("gzip", false)] + [InlineData("br;q=0", false)] + [InlineData("*", true)] + [InlineData("*;q=0", false)] + [InlineData("br;q=0, *", false)] + public void Accepts_brotli_per_the_accept_encoding_header(string acceptEncoding, bool expected) => + Assert.Equal(expected, SiteFileMiddleware.AcceptsBrotli(new StringValues(acceptEncoding))); + + [Fact] + public async Task Matching_if_none_match_returns_not_modified() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var first = await server.Client.GetAsync("/docs/page/"); + var etag = first.Headers.ETag; + Assert.NotNull(etag); + + foreach (var ifNoneMatch in new[] { etag.Tag, $"W/{etag.Tag}", "\"other\", " + etag.Tag, "*" }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("If-None-Match", ifNoneMatch); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.NotModified, response.StatusCode); + Assert.Equal(etag.Tag, response.Headers.ETag?.Tag); + Assert.Equal("no-cache", response.Headers.CacheControl?.ToString()); + Assert.Empty(await response.Content.ReadAsByteArrayAsync()); + } + } + + [Fact] + public async Task Other_if_none_match_returns_the_file() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("If-None-Match", "\"stale\""); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(Page, await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task If_modified_since_applies_only_without_if_none_match() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var first = await server.Client.GetAsync("/docs/page/"); + var lastModified = first.Content.Headers.LastModified; + Assert.NotNull(lastModified); + + using var conditional = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + conditional.Headers.IfModifiedSince = lastModified; + using var notModified = await server.Client.SendAsync(conditional); + Assert.Equal(HttpStatusCode.NotModified, notModified.StatusCode); + + using var both = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + both.Headers.IfModifiedSince = lastModified; + both.Headers.TryAddWithoutValidation("If-None-Match", "\"stale\""); + using var modified = await server.Client.SendAsync(both); + Assert.Equal(HttpStatusCode.OK, modified.StatusCode); + } + + [Fact] + public async Task If_match_and_if_unmodified_since_fail_for_another_version() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var first = await server.Client.GetAsync("/docs/page/"); + var etag = first.Headers.ETag!.Tag; + var lastModified = first.Content.Headers.LastModified!.Value; + + foreach (var (ifMatch, status) in new[] { (etag, HttpStatusCode.OK), ("*", HttpStatusCode.OK), ("\"stale\"", HttpStatusCode.PreconditionFailed), ($"W/{etag}", HttpStatusCode.PreconditionFailed) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.TryAddWithoutValidation("If-Match", ifMatch); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + + foreach (var (ifUnmodifiedSince, status) in new[] { (lastModified, HttpStatusCode.OK), (lastModified.AddDays(-1), HttpStatusCode.PreconditionFailed) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.IfUnmodifiedSince = ifUnmodifiedSince; + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + } + + [Fact] + public async Task If_range_date_does_not_apply_a_range_to_the_brotli_copy() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + using var first = await server.Client.GetAsync("/docs/page/"); + var lastModified = first.Content.Headers.LastModified!.Value; + + foreach (var (acceptEncoding, status) in new[] { ("identity", HttpStatusCode.PartialContent), ("br", HttpStatusCode.OK) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/docs/page/"); + request.Headers.AcceptEncoding.ParseAdd(acceptEncoding); + request.Headers.TryAddWithoutValidation("Range", "bytes=0-1"); + request.Headers.IfRange = new System.Net.Http.Headers.RangeConditionHeaderValue(lastModified); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + } + + [Fact] + public async Task ETag_follows_a_file_that_changes() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("data.json", "{\"v\":1}")); + + using var before = await server.Client.GetAsync("/data.json"); + await File.WriteAllTextAsync(Path.Combine(server.WebRoot, "data.json"), "{\"v\":22}"); + using var after = await server.Client.GetAsync("/data.json"); + + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes("{\"v\":1}")), before.Headers.ETag?.Tag); + Assert.Equal(ETagOf(Encoding.UTF8.GetBytes("{\"v\":22}")), after.Headers.ETag?.Tag); + Assert.Equal("{\"v\":22}", await after.Content.ReadAsStringAsync()); + } + + [Theory] + [InlineData("/_astro/index.Bx1y2z3.js", SiteFileMiddleware.ImmutableCacheControl)] + [InlineData("/_astro/logo.Bx1y2z3.png", SiteFileMiddleware.ImmutableCacheControl)] + [InlineData("/favicon.svg", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/images/hero.webp", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/fonts/site.woff2", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/casts/demo.mp4", SiteFileMiddleware.MediaCacheControl)] + [InlineData("/docs/page/index.html", SiteFileMiddleware.DefaultCacheControl)] + [InlineData("/docs/page.md", SiteFileMiddleware.DefaultCacheControl)] + [InlineData("/pagefind/pagefind.js", SiteFileMiddleware.DefaultCacheControl)] + public async Task Cache_control_depends_on_the_path_and_the_type(string path, string expected) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile(path, "x")); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + // HttpClient reorders the directives it parses; compare what the server sent. + Assert.Equal(expected, response.Headers.NonValidated["Cache-Control"].ToString()); + } + + [Fact] + public async Task Markdown_is_served_as_text_markdown() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SamplePages.SeedFolderPage(root, "docs/page")); + + using var response = await server.Client.GetAsync("/docs/page.md"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("text/markdown", response.Content.Headers.ContentType?.MediaType); + } + + [Fact] + public async Task Head_sends_the_headers_without_the_body() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("docs/page/index.html", Page)); + + using var response = await server.Client.SendAsync(new HttpRequestMessage(HttpMethod.Head, "/docs/page/")); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal(Encoding.UTF8.GetByteCount(Page), response.Content.Headers.ContentLength); + Assert.NotNull(response.Headers.ETag); + Assert.Empty(await response.Content.ReadAsByteArrayAsync()); + } + + [Theory] + [InlineData("bytes=2-5", HttpStatusCode.PartialContent, "2345", "bytes 2-5/10")] + [InlineData("bytes=7-", HttpStatusCode.PartialContent, "789", "bytes 7-9/10")] + [InlineData("bytes=-3", HttpStatusCode.PartialContent, "789", "bytes 7-9/10")] + [InlineData("bytes=8-100", HttpStatusCode.PartialContent, "89", "bytes 8-9/10")] + [InlineData("bytes=0-1, 4-5", HttpStatusCode.OK, "0123456789", null)] + [InlineData("items=0-1", HttpStatusCode.OK, "0123456789", null)] + public async Task Answers_a_single_byte_range(string range, HttpStatusCode status, string body, string? contentRange) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", range); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + Assert.Equal(body, await response.Content.ReadAsStringAsync()); + Assert.Equal(contentRange, response.Content.Headers.ContentRange?.ToString()); + Assert.Equal(["bytes"], response.Headers.AcceptRanges); + } + + [Fact] + public async Task Range_past_the_end_is_not_satisfiable() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + + using var request = new HttpRequestMessage(HttpMethod.Get, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", "bytes=10-"); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(HttpStatusCode.RequestedRangeNotSatisfiable, response.StatusCode); + Assert.Equal("bytes */10", response.Content.Headers.ContentRange?.ToString()); + } + + [Fact] + public async Task If_range_applies_the_range_only_to_the_current_file() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("casts/demo.mp4", "0123456789")); + using var first = await server.Client.GetAsync("/casts/demo.mp4"); + var etag = first.Headers.ETag!.Tag; + + foreach (var (ifRange, status) in new[] { (etag, HttpStatusCode.PartialContent), ("\"stale\"", HttpStatusCode.OK) }) + { + using var request = new HttpRequestMessage(HttpMethod.Get, "/casts/demo.mp4"); + request.Headers.TryAddWithoutValidation("Range", "bytes=0-1"); + request.Headers.TryAddWithoutValidation("If-Range", ifRange); + using var response = await server.Client.SendAsync(request); + + Assert.Equal(status, response.StatusCode); + } + } + + [Theory] + [InlineData("/missing.html")] + [InlineData("/missing/")] + [InlineData("/docs")] + public async Task Missing_files_fall_through(string path) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => SeedCompressedPage(root, "docs/page")); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + + [Theory] + [InlineData("/pagefind/fragment/en_1a2b3c4.pf_fragment")] + [InlineData("/pagefind/wasm.en.pagefind")] + [InlineData("/casts/demo.cast")] + public async Task Files_of_unknown_types_are_served_as_octet_stream(string path) + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile(path, "x")); + + using var response = await server.Client.GetAsync(path); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + Assert.Equal("application/octet-stream", response.Content.Headers.ContentType?.MediaType); + Assert.Equal("x", await response.Content.ReadAsStringAsync()); + } + + [Fact] + public async Task Post_falls_through() + { + await using var server = await AgentReadinessTestServer.StartAsync(root => root.WriteFile("data.json", "{}")); + + using var response = await server.Client.PostAsync("/data.json", new StringContent("{}")); + + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + + [Fact] + public async Task Matched_endpoint_takes_precedence_over_a_file() + { + using var wwwroot = new TempWebRoot(); + wwwroot.WriteFile("api/data.json", "{\"from\":\"file\"}"); + wwwroot.WriteFile("other.json", "{\"from\":\"file\"}"); + + using var host = new HostBuilder() + .ConfigureLogging(static logging => logging.ClearProviders()) + .ConfigureWebHost(web => + { + web.UseTestServer(); + web.UseWebRoot(wwwroot.Path); + web.ConfigureServices(static services => services.AddRouting().AddSiteFiles()); + web.Configure(static app => + { + app.UseRouting(); + app.UseSiteFiles(); + app.UseEndpoints(static endpoints => endpoints.MapGet("/api/data.json", () => Results.Text("endpoint"))); + }); + }) + .Build(); + await host.StartAsync(); + using var client = host.GetTestClient(); + + Assert.Equal("endpoint", await client.GetStringAsync("/api/data.json")); + Assert.Equal("{\"from\":\"file\"}", await client.GetStringAsync("/other.json")); + + await host.StopAsync(); + } + + private static void SeedCompressedPage(TempWebRoot root, string slug) + { + root.WriteFile($"{slug}/index.html", Page); + + using var output = new MemoryStream(); + using (var brotli = new BrotliStream(output, CompressionLevel.SmallestSize, leaveOpen: true)) + { + brotli.Write(Encoding.UTF8.GetBytes(Page)); + } + + File.WriteAllBytes(Path.Combine(root.Path, slug.Replace('/', Path.DirectorySeparatorChar), "index.html.br"), output.ToArray()); + } + + private static string ETagOf(byte[] content) => $"\"{Convert.ToBase64String(System.IO.Hashing.XxHash128.Hash(content))}\""; +}