diff --git a/crates/vertigo-cli/src/serve/mod.rs b/crates/vertigo-cli/src/serve/mod.rs index 52b5ffb4..5872e9ff 100644 --- a/crates/vertigo-cli/src/serve/mod.rs +++ b/crates/vertigo-cli/src/serve/mod.rs @@ -10,6 +10,7 @@ mod serve_opts; mod serve_run; mod server_state; mod timings; +mod vary; mod vertigo_handler; mod vertigo_install; mod wasm; @@ -23,5 +24,6 @@ pub use serve_run::run; pub use server_state::ServerState; #[cfg(feature = "ssr-timings")] pub use timings::SsrTimings; +pub use vary::vary_accept_encoding; pub use vertigo_handler::{request_accept_language, request_cookie, vertigo_handler}; pub use vertigo_install::{vertigo_files, vertigo_install}; diff --git a/crates/vertigo-cli/src/serve/precompress.rs b/crates/vertigo-cli/src/serve/precompress.rs index a1014bf0..b189d2d9 100644 --- a/crates/vertigo-cli/src/serve/precompress.rs +++ b/crates/vertigo-cli/src/serve/precompress.rs @@ -1,13 +1,19 @@ +use actix_files::NamedFile; use actix_web::{ - HttpRequest, HttpResponse, Resource, - guard::{self, GuardContext}, + HttpRequest, HttpResponse, Resource, guard, http::header::{ - self, AcceptEncoding, ETag, Encoding, EntityTag, Header, HttpDate, IfNoneMatch, - LastModified, + self, AcceptEncoding, ETag, Encoding, EntityTag, Header, HeaderValue, HttpDate, + IfNoneMatch, LastModified, }, web::{self, Bytes}, }; -use std::{collections::HashMap, io::Write, path::Path, sync::Arc, time::Instant}; +use std::{ + collections::HashMap, + io::Write, + path::{Path, PathBuf}, + sync::Arc, + time::Instant, +}; use walkdir::{DirEntry, WalkDir}; use crate::build::get_hash; @@ -16,6 +22,8 @@ use crate::build::get_hash; pub type Precompressed = HashMap; pub struct PrecompressedFile { + /// The file as it is, for the clients not accepting brotli + path: PathBuf, content_type: String, brotli: Bytes, etag: EntityTag, @@ -81,6 +89,7 @@ pub fn precompress(dest_dir: impl AsRef, http_root: &str) -> Precompressed files.insert( format!("{}/{url_path}", http_root.trim_end_matches('/')), PrecompressedFile { + path: path.to_path_buf(), content_type: content_type(extension), etag: EntityTag::new_strong(get_hash(&brotli)), last_modified: entry @@ -137,16 +146,16 @@ fn content_type(extension: &str) -> String { } } -/// Serves the files [`precompress`] compressed to the browsers accepting brotli. Anything else -/// falls through to the next service. +/// Serves the files [`precompress`] compressed: to the browsers accepting brotli compressed, to +/// the other clients as they are. Other paths fall through to the next service. pub fn precompressed_files(http_root: &str, files: Arc) -> Resource { let pattern = format!("{}/{{file:.*}}", http_root.trim_end_matches('/')); let served = files.clone(); web::resource(pattern) - .guard(guard::Get()) + .guard(guard::Any(guard::Get()).or(guard::Head())) .guard(guard::fn_guard(move |ctx| { - served.contains_key(ctx.head().uri.path()) && accepts_brotli(ctx) + served.contains_key(ctx.head().uri.path()) })) .to(move |req: HttpRequest| { let files = files.clone(); @@ -154,22 +163,43 @@ pub fn precompressed_files(http_root: &str, files: Arc) -> Resour }) } -fn accepts_brotli(ctx: &GuardContext) -> bool { - // A range is of the file as it is, which `Files` serves - if ctx.head().headers().contains_key(header::RANGE) { +fn accepts_brotli(req: &HttpRequest) -> bool { + // A range is of the file as it is + if req.headers().contains_key(header::RANGE) { return false; } - ctx.header::() + AcceptEncoding::parse(req) + .ok() .and_then(|accept| accept.negotiate([Encoding::brotli()].iter())) == Some(Encoding::brotli()) } -fn respond(req: &HttpRequest, file: Option<&PrecompressedFile>) -> HttpResponse { +/// Both versions say they depend on `Accept-Encoding`, or a cache on the way (a proxy, a CDN) +/// would give everyone the one it got first - f. ex. the file as it is, which a crawler without +/// `Accept-Encoding` asked for. +fn respond(req: &HttpRequest, file: Option<&PrecompressedFile>) -> actix_web::Result { let Some(file) = file else { - return HttpResponse::NotFound().finish(); + return Ok(HttpResponse::NotFound().finish()); + }; + + let mut response = if accepts_brotli(req) { + brotli_response(req, file) + } else { + // What `Files` would send, it has the same defaults + NamedFile::open(&file.path)?.into_response(req) }; + // `Compress`, compressing the file as it is for a client accepting only gzip, appends one + // more, the same + response + .headers_mut() + .append(header::VARY, HeaderValue::from_static("accept-encoding")); + + Ok(response) +} + +fn brotli_response(req: &HttpRequest, file: &PrecompressedFile) -> HttpResponse { let unchanged = match IfNoneMatch::parse(req) { Ok(IfNoneMatch::Any) => true, Ok(IfNoneMatch::Items(etags)) => etags.iter().any(|etag| etag.weak_eq(&file.etag)), @@ -182,9 +212,7 @@ fn respond(req: &HttpRequest, file: Option<&PrecompressedFile>) -> HttpResponse HttpResponse::Ok() }; - response - .insert_header(ETag(file.etag.clone())) - .insert_header((header::VARY, "accept-encoding")); + response.insert_header(ETag(file.etag.clone())); if let Some(last_modified) = file.last_modified { response.insert_header(LastModified(last_modified)); @@ -208,7 +236,7 @@ mod tests { App, dev::ServiceResponse, http::{ - StatusCode, + Method, StatusCode, header::{self, HeaderName}, }, test::{TestRequest, call_service, init_service, read_body}, @@ -249,10 +277,9 @@ mod tests { ) .await; + let plain = || TestRequest::get().uri("/build/assets/app.js"); let get = |accept_encoding: &str| { - TestRequest::get() - .uri("/build/assets/app.js") - .insert_header((header::ACCEPT_ENCODING, accept_encoding.to_string())) + plain().insert_header((header::ACCEPT_ENCODING, accept_encoding.to_string())) }; let response = call_service(&app, get("gzip, deflate, br, zstd").to_request()).await; @@ -268,27 +295,61 @@ mod tests { assert_eq!(decompressed, script); let revalidation = get("br") - .insert_header((header::IF_NONE_MATCH, etag)) + .insert_header((header::IF_NONE_MATCH, etag.clone())) .to_request(); let response = call_service(&app, revalidation).await; assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + assert_eq!(header(&response, header::VARY), Some("accept-encoding")); assert!(read_body(response).await.is_empty()); - // The file as it is, from `Files` - for accept_encoding in ["gzip", "br;q=0, gzip"] { - let response = call_service(&app, get(accept_encoding).to_request()).await; + let head = TestRequest::default() + .method(Method::HEAD) + .uri("/build/assets/app.js") + .insert_header((header::ACCEPT_ENCODING, "br")) + .to_request(); + let response = call_service(&app, head).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(header(&response, header::CONTENT_ENCODING), Some("br")); + assert_eq!(header(&response, header::VARY), Some("accept-encoding")); + + // The file as it is, also saying it depends on `Accept-Encoding`: a cache that got it + // first, f. ex. for a crawler without the header, must not give it to the browsers + let as_it_is = [get("gzip"), get("br;q=0, gzip"), get("identity"), plain()]; + let mut raw_etag = None; + for request in as_it_is { + let response = call_service(&app, request.to_request()).await; assert_eq!(response.status(), StatusCode::OK); assert_eq!(header(&response, header::CONTENT_ENCODING), None); + assert_eq!(header(&response, header::VARY), Some("accept-encoding")); + raw_etag = header(&response, header::ETAG).map(str::to_string); assert_eq!(read_body(response).await, script.as_bytes()); } + let Some(raw_etag) = raw_etag else { + panic!("the file as it is should carry an ETag"); + }; + assert_ne!(raw_etag, etag); + + let revalidation = plain() + .insert_header((header::IF_NONE_MATCH, raw_etag)) + .to_request(); + let response = call_service(&app, revalidation).await; + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + assert_eq!(header(&response, header::VARY), Some("accept-encoding")); let range = get("br") .insert_header((header::RANGE, "bytes=0-9")) .to_request(); let response = call_service(&app, range).await; assert_eq!(response.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(header(&response, header::VARY), Some("accept-encoding")); assert_eq!(read_body(response).await, script.as_bytes()[..10]); + // Others than the precompressed ones are left to `Files` + let response = + call_service(&app, TestRequest::get().uri("/build/logo.png").to_request()).await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(header(&response, header::VARY), None); + std::fs::remove_dir_all(&dest_dir) } } diff --git a/crates/vertigo-cli/src/serve/serve_run.rs b/crates/vertigo-cli/src/serve/serve_run.rs index a3fb5662..fdf00226 100644 --- a/crates/vertigo-cli/src/serve/serve_run.rs +++ b/crates/vertigo-cli/src/serve/serve_run.rs @@ -1,6 +1,6 @@ use actix_web::{ App, HttpServer, - middleware::{Compress, Condition}, + middleware::{Compress, Condition, from_fn}, rt::System, }; use std::{num::NonZeroUsize, time::Duration}; @@ -13,7 +13,7 @@ use crate::serve::{html::local_origin, mount_path::MountConfig}; use super::{ ServeOpts, ServeOptsInner, proxy::install_proxy, server_state::ServerState, - vertigo_install::vertigo_install, + vary::vary_accept_encoding, vertigo_install::vertigo_install, }; async fn wait_for_port(addr: &str, port: u16) { @@ -73,7 +73,13 @@ pub async fn run(opts: ServeOpts, port_watch: Option) -> Result<(), ErrorCo let app = move || { // Already-encoded responses (what `install_proxy` forwards) carry a `Content-Encoding` // and `Compress` leaves those alone. - let mut app = App::new().wrap(Condition::new(!disable_compression, Compress::default())); + let mut app = App::new() + .wrap(Condition::new(!disable_compression, Compress::default())) + // Wrapped around `Compress`, it sees what `Compress` did + .wrap(Condition::new( + !disable_compression, + from_fn(vary_accept_encoding), + )); for (path, target) in &proxy { app = install_proxy(app, path.clone(), target.clone()); diff --git a/crates/vertigo-cli/src/serve/vary.rs b/crates/vertigo-cli/src/serve/vary.rs new file mode 100644 index 00000000..4174a561 --- /dev/null +++ b/crates/vertigo-cli/src/serve/vary.rs @@ -0,0 +1,226 @@ +use actix_web::{ + Error, + body::MessageBody, + dev::{ServiceRequest, ServiceResponse}, + http::{ + StatusCode, + header::{self, HeaderMap, HeaderValue}, + }, + middleware::Next, + mime::{self, Mime}, +}; + +/// Middleware making every response `Compress` would compress for a client accepting it say +/// `Vary: accept-encoding`, also when sent as it is. +/// +/// `Compress` says it only on what it compresses. Sending the same response as it is, to a client +/// without `Accept-Encoding` (a crawler, curl), it says nothing, and a cache on the way (nginx, +/// a CDN) gives that version to everyone. `vertigo serve` uses it, a back-end with its own +/// `Compress` wraps it around that one, i.e. after it: +/// +/// ```no_run +/// use actix_web::{App, middleware::{Compress, from_fn}}; +/// use vertigo_cli::serve::vary_accept_encoding; +/// +/// let app = App::new() +/// .wrap(Compress::default()) +/// .wrap(from_fn(vary_accept_encoding)); +/// ``` +pub async fn vary_accept_encoding( + req: ServiceRequest, + next: Next, +) -> Result, Error> { + let mut res = next.call(req).await?; + if compressible(&res) && !varies_on_encoding(res.headers()) { + res.headers_mut() + .append(header::VARY, HeaderValue::from_static("accept-encoding")); + } + Ok(res) +} + +/// What `Compress` compresses: a response not encoded already, with content, not an image (but +/// SVG) or a video. A revalidation (304) or a range (206) of it too, the response they stand for +/// is compressed. +fn compressible(res: &ServiceResponse) -> bool { + if matches!( + res.status(), + StatusCode::SWITCHING_PROTOCOLS | StatusCode::NO_CONTENT + ) || res.headers().contains_key(header::CONTENT_ENCODING) + { + return false; + } + let content_type = res + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()); + match content_type { + Some(mime) if mime.type_() == mime::IMAGE => mime.subtype() == mime::SVG, + Some(mime) => mime.type_() != mime::VIDEO, + None => true, + } +} + +/// Already says it, f. ex. the precompressed files or what `Compress` compressed +fn varies_on_encoding(headers: &HeaderMap) -> bool { + headers + .get_all(header::VARY) + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(',')) + .map(str::trim) + .any(|name| name == "*" || name.eq_ignore_ascii_case("accept-encoding")) +} + +#[cfg(test)] +mod tests { + use actix_web::{ + App, HttpResponse, + http::{StatusCode, header}, + middleware::{Compress, from_fn}, + test, web, + }; + + use super::vary_accept_encoding; + + /// `Vary` and `Content-Encoding` of the response to a GET of `uri` + async fn get(uri: &str, accept_encoding: Option<&str>) -> (Vec, Option) { + let respond = |status: StatusCode, content_type: &'static str| { + move || async move { + HttpResponse::build(status) + .content_type(content_type) + .body("

Some content

") + } + }; + let app = test::init_service( + App::new() + .wrap(Compress::default()) + .wrap(from_fn(vary_accept_encoding)) + .route("/page", web::get().to(respond(StatusCode::OK, "text/html"))) + .route( + "/page-revalidated", + web::get().to(|| async { + HttpResponse::NotModified() + .content_type("text/html") + .finish() + }), + ) + .route( + "/page-range", + web::get().to(respond(StatusCode::PARTIAL_CONTENT, "text/html")), + ) + .route( + "/logo.png", + web::get().to(respond(StatusCode::OK, "image/png")), + ) + .route( + "/logo.svg", + web::get().to(respond(StatusCode::OK, "image/svg+xml")), + ) + .route( + "/intro.mp4", + web::get().to(respond(StatusCode::OK, "video/mp4")), + ) + .route( + "/proxied", + web::get().to(|| async { + HttpResponse::Ok() + .content_type("text/html") + .insert_header((header::CONTENT_ENCODING, "gzip")) + .body("gzipped upstream") + }), + ) + .route( + "/per-origin", + web::get().to(|| async { + HttpResponse::Ok() + .content_type("application/json") + .insert_header((header::VARY, "Origin")) + .body("{}") + }), + ) + .route( + "/nothing", + web::get().to(|| async { HttpResponse::NoContent().finish() }), + ), + ) + .await; + + let mut req = test::TestRequest::get().uri(uri); + if let Some(accept_encoding) = accept_encoding { + req = req.insert_header((header::ACCEPT_ENCODING, accept_encoding)); + } + let res = test::call_service(&app, req.to_request()).await; + let header = |name| { + res.headers() + .get_all(name) + .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned()) + .collect::>() + }; + (header(header::VARY), header(header::CONTENT_ENCODING).pop()) + } + + #[actix_web::test] + async fn compressed_or_not_the_response_depends_on_accept_encoding() { + let vary = |values: &[&str]| values.iter().map(|value| value.to_string()).collect(); + + // Sent as it is + for accept_encoding in [None, Some("identity"), Some("gzip;q=0")] { + assert_eq!( + get("/page", accept_encoding).await, + (vary(&["accept-encoding"]), None), + "{accept_encoding:?}" + ); + } + // Compressed, `Compress` says it once + assert_eq!( + get("/page", Some("gzip")).await, + (vary(&["accept-encoding"]), Some("gzip".to_string())) + ); + + // The revalidation and the range of a compressed response + assert_eq!( + get("/page-revalidated", None).await, + (vary(&["accept-encoding"]), None) + ); + assert_eq!( + get("/page-range", Some("gzip")).await, + (vary(&["accept-encoding"]), None) + ); + + assert_eq!( + get("/logo.svg", None).await, + (vary(&["accept-encoding"]), None) + ); + + // To the other `Vary` + assert_eq!( + get("/per-origin", None).await, + (vary(&["Origin", "accept-encoding"]), None) + ); + assert_eq!( + get("/per-origin", Some("gzip")).await, + ( + vary(&["Origin", "accept-encoding"]), + Some("gzip".to_string()) + ) + ); + } + + #[actix_web::test] + async fn never_compressed_is_left_as_it_is() { + for uri in ["/logo.png", "/intro.mp4", "/nothing"] { + for accept_encoding in [None, Some("gzip")] { + assert_eq!( + get(uri, accept_encoding).await, + (vec![], None), + "{uri} {accept_encoding:?}" + ); + } + } + // Encoded by the back-end, which says what it depends on + assert_eq!( + get("/proxied", Some("gzip")).await, + (vec![], Some("gzip".to_string())) + ); + } +} diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 37847dc5..b0353712 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,6 +7,9 @@ * `Driver::history_push` - goes to an address with a new entry in the browser's history, as `history_replace` does without one. +* `serve::vary_accept_encoding` - a middleware for a back-end with its own `Compress`, wrapped + around it: what `Compress` sends as it is says `Vary: accept-encoding` too, as `vertigo serve` + does now. ### Changed @@ -67,6 +70,12 @@ none of its files, e.g. the second of two apps sharing the library. * Tailwind classes (`tw!`, `tw=`) in a library crate: likewise, such a build left them out of the bundle. +* `vertigo serve --precompress` (`MountConfigBuilder::precompress`): a precompressed file sent + as it is - to a client without brotli in `Accept-Encoding`, a range, a revalidation - says + `Vary: accept-encoding` too. Without it a cache on the way (nginx, a CDN) kept the version the + first client got, often a crawler's uncompressed one, and gave it to every browser. +* `vertigo serve`: likewise any response compressed on the fly - pages, files without + `--precompress` - says `Vary: accept-encoding` also when sent as it is. ## 0.14.2 - 2026-10-29