From afbe5ce82e03ac9bbfe9d6500c47dd91264c599e Mon Sep 17 00:00:00 2001 From: Abhishek Choudhary Date: Thu, 8 Oct 2026 13:38:31 +0545 Subject: [PATCH 1/7] feat: add the request-decompress plugin Decompresses a gzip or deflate request body and rewrites the request with it, so every plugin that reads the body sees the plain content. forward_compressed keeps the upstream receiving a compressed body. Decompression itself lives in core.request.get_body behind opts.decompress, off by default, so callers that need the bytes as they arrived are unaffected. The inflate helper moves to apisix/utils/gzip.lua, which content-decode.lua now shares instead of carrying its own copy. --- apisix/cli/config.lua | 1 + apisix/core/request.lua | 98 +- apisix/plugins/request-decompress.lua | 182 ++++ apisix/utils/content-decode.lua | 32 +- apisix/utils/gzip.lua | 86 ++ docs/en/latest/config.json | 1 + docs/en/latest/plugins/request-decompress.md | 196 ++++ t/admin/plugins.t | 1 + t/core/request.t | 150 +++ t/plugin/request-decompress.t | 923 +++++++++++++++++++ t/utils/gzip.t | 47 + 11 files changed, 1688 insertions(+), 29 deletions(-) create mode 100644 apisix/plugins/request-decompress.lua create mode 100644 apisix/utils/gzip.lua create mode 100644 docs/en/latest/plugins/request-decompress.md create mode 100644 t/plugin/request-decompress.t create mode 100644 t/utils/gzip.t diff --git a/apisix/cli/config.lua b/apisix/cli/config.lua index ab034eeea865..a8d5d0633200 100644 --- a/apisix/cli/config.lua +++ b/apisix/cli/config.lua @@ -217,6 +217,7 @@ local _M = { "referer-restriction", "csrf", "uri-blocker", + "request-decompress", "request-validation", "chaitin-waf", "multi-auth", diff --git a/apisix/core/request.lua b/apisix/core/request.lua index e04adb672ff6..724bc88c3750 100644 --- a/apisix/core/request.lua +++ b/apisix/core/request.lua @@ -25,6 +25,7 @@ local json = require("apisix.core.json") local io = require("apisix.core.io") local multipart = require("multipart") local core_str = require("apisix.core.string") +local inflate_gzip = require("apisix.utils.gzip").inflate_gzip local req_add_header if ngx.config.subsystem == "http" then local ngx_req = require "ngx.req" @@ -37,7 +38,9 @@ local clear_header = ngx.req.clear_header local tonumber = tonumber local error = error local type = type +local ipairs = ipairs local str_fmt = string.format +local str_gmatch = string.gmatch local str_lower = string.lower local req_read_body = ngx.req.read_body local req_get_body_data = ngx.req.get_body_data @@ -281,7 +284,77 @@ local function test_expect(var) end -function _M.get_body(max_size, ctx) +local IDENTITY_ENCODING = "identity" +-- gzip and deflate are both zlib streams, inflate_gzip auto-detects the header +local SUPPORTED_ENCODINGS = { + gzip = true, + deflate = true, +} + +-- advertised in Accept-Encoding when a request is rejected over its encoding +_M.SUPPORTED_CONTENT_ENCODINGS = "gzip, deflate" + + +local function collect_codings(value, codings) + for coding in str_gmatch(value, "[^,%s]+") do + codings[#codings + 1] = str_lower(coding) + end +end + + +-- codings are returned in the order they were applied to the body +local function get_content_encodings(ctx) + local value = _headers(ctx)["content-encoding"] + if not value then + return nil + end + + local codings = {} + if type(value) == "table" then + for _, v in ipairs(value) do + collect_codings(v, codings) + end + else + collect_codings(value, codings) + end + + return codings +end +_M.get_content_encodings = get_content_encodings + + +-- inflates the body in the reverse order the codings were applied. The third +-- return value names the failure so callers can pick a status code. +local function decompress_body(body, max_size, ctx) + local codings = get_content_encodings(ctx) + if not codings then + return body + end + + for _, coding in ipairs(codings) do + if coding ~= IDENTITY_ENCODING and not SUPPORTED_ENCODINGS[coding] then + return nil, "unsupported content encoding: " .. coding, "unsupported_encoding" + end + end + + for i = #codings, 1, -1 do + if codings[i] ~= IDENTITY_ENCODING then + local plain, err, exceeded = inflate_gzip(body, nil, nil, max_size) + if not plain then + if exceeded then + return nil, err, "too_large" + end + return nil, err, "decompress_failed" + end + body = plain + end + end + + return body +end + + +local function read_body(max_size, ctx) if max_size then local var = ctx and ctx.var or ngx.var local content_length = tonumber(var.http_content_length) @@ -336,6 +409,29 @@ function _M.get_body(max_size, ctx) end +--- Read the request body. +-- +-- `max_size` bounds the body read into memory, and bounds the inflated data as +-- well when decompression is on. `opts.decompress` inflates a body sent with a +-- `Content-Encoding` of gzip or deflate; it is off by default, so callers that +-- need the bytes as they arrived, a signature check among them, are unaffected. +-- On failure the third return value is one of `unsupported_encoding`, +-- `decompress_failed` or `too_large`. +function _M.get_body(max_size, ctx, opts) + local body, err = read_body(max_size, ctx) + if err then + return nil, err + end + + -- a bare `return body` keeps the single return value callers may splat + if not body or not (opts and opts.decompress) then + return body + end + + return decompress_body(body, max_size, ctx) +end + + -- get_request_body_table parses the request body according to its Content-Type -- and caches the result in ctx._request_body_table for the lifetime of the request. -- Supported types: application/json, application/x-www-form-urlencoded, diff --git a/apisix/plugins/request-decompress.lua b/apisix/plugins/request-decompress.lua new file mode 100644 index 000000000000..6d554279051f --- /dev/null +++ b/apisix/plugins/request-decompress.lua @@ -0,0 +1,182 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local core = require("apisix.core") +local gzip = require("apisix.utils.gzip") +local ipairs = ipairs +local ngx_req = ngx.req +local tab_concat = table.concat + +local plugin_name = "request-decompress" + +local IDENTITY_ENCODING = "identity" +-- windowBits 15 emits a zlib stream, which is the deflate coding +local DEFLATE_OPTS = {windowBits = 15} + +local DECOMPRESS_OPTS = {decompress = true} +local DECOMPRESS_ERRORS = { + unsupported_encoding = {status = 415}, + decompress_failed = {status = 400, message = "failed to decompress request body"}, + too_large = {status = 413, message = "request body is too large"}, +} + +local schema = { + type = "object", + properties = { + max_req_body_size = { + type = "integer", + minimum = 1, + default = 1048576, + description = "maximum size in bytes of the decompressed request body; " + .. "a body inflating beyond it is rejected with 413", + }, + forward_compressed = { + type = "boolean", + default = false, + description = "forward a compressed body upstream instead of the decompressed " + .. "one; the plugins on the route still read the decompressed body", + }, + }, +} + + +local _M = { + version = 0.1, + priority = 2850, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + return core.schema.check(schema, conf) +end + + +-- re-applies the codings in the order the client applied them. Returns the +-- compressed body and the codings that were actually applied. +local function compress(body, codings) + local applied = {} + + for _, coding in ipairs(codings) do + if coding ~= IDENTITY_ENCODING then + local opts = coding == "deflate" and DEFLATE_OPTS or nil + local compressed, err = gzip.deflate_gzip(body, nil, opts) + if not compressed then + return nil, nil, err + end + body = compressed + applied[#applied + 1] = coding + end + end + + return body, applied +end + + +-- Content-Length is rewritten along with the body +local function clear_content_length_cache(ctx) + local var_cache = ctx.var and ctx.var._cache + if var_cache then + var_cache.http_content_length = nil + end +end + + +function _M.rewrite(conf, ctx) + -- an uncompressed request is left untouched, its body is never read + local codings = core.request.get_content_encodings(ctx) + if not codings then + return + end + + local original, encoding + if conf.forward_compressed then + encoding = core.request.header(ctx, "Content-Encoding") + local raw, raw_err = core.request.get_body(conf.max_req_body_size, ctx) + if raw_err then + core.log.error("failed reading request body, err: ", raw_err) + return 500, {message = "error reading the request body. err: " .. raw_err} + end + original = raw + end + + local body, err, err_kind = core.request.get_body(conf.max_req_body_size, ctx, DECOMPRESS_OPTS) + if err then + core.log.error("failed reading request body, err: ", err) + + local decompress_err = err_kind and DECOMPRESS_ERRORS[err_kind] + if not decompress_err then + return 500, {message = "error reading the request body. err: " .. err} + end + + if err_kind == "unsupported_encoding" then + core.response.set_header("Accept-Encoding", core.request.SUPPORTED_CONTENT_ENCODINGS) + end + return decompress_err.status, {message = decompress_err.message or err} + end + + if body then + ngx_req.set_body_data(body) + end + core.request.set_header(ctx, "Content-Encoding", nil) + + clear_content_length_cache(ctx) + + if conf.forward_compressed and original then + ctx.request_decompress = { + codings = codings, + encoding = encoding, + original = original, + plain = body, + } + end +end + + +function _M.before_proxy(_, ctx) + local state = ctx.request_decompress + if not state then + return + end + + local current = ngx_req.get_body_data() or "" + + -- nothing rewrote the body, so the bytes as they arrived still apply + if current == state.plain then + ngx_req.set_body_data(state.original) + core.request.set_header(ctx, "Content-Encoding", state.encoding) + clear_content_length_cache(ctx) + return + end + + local body, applied, err = compress(current, state.codings) + if not body then + core.log.error("failed to compress request body, err: ", err) + return 500, {message = "failed to compress request body"} + end + + ngx_req.set_body_data(body) + clear_content_length_cache(ctx) + -- the header names the codings applied here, which is shorter than the list + -- the client sent when one of them was identity + local header = #applied > 0 and tab_concat(applied, ", ") or nil + core.request.set_header(ctx, "Content-Encoding", header) +end + + +return _M diff --git a/apisix/utils/content-decode.lua b/apisix/utils/content-decode.lua index c22c965fd865..6768c65b2b89 100644 --- a/apisix/utils/content-decode.lua +++ b/apisix/utils/content-decode.lua @@ -15,39 +15,15 @@ -- limitations under the License. -- local pcall = pcall -local zlib = require("ffi-zlib") +local inflate_gzip = require("apisix.utils.gzip").inflate_gzip local str_buffer = require("string.buffer") local is_br_libs_loaded, brotli = pcall(require, "brotli") -local content_decode_funcs = {} +local content_decode_funcs = { + gzip = inflate_gzip, +} local _M = {} -local function inflate_gzip(data) - local inputs = str_buffer.new():set(data) - local outputs = str_buffer.new() - - local read_inputs = function(size) - local data = inputs:get(size) - if data == "" then - return nil - end - return data - end - - local write_outputs = function(data) - return outputs:put(data) - end - - local ok, err = zlib.inflateGzip(read_inputs, write_outputs) - if not ok then - return nil, "inflate gzip err: " .. err - end - - return outputs:get() -end -content_decode_funcs.gzip = inflate_gzip - - local function brotli_stream_decode(read_inputs, write_outputs) -- read 64k data per times local read_size = 64 * 1024 diff --git a/apisix/utils/gzip.lua b/apisix/utils/gzip.lua new file mode 100644 index 000000000000..725ba9536aa2 --- /dev/null +++ b/apisix/utils/gzip.lua @@ -0,0 +1,86 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local zlib = require("ffi-zlib") +local str_buffer = require("string.buffer") +local _M = {} + + +-- max_output aborts the stream once the inflated data exceeds it, so an +-- oversized payload is never fully buffered. Third return value flags that case. +function _M.inflate_gzip(data, buf_size, opts, max_output) + local inputs = str_buffer.new():set(data) + local outputs = str_buffer.new() + local written = 0 + local exceeded = false + + local read_inputs = function(size) + local data = inputs:get(size) + if data == "" then + return nil + end + return data + end + + local write_outputs = function(data) + if max_output then + written = written + #data + if written > max_output then + exceeded = true + return nil, "output size limit exceeded" + end + end + return outputs:put(data) + end + + local ok, err = zlib.inflateGzip(read_inputs, write_outputs, buf_size, opts) + if not ok then + if exceeded then + return nil, "inflated data is greater than the maximum size " + .. max_output .. " allowed", true + end + return nil, "inflate gzip err: " .. err + end + + return outputs:get() +end + + +function _M.deflate_gzip(data, buf_size, opts) + local inputs = str_buffer.new():set(data) + local outputs = str_buffer.new() + + local read_inputs = function(size) + local data = inputs:get(size) + if data == "" then + return nil + end + return data + end + + local write_outputs = function(data) + return outputs:put(data) + end + + local ok, err = zlib.deflateGzip(read_inputs, write_outputs, buf_size, opts) + if not ok then + return nil, "deflate gzip err: " .. err + end + + return outputs:get() +end + +return _M diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index bf2424a500ac..cdeb53a2e949 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -117,6 +117,7 @@ "plugins/mocking", "plugins/degraphql", "plugins/body-transformer", + "plugins/request-decompress", "plugins/attach-consumer-label", "plugins/exit-transformer" ] diff --git a/docs/en/latest/plugins/request-decompress.md b/docs/en/latest/plugins/request-decompress.md new file mode 100644 index 000000000000..211f24abaf55 --- /dev/null +++ b/docs/en/latest/plugins/request-decompress.md @@ -0,0 +1,196 @@ +--- +title: request-decompress +keywords: + - Apache APISIX + - API Gateway + - Plugin + - REQUEST DECOMPRESS + - request-decompress +description: The request-decompress Plugin decompresses a request body sent with a Content-Encoding of gzip or deflate, so that the other Plugins on the Route and, optionally, the Upstream receive the plain body. +--- + + + +## Description + +The `request-decompress` Plugin decompresses a request body sent with a `Content-Encoding` of `gzip` or `deflate`, which [RFC 9110](https://www.rfc-editor.org/rfc/rfc9110#section-8.4) allows, and rewrites the request with the plain body. Every Plugin that runs after it reads the decompressed content, so `oas-validator`, `request-validation`, `body-transformer`, `chaitin-waf` and the logging Plugins work on a compressed request the same way they do on a plain one. + +A request without a `Content-Encoding` header passes through untouched and its body is never read, so the same Route serves both compressed and uncompressed clients. + +By default the Upstream also receives the decompressed body and the `Content-Encoding` header is removed. Set `forward_compressed` to `true` to send a compressed body Upstream instead, while the Plugins on the Route still read the plain one. + +## Attributes + +| Name | Type | Required | Default | Valid values | Description | +| ---- | ---- | -------- | ------- | ------------ | ----------- | +| max_req_body_size | integer | False | 1048576 | >= 1 | Maximum size in bytes of the decompressed request body. A body that inflates beyond this size is rejected with a `413`. The limit is applied while decompressing, so an oversized payload is never buffered in full. | +| forward_compressed | boolean | False | false | | If true, the Upstream receives a compressed body instead of the decompressed one. | + +## Rejections + +| Condition | Status | Notes | +| --------- | ------ | ----- | +| A coding other than `gzip`, `deflate` or `identity`, such as `br` | `415` | The response carries an `Accept-Encoding: gzip, deflate` header, as [RFC 9110](https://www.rfc-editor.org/rfc/rfc9110#section-12.5.3) recommends. | +| A malformed compressed body | `400` | | +| A body inflating beyond `max_req_body_size` | `413` | | + +## Examples + +The examples below demonstrate how you can configure `request-decompress` for different scenarios. + +:::note + +You can fetch the `admin_key` from `config.yaml` and save to an environment variable with the following command: + +```bash +admin_key=$(yq '.deployment.admin.admin_key[0].key' conf/config.yaml | sed 's/"//g') +``` + +::: + +### Decompress a request body + +The following example demonstrates how to decompress a gzip request body and forward the plain body to the Upstream. + +Create a Route with the `request-decompress` Plugin: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ + -H "X-API-KEY: ${admin_key}" \ + -d '{ + "id": "request-decompress-route", + "uri": "/anything", + "plugins": { + "request-decompress": {} + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "httpbin.org:80": 1 + } + } + }' +``` + +Send a gzip compressed body to the Route: + +```shell +echo -n '{"name":"world"}' | gzip | curl "http://127.0.0.1:9080/anything" -X POST \ + -H "Content-Type: application/json" \ + -H "Content-Encoding: gzip" \ + --data-binary @- +``` + +You should see a response showing that the Upstream received the plain body and no `Content-Encoding` header: + +```json +{ + "data": "{\"name\":\"world\"}", + "headers": { + "Content-Type": "application/json", + ... + }, + ... +} +``` + +### Validate a compressed request body against an OpenAPI specification + +The following example demonstrates how to validate the decompressed body with the [`oas-validator`](./oas-validator.md) Plugin, so that an invalid request is rejected before it reaches the Upstream. + +Create a Route with both Plugins: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ + -H "X-API-KEY: ${admin_key}" \ + -d '{ + "id": "request-decompress-route", + "uri": "/anything", + "plugins": { + "request-decompress": {}, + "oas-validator": { + "spec": "{\"openapi\":\"3.0.2\",\"info\":{\"title\":\"test\",\"version\":\"1.0\"},\"paths\":{\"/anything\":{\"post\":{\"requestBody\":{\"required\":true,\"content\":{\"application/json\":{\"schema\":{\"type\":\"object\",\"required\":[\"name\"],\"properties\":{\"name\":{\"type\":\"string\"}}}}}},\"responses\":{\"200\":{\"description\":\"ok\"}}}}}}" + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "httpbin.org:80": 1 + } + } + }' +``` + +Send a gzip compressed body that does not match the specification: + +```shell +echo -n '{"nickname":"world"}' | gzip | curl -i "http://127.0.0.1:9080/anything" -X POST \ + -H "Content-Type: application/json" \ + -H "Content-Encoding: gzip" \ + --data-binary @- +``` + +You should receive an `HTTP/1.1 400 Bad Request` response, showing that the specification was checked against the decompressed body. + +### Forward a compressed body to the Upstream + +The following example demonstrates how to keep the Upstream receiving a compressed body while the Plugins on the Route still read the plain one. + +Create a Route with `forward_compressed` enabled: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ + -H "X-API-KEY: ${admin_key}" \ + -d '{ + "id": "request-decompress-route", + "uri": "/anything", + "plugins": { + "request-decompress": { + "forward_compressed": true + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "httpbin.org:80": 1 + } + } + }' +``` + +Send a gzip compressed body to the Route: + +```shell +echo -n '{"name":"world"}' | gzip | curl "http://127.0.0.1:9080/anything" -X POST \ + -H "Content-Type: application/json" \ + -H "Content-Encoding: gzip" \ + --data-binary @- +``` + +The Upstream receives the body still compressed, with the `Content-Encoding` header restored. When no Plugin modified the body, the bytes are forwarded exactly as the client sent them. When a Plugin such as `body-transformer` rewrote the body, it is compressed again with the coding the client used, and the header names the coding that was applied. + +## Plugin order + +`request-decompress` runs in the `rewrite` phase with a priority of `2850`. That places it after the Plugins that reject a request cheaply, so a blocked request is never decompressed, and before every Plugin that reads the request body. + +Two consequences are worth knowing, and both can be changed with [`_meta.priority`](../terminology/plugin.md#plugins-execution-order): + +- `hmac-auth` with `validate_request_body` enabled hashes the decompressed body on a Route that also enables this Plugin. A client that signs the compressed bytes needs `hmac-auth` configured with a priority above `2850`. +- Decompression happens before authentication, so an unauthenticated client can cause decompression work. It is bounded by `max_req_body_size` and by `client_max_body_size`. diff --git a/t/admin/plugins.t b/t/admin/plugins.t index b15b64d6966a..66bee2d40475 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -74,6 +74,7 @@ ua-restriction referer-restriction csrf uri-blocker +request-decompress request-validation chaitin-waf multi-auth diff --git a/t/core/request.t b/t/core/request.t index 806959e0d2ab..52a79ea130c3 100644 --- a/t/core/request.t +++ b/t/core/request.t @@ -577,3 +577,153 @@ x-mixed-case: old count: 1 value: new header: new + + + +=== TEST 19: get_body keeps the raw body unless opts.decompress is set +--- config + location /read { + content_by_lua_block { + local core = require("apisix.core") + ngx.ctx.api_ctx = {} + local ctx = ngx.ctx.api_ctx + + local raw = core.request.get_body(nil, ctx) + ngx.say("raw is the plain body: ", raw == [[{"name":"doggie"}]]) + + local plain, err, kind = core.request.get_body(nil, ctx, {decompress = true}) + ngx.say("plain: ", plain) + ngx.say("err: ", err, ", kind: ", kind) + } + } + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/read", { + method = "POST", + body = gzip.deflate_gzip([[{"name":"doggie"}]]), + headers = {["Content-Encoding"] = "gzip"} + }) + if not res then + ngx.say(err) + return + end + ngx.print(res.body) + } + } +--- response_body +raw is the plain body: false +plain: {"name":"doggie"} +err: nil, kind: nil + + + +=== TEST 20: an unsupported coding is reported back +--- config + location /read { + content_by_lua_block { + local core = require("apisix.core") + ngx.ctx.api_ctx = {} + local body, err, kind = core.request.get_body(nil, ngx.ctx.api_ctx, + {decompress = true}) + ngx.say("body: ", body) + ngx.say("err: ", err) + ngx.say("kind: ", kind) + } + } + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/read", { + method = "POST", + body = "whatever", + headers = {["Content-Encoding"] = "br"} + }) + if not res then + ngx.say(err) + return + end + ngx.print(res.body) + } + } +--- response_body +body: nil +err: unsupported content encoding: br +kind: unsupported_encoding + + + +=== TEST 21: a corrupt stream is reported back +--- config + location /read { + content_by_lua_block { + local core = require("apisix.core") + ngx.ctx.api_ctx = {} + local body, err, kind = core.request.get_body(nil, ngx.ctx.api_ctx, + {decompress = true}) + ngx.say("body: ", body) + ngx.say("kind: ", kind) + } + } + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/read", { + method = "POST", + body = body:sub(1, #body - 6), + headers = {["Content-Encoding"] = "gzip"} + }) + if not res then + ngx.say(err) + return + end + ngx.print(res.body) + } + } +--- response_body +body: nil +kind: decompress_failed + + + +=== TEST 22: max_size bounds the inflated body +--- config + location /read { + content_by_lua_block { + local core = require("apisix.core") + ngx.ctx.api_ctx = {} + local body, err, kind = core.request.get_body(64, ngx.ctx.api_ctx, + {decompress = true}) + ngx.say("body: ", body) + ngx.say("err: ", err) + ngx.say("kind: ", kind) + } + } + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local raw = [[{"name":"]] .. string.rep("d", 2048) .. [["}]] + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/read", { + method = "POST", + body = gzip.deflate_gzip(raw), + headers = {["Content-Encoding"] = "gzip"} + }) + if not res then + ngx.say(err) + return + end + ngx.print(res.body) + } + } +--- response_body +body: nil +err: inflated data is greater than the maximum size 64 allowed +kind: too_large diff --git a/t/plugin/request-decompress.t b/t/plugin/request-decompress.t new file mode 100644 index 000000000000..3c2a16db242a --- /dev/null +++ b/t/plugin/request-decompress.t @@ -0,0 +1,923 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_shuffle(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + if (!$block->error_log && !$block->no_error_log) { + $block->set_value("no_error_log", "[error]\n[alert]"); + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: schema defaults +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.request-decompress") + local conf = {} + local ok, err = plugin.check_schema(conf) + if not ok then + ngx.say(err) + return + end + + ngx.say("max_req_body_size: ", conf.max_req_body_size) + ngx.say("forward_compressed: ", conf.forward_compressed) + } + } +--- response_body +max_req_body_size: 1048576 +forward_compressed: false + + + +=== TEST 2: max_req_body_size must be positive +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.request-decompress") + local ok, err = plugin.check_schema({max_req_body_size = 0}) + ngx.say(ok, " ", err) + } + } +--- response_body +false property "max_req_body_size" validation failed: expected 0 to be at least 1 + + + +=== TEST 3: route with the plugin +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = {} + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 4: gzip body is forwarded uncompressed +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +content-encoding: none +upstream body: {"name":"doggie"} + + + +=== TEST 5: deflate body is forwarded uncompressed +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + -- windowBits 15 emits a zlib stream, which is the deflate coding + local body = gzip.deflate_gzip([[{"name":"doggie"}]], nil, {windowBits = 15}) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "deflate" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +content-encoding: none +upstream body: {"name":"doggie"} + + + +=== TEST 6: chained codings are inflated in reverse order +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip(gzip.deflate_gzip([[{"name":"doggie"}]])) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip, gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +content-encoding: none +upstream body: {"name":"doggie"} + + + +=== TEST 7: identity coding leaves the body alone +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = [[{"name":"doggie"}]], + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "identity" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +content-encoding: none +upstream body: {"name":"doggie"} + + + +=== TEST 8: request without Content-Encoding is untouched +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = [[{"name":"doggie"}]], + headers = { + ["Content-Type"] = "application/json" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +upstream body: {"name":"doggie"} + + + +=== TEST 9: Content-Length is rewritten to the decompressed length +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code = t('/apisix/admin/routes/2', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/print_request_received", + plugins = { + ["request-decompress"] = {} + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + if code >= 300 then + ngx.say("route creation failed: ", code) + return + end + + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local httpc = http.new() + local res, err = httpc:request_uri( + "http://127.0.0.1:1984/print_request_received", { + method = "POST", + body = gzip.deflate_gzip([[{"name":"doggie"}]]), + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + for line in res.body:gmatch("[^\n]+") do + if line:find("^content%-length") or line:find("^content%-encoding") then + ngx.say(line) + end + end + + t('/apisix/admin/routes/2', ngx.HTTP_DELETE) + } + } +--- response_body +content-length: 17 + + + +=== TEST 10: corrupt stream is rejected +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = "not a gzip stream at all", + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 400 +{"message":"failed to decompress request body"} +--- error_log +failed reading request body, err: inflate gzip err + + + +=== TEST 11: unsupported coding is rejected +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = [[{"name":"doggie"}]], + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "br" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("accept-encoding: ", res.headers["Accept-Encoding"] or "none") + ngx.print(res.body) + } + } +--- response_body +status: 415 +accept-encoding: gzip, deflate +{"message":"unsupported content encoding: br"} +--- error_log +failed reading request body, err: unsupported content encoding: br + + + +=== TEST 12: route with a small body size limit +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + max_req_body_size = 64 + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 13: body inflating over max_req_body_size is rejected +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip(string.rep("d", 2048)) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 413 +{"message":"request body is too large"} +--- error_log +inflated data is greater than the maximum size 64 allowed + + + +=== TEST 14: route stacking the plugin with oas-validator +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local spec = core.json.encode({ + openapi = "3.0.2", + info = {title = "test", version = "1.0"}, + paths = { + ["/echo"] = { + post = { + requestBody = { + required = true, + content = { + ["application/json"] = { + schema = { + type = "object", + required = {"name"}, + properties = { + name = {type = "string"} + } + } + } + } + }, + responses = {["200"] = {description = "ok"}} + } + } + } + }) + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = {}, + ["oas-validator"] = {spec = spec} + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 15: oas-validator validates the decompressed body +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +upstream body: {"name":"doggie"} + + + +=== TEST 16: a gzip body failing the spec is rejected by oas-validator +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"lol":"watdis?"}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 400 +{"message":"failed to validate request. "} +--- error_log +error occurred while validating request + + + +=== TEST 17: Content-Encoding on a request with no body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = {} + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + if code >= 300 then + ngx.say("route creation failed: ", code) + return + end + + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "GET", + headers = { + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + } + } +--- response_body +status: 200 +content-encoding: none + + + +=== TEST 18: route forwarding a compressed body, with oas-validator +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local spec = core.json.encode({ + openapi = "3.0.2", + info = {title = "test", version = "1.0"}, + paths = { + ["/echo"] = { + post = { + requestBody = { + required = true, + content = { + ["application/json"] = { + schema = { + type = "object", + required = {"name"}, + properties = { + name = {type = "string"} + } + } + } + } + }, + responses = {["200"] = {description = "ok"}} + } + } + } + }) + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + forward_compressed = true + }, + ["oas-validator"] = {spec = spec} + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 19: the upstream receives the bytes as they arrived +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body unchanged: ", res.body == body) + } + } +--- response_body +status: 200 +content-encoding: gzip +upstream body unchanged: true + + + +=== TEST 20: the decompressed body is still validated +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"lol":"watdis?"}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 400 +{"message":"failed to validate request. "} +--- error_log +error occurred while validating request + + + +=== TEST 21: deflate round-trips +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]], nil, {windowBits = 15}) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "deflate" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body unchanged: ", res.body == body) + } + } +--- response_body +status: 200 +content-encoding: deflate +upstream body unchanged: true + + + +=== TEST 22: chained codings round-trip +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip(gzip.deflate_gzip([[{"name":"doggie"}]])) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip, gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body unchanged: ", res.body == body) + } + } +--- response_body +status: 200 +content-encoding: gzip, gzip +upstream body unchanged: true + + + +=== TEST 23: an uncompressed request on the same route is untouched +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = [[{"name":"doggie"}]], + headers = { + ["Content-Type"] = "application/json" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +content-encoding: none +upstream body: {"name":"doggie"} + + + +=== TEST 24: route forwarding a compressed body, with body-transformer +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + forward_compressed = true + }, + ["body-transformer"] = { + request = { + template = [[{"foo":"{{name .. " world"}}","bar":{{age+10}}}]] + } + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 25: a rewritten body is compressed again before it is forwarded +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"hello","age":20}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body unchanged: ", res.body == body) + ngx.say("upstream body inflates to: ", gzip.inflate_gzip(res.body)) + } + } +--- response_body +status: 200 +content-encoding: gzip +upstream body unchanged: false +upstream body inflates to: {"foo":"hello world","bar":30} diff --git a/t/utils/gzip.t b/t/utils/gzip.t new file mode 100644 index 000000000000..8fce4c8a7757 --- /dev/null +++ b/t/utils/gzip.t @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +log_level("info"); + +run_tests; + +__DATA__ + +=== TEST 1: test gzip compression levels +--- config + location /t { + content_by_lua_block { + -- generate a big string of 4MB + local big_raw = string.rep("h", 1024 * 1024 * 4) + ngx.log(ngx.NOTICE, "original size: ", #big_raw) + + local gzip = require("apisix.utils.gzip") + local data_l1, err = gzip.deflate_gzip(big_raw, nil, { level = 1 }) + assert(err == nil) + local data_l9, err = gzip.deflate_gzip(big_raw, nil, { level = 9 }) + assert(err == nil) + + assert(#data_l9 < #data_l1, "expected level 9: " .. #data_l9 .. " < level 1: " .. #data_l1) + } + } +--- request +GET /t +--- error_code: 200 From 2fbbacf2a4c219d2f3d281e28e61ea530983d649 Mon Sep 17 00:00:00 2001 From: Abhishek Choudhary Date: Thu, 8 Oct 2026 14:35:30 +0545 Subject: [PATCH 2/7] fix: read the current body through core before compressing A plugin that moves the body to a temp file leaves ngx.req.get_body_data() nil, and the upstream then received a compressed empty body. --- apisix/plugins/request-decompress.lua | 3 +- t/plugin/request-decompress.t | 78 +++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/apisix/plugins/request-decompress.lua b/apisix/plugins/request-decompress.lua index 6d554279051f..268f7df88efc 100644 --- a/apisix/plugins/request-decompress.lua +++ b/apisix/plugins/request-decompress.lua @@ -154,7 +154,8 @@ function _M.before_proxy(_, ctx) return end - local current = ngx_req.get_body_data() or "" + -- read through core so a body another plugin moved to a file is found too + local current = core.request.get_body(nil, ctx) or "" -- nothing rewrote the body, so the bytes as they arrived still apply if current == state.plain then diff --git a/t/plugin/request-decompress.t b/t/plugin/request-decompress.t index 3c2a16db242a..1b87392b1bd5 100644 --- a/t/plugin/request-decompress.t +++ b/t/plugin/request-decompress.t @@ -921,3 +921,81 @@ status: 200 content-encoding: gzip upstream body unchanged: false upstream body inflates to: {"foo":"hello world","bar":30} + + + +=== TEST 26: route whose body another plugin moves to a file +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local func = [[return function(conf, ctx) + local path = ngx.config.prefix() .. "logs/request-decompress-body.txt" + local f = assert(io.open(path, "w")) + f:write('{"name":"from-file"}') + f:close() + ngx.req.set_body_file(path) + end]] + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + forward_compressed = true + }, + ["serverless-pre-function"] = { + phase = "access", + functions = {func} + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 27: a body held in a file is compressed from the file +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = gzip.deflate_gzip([[{"name":"doggie"}]]), + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body inflates to: ", gzip.inflate_gzip(res.body)) + } + } +--- response_body +status: 200 +content-encoding: gzip +upstream body inflates to: {"name":"from-file"} From 48eae3306c277bda61e1d8b291922f6db286fc2a Mon Sep 17 00:00:00 2001 From: Abhishek Choudhary Date: Thu, 8 Oct 2026 15:54:06 +0545 Subject: [PATCH 3/7] fix: address the review on the request-decompress plugin inflate_gzip consumed only the first member of a concatenated gzip stream, so a multi-member body was silently truncated. It now inflates every member, bounds their total by one cumulative counter, and rejects bytes trailing the last member. The plugin on both a global rule and a route compressed the body twice. run_policy prefer_route hands the request to the route's instance, and the forwarding state is consumed on first use. A body over max_req_body_size answered 500. read_body now classifies a size violation as too_large and get_body passes the kind on, so the Content-Length, chunked and file-backed paths all answer 413. --- apisix/core/request.lua | 15 +- apisix/plugins/request-decompress.lua | 41 +- apisix/utils/gzip.lua | 56 +- docs/en/latest/plugins/request-decompress.md | 10 +- t/plugin/request-decompress.t | 528 +++++++++++++++++++ t/utils/gzip.t | 110 ++++ 6 files changed, 720 insertions(+), 40 deletions(-) diff --git a/apisix/core/request.lua b/apisix/core/request.lua index 724bc88c3750..e70a6e59575c 100644 --- a/apisix/core/request.lua +++ b/apisix/core/request.lua @@ -354,6 +354,8 @@ local function decompress_body(body, max_size, ctx) end +-- the third return value names a size violation, so a caller can answer 413 +-- rather than 500. A read failure is left unclassified. local function read_body(max_size, ctx) if max_size then local var = ctx and ctx.var or ngx.var @@ -368,7 +370,7 @@ local function read_body(max_size, ctx) clear_header("expect") end - return nil, err + return nil, err, "too_large" end end end @@ -379,7 +381,7 @@ local function read_body(max_size, ctx) if req_body then local ok, err = check_size(#req_body, max_size) if not ok then - return nil, err + return nil, err, "too_large" end return req_body @@ -400,7 +402,7 @@ local function read_body(max_size, ctx) local ok, err = check_size(size, max_size) if not ok then - return nil, err + return nil, err, "too_large" end end @@ -416,11 +418,12 @@ end -- `Content-Encoding` of gzip or deflate; it is off by default, so callers that -- need the bytes as they arrived, a signature check among them, are unaffected. -- On failure the third return value is one of `unsupported_encoding`, --- `decompress_failed` or `too_large`. +-- `decompress_failed` or `too_large`. A body over `max_size` is `too_large` +-- whether the limit was reached while reading it or while inflating it. function _M.get_body(max_size, ctx, opts) - local body, err = read_body(max_size, ctx) + local body, err, err_kind = read_body(max_size, ctx) if err then - return nil, err + return nil, err, err_kind end -- a bare `return body` keeps the single return value callers may splat diff --git a/apisix/plugins/request-decompress.lua b/apisix/plugins/request-decompress.lua index 268f7df88efc..bc2dfea669e2 100644 --- a/apisix/plugins/request-decompress.lua +++ b/apisix/plugins/request-decompress.lua @@ -28,7 +28,7 @@ local IDENTITY_ENCODING = "identity" local DEFLATE_OPTS = {windowBits = 15} local DECOMPRESS_OPTS = {decompress = true} -local DECOMPRESS_ERRORS = { +local BODY_ERRORS = { unsupported_encoding = {status = 415}, decompress_failed = {status = 400, message = "failed to decompress request body"}, too_large = {status = 413, message = "request body is too large"}, @@ -59,6 +59,9 @@ local _M = { priority = 2850, name = plugin_name, schema = schema, + -- a route that configures the plugin owns the request, so the instance from + -- a global rule is dropped and the body is handled exactly once + run_policy = "prefer_route", } @@ -67,6 +70,23 @@ function _M.check_schema(conf) end +-- an unclassified cause is a read failure rather than anything the client did +local function body_error(err, err_kind) + core.log.error("failed reading request body, err: ", err) + + local known = err_kind and BODY_ERRORS[err_kind] + if not known then + return 500, {message = "error reading the request body. err: " .. err} + end + + if err_kind == "unsupported_encoding" then + core.response.set_header("Accept-Encoding", core.request.SUPPORTED_CONTENT_ENCODINGS) + end + + return known.status, {message = known.message or err} +end + + -- re-applies the codings in the order the client applied them. Returns the -- compressed body and the codings that were actually applied. local function compress(body, codings) @@ -107,27 +127,16 @@ function _M.rewrite(conf, ctx) local original, encoding if conf.forward_compressed then encoding = core.request.header(ctx, "Content-Encoding") - local raw, raw_err = core.request.get_body(conf.max_req_body_size, ctx) + local raw, raw_err, raw_kind = core.request.get_body(conf.max_req_body_size, ctx) if raw_err then - core.log.error("failed reading request body, err: ", raw_err) - return 500, {message = "error reading the request body. err: " .. raw_err} + return body_error(raw_err, raw_kind) end original = raw end local body, err, err_kind = core.request.get_body(conf.max_req_body_size, ctx, DECOMPRESS_OPTS) if err then - core.log.error("failed reading request body, err: ", err) - - local decompress_err = err_kind and DECOMPRESS_ERRORS[err_kind] - if not decompress_err then - return 500, {message = "error reading the request body. err: " .. err} - end - - if err_kind == "unsupported_encoding" then - core.response.set_header("Accept-Encoding", core.request.SUPPORTED_CONTENT_ENCODINGS) - end - return decompress_err.status, {message = decompress_err.message or err} + return body_error(err, err_kind) end if body then @@ -153,6 +162,8 @@ function _M.before_proxy(_, ctx) if not state then return end + -- the state is an instruction to carry out once, not a description + ctx.request_decompress = nil -- read through core so a body another plugin moved to a file is found too local current = core.request.get_body(nil, ctx) or "" diff --git a/apisix/utils/gzip.lua b/apisix/utils/gzip.lua index 725ba9536aa2..a3a630a41535 100644 --- a/apisix/utils/gzip.lua +++ b/apisix/utils/gzip.lua @@ -16,44 +16,68 @@ -- local zlib = require("ffi-zlib") local str_buffer = require("string.buffer") +local str_sub = string.sub +local Z_OK = zlib.zlib.Z_OK +local DEFAULT_CHUNK = 16384 local _M = {} --- max_output aborts the stream once the inflated data exceeds it, so an --- oversized payload is never fully buffered. Third return value flags that case. +-- A gzip stream may carry several members concatenated (RFC 1952 section 2.2), +-- and one inflate run stops at the end of the first one. Every member is +-- inflated in turn, so a concatenated payload is never silently truncated and +-- max_output bounds their total. max_output aborts the stream as soon as the +-- running total crosses it, so an oversized payload is never fully buffered; +-- the third return value flags that case. Bytes left after the last member are +-- fed to a further inflate, which rejects them. function _M.inflate_gzip(data, buf_size, opts, max_output) - local inputs = str_buffer.new():set(data) + buf_size = buf_size or DEFAULT_CHUNK local outputs = str_buffer.new() local written = 0 local exceeded = false + local pos = 1 local read_inputs = function(size) - local data = inputs:get(size) - if data == "" then + if pos > #data then return nil end - return data + local chunk = str_sub(data, pos, pos + size - 1) + pos = pos + #chunk + return chunk end - local write_outputs = function(data) + local write_outputs = function(chunk) if max_output then - written = written + #data + written = written + #chunk if written > max_output then exceeded = true return nil, "output size limit exceeded" end end - return outputs:put(data) + return outputs:put(chunk) end - local ok, err = zlib.inflateGzip(read_inputs, write_outputs, buf_size, opts) - if not ok then - if exceeded then - return nil, "inflated data is greater than the maximum size " - .. max_output .. " allowed", true + repeat + local stream, inbuf, outbuf = zlib.createStream(buf_size) + + local init = zlib.initInflate(stream, opts) + if init ~= Z_OK then + zlib.zlib.inflateEnd(stream) + return nil, "inflate gzip err: INIT: " .. zlib.zlib_err(init) end - return nil, "inflate gzip err: " .. err - end + + local ok, err = zlib.inflate(read_inputs, write_outputs, buf_size, + stream, inbuf, outbuf) + if not ok then + if exceeded then + return nil, "inflated data is greater than the maximum size " + .. max_output .. " allowed", true + end + return nil, "inflate gzip err: " .. err + end + + -- the member stopped short of the bytes that begin the next one + pos = pos - stream.avail_in + until pos > #data return outputs:get() end diff --git a/docs/en/latest/plugins/request-decompress.md b/docs/en/latest/plugins/request-decompress.md index 211f24abaf55..29bd6c9c6a6e 100644 --- a/docs/en/latest/plugins/request-decompress.md +++ b/docs/en/latest/plugins/request-decompress.md @@ -34,13 +34,15 @@ The `request-decompress` Plugin decompresses a request body sent with a `Content A request without a `Content-Encoding` header passes through untouched and its body is never read, so the same Route serves both compressed and uncompressed clients. +A gzip body may carry several members concatenated, which [RFC 1952 section 2.2](https://www.rfc-editor.org/rfc/rfc1952#section-2.2) allows. Every member is decompressed and `max_req_body_size` bounds their total. + By default the Upstream also receives the decompressed body and the `Content-Encoding` header is removed. Set `forward_compressed` to `true` to send a compressed body Upstream instead, while the Plugins on the Route still read the plain one. ## Attributes | Name | Type | Required | Default | Valid values | Description | | ---- | ---- | -------- | ------- | ------------ | ----------- | -| max_req_body_size | integer | False | 1048576 | >= 1 | Maximum size in bytes of the decompressed request body. A body that inflates beyond this size is rejected with a `413`. The limit is applied while decompressing, so an oversized payload is never buffered in full. | +| max_req_body_size | integer | False | 1048576 | >= 1 | Maximum size in bytes of the request body, applied both to the body as it arrives and to the decompressed result. A body beyond this size is rejected with a `413`. The limit is applied while decompressing, so an oversized payload is never buffered in full. | | forward_compressed | boolean | False | false | | If true, the Upstream receives a compressed body instead of the decompressed one. | ## Rejections @@ -48,8 +50,8 @@ By default the Upstream also receives the decompressed body and the `Content-Enc | Condition | Status | Notes | | --------- | ------ | ----- | | A coding other than `gzip`, `deflate` or `identity`, such as `br` | `415` | The response carries an `Accept-Encoding: gzip, deflate` header, as [RFC 9110](https://www.rfc-editor.org/rfc/rfc9110#section-12.5.3) recommends. | -| A malformed compressed body | `400` | | -| A body inflating beyond `max_req_body_size` | `413` | | +| A malformed compressed body, or bytes trailing the last gzip member | `400` | | +| A body beyond `max_req_body_size`, as received or once decompressed | `413` | | ## Examples @@ -190,6 +192,8 @@ The Upstream receives the body still compressed, with the `Content-Encoding` hea `request-decompress` runs in the `rewrite` phase with a priority of `2850`. That places it after the Plugins that reject a request cheaply, so a blocked request is never decompressed, and before every Plugin that reads the request body. +A Route that configures the Plugin takes precedence over a Global Rule that also configures it, so the request body is handled once with the Route's settings. + Two consequences are worth knowing, and both can be changed with [`_meta.priority`](../terminology/plugin.md#plugins-execution-order): - `hmac-auth` with `validate_request_body` enabled hashes the decompressed body on a Route that also enables this Plugin. A client that signs the compressed bytes needs `hmac-auth` configured with a priority above `2850`. diff --git a/t/plugin/request-decompress.t b/t/plugin/request-decompress.t index 1b87392b1bd5..579f18b4eeee 100644 --- a/t/plugin/request-decompress.t +++ b/t/plugin/request-decompress.t @@ -999,3 +999,531 @@ passed status: 200 content-encoding: gzip upstream body inflates to: {"name":"from-file"} + + + +=== TEST 28: route with the default body size limit +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = {} + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 29: a concatenated stream is forwarded whole +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + -- one JSON document split across two gzip members + local body = gzip.deflate_gzip([[{"name":]]) + .. gzip.deflate_gzip([["doggie"}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +upstream body: {"name":"doggie"} + + + +=== TEST 30: a corrupt later member is rejected +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) .. "not a gzip member" + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 400 +{"message":"failed to decompress request body"} +--- error_log +failed reading request body, err: inflate gzip err + + + +=== TEST 31: bytes trailing the last member are rejected +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) .. "\0" + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 400 +{"message":"failed to decompress request body"} +--- error_log +failed reading request body, err: inflate gzip err + + + +=== TEST 32: route with a small body size limit +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + max_req_body_size = 64 + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 33: members under the limit individually are rejected on their total +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + -- 48 compressed bytes, under the 64 byte limit, holding two + -- members of 40 bytes each, so only their total breaks it + local member = gzip.deflate_gzip(string.rep("d", 40)) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = member .. member, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 413 +{"message":"request body is too large"} +--- error_log +inflated data is greater than the maximum size 64 allowed + + + +=== TEST 34: a raw body over the limit is a 413, Content-Length present +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = string.rep("d", 65), + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 413 +{"message":"request body is too large"} +--- error_log +request size 65 is greater than the maximum size 64 allowed + + + +=== TEST 35: a raw body over the limit is a 413, chunked +--- config + location /t { + content_by_lua_block { + local body = string.rep("d", 65) + local sock = ngx.socket.tcp() + sock:settimeout(5000) + local ok, err = sock:connect("127.0.0.1", 1984) + if not ok then + ngx.say("connect: ", err) + return + end + + local req = "POST /echo HTTP/1.1\r\n" + .. "Host: 127.0.0.1:1984\r\n" + .. "Content-Type: application/json\r\n" + .. "Content-Encoding: gzip\r\n" + .. "Transfer-Encoding: chunked\r\n" + .. "Connection: close\r\n\r\n" + .. string.format("%x\r\n", #body) .. body .. "\r\n0\r\n\r\n" + + local bytes, err = sock:send(req) + if not bytes then + ngx.say("send: ", err) + return + end + + local line, err = sock:receive("*l") + sock:close() + if not line then + ngx.say("receive: ", err) + return + end + ngx.say("status: ", line:match("^HTTP/1%.1 (%d+)")) + } + } +--- response_body +status: 413 +--- error_log +request size 65 is greater than the maximum size 64 allowed + + + +=== TEST 36: a raw body over the limit is a 413, held in a file +--- config + client_body_in_file_only on; + location /t { + content_by_lua_block { + local body = string.rep("d", 65) + local sock = ngx.socket.tcp() + sock:settimeout(5000) + local ok, err = sock:connect("127.0.0.1", 1984) + if not ok then + ngx.say("connect: ", err) + return + end + + local req = "POST /echo HTTP/1.1\r\n" + .. "Host: 127.0.0.1:1984\r\n" + .. "Content-Type: application/json\r\n" + .. "Content-Encoding: gzip\r\n" + .. "Transfer-Encoding: chunked\r\n" + .. "Connection: close\r\n\r\n" + .. string.format("%x\r\n", #body) .. body .. "\r\n0\r\n\r\n" + + local bytes, err = sock:send(req) + if not bytes then + ngx.say("send: ", err) + return + end + + local line, err = sock:receive("*l") + sock:close() + if not line then + ngx.say("receive: ", err) + return + end + ngx.say("status: ", line:match("^HTTP/1%.1 (%d+)")) + } + } +--- response_body +status: 413 +--- error_log +request size 65 is greater than the maximum size 64 allowed + + + +=== TEST 37: route forwarding a compressed body, with a small limit +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + max_req_body_size = 64, + forward_compressed = true + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 38: the limit also applies when a compressed body is forwarded +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = gzip.deflate_gzip(string.rep("d", 2048)), + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 413 +{"message":"request body is too large"} +--- error_log +inflated data is greater than the maximum size 64 allowed + + + +=== TEST 39: the plugin on a global rule and on a route +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + + local code = t('/apisix/admin/global_rules/1', + ngx.HTTP_PUT, + core.json.encode({ + plugins = { + ["request-decompress"] = { + forward_compressed = true + } + } + }) + ) + if code >= 300 then + ngx.say("global rule creation failed: ", code) + return + end + + code = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + forward_compressed = true + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + if code >= 300 then + ngx.say("route creation failed: ", code) + return + end + + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + -- compressed once, not once per plugin instance + ngx.say("upstream body unchanged: ", res.body == body) + ngx.say("upstream body inflates to: ", gzip.inflate_gzip(res.body)) + + t('/apisix/admin/global_rules/1', ngx.HTTP_DELETE) + } + } +--- response_body +status: 200 +content-encoding: gzip +upstream body unchanged: true +upstream body inflates to: {"name":"doggie"} + + + +=== TEST 40: the plugin on a global rule alone +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + + local code = t('/apisix/admin/global_rules/1', + ngx.HTTP_PUT, + core.json.encode({ + plugins = { + ["request-decompress"] = {} + } + }) + ) + if code >= 300 then + ngx.say("global rule creation failed: ", code) + return + end + + code = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + if code >= 300 then + ngx.say("route creation failed: ", code) + return + end + + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = gzip.deflate_gzip([[{"name":"doggie"}]]), + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body: ", res.body) + + t('/apisix/admin/global_rules/1', ngx.HTTP_DELETE) + } + } +--- response_body +status: 200 +content-encoding: none +upstream body: {"name":"doggie"} diff --git a/t/utils/gzip.t b/t/utils/gzip.t index 8fce4c8a7757..1c3f7afc3f79 100644 --- a/t/utils/gzip.t +++ b/t/utils/gzip.t @@ -45,3 +45,113 @@ __DATA__ --- request GET /t --- error_code: 200 + + + +=== TEST 2: a concatenated stream inflates every member +--- config + location /t { + content_by_lua_block { + local gzip = require("apisix.utils.gzip") + local payload = gzip.deflate_gzip([[{"name":]]) + .. gzip.deflate_gzip([["doggie"}]]) + + local out, err = gzip.inflate_gzip(payload) + ngx.say("err: ", err) + ngx.say("out: ", out) + } + } +--- request +GET /t +--- response_body +err: nil +out: {"name":"doggie"} + + + +=== TEST 3: max_output bounds every member together +--- config + location /t { + content_by_lua_block { + local gzip = require("apisix.utils.gzip") + local parts = {} + for _ = 1, 20 do + parts[#parts + 1] = gzip.deflate_gzip(string.rep("d", 100)) + end + local payload = table.concat(parts) + + -- every member inflates to 100 bytes, their total is 2000 + local out, err, exceeded = gzip.inflate_gzip(payload, nil, nil, 128) + ngx.say("out: ", out) + ngx.say("err: ", err) + ngx.say("exceeded: ", exceeded) + + local all = gzip.inflate_gzip(payload, nil, nil, 4096) + ngx.say("under a 4096 cap: ", #all, " bytes") + } + } +--- request +GET /t +--- response_body +out: nil +err: inflated data is greater than the maximum size 128 allowed +exceeded: true +under a 4096 cap: 2000 bytes + + + +=== TEST 4: a corrupt later member is rejected +--- config + location /t { + content_by_lua_block { + local gzip = require("apisix.utils.gzip") + local payload = gzip.deflate_gzip([[{"a":1}]]) .. "not a gzip member" + + local out, err = gzip.inflate_gzip(payload) + ngx.say("out: ", out) + ngx.say("err: ", err) + } + } +--- request +GET /t +--- response_body +out: nil +err: inflate gzip err: INFLATE: data error + + + +=== TEST 5: bytes trailing the last member are rejected +--- config + location /t { + content_by_lua_block { + local gzip = require("apisix.utils.gzip") + local payload = gzip.deflate_gzip([[{"a":1}]]) .. "\0" + + local out, err = gzip.inflate_gzip(payload) + ngx.say("out: ", out) + ngx.say("err: ", err) + } + } +--- request +GET /t +--- response_body +out: nil +err: inflate gzip err: INFLATE: Data error, no input bytes + + + +=== TEST 6: an empty payload is still an error +--- config + location /t { + content_by_lua_block { + local gzip = require("apisix.utils.gzip") + local out, err = gzip.inflate_gzip("") + ngx.say("out: ", out) + ngx.say("err: ", err) + } + } +--- request +GET /t +--- response_body +out: nil +err: inflate gzip err: INFLATE: Data error, no input bytes From fc7338c80ccaa4439e8273c0fe406073a39368ef Mon Sep 17 00:00:00 2001 From: Abhishek Choudhary Date: Fri, 9 Oct 2026 08:26:31 +0545 Subject: [PATCH 4/7] fix: restore every coding and surface a failed body read core.request.header collapses a repeated header to its first value, so a request with two Content-Encoding lines had a twice compressed body restored under a single coding. The header is now rebuilt from the parsed coding list, which makes the stored header value redundant. An unreadable body file left before_proxy compressing an empty string and forwarding it. The read failure is now an error. --- apisix/plugins/request-decompress.lua | 20 ++-- t/plugin/request-decompress.t | 147 ++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 7 deletions(-) diff --git a/apisix/plugins/request-decompress.lua b/apisix/plugins/request-decompress.lua index bc2dfea669e2..7d63b849e688 100644 --- a/apisix/plugins/request-decompress.lua +++ b/apisix/plugins/request-decompress.lua @@ -124,9 +124,8 @@ function _M.rewrite(conf, ctx) return end - local original, encoding + local original if conf.forward_compressed then - encoding = core.request.header(ctx, "Content-Encoding") local raw, raw_err, raw_kind = core.request.get_body(conf.max_req_body_size, ctx) if raw_err then return body_error(raw_err, raw_kind) @@ -149,7 +148,6 @@ function _M.rewrite(conf, ctx) if conf.forward_compressed and original then ctx.request_decompress = { codings = codings, - encoding = encoding, original = original, plain = body, } @@ -166,19 +164,27 @@ function _M.before_proxy(_, ctx) ctx.request_decompress = nil -- read through core so a body another plugin moved to a file is found too - local current = core.request.get_body(nil, ctx) or "" + local current, err = core.request.get_body(nil, ctx) + if err then + core.log.error("failed reading request body, err: ", err) + return 500, {message = "error reading the request body. err: " .. err} + end + -- a body a plugin emptied is not a body that could not be read + current = current or "" -- nothing rewrote the body, so the bytes as they arrived still apply if current == state.plain then ngx_req.set_body_data(state.original) - core.request.set_header(ctx, "Content-Encoding", state.encoding) + -- the codings as the client applied them, which a repeated + -- Content-Encoding header makes longer than any single header line + core.request.set_header(ctx, "Content-Encoding", tab_concat(state.codings, ", ")) clear_content_length_cache(ctx) return end - local body, applied, err = compress(current, state.codings) + local body, applied, compress_err = compress(current, state.codings) if not body then - core.log.error("failed to compress request body, err: ", err) + core.log.error("failed to compress request body, err: ", compress_err) return 500, {message = "failed to compress request body"} end diff --git a/t/plugin/request-decompress.t b/t/plugin/request-decompress.t index 579f18b4eeee..200de94e9e82 100644 --- a/t/plugin/request-decompress.t +++ b/t/plugin/request-decompress.t @@ -1527,3 +1527,150 @@ upstream body inflates to: {"name":"doggie"} status: 200 content-encoding: none upstream body: {"name":"doggie"} + + + +=== TEST 41: route forwarding a compressed body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + forward_compressed = true + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 42: a repeated Content-Encoding header keeps every coding +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip(gzip.deflate_gzip([[{"name":"doggie"}]])) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + -- two header lines rather than one comma separated value + ["Content-Encoding"] = {"gzip", "gzip"} + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body unchanged: ", res.body == body) + } + } +--- response_body +status: 200 +content-encoding: gzip, gzip +upstream body unchanged: true + + + +=== TEST 43: route whose body another plugin makes unreadable +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local func = [[return function(conf, ctx) + local path = ngx.config.prefix() .. "logs/request-decompress-locked.txt" + local f = assert(io.open(path, "w")) + f:write('{"name":"from-file"}') + f:close() + -- set_body_file checks the file is readable, so it is locked + -- only afterwards, leaving the read in before_proxy to fail + ngx.req.set_body_file(path) + os.execute("chmod 000 " .. path) + end]] + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + forward_compressed = true + }, + ["serverless-pre-function"] = { + phase = "access", + functions = {func} + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 44: a body that cannot be read is an error, not an empty body +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = gzip.deflate_gzip([[{"name":"doggie"}]]), + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + } + } +--- response_body +status: 500 +--- error_log +failed reading request body, err: From 8a2cafc1cd76d2153cd0d4b00748d9579fd3560e Mon Sep 17 00:00:00 2001 From: Abhishek Choudhary Date: Fri, 9 Oct 2026 09:29:30 +0545 Subject: [PATCH 5/7] test: remove the body file instead of locking it CI runs as root, which reads straight through a file whose permissions were dropped, so the read in before_proxy succeeded there and the block saw a 200. Removing the file fails the read for every uid. --- t/plugin/request-decompress.t | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/t/plugin/request-decompress.t b/t/plugin/request-decompress.t index 200de94e9e82..593efdb216d6 100644 --- a/t/plugin/request-decompress.t +++ b/t/plugin/request-decompress.t @@ -1598,21 +1598,23 @@ upstream body unchanged: true -=== TEST 43: route whose body another plugin makes unreadable +=== TEST 43: route whose body file another plugin removes --- config location /t { content_by_lua_block { local t = require("lib.test_admin").test local core = require("apisix.core") local func = [[return function(conf, ctx) - local path = ngx.config.prefix() .. "logs/request-decompress-locked.txt" + local path = ngx.config.prefix() .. "logs/request-decompress-gone.txt" local f = assert(io.open(path, "w")) f:write('{"name":"from-file"}') f:close() - -- set_body_file checks the file is readable, so it is locked - -- only afterwards, leaving the read in before_proxy to fail + -- set_body_file checks the file is readable, so it is removed + -- only afterwards, leaving the read in before_proxy to fail. + -- Removing it rather than dropping its permissions, which root + -- would read straight through ngx.req.set_body_file(path) - os.execute("chmod 000 " .. path) + os.remove(path) end]] local code, body = t('/apisix/admin/routes/1', From daa449f14929acbf24c695513ea2f1f78dedcebb Mon Sep 17 00:00:00 2001 From: Abhishek Choudhary Date: Fri, 9 Oct 2026 14:40:38 +0545 Subject: [PATCH 6/7] refactor: keep request body decompression inside the plugin Content-Encoding parsing and inflation lived in core.request behind opts.decompress, with the plugin as the only caller, so both move into the plugin and core.request.get_body keeps its old signature. What stays in core is the size classification: the Content-Length and body-file checks run inside get_body, so only it can tell a body over the cap from a body that could not be read, and the plugin answers 413 or 500 on that. No new API, one extra return value on the failure path. Signed-off-by: Shreemaan Abhishek --- apisix/core/request.lua | 105 ++-------------------- apisix/plugins/request-decompress.lua | 84 ++++++++++++++--- t/core/request.t | 124 +------------------------- 3 files changed, 82 insertions(+), 231 deletions(-) diff --git a/apisix/core/request.lua b/apisix/core/request.lua index e70a6e59575c..97c74cc7239c 100644 --- a/apisix/core/request.lua +++ b/apisix/core/request.lua @@ -25,7 +25,6 @@ local json = require("apisix.core.json") local io = require("apisix.core.io") local multipart = require("multipart") local core_str = require("apisix.core.string") -local inflate_gzip = require("apisix.utils.gzip").inflate_gzip local req_add_header if ngx.config.subsystem == "http" then local ngx_req = require "ngx.req" @@ -38,9 +37,7 @@ local clear_header = ngx.req.clear_header local tonumber = tonumber local error = error local type = type -local ipairs = ipairs local str_fmt = string.format -local str_gmatch = string.gmatch local str_lower = string.lower local req_read_body = ngx.req.read_body local req_get_body_data = ngx.req.get_body_data @@ -284,79 +281,11 @@ local function test_expect(var) end -local IDENTITY_ENCODING = "identity" --- gzip and deflate are both zlib streams, inflate_gzip auto-detects the header -local SUPPORTED_ENCODINGS = { - gzip = true, - deflate = true, -} - --- advertised in Accept-Encoding when a request is rejected over its encoding -_M.SUPPORTED_CONTENT_ENCODINGS = "gzip, deflate" - - -local function collect_codings(value, codings) - for coding in str_gmatch(value, "[^,%s]+") do - codings[#codings + 1] = str_lower(coding) - end -end - - --- codings are returned in the order they were applied to the body -local function get_content_encodings(ctx) - local value = _headers(ctx)["content-encoding"] - if not value then - return nil - end - - local codings = {} - if type(value) == "table" then - for _, v in ipairs(value) do - collect_codings(v, codings) - end - else - collect_codings(value, codings) - end - - return codings -end -_M.get_content_encodings = get_content_encodings - - --- inflates the body in the reverse order the codings were applied. The third --- return value names the failure so callers can pick a status code. -local function decompress_body(body, max_size, ctx) - local codings = get_content_encodings(ctx) - if not codings then - return body - end - - for _, coding in ipairs(codings) do - if coding ~= IDENTITY_ENCODING and not SUPPORTED_ENCODINGS[coding] then - return nil, "unsupported content encoding: " .. coding, "unsupported_encoding" - end - end - - for i = #codings, 1, -1 do - if codings[i] ~= IDENTITY_ENCODING then - local plain, err, exceeded = inflate_gzip(body, nil, nil, max_size) - if not plain then - if exceeded then - return nil, err, "too_large" - end - return nil, err, "decompress_failed" - end - body = plain - end - end - - return body -end - - --- the third return value names a size violation, so a caller can answer 413 --- rather than 500. A read failure is left unclassified. -local function read_body(max_size, ctx) +--- Read the request body. +-- +-- On a body larger than `max_size` the third return value is `too_large`, so a +-- caller can answer 413 rather than 500. A read failure is left unclassified. +function _M.get_body(max_size, ctx) if max_size then local var = ctx and ctx.var or ngx.var local content_length = tonumber(var.http_content_length) @@ -411,30 +340,6 @@ local function read_body(max_size, ctx) end ---- Read the request body. --- --- `max_size` bounds the body read into memory, and bounds the inflated data as --- well when decompression is on. `opts.decompress` inflates a body sent with a --- `Content-Encoding` of gzip or deflate; it is off by default, so callers that --- need the bytes as they arrived, a signature check among them, are unaffected. --- On failure the third return value is one of `unsupported_encoding`, --- `decompress_failed` or `too_large`. A body over `max_size` is `too_large` --- whether the limit was reached while reading it or while inflating it. -function _M.get_body(max_size, ctx, opts) - local body, err, err_kind = read_body(max_size, ctx) - if err then - return nil, err, err_kind - end - - -- a bare `return body` keeps the single return value callers may splat - if not body or not (opts and opts.decompress) then - return body - end - - return decompress_body(body, max_size, ctx) -end - - -- get_request_body_table parses the request body according to its Content-Type -- and caches the result in ctx._request_body_table for the lifetime of the request. -- Supported types: application/json, application/x-www-form-urlencoded, diff --git a/apisix/plugins/request-decompress.lua b/apisix/plugins/request-decompress.lua index 7d63b849e688..20ae59f4c14b 100644 --- a/apisix/plugins/request-decompress.lua +++ b/apisix/plugins/request-decompress.lua @@ -18,7 +18,10 @@ local core = require("apisix.core") local gzip = require("apisix.utils.gzip") local ipairs = ipairs +local type = type local ngx_req = ngx.req +local str_gmatch = string.gmatch +local str_lower = string.lower local tab_concat = table.concat local plugin_name = "request-decompress" @@ -27,7 +30,14 @@ local IDENTITY_ENCODING = "identity" -- windowBits 15 emits a zlib stream, which is the deflate coding local DEFLATE_OPTS = {windowBits = 15} -local DECOMPRESS_OPTS = {decompress = true} +-- gzip and deflate are both zlib streams, inflate_gzip auto-detects the header +local SUPPORTED_ENCODINGS = { + gzip = true, + deflate = true, +} +-- advertised in Accept-Encoding when a request is rejected over its encoding +local SUPPORTED_CONTENT_ENCODINGS = "gzip, deflate" + local BODY_ERRORS = { unsupported_encoding = {status = 415}, decompress_failed = {status = 400, message = "failed to decompress request body"}, @@ -70,6 +80,36 @@ function _M.check_schema(conf) end +local function collect_codings(value, codings) + for coding in str_gmatch(value, "[^,%s]+") do + codings[#codings + 1] = str_lower(coding) + end +end + + +-- codings are returned in the order they were applied to the body. A repeated +-- Content-Encoding header carries as many codings as a single comma-joined one. +local function get_content_encodings(ctx) + -- read from the header table, since core.request.header collapses a + -- repeated header to its first line + local value = core.request.headers(ctx)["content-encoding"] + if not value then + return nil + end + + local codings = {} + if type(value) == "table" then + for _, v in ipairs(value) do + collect_codings(v, codings) + end + else + collect_codings(value, codings) + end + + return codings +end + + -- an unclassified cause is a read failure rather than anything the client did local function body_error(err, err_kind) core.log.error("failed reading request body, err: ", err) @@ -80,13 +120,30 @@ local function body_error(err, err_kind) end if err_kind == "unsupported_encoding" then - core.response.set_header("Accept-Encoding", core.request.SUPPORTED_CONTENT_ENCODINGS) + core.response.set_header("Accept-Encoding", SUPPORTED_CONTENT_ENCODINGS) end return known.status, {message = known.message or err} end +-- inflates the body in the reverse order the codings were applied, bounding +-- their total output by max_size. The third return value names the failure. +local function decompress(body, max_size, codings) + for i = #codings, 1, -1 do + if codings[i] ~= IDENTITY_ENCODING then + local plain, err, exceeded = gzip.inflate_gzip(body, nil, nil, max_size) + if not plain then + return nil, err, exceeded and "too_large" or "decompress_failed" + end + body = plain + end + end + + return body +end + + -- re-applies the codings in the order the client applied them. Returns the -- compressed body and the codings that were actually applied. local function compress(body, codings) @@ -119,28 +176,33 @@ end function _M.rewrite(conf, ctx) -- an uncompressed request is left untouched, its body is never read - local codings = core.request.get_content_encodings(ctx) + local codings = get_content_encodings(ctx) if not codings then return end - local original - if conf.forward_compressed then - local raw, raw_err, raw_kind = core.request.get_body(conf.max_req_body_size, ctx) - if raw_err then - return body_error(raw_err, raw_kind) + -- a coding we cannot inflate is rejected before the body is read + for _, coding in ipairs(codings) do + if coding ~= IDENTITY_ENCODING and not SUPPORTED_ENCODINGS[coding] then + return body_error("unsupported content encoding: " .. coding, + "unsupported_encoding") end - original = raw end - local body, err, err_kind = core.request.get_body(conf.max_req_body_size, ctx, DECOMPRESS_OPTS) + local original, err, err_kind = core.request.get_body(conf.max_req_body_size, ctx) if err then return body_error(err, err_kind) end - if body then + local body + if original then + body, err, err_kind = decompress(original, conf.max_req_body_size, codings) + if not body then + return body_error(err, err_kind) + end ngx_req.set_body_data(body) end + core.request.set_header(ctx, "Content-Encoding", nil) clear_content_length_cache(ctx) diff --git a/t/core/request.t b/t/core/request.t index 52a79ea130c3..a99311aa5499 100644 --- a/t/core/request.t +++ b/t/core/request.t @@ -580,126 +580,13 @@ header: new -=== TEST 19: get_body keeps the raw body unless opts.decompress is set +=== TEST 19: get_body marks a body over max_size as too_large --- config location /read { content_by_lua_block { local core = require("apisix.core") ngx.ctx.api_ctx = {} - local ctx = ngx.ctx.api_ctx - - local raw = core.request.get_body(nil, ctx) - ngx.say("raw is the plain body: ", raw == [[{"name":"doggie"}]]) - - local plain, err, kind = core.request.get_body(nil, ctx, {decompress = true}) - ngx.say("plain: ", plain) - ngx.say("err: ", err, ", kind: ", kind) - } - } - location /t { - content_by_lua_block { - local http = require("resty.http") - local gzip = require("apisix.utils.gzip") - local httpc = http.new() - local res, err = httpc:request_uri("http://127.0.0.1:1984/read", { - method = "POST", - body = gzip.deflate_gzip([[{"name":"doggie"}]]), - headers = {["Content-Encoding"] = "gzip"} - }) - if not res then - ngx.say(err) - return - end - ngx.print(res.body) - } - } ---- response_body -raw is the plain body: false -plain: {"name":"doggie"} -err: nil, kind: nil - - - -=== TEST 20: an unsupported coding is reported back ---- config - location /read { - content_by_lua_block { - local core = require("apisix.core") - ngx.ctx.api_ctx = {} - local body, err, kind = core.request.get_body(nil, ngx.ctx.api_ctx, - {decompress = true}) - ngx.say("body: ", body) - ngx.say("err: ", err) - ngx.say("kind: ", kind) - } - } - location /t { - content_by_lua_block { - local http = require("resty.http") - local httpc = http.new() - local res, err = httpc:request_uri("http://127.0.0.1:1984/read", { - method = "POST", - body = "whatever", - headers = {["Content-Encoding"] = "br"} - }) - if not res then - ngx.say(err) - return - end - ngx.print(res.body) - } - } ---- response_body -body: nil -err: unsupported content encoding: br -kind: unsupported_encoding - - - -=== TEST 21: a corrupt stream is reported back ---- config - location /read { - content_by_lua_block { - local core = require("apisix.core") - ngx.ctx.api_ctx = {} - local body, err, kind = core.request.get_body(nil, ngx.ctx.api_ctx, - {decompress = true}) - ngx.say("body: ", body) - ngx.say("kind: ", kind) - } - } - location /t { - content_by_lua_block { - local http = require("resty.http") - local gzip = require("apisix.utils.gzip") - local body = gzip.deflate_gzip([[{"name":"doggie"}]]) - local httpc = http.new() - local res, err = httpc:request_uri("http://127.0.0.1:1984/read", { - method = "POST", - body = body:sub(1, #body - 6), - headers = {["Content-Encoding"] = "gzip"} - }) - if not res then - ngx.say(err) - return - end - ngx.print(res.body) - } - } ---- response_body -body: nil -kind: decompress_failed - - - -=== TEST 22: max_size bounds the inflated body ---- config - location /read { - content_by_lua_block { - local core = require("apisix.core") - ngx.ctx.api_ctx = {} - local body, err, kind = core.request.get_body(64, ngx.ctx.api_ctx, - {decompress = true}) + local body, err, kind = core.request.get_body(8, ngx.ctx.api_ctx) ngx.say("body: ", body) ngx.say("err: ", err) ngx.say("kind: ", kind) @@ -708,13 +595,10 @@ kind: decompress_failed location /t { content_by_lua_block { local http = require("resty.http") - local gzip = require("apisix.utils.gzip") - local raw = [[{"name":"]] .. string.rep("d", 2048) .. [["}]] local httpc = http.new() local res, err = httpc:request_uri("http://127.0.0.1:1984/read", { method = "POST", - body = gzip.deflate_gzip(raw), - headers = {["Content-Encoding"] = "gzip"} + body = [[{"name":"doggie"}]], }) if not res then ngx.say(err) @@ -725,5 +609,5 @@ kind: decompress_failed } --- response_body body: nil -err: inflated data is greater than the maximum size 64 allowed +err: request size 17 is greater than the maximum size 8 allowed kind: too_large From 776f34ed3cb4afec3e7789d8684dadde723052b9 Mon Sep 17 00:00:00 2001 From: Abhishek Choudhary Date: Mon, 12 Oct 2026 12:22:32 +0545 Subject: [PATCH 7/7] docs: say which phase stops seeing the plain body forward_compressed restores the compressed body before the request is proxied, so the log phase reads it. Two sentences claimed the plain body for every plugin on the route without that qualification. Signed-off-by: Shreemaan Abhishek --- docs/en/latest/plugins/request-decompress.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/en/latest/plugins/request-decompress.md b/docs/en/latest/plugins/request-decompress.md index 29bd6c9c6a6e..d726d6310526 100644 --- a/docs/en/latest/plugins/request-decompress.md +++ b/docs/en/latest/plugins/request-decompress.md @@ -30,13 +30,13 @@ description: The request-decompress Plugin decompresses a request body sent with ## Description -The `request-decompress` Plugin decompresses a request body sent with a `Content-Encoding` of `gzip` or `deflate`, which [RFC 9110](https://www.rfc-editor.org/rfc/rfc9110#section-8.4) allows, and rewrites the request with the plain body. Every Plugin that runs after it reads the decompressed content, so `oas-validator`, `request-validation`, `body-transformer`, `chaitin-waf` and the logging Plugins work on a compressed request the same way they do on a plain one. +The `request-decompress` Plugin decompresses a request body sent with a `Content-Encoding` of `gzip` or `deflate`, which [RFC 9110](https://www.rfc-editor.org/rfc/rfc9110#section-8.4) allows, and rewrites the request with the plain body. Every Plugin that runs after it reads the decompressed content, so `oas-validator`, `request-validation`, `body-transformer`, `chaitin-waf` and the logging Plugins work on a compressed request the same way they do on a plain one, unless `forward_compressed` is enabled. A request without a `Content-Encoding` header passes through untouched and its body is never read, so the same Route serves both compressed and uncompressed clients. A gzip body may carry several members concatenated, which [RFC 1952 section 2.2](https://www.rfc-editor.org/rfc/rfc1952#section-2.2) allows. Every member is decompressed and `max_req_body_size` bounds their total. -By default the Upstream also receives the decompressed body and the `Content-Encoding` header is removed. Set `forward_compressed` to `true` to send a compressed body Upstream instead, while the Plugins on the Route still read the plain one. +By default the Upstream also receives the decompressed body and the `Content-Encoding` header is removed. Set `forward_compressed` to `true` to send a compressed body Upstream instead, while the Plugins that run before the request is proxied still read the plain one. ## Attributes @@ -198,3 +198,5 @@ Two consequences are worth knowing, and both can be changed with [`_meta.priorit - `hmac-auth` with `validate_request_body` enabled hashes the decompressed body on a Route that also enables this Plugin. A client that signs the compressed bytes needs `hmac-auth` configured with a priority above `2850`. - Decompression happens before authentication, so an unauthenticated client can cause decompression work. It is bounded by `max_req_body_size` and by `client_max_body_size`. + +With `forward_compressed` enabled, the compressed body is restored just before the request is proxied, so the Plugins that read the body in the `log` phase see the compressed bytes. `include_req_body` and a `request_body` entry in a custom log format record them as they went Upstream.