diff --git a/apisix/cli/config.lua b/apisix/cli/config.lua index ab034eeea865..a8d5d0633200 100644 --- a/apisix/cli/config.lua +++ b/apisix/cli/config.lua @@ -217,6 +217,7 @@ local _M = { "referer-restriction", "csrf", "uri-blocker", + "request-decompress", "request-validation", "chaitin-waf", "multi-auth", diff --git a/apisix/core/request.lua b/apisix/core/request.lua index e04adb672ff6..97c74cc7239c 100644 --- a/apisix/core/request.lua +++ b/apisix/core/request.lua @@ -281,6 +281,10 @@ local function test_expect(var) end +--- Read the request body. +-- +-- On a body larger than `max_size` the third return value is `too_large`, so a +-- caller can answer 413 rather than 500. A read failure is left unclassified. function _M.get_body(max_size, ctx) if max_size then local var = ctx and ctx.var or ngx.var @@ -295,7 +299,7 @@ function _M.get_body(max_size, ctx) clear_header("expect") end - return nil, err + return nil, err, "too_large" end end end @@ -306,7 +310,7 @@ function _M.get_body(max_size, ctx) if req_body then local ok, err = check_size(#req_body, max_size) if not ok then - return nil, err + return nil, err, "too_large" end return req_body @@ -327,7 +331,7 @@ function _M.get_body(max_size, ctx) local ok, err = check_size(size, max_size) if not ok then - return nil, err + return nil, err, "too_large" end end diff --git a/apisix/plugins/request-decompress.lua b/apisix/plugins/request-decompress.lua new file mode 100644 index 000000000000..20ae59f4c14b --- /dev/null +++ b/apisix/plugins/request-decompress.lua @@ -0,0 +1,262 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +local core = require("apisix.core") +local gzip = require("apisix.utils.gzip") +local ipairs = ipairs +local type = type +local ngx_req = ngx.req +local str_gmatch = string.gmatch +local str_lower = string.lower +local tab_concat = table.concat + +local plugin_name = "request-decompress" + +local IDENTITY_ENCODING = "identity" +-- windowBits 15 emits a zlib stream, which is the deflate coding +local DEFLATE_OPTS = {windowBits = 15} + +-- gzip and deflate are both zlib streams, inflate_gzip auto-detects the header +local SUPPORTED_ENCODINGS = { + gzip = true, + deflate = true, +} +-- advertised in Accept-Encoding when a request is rejected over its encoding +local SUPPORTED_CONTENT_ENCODINGS = "gzip, deflate" + +local BODY_ERRORS = { + unsupported_encoding = {status = 415}, + decompress_failed = {status = 400, message = "failed to decompress request body"}, + too_large = {status = 413, message = "request body is too large"}, +} + +local schema = { + type = "object", + properties = { + max_req_body_size = { + type = "integer", + minimum = 1, + default = 1048576, + description = "maximum size in bytes of the decompressed request body; " + .. "a body inflating beyond it is rejected with 413", + }, + forward_compressed = { + type = "boolean", + default = false, + description = "forward a compressed body upstream instead of the decompressed " + .. "one; the plugins on the route still read the decompressed body", + }, + }, +} + + +local _M = { + version = 0.1, + priority = 2850, + name = plugin_name, + schema = schema, + -- a route that configures the plugin owns the request, so the instance from + -- a global rule is dropped and the body is handled exactly once + run_policy = "prefer_route", +} + + +function _M.check_schema(conf) + return core.schema.check(schema, conf) +end + + +local function collect_codings(value, codings) + for coding in str_gmatch(value, "[^,%s]+") do + codings[#codings + 1] = str_lower(coding) + end +end + + +-- codings are returned in the order they were applied to the body. A repeated +-- Content-Encoding header carries as many codings as a single comma-joined one. +local function get_content_encodings(ctx) + -- read from the header table, since core.request.header collapses a + -- repeated header to its first line + local value = core.request.headers(ctx)["content-encoding"] + if not value then + return nil + end + + local codings = {} + if type(value) == "table" then + for _, v in ipairs(value) do + collect_codings(v, codings) + end + else + collect_codings(value, codings) + end + + return codings +end + + +-- an unclassified cause is a read failure rather than anything the client did +local function body_error(err, err_kind) + core.log.error("failed reading request body, err: ", err) + + local known = err_kind and BODY_ERRORS[err_kind] + if not known then + return 500, {message = "error reading the request body. err: " .. err} + end + + if err_kind == "unsupported_encoding" then + core.response.set_header("Accept-Encoding", SUPPORTED_CONTENT_ENCODINGS) + end + + return known.status, {message = known.message or err} +end + + +-- inflates the body in the reverse order the codings were applied, bounding +-- their total output by max_size. The third return value names the failure. +local function decompress(body, max_size, codings) + for i = #codings, 1, -1 do + if codings[i] ~= IDENTITY_ENCODING then + local plain, err, exceeded = gzip.inflate_gzip(body, nil, nil, max_size) + if not plain then + return nil, err, exceeded and "too_large" or "decompress_failed" + end + body = plain + end + end + + return body +end + + +-- re-applies the codings in the order the client applied them. Returns the +-- compressed body and the codings that were actually applied. +local function compress(body, codings) + local applied = {} + + for _, coding in ipairs(codings) do + if coding ~= IDENTITY_ENCODING then + local opts = coding == "deflate" and DEFLATE_OPTS or nil + local compressed, err = gzip.deflate_gzip(body, nil, opts) + if not compressed then + return nil, nil, err + end + body = compressed + applied[#applied + 1] = coding + end + end + + return body, applied +end + + +-- Content-Length is rewritten along with the body +local function clear_content_length_cache(ctx) + local var_cache = ctx.var and ctx.var._cache + if var_cache then + var_cache.http_content_length = nil + end +end + + +function _M.rewrite(conf, ctx) + -- an uncompressed request is left untouched, its body is never read + local codings = get_content_encodings(ctx) + if not codings then + return + end + + -- a coding we cannot inflate is rejected before the body is read + for _, coding in ipairs(codings) do + if coding ~= IDENTITY_ENCODING and not SUPPORTED_ENCODINGS[coding] then + return body_error("unsupported content encoding: " .. coding, + "unsupported_encoding") + end + end + + local original, err, err_kind = core.request.get_body(conf.max_req_body_size, ctx) + if err then + return body_error(err, err_kind) + end + + local body + if original then + body, err, err_kind = decompress(original, conf.max_req_body_size, codings) + if not body then + return body_error(err, err_kind) + end + ngx_req.set_body_data(body) + end + + core.request.set_header(ctx, "Content-Encoding", nil) + + clear_content_length_cache(ctx) + + if conf.forward_compressed and original then + ctx.request_decompress = { + codings = codings, + original = original, + plain = body, + } + end +end + + +function _M.before_proxy(_, ctx) + local state = ctx.request_decompress + if not state then + return + end + -- the state is an instruction to carry out once, not a description + ctx.request_decompress = nil + + -- read through core so a body another plugin moved to a file is found too + local current, err = core.request.get_body(nil, ctx) + if err then + core.log.error("failed reading request body, err: ", err) + return 500, {message = "error reading the request body. err: " .. err} + end + -- a body a plugin emptied is not a body that could not be read + current = current or "" + + -- nothing rewrote the body, so the bytes as they arrived still apply + if current == state.plain then + ngx_req.set_body_data(state.original) + -- the codings as the client applied them, which a repeated + -- Content-Encoding header makes longer than any single header line + core.request.set_header(ctx, "Content-Encoding", tab_concat(state.codings, ", ")) + clear_content_length_cache(ctx) + return + end + + local body, applied, compress_err = compress(current, state.codings) + if not body then + core.log.error("failed to compress request body, err: ", compress_err) + return 500, {message = "failed to compress request body"} + end + + ngx_req.set_body_data(body) + clear_content_length_cache(ctx) + -- the header names the codings applied here, which is shorter than the list + -- the client sent when one of them was identity + local header = #applied > 0 and tab_concat(applied, ", ") or nil + core.request.set_header(ctx, "Content-Encoding", header) +end + + +return _M diff --git a/apisix/utils/content-decode.lua b/apisix/utils/content-decode.lua index c22c965fd865..6768c65b2b89 100644 --- a/apisix/utils/content-decode.lua +++ b/apisix/utils/content-decode.lua @@ -15,39 +15,15 @@ -- limitations under the License. -- local pcall = pcall -local zlib = require("ffi-zlib") +local inflate_gzip = require("apisix.utils.gzip").inflate_gzip local str_buffer = require("string.buffer") local is_br_libs_loaded, brotli = pcall(require, "brotli") -local content_decode_funcs = {} +local content_decode_funcs = { + gzip = inflate_gzip, +} local _M = {} -local function inflate_gzip(data) - local inputs = str_buffer.new():set(data) - local outputs = str_buffer.new() - - local read_inputs = function(size) - local data = inputs:get(size) - if data == "" then - return nil - end - return data - end - - local write_outputs = function(data) - return outputs:put(data) - end - - local ok, err = zlib.inflateGzip(read_inputs, write_outputs) - if not ok then - return nil, "inflate gzip err: " .. err - end - - return outputs:get() -end -content_decode_funcs.gzip = inflate_gzip - - local function brotli_stream_decode(read_inputs, write_outputs) -- read 64k data per times local read_size = 64 * 1024 diff --git a/apisix/utils/gzip.lua b/apisix/utils/gzip.lua new file mode 100644 index 000000000000..a3a630a41535 --- /dev/null +++ b/apisix/utils/gzip.lua @@ -0,0 +1,110 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local zlib = require("ffi-zlib") +local str_buffer = require("string.buffer") +local str_sub = string.sub +local Z_OK = zlib.zlib.Z_OK +local DEFAULT_CHUNK = 16384 +local _M = {} + + +-- A gzip stream may carry several members concatenated (RFC 1952 section 2.2), +-- and one inflate run stops at the end of the first one. Every member is +-- inflated in turn, so a concatenated payload is never silently truncated and +-- max_output bounds their total. max_output aborts the stream as soon as the +-- running total crosses it, so an oversized payload is never fully buffered; +-- the third return value flags that case. Bytes left after the last member are +-- fed to a further inflate, which rejects them. +function _M.inflate_gzip(data, buf_size, opts, max_output) + buf_size = buf_size or DEFAULT_CHUNK + local outputs = str_buffer.new() + local written = 0 + local exceeded = false + local pos = 1 + + local read_inputs = function(size) + if pos > #data then + return nil + end + local chunk = str_sub(data, pos, pos + size - 1) + pos = pos + #chunk + return chunk + end + + local write_outputs = function(chunk) + if max_output then + written = written + #chunk + if written > max_output then + exceeded = true + return nil, "output size limit exceeded" + end + end + return outputs:put(chunk) + end + + repeat + local stream, inbuf, outbuf = zlib.createStream(buf_size) + + local init = zlib.initInflate(stream, opts) + if init ~= Z_OK then + zlib.zlib.inflateEnd(stream) + return nil, "inflate gzip err: INIT: " .. zlib.zlib_err(init) + end + + local ok, err = zlib.inflate(read_inputs, write_outputs, buf_size, + stream, inbuf, outbuf) + if not ok then + if exceeded then + return nil, "inflated data is greater than the maximum size " + .. max_output .. " allowed", true + end + return nil, "inflate gzip err: " .. err + end + + -- the member stopped short of the bytes that begin the next one + pos = pos - stream.avail_in + until pos > #data + + return outputs:get() +end + + +function _M.deflate_gzip(data, buf_size, opts) + local inputs = str_buffer.new():set(data) + local outputs = str_buffer.new() + + local read_inputs = function(size) + local data = inputs:get(size) + if data == "" then + return nil + end + return data + end + + local write_outputs = function(data) + return outputs:put(data) + end + + local ok, err = zlib.deflateGzip(read_inputs, write_outputs, buf_size, opts) + if not ok then + return nil, "deflate gzip err: " .. err + end + + return outputs:get() +end + +return _M diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index bf2424a500ac..cdeb53a2e949 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -117,6 +117,7 @@ "plugins/mocking", "plugins/degraphql", "plugins/body-transformer", + "plugins/request-decompress", "plugins/attach-consumer-label", "plugins/exit-transformer" ] diff --git a/docs/en/latest/plugins/request-decompress.md b/docs/en/latest/plugins/request-decompress.md new file mode 100644 index 000000000000..d726d6310526 --- /dev/null +++ b/docs/en/latest/plugins/request-decompress.md @@ -0,0 +1,202 @@ +--- +title: request-decompress +keywords: + - Apache APISIX + - API Gateway + - Plugin + - REQUEST DECOMPRESS + - request-decompress +description: The request-decompress Plugin decompresses a request body sent with a Content-Encoding of gzip or deflate, so that the other Plugins on the Route and, optionally, the Upstream receive the plain body. +--- + + + +## Description + +The `request-decompress` Plugin decompresses a request body sent with a `Content-Encoding` of `gzip` or `deflate`, which [RFC 9110](https://www.rfc-editor.org/rfc/rfc9110#section-8.4) allows, and rewrites the request with the plain body. Every Plugin that runs after it reads the decompressed content, so `oas-validator`, `request-validation`, `body-transformer`, `chaitin-waf` and the logging Plugins work on a compressed request the same way they do on a plain one, unless `forward_compressed` is enabled. + +A request without a `Content-Encoding` header passes through untouched and its body is never read, so the same Route serves both compressed and uncompressed clients. + +A gzip body may carry several members concatenated, which [RFC 1952 section 2.2](https://www.rfc-editor.org/rfc/rfc1952#section-2.2) allows. Every member is decompressed and `max_req_body_size` bounds their total. + +By default the Upstream also receives the decompressed body and the `Content-Encoding` header is removed. Set `forward_compressed` to `true` to send a compressed body Upstream instead, while the Plugins that run before the request is proxied still read the plain one. + +## Attributes + +| Name | Type | Required | Default | Valid values | Description | +| ---- | ---- | -------- | ------- | ------------ | ----------- | +| max_req_body_size | integer | False | 1048576 | >= 1 | Maximum size in bytes of the request body, applied both to the body as it arrives and to the decompressed result. A body beyond this size is rejected with a `413`. The limit is applied while decompressing, so an oversized payload is never buffered in full. | +| forward_compressed | boolean | False | false | | If true, the Upstream receives a compressed body instead of the decompressed one. | + +## Rejections + +| Condition | Status | Notes | +| --------- | ------ | ----- | +| A coding other than `gzip`, `deflate` or `identity`, such as `br` | `415` | The response carries an `Accept-Encoding: gzip, deflate` header, as [RFC 9110](https://www.rfc-editor.org/rfc/rfc9110#section-12.5.3) recommends. | +| A malformed compressed body, or bytes trailing the last gzip member | `400` | | +| A body beyond `max_req_body_size`, as received or once decompressed | `413` | | + +## Examples + +The examples below demonstrate how you can configure `request-decompress` for different scenarios. + +:::note + +You can fetch the `admin_key` from `config.yaml` and save to an environment variable with the following command: + +```bash +admin_key=$(yq '.deployment.admin.admin_key[0].key' conf/config.yaml | sed 's/"//g') +``` + +::: + +### Decompress a request body + +The following example demonstrates how to decompress a gzip request body and forward the plain body to the Upstream. + +Create a Route with the `request-decompress` Plugin: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ + -H "X-API-KEY: ${admin_key}" \ + -d '{ + "id": "request-decompress-route", + "uri": "/anything", + "plugins": { + "request-decompress": {} + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "httpbin.org:80": 1 + } + } + }' +``` + +Send a gzip compressed body to the Route: + +```shell +echo -n '{"name":"world"}' | gzip | curl "http://127.0.0.1:9080/anything" -X POST \ + -H "Content-Type: application/json" \ + -H "Content-Encoding: gzip" \ + --data-binary @- +``` + +You should see a response showing that the Upstream received the plain body and no `Content-Encoding` header: + +```json +{ + "data": "{\"name\":\"world\"}", + "headers": { + "Content-Type": "application/json", + ... + }, + ... +} +``` + +### Validate a compressed request body against an OpenAPI specification + +The following example demonstrates how to validate the decompressed body with the [`oas-validator`](./oas-validator.md) Plugin, so that an invalid request is rejected before it reaches the Upstream. + +Create a Route with both Plugins: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ + -H "X-API-KEY: ${admin_key}" \ + -d '{ + "id": "request-decompress-route", + "uri": "/anything", + "plugins": { + "request-decompress": {}, + "oas-validator": { + "spec": "{\"openapi\":\"3.0.2\",\"info\":{\"title\":\"test\",\"version\":\"1.0\"},\"paths\":{\"/anything\":{\"post\":{\"requestBody\":{\"required\":true,\"content\":{\"application/json\":{\"schema\":{\"type\":\"object\",\"required\":[\"name\"],\"properties\":{\"name\":{\"type\":\"string\"}}}}}},\"responses\":{\"200\":{\"description\":\"ok\"}}}}}}" + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "httpbin.org:80": 1 + } + } + }' +``` + +Send a gzip compressed body that does not match the specification: + +```shell +echo -n '{"nickname":"world"}' | gzip | curl -i "http://127.0.0.1:9080/anything" -X POST \ + -H "Content-Type: application/json" \ + -H "Content-Encoding: gzip" \ + --data-binary @- +``` + +You should receive an `HTTP/1.1 400 Bad Request` response, showing that the specification was checked against the decompressed body. + +### Forward a compressed body to the Upstream + +The following example demonstrates how to keep the Upstream receiving a compressed body while the Plugins on the Route still read the plain one. + +Create a Route with `forward_compressed` enabled: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes" -X PUT \ + -H "X-API-KEY: ${admin_key}" \ + -d '{ + "id": "request-decompress-route", + "uri": "/anything", + "plugins": { + "request-decompress": { + "forward_compressed": true + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "httpbin.org:80": 1 + } + } + }' +``` + +Send a gzip compressed body to the Route: + +```shell +echo -n '{"name":"world"}' | gzip | curl "http://127.0.0.1:9080/anything" -X POST \ + -H "Content-Type: application/json" \ + -H "Content-Encoding: gzip" \ + --data-binary @- +``` + +The Upstream receives the body still compressed, with the `Content-Encoding` header restored. When no Plugin modified the body, the bytes are forwarded exactly as the client sent them. When a Plugin such as `body-transformer` rewrote the body, it is compressed again with the coding the client used, and the header names the coding that was applied. + +## Plugin order + +`request-decompress` runs in the `rewrite` phase with a priority of `2850`. That places it after the Plugins that reject a request cheaply, so a blocked request is never decompressed, and before every Plugin that reads the request body. + +A Route that configures the Plugin takes precedence over a Global Rule that also configures it, so the request body is handled once with the Route's settings. + +Two consequences are worth knowing, and both can be changed with [`_meta.priority`](../terminology/plugin.md#plugins-execution-order): + +- `hmac-auth` with `validate_request_body` enabled hashes the decompressed body on a Route that also enables this Plugin. A client that signs the compressed bytes needs `hmac-auth` configured with a priority above `2850`. +- Decompression happens before authentication, so an unauthenticated client can cause decompression work. It is bounded by `max_req_body_size` and by `client_max_body_size`. + +With `forward_compressed` enabled, the compressed body is restored just before the request is proxied, so the Plugins that read the body in the `log` phase see the compressed bytes. `include_req_body` and a `request_body` entry in a custom log format record them as they went Upstream. diff --git a/t/admin/plugins.t b/t/admin/plugins.t index b15b64d6966a..66bee2d40475 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -74,6 +74,7 @@ ua-restriction referer-restriction csrf uri-blocker +request-decompress request-validation chaitin-waf multi-auth diff --git a/t/core/request.t b/t/core/request.t index 806959e0d2ab..a99311aa5499 100644 --- a/t/core/request.t +++ b/t/core/request.t @@ -577,3 +577,37 @@ x-mixed-case: old count: 1 value: new header: new + + + +=== TEST 19: get_body marks a body over max_size as too_large +--- config + location /read { + content_by_lua_block { + local core = require("apisix.core") + ngx.ctx.api_ctx = {} + local body, err, kind = core.request.get_body(8, ngx.ctx.api_ctx) + ngx.say("body: ", body) + ngx.say("err: ", err) + ngx.say("kind: ", kind) + } + } + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/read", { + method = "POST", + body = [[{"name":"doggie"}]], + }) + if not res then + ngx.say(err) + return + end + ngx.print(res.body) + } + } +--- response_body +body: nil +err: request size 17 is greater than the maximum size 8 allowed +kind: too_large diff --git a/t/plugin/request-decompress.t b/t/plugin/request-decompress.t new file mode 100644 index 000000000000..593efdb216d6 --- /dev/null +++ b/t/plugin/request-decompress.t @@ -0,0 +1,1678 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_shuffle(); +no_root_location(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + if (!$block->error_log && !$block->no_error_log) { + $block->set_value("no_error_log", "[error]\n[alert]"); + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: schema defaults +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.request-decompress") + local conf = {} + local ok, err = plugin.check_schema(conf) + if not ok then + ngx.say(err) + return + end + + ngx.say("max_req_body_size: ", conf.max_req_body_size) + ngx.say("forward_compressed: ", conf.forward_compressed) + } + } +--- response_body +max_req_body_size: 1048576 +forward_compressed: false + + + +=== TEST 2: max_req_body_size must be positive +--- config + location /t { + content_by_lua_block { + local plugin = require("apisix.plugins.request-decompress") + local ok, err = plugin.check_schema({max_req_body_size = 0}) + ngx.say(ok, " ", err) + } + } +--- response_body +false property "max_req_body_size" validation failed: expected 0 to be at least 1 + + + +=== TEST 3: route with the plugin +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = {} + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 4: gzip body is forwarded uncompressed +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +content-encoding: none +upstream body: {"name":"doggie"} + + + +=== TEST 5: deflate body is forwarded uncompressed +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + -- windowBits 15 emits a zlib stream, which is the deflate coding + local body = gzip.deflate_gzip([[{"name":"doggie"}]], nil, {windowBits = 15}) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "deflate" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +content-encoding: none +upstream body: {"name":"doggie"} + + + +=== TEST 6: chained codings are inflated in reverse order +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip(gzip.deflate_gzip([[{"name":"doggie"}]])) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip, gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +content-encoding: none +upstream body: {"name":"doggie"} + + + +=== TEST 7: identity coding leaves the body alone +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = [[{"name":"doggie"}]], + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "identity" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +content-encoding: none +upstream body: {"name":"doggie"} + + + +=== TEST 8: request without Content-Encoding is untouched +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = [[{"name":"doggie"}]], + headers = { + ["Content-Type"] = "application/json" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +upstream body: {"name":"doggie"} + + + +=== TEST 9: Content-Length is rewritten to the decompressed length +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code = t('/apisix/admin/routes/2', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/print_request_received", + plugins = { + ["request-decompress"] = {} + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + if code >= 300 then + ngx.say("route creation failed: ", code) + return + end + + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local httpc = http.new() + local res, err = httpc:request_uri( + "http://127.0.0.1:1984/print_request_received", { + method = "POST", + body = gzip.deflate_gzip([[{"name":"doggie"}]]), + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + for line in res.body:gmatch("[^\n]+") do + if line:find("^content%-length") or line:find("^content%-encoding") then + ngx.say(line) + end + end + + t('/apisix/admin/routes/2', ngx.HTTP_DELETE) + } + } +--- response_body +content-length: 17 + + + +=== TEST 10: corrupt stream is rejected +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = "not a gzip stream at all", + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 400 +{"message":"failed to decompress request body"} +--- error_log +failed reading request body, err: inflate gzip err + + + +=== TEST 11: unsupported coding is rejected +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = [[{"name":"doggie"}]], + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "br" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("accept-encoding: ", res.headers["Accept-Encoding"] or "none") + ngx.print(res.body) + } + } +--- response_body +status: 415 +accept-encoding: gzip, deflate +{"message":"unsupported content encoding: br"} +--- error_log +failed reading request body, err: unsupported content encoding: br + + + +=== TEST 12: route with a small body size limit +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + max_req_body_size = 64 + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 13: body inflating over max_req_body_size is rejected +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip(string.rep("d", 2048)) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 413 +{"message":"request body is too large"} +--- error_log +inflated data is greater than the maximum size 64 allowed + + + +=== TEST 14: route stacking the plugin with oas-validator +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local spec = core.json.encode({ + openapi = "3.0.2", + info = {title = "test", version = "1.0"}, + paths = { + ["/echo"] = { + post = { + requestBody = { + required = true, + content = { + ["application/json"] = { + schema = { + type = "object", + required = {"name"}, + properties = { + name = {type = "string"} + } + } + } + } + }, + responses = {["200"] = {description = "ok"}} + } + } + } + }) + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = {}, + ["oas-validator"] = {spec = spec} + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 15: oas-validator validates the decompressed body +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +upstream body: {"name":"doggie"} + + + +=== TEST 16: a gzip body failing the spec is rejected by oas-validator +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"lol":"watdis?"}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 400 +{"message":"failed to validate request. "} +--- error_log +error occurred while validating request + + + +=== TEST 17: Content-Encoding on a request with no body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = {} + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + if code >= 300 then + ngx.say("route creation failed: ", code) + return + end + + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "GET", + headers = { + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + } + } +--- response_body +status: 200 +content-encoding: none + + + +=== TEST 18: route forwarding a compressed body, with oas-validator +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local spec = core.json.encode({ + openapi = "3.0.2", + info = {title = "test", version = "1.0"}, + paths = { + ["/echo"] = { + post = { + requestBody = { + required = true, + content = { + ["application/json"] = { + schema = { + type = "object", + required = {"name"}, + properties = { + name = {type = "string"} + } + } + } + } + }, + responses = {["200"] = {description = "ok"}} + } + } + } + }) + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + forward_compressed = true + }, + ["oas-validator"] = {spec = spec} + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 19: the upstream receives the bytes as they arrived +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body unchanged: ", res.body == body) + } + } +--- response_body +status: 200 +content-encoding: gzip +upstream body unchanged: true + + + +=== TEST 20: the decompressed body is still validated +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"lol":"watdis?"}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 400 +{"message":"failed to validate request. "} +--- error_log +error occurred while validating request + + + +=== TEST 21: deflate round-trips +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]], nil, {windowBits = 15}) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "deflate" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body unchanged: ", res.body == body) + } + } +--- response_body +status: 200 +content-encoding: deflate +upstream body unchanged: true + + + +=== TEST 22: chained codings round-trip +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip(gzip.deflate_gzip([[{"name":"doggie"}]])) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip, gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body unchanged: ", res.body == body) + } + } +--- response_body +status: 200 +content-encoding: gzip, gzip +upstream body unchanged: true + + + +=== TEST 23: an uncompressed request on the same route is untouched +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = [[{"name":"doggie"}]], + headers = { + ["Content-Type"] = "application/json" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +content-encoding: none +upstream body: {"name":"doggie"} + + + +=== TEST 24: route forwarding a compressed body, with body-transformer +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + forward_compressed = true + }, + ["body-transformer"] = { + request = { + template = [[{"foo":"{{name .. " world"}}","bar":{{age+10}}}]] + } + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 25: a rewritten body is compressed again before it is forwarded +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"hello","age":20}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body unchanged: ", res.body == body) + ngx.say("upstream body inflates to: ", gzip.inflate_gzip(res.body)) + } + } +--- response_body +status: 200 +content-encoding: gzip +upstream body unchanged: false +upstream body inflates to: {"foo":"hello world","bar":30} + + + +=== TEST 26: route whose body another plugin moves to a file +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local func = [[return function(conf, ctx) + local path = ngx.config.prefix() .. "logs/request-decompress-body.txt" + local f = assert(io.open(path, "w")) + f:write('{"name":"from-file"}') + f:close() + ngx.req.set_body_file(path) + end]] + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + forward_compressed = true + }, + ["serverless-pre-function"] = { + phase = "access", + functions = {func} + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 27: a body held in a file is compressed from the file +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = gzip.deflate_gzip([[{"name":"doggie"}]]), + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body inflates to: ", gzip.inflate_gzip(res.body)) + } + } +--- response_body +status: 200 +content-encoding: gzip +upstream body inflates to: {"name":"from-file"} + + + +=== TEST 28: route with the default body size limit +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = {} + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 29: a concatenated stream is forwarded whole +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + -- one JSON document split across two gzip members + local body = gzip.deflate_gzip([[{"name":]]) + .. gzip.deflate_gzip([["doggie"}]]) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("upstream body: ", res.body) + } + } +--- response_body +status: 200 +upstream body: {"name":"doggie"} + + + +=== TEST 30: a corrupt later member is rejected +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) .. "not a gzip member" + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 400 +{"message":"failed to decompress request body"} +--- error_log +failed reading request body, err: inflate gzip err + + + +=== TEST 31: bytes trailing the last member are rejected +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) .. "\0" + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 400 +{"message":"failed to decompress request body"} +--- error_log +failed reading request body, err: inflate gzip err + + + +=== TEST 32: route with a small body size limit +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + max_req_body_size = 64 + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 33: members under the limit individually are rejected on their total +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + -- 48 compressed bytes, under the 64 byte limit, holding two + -- members of 40 bytes each, so only their total breaks it + local member = gzip.deflate_gzip(string.rep("d", 40)) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = member .. member, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 413 +{"message":"request body is too large"} +--- error_log +inflated data is greater than the maximum size 64 allowed + + + +=== TEST 34: a raw body over the limit is a 413, Content-Length present +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = string.rep("d", 65), + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 413 +{"message":"request body is too large"} +--- error_log +request size 65 is greater than the maximum size 64 allowed + + + +=== TEST 35: a raw body over the limit is a 413, chunked +--- config + location /t { + content_by_lua_block { + local body = string.rep("d", 65) + local sock = ngx.socket.tcp() + sock:settimeout(5000) + local ok, err = sock:connect("127.0.0.1", 1984) + if not ok then + ngx.say("connect: ", err) + return + end + + local req = "POST /echo HTTP/1.1\r\n" + .. "Host: 127.0.0.1:1984\r\n" + .. "Content-Type: application/json\r\n" + .. "Content-Encoding: gzip\r\n" + .. "Transfer-Encoding: chunked\r\n" + .. "Connection: close\r\n\r\n" + .. string.format("%x\r\n", #body) .. body .. "\r\n0\r\n\r\n" + + local bytes, err = sock:send(req) + if not bytes then + ngx.say("send: ", err) + return + end + + local line, err = sock:receive("*l") + sock:close() + if not line then + ngx.say("receive: ", err) + return + end + ngx.say("status: ", line:match("^HTTP/1%.1 (%d+)")) + } + } +--- response_body +status: 413 +--- error_log +request size 65 is greater than the maximum size 64 allowed + + + +=== TEST 36: a raw body over the limit is a 413, held in a file +--- config + client_body_in_file_only on; + location /t { + content_by_lua_block { + local body = string.rep("d", 65) + local sock = ngx.socket.tcp() + sock:settimeout(5000) + local ok, err = sock:connect("127.0.0.1", 1984) + if not ok then + ngx.say("connect: ", err) + return + end + + local req = "POST /echo HTTP/1.1\r\n" + .. "Host: 127.0.0.1:1984\r\n" + .. "Content-Type: application/json\r\n" + .. "Content-Encoding: gzip\r\n" + .. "Transfer-Encoding: chunked\r\n" + .. "Connection: close\r\n\r\n" + .. string.format("%x\r\n", #body) .. body .. "\r\n0\r\n\r\n" + + local bytes, err = sock:send(req) + if not bytes then + ngx.say("send: ", err) + return + end + + local line, err = sock:receive("*l") + sock:close() + if not line then + ngx.say("receive: ", err) + return + end + ngx.say("status: ", line:match("^HTTP/1%.1 (%d+)")) + } + } +--- response_body +status: 413 +--- error_log +request size 65 is greater than the maximum size 64 allowed + + + +=== TEST 37: route forwarding a compressed body, with a small limit +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + max_req_body_size = 64, + forward_compressed = true + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 38: the limit also applies when a compressed body is forwarded +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = gzip.deflate_gzip(string.rep("d", 2048)), + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.print(res.body) + } + } +--- response_body +status: 413 +{"message":"request body is too large"} +--- error_log +inflated data is greater than the maximum size 64 allowed + + + +=== TEST 39: the plugin on a global rule and on a route +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + + local code = t('/apisix/admin/global_rules/1', + ngx.HTTP_PUT, + core.json.encode({ + plugins = { + ["request-decompress"] = { + forward_compressed = true + } + } + }) + ) + if code >= 300 then + ngx.say("global rule creation failed: ", code) + return + end + + code = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + forward_compressed = true + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + if code >= 300 then + ngx.say("route creation failed: ", code) + return + end + + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip([[{"name":"doggie"}]]) + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + -- compressed once, not once per plugin instance + ngx.say("upstream body unchanged: ", res.body == body) + ngx.say("upstream body inflates to: ", gzip.inflate_gzip(res.body)) + + t('/apisix/admin/global_rules/1', ngx.HTTP_DELETE) + } + } +--- response_body +status: 200 +content-encoding: gzip +upstream body unchanged: true +upstream body inflates to: {"name":"doggie"} + + + +=== TEST 40: the plugin on a global rule alone +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + + local code = t('/apisix/admin/global_rules/1', + ngx.HTTP_PUT, + core.json.encode({ + plugins = { + ["request-decompress"] = {} + } + }) + ) + if code >= 300 then + ngx.say("global rule creation failed: ", code) + return + end + + code = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + if code >= 300 then + ngx.say("route creation failed: ", code) + return + end + + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = gzip.deflate_gzip([[{"name":"doggie"}]]), + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body: ", res.body) + + t('/apisix/admin/global_rules/1', ngx.HTTP_DELETE) + } + } +--- response_body +status: 200 +content-encoding: none +upstream body: {"name":"doggie"} + + + +=== TEST 41: route forwarding a compressed body +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + forward_compressed = true + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 42: a repeated Content-Encoding header keeps every coding +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + local body = gzip.deflate_gzip(gzip.deflate_gzip([[{"name":"doggie"}]])) + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = body, + headers = { + ["Content-Type"] = "application/json", + -- two header lines rather than one comma separated value + ["Content-Encoding"] = {"gzip", "gzip"} + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + ngx.say("content-encoding: ", res.headers["Content-Encoding"] or "none") + ngx.say("upstream body unchanged: ", res.body == body) + } + } +--- response_body +status: 200 +content-encoding: gzip, gzip +upstream body unchanged: true + + + +=== TEST 43: route whose body file another plugin removes +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local core = require("apisix.core") + local func = [[return function(conf, ctx) + local path = ngx.config.prefix() .. "logs/request-decompress-gone.txt" + local f = assert(io.open(path, "w")) + f:write('{"name":"from-file"}') + f:close() + -- set_body_file checks the file is readable, so it is removed + -- only afterwards, leaving the read in before_proxy to fail. + -- Removing it rather than dropping its permissions, which root + -- would read straight through + ngx.req.set_body_file(path) + os.remove(path) + end]] + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + core.json.encode({ + uri = "/echo", + plugins = { + ["request-decompress"] = { + forward_compressed = true + }, + ["serverless-pre-function"] = { + phase = "access", + functions = {func} + } + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1} + } + }) + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 44: a body that cannot be read is an error, not an empty body +--- config + location /t { + content_by_lua_block { + local http = require("resty.http") + local gzip = require("apisix.utils.gzip") + + local httpc = http.new() + local res, err = httpc:request_uri("http://127.0.0.1:1984/echo", { + method = "POST", + body = gzip.deflate_gzip([[{"name":"doggie"}]]), + headers = { + ["Content-Type"] = "application/json", + ["Content-Encoding"] = "gzip" + } + }) + if not res then + ngx.say(err) + return + end + + ngx.say("status: ", res.status) + } + } +--- response_body +status: 500 +--- error_log +failed reading request body, err: diff --git a/t/utils/gzip.t b/t/utils/gzip.t new file mode 100644 index 000000000000..1c3f7afc3f79 --- /dev/null +++ b/t/utils/gzip.t @@ -0,0 +1,157 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +log_level("info"); + +run_tests; + +__DATA__ + +=== TEST 1: test gzip compression levels +--- config + location /t { + content_by_lua_block { + -- generate a big string of 4MB + local big_raw = string.rep("h", 1024 * 1024 * 4) + ngx.log(ngx.NOTICE, "original size: ", #big_raw) + + local gzip = require("apisix.utils.gzip") + local data_l1, err = gzip.deflate_gzip(big_raw, nil, { level = 1 }) + assert(err == nil) + local data_l9, err = gzip.deflate_gzip(big_raw, nil, { level = 9 }) + assert(err == nil) + + assert(#data_l9 < #data_l1, "expected level 9: " .. #data_l9 .. " < level 1: " .. #data_l1) + } + } +--- request +GET /t +--- error_code: 200 + + + +=== TEST 2: a concatenated stream inflates every member +--- config + location /t { + content_by_lua_block { + local gzip = require("apisix.utils.gzip") + local payload = gzip.deflate_gzip([[{"name":]]) + .. gzip.deflate_gzip([["doggie"}]]) + + local out, err = gzip.inflate_gzip(payload) + ngx.say("err: ", err) + ngx.say("out: ", out) + } + } +--- request +GET /t +--- response_body +err: nil +out: {"name":"doggie"} + + + +=== TEST 3: max_output bounds every member together +--- config + location /t { + content_by_lua_block { + local gzip = require("apisix.utils.gzip") + local parts = {} + for _ = 1, 20 do + parts[#parts + 1] = gzip.deflate_gzip(string.rep("d", 100)) + end + local payload = table.concat(parts) + + -- every member inflates to 100 bytes, their total is 2000 + local out, err, exceeded = gzip.inflate_gzip(payload, nil, nil, 128) + ngx.say("out: ", out) + ngx.say("err: ", err) + ngx.say("exceeded: ", exceeded) + + local all = gzip.inflate_gzip(payload, nil, nil, 4096) + ngx.say("under a 4096 cap: ", #all, " bytes") + } + } +--- request +GET /t +--- response_body +out: nil +err: inflated data is greater than the maximum size 128 allowed +exceeded: true +under a 4096 cap: 2000 bytes + + + +=== TEST 4: a corrupt later member is rejected +--- config + location /t { + content_by_lua_block { + local gzip = require("apisix.utils.gzip") + local payload = gzip.deflate_gzip([[{"a":1}]]) .. "not a gzip member" + + local out, err = gzip.inflate_gzip(payload) + ngx.say("out: ", out) + ngx.say("err: ", err) + } + } +--- request +GET /t +--- response_body +out: nil +err: inflate gzip err: INFLATE: data error + + + +=== TEST 5: bytes trailing the last member are rejected +--- config + location /t { + content_by_lua_block { + local gzip = require("apisix.utils.gzip") + local payload = gzip.deflate_gzip([[{"a":1}]]) .. "\0" + + local out, err = gzip.inflate_gzip(payload) + ngx.say("out: ", out) + ngx.say("err: ", err) + } + } +--- request +GET /t +--- response_body +out: nil +err: inflate gzip err: INFLATE: Data error, no input bytes + + + +=== TEST 6: an empty payload is still an error +--- config + location /t { + content_by_lua_block { + local gzip = require("apisix.utils.gzip") + local out, err = gzip.inflate_gzip("") + ngx.say("out: ", out) + ngx.say("err: ", err) + } + } +--- request +GET /t +--- response_body +out: nil +err: inflate gzip err: INFLATE: Data error, no input bytes