From c7377de0115f12f6b74fa729691d5ce8b807f9d4 Mon Sep 17 00:00:00 2001
From: Nic <qianyong@api7.ai>
Date: Wed, 14 Oct 2026 07:53:12 +0000
Subject: [PATCH 1/6] feat(prometheus): export per-process timers, connection
 slots and Lua memory

---
 apisix/plugins/prometheus/exporter.lua        |  32 +-
 apisix/plugins/prometheus/process_metrics.lua | 378 ++++++++++++++++++
 t/plugin/prometheus-process-metrics.t         | 210 ++++++++++
 3 files changed, 614 insertions(+), 6 deletions(-)
 create mode 100644 apisix/plugins/prometheus/process_metrics.lua
 create mode 100644 t/plugin/prometheus-process-metrics.t

diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index 7fe666c7a7b5..ed42d7bc2293 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -45,6 +45,7 @@ local get_protos = require("apisix.plugins.grpc-transcode.proto").protos
 local service_fetch = require("apisix.http.service").get
 local latency_details = require("apisix.utils.log-util").latency_details_in_ms
 local xrpc = require("apisix.stream.xrpc")
+local process_metrics = require("apisix.plugins.prometheus.process_metrics")
 local unpack = unpack
 local next = next
 local str_sub = string.sub
@@ -500,12 +501,22 @@ local function collect_stream_zone_metrics()
 end
 
 
+local function get_refresh_interval()
+    local attr = plugin.plugin_attr("prometheus")
+    if attr and attr.refresh_interval then
+        return attr.refresh_interval
+    end
+    return DEFAULT_REFRESH_INTERVAL
+end
+
+
 function _M.http_init(prometheus_enabled_in_stream)
     -- todo: support hot reload, we may need to update the lua-prometheus
     -- library
     if ngx.get_phase() ~= "init" and ngx.get_phase() ~= "init_worker"  then
         if prometheus_bkp then
             prometheus = prometheus_bkp
+            process_metrics.start(get_refresh_interval())
         end
         return
     end
@@ -695,6 +706,8 @@ function _M.http_init(prometheus_enabled_in_stream)
             ai_cache_embedding_latency_buckets,
             ai_cache_embedding_latency_exptime)
 
+    process_metrics.init(prometheus)
+
     if prometheus_enabled_in_stream then
         init_stream_metrics()
     end
@@ -703,6 +716,7 @@ end
 
 function _M.stream_init()
     if ngx.get_phase() ~= "init" and ngx.get_phase() ~= "init_worker"  then
+        process_metrics.start(get_refresh_interval())
         return
     end
 
@@ -722,6 +736,9 @@ function _M.stream_init()
     prometheus = base_prometheus.init("prometheus-metrics", metric_prefix)
 
     init_stream_metrics()
+
+    process_metrics.init(prometheus)
+    process_metrics.start(get_refresh_interval())
 end
 
 
@@ -1186,6 +1203,9 @@ end
 
 
 local function collect(yieldable)
+    -- the other processes report their own values on a timer
+    process_metrics.report()
+
     -- collect ngx.shared.DICT status
     shared_dict_status()
 
@@ -1291,16 +1311,14 @@ end
 
 
 local function init_exporter_timer()
+    local refresh_interval = get_refresh_interval()
+
+    process_metrics.start(refresh_interval)
+
     if process.type() ~= "privileged agent" then
         return
     end
 
-    local refresh_interval = DEFAULT_REFRESH_INTERVAL
-    local attr = plugin.plugin_attr("prometheus")
-    if attr and attr.refresh_interval then
-        refresh_interval = attr.refresh_interval
-    end
-
     local cache_exptime = refresh_interval * 2
 
     exporter_timer(false, false, cache_exptime)
@@ -1457,6 +1475,8 @@ end
 
 
 function _M.destroy()
+    process_metrics.stop()
+
     if prometheus ~= nil then
         prometheus_bkp = prometheus
         prometheus = nil
diff --git a/apisix/plugins/prometheus/process_metrics.lua b/apisix/plugins/prometheus/process_metrics.lua
new file mode 100644
index 000000000000..98bcbe6a9533
--- /dev/null
+++ b/apisix/plugins/prometheus/process_metrics.lua
@@ -0,0 +1,378 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+-- Per-process OpenResty internals: Lua timers, connection slots and Lua VM
+-- memory. These values only exist inside the process they describe, while the
+-- exposition is rendered by a single process, so every process writes its own
+-- gauges into the shared metric dict from a timer, and the process rendering
+-- the exposition refreshes its own values right before it does.
+local core          = require("apisix.core")
+local ffi           = require("ffi")
+local process       = require("ngx.process")
+local C             = ffi.C
+local ngx           = ngx
+local timer_at      = ngx.timer.at
+local collectgarbage = collectgarbage
+local tonumber      = tonumber
+local tostring      = tostring
+local pcall         = pcall
+local ipairs        = ipairs
+local select        = select
+
+
+-- The prefix of struct ngx_cycle_s up to write_events. Its layout is the same
+-- in every nginx core shipped by the supported runtimes (1.21.4 to 1.29.x);
+-- the self check in check_cycle_layout() is what keeps a future change from
+-- turning into a wrong number.
+ffi.cdef[[
+typedef struct {
+    uintptr_t       log_level;
+    void           *file;
+    uintptr_t       connection;
+    long            disk_full_time;
+    void           *handler;
+    void           *data;
+    void           *writer;
+    void           *wdata;
+    char           *action;
+    void           *next;
+} apisix_prom_ngx_log_t;
+
+typedef struct {
+    void           *elts;
+    uintptr_t       nelts;
+    size_t          size;
+    uintptr_t       nalloc;
+    void           *pool;
+} apisix_prom_ngx_array_t;
+
+typedef struct {
+    void           *elts;
+    uintptr_t       nelts;
+    void           *next;
+} apisix_prom_ngx_list_part_t;
+
+typedef struct {
+    void                        *last;
+    apisix_prom_ngx_list_part_t  part;
+    size_t                       size;
+    uintptr_t                    nalloc;
+    void                        *pool;
+} apisix_prom_ngx_list_t;
+
+typedef struct {
+    uintptr_t       key;
+    void           *left;
+    void           *right;
+    void           *parent;
+    unsigned char   color;
+    unsigned char   data;
+} apisix_prom_ngx_rbtree_node_t;
+
+typedef struct {
+    void           *root;
+    void           *sentinel;
+    void           *insert;
+} apisix_prom_ngx_rbtree_t;
+
+typedef struct {
+    void           *data;
+    void           *read;
+    void           *write;
+} apisix_prom_ngx_connection_head_t;
+
+typedef struct {
+    void                           *conf_ctx;
+    void                           *pool;
+    void                           *log;
+    apisix_prom_ngx_log_t           new_log;
+    uintptr_t                       log_use_stderr;
+    void                           *files;
+    void                           *free_connections;
+    uintptr_t                       free_connection_n;
+    void                           *modules;
+    uintptr_t                       modules_n;
+    uintptr_t                       modules_used;
+    void                           *reusable_connections_queue[2];
+    uintptr_t                       reusable_connections_n;
+    long                            connections_reuse_time;
+    apisix_prom_ngx_array_t         listening;
+    apisix_prom_ngx_array_t         paths;
+    apisix_prom_ngx_array_t         config_dump;
+    apisix_prom_ngx_rbtree_t        config_dump_rbtree;
+    apisix_prom_ngx_rbtree_node_t   config_dump_sentinel;
+    apisix_prom_ngx_list_t          open_files;
+    apisix_prom_ngx_list_t          shared_memory;
+    uintptr_t                       connection_n;
+    uintptr_t                       files_n;
+    apisix_prom_ngx_connection_head_t *connections;
+    void                           *read_events;
+    void                           *write_events;
+} apisix_prom_ngx_cycle_t;
+
+extern apisix_prom_ngx_cycle_t *ngx_cycle;
+]]
+
+
+-- nginx's own defaults for lua_max_running_timers / lua_max_pending_timers,
+-- in effect when the configuration leaves them out
+local DEFAULT_MAX_RUNNING_TIMERS = 256
+local DEFAULT_MAX_PENDING_TIMERS = 1024
+local DEFAULT_INTERVAL = 15
+
+-- Shared by every process, so a layout mismatch found by one worker keeps the
+-- connection gauges out of the exposition everywhere, including the
+-- privileged agent, whose connection count cannot be checked against config.
+local CONNECTIONS_VERDICT_KEY = "process_metrics:connections_readable"
+
+local _M = {}
+
+local gauges
+local dict
+local process_label
+local no_extra_labels = {}
+local extra_label_names = no_extra_labels
+local extra_label_values
+local subsystem = ngx.config.subsystem
+local verdict_logged = false
+
+local timer_running = false
+local stopped = true
+local interval = DEFAULT_INTERVAL
+
+
+local function get_process_label()
+    if process.type() == "privileged agent" then
+        return "privileged-agent"
+    end
+
+    return "worker-" .. tostring(ngx.worker.id() or 0)
+end
+
+
+local function timers_max()
+    local nginx_config = core.config.local_conf().nginx_config or {}
+    return nginx_config.max_running_timers or DEFAULT_MAX_RUNNING_TIMERS,
+           nginx_config.max_pending_timers or DEFAULT_MAX_PENDING_TIMERS
+end
+
+
+local function read_cycle()
+    local cycle = C.ngx_cycle
+    local connection_n = tonumber(cycle.connection_n)
+    local free_n = tonumber(cycle.free_connection_n)
+
+    -- every connection slot is wired to its own pair of events at startup, so
+    -- this pins down the offsets right after connection_n as well
+    local conns = cycle.connections
+    if connection_n == 0 or free_n > connection_n or conns == nil
+       or conns[0].read ~= cycle.read_events or conns[0].write ~= cycle.write_events
+    then
+        return nil
+    end
+
+    return connection_n, free_n
+end
+
+
+local function check_cycle_layout()
+    local ok, connection_n = pcall(read_cycle)
+    if not ok or not connection_n then
+        return false, "unexpected nginx cycle layout"
+    end
+
+    local event = core.config.local_conf().nginx_config.event
+    local expected = event and tonumber(event.worker_connections)
+    if connection_n ~= expected then
+        return false, "worker connection slots read as " .. tostring(connection_n)
+                      .. ", but worker_connections is " .. tostring(expected)
+    end
+
+    return true
+end
+
+
+-- Only a worker can tell whether the numbers read through FFI are right: its
+-- connection slots must match worker_connections from the configuration.
+local function connections_readable()
+    local readable = dict:get(CONNECTIONS_VERDICT_KEY)
+    if readable ~= nil then
+        return readable
+    end
+
+    if process.type() == "privileged agent" then
+        -- no worker has checked yet
+        return false
+    end
+
+    local ok, err = check_cycle_layout()
+    dict:set(CONNECTIONS_VERDICT_KEY, ok)
+    if not ok then
+        gauges.connections:reset()
+        gauges.connections_max:reset()
+        if not verdict_logged then
+            verdict_logged = true
+            core.log.warn("process connection metrics are disabled: ", err)
+        end
+    end
+
+    return ok
+end
+
+
+local function with_extra_labels(labels)
+    for _, name in ipairs(extra_label_names) do
+        labels[#labels + 1] = name
+    end
+    return labels
+end
+
+
+local function label_values(extra, ...)
+    local values = {...}
+    local n = select("#", ...)
+    for i, value in ipairs(extra) do
+        values[n + i] = value
+    end
+    return values
+end
+
+
+-- extra_labels, when given, appends the same labels to every gauge here:
+-- {names = {...}, values = function() return {...} end}
+function _M.init(registry, extra_labels)
+    dict = registry.dict
+    if extra_labels then
+        extra_label_names = extra_labels.names
+        extra_label_values = extra_labels.values
+    end
+
+    gauges = {
+        timers = registry:gauge("nginx_timers",
+            "Number of Lua timers per process and subsystem, by state",
+            with_extra_labels({"process", "subsystem", "state"})),
+        timers_max = registry:gauge("nginx_timers_max",
+            "Configured maximum number of Lua timers per process, by state",
+            with_extra_labels({"state"})),
+        connections = registry:gauge("nginx_process_connections",
+            "Number of connection slots in use per nginx process",
+            with_extra_labels({"process"})),
+        connections_max = registry:gauge("nginx_process_connections_max",
+            "Number of connection slots per nginx process",
+            with_extra_labels({"process"})),
+        lua_memory = registry:gauge("nginx_lua_memory_bytes",
+            "Memory allocated by the Lua VM per process and subsystem",
+            with_extra_labels({"process", "subsystem"})),
+    }
+
+    -- the verdict describes the binary and the configuration of the processes
+    -- starting now, so each worker checks again rather than trusting the one
+    -- left behind in the shared dict by a reload
+    if subsystem == "http" and process.type() ~= "privileged agent" then
+        dict:delete(CONNECTIONS_VERDICT_KEY)
+    end
+end
+
+
+function _M.report()
+    if not gauges then
+        return
+    end
+
+    process_label = process_label or get_process_label()
+    local extra = extra_label_values and extra_label_values() or no_extra_labels
+
+    local timers = gauges.timers
+    timers:set(ngx.timer.running_count(),
+               label_values(extra, process_label, subsystem, "running"))
+    timers:set(ngx.timer.pending_count(),
+               label_values(extra, process_label, subsystem, "pending"))
+
+    gauges.lua_memory:set(collectgarbage("count") * 1024,
+                          label_values(extra, process_label, subsystem))
+
+    -- the remaining values are per process rather than per Lua VM, so only
+    -- the http VM reports them; it is present in every process that exports
+    if subsystem ~= "http" then
+        return
+    end
+
+    local max_running, max_pending = timers_max()
+    gauges.timers_max:set(max_running, label_values(extra, "running"))
+    gauges.timers_max:set(max_pending, label_values(extra, "pending"))
+
+    if not connections_readable() then
+        return
+    end
+
+    local connection_n, free_n = read_cycle()
+    if not connection_n then
+        return
+    end
+
+    gauges.connections:set(connection_n - free_n, label_values(extra, process_label))
+    gauges.connections_max:set(connection_n, label_values(extra, process_label))
+end
+
+
+local function tick(premature)
+    if premature or stopped then
+        timer_running = false
+        return
+    end
+
+    local ok, err = pcall(_M.report)
+    if not ok then
+        core.log.error("failed to report process metrics: ", err)
+    end
+
+    ok, err = timer_at(interval, tick)
+    if not ok then
+        timer_running = false
+        core.log.error("failed to schedule process metrics: ", err)
+    end
+end
+
+
+-- Idempotent: calling it again while reporting only updates the interval.
+function _M.start(report_interval)
+    if not gauges then
+        return
+    end
+
+    interval = report_interval or DEFAULT_INTERVAL
+    stopped = false
+    if timer_running then
+        return
+    end
+
+    timer_running = true
+    local ok, err = timer_at(0, tick)
+    if not ok then
+        timer_running = false
+        core.log.error("failed to start process metrics: ", err)
+    end
+end
+
+
+-- The pending timer sees this when it fires and does not schedule another.
+function _M.stop()
+    stopped = true
+end
+
+
+return _M
diff --git a/t/plugin/prometheus-process-metrics.t b/t/plugin/prometheus-process-metrics.t
new file mode 100644
index 000000000000..1e4c95e4069c
--- /dev/null
+++ b/t/plugin/prometheus-process-metrics.t
@@ -0,0 +1,210 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+
+    if (!defined $block->request && !defined $block->stream_request) {
+        $block->set_value("request", "GET /t");
+    }
+
+    # t::APISIX renders worker_connections 128 into nginx.conf, so the
+    # configuration has to say the same for the connection gauges to pass
+    # their self check
+    if (!defined $block->extra_yaml_config) {
+        $block->set_value("extra_yaml_config", <<_EOC_);
+stream_plugins:
+    - prometheus
+nginx_config:
+    max_running_timers: 1234
+    max_pending_timers: 5678
+    event:
+        worker_connections: 128
+plugin_attr:
+    prometheus:
+        refresh_interval: 0.1
+_EOC_
+    }
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: pre-create public API route
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/metrics',
+                ngx.HTTP_PUT,
+                [[{
+                    "plugins": {
+                        "public-api": {}
+                    },
+                    "uri": "/apisix/prometheus/metrics"
+                }]]
+                )
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 2: every process reports its timers, connection slots and Lua memory
+--- config
+    location /t {
+        content_by_lua_block {
+            ngx.sleep(0.5)
+            local http = require("resty.http")
+            local httpc = http.new()
+            local res = assert(httpc:request_uri(
+                "http://127.0.0.1:1984/apisix/prometheus/metrics"))
+            local patterns = {
+                [[apisix_nginx_timers\{process="worker-0",subsystem="http",state="running"\} [1-9]\d*]],
+                [[apisix_nginx_timers\{process="worker-0",subsystem="http",state="pending"\} \d+]],
+                [[apisix_nginx_timers\{process="privileged-agent",subsystem="http",state="running"\} \d+]],
+                [[apisix_nginx_timers_max\{state="running"\} 1234\n]],
+                [[apisix_nginx_timers_max\{state="pending"\} 5678\n]],
+                [[apisix_nginx_process_connections\{process="worker-0"\} [1-9]\d*\n]],
+                [[apisix_nginx_process_connections\{process="privileged-agent"\} [1-9]\d*\n]],
+                [[apisix_nginx_process_connections_max\{process="worker-0"\} 128\n]],
+                [[apisix_nginx_process_connections_max\{process="privileged-agent"\} 512\n]],
+                [[apisix_nginx_lua_memory_bytes\{process="worker-0",subsystem="http"\} [1-9]\d*]],
+                [[apisix_nginx_lua_memory_bytes\{process="privileged-agent",subsystem="http"\} [1-9]\d*]],
+            }
+            for _, p in ipairs(patterns) do
+                if not ngx.re.find(res.body, p, "jo") then
+                    ngx.say("missing ", p)
+                end
+            end
+            ngx.say("done")
+        }
+    }
+--- response_body
+done
+
+
+
+=== TEST 3: connection slots held by cosockets show up in the worker's gauge
+--- config
+    location /t {
+        content_by_lua_block {
+            local http = require("resty.http")
+
+            local function worker_connections()
+                ngx.sleep(0.5)
+                local httpc = http.new()
+                local res = assert(httpc:request_uri(
+                    "http://127.0.0.1:1984/apisix/prometheus/metrics"))
+                local m = ngx.re.match(res.body,
+                    [[apisix_nginx_process_connections\{process="worker-0"\} (\d+)]], "jo")
+                return tonumber(m[1])
+            end
+
+            local before = worker_connections()
+
+            local socks = {}
+            for i = 1, 20 do
+                local sock = ngx.socket.tcp()
+                assert(sock:connect("127.0.0.1", 1980))
+                socks[i] = sock
+            end
+
+            local held = worker_connections()
+
+            for _, sock in ipairs(socks) do
+                sock:close()
+            end
+
+            local released = worker_connections()
+
+            ngx.say("held: ", held - before >= 20)
+            ngx.say("released: ", held - released >= 20)
+        }
+    }
+--- response_body
+held: true
+released: true
+
+
+
+=== TEST 4: connection gauges stay out when the slots do not match worker_connections
+--- extra_yaml_config
+plugin_attr:
+    prometheus:
+        refresh_interval: 0.1
+--- config
+    location /t {
+        content_by_lua_block {
+            ngx.sleep(0.5)
+            local http = require("resty.http")
+            local httpc = http.new()
+            local res = assert(httpc:request_uri(
+                "http://127.0.0.1:1984/apisix/prometheus/metrics"))
+            ngx.say(ngx.re.find(res.body, "apisix_nginx_process_connections", "jo") == nil)
+            ngx.say(ngx.re.find(res.body,
+                [[apisix_nginx_timers\{process="worker-0",subsystem="http",state="pending"\}]],
+                "jo") ~= nil)
+        }
+    }
+--- response_body
+true
+true
+--- error_log
+process connection metrics are disabled: worker connection slots read as 128, but worker_connections is 10620
+
+
+
+=== TEST 5: the stream Lua VM of every process reports as well
+--- stream_server_config
+    content_by_lua_block {
+        ngx.req.socket():receive()
+        ngx.sleep(0.5)
+        local http = require("resty.http")
+        local httpc = http.new()
+        local res = assert(httpc:request_uri(
+            "http://127.0.0.1:1984/apisix/prometheus/metrics"))
+        local patterns = {
+            [[apisix_nginx_timers\{process="worker-0",subsystem="stream",state="running"\} [1-9]\d*]],
+            [[apisix_nginx_timers\{process="worker-0",subsystem="stream",state="pending"\} \d+]],
+            [[apisix_nginx_timers\{process="privileged-agent",subsystem="stream",state="pending"\} \d+]],
+            [[apisix_nginx_lua_memory_bytes\{process="worker-0",subsystem="stream"\} [1-9]\d*]],
+            [[apisix_nginx_lua_memory_bytes\{process="privileged-agent",subsystem="stream"\} [1-9]\d*]],
+        }
+        for _, p in ipairs(patterns) do
+            if not ngx.re.find(res.body, p, "jo") then
+                ngx.say("missing ", p)
+            end
+        end
+        ngx.say("done")
+    }
+--- stream_request
+ping
+--- stream_response
+done

From 3949453490eed1fb27c9ad20b0219a2e74b18665 Mon Sep 17 00:00:00 2001
From: Nic <qianyong@api7.ai>
Date: Wed, 14 Oct 2026 07:57:42 +0000
Subject: [PATCH 2/6] feat(prometheus): document process metrics and cover the
 stream-only mode

---
 apisix/plugins/prometheus/exporter.lua        |  2 +-
 apisix/plugins/prometheus/process_metrics.lua | 27 ++++++++++++-------
 docs/en/latest/plugins/prometheus.md          | 17 ++++++++++++
 docs/zh/latest/plugins/prometheus.md          | 17 ++++++++++++
 t/cli/test_prometheus_stream.sh               | 23 ++++++++++++++++
 5 files changed, 75 insertions(+), 11 deletions(-)

diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua
index ed42d7bc2293..4ccc3f56b71c 100644
--- a/apisix/plugins/prometheus/exporter.lua
+++ b/apisix/plugins/prometheus/exporter.lua
@@ -1204,7 +1204,7 @@ end
 
 local function collect(yieldable)
     -- the other processes report their own values on a timer
-    process_metrics.report()
+    process_metrics.refresh()
 
     -- collect ngx.shared.DICT status
     shared_dict_status()
diff --git a/apisix/plugins/prometheus/process_metrics.lua b/apisix/plugins/prometheus/process_metrics.lua
index 98bcbe6a9533..29c95139b5ed 100644
--- a/apisix/plugins/prometheus/process_metrics.lua
+++ b/apisix/plugins/prometheus/process_metrics.lua
@@ -288,11 +288,7 @@ function _M.init(registry, extra_labels)
 end
 
 
-function _M.report()
-    if not gauges then
-        return
-    end
-
+local function report()
     process_label = process_label or get_process_label()
     local extra = extra_label_values and extra_label_values() or no_extra_labels
 
@@ -329,18 +325,29 @@ function _M.report()
 end
 
 
-local function tick(premature)
-    if premature or stopped then
-        timer_running = false
+-- Writes this process's values now. A failure here is logged rather than
+-- raised, so it can never cost the caller the rest of the exposition.
+function _M.refresh()
+    if not gauges then
         return
     end
 
-    local ok, err = pcall(_M.report)
+    local ok, err = pcall(report)
     if not ok then
         core.log.error("failed to report process metrics: ", err)
     end
+end
+
+
+local function tick(premature)
+    if premature or stopped then
+        timer_running = false
+        return
+    end
+
+    _M.refresh()
 
-    ok, err = timer_at(interval, tick)
+    local ok, err = timer_at(interval, tick)
     if not ok then
         timer_running = false
         core.log.error("failed to schedule process metrics: ", err)
diff --git a/docs/en/latest/plugins/prometheus.md b/docs/en/latest/plugins/prometheus.md
index 2b183020ea6c..172bfbd38e45 100644
--- a/docs/en/latest/plugins/prometheus.md
+++ b/docs/en/latest/plugins/prometheus.md
@@ -170,6 +170,11 @@ The following table lists the core metrics registered directly by the `prometheu
 | apisix_node_info                      | gauge     | Information of the APISIX node, such as host name and the current APISIX version.                                                                                                                                                         |
 | apisix_shared_dict_capacity_bytes     | gauge     | The total capacity of an [NGINX shared dictionary](https://github.com/openresty/lua-nginx-module#ngxshareddict).                                                                                                                     |
 | apisix_shared_dict_free_space_bytes   | gauge     | The remaining space in an [NGINX shared dictionary](https://github.com/openresty/lua-nginx-module#ngxshareddict).                                                                                                                   |
+| apisix_nginx_timers                   | gauge     | Number of running and pending Lua timers of each NGINX process, per subsystem.                                                                                                                                                   |
+| apisix_nginx_timers_max               | gauge     | Configured maximum number of running and pending Lua timers per process (`nginx_config.max_running_timers` and `nginx_config.max_pending_timers`).                                                                               |
+| apisix_nginx_process_connections      | gauge     | Number of connection slots in use by each NGINX process, including listening sockets, client connections and outgoing cosocket connections.                                                                                       |
+| apisix_nginx_process_connections_max  | gauge     | Number of connection slots of each NGINX process: `worker_connections` for workers, and 512 for the privileged agent.                                                                                                  |
+| apisix_nginx_lua_memory_bytes         | gauge     | Memory allocated by the Lua VM of each NGINX process, per subsystem.                                                                                                                                                             |
 | apisix_upstream_status                | gauge     | Health check status of upstream nodes, available if health checks are configured on the upstream. A value of `1` represents healthy and `0` represents unhealthy.                                                                 |
 | apisix_stream_connection_total        | counter   | Total number of connections handled per Stream Route.                                                                                                               |
 | apisix_stream_active_connections      | gauge     | Number of Stream sessions currently being proxied, per listening address. Covers TCP connections and UDP sessions alike.                                             |
@@ -438,6 +443,18 @@ The following labels are used to differentiate `apisix_upstream_status` metrics.
 | ip         | IP address of the upstream node.                                                                         |
 | port       | Port number of the node.                                                                            |
 
+### Labels for the NGINX process metrics
+
+`apisix_nginx_timers`, `apisix_nginx_timers_max`, `apisix_nginx_process_connections`, `apisix_nginx_process_connections_max` and `apisix_nginx_lua_memory_bytes` describe the internals of each NGINX process, so a resource running out in a single process, such as the privileged agent, shows up before it causes errors. Every process writes its own values every `refresh_interval`, and the process building the exposition refreshes its own values first.
+
+| Name      | Description                                                                                                                                                                                  |
+| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| process   | The process the value belongs to: `worker-<id>` for a worker process, or `privileged-agent`.                                                                                                |
+| subsystem | `http` or `stream`. Each subsystem has a Lua VM of its own, with its own timers and memory. Only on `apisix_nginx_timers` and `apisix_nginx_lua_memory_bytes`.                              |
+| state     | `running` or `pending`. Only on `apisix_nginx_timers` and `apisix_nginx_timers_max`.                                                                                                         |
+
+The connection metrics are read from NGINX's internal cycle structure. When a worker finds that the number of connection slots it reads does not match `worker_connections` in the configuration, APISIX logs a warning and leaves both connection metrics out for every process instead of exporting wrong numbers.
+
 ## Examples
 
 The examples below demonstrate how you can work with the `prometheus` Plugin for different scenarios.
diff --git a/docs/zh/latest/plugins/prometheus.md b/docs/zh/latest/plugins/prometheus.md
index ce772bfa7fb5..aa4f4939a45f 100644
--- a/docs/zh/latest/plugins/prometheus.md
+++ b/docs/zh/latest/plugins/prometheus.md
@@ -170,6 +170,11 @@ Prometheus 中有不同类型的指标。要了解它们之间的区别，请参
 | apisix_node_info        | gauge     | APISIX 节点的信息，例如主机名和当前的 APISIX 版本号。                                                                                                                                       |
 | apisix_shared_dict_capacity_bytes | gauge     | [NGINX 共享字典](https://github.com/openresty/lua-nginx-module#ngxshareddict) 的总容量。                                                                                     |
 | apisix_shared_dict_free_space_bytes | gauge     | [NGINX 共享字典](https://github.com/openresty/lua-nginx-module#ngxshareddict) 中剩余的空间。                                                                                   |
+| apisix_nginx_timers      | gauge     | 每个 NGINX 进程中运行中和等待中的 Lua 定时器数量，按子系统区分。                                                                                                     |
+| apisix_nginx_timers_max  | gauge     | 每个进程允许的运行中和等待中 Lua 定时器的上限（`nginx_config.max_running_timers` 和 `nginx_config.max_pending_timers`）。                                            |
+| apisix_nginx_process_connections | gauge | 每个 NGINX 进程已占用的连接槽位数，包括监听套接字、客户端连接和 cosocket 发起的出站连接。                                                                           |
+| apisix_nginx_process_connections_max | gauge | 每个 NGINX 进程的连接槽位总数：worker 进程为 `worker_connections`，privileged agent 为 512。                                                                |
+| apisix_nginx_lua_memory_bytes | gauge | 每个 NGINX 进程中 Lua VM 分配的内存，按子系统区分。                                                                                                                  |
 | apisix_upstream_status   | gauge     | 上游节点的健康检查状态，如果在上游配置了健康检查，则可用。值为 `1` 表示健康，`0` 表示不健康。                                                                                   |
 | apisix_stream_connection_total | counter   | 每个 Stream Route 处理的总连接数。                                                                                                                                         |
 | apisix_stream_active_connections | gauge   | 当前正在代理的 Stream 会话数，按监听地址区分。同时覆盖 TCP 连接与 UDP 会话。                                                                                                 |
@@ -426,6 +431,18 @@ APISIX 延迟
 | ip     | 上游节点的 IP 地址。                                                                                                   |
 | port   | 节点的端口号。                                                                                                         |
 
+### NGINX 进程指标的标签
+
+`apisix_nginx_timers`、`apisix_nginx_timers_max`、`apisix_nginx_process_connections`、`apisix_nginx_process_connections_max` 和 `apisix_nginx_lua_memory_bytes` 描述每个 NGINX 进程的内部状态，因此单个进程（例如 privileged agent）的资源即将耗尽时，可以在它引发错误之前被发现。每个进程每隔 `refresh_interval` 写入一次自己的数值，生成指标输出的进程会先刷新自己的数值。
+
+| 名称      | 描述                                                                                                                         |
+| --------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| process   | 数值所属的进程：worker 进程为 `worker-<id>`，privileged agent 为 `privileged-agent`。                                        |
+| subsystem | `http` 或 `stream`。每个子系统有独立的 Lua VM，定时器和内存也各自独立。仅用于 `apisix_nginx_timers` 和 `apisix_nginx_lua_memory_bytes`。 |
+| state     | `running` 或 `pending`。仅用于 `apisix_nginx_timers` 和 `apisix_nginx_timers_max`。                                          |
+
+连接相关指标读取自 NGINX 内部的 cycle 结构。如果某个 worker 读到的连接槽位数与配置中的 `worker_connections` 不一致，APISIX 会记录一条警告，并在所有进程中都不导出这两个连接指标，而不是导出错误的数值。
+
 ## 示例
 
 以下示例演示如何在不同场景中使用 `prometheus` 插件。
diff --git a/t/cli/test_prometheus_stream.sh b/t/cli/test_prometheus_stream.sh
index fcbfb5a2eec3..e2fff34f6c11 100755
--- a/t/cli/test_prometheus_stream.sh
+++ b/t/cli/test_prometheus_stream.sh
@@ -124,3 +124,26 @@ if ! echo "$out" | grep "apisix_node_info{hostname=" > /dev/null; then
 fi
 
 echo "passed: prometheus works when only stream is enabled"
+
+# the http subsystem exists only to export metrics here, yet every process
+# still reports its own Lua VMs and connection slots
+ok=0
+deadline=$(( $(date +%s) + 20 ))
+{ set +x; } 2>/dev/null
+while [ "$(date +%s)" -lt "$deadline" ]; do
+    out="$(curl -s --connect-timeout 1 --max-time 2 http://127.0.0.1:9091/apisix/prometheus/metrics || true)"
+    if echo "$out" | grep -qE 'apisix_nginx_timers\{process="worker-0",subsystem="stream",state="running"\} [1-9]' \
+        && echo "$out" | grep -qE 'apisix_nginx_lua_memory_bytes\{process="privileged-agent",subsystem="http"\} [1-9]' \
+        && echo "$out" | grep -qE 'apisix_nginx_process_connections_max\{process="privileged-agent"\} 512$'; then
+        ok=1
+        break
+    fi
+    sleep 0.5
+done
+set -x
+if [ "$ok" -ne 1 ]; then
+    echo "failed: process metrics are missing when only stream is enabled"
+    exit 1
+fi
+
+echo "passed: process metrics work when only stream is enabled"

From 81721d1fe0dab26a42eda71e3654776ad7b87a31 Mon Sep 17 00:00:00 2001
From: Nic <qianyong@api7.ai>
Date: Wed, 14 Oct 2026 08:34:30 +0000
Subject: [PATCH 3/6] fix(prometheus): apply the timer limits to the
 stream-only export block

---
 apisix/cli/ngx_tpl.lua          |  9 +++++++++
 t/cli/test_prometheus_stream.sh | 16 ++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua
index e7262a3541d6..d872ff3b56eb 100644
--- a/apisix/cli/ngx_tpl.lua
+++ b/apisix/cli/ngx_tpl.lua
@@ -90,6 +90,15 @@ http {
                       .. [=[$prefix/deps/lib/lua/5.1/?.so;;]=]
                       .. [=[{*lua_cpath*};";
 
+    # the same limits as the stream subsystem, which nginx_timers_max
+    # reports for every Lua VM
+    {% if max_pending_timers then %}
+    lua_max_pending_timers {* max_pending_timers *};
+    {% end %}
+    {% if max_running_timers then %}
+    lua_max_running_timers {* max_running_timers *};
+    {% end %}
+
     {% if enabled_stream_plugins["prometheus"] then %}
 
     init_by_lua_block {
diff --git a/t/cli/test_prometheus_stream.sh b/t/cli/test_prometheus_stream.sh
index e2fff34f6c11..754b222c7323 100755
--- a/t/cli/test_prometheus_stream.sh
+++ b/t/cli/test_prometheus_stream.sh
@@ -147,3 +147,19 @@ if [ "$ok" -ne 1 ]; then
 fi
 
 echo "passed: process metrics work when only stream is enabled"
+
+# the export-only http block runs on the same timer limits as the stream
+# subsystem, so the reported limits hold for every Lua VM
+http_block="$(sed -n '/^http {/,/^}/p' conf/nginx.conf)"
+if ! echo "$http_block" | grep -q "lua_max_pending_timers 16384;" \
+    || ! echo "$http_block" | grep -q "lua_max_running_timers 4096;"; then
+    echo "failed: the stream-only http block does not carry the timer limits"
+    exit 1
+fi
+
+if ! echo "$out" | grep -qE 'apisix_nginx_timers_max\{state="pending"\} 16384$'; then
+    echo "failed: nginx_timers_max does not report the configured limit"
+    exit 1
+fi
+
+echo "passed: the stream-only http block carries the timer limits"

From 2f0c948e09d3892e287de6f932ddbb8707fbd0f2 Mon Sep 17 00:00:00 2001
From: Nic <qianyong@api7.ai>
Date: Thu, 15 Oct 2026 03:17:24 +0000
Subject: [PATCH 4/6] fix(prometheus): tolerate a missing nginx_config in the
 connection self check

---
 apisix/plugins/prometheus/process_metrics.lua | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/apisix/plugins/prometheus/process_metrics.lua b/apisix/plugins/prometheus/process_metrics.lua
index 29c95139b5ed..bbd6ad40eddf 100644
--- a/apisix/plugins/prometheus/process_metrics.lua
+++ b/apisix/plugins/prometheus/process_metrics.lua
@@ -164,10 +164,18 @@ local function get_process_label()
 end
 
 
+-- The CLI refuses to start without nginx_config, but a forced re-read of
+-- config.yaml at runtime (plugin reload) can still drop it: `nginx_config: ~`
+-- deletes the default table while merging.
+local function nginx_config()
+    return core.config.local_conf().nginx_config or {}
+end
+
+
 local function timers_max()
-    local nginx_config = core.config.local_conf().nginx_config or {}
-    return nginx_config.max_running_timers or DEFAULT_MAX_RUNNING_TIMERS,
-           nginx_config.max_pending_timers or DEFAULT_MAX_PENDING_TIMERS
+    local conf = nginx_config()
+    return conf.max_running_timers or DEFAULT_MAX_RUNNING_TIMERS,
+           conf.max_pending_timers or DEFAULT_MAX_PENDING_TIMERS
 end
 
 
@@ -195,7 +203,7 @@ local function check_cycle_layout()
         return false, "unexpected nginx cycle layout"
     end
 
-    local event = core.config.local_conf().nginx_config.event
+    local event = nginx_config().event
     local expected = event and tonumber(event.worker_connections)
     if connection_n ~= expected then
         return false, "worker connection slots read as " .. tostring(connection_n)

From 8c89cc82642678f01f3c798a6bfb46530659b732 Mon Sep 17 00:00:00 2001
From: Nic <qianyong@api7.ai>
Date: Thu, 15 Oct 2026 04:02:14 +0000
Subject: [PATCH 5/6] test(prometheus): do not require a timer to be running at
 scrape time

---
 t/cli/test_prometheus_stream.sh       | 2 +-
 t/plugin/prometheus-process-metrics.t | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/t/cli/test_prometheus_stream.sh b/t/cli/test_prometheus_stream.sh
index 754b222c7323..c9b59f7a9cce 100755
--- a/t/cli/test_prometheus_stream.sh
+++ b/t/cli/test_prometheus_stream.sh
@@ -132,7 +132,7 @@ deadline=$(( $(date +%s) + 20 ))
 { set +x; } 2>/dev/null
 while [ "$(date +%s)" -lt "$deadline" ]; do
     out="$(curl -s --connect-timeout 1 --max-time 2 http://127.0.0.1:9091/apisix/prometheus/metrics || true)"
-    if echo "$out" | grep -qE 'apisix_nginx_timers\{process="worker-0",subsystem="stream",state="running"\} [1-9]' \
+    if echo "$out" | grep -qE 'apisix_nginx_timers\{process="worker-0",subsystem="stream",state="running"\} [0-9]+$' \
         && echo "$out" | grep -qE 'apisix_nginx_lua_memory_bytes\{process="privileged-agent",subsystem="http"\} [1-9]' \
         && echo "$out" | grep -qE 'apisix_nginx_process_connections_max\{process="privileged-agent"\} 512$'; then
         ok=1
diff --git a/t/plugin/prometheus-process-metrics.t b/t/plugin/prometheus-process-metrics.t
index 1e4c95e4069c..059343fb006f 100644
--- a/t/plugin/prometheus-process-metrics.t
+++ b/t/plugin/prometheus-process-metrics.t
@@ -86,7 +86,7 @@ passed
             local res = assert(httpc:request_uri(
                 "http://127.0.0.1:1984/apisix/prometheus/metrics"))
             local patterns = {
-                [[apisix_nginx_timers\{process="worker-0",subsystem="http",state="running"\} [1-9]\d*]],
+                [[apisix_nginx_timers\{process="worker-0",subsystem="http",state="running"\} \d+\n]],
                 [[apisix_nginx_timers\{process="worker-0",subsystem="http",state="pending"\} \d+]],
                 [[apisix_nginx_timers\{process="privileged-agent",subsystem="http",state="running"\} \d+]],
                 [[apisix_nginx_timers_max\{state="running"\} 1234\n]],
@@ -191,7 +191,7 @@ process connection metrics are disabled: worker connection slots read as 128, bu
         local res = assert(httpc:request_uri(
             "http://127.0.0.1:1984/apisix/prometheus/metrics"))
         local patterns = {
-            [[apisix_nginx_timers\{process="worker-0",subsystem="stream",state="running"\} [1-9]\d*]],
+            [[apisix_nginx_timers\{process="worker-0",subsystem="stream",state="running"\} \d+\n]],
             [[apisix_nginx_timers\{process="worker-0",subsystem="stream",state="pending"\} \d+]],
             [[apisix_nginx_timers\{process="privileged-agent",subsystem="stream",state="pending"\} \d+]],
             [[apisix_nginx_lua_memory_bytes\{process="worker-0",subsystem="stream"\} [1-9]\d*]],

From 6d33fdb3f54c39e87f239b180ef19eb196af540e Mon Sep 17 00:00:00 2001
From: Nic <qianyong@api7.ai>
Date: Thu, 15 Oct 2026 06:58:57 +0000
Subject: [PATCH 6/6] fix(prometheus): remove the process series of retired
 workers, cover hot reloads

---
 apisix/plugins/prometheus/process_metrics.lua |  51 ++++++++-
 t/cli/test_prometheus_reload.sh               | 106 ++++++++++++++++++
 2 files changed, 156 insertions(+), 1 deletion(-)

diff --git a/apisix/plugins/prometheus/process_metrics.lua b/apisix/plugins/prometheus/process_metrics.lua
index bbd6ad40eddf..ff4ddfb4c35b 100644
--- a/apisix/plugins/prometheus/process_metrics.lua
+++ b/apisix/plugins/prometheus/process_metrics.lua
@@ -31,6 +31,7 @@ local tonumber      = tonumber
 local tostring      = tostring
 local pcall         = pcall
 local ipairs        = ipairs
+local pairs         = pairs
 local select        = select
 
 
@@ -142,6 +143,7 @@ local CONNECTIONS_VERDICT_KEY = "process_metrics:connections_readable"
 local _M = {}
 
 local gauges
+local registry
 local dict
 local process_label
 local no_extra_labels = {}
@@ -153,6 +155,7 @@ local verdict_logged = false
 local timer_running = false
 local stopped = true
 local interval = DEFAULT_INTERVAL
+local retired_cleanups = 0
 
 
 local function get_process_label()
@@ -262,7 +265,8 @@ end
 
 -- extra_labels, when given, appends the same labels to every gauge here:
 -- {names = {...}, values = function() return {...} end}
-function _M.init(registry, extra_labels)
+function _M.init(prometheus_registry, extra_labels)
+    registry = prometheus_registry
     dict = registry.dict
     if extra_labels then
         extra_label_names = extra_labels.names
@@ -347,12 +351,46 @@ function _M.refresh()
 end
 
 
+-- A reload with fewer workers leaves the series of the workers that are gone
+-- in the shared dict, where nothing overwrites them any more. Expiring them is
+-- not an option: some nginx-lua-prometheus versions do not renew the key index
+-- of a series that is set again, so a live series would expire as well.
+local function remove_retired_workers()
+    local names = {}
+    for _, gauge in pairs(gauges) do
+        names[gauge.name] = true
+    end
+
+    local worker_count = ngx.worker.count()
+    local key_index = registry.key_index
+    for _, key in ipairs(key_index:list()) do
+        local name = key:match("^([^{]+){")
+        local id = name and names[name] and key:match('process="worker%-(%d+)"')
+        if id and tonumber(id) >= worker_count then
+            key_index:remove(key)
+            dict:delete(key)
+        end
+    end
+end
+
+
 local function tick(premature)
     if premature or stopped then
         timer_running = false
+        if stopped then
+            core.log.info("process metrics stopped")
+        end
         return
     end
 
+    if retired_cleanups > 0 then
+        retired_cleanups = retired_cleanups - 1
+        local ok, err = pcall(remove_retired_workers)
+        if not ok then
+            core.log.error("failed to remove the metrics of retired workers: ", err)
+        end
+    end
+
     _M.refresh()
 
     local ok, err = timer_at(interval, tick)
@@ -371,6 +409,14 @@ function _M.start(report_interval)
 
     interval = report_interval or DEFAULT_INTERVAL
     stopped = false
+
+    -- one process is enough, and the privileged agent always exists. The second
+    -- pass, one interval later, catches a sample that an exiting worker was
+    -- still writing while the first one ran.
+    if subsystem == "http" and process.type() == "privileged agent" then
+        retired_cleanups = 2
+    end
+
     if timer_running then
         return
     end
@@ -380,7 +426,10 @@ function _M.start(report_interval)
     if not ok then
         timer_running = false
         core.log.error("failed to start process metrics: ", err)
+        return
     end
+
+    core.log.info("process metrics started, reporting every ", interval, "s")
 end
 
 
diff --git a/t/cli/test_prometheus_reload.sh b/t/cli/test_prometheus_reload.sh
index 7a8b1a1f810f..f4ee9f9e459d 100755
--- a/t/cli/test_prometheus_reload.sh
+++ b/t/cli/test_prometheus_reload.sh
@@ -89,3 +89,109 @@ if curl -i http://127.0.0.1:9091/apisix/prometheus/metrics | grep "{}" > /dev/nu
     echo "failed: metrics should not contain '{}' when prometheus is enabled"
     exit 1
 fi
+
+# The per-process metrics follow the plugin through hot reloads: each process
+# runs exactly one reporting timer while prometheus is enabled and none while
+# it is disabled.
+make stop
+sleep 0.5
+rm logs/error.log || true
+
+write_conf() {
+    echo "
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  admin:
+    admin_key: null
+apisix:
+  node_listen: 1984
+nginx_config:
+  error_log_level: info
+  worker_processes: $1
+plugins:
+  - $2
+plugin_attr:
+  prometheus:
+    refresh_interval: 1
+" > conf/config.yaml
+}
+
+# wait_for <description> <command...>: polls for up to 20 seconds
+wait_for() {
+    local desc=$1
+    shift
+    local deadline=$(( $(date +%s) + 20 ))
+    { set +x; } 2>/dev/null
+    while [ "$(date +%s)" -lt "$deadline" ]; do
+        if "$@"; then
+            set -x
+            return 0
+        fi
+        sleep 0.5
+    done
+    set -x
+    echo "failed: $desc"
+    exit 1
+}
+
+metrics() {
+    curl -s --connect-timeout 1 --max-time 2 http://127.0.0.1:9091/apisix/prometheus/metrics || true
+}
+
+# 2 workers and the privileged agent, http only
+started_is() { [ "$(grep -c 'process metrics started' logs/error.log)" -eq "$1" ]; }
+stopped_is() { [ "$(grep -c 'process metrics stopped' logs/error.log)" -eq "$1" ]; }
+reports_workers() {
+    local out
+    out="$(metrics)"
+    echo "$out" | grep -qE 'apisix_nginx_lua_memory_bytes\{process="worker-0",subsystem="http"\} [0-9]' \
+        && echo "$out" | grep -qE 'apisix_nginx_lua_memory_bytes\{process="worker-1",subsystem="http"\} [0-9]' \
+        && echo "$out" | grep -qE 'apisix_nginx_lua_memory_bytes\{process="privileged-agent",subsystem="http"\} [0-9]'
+}
+exports_nothing() { metrics | grep -q "{}"; }
+
+write_conf 2 prometheus
+make run
+
+wait_for "every process reports when prometheus is enabled" reports_workers
+wait_for "one reporting timer per process" started_is 3
+
+write_conf 2 ip-restriction
+curl -s -o /dev/null http://127.0.0.1:9090/v1/plugins/reload -XPUT
+
+wait_for "every reporting timer stops when prometheus is disabled" stopped_is 3
+wait_for "nothing is exported when prometheus is disabled" exports_nothing
+
+write_conf 2 prometheus
+curl -s -o /dev/null http://127.0.0.1:9090/v1/plugins/reload -XPUT
+
+wait_for "reporting resumes when prometheus is enabled again" started_is 6
+wait_for "every process reports again" reports_workers
+
+# reloading with prometheus still enabled keeps the running timer
+curl -s -o /dev/null http://127.0.0.1:9090/v1/plugins/reload -XPUT
+sleep 3
+if ! started_is 6 || ! stopped_is 3; then
+    echo "failed: a plugin reload must not start a second reporting timer"
+    exit 1
+fi
+
+echo "passed: process metrics follow the plugin through hot reloads"
+
+# Scaling down must not leave the series of the workers that are gone.
+write_conf 1 prometheus
+make reload
+
+no_retired_worker() {
+    local out
+    out="$(metrics)"
+    echo "$out" | grep -qE 'apisix_nginx_lua_memory_bytes\{process="worker-0",subsystem="http"\} [0-9]' \
+        && ! echo "$out" | grep -q 'process="worker-1"'
+}
+wait_for "the series of a retired worker are removed" no_retired_worker
+
+make stop
+
+echo "passed: the series of retired workers are removed"