From a85651ba2628f155acdfc876e630c35ec61f6946 Mon Sep 17 00:00:00 2001 From: Tal Weiss <73443663+talwgx@users.noreply.github.com> Date: Sun, 25 Oct 2026 20:48:52 -0400 Subject: [PATCH 1/2] Rate regulator: at most 500 keep-count lines per service per window past its total A keep-count line (one of the first K held lines of an ERROR, CRITICAL or FATAL pattern in a window) now reserves the service total like any line. Past the total, a service passes at most keepMaxOverTotal of them per window (default 500, counter rg_kx_); the rest take the service's action. 0 removes the bound. Lines an active mute entry passes are unchanged. Both cap-file variants; config.yaml and schema.json carry the key. Co-Authored-By: Claude Opus 5.5 --- pipelines/run/receive/rate/config.yaml | 24 +++++++++++++------ pipelines/run/receive/rate/rate-object-cap.js | 15 ++++++++++-- .../receive/rate/rate-object-lookup-cap.js | 15 ++++++++++-- pipelines/run/receive/rate/schema.json | 13 ++++++++-- 4 files changed, 54 insertions(+), 13 deletions(-) diff --git a/pipelines/run/receive/rate/config.yaml b/pipelines/run/receive/rate/config.yaml index 56ce547..f14ac8b 100644 --- a/pipelines/run/receive/rate/config.yaml +++ b/pipelines/run/receive/rate/config.yaml @@ -78,7 +78,9 @@ rateReceiver: # 'keepCounts' is the number of lines per pattern per window a level passes once # the pattern is over its cap or its service over its total, also under a mute # entry, in place of its 'severityFloors' draw. ERROR, CRITICAL and FATAL keep - # a count of 50 even when removed here; -1 gives a level its floor instead. + # a count of 50 even when removed here; -1 gives a level its floor instead. In + # a service with a total, the lines count toward it and 'keepMaxOverTotal' + # bounds them past it. keepCounts: - ERROR=50 - CRITICAL=50 # the level enrichment maps fatal/critical -> CRITICAL @@ -118,6 +120,13 @@ rateReceiver: # bound, so every sample line passes. baselineMaxOverTotal: 50 + # 'keepMaxOverTotal' is the most 'keepCounts' lines a service passes per window + # once it has passed its total; the rest take the service's action. Within the + # total they pass like any line. It bounds a service that prints many new + # error patterns in one window. Counted in lines whatever 'unit' is. 0 removes + # the bound, so every keep-count line passes. + keepMaxOverTotal: 500 + # 'ingestionCostPerGB' is used only to render savings metrics in dollars; it does # not affect the keep/drop decision (the decision is share-based). ingestionCostPerGB: 1.5 @@ -262,12 +271,13 @@ rateReceiver: # takes the service's action. The 'baselineCount' sample counts toward the # total like any line, and once the total is spent the service passes at most # 'baselineMaxOverTotal' more sample lines in the window. The 'keepCounts' - # lines and the lines a mute entry passes still pass and count toward the - # total, so an error storm or a muted pattern can exceed it by that much. Read - # by the cap-file regulators, for containers with a cap. A row keyed '*' - # applies to every service without a row of its own; a row of 0 exempts its - # service, and a '*' row of 0 gives every unlisted service no total. A missing - # file loads as an empty table. + # lines count toward the total the same way, and once it is spent the service + # passes at most 'keepMaxOverTotal' more of them in the window. The lines a + # mute entry passes still pass and count toward the total, so a muted pattern + # can exceed it by that much. Read by the cap-file regulators, for containers + # with a cap. A row keyed '*' applies to every service without a row of its + # own; a row of 0 exempts its service, and a '*' row of 0 gives every unlisted + # service no total. A missing file loads as an empty table. # # File format (CSV; header row + one comma-separated entry per row): # container,total diff --git a/pipelines/run/receive/rate/rate-object-cap.js b/pipelines/run/receive/rate/rate-object-cap.js index 1fa834c..4c531c5 100644 --- a/pipelines/run/receive/rate/rate-object-cap.js +++ b/pipelines/run/receive/rate/rate-object-cap.js @@ -266,10 +266,21 @@ export class rateReceiverCapObject extends TenXObject { if (!keep && inSample && sampleOver > 0 && TenXCounter.getAndInc("rg_sx_" + container, 1, windowMs) < sampleOver) keep = true; // Error-class levels with a keep count: the first K lines of a pattern - // held by its cap or by the service total pass each window. + // held by its cap or by the service total pass each window, within the + // service total like any line, and past it up to + // rateReceiverKeepMaxOverTotal keep lines per service per window. var keptBefore = -1; if (!keep && keepCount >= 0) keptBefore = TenXCounter.getAndInc("rg_kc_" + key, 1, windowMs); - if (keptBefore >= 0 && keptBefore < keepCount) keep = true; + var inKeep = false; + if (keptBefore >= 0 && keptBefore < keepCount) inKeep = true; + var keepOver = TenXEnv.get("rateReceiverKeepMaxOverTotal", 500); + if (inKeep && serviceTotal <= 0) keep = true; + if (inKeep && keepOver <= 0) keep = true; + if (inKeep && !keep) passedBefore = TenXCounter.getAndInc("rg_pas_" + container, units, windowMs); + if (inKeep && !keep && (passedBefore + units) <= serviceTotal) counted = true; + if (inKeep && !keep && !counted) TenXCounter.getAndInc("rg_pas_" + container, 0 - units, windowMs); + if (inKeep && counted) keep = true; + if (inKeep && !keep && TenXCounter.getAndInc("rg_kx_" + container, 1, windowMs) < keepOver) keep = true; if (keep && serviceTotal > 0 && !counted) TenXCounter.getAndInc("rg_pas_" + container, units, windowMs); if (keep) return true; diff --git a/pipelines/run/receive/rate/rate-object-lookup-cap.js b/pipelines/run/receive/rate/rate-object-lookup-cap.js index 60d2569..f9d2508 100644 --- a/pipelines/run/receive/rate/rate-object-lookup-cap.js +++ b/pipelines/run/receive/rate/rate-object-lookup-cap.js @@ -242,10 +242,21 @@ export class rateReceiverLookupCapObject extends TenXObject { if (!keep && inSample && sampleOver > 0 && TenXCounter.getAndInc("rg_sx_" + container, 1, windowMs) < sampleOver) keep = true; // Error-class levels with a keep count: the first K lines of a pattern - // held by its cap or by the service total pass each window. + // held by its cap or by the service total pass each window, within the + // service total like any line, and past it up to + // rateReceiverKeepMaxOverTotal keep lines per service per window. var keptBefore = -1; if (!keep && keepCount >= 0) keptBefore = TenXCounter.getAndInc("rg_kc_" + key, 1, windowMs); - if (keptBefore >= 0 && keptBefore < keepCount) keep = true; + var inKeep = false; + if (keptBefore >= 0 && keptBefore < keepCount) inKeep = true; + var keepOver = TenXEnv.get("rateReceiverKeepMaxOverTotal", 500); + if (inKeep && serviceTotal <= 0) keep = true; + if (inKeep && keepOver <= 0) keep = true; + if (inKeep && !keep) passedBefore = TenXCounter.getAndInc("rg_pas_" + container, units, windowMs); + if (inKeep && !keep && (passedBefore + units) <= serviceTotal) counted = true; + if (inKeep && !keep && !counted) TenXCounter.getAndInc("rg_pas_" + container, 0 - units, windowMs); + if (inKeep && counted) keep = true; + if (inKeep && !keep && TenXCounter.getAndInc("rg_kx_" + container, 1, windowMs) < keepOver) keep = true; if (keep && serviceTotal > 0 && !counted) TenXCounter.getAndInc("rg_pas_" + container, units, windowMs); if (keep) return true; diff --git a/pipelines/run/receive/rate/schema.json b/pipelines/run/receive/rate/schema.json index cd5f8eb..9849c4b 100644 --- a/pipelines/run/receive/rate/schema.json +++ b/pipelines/run/receive/rate/schema.json @@ -76,7 +76,7 @@ "string", "null" ], - "markdownDescription" : "Lines per pattern per window that pass for a level once the pattern is held\n\nDefines a map of [severity levels](https://doc.log10x.com/run/initialize/level/) to a count: the number of lines of each pattern of that level that pass per window once the pattern is over its cap or its container is over its total (`rateReceiverTotalLookupFile`), and under an active mute-file entry. Lines past the count take the container's action, or the mute's sample rate under a mute. A listed level skips its `rateReceiverSeverityFloors` draw. The count is per pattern, per container, per regulator instance (one per node), per window (`rateReceiverResetIntervalMs`). The lines it passes count toward the container's total, so an error storm can exceed the total by the count times the storming patterns. Keys must match the level vocabulary emitted by the level enrichment. The module's settings set `ERROR=50`, `CRITICAL=50` and `FATAL=50`, and entries merge by level, so those three levels always keep a count (50 unless set); an entry for one of them in the config or on the command line comes later and wins, and `-1` gives it its floor instead. Any other level not listed follows its floor. For example: ``` yaml keepCounts: - ERROR=50 - CRITICAL=50 - FATAL=50 ```", + "markdownDescription" : "Lines per pattern per window that pass for a level once the pattern is held\n\nDefines a map of [severity levels](https://doc.log10x.com/run/initialize/level/) to a count: the number of lines of each pattern of that level that pass per window once the pattern is over its cap or its container is over its total (`rateReceiverTotalLookupFile`), and under an active mute-file entry. Lines past the count take the container's action, or the mute's sample rate under a mute. A listed level skips its `rateReceiverSeverityFloors` draw. The count is per pattern, per container, per regulator instance (one per node), per window (`rateReceiverResetIntervalMs`). The lines it passes count toward the container's total, and once the total is spent the container passes at most `rateReceiverKeepMaxOverTotal` more of them in the window. Keys must match the level vocabulary emitted by the level enrichment. The module's settings set `ERROR=50`, `CRITICAL=50` and `FATAL=50`, and entries merge by level, so those three levels always keep a count (50 unless set); an entry for one of them in the config or on the command line comes later and wins, and `-1` gives it its floor instead. Any other level not listed follows its floor. For example: ``` yaml keepCounts: - ERROR=50 - CRITICAL=50 - FATAL=50 ```", "items" : { "type" : "string" } @@ -147,7 +147,7 @@ "string", "null" ], - "markdownDescription" : "Declarative per-service total file, keyed by the `rateReceiverContainerField` value; bounds what each capped service passes per window\n\nDefines the path to a declarative **per-service total file** (sibling to the cap file). Each entry bounds what a service passes per window (`rateReceiverResetIntervalMs`) on this regulator instance, in `rateReceiverUnit`. Read by the cap-file regulator classes, for containers with a resolved cap. Two rules apply once a service has a total: - **Largest first.** When the service's caps alone let more than its total through in the previous window, every pattern's cap scales by total / that amount for the whole window, or by what they let through so far this window once that is larger, so the service's largest patterns are held first and its small ones keep passing. The scale follows the previous window, so the window after a storm is scaled by the storm for its whole length and recovers the window after. The first window after a quiet one fills its total in arrival order before the scale applies. A service whose first lines of this window and of the last window it sent in are two window lengths or more apart scales by total / its units so far this window instead. - **Hard bound.** Once the lines the service passed reach its total, every further line takes the service's action (`rateReceiverActionLookupFile`). The `rateReceiverBaselineCount` sample counts toward the total like any line, and once the total is spent the service passes at most `rateReceiverBaselineMaxOverTotal` more sample lines in the window. The `rateReceiverKeepCounts` lines and the lines an active mute entry passes always pass and count toward the total, so an error storm or a muted pattern can exceed it by that much: the keep count is per pattern, with no limit per service. **File format** (CSV; header row + one comma-separated entry per row): ``` container,total ,[:][:] *,[:][:] ``` - `container`, value of `rateReceiverContainerField` (the k8s container = the service), or `*` for every service without a row of its own. - `total`, the service's total per window on this instance, in `rateReceiverUnit`. `0` exempts the service; a `*` row of `0` gives every service without a row of its own no total. - `untilEpochSec`, optional Unix-epoch (seconds) expiry; past it the entry is a no-op. - `reason`, optional free-text for audit. Must not contain commas (would break CSV parsing). A missing file loads as an empty table, so no service has a total until a row names it. Each regulator instance holds its own total; the MCP's budget run (`log10x_setup_recurring`) divides a fleet total across instances. **From a policy repo.** With `GH_ENABLED=true`, the default is the GitHub policy lane's mirrored copy of the repo's `log10x/policy/totals.csv`: `${GH_DEST}/log10x/policy/totals.csv`, with `GH_DEST` defaulting to `${java.io.tmpdir}/tenx/github` (`pipelines/gitops/config.yaml`). Each sync rewrites the copy in place, so the lookup hot-reloads. `GH_ENABLED` turns on with `true`, `True` or `TRUE`, as the lane reads it, and with the Kubernetes ConfigMap lane also on the GitHub copy wins: enable one lane. (Default: )", + "markdownDescription" : "Declarative per-service total file, keyed by the `rateReceiverContainerField` value; bounds what each capped service passes per window\n\nDefines the path to a declarative **per-service total file** (sibling to the cap file). Each entry bounds what a service passes per window (`rateReceiverResetIntervalMs`) on this regulator instance, in `rateReceiverUnit`. Read by the cap-file regulator classes, for containers with a resolved cap. Two rules apply once a service has a total: - **Largest first.** When the service's caps alone let more than its total through in the previous window, every pattern's cap scales by total / that amount for the whole window, or by what they let through so far this window once that is larger, so the service's largest patterns are held first and its small ones keep passing. The scale follows the previous window, so the window after a storm is scaled by the storm for its whole length and recovers the window after. The first window after a quiet one fills its total in arrival order before the scale applies. A service whose first lines of this window and of the last window it sent in are two window lengths or more apart scales by total / its units so far this window instead. - **Hard bound.** Once the lines the service passed reach its total, every further line takes the service's action (`rateReceiverActionLookupFile`). The `rateReceiverBaselineCount` sample counts toward the total like any line, and once the total is spent the service passes at most `rateReceiverBaselineMaxOverTotal` more sample lines in the window. The `rateReceiverKeepCounts` lines count toward the total the same way, and once it is spent the service passes at most `rateReceiverKeepMaxOverTotal` more of them in the window. The lines an active mute entry passes always pass and count toward the total, so a muted pattern can exceed it by that much. **File format** (CSV; header row + one comma-separated entry per row): ``` container,total ,[:][:] *,[:][:] ``` - `container`, value of `rateReceiverContainerField` (the k8s container = the service), or `*` for every service without a row of its own. - `total`, the service's total per window on this instance, in `rateReceiverUnit`. `0` exempts the service; a `*` row of `0` gives every service without a row of its own no total. - `untilEpochSec`, optional Unix-epoch (seconds) expiry; past it the entry is a no-op. - `reason`, optional free-text for audit. Must not contain commas (would break CSV parsing). A missing file loads as an empty table, so no service has a total until a row names it. Each regulator instance holds its own total; the MCP's budget run (`log10x_setup_recurring`) divides a fleet total across instances. **From a policy repo.** With `GH_ENABLED=true`, the default is the GitHub policy lane's mirrored copy of the repo's `log10x/policy/totals.csv`: `${GH_DEST}/log10x/policy/totals.csv`, with `GH_DEST` defaulting to `${java.io.tmpdir}/tenx/github` (`pipelines/gitops/config.yaml`). Each sync rewrites the copy in place, so the lookup hot-reloads. `GH_ENABLED` turns on with `true`, `True` or `TRUE`, as the lane reads it, and with the Kubernetes ConfigMap lane also on the GitHub copy wins: enable one lane. (Default: )", "default" : "" }, "configGenerationFile" : { @@ -185,6 +185,15 @@ "markdownDescription" : "Sample lines a service passes per window once it has passed its total\n\nDefines how many `rateReceiverBaselineCount` sample lines a service passes per window on this regulator instance once it has passed its total (`rateReceiverTotalLookupFile`). Within the total the sample passes like any line. Past it, sample lines pass up to this count, and the rest take the regular path: the keep count for the levels in `rateReceiverKeepCounts`, then the service's action. This bounds a service that prints many new patterns in one window, each of which would otherwise pass its sample over the total. The count is in lines whatever `rateReceiverUnit` is, like the sample and the keep counts, so under `bytes` a service can exceed its total by this many of its own lines. `0` removes the bound, so every sample line passes. Read by the cap-file regulators, for services with a total. (Accepts number or string with $= prefix for runtime evaluation) (Default: 50)", "default" : 50 }, + "keepMaxOverTotal" : { + "type" : [ + "number", + "string", + "null" + ], + "markdownDescription" : "Keep-count lines a service passes per window once it has passed its total\n\nDefines how many `rateReceiverKeepCounts` lines a service passes per window on this regulator instance once it has passed its total (`rateReceiverTotalLookupFile`). Within the total a keep-count line passes like any line. Past it, keep-count lines pass up to this count, and the rest take the service's action. This bounds a service that prints many new error patterns in one window, each of which would otherwise pass its keep count over the total. The count is in lines whatever `rateReceiverUnit` is, like the keep counts, so under `bytes` a service can exceed its total by this many of its own lines. `0` removes the bound, so every keep-count line passes. The lines an active mute entry passes follow the entry and are not counted here. Read by the cap-file regulators, for services with a total. (Accepts number or string with $= prefix for runtime evaluation) (Default: 500)", + "default" : 500 + }, "absoluteCap" : { "type" : [ "number", From d0a996140ecbe6cc1cc06b4cd5c5514790774664 Mon Sep 17 00:00:00 2001 From: Tal Weiss <73443663+talwgx@users.noreply.github.com> Date: Sun, 25 Oct 2026 21:06:09 -0400 Subject: [PATCH 2/2] Regulator gate: three keep-bound cases; engines run detached and named e1: 1,422 new ERROR patterns of 7 lines over a 1-byte cap and a 100-line total pass 650 (the total, 50 sample lines, 500 keep-count lines), not 9,954. e2: keepMaxOverTotal 0 passes all 9,954. e3: e1 with a mute file present, the mute and cap regulator. On edge-10x 1.1.150: e1 and e3 fail on main and pass on the fix; k1 to b3 and w1 pass on both. run.sh starts each engine with docker run -d --name (PREFIX) and removes it with docker rm -f -v, waits for the Forward input's listening line before a paced case sends, and defaults to edge-10x 1.1.150. Co-Authored-By: Claude Opus 5.5 --- tests/regulator/cases.py | 68 ++++++++++++++++++++++++++++++++++++++++ tests/regulator/run.sh | 28 +++++++++++------ 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/tests/regulator/cases.py b/tests/regulator/cases.py index ef591ff..4606f04 100644 --- a/tests/regulator/cases.py +++ b/tests/regulator/cases.py @@ -301,6 +301,66 @@ def case_b3_sample_bound_mute(): return lines, files, args, env, {"pass_by_container": {"svc": 100}} +BURST_PATTERNS = 1422 +BURST_LINES = 7 +TRIPLE_WIDTH = 3 * max(len(w) for w in WORDS) + 8 + + +def triple_words(p): + """Three different words per pattern, in order: 6,840 distinct triples of the 20 words.""" + a = p // 342 + rest = [w for w in range(20) if w != a] + b = rest[(p % 342) // 18] + c = [w for w in rest if w != b][p % 18] + return WORDS[a], WORDS[b], WORDS[c] + + +def burst_line(p, i): + w = "%s %s %s" % triple_words(p) + return record("svc", "2026-10-18T10:00:00.000Z ERROR %s migration failed id=%0*d" % (w, TRIPLE_WIDTH - len(w), i)) + + +def error_burst(extra_args=(), files_extra=None): + """1,422 new ERROR patterns of 7 lines each in one window, in a service whose total is 100 + lines and whose cap is 1 byte, so every line is over its cap: each pattern's first 5 lines + are its sample and the next 2 reach the keep count.""" + lines = shuffled([burst_line(p, p * 10 + i) for p in range(BURST_PATTERNS) for i in range(BURST_LINES)], 43) + b = size(lines[0]) + assert len({size(l) for l in lines}) == 1 + files = {"caps.csv": csv("container,cap", [("svc", 1)]), + "totals.csv": csv("container,total", [("svc", 100 * b)]), + "actions.csv": csv("container,action", [("svc", "tier_down")])} + files.update(files_extra or {}) + return lines, files, ["rateReceiverWarmupMs", "0"] + list(extra_args), ["CAP", "ACTION", "TOTAL"] + + +def case_e1_keep_bound(): + """1,422 new ERROR patterns, a 100-line total: the service passes its total, then at most + rateReceiverBaselineMaxOverTotal (50) sample lines and rateReceiverKeepMaxOverTotal (500) + keep-count lines, 650 in all; the rest take the service's action.""" + lines, files, args, env = error_burst() + return lines, files, args, env, {"pass_by_container": {"svc": 650}, "routes": {"svc": {"tier_down"}}, + "distinct_patterns": {"svc": BURST_PATTERNS}} + + +def case_e2_keep_bound_off(): + """rateReceiverKeepMaxOverTotal 0 removes the bound: every line passes, the sample past the + sample bound by the keep count (the option is new, so the case runs on the fix only).""" + lines, files, args, env = error_burst(["rateReceiverKeepMaxOverTotal", "0"]) + return lines, files, args, env, {"pass_by_container": {"svc": BURST_PATTERNS * BURST_LINES}, + "distinct_patterns": {"svc": BURST_PATTERNS}, "config": "fix-only"} + + +def case_e3_keep_bound_mute(): + """the same burst with a mute file present (the mute + cap variant), the mute entry for a + pattern that never prints: 650 pass.""" + lines, files, args, env = error_burst( + ["rateReceiverLookupFile", "/policy/mutes.csv"], + {"mutes.csv": csv("fieldSet,value", [("never_printed_pattern", "0:%d:test" % FUTURE)])}) + return lines, files, args, env, {"pass_by_container": {"svc": 650}, "routes": {"svc": {"tier_down"}}, + "distinct_patterns": {"svc": BURST_PATTERNS}} + + def case_d1_defaults(): """4a.4: no command-line overrides. A capped container first seen this window is held.""" lines = [error_line("svc", i) for i in range(1000)] @@ -433,6 +493,9 @@ def case_w4_windows_late(): "b1-sample-bound": case_b1_sample_bound, "b2-sample-bound-off": case_b2_sample_bound_off, "b3-sample-bound-mute": case_b3_sample_bound_mute, + "e1-keep-bound": case_e1_keep_bound, + "e2-keep-bound-off": case_e2_keep_bound_off, + "e3-keep-bound-mute": case_e3_keep_bound_mute, "w1-windows": case_w1_windows, "w2-windows-mute": case_w2_windows_mute, "w3-windows-scaled": case_w3_windows_scaled, @@ -590,6 +653,11 @@ def check(name, out): facts.append("window %d large patterns passed %d to %d lines each" % ( i + 1, min(got[p] for p in moved), max(got[p] for p in moved))) start += n + for c, want in expect.get("distinct_patterns", {}).items(): + got = len({r["message_pattern"] for r in rows if r["k8s_container"] == c}) + facts.append("%s patterns %d (want %d)" % (c, got, want)) + if got != want: + problems.append("%s has %d patterns, want %d" % (c, got, want)) for c, want in expect.get("pass_by_container", {}).items(): got = sum(1 for r in passed if r["k8s_container"] == c) facts.append("%s passed %d (want %d)" % (c, got, want)) diff --git a/tests/regulator/run.sh b/tests/regulator/run.sh index f236357..d38662c 100755 --- a/tests/regulator/run.sh +++ b/tests/regulator/run.sh @@ -15,21 +15,23 @@ # Environment: # MODULES_DIR modules checkout to mount for the fix arm (required) # CONFIG_DIR config checkout to mount for the fix arm (default: this one) -# IMAGE engine image (default: edge-10x 1.1.145, by digest) +# IMAGE engine image (default: edge-10x 1.1.150, by digest) # OUT output folder (default: a new temporary folder) # ARMS arms to run (default: "main fix"; add "before" with BEFORE_REF) # BEFORE_REF config commit for the before arm, extracted with git archive # PORT host port for the paced cases' Forward input (default: 24298) +# PREFIX container name prefix (default: tenx-regulator) set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CONFIG_DIR="$(cd "${CONFIG_DIR:-$HERE/../..}" && pwd)" : "${MODULES_DIR:?set MODULES_DIR to a modules checkout}" MODULES_DIR="$(cd "$MODULES_DIR" && pwd)" -IMAGE="${IMAGE:-log10x/edge-10x:1.1.145@sha256:aec232a2f7d9da8277a529ea9d844267f8c6f10bb241f088c1fbe774b8b9949e}" +IMAGE="${IMAGE:-ghcr.io/log-10x/edge-10x:1.1.150@sha256:3e755bbc14780a27e643e09b76dfd7ec9616cab67833f70daf9420eff646d350}" OUT="${OUT:-$(mktemp -d)}" ARMS="${ARMS:-main fix}" PORT="${PORT:-24298}" +PREFIX="${PREFIX:-tenx-regulator}" CASES=("$@") [ ${#CASES[@]} -gt 0 ] || mapfile -t CASES < <(python3 -I "$HERE/cases.py" list) @@ -78,24 +80,32 @@ PY local args=() while IFS= read -r a; do [ -n "$a" ] && args+=("$a"); done < "$dir/args" if [ -s "$dir/pace" ]; then run_paced "$name-$arm" "$dir" envs vols args; return; fi + local cname="$PREFIX-$name-$arm" code + docker rm -f -v "$cname" >/dev/null 2>&1 || true set +e - docker run --rm "${envs[@]}" "${vols[@]}" "$IMAGE" @/cfg/replay.yaml ${args[@]+"${args[@]}"} > "$dir/engine.log" 2>&1 - echo $? > "$dir/exit" + if docker run -d --name "$cname" "${envs[@]}" "${vols[@]}" "$IMAGE" @/cfg/replay.yaml ${args[@]+"${args[@]}"} >/dev/null; then + code="$(timeout 600 docker wait "$cname")" || code=124 + docker logs "$cname" > "$dir/engine.log" 2>&1 + else + code=1; echo "docker run failed" > "$dir/engine.log" + fi + docker rm -f -v "$cname" >/dev/null 2>&1 + echo "$code" > "$dir/exit" set -e } # A paced case: the engine runs on paced.yaml (the Forward input) and cases.py sends the # batches. The exit file says 0 once the pipeline started. run_paced() { - local cname="tenx-regulator-$1" dir="$2" + local cname="$PREFIX-$1" dir="$2" local -n p_envs="$3" p_vols="$4" p_args="$5" - docker rm -f "$cname" >/dev/null 2>&1 || true + docker rm -f -v "$cname" >/dev/null 2>&1 || true docker run -d --name "$cname" -p "127.0.0.1:$PORT:24224" "${p_envs[@]}" "${p_vols[@]}" \ -v "$HERE/paced.yaml":/cfg/paced.yaml:ro "$IMAGE" @/cfg/paced.yaml ${p_args[@]+"${p_args[@]}"} >/dev/null local t=0 - until docker logs "$cname" 2>&1 | grep -q "pipeline units:"; do + until docker logs "$cname" 2>&1 | grep -q "Forward protocol server listening"; do if [ "$t" -ge 120 ] || ! docker ps -q -f "name=^$cname$" | grep -q .; then - docker logs "$cname" > "$dir/engine.log" 2>&1; docker rm -f "$cname" >/dev/null 2>&1 + docker logs "$cname" > "$dir/engine.log" 2>&1; docker rm -f -v "$cname" >/dev/null 2>&1 echo 1 > "$dir/exit"; return fi sleep 2; t=$((t + 2)) @@ -105,7 +115,7 @@ run_paced() { sleep 8 docker stop -t 20 "$cname" >/dev/null docker logs "$cname" > "$dir/engine.log" 2>&1 - docker rm "$cname" >/dev/null + docker rm -f -v "$cname" >/dev/null echo 0 > "$dir/exit" }