diff --git a/elastic-hot-tier-budget/README.md b/elastic-hot-tier-budget/README.md index d2b98ea..9c2a8d5 100644 --- a/elastic-hot-tier-budget/README.md +++ b/elastic-hot-tier-budget/README.md @@ -160,3 +160,28 @@ Warm peaked at 20.7 MB in arm A (08:34:51) and 9.1 MB in arm B (08:33:11). Each **Rig additions.** `compose.warm.yaml`, `phases-two-arm-warm.sh` (the 2026-10-24 schedule with warm placement; containers and volumes removed before and after), `es-setup.mjs` and `es-setup-b.mjs` (`TD_PLACEMENT=warm` renders the recipe with the option and checks each bootstrap index sits on a warm node), `meter.py` (each shard's node from `_cat/shards`: per stream, store on hot and on warm; per arm, the warm node's store), `score-two-arm.py` (the warm lines beside the unchanged criteria), `run.sh clean`. Not shown: the cost of a warm tier, a measured Elastic Cloud bill, 300 nodes, compact, autoscaling, or a hold longer than the hot retention. + +## Grouping across containers on the E103 fix, arm A, 2026-10-26 + +The question: on the rig where the defect was found, does the E103 fix return any record holding lines from more than one container? The two-arm smoke's schedule (every phase at a tenth of its length, the dry run at 240 seconds) ran for arm A alone from 00:10 to 00:36 UTC. Results are in `results-e103-smoke/`. + +The engine is commit `0cd458f0f` (log-10x/engine, branch `fix/e103-sequence-key`) with modules commit `6c69d90a` (log-10x/modules, same branch). They ran in the image `local/edge-10x:e103` (sha256:8f74b8bbec2e…), which runs the engine jar on a JVM, so the Receiver had `TENX_JAVA_HEAP=320M` inside its rendered 512 MB limit. Everything else is the 2026-10-24 smoke: the same feed (sha256 24e02233a797…), log10x-mcp 1.30.84 (d6fa2b1), the same ILM policies and the same policy branch. + +**Result: no cross-container record, and delivery is exact.** + +| Count | Value | +|---|---:| +| Lines fed, by seq | 101,663 | +| Non-blank lines fed | 101,637 | +| Records returned | 79,415, plus the Receiver's connect message | +| Records holding lines from more than one container | 0 | +| Non-blank lines missing (sent, in no record) | 0 | +| Lines duplicated or unmatched (returned, no unused sent line) | 0 | +| Records with lines out of feed order | 0 | +| Elasticsearch documents (`tenx-app`, `tenx-tierdown`) | 79,416, with line counts summing to 101,637 | + +**How the count is made.** Fluent Bit keeps every returned record whole in this run. `score-e103.py` matches each record's lines to the lines sent and counts the records whose lines came from more than one container. The 2026-10-24 count (71 of 848,190, and 11 of 78,852 in that day's smoke) treated each record as the run of seqs up to the next record's seq. That holds only while lines are grouped in arrival order. Here it reads 0, with 2 records whose runs do not match their line counts. The scorer's self-test groups the run's feed with one head rule both ways. In arrival order it finds 4,333 cross-container records by lines and the same 4,333 by the run-based count. Per container it finds none by lines, while the run-based count reports 4,322. + +In this run, kafka's four startup lines (seq 2751 to 2754) came back as four kafka records, one line each. The jaeger record at seq 2750 holds only its own line. + +**Rig additions.** `ARM_A_ONLY=1` in `phases-two-arm.sh` runs arm A alone: no cluster B, no copy to it, and every returned record written whole to `data/returned.jsonl`. `gen-receiver-values.mjs` passes `TENX_JAVA_HEAP` to the Receiver. `score-e103.py` makes the count above. diff --git a/elastic-hot-tier-budget/results-e103-smoke/arm-b-rule.json b/elastic-hot-tier-budget/results-e103-smoke/arm-b-rule.json new file mode 100644 index 0000000..099e6d0 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/arm-b-rule.json @@ -0,0 +1,232 @@ +{ + "source": "results-two-arm/replay/source.jsonl", + "speed": 8.0, + "minutes": 20.0, + "source_seconds": 9600.0, + "lines": 67200, + "bytes": 13282731, + "top_half": [ + "opentelemetry-collector" + ], + "top_half_bytes": 9149163, + "top_half_share": 0.6888, + "regex": "^(opentelemetry-collector)$", + "services": [ + { + "service": "opentelemetry-collector", + "bytes": 9149163, + "lines": 32427, + "share": 0.6888 + }, + { + "service": "kafka", + "bytes": 1891441, + "lines": 10216, + "share": 0.1424 + }, + { + "service": "cart", + "bytes": 718696, + "lines": 12130, + "share": 0.0541 + }, + { + "service": "ad", + "bytes": 353362, + "lines": 2180, + "share": 0.0266 + }, + { + "service": "recommendation", + "bytes": 317242, + "lines": 1034, + "share": 0.0239 + }, + { + "service": "frontend", + "bytes": 292949, + "lines": 4934, + "share": 0.0221 + }, + { + "service": "grafana", + "bytes": 292610, + "lines": 1861, + "share": 0.022 + }, + { + "service": "fluentd-10x", + "bytes": 85930, + "lines": 641, + "share": 0.0065 + }, + { + "service": "opensearch", + "bytes": 49164, + "lines": 363, + "share": 0.0037 + }, + { + "service": "kube-proxy", + "bytes": 28157, + "lines": 245, + "share": 0.0021 + }, + { + "service": "frontend-proxy", + "bytes": 27512, + "lines": 121, + "share": 0.0021 + }, + { + "service": "jaeger", + "bytes": 17627, + "lines": 87, + "share": 0.0013 + }, + { + "service": "valkey-cart", + "bytes": 13212, + "lines": 192, + "share": 0.001 + }, + { + "service": "prometheus-server", + "bytes": 10711, + "lines": 67, + "share": 0.0008 + }, + { + "service": "metrics-server", + "bytes": 5148, + "lines": 35, + "share": 0.0004 + }, + { + "service": "coredns", + "bytes": 4099, + "lines": 49, + "share": 0.0003 + }, + { + "service": "accounting", + "bytes": 3653, + "lines": 76, + "share": 0.0003 + }, + { + "service": "postgresql", + "bytes": 3543, + "lines": 64, + "share": 0.0003 + }, + { + "service": "load-generator", + "bytes": 2640, + "lines": 45, + "share": 0.0002 + }, + { + "service": "wait-for-kafka", + "bytes": 2592, + "lines": 150, + "share": 0.0002 + }, + { + "service": "init-config", + "bytes": 2565, + "lines": 110, + "share": 0.0002 + }, + { + "service": "grafana-sc-dashboard", + "bytes": 1862, + "lines": 14, + "share": 0.0001 + }, + { + "service": "grafana-sc-alerts", + "bytes": 1327, + "lines": 10, + "share": 0.0001 + }, + { + "service": "flagd", + "bytes": 1322, + "lines": 16, + "share": 0.0001 + }, + { + "service": "grafana-sc-datasources", + "bytes": 1322, + "lines": 10, + "share": 0.0001 + }, + { + "service": "payment", + "bytes": 1104, + "lines": 34, + "share": 0.0001 + }, + { + "service": "email", + "bytes": 766, + "lines": 13, + "share": 0.0001 + }, + { + "service": "image-provider", + "bytes": 650, + "lines": 40, + "share": 0.0 + }, + { + "service": "aws-vpc-cni-init", + "bytes": 556, + "lines": 7, + "share": 0.0 + }, + { + "service": "fraud-detection", + "bytes": 543, + "lines": 6, + "share": 0.0 + }, + { + "service": "aws-node", + "bytes": 388, + "lines": 6, + "share": 0.0 + }, + { + "service": "wait-for-valkey-cart", + "bytes": 268, + "lines": 11, + "share": 0.0 + }, + { + "service": "currency", + "bytes": 265, + "lines": 1, + "share": 0.0 + }, + { + "service": "flagd-ui", + "bytes": 194, + "lines": 3, + "share": 0.0 + }, + { + "service": "aws-eks-nodeagent", + "bytes": 122, + "lines": 1, + "share": 0.0 + }, + { + "service": "quote", + "bytes": 26, + "lines": 1, + "share": 0.0 + } + ] +} \ No newline at end of file diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/actions.csv new file mode 100644 index 0000000..b86a442 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/actions.csv @@ -0,0 +1,2 @@ +container,action +tenx-seed,pass::install seed diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/caps.csv new file mode 100644 index 0000000..e74fc03 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/caps.csv @@ -0,0 +1,2 @@ +container,cap +tenx-seed,1::install seed diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/commit.json new file mode 100644 index 0000000..68acd18 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/commit.json @@ -0,0 +1 @@ +{"before":"78c5eea4ed9fd3c0b1fef9757f4904e9284207d1","after":"78c5eea4ed9fd3c0b1fef9757f4904e9284207d1","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/config-generation.csv new file mode 100644 index 0000000..cc83fe1 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,seed diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/latest.md new file mode 100644 index 0000000..e69de29 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/report.txt new file mode 100644 index 0000000..7762ce1 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/report.txt @@ -0,0 +1,54 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp [DRY RUN] +[tenx-recur] Hot tier: 1 MB of 1000 GB (1000 GB set in the policy) on 1 hot node; at these limits it settles at about 58 MB within 2 hours, with 0 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 1 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 0.88 bytes of store kept, 0.88 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 784 MB a day, under the 13589 GB a day that fits the size; every statement keeps its lines, with room to grow 2.0 times. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 597 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:14:47.000Z + +Hot tier: 1 MB of 1000 GB (1000 GB set in the policy) on 1 hot node; at these limits it settles at about 58 MB within 2 hours, with 0 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 1 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 0.88 bytes of store kept, 0.88 as tier_down (measured on the indices on hot). +- The Receiver's statements send 784 MB a day, under the 13589 GB a day that fits the size; every statement keeps its lines, with room to grow 2.0 times. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 597 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 8 MB | 16 MB | 4359 | 43432 | +| ad | the frozen tier | 1 | 10 MB | 20 MB | 25377 | 56529 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 832 | 832 | +| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 748 | 3948 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 2 MB | 761 | 5026 | +| cart | the frozen tier | 1 | 19 MB | 37 MB | 63511 | 104041 | +| coredns | the frozen tier | 1 | 6 MB | 11 MB | 12287 | 31658 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1186 | 1186 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1705 | 3458 | +| flagd | the frozen tier | 1 | 2 MB | 4 MB | 959 | 11159 | +| flagd-ui | the frozen tier | 1 | 0 MB | 1 MB | 706 | 1974 | +| fraud-detection | the frozen tier | 1 | 1 MB | 1 MB | 1386 | 2949 | +| frontend | the frozen tier | 1 | 5 MB | 10 MB | 20593 | 27902 | +| frontend-proxy | the frozen tier | 1 | 23 MB | 46 MB | 6613 | 129136 | +| grafana | the frozen tier | 1 | 251 MB | 502 MB | 6064 | 1395753 | +| grafana-sc-alerts | the frozen tier | 1 | 2 MB | 3 MB | 1056 | 8637 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 4 MB | 2150 | 12214 | +| grafana-sc-datasources | the frozen tier | 1 | 2 MB | 3 MB | 1085 | 8756 | +| image-provider | the frozen tier | 1 | 4 MB | 8 MB | 4554 | 21955 | +| init-config | the frozen tier | 1 | 1 MB | 3 MB | 7256 | 7256 | +| jaeger | the frozen tier | 1 | 14 MB | 27 MB | 4992 | 76313 | +| kafka | the frozen tier | 1 | 125 MB | 251 MB | 59246 | 696146 | +| kube-proxy | the frozen tier | 1 | 33 MB | 65 MB | 44101 | 180621 | +| load-generator | the frozen tier | 1 | 2 MB | 4 MB | 9159 | 9994 | +| metrics-server | the frozen tier | 1 | 6 MB | 11 MB | 4501 | 31281 | +| opensearch | the frozen tier | 1 | 43 MB | 86 MB | 11605 | 237669 | +| opentelemetry-collector | the frozen tier | 1 | 209 MB | 417 MB | 524270 | 1159227 | +| wait-for-kafka | the frozen tier | 1 | 15 MB | 30 MB | 80530 | 82286 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 2 MB | 5798 | 6418 | +| `*` (any other service) | the frozen tier | 1 | | 13588 GB | 37744131243 | 37744131243 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/run.json new file mode 100644 index 0000000..058621d --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:14:51.625Z","status":"dry_run","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"dry-run: Hot tier: 1 MB of 1000 GB (1000 GB set in the policy) on 1 hot node; at these limits it settles at about 58 MB within 2 hours, with 0 MB a day of counted lines moved to the frozen tier."} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/totals.csv new file mode 100644 index 0000000..6d41912 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001446Z/totals.csv @@ -0,0 +1,2 @@ +container,total +tenx-seed,1::install seed diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/actions.csv new file mode 100644 index 0000000..69b7e87 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/actions.csv @@ -0,0 +1,31 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/caps.csv new file mode 100644 index 0000000..d0e571c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/caps.csv @@ -0,0 +1,31 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/commit.diff new file mode 100644 index 0000000..28f1e80 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/commit.diff @@ -0,0 +1,171 @@ +diff --git a/log10x/policy/actions.csv b/log10x/policy/actions.csv +index b86a442..69b7e87 100644 +--- a/log10x/policy/actions.csv ++++ b/log10x/policy/actions.csv +@@ -1,2 +1,31 @@ + container,action +-tenx-seed,pass::install seed ++accounting,tier_down::budget run ++ad,tier_down::budget run ++aws-eks-nodeagent,tier_down::budget run ++aws-node,tier_down::budget run ++aws-vpc-cni-init,tier_down::budget run ++cart,tier_down::budget run ++coredns,tier_down::budget run ++currency,tier_down::budget run ++email,tier_down::budget run ++flagd,tier_down::budget run ++flagd-ui,tier_down::budget run ++fraud-detection,tier_down::budget run ++frontend,tier_down::budget run ++frontend-proxy,tier_down::budget run ++grafana,tier_down::budget run ++grafana-sc-alerts,tier_down::budget run ++grafana-sc-dashboard,tier_down::budget run ++grafana-sc-datasources,tier_down::budget run ++image-provider,tier_down::budget run ++init-config,tier_down::budget run ++jaeger,tier_down::budget run ++kafka,tier_down::budget run ++kube-proxy,tier_down::budget run ++load-generator,tier_down::budget run ++metrics-server,tier_down::budget run ++opensearch,tier_down::budget run ++opentelemetry-collector,tier_down::budget run ++wait-for-kafka,tier_down::budget run ++wait-for-valkey-cart,tier_down::budget run ++*,tier_down::budget run +diff --git a/log10x/policy/caps.csv b/log10x/policy/caps.csv +index e74fc03..d0e571c 100644 +--- a/log10x/policy/caps.csv ++++ b/log10x/policy/caps.csv +@@ -1,2 +1,31 @@ + container,cap +-tenx-seed,1::install seed ++accounting,1::budget run ++ad,1::budget run ++aws-eks-nodeagent,1::budget run ++aws-node,1::budget run ++aws-vpc-cni-init,1::budget run ++cart,1::budget run ++coredns,1::budget run ++currency,1::budget run ++email,1::budget run ++flagd,1::budget run ++flagd-ui,1::budget run ++fraud-detection,1::budget run ++frontend,1::budget run ++frontend-proxy,1::budget run ++grafana,1::budget run ++grafana-sc-alerts,1::budget run ++grafana-sc-dashboard,1::budget run ++grafana-sc-datasources,1::budget run ++image-provider,1::budget run ++init-config,1::budget run ++jaeger,1::budget run ++kafka,1::budget run ++kube-proxy,1::budget run ++load-generator,1::budget run ++metrics-server,1::budget run ++opensearch,1::budget run ++opentelemetry-collector,1::budget run ++wait-for-kafka,1::budget run ++wait-for-valkey-cart,1::budget run ++*,1::budget run +diff --git a/log10x/policy/config-generation.csv b/log10x/policy/config-generation.csv +index cc83fe1..ddb50d6 100644 +--- a/log10x/policy/config-generation.csv ++++ b/log10x/policy/config-generation.csv +@@ -1,2 +1,2 @@ + key,value +-generation,seed ++generation,88a2a5ef6083 +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 6d41912..5ba2144 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,2 +1,31 @@ + container,total +-tenx-seed,1::install seed ++accounting,18023::budget run ++ad,7339::budget run ++aws-eks-nodeagent,405::budget run ++aws-node,1921::budget run ++aws-vpc-cni-init,2446::budget run ++cart,5928::budget run ++coredns,8044::budget run ++currency,577::budget run ++email,1682::budget run ++flagd,5430::budget run ++flagd-ui,960::budget run ++fraud-detection,1435::budget run ++frontend,13578::budget run ++frontend-proxy,62843::budget run ++grafana,676544::budget run ++grafana-sc-alerts,4203::budget run ++grafana-sc-dashboard,5943::budget run ++grafana-sc-datasources,4261::budget run ++image-provider,9150::budget run ++init-config,3531::budget run ++jaeger,37137::budget run ++kafka,135731::budget run ++kube-proxy,38945::budget run ++load-generator,4863::budget run ++metrics-server,15222::budget run ++opensearch,105815::budget run ++opentelemetry-collector,453312::budget run ++wait-for-kafka,1893::budget run ++wait-for-valkey-cart,1401::budget run ++*,5928::budget run +diff --git a/reports/latest.md b/reports/latest.md +new file mode 100644 +index 0000000..34cef3c +--- /dev/null ++++ b/reports/latest.md +@@ -0,0 +1,46 @@ ++# Hot-tier run: 2026-10-26T00:14:52.000Z ++ ++Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 67 MB within 2 hours, with 191 MB a day of counted lines moved to the frozen tier. ++ ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. ++- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). ++- Each counted byte takes 1.03 bytes of store kept, 1.03 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 777 MB a day; 281 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 586 MB a day on hot whatever the limits; the size holds down to that floor. ++- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 586 MB a day on hot, more than the 281 MB a day that fits, so hot settles above 41 MB at about 67 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++ ++| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | ++|---|---|---|---|---|---|---| ++| accounting | the frozen tier | 1 | 8 MB | 6 MB | 1 | 18023 | ++| ad | the frozen tier | 1 | 10 MB | 3 MB | 1 | 7339 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 405 | ++| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1921 | ++| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2446 | ++| cart | the frozen tier | 1 | 18 MB | 2 MB | 1 | 5928 | ++| coredns | the frozen tier | 1 | 6 MB | 3 MB | 1 | 8044 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 577 | ++| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1682 | ++| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5430 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 960 | ++| fraud-detection | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1435 | ++| frontend | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13578 | ++| frontend-proxy | the frozen tier | 1 | 23 MB | 23 MB | 1 | 62843 | ++| grafana | the frozen tier | 1 | 245 MB | 244 MB | 1 | 676544 | ++| grafana-sc-alerts | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4203 | ++| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5943 | ++| grafana-sc-datasources | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4261 | ++| image-provider | the frozen tier | 1 | 4 MB | 3 MB | 1 | 9150 | ++| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3531 | ++| jaeger | the frozen tier | 1 | 13 MB | 13 MB | 1 | 37137 | ++| kafka | the frozen tier | 1 | 122 MB | 49 MB | 1 | 135731 | ++| kube-proxy | the frozen tier | 1 | 32 MB | 14 MB | 1 | 38945 | ++| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4863 | ++| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 15222 | ++| opensearch | the frozen tier | 1 | 42 MB | 38 MB | 1 | 105815 | ++| opentelemetry-collector | the frozen tier | 1 | 217 MB | 163 MB | 1 | 453312 | ++| wait-for-kafka | the frozen tier | 1 | 14 MB | 1 MB | 1 | 1893 | ++| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1401 | ++| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5928 | ++ ++Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/commit.json new file mode 100644 index 0000000..2937a85 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/commit.json @@ -0,0 +1 @@ +{"before":"78c5eea4ed9fd3c0b1fef9757f4904e9284207d1","after":"afa4fe44ce9771c9dc07d8b99ae01c3444ab6d24","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/config-generation.csv new file mode 100644 index 0000000..ddb50d6 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,88a2a5ef6083 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/latest.md new file mode 100644 index 0000000..34cef3c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/latest.md @@ -0,0 +1,46 @@ +# Hot-tier run: 2026-10-26T00:14:52.000Z + +Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 67 MB within 2 hours, with 191 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.03 bytes of store kept, 1.03 as tier_down (measured on the indices on hot). +- The Receiver's statements send 777 MB a day; 281 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 586 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 586 MB a day on hot, more than the 281 MB a day that fits, so hot settles above 41 MB at about 67 MB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 8 MB | 6 MB | 1 | 18023 | +| ad | the frozen tier | 1 | 10 MB | 3 MB | 1 | 7339 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 405 | +| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1921 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2446 | +| cart | the frozen tier | 1 | 18 MB | 2 MB | 1 | 5928 | +| coredns | the frozen tier | 1 | 6 MB | 3 MB | 1 | 8044 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 577 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1682 | +| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5430 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 960 | +| fraud-detection | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1435 | +| frontend | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13578 | +| frontend-proxy | the frozen tier | 1 | 23 MB | 23 MB | 1 | 62843 | +| grafana | the frozen tier | 1 | 245 MB | 244 MB | 1 | 676544 | +| grafana-sc-alerts | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4203 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5943 | +| grafana-sc-datasources | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4261 | +| image-provider | the frozen tier | 1 | 4 MB | 3 MB | 1 | 9150 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3531 | +| jaeger | the frozen tier | 1 | 13 MB | 13 MB | 1 | 37137 | +| kafka | the frozen tier | 1 | 122 MB | 49 MB | 1 | 135731 | +| kube-proxy | the frozen tier | 1 | 32 MB | 14 MB | 1 | 38945 | +| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4863 | +| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 15222 | +| opensearch | the frozen tier | 1 | 42 MB | 38 MB | 1 | 105815 | +| opentelemetry-collector | the frozen tier | 1 | 217 MB | 163 MB | 1 | 453312 | +| wait-for-kafka | the frozen tier | 1 | 14 MB | 1 MB | 1 | 1893 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1401 | +| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5928 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/report.txt new file mode 100644 index 0000000..9b10164 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/report.txt @@ -0,0 +1,55 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 67 MB within 2 hours, with 191 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.03 bytes of store kept, 1.03 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 777 MB a day; 281 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 586 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:14:52.000Z + +Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 67 MB within 2 hours, with 191 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.03 bytes of store kept, 1.03 as tier_down (measured on the indices on hot). +- The Receiver's statements send 777 MB a day; 281 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 586 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 586 MB a day on hot, more than the 281 MB a day that fits, so hot settles above 41 MB at about 67 MB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 8 MB | 6 MB | 1 | 18023 | +| ad | the frozen tier | 1 | 10 MB | 3 MB | 1 | 7339 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 405 | +| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1921 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2446 | +| cart | the frozen tier | 1 | 18 MB | 2 MB | 1 | 5928 | +| coredns | the frozen tier | 1 | 6 MB | 3 MB | 1 | 8044 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 577 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1682 | +| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5430 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 960 | +| fraud-detection | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1435 | +| frontend | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13578 | +| frontend-proxy | the frozen tier | 1 | 23 MB | 23 MB | 1 | 62843 | +| grafana | the frozen tier | 1 | 245 MB | 244 MB | 1 | 676544 | +| grafana-sc-alerts | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4203 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5943 | +| grafana-sc-datasources | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4261 | +| image-provider | the frozen tier | 1 | 4 MB | 3 MB | 1 | 9150 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3531 | +| jaeger | the frozen tier | 1 | 13 MB | 13 MB | 1 | 37137 | +| kafka | the frozen tier | 1 | 122 MB | 49 MB | 1 | 135731 | +| kube-proxy | the frozen tier | 1 | 32 MB | 14 MB | 1 | 38945 | +| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4863 | +| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 15222 | +| opensearch | the frozen tier | 1 | 42 MB | 38 MB | 1 | 105815 | +| opentelemetry-collector | the frozen tier | 1 | 217 MB | 163 MB | 1 | 453312 | +| wait-for-kafka | the frozen tier | 1 | 14 MB | 1 MB | 1 | 1893 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1401 | +| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5928 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/run.json new file mode 100644 index 0000000..d48f601 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:14:57.541Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 67 MB within 2 hours, with 191 MB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/totals.csv new file mode 100644 index 0000000..5ba2144 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001452Z/totals.csv @@ -0,0 +1,31 @@ +container,total +accounting,18023::budget run +ad,7339::budget run +aws-eks-nodeagent,405::budget run +aws-node,1921::budget run +aws-vpc-cni-init,2446::budget run +cart,5928::budget run +coredns,8044::budget run +currency,577::budget run +email,1682::budget run +flagd,5430::budget run +flagd-ui,960::budget run +fraud-detection,1435::budget run +frontend,13578::budget run +frontend-proxy,62843::budget run +grafana,676544::budget run +grafana-sc-alerts,4203::budget run +grafana-sc-dashboard,5943::budget run +grafana-sc-datasources,4261::budget run +image-provider,9150::budget run +init-config,3531::budget run +jaeger,37137::budget run +kafka,135731::budget run +kube-proxy,38945::budget run +load-generator,4863::budget run +metrics-server,15222::budget run +opensearch,105815::budget run +opentelemetry-collector,453312::budget run +wait-for-kafka,1893::budget run +wait-for-valkey-cart,1401::budget run +*,5928::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/actions.csv new file mode 100644 index 0000000..69b7e87 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/actions.csv @@ -0,0 +1,31 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/caps.csv new file mode 100644 index 0000000..d0e571c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/caps.csv @@ -0,0 +1,31 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/commit.diff new file mode 100644 index 0000000..3097f6f --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/commit.diff @@ -0,0 +1,153 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 5ba2144..82a2527 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,31 +1,31 @@ + container,total +-accounting,18023::budget run +-ad,7339::budget run +-aws-eks-nodeagent,405::budget run +-aws-node,1921::budget run +-aws-vpc-cni-init,2446::budget run +-cart,5928::budget run +-coredns,8044::budget run +-currency,577::budget run +-email,1682::budget run +-flagd,5430::budget run +-flagd-ui,960::budget run +-fraud-detection,1435::budget run +-frontend,13578::budget run +-frontend-proxy,62843::budget run +-grafana,676544::budget run +-grafana-sc-alerts,4203::budget run +-grafana-sc-dashboard,5943::budget run +-grafana-sc-datasources,4261::budget run +-image-provider,9150::budget run +-init-config,3531::budget run +-jaeger,37137::budget run +-kafka,135731::budget run +-kube-proxy,38945::budget run +-load-generator,4863::budget run +-metrics-server,15222::budget run +-opensearch,105815::budget run +-opentelemetry-collector,453312::budget run +-wait-for-kafka,1893::budget run +-wait-for-valkey-cart,1401::budget run +-*,5928::budget run ++accounting,17541::budget run ++ad,7243::budget run ++aws-eks-nodeagent,390::budget run ++aws-node,1852::budget run ++aws-vpc-cni-init,2357::budget run ++cart,5824::budget run ++coredns,7919::budget run ++currency,556::budget run ++email,1622::budget run ++flagd,5234::budget run ++flagd-ui,926::budget run ++fraud-detection,1383::budget run ++frontend,13088::budget run ++frontend-proxy,60576::budget run ++grafana,652799::budget run ++grafana-sc-alerts,4051::budget run ++grafana-sc-dashboard,5729::budget run ++grafana-sc-datasources,4107::budget run ++image-provider,8890::budget run ++init-config,3403::budget run ++jaeger,35797::budget run ++kafka,132522::budget run ++kube-proxy,37754::budget run ++load-generator,4688::budget run ++metrics-server,14673::budget run ++opensearch,102386::budget run ++opentelemetry-collector,450790::budget run ++wait-for-kafka,1862::budget run ++wait-for-valkey-cart,1390::budget run ++*,5729::budget run +diff --git a/reports/latest.md b/reports/latest.md +index 34cef3c..e7c9003 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,46 +1,46 @@ +-# Hot-tier run: 2026-10-26T00:14:52.000Z ++# Hot-tier run: 2026-10-26T00:14:59.000Z + +-Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 67 MB within 2 hours, with 191 MB a day of counted lines moved to the frozen tier. ++Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 65 MB within 2 hours, with 204 MB a day of counted lines moved to the frozen tier. + + - Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.03 bytes of store kept, 1.03 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 777 MB a day; 281 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 586 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.01 bytes of store kept, 1.01 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 775 MB a day; 280 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 571 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 586 MB a day on hot, more than the 281 MB a day that fits, so hot settles above 41 MB at about 67 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 571 MB a day on hot, more than the 280 MB a day that fits, so hot settles above 41 MB at about 65 MB. Raise the size, move the noisiest services to offload, or lower the sample. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 8 MB | 6 MB | 1 | 18023 | +-| ad | the frozen tier | 1 | 10 MB | 3 MB | 1 | 7339 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 405 | +-| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1921 | +-| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2446 | +-| cart | the frozen tier | 1 | 18 MB | 2 MB | 1 | 5928 | +-| coredns | the frozen tier | 1 | 6 MB | 3 MB | 1 | 8044 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 577 | +-| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1682 | +-| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5430 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 960 | +-| fraud-detection | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1435 | +-| frontend | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13578 | +-| frontend-proxy | the frozen tier | 1 | 23 MB | 23 MB | 1 | 62843 | +-| grafana | the frozen tier | 1 | 245 MB | 244 MB | 1 | 676544 | +-| grafana-sc-alerts | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4203 | +-| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5943 | +-| grafana-sc-datasources | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4261 | +-| image-provider | the frozen tier | 1 | 4 MB | 3 MB | 1 | 9150 | +-| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3531 | +-| jaeger | the frozen tier | 1 | 13 MB | 13 MB | 1 | 37137 | +-| kafka | the frozen tier | 1 | 122 MB | 49 MB | 1 | 135731 | +-| kube-proxy | the frozen tier | 1 | 32 MB | 14 MB | 1 | 38945 | +-| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4863 | +-| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 15222 | +-| opensearch | the frozen tier | 1 | 42 MB | 38 MB | 1 | 105815 | +-| opentelemetry-collector | the frozen tier | 1 | 217 MB | 163 MB | 1 | 453312 | +-| wait-for-kafka | the frozen tier | 1 | 14 MB | 1 MB | 1 | 1893 | +-| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1401 | +-| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5928 | ++| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 17541 | ++| ad | the frozen tier | 1 | 10 MB | 3 MB | 1 | 7243 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 390 | ++| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1852 | ++| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2357 | ++| cart | the frozen tier | 1 | 18 MB | 2 MB | 1 | 5824 | ++| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7919 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 556 | ++| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1622 | ++| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5234 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 926 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1383 | ++| frontend | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13088 | ++| frontend-proxy | the frozen tier | 1 | 22 MB | 22 MB | 1 | 60576 | ++| grafana | the frozen tier | 1 | 236 MB | 235 MB | 1 | 652799 | ++| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4051 | ++| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5729 | ++| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4107 | ++| image-provider | the frozen tier | 1 | 4 MB | 3 MB | 1 | 8890 | ++| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3403 | ++| jaeger | the frozen tier | 1 | 13 MB | 13 MB | 1 | 35797 | ++| kafka | the frozen tier | 1 | 118 MB | 48 MB | 1 | 132522 | ++| kube-proxy | the frozen tier | 1 | 31 MB | 14 MB | 1 | 37754 | ++| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4688 | ++| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14673 | ++| opensearch | the frozen tier | 1 | 40 MB | 37 MB | 1 | 102386 | ++| opentelemetry-collector | the frozen tier | 1 | 235 MB | 162 MB | 1 | 450790 | ++| wait-for-kafka | the frozen tier | 1 | 14 MB | 1 MB | 1 | 1862 | ++| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1390 | ++| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5729 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/commit.json new file mode 100644 index 0000000..e782608 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/commit.json @@ -0,0 +1 @@ +{"before":"afa4fe44ce9771c9dc07d8b99ae01c3444ab6d24","after":"fc28e9434dc230ca97fc10af9173ef1c22d14180","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/config-generation.csv new file mode 100644 index 0000000..ddb50d6 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,88a2a5ef6083 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/latest.md new file mode 100644 index 0000000..e7c9003 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/latest.md @@ -0,0 +1,46 @@ +# Hot-tier run: 2026-10-26T00:14:59.000Z + +Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 65 MB within 2 hours, with 204 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.01 bytes of store kept, 1.01 as tier_down (measured on the indices on hot). +- The Receiver's statements send 775 MB a day; 280 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 571 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 571 MB a day on hot, more than the 280 MB a day that fits, so hot settles above 41 MB at about 65 MB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 17541 | +| ad | the frozen tier | 1 | 10 MB | 3 MB | 1 | 7243 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 390 | +| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1852 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2357 | +| cart | the frozen tier | 1 | 18 MB | 2 MB | 1 | 5824 | +| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7919 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 556 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1622 | +| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5234 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 926 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1383 | +| frontend | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13088 | +| frontend-proxy | the frozen tier | 1 | 22 MB | 22 MB | 1 | 60576 | +| grafana | the frozen tier | 1 | 236 MB | 235 MB | 1 | 652799 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4051 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5729 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4107 | +| image-provider | the frozen tier | 1 | 4 MB | 3 MB | 1 | 8890 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3403 | +| jaeger | the frozen tier | 1 | 13 MB | 13 MB | 1 | 35797 | +| kafka | the frozen tier | 1 | 118 MB | 48 MB | 1 | 132522 | +| kube-proxy | the frozen tier | 1 | 31 MB | 14 MB | 1 | 37754 | +| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4688 | +| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14673 | +| opensearch | the frozen tier | 1 | 40 MB | 37 MB | 1 | 102386 | +| opentelemetry-collector | the frozen tier | 1 | 235 MB | 162 MB | 1 | 450790 | +| wait-for-kafka | the frozen tier | 1 | 14 MB | 1 MB | 1 | 1862 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1390 | +| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5729 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/report.txt new file mode 100644 index 0000000..f1a4dc5 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/report.txt @@ -0,0 +1,55 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 65 MB within 2 hours, with 204 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.01 bytes of store kept, 1.01 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 775 MB a day; 280 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 571 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:14:59.000Z + +Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 65 MB within 2 hours, with 204 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.01 bytes of store kept, 1.01 as tier_down (measured on the indices on hot). +- The Receiver's statements send 775 MB a day; 280 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 571 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 571 MB a day on hot, more than the 280 MB a day that fits, so hot settles above 41 MB at about 65 MB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 17541 | +| ad | the frozen tier | 1 | 10 MB | 3 MB | 1 | 7243 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 390 | +| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1852 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2357 | +| cart | the frozen tier | 1 | 18 MB | 2 MB | 1 | 5824 | +| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7919 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 556 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1622 | +| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5234 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 926 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1383 | +| frontend | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13088 | +| frontend-proxy | the frozen tier | 1 | 22 MB | 22 MB | 1 | 60576 | +| grafana | the frozen tier | 1 | 236 MB | 235 MB | 1 | 652799 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4051 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5729 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4107 | +| image-provider | the frozen tier | 1 | 4 MB | 3 MB | 1 | 8890 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3403 | +| jaeger | the frozen tier | 1 | 13 MB | 13 MB | 1 | 35797 | +| kafka | the frozen tier | 1 | 118 MB | 48 MB | 1 | 132522 | +| kube-proxy | the frozen tier | 1 | 31 MB | 14 MB | 1 | 37754 | +| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4688 | +| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14673 | +| opensearch | the frozen tier | 1 | 40 MB | 37 MB | 1 | 102386 | +| opentelemetry-collector | the frozen tier | 1 | 235 MB | 162 MB | 1 | 450790 | +| wait-for-kafka | the frozen tier | 1 | 14 MB | 1 MB | 1 | 1862 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1390 | +| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5729 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/run.json new file mode 100644 index 0000000..f612da7 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:15:03.346Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 65 MB within 2 hours, with 204 MB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/totals.csv new file mode 100644 index 0000000..82a2527 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001458Z/totals.csv @@ -0,0 +1,31 @@ +container,total +accounting,17541::budget run +ad,7243::budget run +aws-eks-nodeagent,390::budget run +aws-node,1852::budget run +aws-vpc-cni-init,2357::budget run +cart,5824::budget run +coredns,7919::budget run +currency,556::budget run +email,1622::budget run +flagd,5234::budget run +flagd-ui,926::budget run +fraud-detection,1383::budget run +frontend,13088::budget run +frontend-proxy,60576::budget run +grafana,652799::budget run +grafana-sc-alerts,4051::budget run +grafana-sc-dashboard,5729::budget run +grafana-sc-datasources,4107::budget run +image-provider,8890::budget run +init-config,3403::budget run +jaeger,35797::budget run +kafka,132522::budget run +kube-proxy,37754::budget run +load-generator,4688::budget run +metrics-server,14673::budget run +opensearch,102386::budget run +opentelemetry-collector,450790::budget run +wait-for-kafka,1862::budget run +wait-for-valkey-cart,1390::budget run +*,5729::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/actions.csv new file mode 100644 index 0000000..69b7e87 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/actions.csv @@ -0,0 +1,31 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/caps.csv new file mode 100644 index 0000000..d0e571c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/caps.csv @@ -0,0 +1,31 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/commit.diff new file mode 100644 index 0000000..dac3b94 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/commit.diff @@ -0,0 +1,153 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 82a2527..4e39f04 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,31 +1,31 @@ + container,total +-accounting,17541::budget run +-ad,7243::budget run +-aws-eks-nodeagent,390::budget run +-aws-node,1852::budget run +-aws-vpc-cni-init,2357::budget run +-cart,5824::budget run +-coredns,7919::budget run +-currency,556::budget run +-email,1622::budget run +-flagd,5234::budget run +-flagd-ui,926::budget run +-fraud-detection,1383::budget run +-frontend,13088::budget run +-frontend-proxy,60576::budget run +-grafana,652799::budget run +-grafana-sc-alerts,4051::budget run +-grafana-sc-dashboard,5729::budget run +-grafana-sc-datasources,4107::budget run +-image-provider,8890::budget run +-init-config,3403::budget run +-jaeger,35797::budget run +-kafka,132522::budget run +-kube-proxy,37754::budget run +-load-generator,4688::budget run +-metrics-server,14673::budget run +-opensearch,102386::budget run +-opentelemetry-collector,450790::budget run +-wait-for-kafka,1862::budget run +-wait-for-valkey-cart,1390::budget run +-*,5729::budget run ++accounting,16910::budget run ++ad,7055::budget run ++aws-eks-nodeagent,371::budget run ++aws-node,1761::budget run ++aws-vpc-cni-init,2242::budget run ++cart,5688::budget run ++coredns,7756::budget run ++currency,529::budget run ++email,1542::budget run ++flagd,4978::budget run ++flagd-ui,880::budget run ++fraud-detection,1315::budget run ++frontend,12447::budget run ++frontend-proxy,57608::budget run ++grafana,621133::budget run ++grafana-sc-alerts,3853::budget run ++grafana-sc-dashboard,5448::budget run ++grafana-sc-datasources,3906::budget run ++image-provider,8550::budget run ++init-config,3237::budget run ++jaeger,34043::budget run ++kafka,128256::budget run ++kube-proxy,36137::budget run ++load-generator,4458::budget run ++metrics-server,13954::budget run ++opensearch,97781::budget run ++opentelemetry-collector,449225::budget run ++wait-for-kafka,1822::budget run ++wait-for-valkey-cart,1376::budget run ++*,5448::budget run +diff --git a/reports/latest.md b/reports/latest.md +index e7c9003..747fe0a 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,46 +1,46 @@ +-# Hot-tier run: 2026-10-26T00:14:59.000Z ++# Hot-tier run: 2026-10-26T00:15:09.000Z + +-Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 65 MB within 2 hours, with 204 MB a day of counted lines moved to the frozen tier. ++Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 68 MB within 2 hours, with 217 MB a day of counted lines moved to the frozen tier. + + - Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.01 bytes of store kept, 1.01 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 775 MB a day; 280 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 571 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.06 bytes of store kept, 1.06 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 769 MB a day; 244 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 552 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 571 MB a day on hot, more than the 280 MB a day that fits, so hot settles above 41 MB at about 65 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 552 MB a day on hot, more than the 244 MB a day that fits, so hot settles above 41 MB at about 68 MB. Raise the size, move the noisiest services to offload, or lower the sample. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 17541 | +-| ad | the frozen tier | 1 | 10 MB | 3 MB | 1 | 7243 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 390 | +-| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1852 | +-| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2357 | +-| cart | the frozen tier | 1 | 18 MB | 2 MB | 1 | 5824 | +-| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7919 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 556 | +-| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1622 | +-| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5234 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 926 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1383 | +-| frontend | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13088 | +-| frontend-proxy | the frozen tier | 1 | 22 MB | 22 MB | 1 | 60576 | +-| grafana | the frozen tier | 1 | 236 MB | 235 MB | 1 | 652799 | +-| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4051 | +-| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5729 | +-| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4107 | +-| image-provider | the frozen tier | 1 | 4 MB | 3 MB | 1 | 8890 | +-| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3403 | +-| jaeger | the frozen tier | 1 | 13 MB | 13 MB | 1 | 35797 | +-| kafka | the frozen tier | 1 | 118 MB | 48 MB | 1 | 132522 | +-| kube-proxy | the frozen tier | 1 | 31 MB | 14 MB | 1 | 37754 | +-| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4688 | +-| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14673 | +-| opensearch | the frozen tier | 1 | 40 MB | 37 MB | 1 | 102386 | +-| opentelemetry-collector | the frozen tier | 1 | 235 MB | 162 MB | 1 | 450790 | +-| wait-for-kafka | the frozen tier | 1 | 14 MB | 1 MB | 1 | 1862 | +-| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1390 | +-| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5729 | ++| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 16910 | ++| ad | the frozen tier | 1 | 9 MB | 3 MB | 1 | 7055 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 371 | ++| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1761 | ++| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2242 | ++| cart | the frozen tier | 1 | 17 MB | 2 MB | 1 | 5688 | ++| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7756 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 529 | ++| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1542 | ++| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4978 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 880 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1315 | ++| frontend | the frozen tier | 1 | 4 MB | 4 MB | 1 | 12447 | ++| frontend-proxy | the frozen tier | 1 | 21 MB | 21 MB | 1 | 57608 | ++| grafana | the frozen tier | 1 | 224 MB | 224 MB | 1 | 621133 | ++| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3853 | ++| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5448 | ++| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3906 | ++| image-provider | the frozen tier | 1 | 4 MB | 3 MB | 1 | 8550 | ++| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3237 | ++| jaeger | the frozen tier | 1 | 12 MB | 12 MB | 1 | 34043 | ++| kafka | the frozen tier | 1 | 112 MB | 46 MB | 1 | 128256 | ++| kube-proxy | the frozen tier | 1 | 29 MB | 13 MB | 1 | 36137 | ++| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4458 | ++| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13954 | ++| opensearch | the frozen tier | 1 | 38 MB | 35 MB | 1 | 97781 | ++| opentelemetry-collector | the frozen tier | 1 | 256 MB | 162 MB | 1 | 449225 | ++| wait-for-kafka | the frozen tier | 1 | 13 MB | 1 MB | 1 | 1822 | ++| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1376 | ++| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5448 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/commit.json new file mode 100644 index 0000000..de470b9 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/commit.json @@ -0,0 +1 @@ +{"before":"fc28e9434dc230ca97fc10af9173ef1c22d14180","after":"09309365d88d13c5d5b6f39a888e2cbd434ba3cb","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/config-generation.csv new file mode 100644 index 0000000..ddb50d6 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,88a2a5ef6083 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/latest.md new file mode 100644 index 0000000..747fe0a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/latest.md @@ -0,0 +1,46 @@ +# Hot-tier run: 2026-10-26T00:15:09.000Z + +Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 68 MB within 2 hours, with 217 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.06 bytes of store kept, 1.06 as tier_down (measured on the indices on hot). +- The Receiver's statements send 769 MB a day; 244 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 552 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 552 MB a day on hot, more than the 244 MB a day that fits, so hot settles above 41 MB at about 68 MB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 16910 | +| ad | the frozen tier | 1 | 9 MB | 3 MB | 1 | 7055 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 371 | +| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1761 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2242 | +| cart | the frozen tier | 1 | 17 MB | 2 MB | 1 | 5688 | +| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7756 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 529 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1542 | +| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4978 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 880 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1315 | +| frontend | the frozen tier | 1 | 4 MB | 4 MB | 1 | 12447 | +| frontend-proxy | the frozen tier | 1 | 21 MB | 21 MB | 1 | 57608 | +| grafana | the frozen tier | 1 | 224 MB | 224 MB | 1 | 621133 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3853 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5448 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3906 | +| image-provider | the frozen tier | 1 | 4 MB | 3 MB | 1 | 8550 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3237 | +| jaeger | the frozen tier | 1 | 12 MB | 12 MB | 1 | 34043 | +| kafka | the frozen tier | 1 | 112 MB | 46 MB | 1 | 128256 | +| kube-proxy | the frozen tier | 1 | 29 MB | 13 MB | 1 | 36137 | +| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4458 | +| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13954 | +| opensearch | the frozen tier | 1 | 38 MB | 35 MB | 1 | 97781 | +| opentelemetry-collector | the frozen tier | 1 | 256 MB | 162 MB | 1 | 449225 | +| wait-for-kafka | the frozen tier | 1 | 13 MB | 1 MB | 1 | 1822 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1376 | +| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5448 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/report.txt new file mode 100644 index 0000000..ced0f3e --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/report.txt @@ -0,0 +1,55 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 68 MB within 2 hours, with 217 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.06 bytes of store kept, 1.06 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 769 MB a day; 244 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 552 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:15:09.000Z + +Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 68 MB within 2 hours, with 217 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.06 bytes of store kept, 1.06 as tier_down (measured on the indices on hot). +- The Receiver's statements send 769 MB a day; 244 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 552 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 552 MB a day on hot, more than the 244 MB a day that fits, so hot settles above 41 MB at about 68 MB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 16910 | +| ad | the frozen tier | 1 | 9 MB | 3 MB | 1 | 7055 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 371 | +| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1761 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2242 | +| cart | the frozen tier | 1 | 17 MB | 2 MB | 1 | 5688 | +| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7756 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 529 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1542 | +| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4978 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 880 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1315 | +| frontend | the frozen tier | 1 | 4 MB | 4 MB | 1 | 12447 | +| frontend-proxy | the frozen tier | 1 | 21 MB | 21 MB | 1 | 57608 | +| grafana | the frozen tier | 1 | 224 MB | 224 MB | 1 | 621133 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3853 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5448 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3906 | +| image-provider | the frozen tier | 1 | 4 MB | 3 MB | 1 | 8550 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3237 | +| jaeger | the frozen tier | 1 | 12 MB | 12 MB | 1 | 34043 | +| kafka | the frozen tier | 1 | 112 MB | 46 MB | 1 | 128256 | +| kube-proxy | the frozen tier | 1 | 29 MB | 13 MB | 1 | 36137 | +| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4458 | +| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13954 | +| opensearch | the frozen tier | 1 | 38 MB | 35 MB | 1 | 97781 | +| opentelemetry-collector | the frozen tier | 1 | 256 MB | 162 MB | 1 | 449225 | +| wait-for-kafka | the frozen tier | 1 | 13 MB | 1 MB | 1 | 1822 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1376 | +| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5448 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/run.json new file mode 100644 index 0000000..88d528c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:15:14.574Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 68 MB within 2 hours, with 217 MB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/totals.csv new file mode 100644 index 0000000..4e39f04 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001509Z/totals.csv @@ -0,0 +1,31 @@ +container,total +accounting,16910::budget run +ad,7055::budget run +aws-eks-nodeagent,371::budget run +aws-node,1761::budget run +aws-vpc-cni-init,2242::budget run +cart,5688::budget run +coredns,7756::budget run +currency,529::budget run +email,1542::budget run +flagd,4978::budget run +flagd-ui,880::budget run +fraud-detection,1315::budget run +frontend,12447::budget run +frontend-proxy,57608::budget run +grafana,621133::budget run +grafana-sc-alerts,3853::budget run +grafana-sc-dashboard,5448::budget run +grafana-sc-datasources,3906::budget run +image-provider,8550::budget run +init-config,3237::budget run +jaeger,34043::budget run +kafka,128256::budget run +kube-proxy,36137::budget run +load-generator,4458::budget run +metrics-server,13954::budget run +opensearch,97781::budget run +opentelemetry-collector,449225::budget run +wait-for-kafka,1822::budget run +wait-for-valkey-cart,1376::budget run +*,5448::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/actions.csv new file mode 100644 index 0000000..69b7e87 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/actions.csv @@ -0,0 +1,31 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/caps.csv new file mode 100644 index 0000000..d0e571c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/caps.csv @@ -0,0 +1,31 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/commit.diff new file mode 100644 index 0000000..bb7e87f --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/commit.diff @@ -0,0 +1,153 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 4e39f04..c119b0a 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,31 +1,31 @@ + container,total +-accounting,16910::budget run +-ad,7055::budget run +-aws-eks-nodeagent,371::budget run +-aws-node,1761::budget run +-aws-vpc-cni-init,2242::budget run +-cart,5688::budget run +-coredns,7756::budget run +-currency,529::budget run +-email,1542::budget run +-flagd,4978::budget run +-flagd-ui,880::budget run +-fraud-detection,1315::budget run +-frontend,12447::budget run +-frontend-proxy,57608::budget run +-grafana,621133::budget run +-grafana-sc-alerts,3853::budget run +-grafana-sc-dashboard,5448::budget run +-grafana-sc-datasources,3906::budget run +-image-provider,8550::budget run +-init-config,3237::budget run +-jaeger,34043::budget run +-kafka,128256::budget run +-kube-proxy,36137::budget run +-load-generator,4458::budget run +-metrics-server,13954::budget run +-opensearch,97781::budget run +-opentelemetry-collector,449225::budget run +-wait-for-kafka,1822::budget run +-wait-for-valkey-cart,1376::budget run +-*,5448::budget run ++accounting,16231::budget run ++ad,6852::budget run ++aws-eks-nodeagent,350::budget run ++aws-node,1663::budget run ++aws-vpc-cni-init,2117::budget run ++cart,5541::budget run ++coredns,7580::budget run ++currency,499::budget run ++email,1457::budget run ++flagd,4702::budget run ++flagd-ui,831::budget run ++fraud-detection,1242::budget run ++frontend,11756::budget run ++frontend-proxy,54410::budget run ++grafana,587003::budget run ++grafana-sc-alerts,3639::budget run ++grafana-sc-dashboard,5146::budget run ++grafana-sc-datasources,3689::budget run ++image-provider,8183::budget run ++init-config,3057::budget run ++jaeger,32153::budget run ++kafka,123658::budget run ++kube-proxy,34394::budget run ++load-generator,4211::budget run ++metrics-server,13179::budget run ++opensearch,92817::budget run ++opentelemetry-collector,456456::budget run ++wait-for-kafka,1778::budget run ++wait-for-valkey-cart,1360::budget run ++*,5146::budget run +diff --git a/reports/latest.md b/reports/latest.md +index 747fe0a..7527e04 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,46 +1,46 @@ +-# Hot-tier run: 2026-10-26T00:15:09.000Z ++# Hot-tier run: 2026-10-26T00:15:21.000Z + +-Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 68 MB within 2 hours, with 217 MB a day of counted lines moved to the frozen tier. ++Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 70 MB within 2 hours, with 312 MB a day of counted lines moved to the frozen tier. + + - Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.06 bytes of store kept, 1.06 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 769 MB a day; 244 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 552 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.13 bytes of store kept, 0.75 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 847 MB a day; 223 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 535 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 552 MB a day on hot, more than the 244 MB a day that fits, so hot settles above 41 MB at about 68 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 535 MB a day on hot, more than the 223 MB a day that fits, so hot settles above 41 MB at about 70 MB. Raise the size, move the noisiest services to offload, or lower the sample. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 16910 | +-| ad | the frozen tier | 1 | 9 MB | 3 MB | 1 | 7055 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 371 | +-| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1761 | +-| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2242 | +-| cart | the frozen tier | 1 | 17 MB | 2 MB | 1 | 5688 | +-| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7756 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 529 | +-| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1542 | +-| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4978 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 880 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1315 | +-| frontend | the frozen tier | 1 | 4 MB | 4 MB | 1 | 12447 | +-| frontend-proxy | the frozen tier | 1 | 21 MB | 21 MB | 1 | 57608 | +-| grafana | the frozen tier | 1 | 224 MB | 224 MB | 1 | 621133 | +-| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3853 | +-| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5448 | +-| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3906 | +-| image-provider | the frozen tier | 1 | 4 MB | 3 MB | 1 | 8550 | +-| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3237 | +-| jaeger | the frozen tier | 1 | 12 MB | 12 MB | 1 | 34043 | +-| kafka | the frozen tier | 1 | 112 MB | 46 MB | 1 | 128256 | +-| kube-proxy | the frozen tier | 1 | 29 MB | 13 MB | 1 | 36137 | +-| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4458 | +-| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13954 | +-| opensearch | the frozen tier | 1 | 38 MB | 35 MB | 1 | 97781 | +-| opentelemetry-collector | the frozen tier | 1 | 256 MB | 162 MB | 1 | 449225 | +-| wait-for-kafka | the frozen tier | 1 | 13 MB | 1 MB | 1 | 1822 | +-| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1376 | +-| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5448 | ++| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 16231 | ++| ad | the frozen tier | 1 | 9 MB | 2 MB | 1 | 6852 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 350 | ++| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1663 | ++| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2117 | ++| cart | the frozen tier | 1 | 16 MB | 2 MB | 1 | 5541 | ++| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7580 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 499 | ++| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1457 | ++| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4702 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 831 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1242 | ++| frontend | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11756 | ++| frontend-proxy | the frozen tier | 1 | 20 MB | 20 MB | 1 | 54410 | ++| grafana | the frozen tier | 1 | 212 MB | 211 MB | 1 | 587003 | ++| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3639 | ++| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5146 | ++| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3689 | ++| image-provider | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8183 | ++| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3057 | ++| jaeger | the frozen tier | 1 | 12 MB | 12 MB | 1 | 32153 | ++| kafka | the frozen tier | 1 | 106 MB | 45 MB | 1 | 123658 | ++| kube-proxy | the frozen tier | 1 | 27 MB | 12 MB | 1 | 34394 | ++| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4211 | ++| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13179 | ++| opensearch | the frozen tier | 1 | 36 MB | 33 MB | 1 | 92817 | ++| opentelemetry-collector | the frozen tier | 1 | 362 MB | 164 MB | 1 | 456456 | ++| wait-for-kafka | the frozen tier | 1 | 12 MB | 1 MB | 1 | 1778 | ++| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1360 | ++| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5146 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/commit.json new file mode 100644 index 0000000..b939c2b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/commit.json @@ -0,0 +1 @@ +{"before":"09309365d88d13c5d5b6f39a888e2cbd434ba3cb","after":"c72d0a2be4f4c292919e02a324ca752708d2274b","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/config-generation.csv new file mode 100644 index 0000000..ddb50d6 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,88a2a5ef6083 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/latest.md new file mode 100644 index 0000000..7527e04 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/latest.md @@ -0,0 +1,46 @@ +# Hot-tier run: 2026-10-26T00:15:21.000Z + +Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 70 MB within 2 hours, with 312 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.13 bytes of store kept, 0.75 as tier_down (measured on the indices on hot). +- The Receiver's statements send 847 MB a day; 223 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 535 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 535 MB a day on hot, more than the 223 MB a day that fits, so hot settles above 41 MB at about 70 MB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 16231 | +| ad | the frozen tier | 1 | 9 MB | 2 MB | 1 | 6852 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 350 | +| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1663 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2117 | +| cart | the frozen tier | 1 | 16 MB | 2 MB | 1 | 5541 | +| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7580 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 499 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1457 | +| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4702 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 831 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1242 | +| frontend | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11756 | +| frontend-proxy | the frozen tier | 1 | 20 MB | 20 MB | 1 | 54410 | +| grafana | the frozen tier | 1 | 212 MB | 211 MB | 1 | 587003 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3639 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5146 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3689 | +| image-provider | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8183 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3057 | +| jaeger | the frozen tier | 1 | 12 MB | 12 MB | 1 | 32153 | +| kafka | the frozen tier | 1 | 106 MB | 45 MB | 1 | 123658 | +| kube-proxy | the frozen tier | 1 | 27 MB | 12 MB | 1 | 34394 | +| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4211 | +| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13179 | +| opensearch | the frozen tier | 1 | 36 MB | 33 MB | 1 | 92817 | +| opentelemetry-collector | the frozen tier | 1 | 362 MB | 164 MB | 1 | 456456 | +| wait-for-kafka | the frozen tier | 1 | 12 MB | 1 MB | 1 | 1778 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1360 | +| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5146 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/report.txt new file mode 100644 index 0000000..1c09b21 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/report.txt @@ -0,0 +1,55 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 70 MB within 2 hours, with 312 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.13 bytes of store kept, 0.75 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 847 MB a day; 223 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 535 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:15:21.000Z + +Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 70 MB within 2 hours, with 312 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.13 bytes of store kept, 0.75 as tier_down (measured on the indices on hot). +- The Receiver's statements send 847 MB a day; 223 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 535 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 535 MB a day on hot, more than the 223 MB a day that fits, so hot settles above 41 MB at about 70 MB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 16231 | +| ad | the frozen tier | 1 | 9 MB | 2 MB | 1 | 6852 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 350 | +| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1663 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2117 | +| cart | the frozen tier | 1 | 16 MB | 2 MB | 1 | 5541 | +| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7580 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 499 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1457 | +| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4702 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 831 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1242 | +| frontend | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11756 | +| frontend-proxy | the frozen tier | 1 | 20 MB | 20 MB | 1 | 54410 | +| grafana | the frozen tier | 1 | 212 MB | 211 MB | 1 | 587003 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3639 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5146 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3689 | +| image-provider | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8183 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3057 | +| jaeger | the frozen tier | 1 | 12 MB | 12 MB | 1 | 32153 | +| kafka | the frozen tier | 1 | 106 MB | 45 MB | 1 | 123658 | +| kube-proxy | the frozen tier | 1 | 27 MB | 12 MB | 1 | 34394 | +| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4211 | +| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13179 | +| opensearch | the frozen tier | 1 | 36 MB | 33 MB | 1 | 92817 | +| opentelemetry-collector | the frozen tier | 1 | 362 MB | 164 MB | 1 | 456456 | +| wait-for-kafka | the frozen tier | 1 | 12 MB | 1 MB | 1 | 1778 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1360 | +| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5146 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/run.json new file mode 100644 index 0000000..6c8613f --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:15:26.533Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 70 MB within 2 hours, with 312 MB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/totals.csv new file mode 100644 index 0000000..c119b0a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001520Z/totals.csv @@ -0,0 +1,31 @@ +container,total +accounting,16231::budget run +ad,6852::budget run +aws-eks-nodeagent,350::budget run +aws-node,1663::budget run +aws-vpc-cni-init,2117::budget run +cart,5541::budget run +coredns,7580::budget run +currency,499::budget run +email,1457::budget run +flagd,4702::budget run +flagd-ui,831::budget run +fraud-detection,1242::budget run +frontend,11756::budget run +frontend-proxy,54410::budget run +grafana,587003::budget run +grafana-sc-alerts,3639::budget run +grafana-sc-dashboard,5146::budget run +grafana-sc-datasources,3689::budget run +image-provider,8183::budget run +init-config,3057::budget run +jaeger,32153::budget run +kafka,123658::budget run +kube-proxy,34394::budget run +load-generator,4211::budget run +metrics-server,13179::budget run +opensearch,92817::budget run +opentelemetry-collector,456456::budget run +wait-for-kafka,1778::budget run +wait-for-valkey-cart,1360::budget run +*,5146::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/commit.diff new file mode 100644 index 0000000..d44a045 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/commit.diff @@ -0,0 +1,226 @@ +diff --git a/log10x/policy/actions.csv b/log10x/policy/actions.csv +index 69b7e87..ca15017 100644 +--- a/log10x/policy/actions.csv ++++ b/log10x/policy/actions.csv +@@ -10,6 +10,7 @@ currency,tier_down::budget run + email,tier_down::budget run + flagd,tier_down::budget run + flagd-ui,tier_down::budget run ++fluentd-10x,tier_down::budget run + fraud-detection,tier_down::budget run + frontend,tier_down::budget run + frontend-proxy,tier_down::budget run +@@ -26,6 +27,12 @@ load-generator,tier_down::budget run + metrics-server,tier_down::budget run + opensearch,tier_down::budget run + opentelemetry-collector,tier_down::budget run ++payment,tier_down::budget run ++postgresql,tier_down::budget run ++prometheus-server,tier_down::budget run ++quote,tier_down::budget run ++recommendation,tier_down::budget run ++valkey-cart,tier_down::budget run + wait-for-kafka,tier_down::budget run + wait-for-valkey-cart,tier_down::budget run + *,tier_down::budget run +diff --git a/log10x/policy/caps.csv b/log10x/policy/caps.csv +index d0e571c..f92f886 100644 +--- a/log10x/policy/caps.csv ++++ b/log10x/policy/caps.csv +@@ -10,6 +10,7 @@ currency,1::budget run + email,1::budget run + flagd,1::budget run + flagd-ui,1::budget run ++fluentd-10x,1::budget run + fraud-detection,1::budget run + frontend,1::budget run + frontend-proxy,1::budget run +@@ -26,6 +27,12 @@ load-generator,1::budget run + metrics-server,1::budget run + opensearch,1::budget run + opentelemetry-collector,1::budget run ++payment,1::budget run ++postgresql,1::budget run ++prometheus-server,1::budget run ++quote,1::budget run ++recommendation,1::budget run ++valkey-cart,1::budget run + wait-for-kafka,1::budget run + wait-for-valkey-cart,1::budget run + *,1::budget run +diff --git a/log10x/policy/config-generation.csv b/log10x/policy/config-generation.csv +index ddb50d6..89e536b 100644 +--- a/log10x/policy/config-generation.csv ++++ b/log10x/policy/config-generation.csv +@@ -1,2 +1,2 @@ + key,value +-generation,88a2a5ef6083 ++generation,4236ef75e377 +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index c119b0a..c75da71 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,31 +1,38 @@ + container,total +-accounting,16231::budget run +-ad,6852::budget run +-aws-eks-nodeagent,350::budget run +-aws-node,1663::budget run +-aws-vpc-cni-init,2117::budget run +-cart,5541::budget run +-coredns,7580::budget run +-currency,499::budget run +-email,1457::budget run +-flagd,4702::budget run +-flagd-ui,831::budget run +-fraud-detection,1242::budget run +-frontend,11756::budget run +-frontend-proxy,54410::budget run +-grafana,587003::budget run +-grafana-sc-alerts,3639::budget run +-grafana-sc-dashboard,5146::budget run +-grafana-sc-datasources,3689::budget run +-image-provider,8183::budget run +-init-config,3057::budget run +-jaeger,32153::budget run +-kafka,123658::budget run +-kube-proxy,34394::budget run +-load-generator,4211::budget run +-metrics-server,13179::budget run +-opensearch,92817::budget run +-opentelemetry-collector,456456::budget run +-wait-for-kafka,1778::budget run +-wait-for-valkey-cart,1360::budget run +-*,5146::budget run ++accounting,14129::budget run ++ad,6743::budget run ++aws-eks-nodeagent,287::budget run ++aws-node,1361::budget run ++aws-vpc-cni-init,1732::budget run ++cart,5115::budget run ++coredns,7035::budget run ++currency,408::budget run ++email,1192::budget run ++flagd,3847::budget run ++flagd-ui,680::budget run ++fluentd-10x,23495::budget run ++fraud-detection,1016::budget run ++frontend,23318::budget run ++frontend-proxy,44521::budget run ++grafana,482478::budget run ++grafana-sc-alerts,2977::budget run ++grafana-sc-dashboard,4210::budget run ++grafana-sc-datasources,3018::budget run ++image-provider,6959::budget run ++init-config,2501::budget run ++jaeger,26309::budget run ++kafka,109746::budget run ++kube-proxy,29007::budget run ++load-generator,3445::budget run ++metrics-server,10784::budget run ++opensearch,78270::budget run ++opentelemetry-collector,455772::budget run ++payment,1171::budget run ++postgresql,10651::budget run ++prometheus-server,8542::budget run ++quote,189::budget run ++recommendation,3226::budget run ++valkey-cart,4089::budget run ++wait-for-kafka,1644::budget run ++wait-for-valkey-cart,1313::budget run ++*,4150::budget run +diff --git a/reports/latest.md b/reports/latest.md +index 7527e04..f123a4f 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,46 +1,53 @@ +-# Hot-tier run: 2026-10-26T00:15:21.000Z ++# Hot-tier run: 2026-10-26T00:16:09.000Z + +-Hot tier: 2 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 70 MB within 2 hours, with 312 MB a day of counted lines moved to the frozen tier. ++Hot tier: 3 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 74 MB within 2 hours, with 502 MB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 1. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 1 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.13 bytes of store kept, 0.75 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 847 MB a day; 223 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 535 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.20 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 999 MB a day; 164 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 497 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 535 MB a day on hot, more than the 223 MB a day that fits, so hot settles above 41 MB at about 70 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 497 MB a day on hot, more than the 164 MB a day that fits, so hot settles above 41 MB at about 74 MB. Raise the size, move the noisiest services to offload, or lower the sample. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 16231 | +-| ad | the frozen tier | 1 | 9 MB | 2 MB | 1 | 6852 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 350 | +-| aws-node | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1663 | +-| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2117 | +-| cart | the frozen tier | 1 | 16 MB | 2 MB | 1 | 5541 | +-| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7580 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 499 | +-| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1457 | +-| flagd | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4702 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 831 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1242 | +-| frontend | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11756 | +-| frontend-proxy | the frozen tier | 1 | 20 MB | 20 MB | 1 | 54410 | +-| grafana | the frozen tier | 1 | 212 MB | 211 MB | 1 | 587003 | +-| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3639 | +-| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5146 | +-| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3689 | +-| image-provider | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8183 | +-| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3057 | +-| jaeger | the frozen tier | 1 | 12 MB | 12 MB | 1 | 32153 | +-| kafka | the frozen tier | 1 | 106 MB | 45 MB | 1 | 123658 | +-| kube-proxy | the frozen tier | 1 | 27 MB | 12 MB | 1 | 34394 | +-| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4211 | +-| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13179 | +-| opensearch | the frozen tier | 1 | 36 MB | 33 MB | 1 | 92817 | +-| opentelemetry-collector | the frozen tier | 1 | 362 MB | 164 MB | 1 | 456456 | +-| wait-for-kafka | the frozen tier | 1 | 12 MB | 1 MB | 1 | 1778 | +-| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1360 | +-| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5146 | ++| accounting | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14129 | ++| ad | the frozen tier | 1 | 16 MB | 2 MB | 1 | 6743 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 287 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1361 | ++| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1732 | ++| cart | the frozen tier | 1 | 37 MB | 2 MB | 1 | 5115 | ++| coredns | the frozen tier | 1 | 4 MB | 3 MB | 1 | 7035 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 408 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1192 | ++| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3847 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 680 | ++| fluentd-10x | the frozen tier | 1 | 15 MB | 8 MB | 1 | 23495 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1016 | ++| frontend | the frozen tier | 1 | 8 MB | 8 MB | 1 | 23318 | ++| frontend-proxy | the frozen tier | 1 | 16 MB | 16 MB | 1 | 44521 | ++| grafana | the frozen tier | 1 | 174 MB | 174 MB | 1 | 482478 | ++| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2977 | ++| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4210 | ++| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3018 | ++| image-provider | the frozen tier | 1 | 3 MB | 3 MB | 1 | 6959 | ++| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2501 | ++| jaeger | the frozen tier | 1 | 9 MB | 9 MB | 1 | 26309 | ++| kafka | the frozen tier | 1 | 128 MB | 40 MB | 1 | 109746 | ++| kube-proxy | the frozen tier | 1 | 22 MB | 10 MB | 1 | 29007 | ++| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3445 | ++| metrics-server | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10784 | ++| opensearch | the frozen tier | 1 | 30 MB | 28 MB | 1 | 78270 | ++| opentelemetry-collector | the frozen tier | 1 | 486 MB | 164 MB | 1 | 455772 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1171 | ++| postgresql | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10651 | ++| prometheus-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8542 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 189 | ++| recommendation | the frozen tier | 1 | 13 MB | 1 MB | 1 | 3226 | ++| valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4089 | ++| wait-for-kafka | the frozen tier | 1 | 10 MB | 1 MB | 1 | 1644 | ++| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1313 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 4150 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/commit.json new file mode 100644 index 0000000..e72cdeb --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/commit.json @@ -0,0 +1 @@ +{"before":"c72d0a2be4f4c292919e02a324ca752708d2274b","after":"79cc5655b373e5323862134669c10bd6fc64c335","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/latest.md new file mode 100644 index 0000000..f123a4f --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/latest.md @@ -0,0 +1,53 @@ +# Hot-tier run: 2026-10-26T00:16:09.000Z + +Hot tier: 3 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 74 MB within 2 hours, with 502 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 1 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.20 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +- The Receiver's statements send 999 MB a day; 164 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 497 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 497 MB a day on hot, more than the 164 MB a day that fits, so hot settles above 41 MB at about 74 MB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14129 | +| ad | the frozen tier | 1 | 16 MB | 2 MB | 1 | 6743 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 287 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1361 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1732 | +| cart | the frozen tier | 1 | 37 MB | 2 MB | 1 | 5115 | +| coredns | the frozen tier | 1 | 4 MB | 3 MB | 1 | 7035 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 408 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1192 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3847 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 680 | +| fluentd-10x | the frozen tier | 1 | 15 MB | 8 MB | 1 | 23495 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1016 | +| frontend | the frozen tier | 1 | 8 MB | 8 MB | 1 | 23318 | +| frontend-proxy | the frozen tier | 1 | 16 MB | 16 MB | 1 | 44521 | +| grafana | the frozen tier | 1 | 174 MB | 174 MB | 1 | 482478 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2977 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4210 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3018 | +| image-provider | the frozen tier | 1 | 3 MB | 3 MB | 1 | 6959 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2501 | +| jaeger | the frozen tier | 1 | 9 MB | 9 MB | 1 | 26309 | +| kafka | the frozen tier | 1 | 128 MB | 40 MB | 1 | 109746 | +| kube-proxy | the frozen tier | 1 | 22 MB | 10 MB | 1 | 29007 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3445 | +| metrics-server | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10784 | +| opensearch | the frozen tier | 1 | 30 MB | 28 MB | 1 | 78270 | +| opentelemetry-collector | the frozen tier | 1 | 486 MB | 164 MB | 1 | 455772 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1171 | +| postgresql | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10651 | +| prometheus-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8542 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 189 | +| recommendation | the frozen tier | 1 | 13 MB | 1 MB | 1 | 3226 | +| valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4089 | +| wait-for-kafka | the frozen tier | 1 | 10 MB | 1 MB | 1 | 1644 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1313 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 4150 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/report.txt new file mode 100644 index 0000000..d2938f6 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/report.txt @@ -0,0 +1,62 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 3 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 74 MB within 2 hours, with 502 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 1 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.20 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 999 MB a day; 164 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 497 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:16:09.000Z + +Hot tier: 3 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 74 MB within 2 hours, with 502 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 1 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.20 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +- The Receiver's statements send 999 MB a day; 164 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 497 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 497 MB a day on hot, more than the 164 MB a day that fits, so hot settles above 41 MB at about 74 MB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14129 | +| ad | the frozen tier | 1 | 16 MB | 2 MB | 1 | 6743 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 287 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1361 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1732 | +| cart | the frozen tier | 1 | 37 MB | 2 MB | 1 | 5115 | +| coredns | the frozen tier | 1 | 4 MB | 3 MB | 1 | 7035 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 408 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1192 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3847 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 680 | +| fluentd-10x | the frozen tier | 1 | 15 MB | 8 MB | 1 | 23495 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1016 | +| frontend | the frozen tier | 1 | 8 MB | 8 MB | 1 | 23318 | +| frontend-proxy | the frozen tier | 1 | 16 MB | 16 MB | 1 | 44521 | +| grafana | the frozen tier | 1 | 174 MB | 174 MB | 1 | 482478 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2977 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4210 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3018 | +| image-provider | the frozen tier | 1 | 3 MB | 3 MB | 1 | 6959 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2501 | +| jaeger | the frozen tier | 1 | 9 MB | 9 MB | 1 | 26309 | +| kafka | the frozen tier | 1 | 128 MB | 40 MB | 1 | 109746 | +| kube-proxy | the frozen tier | 1 | 22 MB | 10 MB | 1 | 29007 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3445 | +| metrics-server | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10784 | +| opensearch | the frozen tier | 1 | 30 MB | 28 MB | 1 | 78270 | +| opentelemetry-collector | the frozen tier | 1 | 486 MB | 164 MB | 1 | 455772 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1171 | +| postgresql | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10651 | +| prometheus-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8542 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 189 | +| recommendation | the frozen tier | 1 | 13 MB | 1 MB | 1 | 3226 | +| valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4089 | +| wait-for-kafka | the frozen tier | 1 | 10 MB | 1 MB | 1 | 1644 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1313 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 4150 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/run.json new file mode 100644 index 0000000..30067a3 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:16:17.367Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 3 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 74 MB within 2 hours, with 502 MB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/totals.csv new file mode 100644 index 0000000..c75da71 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001609Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,14129::budget run +ad,6743::budget run +aws-eks-nodeagent,287::budget run +aws-node,1361::budget run +aws-vpc-cni-init,1732::budget run +cart,5115::budget run +coredns,7035::budget run +currency,408::budget run +email,1192::budget run +flagd,3847::budget run +flagd-ui,680::budget run +fluentd-10x,23495::budget run +fraud-detection,1016::budget run +frontend,23318::budget run +frontend-proxy,44521::budget run +grafana,482478::budget run +grafana-sc-alerts,2977::budget run +grafana-sc-dashboard,4210::budget run +grafana-sc-datasources,3018::budget run +image-provider,6959::budget run +init-config,2501::budget run +jaeger,26309::budget run +kafka,109746::budget run +kube-proxy,29007::budget run +load-generator,3445::budget run +metrics-server,10784::budget run +opensearch,78270::budget run +opentelemetry-collector,455772::budget run +payment,1171::budget run +postgresql,10651::budget run +prometheus-server,8542::budget run +quote,189::budget run +recommendation,3226::budget run +valkey-cart,4089::budget run +wait-for-kafka,1644::budget run +wait-for-valkey-cart,1313::budget run +*,4150::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/commit.diff new file mode 100644 index 0000000..ba60530 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/commit.diff @@ -0,0 +1,183 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index c75da71..3b4557c 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,14129::budget run +-ad,6743::budget run +-aws-eks-nodeagent,287::budget run +-aws-node,1361::budget run +-aws-vpc-cni-init,1732::budget run +-cart,5115::budget run +-coredns,7035::budget run +-currency,408::budget run +-email,1192::budget run +-flagd,3847::budget run +-flagd-ui,680::budget run +-fluentd-10x,23495::budget run +-fraud-detection,1016::budget run +-frontend,23318::budget run +-frontend-proxy,44521::budget run +-grafana,482478::budget run +-grafana-sc-alerts,2977::budget run +-grafana-sc-dashboard,4210::budget run +-grafana-sc-datasources,3018::budget run +-image-provider,6959::budget run +-init-config,2501::budget run +-jaeger,26309::budget run +-kafka,109746::budget run +-kube-proxy,29007::budget run +-load-generator,3445::budget run +-metrics-server,10784::budget run +-opensearch,78270::budget run +-opentelemetry-collector,455772::budget run +-payment,1171::budget run +-postgresql,10651::budget run +-prometheus-server,8542::budget run +-quote,189::budget run +-recommendation,3226::budget run +-valkey-cart,4089::budget run +-wait-for-kafka,1644::budget run +-wait-for-valkey-cart,1313::budget run +-*,4150::budget run ++accounting,12086::budget run ++ad,6466::budget run ++aws-eks-nodeagent,233::budget run ++aws-node,1105::budget run ++aws-vpc-cni-init,1407::budget run ++cart,4731::budget run ++coredns,6333::budget run ++currency,332::budget run ++email,968::budget run ++flagd,3125::budget run ++flagd-ui,552::budget run ++fluentd-10x,19845::budget run ++fraud-detection,826::budget run ++frontend,33186::budget run ++frontend-proxy,36395::budget run ++grafana,394254::budget run ++grafana-sc-alerts,2419::budget run ++grafana-sc-dashboard,3420::budget run ++grafana-sc-datasources,2452::budget run ++image-provider,5833::budget run ++init-config,2032::budget run ++jaeger,21373::budget run ++kafka,97038::budget run ++kube-proxy,24457::budget run ++load-generator,2799::budget run ++metrics-server,8761::budget run ++opensearch,65507::budget run ++opentelemetry-collector,385112::budget run ++payment,951::budget run ++postgresql,8829::budget run ++prometheus-server,7441::budget run ++quote,154::budget run ++recommendation,3090::budget run ++valkey-cart,5228::budget run ++wait-for-kafka,1530::budget run ++wait-for-valkey-cart,1273::budget run ++*,4076::budget run +diff --git a/reports/latest.md b/reports/latest.md +index f123a4f..cbb1600 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,53 +1,54 @@ +-# Hot-tier run: 2026-10-26T00:16:09.000Z ++# Hot-tier run: 2026-10-26T00:17:10.000Z + +-Hot tier: 3 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 74 MB within 2 hours, with 502 MB a day of counted lines moved to the frozen tier. ++Hot tier: 4 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 96 MB within 2 hours, with 730 MB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 1 MB in 1. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 2 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.20 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 999 MB a day; 164 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 497 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.17 bytes of store kept, 0.91 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 1.2 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 422 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 497 MB a day on hot, more than the 164 MB a day that fits, so hot settles above 41 MB at about 74 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 422 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 96 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14129 | +-| ad | the frozen tier | 1 | 16 MB | 2 MB | 1 | 6743 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 287 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1361 | +-| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1732 | +-| cart | the frozen tier | 1 | 37 MB | 2 MB | 1 | 5115 | +-| coredns | the frozen tier | 1 | 4 MB | 3 MB | 1 | 7035 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 408 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1192 | +-| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3847 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 680 | +-| fluentd-10x | the frozen tier | 1 | 15 MB | 8 MB | 1 | 23495 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1016 | +-| frontend | the frozen tier | 1 | 8 MB | 8 MB | 1 | 23318 | +-| frontend-proxy | the frozen tier | 1 | 16 MB | 16 MB | 1 | 44521 | +-| grafana | the frozen tier | 1 | 174 MB | 174 MB | 1 | 482478 | +-| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2977 | +-| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4210 | +-| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3018 | +-| image-provider | the frozen tier | 1 | 3 MB | 3 MB | 1 | 6959 | +-| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2501 | +-| jaeger | the frozen tier | 1 | 9 MB | 9 MB | 1 | 26309 | +-| kafka | the frozen tier | 1 | 128 MB | 40 MB | 1 | 109746 | +-| kube-proxy | the frozen tier | 1 | 22 MB | 10 MB | 1 | 29007 | +-| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3445 | +-| metrics-server | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10784 | +-| opensearch | the frozen tier | 1 | 30 MB | 28 MB | 1 | 78270 | +-| opentelemetry-collector | the frozen tier | 1 | 486 MB | 164 MB | 1 | 455772 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1171 | +-| postgresql | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10651 | +-| prometheus-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8542 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 189 | +-| recommendation | the frozen tier | 1 | 13 MB | 1 MB | 1 | 3226 | +-| valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4089 | +-| wait-for-kafka | the frozen tier | 1 | 10 MB | 1 MB | 1 | 1644 | +-| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1313 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 4150 | ++| accounting | the frozen tier | 1 | 4 MB | 4 MB | 1 | 12086 | ++| ad | the frozen tier | 1 | 25 MB | 2 MB | 1 | 6466 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 233 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1105 | ++| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1407 | ++| cart | the frozen tier | 1 | 54 MB | 2 MB | 1 | 4731 | ++| coredns | the frozen tier | 1 | 3 MB | 2 MB | 1 | 6333 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 332 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 968 | ++| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3125 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 552 | ++| fluentd-10x | the frozen tier | 1 | 14 MB | 7 MB | 1 | 19845 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 826 | ++| frontend | the frozen tier | 1 | 12 MB | 12 MB | 1 | 33186 | ++| frontend-proxy | the frozen tier | 1 | 13 MB | 13 MB | 1 | 36395 | ++| grafana | the frozen tier | 1 | 143 MB | 142 MB | 1 | 394254 | ++| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2419 | ++| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3420 | ++| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2452 | ++| image-provider | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5833 | ++| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2032 | ++| jaeger | the frozen tier | 1 | 8 MB | 8 MB | 1 | 21373 | ++| kafka | the frozen tier | 1 | 160 MB | 35 MB | 1 | 97038 | ++| kube-proxy | the frozen tier | 1 | 18 MB | 9 MB | 1 | 24457 | ++| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2799 | ++| metrics-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8761 | ++| opensearch | the frozen tier | 1 | 25 MB | 24 MB | 1 | 65507 | ++| opentelemetry-collector | the frozen tier | 1 | 624 MB | 139 MB | 1 | 385112 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 951 | ++| postgresql | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8829 | ++| prometheus-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7441 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 154 | ++| recommendation | the frozen tier | 1 | 18 MB | 1 MB | 1 | 3090 | ++| valkey-cart | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5228 | ++| wait-for-kafka | the frozen tier | 1 | 8 MB | 1 MB | 1 | 1530 | ++| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1273 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 4076 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/commit.json new file mode 100644 index 0000000..1ac6abf --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/commit.json @@ -0,0 +1 @@ +{"before":"79cc5655b373e5323862134669c10bd6fc64c335","after":"89bb297cf1b8d141aada6b3138632a1f990a404d","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/latest.md new file mode 100644 index 0000000..cbb1600 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/latest.md @@ -0,0 +1,54 @@ +# Hot-tier run: 2026-10-26T00:17:10.000Z + +Hot tier: 4 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 96 MB within 2 hours, with 730 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 2 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.17 bytes of store kept, 0.91 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.2 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 422 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 422 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 96 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 4 MB | 4 MB | 1 | 12086 | +| ad | the frozen tier | 1 | 25 MB | 2 MB | 1 | 6466 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 233 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1105 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1407 | +| cart | the frozen tier | 1 | 54 MB | 2 MB | 1 | 4731 | +| coredns | the frozen tier | 1 | 3 MB | 2 MB | 1 | 6333 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 332 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 968 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3125 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 552 | +| fluentd-10x | the frozen tier | 1 | 14 MB | 7 MB | 1 | 19845 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 826 | +| frontend | the frozen tier | 1 | 12 MB | 12 MB | 1 | 33186 | +| frontend-proxy | the frozen tier | 1 | 13 MB | 13 MB | 1 | 36395 | +| grafana | the frozen tier | 1 | 143 MB | 142 MB | 1 | 394254 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2419 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3420 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2452 | +| image-provider | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5833 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2032 | +| jaeger | the frozen tier | 1 | 8 MB | 8 MB | 1 | 21373 | +| kafka | the frozen tier | 1 | 160 MB | 35 MB | 1 | 97038 | +| kube-proxy | the frozen tier | 1 | 18 MB | 9 MB | 1 | 24457 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2799 | +| metrics-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8761 | +| opensearch | the frozen tier | 1 | 25 MB | 24 MB | 1 | 65507 | +| opentelemetry-collector | the frozen tier | 1 | 624 MB | 139 MB | 1 | 385112 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 951 | +| postgresql | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8829 | +| prometheus-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7441 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 154 | +| recommendation | the frozen tier | 1 | 18 MB | 1 MB | 1 | 3090 | +| valkey-cart | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5228 | +| wait-for-kafka | the frozen tier | 1 | 8 MB | 1 MB | 1 | 1530 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1273 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 4076 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/report.txt new file mode 100644 index 0000000..667247e --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/report.txt @@ -0,0 +1,63 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 4 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 96 MB within 2 hours, with 730 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 2 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.17 bytes of store kept, 0.91 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1.2 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 422 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:17:10.000Z + +Hot tier: 4 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 96 MB within 2 hours, with 730 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 2 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.17 bytes of store kept, 0.91 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.2 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 422 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 422 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 96 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 4 MB | 4 MB | 1 | 12086 | +| ad | the frozen tier | 1 | 25 MB | 2 MB | 1 | 6466 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 233 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1105 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1407 | +| cart | the frozen tier | 1 | 54 MB | 2 MB | 1 | 4731 | +| coredns | the frozen tier | 1 | 3 MB | 2 MB | 1 | 6333 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 332 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 968 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3125 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 552 | +| fluentd-10x | the frozen tier | 1 | 14 MB | 7 MB | 1 | 19845 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 826 | +| frontend | the frozen tier | 1 | 12 MB | 12 MB | 1 | 33186 | +| frontend-proxy | the frozen tier | 1 | 13 MB | 13 MB | 1 | 36395 | +| grafana | the frozen tier | 1 | 143 MB | 142 MB | 1 | 394254 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2419 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3420 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2452 | +| image-provider | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5833 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2032 | +| jaeger | the frozen tier | 1 | 8 MB | 8 MB | 1 | 21373 | +| kafka | the frozen tier | 1 | 160 MB | 35 MB | 1 | 97038 | +| kube-proxy | the frozen tier | 1 | 18 MB | 9 MB | 1 | 24457 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2799 | +| metrics-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8761 | +| opensearch | the frozen tier | 1 | 25 MB | 24 MB | 1 | 65507 | +| opentelemetry-collector | the frozen tier | 1 | 624 MB | 139 MB | 1 | 385112 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 951 | +| postgresql | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8829 | +| prometheus-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7441 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 154 | +| recommendation | the frozen tier | 1 | 18 MB | 1 MB | 1 | 3090 | +| valkey-cart | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5228 | +| wait-for-kafka | the frozen tier | 1 | 8 MB | 1 MB | 1 | 1530 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1273 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 4076 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/run.json new file mode 100644 index 0000000..20005cf --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:17:16.251Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 4 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 96 MB within 2 hours, with 730 MB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/totals.csv new file mode 100644 index 0000000..3b4557c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001709Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,12086::budget run +ad,6466::budget run +aws-eks-nodeagent,233::budget run +aws-node,1105::budget run +aws-vpc-cni-init,1407::budget run +cart,4731::budget run +coredns,6333::budget run +currency,332::budget run +email,968::budget run +flagd,3125::budget run +flagd-ui,552::budget run +fluentd-10x,19845::budget run +fraud-detection,826::budget run +frontend,33186::budget run +frontend-proxy,36395::budget run +grafana,394254::budget run +grafana-sc-alerts,2419::budget run +grafana-sc-dashboard,3420::budget run +grafana-sc-datasources,2452::budget run +image-provider,5833::budget run +init-config,2032::budget run +jaeger,21373::budget run +kafka,97038::budget run +kube-proxy,24457::budget run +load-generator,2799::budget run +metrics-server,8761::budget run +opensearch,65507::budget run +opentelemetry-collector,385112::budget run +payment,951::budget run +postgresql,8829::budget run +prometheus-server,7441::budget run +quote,154::budget run +recommendation,3090::budget run +valkey-cart,5228::budget run +wait-for-kafka,1530::budget run +wait-for-valkey-cart,1273::budget run +*,4076::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/commit.diff new file mode 100644 index 0000000..bd91b25 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/commit.diff @@ -0,0 +1,182 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 3b4557c..1c91da8 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,12086::budget run +-ad,6466::budget run +-aws-eks-nodeagent,233::budget run +-aws-node,1105::budget run +-aws-vpc-cni-init,1407::budget run +-cart,4731::budget run +-coredns,6333::budget run +-currency,332::budget run +-email,968::budget run +-flagd,3125::budget run +-flagd-ui,552::budget run +-fluentd-10x,19845::budget run +-fraud-detection,826::budget run +-frontend,33186::budget run +-frontend-proxy,36395::budget run +-grafana,394254::budget run +-grafana-sc-alerts,2419::budget run +-grafana-sc-dashboard,3420::budget run +-grafana-sc-datasources,2452::budget run +-image-provider,5833::budget run +-init-config,2032::budget run +-jaeger,21373::budget run +-kafka,97038::budget run +-kube-proxy,24457::budget run +-load-generator,2799::budget run +-metrics-server,8761::budget run +-opensearch,65507::budget run +-opentelemetry-collector,385112::budget run +-payment,951::budget run +-postgresql,8829::budget run +-prometheus-server,7441::budget run +-quote,154::budget run +-recommendation,3090::budget run +-valkey-cart,5228::budget run +-wait-for-kafka,1530::budget run +-wait-for-valkey-cart,1273::budget run +-*,4076::budget run ++accounting,10322::budget run ++ad,6284::budget run ++aws-eks-nodeagent,197::budget run ++aws-node,938::budget run ++aws-vpc-cni-init,1194::budget run ++cart,4481::budget run ++coredns,5748::budget run ++currency,281::budget run ++email,821::budget run ++flagd,2652::budget run ++flagd-ui,469::budget run ++fluentd-10x,17453::budget run ++fraud-detection,700::budget run ++frontend,34888::budget run ++frontend-proxy,30885::budget run ++grafana,336872::budget run ++grafana-sc-alerts,2052::budget run ++grafana-sc-dashboard,2902::budget run ++grafana-sc-datasources,2081::budget run ++image-provider,5095::budget run ++init-config,1724::budget run ++jaeger,21078::budget run ++kafka,88688::budget run ++kube-proxy,21474::budget run ++load-generator,2375::budget run ++metrics-server,7434::budget run ++opensearch,56884::budget run ++opentelemetry-collector,332220::budget run ++payment,807::budget run ++postgresql,7519::budget run ++prometheus-server,7167::budget run ++quote,130::budget run ++recommendation,3001::budget run ++valkey-cart,6053::budget run ++wait-for-kafka,1456::budget run ++wait-for-valkey-cart,1247::budget run ++*,3741::budget run +diff --git a/reports/latest.md b/reports/latest.md +index cbb1600..4933fb2 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,54 +1,54 @@ +-# Hot-tier run: 2026-10-26T00:17:10.000Z ++# Hot-tier run: 2026-10-26T00:18:08.000Z + +-Hot tier: 4 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 96 MB within 2 hours, with 730 MB a day of counted lines moved to the frozen tier. ++Hot tier: 5 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 92 MB within 2 hours, with 880 MB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 2 MB in 1. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 1 indices; its tier_down lines not yet moved: 3 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.17 bytes of store kept, 0.91 as tier_down (measured on the indices on hot). ++- Each counted byte takes 1.26 bytes of store kept, 0.73 as tier_down (measured on the indices on hot). + - The Receiver's statements send 1.2 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 422 MB a day on hot whatever the limits; the size holds down to that floor. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 369 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 422 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 96 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 369 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 92 MB. Raise the size, move the noisiest services to offload, or lower the sample. + - Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 4 MB | 4 MB | 1 | 12086 | +-| ad | the frozen tier | 1 | 25 MB | 2 MB | 1 | 6466 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 233 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1105 | +-| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1407 | +-| cart | the frozen tier | 1 | 54 MB | 2 MB | 1 | 4731 | +-| coredns | the frozen tier | 1 | 3 MB | 2 MB | 1 | 6333 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 332 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 968 | +-| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3125 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 552 | +-| fluentd-10x | the frozen tier | 1 | 14 MB | 7 MB | 1 | 19845 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 826 | +-| frontend | the frozen tier | 1 | 12 MB | 12 MB | 1 | 33186 | +-| frontend-proxy | the frozen tier | 1 | 13 MB | 13 MB | 1 | 36395 | +-| grafana | the frozen tier | 1 | 143 MB | 142 MB | 1 | 394254 | +-| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2419 | +-| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3420 | +-| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2452 | +-| image-provider | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5833 | +-| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2032 | +-| jaeger | the frozen tier | 1 | 8 MB | 8 MB | 1 | 21373 | +-| kafka | the frozen tier | 1 | 160 MB | 35 MB | 1 | 97038 | +-| kube-proxy | the frozen tier | 1 | 18 MB | 9 MB | 1 | 24457 | +-| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2799 | +-| metrics-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8761 | +-| opensearch | the frozen tier | 1 | 25 MB | 24 MB | 1 | 65507 | +-| opentelemetry-collector | the frozen tier | 1 | 624 MB | 139 MB | 1 | 385112 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 951 | +-| postgresql | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8829 | +-| prometheus-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7441 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 154 | +-| recommendation | the frozen tier | 1 | 18 MB | 1 MB | 1 | 3090 | +-| valkey-cart | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5228 | +-| wait-for-kafka | the frozen tier | 1 | 8 MB | 1 MB | 1 | 1530 | +-| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1273 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 4076 | ++| accounting | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10322 | ++| ad | the frozen tier | 1 | 33 MB | 2 MB | 1 | 6284 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 197 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 938 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1194 | ++| cart | the frozen tier | 1 | 68 MB | 2 MB | 1 | 4481 | ++| coredns | the frozen tier | 1 | 3 MB | 2 MB | 1 | 5748 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 281 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 821 | ++| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2652 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 469 | ++| fluentd-10x | the frozen tier | 1 | 14 MB | 6 MB | 1 | 17453 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 700 | ++| frontend | the frozen tier | 1 | 15 MB | 13 MB | 1 | 34888 | ++| frontend-proxy | the frozen tier | 1 | 11 MB | 11 MB | 1 | 30885 | ++| grafana | the frozen tier | 1 | 123 MB | 121 MB | 1 | 336872 | ++| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2052 | ++| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2902 | ++| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2081 | ++| image-provider | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5095 | ++| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1724 | ++| jaeger | the frozen tier | 1 | 8 MB | 8 MB | 1 | 21078 | ++| kafka | the frozen tier | 1 | 178 MB | 32 MB | 1 | 88688 | ++| kube-proxy | the frozen tier | 1 | 15 MB | 8 MB | 1 | 21474 | ++| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2375 | ++| metrics-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7434 | ++| opensearch | the frozen tier | 1 | 21 MB | 20 MB | 1 | 56884 | ++| opentelemetry-collector | the frozen tier | 1 | 709 MB | 120 MB | 1 | 332220 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 807 | ++| postgresql | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7519 | ++| prometheus-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7167 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 130 | ++| recommendation | the frozen tier | 1 | 21 MB | 1 MB | 1 | 3001 | ++| valkey-cart | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6053 | ++| wait-for-kafka | the frozen tier | 1 | 7 MB | 1 MB | 1 | 1456 | ++| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1247 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3741 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/commit.json new file mode 100644 index 0000000..1c86251 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/commit.json @@ -0,0 +1 @@ +{"before":"89bb297cf1b8d141aada6b3138632a1f990a404d","after":"ee058f9c351a20cf6521dbb4ebf7f387177c95a7","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/latest.md new file mode 100644 index 0000000..4933fb2 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/latest.md @@ -0,0 +1,54 @@ +# Hot-tier run: 2026-10-26T00:18:08.000Z + +Hot tier: 5 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 92 MB within 2 hours, with 880 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 1 indices; its tier_down lines not yet moved: 3 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.26 bytes of store kept, 0.73 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.2 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 369 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 369 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 92 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10322 | +| ad | the frozen tier | 1 | 33 MB | 2 MB | 1 | 6284 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 197 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 938 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1194 | +| cart | the frozen tier | 1 | 68 MB | 2 MB | 1 | 4481 | +| coredns | the frozen tier | 1 | 3 MB | 2 MB | 1 | 5748 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 281 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 821 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2652 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 469 | +| fluentd-10x | the frozen tier | 1 | 14 MB | 6 MB | 1 | 17453 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 700 | +| frontend | the frozen tier | 1 | 15 MB | 13 MB | 1 | 34888 | +| frontend-proxy | the frozen tier | 1 | 11 MB | 11 MB | 1 | 30885 | +| grafana | the frozen tier | 1 | 123 MB | 121 MB | 1 | 336872 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2052 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2902 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2081 | +| image-provider | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5095 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1724 | +| jaeger | the frozen tier | 1 | 8 MB | 8 MB | 1 | 21078 | +| kafka | the frozen tier | 1 | 178 MB | 32 MB | 1 | 88688 | +| kube-proxy | the frozen tier | 1 | 15 MB | 8 MB | 1 | 21474 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2375 | +| metrics-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7434 | +| opensearch | the frozen tier | 1 | 21 MB | 20 MB | 1 | 56884 | +| opentelemetry-collector | the frozen tier | 1 | 709 MB | 120 MB | 1 | 332220 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 807 | +| postgresql | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7519 | +| prometheus-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7167 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 130 | +| recommendation | the frozen tier | 1 | 21 MB | 1 MB | 1 | 3001 | +| valkey-cart | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6053 | +| wait-for-kafka | the frozen tier | 1 | 7 MB | 1 MB | 1 | 1456 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1247 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3741 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/report.txt new file mode 100644 index 0000000..0aaffaf --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/report.txt @@ -0,0 +1,63 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 5 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 92 MB within 2 hours, with 880 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 1 indices; its tier_down lines not yet moved: 3 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.26 bytes of store kept, 0.73 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1.2 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 369 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:18:08.000Z + +Hot tier: 5 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 92 MB within 2 hours, with 880 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 1 indices; its tier_down lines not yet moved: 3 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.26 bytes of store kept, 0.73 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.2 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 369 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 369 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 92 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10322 | +| ad | the frozen tier | 1 | 33 MB | 2 MB | 1 | 6284 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 197 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 938 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1194 | +| cart | the frozen tier | 1 | 68 MB | 2 MB | 1 | 4481 | +| coredns | the frozen tier | 1 | 3 MB | 2 MB | 1 | 5748 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 281 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 821 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2652 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 469 | +| fluentd-10x | the frozen tier | 1 | 14 MB | 6 MB | 1 | 17453 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 700 | +| frontend | the frozen tier | 1 | 15 MB | 13 MB | 1 | 34888 | +| frontend-proxy | the frozen tier | 1 | 11 MB | 11 MB | 1 | 30885 | +| grafana | the frozen tier | 1 | 123 MB | 121 MB | 1 | 336872 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2052 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2902 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2081 | +| image-provider | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5095 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1724 | +| jaeger | the frozen tier | 1 | 8 MB | 8 MB | 1 | 21078 | +| kafka | the frozen tier | 1 | 178 MB | 32 MB | 1 | 88688 | +| kube-proxy | the frozen tier | 1 | 15 MB | 8 MB | 1 | 21474 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2375 | +| metrics-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7434 | +| opensearch | the frozen tier | 1 | 21 MB | 20 MB | 1 | 56884 | +| opentelemetry-collector | the frozen tier | 1 | 709 MB | 120 MB | 1 | 332220 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 807 | +| postgresql | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7519 | +| prometheus-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7167 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 130 | +| recommendation | the frozen tier | 1 | 21 MB | 1 MB | 1 | 3001 | +| valkey-cart | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6053 | +| wait-for-kafka | the frozen tier | 1 | 7 MB | 1 MB | 1 | 1456 | +| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1247 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3741 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/run.json new file mode 100644 index 0000000..5e6b7b9 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:18:12.226Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 5 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 92 MB within 2 hours, with 880 MB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/totals.csv new file mode 100644 index 0000000..1c91da8 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001808Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,10322::budget run +ad,6284::budget run +aws-eks-nodeagent,197::budget run +aws-node,938::budget run +aws-vpc-cni-init,1194::budget run +cart,4481::budget run +coredns,5748::budget run +currency,281::budget run +email,821::budget run +flagd,2652::budget run +flagd-ui,469::budget run +fluentd-10x,17453::budget run +fraud-detection,700::budget run +frontend,34888::budget run +frontend-proxy,30885::budget run +grafana,336872::budget run +grafana-sc-alerts,2052::budget run +grafana-sc-dashboard,2902::budget run +grafana-sc-datasources,2081::budget run +image-provider,5095::budget run +init-config,1724::budget run +jaeger,21078::budget run +kafka,88688::budget run +kube-proxy,21474::budget run +load-generator,2375::budget run +metrics-server,7434::budget run +opensearch,56884::budget run +opentelemetry-collector,332220::budget run +payment,807::budget run +postgresql,7519::budget run +prometheus-server,7167::budget run +quote,130::budget run +recommendation,3001::budget run +valkey-cart,6053::budget run +wait-for-kafka,1456::budget run +wait-for-valkey-cart,1247::budget run +*,3741::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/commit.diff new file mode 100644 index 0000000..83ed756 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/commit.diff @@ -0,0 +1,182 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 1c91da8..c6bc53c 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,10322::budget run +-ad,6284::budget run +-aws-eks-nodeagent,197::budget run +-aws-node,938::budget run +-aws-vpc-cni-init,1194::budget run +-cart,4481::budget run +-coredns,5748::budget run +-currency,281::budget run +-email,821::budget run +-flagd,2652::budget run +-flagd-ui,469::budget run +-fluentd-10x,17453::budget run +-fraud-detection,700::budget run +-frontend,34888::budget run +-frontend-proxy,30885::budget run +-grafana,336872::budget run +-grafana-sc-alerts,2052::budget run +-grafana-sc-dashboard,2902::budget run +-grafana-sc-datasources,2081::budget run +-image-provider,5095::budget run +-init-config,1724::budget run +-jaeger,21078::budget run +-kafka,88688::budget run +-kube-proxy,21474::budget run +-load-generator,2375::budget run +-metrics-server,7434::budget run +-opensearch,56884::budget run +-opentelemetry-collector,332220::budget run +-payment,807::budget run +-postgresql,7519::budget run +-prometheus-server,7167::budget run +-quote,130::budget run +-recommendation,3001::budget run +-valkey-cart,6053::budget run +-wait-for-kafka,1456::budget run +-wait-for-valkey-cart,1247::budget run +-*,3741::budget run ++accounting,8904::budget run ++ad,6144::budget run ++aws-eks-nodeagent,170::budget run ++aws-node,809::budget run ++aws-vpc-cni-init,1030::budget run ++cart,4289::budget run ++coredns,5266::budget run ++currency,243::budget run ++email,709::budget run ++flagd,2288::budget run ++flagd-ui,404::budget run ++fluentd-10x,15612::budget run ++fraud-detection,604::budget run ++frontend,34653::budget run ++frontend-proxy,26643::budget run ++grafana,292016::budget run ++grafana-sc-alerts,1770::budget run ++grafana-sc-dashboard,2504::budget run ++grafana-sc-datasources,1795::budget run ++image-provider,4501::budget run ++init-config,1487::budget run ++jaeger,18183::budget run ++kafka,82001::budget run ++kube-proxy,19177::budget run ++load-generator,2049::budget run ++metrics-server,6413::budget run ++opensearch,50179::budget run ++opentelemetry-collector,291405::budget run ++payment,696::budget run ++postgresql,6486::budget run ++prometheus-server,6366::budget run ++quote,112::budget run ++recommendation,2932::budget run ++valkey-cart,6616::budget run ++wait-for-kafka,1399::budget run ++wait-for-valkey-cart,1226::budget run ++*,3611::budget run +diff --git a/reports/latest.md b/reports/latest.md +index 4933fb2..f03decb 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,54 +1,54 @@ +-# Hot-tier run: 2026-10-26T00:18:08.000Z ++# Hot-tier run: 2026-10-26T00:19:09.000Z + +-Hot tier: 5 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 92 MB within 2 hours, with 880 MB a day of counted lines moved to the frozen tier. ++Hot tier: 5 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 85 MB within 2 hours, with 992 MB a day of counted lines moved to the frozen tier. + + - Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 1 indices; its tier_down lines not yet moved: 3 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.26 bytes of store kept, 0.73 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 1.2 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 369 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.27 bytes of store kept, 0.61 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 1.3 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 327 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 369 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 92 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 327 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 85 MB. Raise the size, move the noisiest services to offload, or lower the sample. + - Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10322 | +-| ad | the frozen tier | 1 | 33 MB | 2 MB | 1 | 6284 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 197 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 938 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1194 | +-| cart | the frozen tier | 1 | 68 MB | 2 MB | 1 | 4481 | +-| coredns | the frozen tier | 1 | 3 MB | 2 MB | 1 | 5748 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 281 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 821 | +-| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2652 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 469 | +-| fluentd-10x | the frozen tier | 1 | 14 MB | 6 MB | 1 | 17453 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 700 | +-| frontend | the frozen tier | 1 | 15 MB | 13 MB | 1 | 34888 | +-| frontend-proxy | the frozen tier | 1 | 11 MB | 11 MB | 1 | 30885 | +-| grafana | the frozen tier | 1 | 123 MB | 121 MB | 1 | 336872 | +-| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2052 | +-| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2902 | +-| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2081 | +-| image-provider | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5095 | +-| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1724 | +-| jaeger | the frozen tier | 1 | 8 MB | 8 MB | 1 | 21078 | +-| kafka | the frozen tier | 1 | 178 MB | 32 MB | 1 | 88688 | +-| kube-proxy | the frozen tier | 1 | 15 MB | 8 MB | 1 | 21474 | +-| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2375 | +-| metrics-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7434 | +-| opensearch | the frozen tier | 1 | 21 MB | 20 MB | 1 | 56884 | +-| opentelemetry-collector | the frozen tier | 1 | 709 MB | 120 MB | 1 | 332220 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 807 | +-| postgresql | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7519 | +-| prometheus-server | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7167 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 130 | +-| recommendation | the frozen tier | 1 | 21 MB | 1 MB | 1 | 3001 | +-| valkey-cart | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6053 | +-| wait-for-kafka | the frozen tier | 1 | 7 MB | 1 MB | 1 | 1456 | +-| wait-for-valkey-cart | the frozen tier | 1 | 1 MB | 0 MB | 1 | 1247 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3741 | ++| accounting | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8904 | ++| ad | the frozen tier | 1 | 36 MB | 2 MB | 1 | 6144 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 170 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 809 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1030 | ++| cart | the frozen tier | 1 | 79 MB | 2 MB | 1 | 4289 | ++| coredns | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5266 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 243 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 709 | ++| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2288 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 404 | ++| fluentd-10x | the frozen tier | 1 | 14 MB | 6 MB | 1 | 15612 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 604 | ++| frontend | the frozen tier | 1 | 18 MB | 12 MB | 1 | 34653 | ++| frontend-proxy | the frozen tier | 1 | 10 MB | 10 MB | 1 | 26643 | ++| grafana | the frozen tier | 1 | 107 MB | 105 MB | 1 | 292016 | ++| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1770 | ++| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2504 | ++| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1795 | ++| image-provider | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4501 | ++| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1487 | ++| jaeger | the frozen tier | 1 | 7 MB | 7 MB | 1 | 18183 | ++| kafka | the frozen tier | 1 | 192 MB | 30 MB | 1 | 82001 | ++| kube-proxy | the frozen tier | 1 | 13 MB | 7 MB | 1 | 19177 | ++| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2049 | ++| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6413 | ++| opensearch | the frozen tier | 1 | 18 MB | 18 MB | 1 | 50179 | ++| opentelemetry-collector | the frozen tier | 1 | 772 MB | 105 MB | 1 | 291405 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 696 | ++| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6486 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6366 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 112 | ++| recommendation | the frozen tier | 1 | 24 MB | 1 MB | 1 | 2932 | ++| valkey-cart | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6616 | ++| wait-for-kafka | the frozen tier | 1 | 6 MB | 1 MB | 1 | 1399 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1226 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3611 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/commit.json new file mode 100644 index 0000000..77c815c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/commit.json @@ -0,0 +1 @@ +{"before":"ee058f9c351a20cf6521dbb4ebf7f387177c95a7","after":"1142b2983785d73525518e634d446e2336d4e0e1","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/latest.md new file mode 100644 index 0000000..f03decb --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/latest.md @@ -0,0 +1,54 @@ +# Hot-tier run: 2026-10-26T00:19:09.000Z + +Hot tier: 5 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 85 MB within 2 hours, with 992 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 1 indices; its tier_down lines not yet moved: 3 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.27 bytes of store kept, 0.61 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.3 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 327 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 327 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 85 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8904 | +| ad | the frozen tier | 1 | 36 MB | 2 MB | 1 | 6144 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 170 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 809 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1030 | +| cart | the frozen tier | 1 | 79 MB | 2 MB | 1 | 4289 | +| coredns | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5266 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 243 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 709 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2288 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 404 | +| fluentd-10x | the frozen tier | 1 | 14 MB | 6 MB | 1 | 15612 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 604 | +| frontend | the frozen tier | 1 | 18 MB | 12 MB | 1 | 34653 | +| frontend-proxy | the frozen tier | 1 | 10 MB | 10 MB | 1 | 26643 | +| grafana | the frozen tier | 1 | 107 MB | 105 MB | 1 | 292016 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1770 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2504 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1795 | +| image-provider | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4501 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1487 | +| jaeger | the frozen tier | 1 | 7 MB | 7 MB | 1 | 18183 | +| kafka | the frozen tier | 1 | 192 MB | 30 MB | 1 | 82001 | +| kube-proxy | the frozen tier | 1 | 13 MB | 7 MB | 1 | 19177 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2049 | +| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6413 | +| opensearch | the frozen tier | 1 | 18 MB | 18 MB | 1 | 50179 | +| opentelemetry-collector | the frozen tier | 1 | 772 MB | 105 MB | 1 | 291405 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 696 | +| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6486 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6366 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 112 | +| recommendation | the frozen tier | 1 | 24 MB | 1 MB | 1 | 2932 | +| valkey-cart | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6616 | +| wait-for-kafka | the frozen tier | 1 | 6 MB | 1 MB | 1 | 1399 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1226 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3611 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/report.txt new file mode 100644 index 0000000..9b93685 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/report.txt @@ -0,0 +1,63 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 5 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 85 MB within 2 hours, with 992 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 1 indices; its tier_down lines not yet moved: 3 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.27 bytes of store kept, 0.61 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1.3 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 327 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:19:09.000Z + +Hot tier: 5 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 85 MB within 2 hours, with 992 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 1 indices; its tier_down lines not yet moved: 3 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.27 bytes of store kept, 0.61 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.3 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 327 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 327 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 85 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8904 | +| ad | the frozen tier | 1 | 36 MB | 2 MB | 1 | 6144 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 170 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 809 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1030 | +| cart | the frozen tier | 1 | 79 MB | 2 MB | 1 | 4289 | +| coredns | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5266 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 243 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 709 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2288 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 404 | +| fluentd-10x | the frozen tier | 1 | 14 MB | 6 MB | 1 | 15612 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 604 | +| frontend | the frozen tier | 1 | 18 MB | 12 MB | 1 | 34653 | +| frontend-proxy | the frozen tier | 1 | 10 MB | 10 MB | 1 | 26643 | +| grafana | the frozen tier | 1 | 107 MB | 105 MB | 1 | 292016 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1770 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2504 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1795 | +| image-provider | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4501 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1487 | +| jaeger | the frozen tier | 1 | 7 MB | 7 MB | 1 | 18183 | +| kafka | the frozen tier | 1 | 192 MB | 30 MB | 1 | 82001 | +| kube-proxy | the frozen tier | 1 | 13 MB | 7 MB | 1 | 19177 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2049 | +| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6413 | +| opensearch | the frozen tier | 1 | 18 MB | 18 MB | 1 | 50179 | +| opentelemetry-collector | the frozen tier | 1 | 772 MB | 105 MB | 1 | 291405 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 696 | +| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6486 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6366 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 112 | +| recommendation | the frozen tier | 1 | 24 MB | 1 MB | 1 | 2932 | +| valkey-cart | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6616 | +| wait-for-kafka | the frozen tier | 1 | 6 MB | 1 MB | 1 | 1399 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1226 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3611 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/run.json new file mode 100644 index 0000000..10570a6 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:19:12.559Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 5 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 85 MB within 2 hours, with 992 MB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/totals.csv new file mode 100644 index 0000000..c6bc53c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T001908Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,8904::budget run +ad,6144::budget run +aws-eks-nodeagent,170::budget run +aws-node,809::budget run +aws-vpc-cni-init,1030::budget run +cart,4289::budget run +coredns,5266::budget run +currency,243::budget run +email,709::budget run +flagd,2288::budget run +flagd-ui,404::budget run +fluentd-10x,15612::budget run +fraud-detection,604::budget run +frontend,34653::budget run +frontend-proxy,26643::budget run +grafana,292016::budget run +grafana-sc-alerts,1770::budget run +grafana-sc-dashboard,2504::budget run +grafana-sc-datasources,1795::budget run +image-provider,4501::budget run +init-config,1487::budget run +jaeger,18183::budget run +kafka,82001::budget run +kube-proxy,19177::budget run +load-generator,2049::budget run +metrics-server,6413::budget run +opensearch,50179::budget run +opentelemetry-collector,291405::budget run +payment,696::budget run +postgresql,6486::budget run +prometheus-server,6366::budget run +quote,112::budget run +recommendation,2932::budget run +valkey-cart,6616::budget run +wait-for-kafka,1399::budget run +wait-for-valkey-cart,1226::budget run +*,3611::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/commit.diff new file mode 100644 index 0000000..55f44bc --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/commit.diff @@ -0,0 +1,183 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index c6bc53c..4aaf2df 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,8904::budget run +-ad,6144::budget run +-aws-eks-nodeagent,170::budget run +-aws-node,809::budget run +-aws-vpc-cni-init,1030::budget run +-cart,4289::budget run +-coredns,5266::budget run +-currency,243::budget run +-email,709::budget run +-flagd,2288::budget run +-flagd-ui,404::budget run +-fluentd-10x,15612::budget run +-fraud-detection,604::budget run +-frontend,34653::budget run +-frontend-proxy,26643::budget run +-grafana,292016::budget run +-grafana-sc-alerts,1770::budget run +-grafana-sc-dashboard,2504::budget run +-grafana-sc-datasources,1795::budget run +-image-provider,4501::budget run +-init-config,1487::budget run +-jaeger,18183::budget run +-kafka,82001::budget run +-kube-proxy,19177::budget run +-load-generator,2049::budget run +-metrics-server,6413::budget run +-opensearch,50179::budget run +-opentelemetry-collector,291405::budget run +-payment,696::budget run +-postgresql,6486::budget run +-prometheus-server,6366::budget run +-quote,112::budget run +-recommendation,2932::budget run +-valkey-cart,6616::budget run +-wait-for-kafka,1399::budget run +-wait-for-valkey-cart,1226::budget run +-*,3611::budget run ++accounting,7844::budget run ++ad,6038::budget run ++aws-eks-nodeagent,150::budget run ++aws-node,713::budget run ++aws-vpc-cni-init,907::budget run ++cart,4146::budget run ++coredns,4793::budget run ++currency,214::budget run ++email,624::budget run ++flagd,2015::budget run ++flagd-ui,356::budget run ++fluentd-10x,14235::budget run ++fraud-detection,532::budget run ++frontend,34477::budget run ++frontend-proxy,23472::budget run ++grafana,258503::budget run ++grafana-sc-alerts,1560::budget run ++grafana-sc-dashboard,2206::budget run ++grafana-sc-datasources,1581::budget run ++image-provider,3965::budget run ++init-config,1310::budget run ++jaeger,16019::budget run ++kafka,76844::budget run ++kube-proxy,17460::budget run ++load-generator,1805::budget run ++metrics-server,5650::budget run ++opensearch,44976::budget run ++opentelemetry-collector,260895::budget run ++payment,613::budget run ++postgresql,5714::budget run ++prometheus-server,5770::budget run ++quote,99::budget run ++recommendation,2881::budget run ++valkey-cart,7058::budget run ++wait-for-kafka,1356::budget run ++wait-for-valkey-cart,1159::budget run ++*,3423::budget run +diff --git a/reports/latest.md b/reports/latest.md +index f03decb..b1c9d23 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,54 +1,54 @@ +-# Hot-tier run: 2026-10-26T00:19:09.000Z ++# Hot-tier run: 2026-10-26T00:20:09.000Z + +-Hot tier: 5 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 85 MB within 2 hours, with 992 MB a day of counted lines moved to the frozen tier. ++Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 158 MB within 2 hours, with 1.1 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 1 indices; its tier_down lines not yet moved: 3 MB in 1. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 1 indices; its tier_down lines not yet moved: 6 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.27 bytes of store kept, 0.61 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 1.3 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 327 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 2.44 bytes of store kept, 1.08 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 1.4 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 294 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 327 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 85 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 294 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 158 MB. Raise the size, move the noisiest services to offload, or lower the sample. + - Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8904 | +-| ad | the frozen tier | 1 | 36 MB | 2 MB | 1 | 6144 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 170 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 809 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1030 | +-| cart | the frozen tier | 1 | 79 MB | 2 MB | 1 | 4289 | +-| coredns | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5266 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 243 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 709 | +-| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2288 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 404 | +-| fluentd-10x | the frozen tier | 1 | 14 MB | 6 MB | 1 | 15612 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 604 | +-| frontend | the frozen tier | 1 | 18 MB | 12 MB | 1 | 34653 | +-| frontend-proxy | the frozen tier | 1 | 10 MB | 10 MB | 1 | 26643 | +-| grafana | the frozen tier | 1 | 107 MB | 105 MB | 1 | 292016 | +-| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1770 | +-| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2504 | +-| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1795 | +-| image-provider | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4501 | +-| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1487 | +-| jaeger | the frozen tier | 1 | 7 MB | 7 MB | 1 | 18183 | +-| kafka | the frozen tier | 1 | 192 MB | 30 MB | 1 | 82001 | +-| kube-proxy | the frozen tier | 1 | 13 MB | 7 MB | 1 | 19177 | +-| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2049 | +-| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6413 | +-| opensearch | the frozen tier | 1 | 18 MB | 18 MB | 1 | 50179 | +-| opentelemetry-collector | the frozen tier | 1 | 772 MB | 105 MB | 1 | 291405 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 696 | +-| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6486 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6366 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 112 | +-| recommendation | the frozen tier | 1 | 24 MB | 1 MB | 1 | 2932 | +-| valkey-cart | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6616 | +-| wait-for-kafka | the frozen tier | 1 | 6 MB | 1 MB | 1 | 1399 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1226 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3611 | ++| accounting | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7844 | ++| ad | the frozen tier | 1 | 39 MB | 2 MB | 1 | 6038 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 150 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 713 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 907 | ++| cart | the frozen tier | 1 | 88 MB | 1 MB | 1 | 4146 | ++| coredns | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4793 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 214 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 624 | ++| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2015 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 356 | ++| fluentd-10x | the frozen tier | 1 | 14 MB | 5 MB | 1 | 14235 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 532 | ++| frontend | the frozen tier | 1 | 19 MB | 12 MB | 1 | 34477 | ++| frontend-proxy | the frozen tier | 1 | 8 MB | 8 MB | 1 | 23472 | ++| grafana | the frozen tier | 1 | 96 MB | 93 MB | 1 | 258503 | ++| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1560 | ++| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2206 | ++| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1581 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3965 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1310 | ++| jaeger | the frozen tier | 1 | 6 MB | 6 MB | 1 | 16019 | ++| kafka | the frozen tier | 1 | 205 MB | 28 MB | 1 | 76844 | ++| kube-proxy | the frozen tier | 1 | 12 MB | 6 MB | 1 | 17460 | ++| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1805 | ++| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5650 | ++| opensearch | the frozen tier | 1 | 16 MB | 16 MB | 1 | 44976 | ++| opentelemetry-collector | the frozen tier | 1 | 826 MB | 94 MB | 1 | 260895 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 613 | ++| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5714 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5770 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 99 | ++| recommendation | the frozen tier | 1 | 27 MB | 1 MB | 1 | 2881 | ++| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7058 | ++| wait-for-kafka | the frozen tier | 1 | 5 MB | 0 MB | 1 | 1356 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1159 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3423 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/commit.json new file mode 100644 index 0000000..7ae7b18 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/commit.json @@ -0,0 +1 @@ +{"before":"1142b2983785d73525518e634d446e2336d4e0e1","after":"9a2ac1b0feb8702e9586825b0ce1a52f4808dc31","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/latest.md new file mode 100644 index 0000000..b1c9d23 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/latest.md @@ -0,0 +1,54 @@ +# Hot-tier run: 2026-10-26T00:20:09.000Z + +Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 158 MB within 2 hours, with 1.1 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 1 indices; its tier_down lines not yet moved: 6 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 2.44 bytes of store kept, 1.08 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.4 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 294 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 294 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 158 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7844 | +| ad | the frozen tier | 1 | 39 MB | 2 MB | 1 | 6038 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 150 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 713 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 907 | +| cart | the frozen tier | 1 | 88 MB | 1 MB | 1 | 4146 | +| coredns | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4793 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 214 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 624 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2015 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 356 | +| fluentd-10x | the frozen tier | 1 | 14 MB | 5 MB | 1 | 14235 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 532 | +| frontend | the frozen tier | 1 | 19 MB | 12 MB | 1 | 34477 | +| frontend-proxy | the frozen tier | 1 | 8 MB | 8 MB | 1 | 23472 | +| grafana | the frozen tier | 1 | 96 MB | 93 MB | 1 | 258503 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1560 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2206 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1581 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3965 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1310 | +| jaeger | the frozen tier | 1 | 6 MB | 6 MB | 1 | 16019 | +| kafka | the frozen tier | 1 | 205 MB | 28 MB | 1 | 76844 | +| kube-proxy | the frozen tier | 1 | 12 MB | 6 MB | 1 | 17460 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1805 | +| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5650 | +| opensearch | the frozen tier | 1 | 16 MB | 16 MB | 1 | 44976 | +| opentelemetry-collector | the frozen tier | 1 | 826 MB | 94 MB | 1 | 260895 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 613 | +| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5714 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5770 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 99 | +| recommendation | the frozen tier | 1 | 27 MB | 1 MB | 1 | 2881 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7058 | +| wait-for-kafka | the frozen tier | 1 | 5 MB | 0 MB | 1 | 1356 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1159 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3423 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/report.txt new file mode 100644 index 0000000..b59f4f4 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/report.txt @@ -0,0 +1,63 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 158 MB within 2 hours, with 1.1 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 1 indices; its tier_down lines not yet moved: 6 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 2.44 bytes of store kept, 1.08 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1.4 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 294 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:20:09.000Z + +Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 158 MB within 2 hours, with 1.1 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 1 indices; its tier_down lines not yet moved: 6 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 2.44 bytes of store kept, 1.08 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.4 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 294 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 294 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 158 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7844 | +| ad | the frozen tier | 1 | 39 MB | 2 MB | 1 | 6038 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 150 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 713 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 907 | +| cart | the frozen tier | 1 | 88 MB | 1 MB | 1 | 4146 | +| coredns | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4793 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 214 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 624 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2015 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 356 | +| fluentd-10x | the frozen tier | 1 | 14 MB | 5 MB | 1 | 14235 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 532 | +| frontend | the frozen tier | 1 | 19 MB | 12 MB | 1 | 34477 | +| frontend-proxy | the frozen tier | 1 | 8 MB | 8 MB | 1 | 23472 | +| grafana | the frozen tier | 1 | 96 MB | 93 MB | 1 | 258503 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1560 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2206 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1581 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3965 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1310 | +| jaeger | the frozen tier | 1 | 6 MB | 6 MB | 1 | 16019 | +| kafka | the frozen tier | 1 | 205 MB | 28 MB | 1 | 76844 | +| kube-proxy | the frozen tier | 1 | 12 MB | 6 MB | 1 | 17460 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1805 | +| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5650 | +| opensearch | the frozen tier | 1 | 16 MB | 16 MB | 1 | 44976 | +| opentelemetry-collector | the frozen tier | 1 | 826 MB | 94 MB | 1 | 260895 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 613 | +| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5714 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5770 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 99 | +| recommendation | the frozen tier | 1 | 27 MB | 1 MB | 1 | 2881 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7058 | +| wait-for-kafka | the frozen tier | 1 | 5 MB | 0 MB | 1 | 1356 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1159 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3423 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/run.json new file mode 100644 index 0000000..356eead --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:20:12.384Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 158 MB within 2 hours, with 1.1 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/totals.csv new file mode 100644 index 0000000..4aaf2df --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002009Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,7844::budget run +ad,6038::budget run +aws-eks-nodeagent,150::budget run +aws-node,713::budget run +aws-vpc-cni-init,907::budget run +cart,4146::budget run +coredns,4793::budget run +currency,214::budget run +email,624::budget run +flagd,2015::budget run +flagd-ui,356::budget run +fluentd-10x,14235::budget run +fraud-detection,532::budget run +frontend,34477::budget run +frontend-proxy,23472::budget run +grafana,258503::budget run +grafana-sc-alerts,1560::budget run +grafana-sc-dashboard,2206::budget run +grafana-sc-datasources,1581::budget run +image-provider,3965::budget run +init-config,1310::budget run +jaeger,16019::budget run +kafka,76844::budget run +kube-proxy,17460::budget run +load-generator,1805::budget run +metrics-server,5650::budget run +opensearch,44976::budget run +opentelemetry-collector,260895::budget run +payment,613::budget run +postgresql,5714::budget run +prometheus-server,5770::budget run +quote,99::budget run +recommendation,2881::budget run +valkey-cart,7058::budget run +wait-for-kafka,1356::budget run +wait-for-valkey-cart,1159::budget run +*,3423::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/commit.diff new file mode 100644 index 0000000..5927557 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/commit.diff @@ -0,0 +1,182 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 4aaf2df..abd84ef 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,7844::budget run +-ad,6038::budget run +-aws-eks-nodeagent,150::budget run +-aws-node,713::budget run +-aws-vpc-cni-init,907::budget run +-cart,4146::budget run +-coredns,4793::budget run +-currency,214::budget run +-email,624::budget run +-flagd,2015::budget run +-flagd-ui,356::budget run +-fluentd-10x,14235::budget run +-fraud-detection,532::budget run +-frontend,34477::budget run +-frontend-proxy,23472::budget run +-grafana,258503::budget run +-grafana-sc-alerts,1560::budget run +-grafana-sc-dashboard,2206::budget run +-grafana-sc-datasources,1581::budget run +-image-provider,3965::budget run +-init-config,1310::budget run +-jaeger,16019::budget run +-kafka,76844::budget run +-kube-proxy,17460::budget run +-load-generator,1805::budget run +-metrics-server,5650::budget run +-opensearch,44976::budget run +-opentelemetry-collector,260895::budget run +-payment,613::budget run +-postgresql,5714::budget run +-prometheus-server,5770::budget run +-quote,99::budget run +-recommendation,2881::budget run +-valkey-cart,7058::budget run +-wait-for-kafka,1356::budget run +-wait-for-valkey-cart,1159::budget run +-*,3423::budget run ++accounting,7010::budget run ++ad,5955::budget run ++aws-eks-nodeagent,134::budget run ++aws-node,637::budget run ++aws-vpc-cni-init,811::budget run ++cart,4033::budget run ++coredns,4421::budget run ++currency,191::budget run ++email,558::budget run ++flagd,1801::budget run ++flagd-ui,318::budget run ++fluentd-10x,14184::budget run ++fraud-detection,476::budget run ++frontend,34338::budget run ++frontend-proxy,20975::budget run ++grafana,232242::budget run ++grafana-sc-alerts,1394::budget run ++grafana-sc-dashboard,1971::budget run ++grafana-sc-datasources,1413::budget run ++image-provider,3543::budget run ++init-config,1171::budget run ++jaeger,14315::budget run ++kafka,72784::budget run ++kube-proxy,16109::budget run ++load-generator,1613::budget run ++metrics-server,5049::budget run ++opensearch,41262::budget run ++opentelemetry-collector,236874::budget run ++payment,548::budget run ++postgresql,5106::budget run ++prometheus-server,5590::budget run ++quote,88::budget run ++recommendation,2840::budget run ++valkey-cart,7271::budget run ++wait-for-kafka,1322::budget run ++wait-for-valkey-cart,1035::budget run ++*,3192::budget run +diff --git a/reports/latest.md b/reports/latest.md +index b1c9d23..511915a 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,54 +1,54 @@ +-# Hot-tier run: 2026-10-26T00:20:09.000Z ++# Hot-tier run: 2026-10-26T00:21:09.000Z + +-Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 158 MB within 2 hours, with 1.1 GB a day of counted lines moved to the frozen tier. ++Hot tier: 9 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 111 MB within 2 hours, with 1.1 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 1 indices; its tier_down lines not yet moved: 6 MB in 1. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 1 indices; its tier_down lines not yet moved: 4 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 2.44 bytes of store kept, 1.08 as tier_down (measured on the indices on hot). ++- Each counted byte takes 2.34 bytes of store kept, 0.60 as tier_down (measured on the indices on hot). + - The Receiver's statements send 1.4 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 294 MB a day on hot whatever the limits; the size holds down to that floor. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 270 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 294 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 158 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 270 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 111 MB. Raise the size, move the noisiest services to offload, or lower the sample. + - Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7844 | +-| ad | the frozen tier | 1 | 39 MB | 2 MB | 1 | 6038 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 150 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 713 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 907 | +-| cart | the frozen tier | 1 | 88 MB | 1 MB | 1 | 4146 | +-| coredns | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4793 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 214 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 624 | +-| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2015 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 356 | +-| fluentd-10x | the frozen tier | 1 | 14 MB | 5 MB | 1 | 14235 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 532 | +-| frontend | the frozen tier | 1 | 19 MB | 12 MB | 1 | 34477 | +-| frontend-proxy | the frozen tier | 1 | 8 MB | 8 MB | 1 | 23472 | +-| grafana | the frozen tier | 1 | 96 MB | 93 MB | 1 | 258503 | +-| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1560 | +-| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2206 | +-| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1581 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3965 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1310 | +-| jaeger | the frozen tier | 1 | 6 MB | 6 MB | 1 | 16019 | +-| kafka | the frozen tier | 1 | 205 MB | 28 MB | 1 | 76844 | +-| kube-proxy | the frozen tier | 1 | 12 MB | 6 MB | 1 | 17460 | +-| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1805 | +-| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5650 | +-| opensearch | the frozen tier | 1 | 16 MB | 16 MB | 1 | 44976 | +-| opentelemetry-collector | the frozen tier | 1 | 826 MB | 94 MB | 1 | 260895 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 613 | +-| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5714 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5770 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 99 | +-| recommendation | the frozen tier | 1 | 27 MB | 1 MB | 1 | 2881 | +-| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7058 | +-| wait-for-kafka | the frozen tier | 1 | 5 MB | 0 MB | 1 | 1356 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1159 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3423 | ++| accounting | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7010 | ++| ad | the frozen tier | 1 | 40 MB | 2 MB | 1 | 5955 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 134 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 637 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 811 | ++| cart | the frozen tier | 1 | 94 MB | 1 MB | 1 | 4033 | ++| coredns | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4421 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 191 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 558 | ++| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1801 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 318 | ++| fluentd-10x | the frozen tier | 1 | 14 MB | 5 MB | 1 | 14184 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 476 | ++| frontend | the frozen tier | 1 | 21 MB | 12 MB | 1 | 34338 | ++| frontend-proxy | the frozen tier | 1 | 8 MB | 8 MB | 1 | 20975 | ++| grafana | the frozen tier | 1 | 87 MB | 84 MB | 1 | 232242 | ++| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1394 | ++| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1971 | ++| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1413 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3543 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1171 | ++| jaeger | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14315 | ++| kafka | the frozen tier | 1 | 213 MB | 26 MB | 1 | 72784 | ++| kube-proxy | the frozen tier | 1 | 11 MB | 6 MB | 1 | 16109 | ++| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1613 | ++| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5049 | ++| opensearch | the frozen tier | 1 | 15 MB | 15 MB | 1 | 41262 | ++| opentelemetry-collector | the frozen tier | 1 | 861 MB | 85 MB | 1 | 236874 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 548 | ++| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5106 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5590 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 88 | ++| recommendation | the frozen tier | 1 | 29 MB | 1 MB | 1 | 2840 | ++| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7271 | ++| wait-for-kafka | the frozen tier | 1 | 5 MB | 0 MB | 1 | 1322 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1035 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3192 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/commit.json new file mode 100644 index 0000000..d5993f9 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/commit.json @@ -0,0 +1 @@ +{"before":"9a2ac1b0feb8702e9586825b0ce1a52f4808dc31","after":"bdd584edda39a3913bf210dd96714051a38b304b","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/latest.md new file mode 100644 index 0000000..511915a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/latest.md @@ -0,0 +1,54 @@ +# Hot-tier run: 2026-10-26T00:21:09.000Z + +Hot tier: 9 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 111 MB within 2 hours, with 1.1 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 1 indices; its tier_down lines not yet moved: 4 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 2.34 bytes of store kept, 0.60 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.4 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 270 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 270 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 111 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7010 | +| ad | the frozen tier | 1 | 40 MB | 2 MB | 1 | 5955 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 134 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 637 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 811 | +| cart | the frozen tier | 1 | 94 MB | 1 MB | 1 | 4033 | +| coredns | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4421 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 191 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 558 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1801 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 318 | +| fluentd-10x | the frozen tier | 1 | 14 MB | 5 MB | 1 | 14184 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 476 | +| frontend | the frozen tier | 1 | 21 MB | 12 MB | 1 | 34338 | +| frontend-proxy | the frozen tier | 1 | 8 MB | 8 MB | 1 | 20975 | +| grafana | the frozen tier | 1 | 87 MB | 84 MB | 1 | 232242 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1394 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1971 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1413 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3543 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1171 | +| jaeger | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14315 | +| kafka | the frozen tier | 1 | 213 MB | 26 MB | 1 | 72784 | +| kube-proxy | the frozen tier | 1 | 11 MB | 6 MB | 1 | 16109 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1613 | +| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5049 | +| opensearch | the frozen tier | 1 | 15 MB | 15 MB | 1 | 41262 | +| opentelemetry-collector | the frozen tier | 1 | 861 MB | 85 MB | 1 | 236874 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 548 | +| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5106 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5590 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 88 | +| recommendation | the frozen tier | 1 | 29 MB | 1 MB | 1 | 2840 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7271 | +| wait-for-kafka | the frozen tier | 1 | 5 MB | 0 MB | 1 | 1322 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1035 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3192 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/report.txt new file mode 100644 index 0000000..2562da0 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/report.txt @@ -0,0 +1,63 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 9 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 111 MB within 2 hours, with 1.1 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 1 indices; its tier_down lines not yet moved: 4 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 2.34 bytes of store kept, 0.60 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1.4 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 270 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:21:09.000Z + +Hot tier: 9 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 111 MB within 2 hours, with 1.1 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 1 indices; its tier_down lines not yet moved: 4 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 2.34 bytes of store kept, 0.60 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.4 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 270 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 270 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 111 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7010 | +| ad | the frozen tier | 1 | 40 MB | 2 MB | 1 | 5955 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 134 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 637 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 811 | +| cart | the frozen tier | 1 | 94 MB | 1 MB | 1 | 4033 | +| coredns | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4421 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 191 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 558 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1801 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 318 | +| fluentd-10x | the frozen tier | 1 | 14 MB | 5 MB | 1 | 14184 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 476 | +| frontend | the frozen tier | 1 | 21 MB | 12 MB | 1 | 34338 | +| frontend-proxy | the frozen tier | 1 | 8 MB | 8 MB | 1 | 20975 | +| grafana | the frozen tier | 1 | 87 MB | 84 MB | 1 | 232242 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1394 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1971 | +| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1413 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3543 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1171 | +| jaeger | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14315 | +| kafka | the frozen tier | 1 | 213 MB | 26 MB | 1 | 72784 | +| kube-proxy | the frozen tier | 1 | 11 MB | 6 MB | 1 | 16109 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1613 | +| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5049 | +| opensearch | the frozen tier | 1 | 15 MB | 15 MB | 1 | 41262 | +| opentelemetry-collector | the frozen tier | 1 | 861 MB | 85 MB | 1 | 236874 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 548 | +| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5106 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5590 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 88 | +| recommendation | the frozen tier | 1 | 29 MB | 1 MB | 1 | 2840 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7271 | +| wait-for-kafka | the frozen tier | 1 | 5 MB | 0 MB | 1 | 1322 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1035 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3192 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/run.json new file mode 100644 index 0000000..a99a062 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:21:12.345Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 9 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 111 MB within 2 hours, with 1.1 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/totals.csv new file mode 100644 index 0000000..abd84ef --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002108Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,7010::budget run +ad,5955::budget run +aws-eks-nodeagent,134::budget run +aws-node,637::budget run +aws-vpc-cni-init,811::budget run +cart,4033::budget run +coredns,4421::budget run +currency,191::budget run +email,558::budget run +flagd,1801::budget run +flagd-ui,318::budget run +fluentd-10x,14184::budget run +fraud-detection,476::budget run +frontend,34338::budget run +frontend-proxy,20975::budget run +grafana,232242::budget run +grafana-sc-alerts,1394::budget run +grafana-sc-dashboard,1971::budget run +grafana-sc-datasources,1413::budget run +image-provider,3543::budget run +init-config,1171::budget run +jaeger,14315::budget run +kafka,72784::budget run +kube-proxy,16109::budget run +load-generator,1613::budget run +metrics-server,5049::budget run +opensearch,41262::budget run +opentelemetry-collector,236874::budget run +payment,548::budget run +postgresql,5106::budget run +prometheus-server,5590::budget run +quote,88::budget run +recommendation,2840::budget run +valkey-cart,7271::budget run +wait-for-kafka,1322::budget run +wait-for-valkey-cart,1035::budget run +*,3192::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/commit.diff new file mode 100644 index 0000000..78f387b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/commit.diff @@ -0,0 +1,183 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index abd84ef..e999cd0 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,7010::budget run +-ad,5955::budget run +-aws-eks-nodeagent,134::budget run +-aws-node,637::budget run +-aws-vpc-cni-init,811::budget run +-cart,4033::budget run +-coredns,4421::budget run +-currency,191::budget run +-email,558::budget run +-flagd,1801::budget run +-flagd-ui,318::budget run +-fluentd-10x,14184::budget run +-fraud-detection,476::budget run +-frontend,34338::budget run +-frontend-proxy,20975::budget run +-grafana,232242::budget run +-grafana-sc-alerts,1394::budget run +-grafana-sc-dashboard,1971::budget run +-grafana-sc-datasources,1413::budget run +-image-provider,3543::budget run +-init-config,1171::budget run +-jaeger,14315::budget run +-kafka,72784::budget run +-kube-proxy,16109::budget run +-load-generator,1613::budget run +-metrics-server,5049::budget run +-opensearch,41262::budget run +-opentelemetry-collector,236874::budget run +-payment,548::budget run +-postgresql,5106::budget run +-prometheus-server,5590::budget run +-quote,88::budget run +-recommendation,2840::budget run +-valkey-cart,7271::budget run +-wait-for-kafka,1322::budget run +-wait-for-valkey-cart,1035::budget run +-*,3192::budget run ++accounting,6346::budget run ++ad,5889::budget run ++aws-eks-nodeagent,121::budget run ++aws-node,576::budget run ++aws-vpc-cni-init,734::budget run ++cart,3942::budget run ++coredns,4125::budget run ++currency,173::budget run ++email,505::budget run ++flagd,1630::budget run ++flagd-ui,288::budget run ++fluentd-10x,14024::budget run ++fraud-detection,430::budget run ++frontend,34227::budget run ++frontend-proxy,18989::budget run ++grafana,211560::budget run ++grafana-sc-alerts,1262::budget run ++grafana-sc-dashboard,1784::budget run ++grafana-sc-datasources,1279::budget run ++image-provider,3208::budget run ++init-config,1060::budget run ++jaeger,12960::budget run ++kafka,69554::budget run ++kube-proxy,15034::budget run ++load-generator,1460::budget run ++metrics-server,4571::budget run ++opensearch,37911::budget run ++opentelemetry-collector,217765::budget run ++payment,496::budget run ++postgresql,4623::budget run ++prometheus-server,5320::budget run ++quote,80::budget run ++recommendation,2808::budget run ++valkey-cart,7193::budget run ++wait-for-kafka,1295::budget run ++wait-for-valkey-cart,937::budget run ++*,3008::budget run +diff --git a/reports/latest.md b/reports/latest.md +index 511915a..c09b236 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,54 +1,54 @@ +-# Hot-tier run: 2026-10-26T00:21:09.000Z ++# Hot-tier run: 2026-10-26T00:22:08.000Z + +-Hot tier: 9 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 111 MB within 2 hours, with 1.1 GB a day of counted lines moved to the frozen tier. ++Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 146 MB within 2 hours, with 1.2 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 1 indices; its tier_down lines not yet moved: 4 MB in 1. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 2 indices; its tier_down lines not yet moved: 10 MB in 2. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 2.34 bytes of store kept, 0.60 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 1.4 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 270 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.17 bytes of store kept, 1.20 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 250 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 270 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 111 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 250 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 146 MB. Raise the size, move the noisiest services to offload, or lower the sample. + - Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7010 | +-| ad | the frozen tier | 1 | 40 MB | 2 MB | 1 | 5955 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 134 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 637 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 811 | +-| cart | the frozen tier | 1 | 94 MB | 1 MB | 1 | 4033 | +-| coredns | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4421 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 191 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 558 | +-| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1801 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 318 | +-| fluentd-10x | the frozen tier | 1 | 14 MB | 5 MB | 1 | 14184 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 476 | +-| frontend | the frozen tier | 1 | 21 MB | 12 MB | 1 | 34338 | +-| frontend-proxy | the frozen tier | 1 | 8 MB | 8 MB | 1 | 20975 | +-| grafana | the frozen tier | 1 | 87 MB | 84 MB | 1 | 232242 | +-| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1394 | +-| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1971 | +-| grafana-sc-datasources | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1413 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3543 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1171 | +-| jaeger | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14315 | +-| kafka | the frozen tier | 1 | 213 MB | 26 MB | 1 | 72784 | +-| kube-proxy | the frozen tier | 1 | 11 MB | 6 MB | 1 | 16109 | +-| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1613 | +-| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5049 | +-| opensearch | the frozen tier | 1 | 15 MB | 15 MB | 1 | 41262 | +-| opentelemetry-collector | the frozen tier | 1 | 861 MB | 85 MB | 1 | 236874 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 548 | +-| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5106 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5590 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 88 | +-| recommendation | the frozen tier | 1 | 29 MB | 1 MB | 1 | 2840 | +-| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7271 | +-| wait-for-kafka | the frozen tier | 1 | 5 MB | 0 MB | 1 | 1322 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1035 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3192 | ++| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6346 | ++| ad | the frozen tier | 1 | 43 MB | 2 MB | 1 | 5889 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 121 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 576 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 734 | ++| cart | the frozen tier | 1 | 98 MB | 1 MB | 1 | 3942 | ++| coredns | the frozen tier | 1 | 2 MB | 1 MB | 1 | 4125 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 173 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 505 | ++| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1630 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 288 | ++| fluentd-10x | the frozen tier | 1 | 14 MB | 5 MB | 1 | 14024 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 430 | ++| frontend | the frozen tier | 1 | 22 MB | 12 MB | 1 | 34227 | ++| frontend-proxy | the frozen tier | 1 | 7 MB | 7 MB | 1 | 18989 | ++| grafana | the frozen tier | 1 | 79 MB | 76 MB | 1 | 211560 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1262 | ++| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1784 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1279 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3208 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1060 | ++| jaeger | the frozen tier | 1 | 5 MB | 5 MB | 1 | 12960 | ++| kafka | the frozen tier | 1 | 220 MB | 25 MB | 1 | 69554 | ++| kube-proxy | the frozen tier | 1 | 10 MB | 5 MB | 1 | 15034 | ++| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1460 | ++| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4571 | ++| opensearch | the frozen tier | 1 | 14 MB | 14 MB | 1 | 37911 | ++| opentelemetry-collector | the frozen tier | 1 | 897 MB | 78 MB | 1 | 217765 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 496 | ++| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4623 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5320 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 80 | ++| recommendation | the frozen tier | 1 | 31 MB | 1 MB | 1 | 2808 | ++| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7193 | ++| wait-for-kafka | the frozen tier | 1 | 4 MB | 0 MB | 1 | 1295 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 937 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3008 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/commit.json new file mode 100644 index 0000000..564bb35 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/commit.json @@ -0,0 +1 @@ +{"before":"bdd584edda39a3913bf210dd96714051a38b304b","after":"9f9729f27515d4a4921495cafd93544e476a69e0","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/latest.md new file mode 100644 index 0000000..c09b236 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/latest.md @@ -0,0 +1,54 @@ +# Hot-tier run: 2026-10-26T00:22:08.000Z + +Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 146 MB within 2 hours, with 1.2 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 2 indices; its tier_down lines not yet moved: 10 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.17 bytes of store kept, 1.20 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 250 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 250 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 146 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6346 | +| ad | the frozen tier | 1 | 43 MB | 2 MB | 1 | 5889 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 121 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 576 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 734 | +| cart | the frozen tier | 1 | 98 MB | 1 MB | 1 | 3942 | +| coredns | the frozen tier | 1 | 2 MB | 1 MB | 1 | 4125 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 173 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 505 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1630 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 288 | +| fluentd-10x | the frozen tier | 1 | 14 MB | 5 MB | 1 | 14024 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 430 | +| frontend | the frozen tier | 1 | 22 MB | 12 MB | 1 | 34227 | +| frontend-proxy | the frozen tier | 1 | 7 MB | 7 MB | 1 | 18989 | +| grafana | the frozen tier | 1 | 79 MB | 76 MB | 1 | 211560 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1262 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1784 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1279 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3208 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1060 | +| jaeger | the frozen tier | 1 | 5 MB | 5 MB | 1 | 12960 | +| kafka | the frozen tier | 1 | 220 MB | 25 MB | 1 | 69554 | +| kube-proxy | the frozen tier | 1 | 10 MB | 5 MB | 1 | 15034 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1460 | +| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4571 | +| opensearch | the frozen tier | 1 | 14 MB | 14 MB | 1 | 37911 | +| opentelemetry-collector | the frozen tier | 1 | 897 MB | 78 MB | 1 | 217765 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 496 | +| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4623 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5320 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 80 | +| recommendation | the frozen tier | 1 | 31 MB | 1 MB | 1 | 2808 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7193 | +| wait-for-kafka | the frozen tier | 1 | 4 MB | 0 MB | 1 | 1295 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 937 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3008 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/report.txt new file mode 100644 index 0000000..21c65c2 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/report.txt @@ -0,0 +1,63 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 146 MB within 2 hours, with 1.2 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 2 indices; its tier_down lines not yet moved: 10 MB in 2. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.17 bytes of store kept, 1.20 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 250 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:22:08.000Z + +Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 146 MB within 2 hours, with 1.2 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 2 indices; its tier_down lines not yet moved: 10 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.17 bytes of store kept, 1.20 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 250 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 250 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 146 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6346 | +| ad | the frozen tier | 1 | 43 MB | 2 MB | 1 | 5889 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 121 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 576 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 734 | +| cart | the frozen tier | 1 | 98 MB | 1 MB | 1 | 3942 | +| coredns | the frozen tier | 1 | 2 MB | 1 MB | 1 | 4125 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 173 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 505 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1630 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 288 | +| fluentd-10x | the frozen tier | 1 | 14 MB | 5 MB | 1 | 14024 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 430 | +| frontend | the frozen tier | 1 | 22 MB | 12 MB | 1 | 34227 | +| frontend-proxy | the frozen tier | 1 | 7 MB | 7 MB | 1 | 18989 | +| grafana | the frozen tier | 1 | 79 MB | 76 MB | 1 | 211560 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1262 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1784 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1279 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3208 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1060 | +| jaeger | the frozen tier | 1 | 5 MB | 5 MB | 1 | 12960 | +| kafka | the frozen tier | 1 | 220 MB | 25 MB | 1 | 69554 | +| kube-proxy | the frozen tier | 1 | 10 MB | 5 MB | 1 | 15034 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1460 | +| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4571 | +| opensearch | the frozen tier | 1 | 14 MB | 14 MB | 1 | 37911 | +| opentelemetry-collector | the frozen tier | 1 | 897 MB | 78 MB | 1 | 217765 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 496 | +| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4623 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5320 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 80 | +| recommendation | the frozen tier | 1 | 31 MB | 1 MB | 1 | 2808 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7193 | +| wait-for-kafka | the frozen tier | 1 | 4 MB | 0 MB | 1 | 1295 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 937 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3008 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/run.json new file mode 100644 index 0000000..73a1d7f --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:22:11.706Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 146 MB within 2 hours, with 1.2 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/totals.csv new file mode 100644 index 0000000..e999cd0 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002208Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,6346::budget run +ad,5889::budget run +aws-eks-nodeagent,121::budget run +aws-node,576::budget run +aws-vpc-cni-init,734::budget run +cart,3942::budget run +coredns,4125::budget run +currency,173::budget run +email,505::budget run +flagd,1630::budget run +flagd-ui,288::budget run +fluentd-10x,14024::budget run +fraud-detection,430::budget run +frontend,34227::budget run +frontend-proxy,18989::budget run +grafana,211560::budget run +grafana-sc-alerts,1262::budget run +grafana-sc-dashboard,1784::budget run +grafana-sc-datasources,1279::budget run +image-provider,3208::budget run +init-config,1060::budget run +jaeger,12960::budget run +kafka,69554::budget run +kube-proxy,15034::budget run +load-generator,1460::budget run +metrics-server,4571::budget run +opensearch,37911::budget run +opentelemetry-collector,217765::budget run +payment,496::budget run +postgresql,4623::budget run +prometheus-server,5320::budget run +quote,80::budget run +recommendation,2808::budget run +valkey-cart,7193::budget run +wait-for-kafka,1295::budget run +wait-for-valkey-cart,937::budget run +*,3008::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/commit.diff new file mode 100644 index 0000000..3e7eef2 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/commit.diff @@ -0,0 +1,182 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index e999cd0..76eabb7 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,6346::budget run +-ad,5889::budget run +-aws-eks-nodeagent,121::budget run +-aws-node,576::budget run +-aws-vpc-cni-init,734::budget run +-cart,3942::budget run +-coredns,4125::budget run +-currency,173::budget run +-email,505::budget run +-flagd,1630::budget run +-flagd-ui,288::budget run +-fluentd-10x,14024::budget run +-fraud-detection,430::budget run +-frontend,34227::budget run +-frontend-proxy,18989::budget run +-grafana,211560::budget run +-grafana-sc-alerts,1262::budget run +-grafana-sc-dashboard,1784::budget run +-grafana-sc-datasources,1279::budget run +-image-provider,3208::budget run +-init-config,1060::budget run +-jaeger,12960::budget run +-kafka,69554::budget run +-kube-proxy,15034::budget run +-load-generator,1460::budget run +-metrics-server,4571::budget run +-opensearch,37911::budget run +-opentelemetry-collector,217765::budget run +-payment,496::budget run +-postgresql,4623::budget run +-prometheus-server,5320::budget run +-quote,80::budget run +-recommendation,2808::budget run +-valkey-cart,7193::budget run +-wait-for-kafka,1295::budget run +-wait-for-valkey-cart,937::budget run +-*,3008::budget run ++accounting,5747::budget run ++ad,5829::budget run ++aws-eks-nodeagent,110::budget run ++aws-node,522::budget run ++aws-vpc-cni-init,665::budget run ++cart,3862::budget run ++coredns,3858::budget run ++currency,156::budget run ++email,457::budget run ++flagd,1476::budget run ++flagd-ui,261::budget run ++fluentd-10x,13245::budget run ++fraud-detection,390::budget run ++frontend,34127::budget run ++frontend-proxy,17195::budget run ++grafana,192810::budget run ++grafana-sc-alerts,1142::budget run ++grafana-sc-dashboard,1616::budget run ++grafana-sc-datasources,1158::budget run ++image-provider,2905::budget run ++init-config,960::budget run ++jaeger,11735::budget run ++kafka,66640::budget run ++kube-proxy,14063::budget run ++load-generator,1322::budget run ++metrics-server,4139::budget run ++opensearch,34702::budget run ++opentelemetry-collector,200507::budget run ++payment,449::budget run ++postgresql,4186::budget run ++prometheus-server,4820::budget run ++quote,72::budget run ++recommendation,2779::budget run ++valkey-cart,7123::budget run ++wait-for-kafka,1271::budget run ++wait-for-valkey-cart,849::budget run ++*,2842::budget run +diff --git a/reports/latest.md b/reports/latest.md +index c09b236..c1f02fa 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,54 +1,54 @@ +-# Hot-tier run: 2026-10-26T00:22:08.000Z ++# Hot-tier run: 2026-10-26T00:23:13.000Z + +-Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 146 MB within 2 hours, with 1.2 GB a day of counted lines moved to the frozen tier. ++Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 138 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 2 indices; its tier_down lines not yet moved: 10 MB in 2. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 11 MB in 2. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.17 bytes of store kept, 1.20 as tier_down (measured on the indices on hot). ++- Each counted byte takes 1.20 bytes of store kept, 1.10 as tier_down (measured on the indices on hot). + - The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 250 MB a day on hot whatever the limits; the size holds down to that floor. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 232 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 250 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 146 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 232 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 138 MB. Raise the size, move the noisiest services to offload, or lower the sample. + - Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 6346 | +-| ad | the frozen tier | 1 | 43 MB | 2 MB | 1 | 5889 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 121 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 576 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 734 | +-| cart | the frozen tier | 1 | 98 MB | 1 MB | 1 | 3942 | +-| coredns | the frozen tier | 1 | 2 MB | 1 MB | 1 | 4125 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 173 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 505 | +-| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1630 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 288 | +-| fluentd-10x | the frozen tier | 1 | 14 MB | 5 MB | 1 | 14024 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 430 | +-| frontend | the frozen tier | 1 | 22 MB | 12 MB | 1 | 34227 | +-| frontend-proxy | the frozen tier | 1 | 7 MB | 7 MB | 1 | 18989 | +-| grafana | the frozen tier | 1 | 79 MB | 76 MB | 1 | 211560 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1262 | +-| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1784 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1279 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3208 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1060 | +-| jaeger | the frozen tier | 1 | 5 MB | 5 MB | 1 | 12960 | +-| kafka | the frozen tier | 1 | 220 MB | 25 MB | 1 | 69554 | +-| kube-proxy | the frozen tier | 1 | 10 MB | 5 MB | 1 | 15034 | +-| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1460 | +-| metrics-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4571 | +-| opensearch | the frozen tier | 1 | 14 MB | 14 MB | 1 | 37911 | +-| opentelemetry-collector | the frozen tier | 1 | 897 MB | 78 MB | 1 | 217765 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 496 | +-| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4623 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5320 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 80 | +-| recommendation | the frozen tier | 1 | 31 MB | 1 MB | 1 | 2808 | +-| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7193 | +-| wait-for-kafka | the frozen tier | 1 | 4 MB | 0 MB | 1 | 1295 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 937 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 3008 | ++| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5747 | ++| ad | the frozen tier | 1 | 45 MB | 2 MB | 1 | 5829 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 110 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 522 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 665 | ++| cart | the frozen tier | 1 | 103 MB | 1 MB | 1 | 3862 | ++| coredns | the frozen tier | 1 | 2 MB | 1 MB | 1 | 3858 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 156 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 457 | ++| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1476 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 261 | ++| fluentd-10x | the frozen tier | 1 | 13 MB | 5 MB | 1 | 13245 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 390 | ++| frontend | the frozen tier | 1 | 23 MB | 12 MB | 1 | 34127 | ++| frontend-proxy | the frozen tier | 1 | 6 MB | 6 MB | 1 | 17195 | ++| grafana | the frozen tier | 1 | 73 MB | 69 MB | 1 | 192810 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1142 | ++| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1616 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1158 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2905 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 960 | ++| jaeger | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11735 | ++| kafka | the frozen tier | 1 | 225 MB | 24 MB | 1 | 66640 | ++| kube-proxy | the frozen tier | 1 | 9 MB | 5 MB | 1 | 14063 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1322 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4139 | ++| opensearch | the frozen tier | 1 | 13 MB | 12 MB | 1 | 34702 | ++| opentelemetry-collector | the frozen tier | 1 | 923 MB | 72 MB | 1 | 200507 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 449 | ++| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4186 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4820 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 72 | ++| recommendation | the frozen tier | 1 | 33 MB | 1 MB | 1 | 2779 | ++| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7123 | ++| wait-for-kafka | the frozen tier | 1 | 4 MB | 0 MB | 1 | 1271 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 849 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2842 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/commit.json new file mode 100644 index 0000000..bf0a8af --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/commit.json @@ -0,0 +1 @@ +{"before":"9f9729f27515d4a4921495cafd93544e476a69e0","after":"f2bec2d3ce3661d12bbf371e2342fc6f93e6b593","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/latest.md new file mode 100644 index 0000000..c1f02fa --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/latest.md @@ -0,0 +1,54 @@ +# Hot-tier run: 2026-10-26T00:23:13.000Z + +Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 138 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 11 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.20 bytes of store kept, 1.10 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 232 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 232 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 138 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5747 | +| ad | the frozen tier | 1 | 45 MB | 2 MB | 1 | 5829 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 110 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 522 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 665 | +| cart | the frozen tier | 1 | 103 MB | 1 MB | 1 | 3862 | +| coredns | the frozen tier | 1 | 2 MB | 1 MB | 1 | 3858 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 156 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 457 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1476 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 261 | +| fluentd-10x | the frozen tier | 1 | 13 MB | 5 MB | 1 | 13245 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 390 | +| frontend | the frozen tier | 1 | 23 MB | 12 MB | 1 | 34127 | +| frontend-proxy | the frozen tier | 1 | 6 MB | 6 MB | 1 | 17195 | +| grafana | the frozen tier | 1 | 73 MB | 69 MB | 1 | 192810 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1142 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1616 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1158 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2905 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 960 | +| jaeger | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11735 | +| kafka | the frozen tier | 1 | 225 MB | 24 MB | 1 | 66640 | +| kube-proxy | the frozen tier | 1 | 9 MB | 5 MB | 1 | 14063 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1322 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4139 | +| opensearch | the frozen tier | 1 | 13 MB | 12 MB | 1 | 34702 | +| opentelemetry-collector | the frozen tier | 1 | 923 MB | 72 MB | 1 | 200507 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 449 | +| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4186 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4820 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 72 | +| recommendation | the frozen tier | 1 | 33 MB | 1 MB | 1 | 2779 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7123 | +| wait-for-kafka | the frozen tier | 1 | 4 MB | 0 MB | 1 | 1271 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 849 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2842 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/report.txt new file mode 100644 index 0000000..9cbaea2 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/report.txt @@ -0,0 +1,63 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 138 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 11 MB in 2. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.20 bytes of store kept, 1.10 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 232 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:23:13.000Z + +Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 138 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 11 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.20 bytes of store kept, 1.10 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 232 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 232 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 138 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5747 | +| ad | the frozen tier | 1 | 45 MB | 2 MB | 1 | 5829 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 110 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 522 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 665 | +| cart | the frozen tier | 1 | 103 MB | 1 MB | 1 | 3862 | +| coredns | the frozen tier | 1 | 2 MB | 1 MB | 1 | 3858 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 156 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 457 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1476 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 261 | +| fluentd-10x | the frozen tier | 1 | 13 MB | 5 MB | 1 | 13245 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 390 | +| frontend | the frozen tier | 1 | 23 MB | 12 MB | 1 | 34127 | +| frontend-proxy | the frozen tier | 1 | 6 MB | 6 MB | 1 | 17195 | +| grafana | the frozen tier | 1 | 73 MB | 69 MB | 1 | 192810 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1142 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1616 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1158 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2905 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 960 | +| jaeger | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11735 | +| kafka | the frozen tier | 1 | 225 MB | 24 MB | 1 | 66640 | +| kube-proxy | the frozen tier | 1 | 9 MB | 5 MB | 1 | 14063 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1322 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4139 | +| opensearch | the frozen tier | 1 | 13 MB | 12 MB | 1 | 34702 | +| opentelemetry-collector | the frozen tier | 1 | 923 MB | 72 MB | 1 | 200507 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 449 | +| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4186 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4820 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 72 | +| recommendation | the frozen tier | 1 | 33 MB | 1 MB | 1 | 2779 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7123 | +| wait-for-kafka | the frozen tier | 1 | 4 MB | 0 MB | 1 | 1271 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 849 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2842 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/run.json new file mode 100644 index 0000000..8b3be11 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:23:16.128Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 138 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/totals.csv new file mode 100644 index 0000000..76eabb7 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002313Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,5747::budget run +ad,5829::budget run +aws-eks-nodeagent,110::budget run +aws-node,522::budget run +aws-vpc-cni-init,665::budget run +cart,3862::budget run +coredns,3858::budget run +currency,156::budget run +email,457::budget run +flagd,1476::budget run +flagd-ui,261::budget run +fluentd-10x,13245::budget run +fraud-detection,390::budget run +frontend,34127::budget run +frontend-proxy,17195::budget run +grafana,192810::budget run +grafana-sc-alerts,1142::budget run +grafana-sc-dashboard,1616::budget run +grafana-sc-datasources,1158::budget run +image-provider,2905::budget run +init-config,960::budget run +jaeger,11735::budget run +kafka,66640::budget run +kube-proxy,14063::budget run +load-generator,1322::budget run +metrics-server,4139::budget run +opensearch,34702::budget run +opentelemetry-collector,200507::budget run +payment,449::budget run +postgresql,4186::budget run +prometheus-server,4820::budget run +quote,72::budget run +recommendation,2779::budget run +valkey-cart,7123::budget run +wait-for-kafka,1271::budget run +wait-for-valkey-cart,849::budget run +*,2842::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/commit.diff new file mode 100644 index 0000000..5bc14e3 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/commit.diff @@ -0,0 +1,182 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 76eabb7..b85be66 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,5747::budget run +-ad,5829::budget run +-aws-eks-nodeagent,110::budget run +-aws-node,522::budget run +-aws-vpc-cni-init,665::budget run +-cart,3862::budget run +-coredns,3858::budget run +-currency,156::budget run +-email,457::budget run +-flagd,1476::budget run +-flagd-ui,261::budget run +-fluentd-10x,13245::budget run +-fraud-detection,390::budget run +-frontend,34127::budget run +-frontend-proxy,17195::budget run +-grafana,192810::budget run +-grafana-sc-alerts,1142::budget run +-grafana-sc-dashboard,1616::budget run +-grafana-sc-datasources,1158::budget run +-image-provider,2905::budget run +-init-config,960::budget run +-jaeger,11735::budget run +-kafka,66640::budget run +-kube-proxy,14063::budget run +-load-generator,1322::budget run +-metrics-server,4139::budget run +-opensearch,34702::budget run +-opentelemetry-collector,200507::budget run +-payment,449::budget run +-postgresql,4186::budget run +-prometheus-server,4820::budget run +-quote,72::budget run +-recommendation,2779::budget run +-valkey-cart,7123::budget run +-wait-for-kafka,1271::budget run +-wait-for-valkey-cart,849::budget run +-*,2842::budget run ++accounting,5293::budget run ++ad,5784::budget run ++aws-eks-nodeagent,101::budget run ++aws-node,481::budget run ++aws-vpc-cni-init,612::budget run ++cart,3801::budget run ++coredns,3655::budget run ++currency,144::budget run ++email,421::budget run ++flagd,1360::budget run ++flagd-ui,240::budget run ++fluentd-10x,12656::budget run ++fraud-detection,359::budget run ++frontend,34051::budget run ++frontend-proxy,15837::budget run ++grafana,178420::budget run ++grafana-sc-alerts,1052::budget run ++grafana-sc-dashboard,1488::budget run ++grafana-sc-datasources,1067::budget run ++image-provider,2675::budget run ++init-config,884::budget run ++jaeger,10809::budget run ++kafka,64436::budget run ++kube-proxy,13328::budget run ++load-generator,1218::budget run ++metrics-server,3812::budget run ++opensearch,32073::budget run ++opentelemetry-collector,187441::budget run ++payment,414::budget run ++postgresql,3855::budget run ++prometheus-server,4548::budget run ++quote,67::budget run ++recommendation,2757::budget run ++valkey-cart,7071::budget run ++wait-for-kafka,1253::budget run ++wait-for-valkey-cart,782::budget run ++*,2716::budget run +diff --git a/reports/latest.md b/reports/latest.md +index c1f02fa..5c51c5b 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,54 +1,54 @@ +-# Hot-tier run: 2026-10-26T00:23:13.000Z ++# Hot-tier run: 2026-10-26T00:24:12.000Z + +-Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 138 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. ++Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 143 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 11 MB in 2. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 2. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.20 bytes of store kept, 1.10 as tier_down (measured on the indices on hot). ++- Each counted byte takes 1.38 bytes of store kept, 1.09 as tier_down (measured on the indices on hot). + - The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 232 MB a day on hot whatever the limits; the size holds down to that floor. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 218 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 232 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 138 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 218 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 143 MB. Raise the size, move the noisiest services to offload, or lower the sample. + - Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5747 | +-| ad | the frozen tier | 1 | 45 MB | 2 MB | 1 | 5829 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 110 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 522 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 665 | +-| cart | the frozen tier | 1 | 103 MB | 1 MB | 1 | 3862 | +-| coredns | the frozen tier | 1 | 2 MB | 1 MB | 1 | 3858 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 156 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 457 | +-| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1476 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 261 | +-| fluentd-10x | the frozen tier | 1 | 13 MB | 5 MB | 1 | 13245 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 390 | +-| frontend | the frozen tier | 1 | 23 MB | 12 MB | 1 | 34127 | +-| frontend-proxy | the frozen tier | 1 | 6 MB | 6 MB | 1 | 17195 | +-| grafana | the frozen tier | 1 | 73 MB | 69 MB | 1 | 192810 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1142 | +-| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1616 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1158 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2905 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 960 | +-| jaeger | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11735 | +-| kafka | the frozen tier | 1 | 225 MB | 24 MB | 1 | 66640 | +-| kube-proxy | the frozen tier | 1 | 9 MB | 5 MB | 1 | 14063 | +-| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1322 | +-| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4139 | +-| opensearch | the frozen tier | 1 | 13 MB | 12 MB | 1 | 34702 | +-| opentelemetry-collector | the frozen tier | 1 | 923 MB | 72 MB | 1 | 200507 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 449 | +-| postgresql | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4186 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4820 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 72 | +-| recommendation | the frozen tier | 1 | 33 MB | 1 MB | 1 | 2779 | +-| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7123 | +-| wait-for-kafka | the frozen tier | 1 | 4 MB | 0 MB | 1 | 1271 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 849 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2842 | ++| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5293 | ++| ad | the frozen tier | 1 | 46 MB | 2 MB | 1 | 5784 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 101 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 481 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 612 | ++| cart | the frozen tier | 1 | 107 MB | 1 MB | 1 | 3801 | ++| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3655 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 144 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 421 | ++| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1360 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 240 | ++| fluentd-10x | the frozen tier | 1 | 13 MB | 5 MB | 1 | 12656 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 359 | ++| frontend | the frozen tier | 1 | 24 MB | 12 MB | 1 | 34051 | ++| frontend-proxy | the frozen tier | 1 | 6 MB | 6 MB | 1 | 15837 | ++| grafana | the frozen tier | 1 | 68 MB | 64 MB | 1 | 178420 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1052 | ++| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1488 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1067 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2675 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 884 | ++| jaeger | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10809 | ++| kafka | the frozen tier | 1 | 230 MB | 23 MB | 1 | 64436 | ++| kube-proxy | the frozen tier | 1 | 8 MB | 5 MB | 1 | 13328 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1218 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3812 | ++| opensearch | the frozen tier | 1 | 12 MB | 12 MB | 1 | 32073 | ++| opentelemetry-collector | the frozen tier | 1 | 939 MB | 67 MB | 1 | 187441 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 414 | ++| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3855 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4548 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 67 | ++| recommendation | the frozen tier | 1 | 33 MB | 1 MB | 1 | 2757 | ++| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7071 | ++| wait-for-kafka | the frozen tier | 1 | 4 MB | 0 MB | 1 | 1253 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 782 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2716 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/commit.json new file mode 100644 index 0000000..0196487 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/commit.json @@ -0,0 +1 @@ +{"before":"f2bec2d3ce3661d12bbf371e2342fc6f93e6b593","after":"2ce259a177b753a0651d30d40ad14706d29f4f71","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/latest.md new file mode 100644 index 0000000..5c51c5b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/latest.md @@ -0,0 +1,54 @@ +# Hot-tier run: 2026-10-26T00:24:12.000Z + +Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 143 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.38 bytes of store kept, 1.09 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 218 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 218 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 143 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5293 | +| ad | the frozen tier | 1 | 46 MB | 2 MB | 1 | 5784 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 101 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 481 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 612 | +| cart | the frozen tier | 1 | 107 MB | 1 MB | 1 | 3801 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3655 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 144 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 421 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1360 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 240 | +| fluentd-10x | the frozen tier | 1 | 13 MB | 5 MB | 1 | 12656 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 359 | +| frontend | the frozen tier | 1 | 24 MB | 12 MB | 1 | 34051 | +| frontend-proxy | the frozen tier | 1 | 6 MB | 6 MB | 1 | 15837 | +| grafana | the frozen tier | 1 | 68 MB | 64 MB | 1 | 178420 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1052 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1488 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1067 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2675 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 884 | +| jaeger | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10809 | +| kafka | the frozen tier | 1 | 230 MB | 23 MB | 1 | 64436 | +| kube-proxy | the frozen tier | 1 | 8 MB | 5 MB | 1 | 13328 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1218 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3812 | +| opensearch | the frozen tier | 1 | 12 MB | 12 MB | 1 | 32073 | +| opentelemetry-collector | the frozen tier | 1 | 939 MB | 67 MB | 1 | 187441 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 414 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3855 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4548 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 67 | +| recommendation | the frozen tier | 1 | 33 MB | 1 MB | 1 | 2757 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7071 | +| wait-for-kafka | the frozen tier | 1 | 4 MB | 0 MB | 1 | 1253 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 782 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2716 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/report.txt new file mode 100644 index 0000000..ebc9061 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/report.txt @@ -0,0 +1,63 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 143 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 2. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.38 bytes of store kept, 1.09 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 218 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:24:12.000Z + +Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 143 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.38 bytes of store kept, 1.09 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 218 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 218 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 143 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5293 | +| ad | the frozen tier | 1 | 46 MB | 2 MB | 1 | 5784 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 101 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 481 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 612 | +| cart | the frozen tier | 1 | 107 MB | 1 MB | 1 | 3801 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3655 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 144 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 421 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1360 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 240 | +| fluentd-10x | the frozen tier | 1 | 13 MB | 5 MB | 1 | 12656 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 359 | +| frontend | the frozen tier | 1 | 24 MB | 12 MB | 1 | 34051 | +| frontend-proxy | the frozen tier | 1 | 6 MB | 6 MB | 1 | 15837 | +| grafana | the frozen tier | 1 | 68 MB | 64 MB | 1 | 178420 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1052 | +| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1488 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1067 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2675 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 884 | +| jaeger | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10809 | +| kafka | the frozen tier | 1 | 230 MB | 23 MB | 1 | 64436 | +| kube-proxy | the frozen tier | 1 | 8 MB | 5 MB | 1 | 13328 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1218 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3812 | +| opensearch | the frozen tier | 1 | 12 MB | 12 MB | 1 | 32073 | +| opentelemetry-collector | the frozen tier | 1 | 939 MB | 67 MB | 1 | 187441 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 414 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3855 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4548 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 67 | +| recommendation | the frozen tier | 1 | 33 MB | 1 MB | 1 | 2757 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7071 | +| wait-for-kafka | the frozen tier | 1 | 4 MB | 0 MB | 1 | 1253 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 782 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2716 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/run.json new file mode 100644 index 0000000..8c51b74 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:24:15.085Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 143 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/totals.csv new file mode 100644 index 0000000..b85be66 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002412Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,5293::budget run +ad,5784::budget run +aws-eks-nodeagent,101::budget run +aws-node,481::budget run +aws-vpc-cni-init,612::budget run +cart,3801::budget run +coredns,3655::budget run +currency,144::budget run +email,421::budget run +flagd,1360::budget run +flagd-ui,240::budget run +fluentd-10x,12656::budget run +fraud-detection,359::budget run +frontend,34051::budget run +frontend-proxy,15837::budget run +grafana,178420::budget run +grafana-sc-alerts,1052::budget run +grafana-sc-dashboard,1488::budget run +grafana-sc-datasources,1067::budget run +image-provider,2675::budget run +init-config,884::budget run +jaeger,10809::budget run +kafka,64436::budget run +kube-proxy,13328::budget run +load-generator,1218::budget run +metrics-server,3812::budget run +opensearch,32073::budget run +opentelemetry-collector,187441::budget run +payment,414::budget run +postgresql,3855::budget run +prometheus-server,4548::budget run +quote,67::budget run +recommendation,2757::budget run +valkey-cart,7071::budget run +wait-for-kafka,1253::budget run +wait-for-valkey-cart,782::budget run +*,2716::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/commit.diff new file mode 100644 index 0000000..df5779f --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/commit.diff @@ -0,0 +1,181 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index b85be66..fa65bf7 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,5293::budget run +-ad,5784::budget run +-aws-eks-nodeagent,101::budget run +-aws-node,481::budget run +-aws-vpc-cni-init,612::budget run +-cart,3801::budget run +-coredns,3655::budget run +-currency,144::budget run +-email,421::budget run +-flagd,1360::budget run +-flagd-ui,240::budget run +-fluentd-10x,12656::budget run +-fraud-detection,359::budget run +-frontend,34051::budget run +-frontend-proxy,15837::budget run +-grafana,178420::budget run +-grafana-sc-alerts,1052::budget run +-grafana-sc-dashboard,1488::budget run +-grafana-sc-datasources,1067::budget run +-image-provider,2675::budget run +-init-config,884::budget run +-jaeger,10809::budget run +-kafka,64436::budget run +-kube-proxy,13328::budget run +-load-generator,1218::budget run +-metrics-server,3812::budget run +-opensearch,32073::budget run +-opentelemetry-collector,187441::budget run +-payment,414::budget run +-postgresql,3855::budget run +-prometheus-server,4548::budget run +-quote,67::budget run +-recommendation,2757::budget run +-valkey-cart,7071::budget run +-wait-for-kafka,1253::budget run +-wait-for-valkey-cart,782::budget run +-*,2716::budget run ++accounting,4912::budget run ++ad,5746::budget run ++aws-eks-nodeagent,94::budget run ++aws-node,446::budget run ++aws-vpc-cni-init,568::budget run ++cart,3750::budget run ++coredns,3485::budget run ++currency,134::budget run ++email,391::budget run ++flagd,1262::budget run ++flagd-ui,223::budget run ++fluentd-10x,12161::budget run ++fraud-detection,333::budget run ++frontend,33987::budget run ++frontend-proxy,14696::budget run ++grafana,166340::budget run ++grafana-sc-alerts,976::budget run ++grafana-sc-dashboard,1381::budget run ++grafana-sc-datasources,990::budget run ++image-provider,2483::budget run ++init-config,820::budget run ++jaeger,10030::budget run ++kafka,62586::budget run ++kube-proxy,12710::budget run ++load-generator,1130::budget run ++metrics-server,3537::budget run ++opensearch,30067::budget run ++opentelemetry-collector,176464::budget run ++payment,384::budget run ++postgresql,3578::budget run ++prometheus-server,4502::budget run ++quote,62::budget run ++recommendation,2738::budget run ++valkey-cart,7026::budget run ++wait-for-kafka,1237::budget run ++wait-for-valkey-cart,725::budget run ++*,2610::budget run +diff --git a/reports/latest.md b/reports/latest.md +index 5c51c5b..f91cee5 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,54 +1,54 @@ +-# Hot-tier run: 2026-10-26T00:24:12.000Z ++# Hot-tier run: 2026-10-26T00:25:10.000Z + +-Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 143 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. ++Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 133 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. + + - Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 2. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.38 bytes of store kept, 1.09 as tier_down (measured on the indices on hot). ++- Each counted byte takes 1.25 bytes of store kept, 1.01 as tier_down (measured on the indices on hot). + - The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 218 MB a day on hot whatever the limits; the size holds down to that floor. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 206 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 218 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 143 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 206 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 133 MB. Raise the size, move the noisiest services to offload, or lower the sample. + - Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5293 | +-| ad | the frozen tier | 1 | 46 MB | 2 MB | 1 | 5784 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 101 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 481 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 612 | +-| cart | the frozen tier | 1 | 107 MB | 1 MB | 1 | 3801 | +-| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3655 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 144 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 421 | +-| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1360 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 240 | +-| fluentd-10x | the frozen tier | 1 | 13 MB | 5 MB | 1 | 12656 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 359 | +-| frontend | the frozen tier | 1 | 24 MB | 12 MB | 1 | 34051 | +-| frontend-proxy | the frozen tier | 1 | 6 MB | 6 MB | 1 | 15837 | +-| grafana | the frozen tier | 1 | 68 MB | 64 MB | 1 | 178420 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1052 | +-| grafana-sc-dashboard | the frozen tier | 1 | 1 MB | 1 MB | 1 | 1488 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1067 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2675 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 884 | +-| jaeger | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10809 | +-| kafka | the frozen tier | 1 | 230 MB | 23 MB | 1 | 64436 | +-| kube-proxy | the frozen tier | 1 | 8 MB | 5 MB | 1 | 13328 | +-| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1218 | +-| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3812 | +-| opensearch | the frozen tier | 1 | 12 MB | 12 MB | 1 | 32073 | +-| opentelemetry-collector | the frozen tier | 1 | 939 MB | 67 MB | 1 | 187441 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 414 | +-| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3855 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4548 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 67 | +-| recommendation | the frozen tier | 1 | 33 MB | 1 MB | 1 | 2757 | +-| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7071 | +-| wait-for-kafka | the frozen tier | 1 | 4 MB | 0 MB | 1 | 1253 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 782 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2716 | ++| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4912 | ++| ad | the frozen tier | 1 | 47 MB | 2 MB | 1 | 5746 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 94 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 446 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 568 | ++| cart | the frozen tier | 1 | 111 MB | 1 MB | 1 | 3750 | ++| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3485 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 134 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 391 | ++| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1262 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 223 | ++| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 12161 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 333 | ++| frontend | the frozen tier | 1 | 26 MB | 12 MB | 1 | 33987 | ++| frontend-proxy | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14696 | ++| grafana | the frozen tier | 1 | 63 MB | 60 MB | 1 | 166340 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 976 | ++| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1381 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 990 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2483 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 820 | ++| jaeger | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10030 | ++| kafka | the frozen tier | 1 | 233 MB | 23 MB | 1 | 62586 | ++| kube-proxy | the frozen tier | 1 | 8 MB | 5 MB | 1 | 12710 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1130 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3537 | ++| opensearch | the frozen tier | 1 | 11 MB | 11 MB | 1 | 30067 | ++| opentelemetry-collector | the frozen tier | 1 | 957 MB | 64 MB | 1 | 176464 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 384 | ++| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3578 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4502 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 62 | ++| recommendation | the frozen tier | 1 | 33 MB | 1 MB | 1 | 2738 | ++| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7026 | ++| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1237 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 725 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2610 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/commit.json new file mode 100644 index 0000000..4860a34 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/commit.json @@ -0,0 +1 @@ +{"before":"2ce259a177b753a0651d30d40ad14706d29f4f71","after":"c50b3ed2068a410b712650c9b6d1b372f6eb9361","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/latest.md new file mode 100644 index 0000000..f91cee5 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/latest.md @@ -0,0 +1,54 @@ +# Hot-tier run: 2026-10-26T00:25:10.000Z + +Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 133 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.25 bytes of store kept, 1.01 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 206 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 206 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 133 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4912 | +| ad | the frozen tier | 1 | 47 MB | 2 MB | 1 | 5746 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 94 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 446 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 568 | +| cart | the frozen tier | 1 | 111 MB | 1 MB | 1 | 3750 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3485 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 134 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 391 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1262 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 223 | +| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 12161 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 333 | +| frontend | the frozen tier | 1 | 26 MB | 12 MB | 1 | 33987 | +| frontend-proxy | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14696 | +| grafana | the frozen tier | 1 | 63 MB | 60 MB | 1 | 166340 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 976 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1381 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 990 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2483 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 820 | +| jaeger | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10030 | +| kafka | the frozen tier | 1 | 233 MB | 23 MB | 1 | 62586 | +| kube-proxy | the frozen tier | 1 | 8 MB | 5 MB | 1 | 12710 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1130 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3537 | +| opensearch | the frozen tier | 1 | 11 MB | 11 MB | 1 | 30067 | +| opentelemetry-collector | the frozen tier | 1 | 957 MB | 64 MB | 1 | 176464 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 384 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3578 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4502 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 62 | +| recommendation | the frozen tier | 1 | 33 MB | 1 MB | 1 | 2738 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7026 | +| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1237 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 725 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2610 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/report.txt new file mode 100644 index 0000000..3dddbd0 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/report.txt @@ -0,0 +1,63 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 133 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 2. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.25 bytes of store kept, 1.01 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 206 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:25:10.000Z + +Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 133 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.25 bytes of store kept, 1.01 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 206 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 206 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 133 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4912 | +| ad | the frozen tier | 1 | 47 MB | 2 MB | 1 | 5746 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 94 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 446 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 568 | +| cart | the frozen tier | 1 | 111 MB | 1 MB | 1 | 3750 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3485 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 134 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 391 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1262 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 223 | +| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 12161 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 333 | +| frontend | the frozen tier | 1 | 26 MB | 12 MB | 1 | 33987 | +| frontend-proxy | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14696 | +| grafana | the frozen tier | 1 | 63 MB | 60 MB | 1 | 166340 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 976 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1381 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 990 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2483 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 820 | +| jaeger | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10030 | +| kafka | the frozen tier | 1 | 233 MB | 23 MB | 1 | 62586 | +| kube-proxy | the frozen tier | 1 | 8 MB | 5 MB | 1 | 12710 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1130 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3537 | +| opensearch | the frozen tier | 1 | 11 MB | 11 MB | 1 | 30067 | +| opentelemetry-collector | the frozen tier | 1 | 957 MB | 64 MB | 1 | 176464 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 384 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3578 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4502 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 62 | +| recommendation | the frozen tier | 1 | 33 MB | 1 MB | 1 | 2738 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7026 | +| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1237 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 725 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2610 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/run.json new file mode 100644 index 0000000..5a4388e --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:25:13.480Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 133 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/totals.csv new file mode 100644 index 0000000..fa65bf7 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002510Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,4912::budget run +ad,5746::budget run +aws-eks-nodeagent,94::budget run +aws-node,446::budget run +aws-vpc-cni-init,568::budget run +cart,3750::budget run +coredns,3485::budget run +currency,134::budget run +email,391::budget run +flagd,1262::budget run +flagd-ui,223::budget run +fluentd-10x,12161::budget run +fraud-detection,333::budget run +frontend,33987::budget run +frontend-proxy,14696::budget run +grafana,166340::budget run +grafana-sc-alerts,976::budget run +grafana-sc-dashboard,1381::budget run +grafana-sc-datasources,990::budget run +image-provider,2483::budget run +init-config,820::budget run +jaeger,10030::budget run +kafka,62586::budget run +kube-proxy,12710::budget run +load-generator,1130::budget run +metrics-server,3537::budget run +opensearch,30067::budget run +opentelemetry-collector,176464::budget run +payment,384::budget run +postgresql,3578::budget run +prometheus-server,4502::budget run +quote,62::budget run +recommendation,2738::budget run +valkey-cart,7026::budget run +wait-for-kafka,1237::budget run +wait-for-valkey-cart,725::budget run +*,2610::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/commit.diff new file mode 100644 index 0000000..7d59e65 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/commit.diff @@ -0,0 +1,182 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index fa65bf7..199b92a 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,4912::budget run +-ad,5746::budget run +-aws-eks-nodeagent,94::budget run +-aws-node,446::budget run +-aws-vpc-cni-init,568::budget run +-cart,3750::budget run +-coredns,3485::budget run +-currency,134::budget run +-email,391::budget run +-flagd,1262::budget run +-flagd-ui,223::budget run +-fluentd-10x,12161::budget run +-fraud-detection,333::budget run +-frontend,33987::budget run +-frontend-proxy,14696::budget run +-grafana,166340::budget run +-grafana-sc-alerts,976::budget run +-grafana-sc-dashboard,1381::budget run +-grafana-sc-datasources,990::budget run +-image-provider,2483::budget run +-init-config,820::budget run +-jaeger,10030::budget run +-kafka,62586::budget run +-kube-proxy,12710::budget run +-load-generator,1130::budget run +-metrics-server,3537::budget run +-opensearch,30067::budget run +-opentelemetry-collector,176464::budget run +-payment,384::budget run +-postgresql,3578::budget run +-prometheus-server,4502::budget run +-quote,62::budget run +-recommendation,2738::budget run +-valkey-cart,7026::budget run +-wait-for-kafka,1237::budget run +-wait-for-valkey-cart,725::budget run +-*,2610::budget run ++accounting,4576::budget run ++ad,5712::budget run ++aws-eks-nodeagent,87::budget run ++aws-node,416::budget run ++aws-vpc-cni-init,529::budget run ++cart,3704::budget run ++coredns,3335::budget run ++currency,124::budget run ++email,364::budget run ++flagd,1175::budget run ++flagd-ui,208::budget run ++fluentd-10x,11725::budget run ++fraud-detection,310::budget run ++frontend,33931::budget run ++frontend-proxy,13693::budget run ++grafana,156007::budget run ++grafana-sc-alerts,910::budget run ++grafana-sc-dashboard,1287::budget run ++grafana-sc-datasources,922::budget run ++image-provider,2313::budget run ++init-config,764::budget run ++jaeger,9345::budget run ++kafka,60957::budget run ++kube-proxy,12167::budget run ++load-generator,1053::budget run ++metrics-server,3296::budget run ++opensearch,28546::budget run ++opentelemetry-collector,166809::budget run ++payment,358::budget run ++postgresql,3333::budget run ++prometheus-server,4310::budget run ++quote,58::budget run ++recommendation,2722::budget run ++valkey-cart,6987::budget run ++wait-for-kafka,1224::budget run ++wait-for-valkey-cart,676::budget run ++*,2518::budget run +diff --git a/reports/latest.md b/reports/latest.md +index f91cee5..a9276a7 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,54 +1,54 @@ +-# Hot-tier run: 2026-10-26T00:25:10.000Z ++# Hot-tier run: 2026-10-26T00:26:09.000Z + +-Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 133 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. ++Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 128 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 2. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 13 MB in 2. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.25 bytes of store kept, 1.01 as tier_down (measured on the indices on hot). ++- Each counted byte takes 1.28 bytes of store kept, 0.95 as tier_down (measured on the indices on hot). + - The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 206 MB a day on hot whatever the limits; the size holds down to that floor. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 196 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 206 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 133 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 196 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 128 MB. Raise the size, move the noisiest services to offload, or lower the sample. + - Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4912 | +-| ad | the frozen tier | 1 | 47 MB | 2 MB | 1 | 5746 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 94 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 446 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 568 | +-| cart | the frozen tier | 1 | 111 MB | 1 MB | 1 | 3750 | +-| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3485 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 134 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 391 | +-| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1262 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 223 | +-| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 12161 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 333 | +-| frontend | the frozen tier | 1 | 26 MB | 12 MB | 1 | 33987 | +-| frontend-proxy | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14696 | +-| grafana | the frozen tier | 1 | 63 MB | 60 MB | 1 | 166340 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 976 | +-| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1381 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 990 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2483 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 820 | +-| jaeger | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10030 | +-| kafka | the frozen tier | 1 | 233 MB | 23 MB | 1 | 62586 | +-| kube-proxy | the frozen tier | 1 | 8 MB | 5 MB | 1 | 12710 | +-| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1130 | +-| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3537 | +-| opensearch | the frozen tier | 1 | 11 MB | 11 MB | 1 | 30067 | +-| opentelemetry-collector | the frozen tier | 1 | 957 MB | 64 MB | 1 | 176464 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 384 | +-| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3578 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4502 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 62 | +-| recommendation | the frozen tier | 1 | 33 MB | 1 MB | 1 | 2738 | +-| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7026 | +-| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1237 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 725 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2610 | ++| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4576 | ++| ad | the frozen tier | 1 | 49 MB | 2 MB | 1 | 5712 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 87 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 416 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 529 | ++| cart | the frozen tier | 1 | 113 MB | 1 MB | 1 | 3704 | ++| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3335 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 124 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 364 | ++| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1175 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 208 | ++| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 11725 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 310 | ++| frontend | the frozen tier | 1 | 26 MB | 12 MB | 1 | 33931 | ++| frontend-proxy | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13693 | ++| grafana | the frozen tier | 1 | 60 MB | 56 MB | 1 | 156007 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 910 | ++| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1287 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 922 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2313 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 764 | ++| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 1 | 9345 | ++| kafka | the frozen tier | 1 | 236 MB | 22 MB | 1 | 60957 | ++| kube-proxy | the frozen tier | 1 | 7 MB | 4 MB | 1 | 12167 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1053 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3296 | ++| opensearch | the frozen tier | 1 | 10 MB | 10 MB | 1 | 28546 | ++| opentelemetry-collector | the frozen tier | 1 | 972 MB | 60 MB | 1 | 166809 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 358 | ++| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3333 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4310 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 58 | ++| recommendation | the frozen tier | 1 | 34 MB | 1 MB | 1 | 2722 | ++| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 6987 | ++| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1224 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 676 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2518 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/commit.json new file mode 100644 index 0000000..8665853 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/commit.json @@ -0,0 +1 @@ +{"before":"c50b3ed2068a410b712650c9b6d1b372f6eb9361","after":"20a17566f950916eec1fab37d48e0d4b52d32cba","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/latest.md new file mode 100644 index 0000000..a9276a7 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/latest.md @@ -0,0 +1,54 @@ +# Hot-tier run: 2026-10-26T00:26:09.000Z + +Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 128 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 13 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.28 bytes of store kept, 0.95 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 196 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 196 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 128 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4576 | +| ad | the frozen tier | 1 | 49 MB | 2 MB | 1 | 5712 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 87 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 416 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 529 | +| cart | the frozen tier | 1 | 113 MB | 1 MB | 1 | 3704 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3335 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 124 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 364 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1175 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 208 | +| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 11725 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 310 | +| frontend | the frozen tier | 1 | 26 MB | 12 MB | 1 | 33931 | +| frontend-proxy | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13693 | +| grafana | the frozen tier | 1 | 60 MB | 56 MB | 1 | 156007 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 910 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1287 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 922 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2313 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 764 | +| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 1 | 9345 | +| kafka | the frozen tier | 1 | 236 MB | 22 MB | 1 | 60957 | +| kube-proxy | the frozen tier | 1 | 7 MB | 4 MB | 1 | 12167 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1053 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3296 | +| opensearch | the frozen tier | 1 | 10 MB | 10 MB | 1 | 28546 | +| opentelemetry-collector | the frozen tier | 1 | 972 MB | 60 MB | 1 | 166809 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 358 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3333 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4310 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 58 | +| recommendation | the frozen tier | 1 | 34 MB | 1 MB | 1 | 2722 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 6987 | +| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1224 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 676 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2518 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/report.txt new file mode 100644 index 0000000..9b0490d --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/report.txt @@ -0,0 +1,63 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 128 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 13 MB in 2. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.28 bytes of store kept, 0.95 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 196 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:26:09.000Z + +Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 128 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 13 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.28 bytes of store kept, 0.95 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 196 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 196 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 128 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4576 | +| ad | the frozen tier | 1 | 49 MB | 2 MB | 1 | 5712 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 87 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 416 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 529 | +| cart | the frozen tier | 1 | 113 MB | 1 MB | 1 | 3704 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3335 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 124 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 364 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1175 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 208 | +| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 11725 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 310 | +| frontend | the frozen tier | 1 | 26 MB | 12 MB | 1 | 33931 | +| frontend-proxy | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13693 | +| grafana | the frozen tier | 1 | 60 MB | 56 MB | 1 | 156007 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 910 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1287 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 922 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2313 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 764 | +| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 1 | 9345 | +| kafka | the frozen tier | 1 | 236 MB | 22 MB | 1 | 60957 | +| kube-proxy | the frozen tier | 1 | 7 MB | 4 MB | 1 | 12167 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1053 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3296 | +| opensearch | the frozen tier | 1 | 10 MB | 10 MB | 1 | 28546 | +| opentelemetry-collector | the frozen tier | 1 | 972 MB | 60 MB | 1 | 166809 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 358 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3333 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4310 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 58 | +| recommendation | the frozen tier | 1 | 34 MB | 1 MB | 1 | 2722 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 6987 | +| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1224 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 676 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2518 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/run.json new file mode 100644 index 0000000..775e43e --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:26:12.142Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 128 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/totals.csv new file mode 100644 index 0000000..199b92a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002609Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,4576::budget run +ad,5712::budget run +aws-eks-nodeagent,87::budget run +aws-node,416::budget run +aws-vpc-cni-init,529::budget run +cart,3704::budget run +coredns,3335::budget run +currency,124::budget run +email,364::budget run +flagd,1175::budget run +flagd-ui,208::budget run +fluentd-10x,11725::budget run +fraud-detection,310::budget run +frontend,33931::budget run +frontend-proxy,13693::budget run +grafana,156007::budget run +grafana-sc-alerts,910::budget run +grafana-sc-dashboard,1287::budget run +grafana-sc-datasources,922::budget run +image-provider,2313::budget run +init-config,764::budget run +jaeger,9345::budget run +kafka,60957::budget run +kube-proxy,12167::budget run +load-generator,1053::budget run +metrics-server,3296::budget run +opensearch,28546::budget run +opentelemetry-collector,166809::budget run +payment,358::budget run +postgresql,3333::budget run +prometheus-server,4310::budget run +quote,58::budget run +recommendation,2722::budget run +valkey-cart,6987::budget run +wait-for-kafka,1224::budget run +wait-for-valkey-cart,676::budget run +*,2518::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/commit.diff new file mode 100644 index 0000000..29f32db --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/commit.diff @@ -0,0 +1,183 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 199b92a..78afeeb 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,4576::budget run +-ad,5712::budget run +-aws-eks-nodeagent,87::budget run +-aws-node,416::budget run +-aws-vpc-cni-init,529::budget run +-cart,3704::budget run +-coredns,3335::budget run +-currency,124::budget run +-email,364::budget run +-flagd,1175::budget run +-flagd-ui,208::budget run +-fluentd-10x,11725::budget run +-fraud-detection,310::budget run +-frontend,33931::budget run +-frontend-proxy,13693::budget run +-grafana,156007::budget run +-grafana-sc-alerts,910::budget run +-grafana-sc-dashboard,1287::budget run +-grafana-sc-datasources,922::budget run +-image-provider,2313::budget run +-init-config,764::budget run +-jaeger,9345::budget run +-kafka,60957::budget run +-kube-proxy,12167::budget run +-load-generator,1053::budget run +-metrics-server,3296::budget run +-opensearch,28546::budget run +-opentelemetry-collector,166809::budget run +-payment,358::budget run +-postgresql,3333::budget run +-prometheus-server,4310::budget run +-quote,58::budget run +-recommendation,2722::budget run +-valkey-cart,6987::budget run +-wait-for-kafka,1224::budget run +-wait-for-valkey-cart,676::budget run +-*,2518::budget run ++accounting,4261::budget run ++ad,5681::budget run ++aws-eks-nodeagent,81::budget run ++aws-node,387::budget run ++aws-vpc-cni-init,493::budget run ++cart,3661::budget run ++coredns,3105::budget run ++currency,116::budget run ++email,339::budget run ++flagd,1094::budget run ++flagd-ui,193::budget run ++fluentd-10x,11315::budget run ++fraud-detection,289::budget run ++frontend,33879::budget run ++frontend-proxy,12749::budget run ++grafana,145931::budget run ++grafana-sc-alerts,847::budget run ++grafana-sc-dashboard,1198::budget run ++grafana-sc-datasources,859::budget run ++image-provider,2154::budget run ++init-config,711::budget run ++jaeger,8701::budget run ++kafka,59424::budget run ++kube-proxy,11656::budget run ++load-generator,980::budget run ++metrics-server,3068::budget run ++opensearch,26762::budget run ++opentelemetry-collector,157724::budget run ++payment,333::budget run ++postgresql,3103::budget run ++prometheus-server,4101::budget run ++quote,54::budget run ++recommendation,2707::budget run ++valkey-cart,6950::budget run ++wait-for-kafka,1211::budget run ++wait-for-valkey-cart,629::budget run ++*,2430::budget run +diff --git a/reports/latest.md b/reports/latest.md +index a9276a7..0419425 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,54 +1,54 @@ +-# Hot-tier run: 2026-10-26T00:26:09.000Z ++# Hot-tier run: 2026-10-26T00:27:13.000Z + +-Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 128 MB within 2 hours, with 1.3 GB a day of counted lines moved to the frozen tier. ++Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 89 MB within 2 hours, with 1.4 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 13 MB in 2. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 2. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.28 bytes of store kept, 0.95 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 1.5 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 196 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.29 bytes of store kept, 0.60 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 1.6 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 186 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 196 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 128 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 186 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 89 MB. Raise the size, move the noisiest services to offload, or lower the sample. + - Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4576 | +-| ad | the frozen tier | 1 | 49 MB | 2 MB | 1 | 5712 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 87 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 416 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 529 | +-| cart | the frozen tier | 1 | 113 MB | 1 MB | 1 | 3704 | +-| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3335 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 124 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 364 | +-| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1175 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 208 | +-| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 11725 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 310 | +-| frontend | the frozen tier | 1 | 26 MB | 12 MB | 1 | 33931 | +-| frontend-proxy | the frozen tier | 1 | 5 MB | 5 MB | 1 | 13693 | +-| grafana | the frozen tier | 1 | 60 MB | 56 MB | 1 | 156007 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 910 | +-| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1287 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 922 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2313 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 764 | +-| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 1 | 9345 | +-| kafka | the frozen tier | 1 | 236 MB | 22 MB | 1 | 60957 | +-| kube-proxy | the frozen tier | 1 | 7 MB | 4 MB | 1 | 12167 | +-| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1053 | +-| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3296 | +-| opensearch | the frozen tier | 1 | 10 MB | 10 MB | 1 | 28546 | +-| opentelemetry-collector | the frozen tier | 1 | 972 MB | 60 MB | 1 | 166809 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 358 | +-| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3333 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4310 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 58 | +-| recommendation | the frozen tier | 1 | 34 MB | 1 MB | 1 | 2722 | +-| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 6987 | +-| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1224 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 676 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2518 | ++| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4261 | ++| ad | the frozen tier | 1 | 49 MB | 2 MB | 1 | 5681 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 81 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 387 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 493 | ++| cart | the frozen tier | 1 | 115 MB | 1 MB | 1 | 3661 | ++| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3105 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 116 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 339 | ++| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1094 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 193 | ++| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 11315 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 289 | ++| frontend | the frozen tier | 1 | 27 MB | 12 MB | 1 | 33879 | ++| frontend-proxy | the frozen tier | 1 | 5 MB | 5 MB | 1 | 12749 | ++| grafana | the frozen tier | 1 | 56 MB | 53 MB | 1 | 145931 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 847 | ++| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1198 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 859 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2154 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 711 | ++| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8701 | ++| kafka | the frozen tier | 1 | 239 MB | 21 MB | 1 | 59424 | ++| kube-proxy | the frozen tier | 1 | 7 MB | 4 MB | 1 | 11656 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 980 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3068 | ++| opensearch | the frozen tier | 1 | 10 MB | 10 MB | 1 | 26762 | ++| opentelemetry-collector | the frozen tier | 1 | 986 MB | 57 MB | 1 | 157724 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 333 | ++| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3103 | ++| prometheus-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4101 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 54 | ++| recommendation | the frozen tier | 1 | 35 MB | 1 MB | 1 | 2707 | ++| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 6950 | ++| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1211 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 629 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2430 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/commit.json new file mode 100644 index 0000000..41e4f92 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/commit.json @@ -0,0 +1 @@ +{"before":"20a17566f950916eec1fab37d48e0d4b52d32cba","after":"82bd4fb4729c3b7ca6442d2d1811919257a19633","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/latest.md new file mode 100644 index 0000000..0419425 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/latest.md @@ -0,0 +1,54 @@ +# Hot-tier run: 2026-10-26T00:27:13.000Z + +Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 89 MB within 2 hours, with 1.4 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.29 bytes of store kept, 0.60 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.6 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 186 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 186 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 89 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4261 | +| ad | the frozen tier | 1 | 49 MB | 2 MB | 1 | 5681 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 81 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 387 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 493 | +| cart | the frozen tier | 1 | 115 MB | 1 MB | 1 | 3661 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3105 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 116 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 339 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1094 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 193 | +| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 11315 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 289 | +| frontend | the frozen tier | 1 | 27 MB | 12 MB | 1 | 33879 | +| frontend-proxy | the frozen tier | 1 | 5 MB | 5 MB | 1 | 12749 | +| grafana | the frozen tier | 1 | 56 MB | 53 MB | 1 | 145931 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 847 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1198 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 859 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2154 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 711 | +| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8701 | +| kafka | the frozen tier | 1 | 239 MB | 21 MB | 1 | 59424 | +| kube-proxy | the frozen tier | 1 | 7 MB | 4 MB | 1 | 11656 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 980 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3068 | +| opensearch | the frozen tier | 1 | 10 MB | 10 MB | 1 | 26762 | +| opentelemetry-collector | the frozen tier | 1 | 986 MB | 57 MB | 1 | 157724 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 333 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3103 | +| prometheus-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4101 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 54 | +| recommendation | the frozen tier | 1 | 35 MB | 1 MB | 1 | 2707 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 6950 | +| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1211 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 629 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2430 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/report.txt new file mode 100644 index 0000000..4e927d7 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/report.txt @@ -0,0 +1,63 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 89 MB within 2 hours, with 1.4 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 2. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.29 bytes of store kept, 0.60 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1.6 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 186 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:27:13.000Z + +Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 89 MB within 2 hours, with 1.4 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.29 bytes of store kept, 0.60 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.6 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 186 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 186 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 89 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4261 | +| ad | the frozen tier | 1 | 49 MB | 2 MB | 1 | 5681 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 81 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 387 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 493 | +| cart | the frozen tier | 1 | 115 MB | 1 MB | 1 | 3661 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3105 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 116 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 339 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1094 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 193 | +| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 11315 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 289 | +| frontend | the frozen tier | 1 | 27 MB | 12 MB | 1 | 33879 | +| frontend-proxy | the frozen tier | 1 | 5 MB | 5 MB | 1 | 12749 | +| grafana | the frozen tier | 1 | 56 MB | 53 MB | 1 | 145931 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 847 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1198 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 859 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2154 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 711 | +| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8701 | +| kafka | the frozen tier | 1 | 239 MB | 21 MB | 1 | 59424 | +| kube-proxy | the frozen tier | 1 | 7 MB | 4 MB | 1 | 11656 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 980 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3068 | +| opensearch | the frozen tier | 1 | 10 MB | 10 MB | 1 | 26762 | +| opentelemetry-collector | the frozen tier | 1 | 986 MB | 57 MB | 1 | 157724 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 333 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3103 | +| prometheus-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4101 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 54 | +| recommendation | the frozen tier | 1 | 35 MB | 1 MB | 1 | 2707 | +| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 6950 | +| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1211 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 629 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2430 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/run.json new file mode 100644 index 0000000..7fabb8b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:27:15.840Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 89 MB within 2 hours, with 1.4 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/totals.csv new file mode 100644 index 0000000..78afeeb --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002712Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,4261::budget run +ad,5681::budget run +aws-eks-nodeagent,81::budget run +aws-node,387::budget run +aws-vpc-cni-init,493::budget run +cart,3661::budget run +coredns,3105::budget run +currency,116::budget run +email,339::budget run +flagd,1094::budget run +flagd-ui,193::budget run +fluentd-10x,11315::budget run +fraud-detection,289::budget run +frontend,33879::budget run +frontend-proxy,12749::budget run +grafana,145931::budget run +grafana-sc-alerts,847::budget run +grafana-sc-dashboard,1198::budget run +grafana-sc-datasources,859::budget run +image-provider,2154::budget run +init-config,711::budget run +jaeger,8701::budget run +kafka,59424::budget run +kube-proxy,11656::budget run +load-generator,980::budget run +metrics-server,3068::budget run +opensearch,26762::budget run +opentelemetry-collector,157724::budget run +payment,333::budget run +postgresql,3103::budget run +prometheus-server,4101::budget run +quote,54::budget run +recommendation,2707::budget run +valkey-cart,6950::budget run +wait-for-kafka,1211::budget run +wait-for-valkey-cart,629::budget run +*,2430::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/caps.csv new file mode 100644 index 0000000..f92f886 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +image-provider,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +valkey-cart,1::budget run +wait-for-kafka,1::budget run +wait-for-valkey-cart,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/commit.diff new file mode 100644 index 0000000..4e95586 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/commit.diff @@ -0,0 +1,181 @@ +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 78afeeb..57a56b7 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,4261::budget run +-ad,5681::budget run +-aws-eks-nodeagent,81::budget run +-aws-node,387::budget run +-aws-vpc-cni-init,493::budget run +-cart,3661::budget run +-coredns,3105::budget run +-currency,116::budget run +-email,339::budget run +-flagd,1094::budget run +-flagd-ui,193::budget run +-fluentd-10x,11315::budget run +-fraud-detection,289::budget run +-frontend,33879::budget run +-frontend-proxy,12749::budget run +-grafana,145931::budget run +-grafana-sc-alerts,847::budget run +-grafana-sc-dashboard,1198::budget run +-grafana-sc-datasources,859::budget run +-image-provider,2154::budget run +-init-config,711::budget run +-jaeger,8701::budget run +-kafka,59424::budget run +-kube-proxy,11656::budget run +-load-generator,980::budget run +-metrics-server,3068::budget run +-opensearch,26762::budget run +-opentelemetry-collector,157724::budget run +-payment,333::budget run +-postgresql,3103::budget run +-prometheus-server,4101::budget run +-quote,54::budget run +-recommendation,2707::budget run +-valkey-cart,6950::budget run +-wait-for-kafka,1211::budget run +-wait-for-valkey-cart,629::budget run +-*,2430::budget run ++accounting,4010::budget run ++ad,5656::budget run ++aws-eks-nodeagent,76::budget run ++aws-node,364::budget run ++aws-vpc-cni-init,464::budget run ++cart,3627::budget run ++coredns,2923::budget run ++currency,109::budget run ++email,319::budget run ++flagd,1030::budget run ++flagd-ui,182::budget run ++fluentd-10x,10989::budget run ++fraud-detection,272::budget run ++frontend,33837::budget run ++frontend-proxy,11999::budget run ++grafana,138583::budget run ++grafana-sc-alerts,797::budget run ++grafana-sc-dashboard,1127::budget run ++grafana-sc-datasources,808::budget run ++image-provider,2027::budget run ++init-config,670::budget run ++jaeger,8189::budget run ++kafka,58207::budget run ++kube-proxy,11250::budget run ++load-generator,922::budget run ++metrics-server,2888::budget run ++opensearch,25520::budget run ++opentelemetry-collector,150509::budget run ++payment,313::budget run ++postgresql,2921::budget run ++prometheus-server,3988::budget run ++quote,50::budget run ++recommendation,2695::budget run ++valkey-cart,6921::budget run ++wait-for-kafka,1201::budget run ++wait-for-valkey-cart,592::budget run ++*,2361::budget run +diff --git a/reports/latest.md b/reports/latest.md +index 0419425..c72568e 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,54 +1,54 @@ +-# Hot-tier run: 2026-10-26T00:27:13.000Z ++# Hot-tier run: 2026-10-26T00:28:11.000Z + +-Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 89 MB within 2 hours, with 1.4 GB a day of counted lines moved to the frozen tier. ++Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 89 MB within 2 hours, with 1.5 GB a day of counted lines moved to the frozen tier. + + - Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 2. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.29 bytes of store kept, 0.60 as tier_down (measured on the indices on hot). ++- Each counted byte takes 1.45 bytes of store kept, 0.55 as tier_down (measured on the indices on hot). + - The Receiver's statements send 1.6 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 186 MB a day on hot whatever the limits; the size holds down to that floor. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 179 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 186 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 89 MB. Raise the size, move the noisiest services to offload, or lower the sample. ++- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 179 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 89 MB. Raise the size, move the noisiest services to offload, or lower the sample. + - Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4261 | +-| ad | the frozen tier | 1 | 49 MB | 2 MB | 1 | 5681 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 81 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 387 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 493 | +-| cart | the frozen tier | 1 | 115 MB | 1 MB | 1 | 3661 | +-| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3105 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 116 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 339 | +-| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1094 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 193 | +-| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 11315 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 289 | +-| frontend | the frozen tier | 1 | 27 MB | 12 MB | 1 | 33879 | +-| frontend-proxy | the frozen tier | 1 | 5 MB | 5 MB | 1 | 12749 | +-| grafana | the frozen tier | 1 | 56 MB | 53 MB | 1 | 145931 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 847 | +-| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1198 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 859 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2154 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 711 | +-| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8701 | +-| kafka | the frozen tier | 1 | 239 MB | 21 MB | 1 | 59424 | +-| kube-proxy | the frozen tier | 1 | 7 MB | 4 MB | 1 | 11656 | +-| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 980 | +-| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3068 | +-| opensearch | the frozen tier | 1 | 10 MB | 10 MB | 1 | 26762 | +-| opentelemetry-collector | the frozen tier | 1 | 986 MB | 57 MB | 1 | 157724 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 333 | +-| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3103 | +-| prometheus-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4101 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 54 | +-| recommendation | the frozen tier | 1 | 35 MB | 1 MB | 1 | 2707 | +-| valkey-cart | the frozen tier | 1 | 3 MB | 3 MB | 1 | 6950 | +-| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1211 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 629 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2430 | ++| accounting | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4010 | ++| ad | the frozen tier | 1 | 51 MB | 2 MB | 1 | 5656 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 76 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 364 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 464 | ++| cart | the frozen tier | 1 | 125 MB | 1 MB | 1 | 3627 | ++| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2923 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 109 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 319 | ++| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1030 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 182 | ++| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 10989 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 272 | ++| frontend | the frozen tier | 1 | 29 MB | 12 MB | 1 | 33837 | ++| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11999 | ++| grafana | the frozen tier | 1 | 54 MB | 50 MB | 1 | 138583 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 797 | ++| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1127 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 808 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2027 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 670 | ++| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8189 | ++| kafka | the frozen tier | 1 | 254 MB | 21 MB | 1 | 58207 | ++| kube-proxy | the frozen tier | 1 | 6 MB | 4 MB | 1 | 11250 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 922 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2888 | ++| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 1 | 25520 | ++| opentelemetry-collector | the frozen tier | 1 | 1 GB | 54 MB | 1 | 150509 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 313 | ++| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2921 | ++| prometheus-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3988 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 50 | ++| recommendation | the frozen tier | 1 | 37 MB | 1 MB | 1 | 2695 | ++| valkey-cart | the frozen tier | 1 | 3 MB | 2 MB | 1 | 6921 | ++| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1201 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 592 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2361 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/commit.json new file mode 100644 index 0000000..7f76808 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/commit.json @@ -0,0 +1 @@ +{"before":"82bd4fb4729c3b7ca6442d2d1811919257a19633","after":"b38595b6f85a4836e82df549be73476fb17d161f","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/config-generation.csv new file mode 100644 index 0000000..89e536b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,4236ef75e377 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/latest.md new file mode 100644 index 0000000..c72568e --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/latest.md @@ -0,0 +1,54 @@ +# Hot-tier run: 2026-10-26T00:28:11.000Z + +Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 89 MB within 2 hours, with 1.5 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.45 bytes of store kept, 0.55 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.6 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 179 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 179 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 89 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4010 | +| ad | the frozen tier | 1 | 51 MB | 2 MB | 1 | 5656 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 76 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 364 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 464 | +| cart | the frozen tier | 1 | 125 MB | 1 MB | 1 | 3627 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2923 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 109 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 319 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1030 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 182 | +| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 10989 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 272 | +| frontend | the frozen tier | 1 | 29 MB | 12 MB | 1 | 33837 | +| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11999 | +| grafana | the frozen tier | 1 | 54 MB | 50 MB | 1 | 138583 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 797 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1127 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 808 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2027 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 670 | +| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8189 | +| kafka | the frozen tier | 1 | 254 MB | 21 MB | 1 | 58207 | +| kube-proxy | the frozen tier | 1 | 6 MB | 4 MB | 1 | 11250 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 922 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2888 | +| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 1 | 25520 | +| opentelemetry-collector | the frozen tier | 1 | 1 GB | 54 MB | 1 | 150509 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 313 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2921 | +| prometheus-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3988 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 50 | +| recommendation | the frozen tier | 1 | 37 MB | 1 MB | 1 | 2695 | +| valkey-cart | the frozen tier | 1 | 3 MB | 2 MB | 1 | 6921 | +| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1201 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 592 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2361 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/report.txt new file mode 100644 index 0000000..a32666d --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/report.txt @@ -0,0 +1,63 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 89 MB within 2 hours, with 1.5 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 2. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +[tenx-recur] Each counted byte takes 1.45 bytes of store kept, 0.55 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1.6 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 179 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:28:11.000Z + +Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 89 MB within 2 hours, with 1.5 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +- Each counted byte takes 1.45 bytes of store kept, 0.55 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1.6 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 179 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 179 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 89 MB. Raise the size, move the noisiest services to offload, or lower the sample. +- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4010 | +| ad | the frozen tier | 1 | 51 MB | 2 MB | 1 | 5656 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 76 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 364 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 464 | +| cart | the frozen tier | 1 | 125 MB | 1 MB | 1 | 3627 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2923 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 109 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 319 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1030 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 182 | +| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 10989 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 272 | +| frontend | the frozen tier | 1 | 29 MB | 12 MB | 1 | 33837 | +| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11999 | +| grafana | the frozen tier | 1 | 54 MB | 50 MB | 1 | 138583 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 797 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1127 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 808 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2027 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 670 | +| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8189 | +| kafka | the frozen tier | 1 | 254 MB | 21 MB | 1 | 58207 | +| kube-proxy | the frozen tier | 1 | 6 MB | 4 MB | 1 | 11250 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 922 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2888 | +| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 1 | 25520 | +| opentelemetry-collector | the frozen tier | 1 | 1 GB | 54 MB | 1 | 150509 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 313 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2921 | +| prometheus-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3988 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 50 | +| recommendation | the frozen tier | 1 | 37 MB | 1 MB | 1 | 2695 | +| valkey-cart | the frozen tier | 1 | 3 MB | 2 MB | 1 | 6921 | +| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1201 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 592 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2361 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/run.json new file mode 100644 index 0000000..fd94b6b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:28:14.437Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 89 MB within 2 hours, with 1.5 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/totals.csv new file mode 100644 index 0000000..57a56b7 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002811Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,4010::budget run +ad,5656::budget run +aws-eks-nodeagent,76::budget run +aws-node,364::budget run +aws-vpc-cni-init,464::budget run +cart,3627::budget run +coredns,2923::budget run +currency,109::budget run +email,319::budget run +flagd,1030::budget run +flagd-ui,182::budget run +fluentd-10x,10989::budget run +fraud-detection,272::budget run +frontend,33837::budget run +frontend-proxy,11999::budget run +grafana,138583::budget run +grafana-sc-alerts,797::budget run +grafana-sc-dashboard,1127::budget run +grafana-sc-datasources,808::budget run +image-provider,2027::budget run +init-config,670::budget run +jaeger,8189::budget run +kafka,58207::budget run +kube-proxy,11250::budget run +load-generator,922::budget run +metrics-server,2888::budget run +opensearch,25520::budget run +opentelemetry-collector,150509::budget run +payment,313::budget run +postgresql,2921::budget run +prometheus-server,3988::budget run +quote,50::budget run +recommendation,2695::budget run +valkey-cart,6921::budget run +wait-for-kafka,1201::budget run +wait-for-valkey-cart,592::budget run +*,2361::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/caps.csv new file mode 100644 index 0000000..f4acd8c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,5991::budget run +ad,5991::budget run +aws-eks-nodeagent,5991::budget run +aws-node,5991::budget run +aws-vpc-cni-init,5991::budget run +cart,5991::budget run +coredns,5991::budget run +currency,5991::budget run +email,5991::budget run +flagd,5991::budget run +flagd-ui,5991::budget run +fluentd-10x,5991::budget run +fraud-detection,5991::budget run +frontend,5991::budget run +frontend-proxy,5991::budget run +grafana,5991::budget run +grafana-sc-alerts,5991::budget run +grafana-sc-dashboard,5991::budget run +grafana-sc-datasources,5991::budget run +image-provider,5991::budget run +init-config,5991::budget run +jaeger,5991::budget run +kafka,5991::budget run +kube-proxy,5991::budget run +load-generator,5991::budget run +metrics-server,5991::budget run +opensearch,5991::budget run +opentelemetry-collector,5991::budget run +payment,5991::budget run +postgresql,5991::budget run +prometheus-server,5991::budget run +quote,5991::budget run +recommendation,5991::budget run +valkey-cart,5991::budget run +wait-for-kafka,5991::budget run +wait-for-valkey-cart,5991::budget run +*,5991::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/commit.diff new file mode 100644 index 0000000..75e9f36 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/commit.diff @@ -0,0 +1,271 @@ +diff --git a/log10x/policy/caps.csv b/log10x/policy/caps.csv +index f92f886..f4acd8c 100644 +--- a/log10x/policy/caps.csv ++++ b/log10x/policy/caps.csv +@@ -1,38 +1,38 @@ + container,cap +-accounting,1::budget run +-ad,1::budget run +-aws-eks-nodeagent,1::budget run +-aws-node,1::budget run +-aws-vpc-cni-init,1::budget run +-cart,1::budget run +-coredns,1::budget run +-currency,1::budget run +-email,1::budget run +-flagd,1::budget run +-flagd-ui,1::budget run +-fluentd-10x,1::budget run +-fraud-detection,1::budget run +-frontend,1::budget run +-frontend-proxy,1::budget run +-grafana,1::budget run +-grafana-sc-alerts,1::budget run +-grafana-sc-dashboard,1::budget run +-grafana-sc-datasources,1::budget run +-image-provider,1::budget run +-init-config,1::budget run +-jaeger,1::budget run +-kafka,1::budget run +-kube-proxy,1::budget run +-load-generator,1::budget run +-metrics-server,1::budget run +-opensearch,1::budget run +-opentelemetry-collector,1::budget run +-payment,1::budget run +-postgresql,1::budget run +-prometheus-server,1::budget run +-quote,1::budget run +-recommendation,1::budget run +-valkey-cart,1::budget run +-wait-for-kafka,1::budget run +-wait-for-valkey-cart,1::budget run +-*,1::budget run ++accounting,5991::budget run ++ad,5991::budget run ++aws-eks-nodeagent,5991::budget run ++aws-node,5991::budget run ++aws-vpc-cni-init,5991::budget run ++cart,5991::budget run ++coredns,5991::budget run ++currency,5991::budget run ++email,5991::budget run ++flagd,5991::budget run ++flagd-ui,5991::budget run ++fluentd-10x,5991::budget run ++fraud-detection,5991::budget run ++frontend,5991::budget run ++frontend-proxy,5991::budget run ++grafana,5991::budget run ++grafana-sc-alerts,5991::budget run ++grafana-sc-dashboard,5991::budget run ++grafana-sc-datasources,5991::budget run ++image-provider,5991::budget run ++init-config,5991::budget run ++jaeger,5991::budget run ++kafka,5991::budget run ++kube-proxy,5991::budget run ++load-generator,5991::budget run ++metrics-server,5991::budget run ++opensearch,5991::budget run ++opentelemetry-collector,5991::budget run ++payment,5991::budget run ++postgresql,5991::budget run ++prometheus-server,5991::budget run ++quote,5991::budget run ++recommendation,5991::budget run ++valkey-cart,5991::budget run ++wait-for-kafka,5991::budget run ++wait-for-valkey-cart,5991::budget run ++*,5991::budget run +diff --git a/log10x/policy/config-generation.csv b/log10x/policy/config-generation.csv +index 89e536b..e07854e 100644 +--- a/log10x/policy/config-generation.csv ++++ b/log10x/policy/config-generation.csv +@@ -1,2 +1,2 @@ + key,value +-generation,4236ef75e377 ++generation,e4464048f2c6 +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 57a56b7..bd29180 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,4010::budget run +-ad,5656::budget run +-aws-eks-nodeagent,76::budget run +-aws-node,364::budget run +-aws-vpc-cni-init,464::budget run +-cart,3627::budget run +-coredns,2923::budget run +-currency,109::budget run +-email,319::budget run +-flagd,1030::budget run +-flagd-ui,182::budget run +-fluentd-10x,10989::budget run +-fraud-detection,272::budget run +-frontend,33837::budget run +-frontend-proxy,11999::budget run +-grafana,138583::budget run +-grafana-sc-alerts,797::budget run +-grafana-sc-dashboard,1127::budget run +-grafana-sc-datasources,808::budget run +-image-provider,2027::budget run +-init-config,670::budget run +-jaeger,8189::budget run +-kafka,58207::budget run +-kube-proxy,11250::budget run +-load-generator,922::budget run +-metrics-server,2888::budget run +-opensearch,25520::budget run +-opentelemetry-collector,150509::budget run +-payment,313::budget run +-postgresql,2921::budget run +-prometheus-server,3988::budget run +-quote,50::budget run +-recommendation,2695::budget run +-valkey-cart,6921::budget run +-wait-for-kafka,1201::budget run +-wait-for-valkey-cart,592::budget run +-*,2361::budget run ++accounting,3784::budget run ++ad,18352::budget run ++aws-eks-nodeagent,72::budget run ++aws-node,344::budget run ++aws-vpc-cni-init,437::budget run ++cart,12499::budget run ++coredns,2758::budget run ++currency,103::budget run ++email,301::budget run ++flagd,972::budget run ++flagd-ui,172::budget run ++fluentd-10x,11904::budget run ++fraud-detection,256::budget run ++frontend,33800::budget run ++frontend-proxy,11321::budget run ++grafana,136482::budget run ++grafana-sc-alerts,752::budget run ++grafana-sc-dashboard,1064::budget run ++grafana-sc-datasources,762::budget run ++image-provider,1912::budget run ++init-config,632::budget run ++jaeger,7726::budget run ++kafka,127170::budget run ++kube-proxy,16828::budget run ++load-generator,870::budget run ++metrics-server,2725::budget run ++opensearch,26157::budget run ++opentelemetry-collector,155892::budget run ++payment,296::budget run ++postgresql,2670::budget run ++prometheus-server,5347::budget run ++quote,47::budget run ++recommendation,6175::budget run ++valkey-cart,13340::budget run ++wait-for-kafka,6144::budget run ++wait-for-valkey-cart,559::budget run ++*,2697::budget run +diff --git a/reports/latest.md b/reports/latest.md +index c72568e..f56804e 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,54 +1,52 @@ +-# Hot-tier run: 2026-10-26T00:28:11.000Z ++# Hot-tier run: 2026-10-26T00:29:10.000Z + +-Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 89 MB within 2 hours, with 1.5 GB a day of counted lines moved to the frozen tier. ++Hot tier: 11 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.2 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 2. +-- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 2 hours (default (no tier_down index has been moved yet)). +-- Each counted byte takes 1.45 bytes of store kept, 0.55 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 1.6 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 179 MB a day on hot whatever the limits; the size holds down to that floor. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 7 MB in 1. ++- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). ++- Each counted byte takes 1.33 bytes of store kept, 0.63 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 2.4 GB a day; 220 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 171 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +-- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 179 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 41 MB at about 89 MB. Raise the size, move the noisiest services to offload, or lower the sample. +-- Warning: Lines outside the Receiver and the tier_down lines waiting to move already fill 41 MB, so the Receiver's services keep nothing beyond the always-pass sample of 5 lines a statement a window. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4010 | +-| ad | the frozen tier | 1 | 51 MB | 2 MB | 1 | 5656 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 76 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 364 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1 | 464 | +-| cart | the frozen tier | 1 | 125 MB | 1 MB | 1 | 3627 | +-| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2923 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 109 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1 | 319 | +-| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1030 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 182 | +-| fluentd-10x | the frozen tier | 1 | 13 MB | 4 MB | 1 | 10989 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1 | 272 | +-| frontend | the frozen tier | 1 | 29 MB | 12 MB | 1 | 33837 | +-| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11999 | +-| grafana | the frozen tier | 1 | 54 MB | 50 MB | 1 | 138583 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1 | 797 | +-| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1127 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1 | 808 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2027 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1 | 670 | +-| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 1 | 8189 | +-| kafka | the frozen tier | 1 | 254 MB | 21 MB | 1 | 58207 | +-| kube-proxy | the frozen tier | 1 | 6 MB | 4 MB | 1 | 11250 | +-| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1 | 922 | +-| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2888 | +-| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 1 | 25520 | +-| opentelemetry-collector | the frozen tier | 1 | 1 GB | 54 MB | 1 | 150509 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 313 | +-| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2921 | +-| prometheus-server | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3988 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 50 | +-| recommendation | the frozen tier | 1 | 37 MB | 1 MB | 1 | 2695 | +-| valkey-cart | the frozen tier | 1 | 3 MB | 2 MB | 1 | 6921 | +-| wait-for-kafka | the frozen tier | 1 | 3 MB | 0 MB | 1 | 1201 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1 | 592 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1 | 2361 | ++| accounting | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 3784 | ++| ad | the frozen tier | 1 | 80 MB | 7 MB | 5991 | 18352 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 72 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 344 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 437 | ++| cart | the frozen tier | 1 | 200 MB | 4 MB | 5991 | 12499 | ++| coredns | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 2758 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 103 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 301 | ++| flagd | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 972 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 172 | ++| fluentd-10x | the frozen tier | 1 | 16 MB | 4 MB | 5991 | 11904 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 256 | ++| frontend | the frozen tier | 1 | 46 MB | 12 MB | 5991 | 33800 | ++| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 5991 | 11321 | ++| grafana | the frozen tier | 1 | 54 MB | 49 MB | 5991 | 136482 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 752 | ++| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 1064 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 762 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 1912 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 632 | ++| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 5991 | 7726 | ++| kafka | the frozen tier | 1 | 383 MB | 46 MB | 5991 | 127170 | ++| kube-proxy | the frozen tier | 1 | 6 MB | 6 MB | 5991 | 16828 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 870 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 2725 | ++| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 5991 | 26157 | ++| opentelemetry-collector | the frozen tier | 1 | 1.6 GB | 56 MB | 5991 | 155892 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 296 | ++| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 2670 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 5991 | 5347 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 47 | ++| recommendation | the frozen tier | 1 | 55 MB | 2 MB | 5991 | 6175 | ++| valkey-cart | the frozen tier | 1 | 5 MB | 5 MB | 5991 | 13340 | ++| wait-for-kafka | the frozen tier | 1 | 3 MB | 2 MB | 5991 | 6144 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 559 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 5991 | 2697 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/commit.json new file mode 100644 index 0000000..e2ce125 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/commit.json @@ -0,0 +1 @@ +{"before":"b38595b6f85a4836e82df549be73476fb17d161f","after":"8750a9199bc0d9c79f8dbab55ee4b043473189ec","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/config-generation.csv new file mode 100644 index 0000000..e07854e --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,e4464048f2c6 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/latest.md new file mode 100644 index 0000000..f56804e --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/latest.md @@ -0,0 +1,52 @@ +# Hot-tier run: 2026-10-26T00:29:10.000Z + +Hot tier: 11 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.2 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 7 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.33 bytes of store kept, 0.63 as tier_down (measured on the indices on hot). +- The Receiver's statements send 2.4 GB a day; 220 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 171 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 3784 | +| ad | the frozen tier | 1 | 80 MB | 7 MB | 5991 | 18352 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 72 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 344 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 437 | +| cart | the frozen tier | 1 | 200 MB | 4 MB | 5991 | 12499 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 2758 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 103 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 301 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 972 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 172 | +| fluentd-10x | the frozen tier | 1 | 16 MB | 4 MB | 5991 | 11904 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 256 | +| frontend | the frozen tier | 1 | 46 MB | 12 MB | 5991 | 33800 | +| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 5991 | 11321 | +| grafana | the frozen tier | 1 | 54 MB | 49 MB | 5991 | 136482 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 752 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 1064 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 762 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 1912 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 632 | +| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 5991 | 7726 | +| kafka | the frozen tier | 1 | 383 MB | 46 MB | 5991 | 127170 | +| kube-proxy | the frozen tier | 1 | 6 MB | 6 MB | 5991 | 16828 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 870 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 2725 | +| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 5991 | 26157 | +| opentelemetry-collector | the frozen tier | 1 | 1.6 GB | 56 MB | 5991 | 155892 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 296 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 2670 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 5991 | 5347 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 47 | +| recommendation | the frozen tier | 1 | 55 MB | 2 MB | 5991 | 6175 | +| valkey-cart | the frozen tier | 1 | 5 MB | 5 MB | 5991 | 13340 | +| wait-for-kafka | the frozen tier | 1 | 3 MB | 2 MB | 5991 | 6144 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 559 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 5991 | 2697 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/report.txt new file mode 100644 index 0000000..ee71e0b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/report.txt @@ -0,0 +1,61 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 11 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.2 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 7 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +[tenx-recur] Each counted byte takes 1.33 bytes of store kept, 0.63 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 2.4 GB a day; 220 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 171 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:29:10.000Z + +Hot tier: 11 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.2 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 7 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.33 bytes of store kept, 0.63 as tier_down (measured on the indices on hot). +- The Receiver's statements send 2.4 GB a day; 220 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 171 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 3784 | +| ad | the frozen tier | 1 | 80 MB | 7 MB | 5991 | 18352 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 72 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 344 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 437 | +| cart | the frozen tier | 1 | 200 MB | 4 MB | 5991 | 12499 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 2758 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 103 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 301 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 972 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 172 | +| fluentd-10x | the frozen tier | 1 | 16 MB | 4 MB | 5991 | 11904 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 256 | +| frontend | the frozen tier | 1 | 46 MB | 12 MB | 5991 | 33800 | +| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 5991 | 11321 | +| grafana | the frozen tier | 1 | 54 MB | 49 MB | 5991 | 136482 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 752 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 1064 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 762 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 1912 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 632 | +| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 5991 | 7726 | +| kafka | the frozen tier | 1 | 383 MB | 46 MB | 5991 | 127170 | +| kube-proxy | the frozen tier | 1 | 6 MB | 6 MB | 5991 | 16828 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 870 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 2725 | +| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 5991 | 26157 | +| opentelemetry-collector | the frozen tier | 1 | 1.6 GB | 56 MB | 5991 | 155892 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 296 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 2670 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 5991 | 5347 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 47 | +| recommendation | the frozen tier | 1 | 55 MB | 2 MB | 5991 | 6175 | +| valkey-cart | the frozen tier | 1 | 5 MB | 5 MB | 5991 | 13340 | +| wait-for-kafka | the frozen tier | 1 | 3 MB | 2 MB | 5991 | 6144 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 559 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 5991 | 2697 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/run.json new file mode 100644 index 0000000..8ee0f7b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:29:13.092Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 11 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.2 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/totals.csv new file mode 100644 index 0000000..bd29180 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T002909Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,3784::budget run +ad,18352::budget run +aws-eks-nodeagent,72::budget run +aws-node,344::budget run +aws-vpc-cni-init,437::budget run +cart,12499::budget run +coredns,2758::budget run +currency,103::budget run +email,301::budget run +flagd,972::budget run +flagd-ui,172::budget run +fluentd-10x,11904::budget run +fraud-detection,256::budget run +frontend,33800::budget run +frontend-proxy,11321::budget run +grafana,136482::budget run +grafana-sc-alerts,752::budget run +grafana-sc-dashboard,1064::budget run +grafana-sc-datasources,762::budget run +image-provider,1912::budget run +init-config,632::budget run +jaeger,7726::budget run +kafka,127170::budget run +kube-proxy,16828::budget run +load-generator,870::budget run +metrics-server,2725::budget run +opensearch,26157::budget run +opentelemetry-collector,155892::budget run +payment,296::budget run +postgresql,2670::budget run +prometheus-server,5347::budget run +quote,47::budget run +recommendation,6175::budget run +valkey-cart,13340::budget run +wait-for-kafka,6144::budget run +wait-for-valkey-cart,559::budget run +*,2697::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/caps.csv new file mode 100644 index 0000000..2fd505d --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,5154::budget run +ad,5154::budget run +aws-eks-nodeagent,5154::budget run +aws-node,5154::budget run +aws-vpc-cni-init,5154::budget run +cart,5154::budget run +coredns,5154::budget run +currency,5154::budget run +email,5154::budget run +flagd,5154::budget run +flagd-ui,5154::budget run +fluentd-10x,5154::budget run +fraud-detection,5154::budget run +frontend,5154::budget run +frontend-proxy,5154::budget run +grafana,5154::budget run +grafana-sc-alerts,5154::budget run +grafana-sc-dashboard,5154::budget run +grafana-sc-datasources,5154::budget run +image-provider,5154::budget run +init-config,5154::budget run +jaeger,5154::budget run +kafka,5154::budget run +kube-proxy,5154::budget run +load-generator,5154::budget run +metrics-server,5154::budget run +opensearch,5154::budget run +opentelemetry-collector,5154::budget run +payment,5154::budget run +postgresql,5154::budget run +prometheus-server,5154::budget run +quote,5154::budget run +recommendation,5154::budget run +valkey-cart,5154::budget run +wait-for-kafka,5154::budget run +wait-for-valkey-cart,5154::budget run +*,5154::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/commit.diff new file mode 100644 index 0000000..a51c196 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/commit.diff @@ -0,0 +1,268 @@ +diff --git a/log10x/policy/caps.csv b/log10x/policy/caps.csv +index f4acd8c..2fd505d 100644 +--- a/log10x/policy/caps.csv ++++ b/log10x/policy/caps.csv +@@ -1,38 +1,38 @@ + container,cap +-accounting,5991::budget run +-ad,5991::budget run +-aws-eks-nodeagent,5991::budget run +-aws-node,5991::budget run +-aws-vpc-cni-init,5991::budget run +-cart,5991::budget run +-coredns,5991::budget run +-currency,5991::budget run +-email,5991::budget run +-flagd,5991::budget run +-flagd-ui,5991::budget run +-fluentd-10x,5991::budget run +-fraud-detection,5991::budget run +-frontend,5991::budget run +-frontend-proxy,5991::budget run +-grafana,5991::budget run +-grafana-sc-alerts,5991::budget run +-grafana-sc-dashboard,5991::budget run +-grafana-sc-datasources,5991::budget run +-image-provider,5991::budget run +-init-config,5991::budget run +-jaeger,5991::budget run +-kafka,5991::budget run +-kube-proxy,5991::budget run +-load-generator,5991::budget run +-metrics-server,5991::budget run +-opensearch,5991::budget run +-opentelemetry-collector,5991::budget run +-payment,5991::budget run +-postgresql,5991::budget run +-prometheus-server,5991::budget run +-quote,5991::budget run +-recommendation,5991::budget run +-valkey-cart,5991::budget run +-wait-for-kafka,5991::budget run +-wait-for-valkey-cart,5991::budget run +-*,5991::budget run ++accounting,5154::budget run ++ad,5154::budget run ++aws-eks-nodeagent,5154::budget run ++aws-node,5154::budget run ++aws-vpc-cni-init,5154::budget run ++cart,5154::budget run ++coredns,5154::budget run ++currency,5154::budget run ++email,5154::budget run ++flagd,5154::budget run ++flagd-ui,5154::budget run ++fluentd-10x,5154::budget run ++fraud-detection,5154::budget run ++frontend,5154::budget run ++frontend-proxy,5154::budget run ++grafana,5154::budget run ++grafana-sc-alerts,5154::budget run ++grafana-sc-dashboard,5154::budget run ++grafana-sc-datasources,5154::budget run ++image-provider,5154::budget run ++init-config,5154::budget run ++jaeger,5154::budget run ++kafka,5154::budget run ++kube-proxy,5154::budget run ++load-generator,5154::budget run ++metrics-server,5154::budget run ++opensearch,5154::budget run ++opentelemetry-collector,5154::budget run ++payment,5154::budget run ++postgresql,5154::budget run ++prometheus-server,5154::budget run ++quote,5154::budget run ++recommendation,5154::budget run ++valkey-cart,5154::budget run ++wait-for-kafka,5154::budget run ++wait-for-valkey-cart,5154::budget run ++*,5154::budget run +diff --git a/log10x/policy/config-generation.csv b/log10x/policy/config-generation.csv +index e07854e..60fc9b6 100644 +--- a/log10x/policy/config-generation.csv ++++ b/log10x/policy/config-generation.csv +@@ -1,2 +1,2 @@ + key,value +-generation,e4464048f2c6 ++generation,7d2a21a237e4 +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index bd29180..a767c2d 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,3784::budget run +-ad,18352::budget run +-aws-eks-nodeagent,72::budget run +-aws-node,344::budget run +-aws-vpc-cni-init,437::budget run +-cart,12499::budget run +-coredns,2758::budget run +-currency,103::budget run +-email,301::budget run +-flagd,972::budget run +-flagd-ui,172::budget run +-fluentd-10x,11904::budget run +-fraud-detection,256::budget run +-frontend,33800::budget run +-frontend-proxy,11321::budget run +-grafana,136482::budget run +-grafana-sc-alerts,752::budget run +-grafana-sc-dashboard,1064::budget run +-grafana-sc-datasources,762::budget run +-image-provider,1912::budget run +-init-config,632::budget run +-jaeger,7726::budget run +-kafka,127170::budget run +-kube-proxy,16828::budget run +-load-generator,870::budget run +-metrics-server,2725::budget run +-opensearch,26157::budget run +-opentelemetry-collector,155892::budget run +-payment,296::budget run +-postgresql,2670::budget run +-prometheus-server,5347::budget run +-quote,47::budget run +-recommendation,6175::budget run +-valkey-cart,13340::budget run +-wait-for-kafka,6144::budget run +-wait-for-valkey-cart,559::budget run +-*,2697::budget run ++accounting,3585::budget run ++ad,15821::budget run ++aws-eks-nodeagent,68::budget run ++aws-node,325::budget run ++aws-vpc-cni-init,414::budget run ++cart,10798::budget run ++coredns,2613::budget run ++currency,97::budget run ++email,285::budget run ++flagd,921::budget run ++flagd-ui,162::budget run ++fluentd-10x,10944::budget run ++fraud-detection,243::budget run ++frontend,33767::budget run ++frontend-proxy,10726::budget run ++grafana,130285::budget run ++grafana-sc-alerts,712::budget run ++grafana-sc-dashboard,1008::budget run ++grafana-sc-datasources,722::budget run ++image-provider,1812::budget run ++init-config,598::budget run ++jaeger,7320::budget run ++kafka,113697::budget run ++kube-proxy,16151::budget run ++load-generator,825::budget run ++metrics-server,2582::budget run ++opensearch,25572::budget run ++opentelemetry-collector,146596::budget run ++payment,280::budget run ++postgresql,2529::budget run ++prometheus-server,5994::budget run ++quote,45::budget run ++recommendation,5328::budget run ++valkey-cart,15775::budget run ++wait-for-kafka,5299::budget run ++wait-for-valkey-cart,529::budget run ++*,2555::budget run +diff --git a/reports/latest.md b/reports/latest.md +index f56804e..1625f7d 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,52 +1,52 @@ +-# Hot-tier run: 2026-10-26T00:29:10.000Z ++# Hot-tier run: 2026-10-26T00:30:08.000Z + +-Hot tier: 11 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.2 GB a day of counted lines moved to the frozen tier. ++Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.5 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 7 MB in 1. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 8 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +-- Each counted byte takes 1.33 bytes of store kept, 0.63 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 2.4 GB a day; 220 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 171 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.39 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 2.7 GB a day; 207 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 165 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 3784 | +-| ad | the frozen tier | 1 | 80 MB | 7 MB | 5991 | 18352 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 72 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 344 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 437 | +-| cart | the frozen tier | 1 | 200 MB | 4 MB | 5991 | 12499 | +-| coredns | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 2758 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 103 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 301 | +-| flagd | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 972 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 172 | +-| fluentd-10x | the frozen tier | 1 | 16 MB | 4 MB | 5991 | 11904 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 256 | +-| frontend | the frozen tier | 1 | 46 MB | 12 MB | 5991 | 33800 | +-| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 5991 | 11321 | +-| grafana | the frozen tier | 1 | 54 MB | 49 MB | 5991 | 136482 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 752 | +-| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 1064 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 762 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 1912 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 632 | +-| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 5991 | 7726 | +-| kafka | the frozen tier | 1 | 383 MB | 46 MB | 5991 | 127170 | +-| kube-proxy | the frozen tier | 1 | 6 MB | 6 MB | 5991 | 16828 | +-| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 870 | +-| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 2725 | +-| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 5991 | 26157 | +-| opentelemetry-collector | the frozen tier | 1 | 1.6 GB | 56 MB | 5991 | 155892 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 296 | +-| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 5991 | 2670 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 5991 | 5347 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 47 | +-| recommendation | the frozen tier | 1 | 55 MB | 2 MB | 5991 | 6175 | +-| valkey-cart | the frozen tier | 1 | 5 MB | 5 MB | 5991 | 13340 | +-| wait-for-kafka | the frozen tier | 1 | 3 MB | 2 MB | 5991 | 6144 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 5991 | 559 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 5991 | 2697 | ++| accounting | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 3585 | ++| ad | the frozen tier | 1 | 91 MB | 6 MB | 5154 | 15821 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 68 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 325 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 414 | ++| cart | the frozen tier | 1 | 225 MB | 4 MB | 5154 | 10798 | ++| coredns | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 2613 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 97 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 285 | ++| flagd | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 921 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 162 | ++| fluentd-10x | the frozen tier | 1 | 18 MB | 4 MB | 5154 | 10944 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 243 | ++| frontend | the frozen tier | 1 | 53 MB | 12 MB | 5154 | 33767 | ++| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 5154 | 10726 | ++| grafana | the frozen tier | 1 | 53 MB | 47 MB | 5154 | 130285 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 712 | ++| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 1008 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 722 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 1812 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 598 | ++| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 5154 | 7320 | ++| kafka | the frozen tier | 1 | 423 MB | 41 MB | 5154 | 113697 | ++| kube-proxy | the frozen tier | 1 | 6 MB | 6 MB | 5154 | 16151 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 825 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 2582 | ++| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 5154 | 25572 | ++| opentelemetry-collector | the frozen tier | 1 | 1.7 GB | 53 MB | 5154 | 146596 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 280 | ++| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 2529 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 5154 | 5994 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 45 | ++| recommendation | the frozen tier | 1 | 59 MB | 2 MB | 5154 | 5328 | ++| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 5154 | 15775 | ++| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 5154 | 5299 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 529 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 5154 | 2555 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/commit.json new file mode 100644 index 0000000..6a401c0 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/commit.json @@ -0,0 +1 @@ +{"before":"8750a9199bc0d9c79f8dbab55ee4b043473189ec","after":"0cf7d84d09c22a6a34f532f78e9bfaeb4fb311c5","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/config-generation.csv new file mode 100644 index 0000000..60fc9b6 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,7d2a21a237e4 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/latest.md new file mode 100644 index 0000000..1625f7d --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/latest.md @@ -0,0 +1,52 @@ +# Hot-tier run: 2026-10-26T00:30:08.000Z + +Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.5 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 8 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.39 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +- The Receiver's statements send 2.7 GB a day; 207 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 165 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 3585 | +| ad | the frozen tier | 1 | 91 MB | 6 MB | 5154 | 15821 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 68 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 325 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 414 | +| cart | the frozen tier | 1 | 225 MB | 4 MB | 5154 | 10798 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 2613 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 97 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 285 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 921 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 162 | +| fluentd-10x | the frozen tier | 1 | 18 MB | 4 MB | 5154 | 10944 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 243 | +| frontend | the frozen tier | 1 | 53 MB | 12 MB | 5154 | 33767 | +| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 5154 | 10726 | +| grafana | the frozen tier | 1 | 53 MB | 47 MB | 5154 | 130285 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 712 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 1008 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 722 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 1812 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 598 | +| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 5154 | 7320 | +| kafka | the frozen tier | 1 | 423 MB | 41 MB | 5154 | 113697 | +| kube-proxy | the frozen tier | 1 | 6 MB | 6 MB | 5154 | 16151 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 825 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 2582 | +| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 5154 | 25572 | +| opentelemetry-collector | the frozen tier | 1 | 1.7 GB | 53 MB | 5154 | 146596 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 280 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 2529 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 5154 | 5994 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 45 | +| recommendation | the frozen tier | 1 | 59 MB | 2 MB | 5154 | 5328 | +| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 5154 | 15775 | +| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 5154 | 5299 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 529 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 5154 | 2555 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/report.txt new file mode 100644 index 0000000..732195e --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/report.txt @@ -0,0 +1,61 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.5 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 8 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +[tenx-recur] Each counted byte takes 1.39 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 2.7 GB a day; 207 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 165 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:30:08.000Z + +Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.5 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 8 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.39 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +- The Receiver's statements send 2.7 GB a day; 207 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 165 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 3585 | +| ad | the frozen tier | 1 | 91 MB | 6 MB | 5154 | 15821 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 68 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 325 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 414 | +| cart | the frozen tier | 1 | 225 MB | 4 MB | 5154 | 10798 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 2613 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 97 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 285 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 921 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 162 | +| fluentd-10x | the frozen tier | 1 | 18 MB | 4 MB | 5154 | 10944 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 243 | +| frontend | the frozen tier | 1 | 53 MB | 12 MB | 5154 | 33767 | +| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 5154 | 10726 | +| grafana | the frozen tier | 1 | 53 MB | 47 MB | 5154 | 130285 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 712 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 1008 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 722 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 1812 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 598 | +| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 5154 | 7320 | +| kafka | the frozen tier | 1 | 423 MB | 41 MB | 5154 | 113697 | +| kube-proxy | the frozen tier | 1 | 6 MB | 6 MB | 5154 | 16151 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 825 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 2582 | +| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 5154 | 25572 | +| opentelemetry-collector | the frozen tier | 1 | 1.7 GB | 53 MB | 5154 | 146596 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 280 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 2529 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 5154 | 5994 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 45 | +| recommendation | the frozen tier | 1 | 59 MB | 2 MB | 5154 | 5328 | +| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 5154 | 15775 | +| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 5154 | 5299 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 529 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 5154 | 2555 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/run.json new file mode 100644 index 0000000..2fc4300 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:30:11.741Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.5 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/totals.csv new file mode 100644 index 0000000..a767c2d --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003008Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,3585::budget run +ad,15821::budget run +aws-eks-nodeagent,68::budget run +aws-node,325::budget run +aws-vpc-cni-init,414::budget run +cart,10798::budget run +coredns,2613::budget run +currency,97::budget run +email,285::budget run +flagd,921::budget run +flagd-ui,162::budget run +fluentd-10x,10944::budget run +fraud-detection,243::budget run +frontend,33767::budget run +frontend-proxy,10726::budget run +grafana,130285::budget run +grafana-sc-alerts,712::budget run +grafana-sc-dashboard,1008::budget run +grafana-sc-datasources,722::budget run +image-provider,1812::budget run +init-config,598::budget run +jaeger,7320::budget run +kafka,113697::budget run +kube-proxy,16151::budget run +load-generator,825::budget run +metrics-server,2582::budget run +opensearch,25572::budget run +opentelemetry-collector,146596::budget run +payment,280::budget run +postgresql,2529::budget run +prometheus-server,5994::budget run +quote,45::budget run +recommendation,5328::budget run +valkey-cart,15775::budget run +wait-for-kafka,5299::budget run +wait-for-valkey-cart,529::budget run +*,2555::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/caps.csv new file mode 100644 index 0000000..eb318b1 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,3081::budget run +ad,3081::budget run +aws-eks-nodeagent,3081::budget run +aws-node,3081::budget run +aws-vpc-cni-init,3081::budget run +cart,3081::budget run +coredns,3081::budget run +currency,3081::budget run +email,3081::budget run +flagd,3081::budget run +flagd-ui,3081::budget run +fluentd-10x,3081::budget run +fraud-detection,3081::budget run +frontend,3081::budget run +frontend-proxy,3081::budget run +grafana,3081::budget run +grafana-sc-alerts,3081::budget run +grafana-sc-dashboard,3081::budget run +grafana-sc-datasources,3081::budget run +image-provider,3081::budget run +init-config,3081::budget run +jaeger,3081::budget run +kafka,3081::budget run +kube-proxy,3081::budget run +load-generator,3081::budget run +metrics-server,3081::budget run +opensearch,3081::budget run +opentelemetry-collector,3081::budget run +payment,3081::budget run +postgresql,3081::budget run +prometheus-server,3081::budget run +quote,3081::budget run +recommendation,3081::budget run +valkey-cart,3081::budget run +wait-for-kafka,3081::budget run +wait-for-valkey-cart,3081::budget run +*,3081::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/commit.diff new file mode 100644 index 0000000..7dccf00 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/commit.diff @@ -0,0 +1,268 @@ +diff --git a/log10x/policy/caps.csv b/log10x/policy/caps.csv +index 2fd505d..eb318b1 100644 +--- a/log10x/policy/caps.csv ++++ b/log10x/policy/caps.csv +@@ -1,38 +1,38 @@ + container,cap +-accounting,5154::budget run +-ad,5154::budget run +-aws-eks-nodeagent,5154::budget run +-aws-node,5154::budget run +-aws-vpc-cni-init,5154::budget run +-cart,5154::budget run +-coredns,5154::budget run +-currency,5154::budget run +-email,5154::budget run +-flagd,5154::budget run +-flagd-ui,5154::budget run +-fluentd-10x,5154::budget run +-fraud-detection,5154::budget run +-frontend,5154::budget run +-frontend-proxy,5154::budget run +-grafana,5154::budget run +-grafana-sc-alerts,5154::budget run +-grafana-sc-dashboard,5154::budget run +-grafana-sc-datasources,5154::budget run +-image-provider,5154::budget run +-init-config,5154::budget run +-jaeger,5154::budget run +-kafka,5154::budget run +-kube-proxy,5154::budget run +-load-generator,5154::budget run +-metrics-server,5154::budget run +-opensearch,5154::budget run +-opentelemetry-collector,5154::budget run +-payment,5154::budget run +-postgresql,5154::budget run +-prometheus-server,5154::budget run +-quote,5154::budget run +-recommendation,5154::budget run +-valkey-cart,5154::budget run +-wait-for-kafka,5154::budget run +-wait-for-valkey-cart,5154::budget run +-*,5154::budget run ++accounting,3081::budget run ++ad,3081::budget run ++aws-eks-nodeagent,3081::budget run ++aws-node,3081::budget run ++aws-vpc-cni-init,3081::budget run ++cart,3081::budget run ++coredns,3081::budget run ++currency,3081::budget run ++email,3081::budget run ++flagd,3081::budget run ++flagd-ui,3081::budget run ++fluentd-10x,3081::budget run ++fraud-detection,3081::budget run ++frontend,3081::budget run ++frontend-proxy,3081::budget run ++grafana,3081::budget run ++grafana-sc-alerts,3081::budget run ++grafana-sc-dashboard,3081::budget run ++grafana-sc-datasources,3081::budget run ++image-provider,3081::budget run ++init-config,3081::budget run ++jaeger,3081::budget run ++kafka,3081::budget run ++kube-proxy,3081::budget run ++load-generator,3081::budget run ++metrics-server,3081::budget run ++opensearch,3081::budget run ++opentelemetry-collector,3081::budget run ++payment,3081::budget run ++postgresql,3081::budget run ++prometheus-server,3081::budget run ++quote,3081::budget run ++recommendation,3081::budget run ++valkey-cart,3081::budget run ++wait-for-kafka,3081::budget run ++wait-for-valkey-cart,3081::budget run ++*,3081::budget run +diff --git a/log10x/policy/config-generation.csv b/log10x/policy/config-generation.csv +index 60fc9b6..417d31c 100644 +--- a/log10x/policy/config-generation.csv ++++ b/log10x/policy/config-generation.csv +@@ -1,2 +1,2 @@ + key,value +-generation,7d2a21a237e4 ++generation,53f9121356c1 +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index a767c2d..969299a 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,3585::budget run +-ad,15821::budget run +-aws-eks-nodeagent,68::budget run +-aws-node,325::budget run +-aws-vpc-cni-init,414::budget run +-cart,10798::budget run +-coredns,2613::budget run +-currency,97::budget run +-email,285::budget run +-flagd,921::budget run +-flagd-ui,162::budget run +-fluentd-10x,10944::budget run +-fraud-detection,243::budget run +-frontend,33767::budget run +-frontend-proxy,10726::budget run +-grafana,130285::budget run +-grafana-sc-alerts,712::budget run +-grafana-sc-dashboard,1008::budget run +-grafana-sc-datasources,722::budget run +-image-provider,1812::budget run +-init-config,598::budget run +-jaeger,7320::budget run +-kafka,113697::budget run +-kube-proxy,16151::budget run +-load-generator,825::budget run +-metrics-server,2582::budget run +-opensearch,25572::budget run +-opentelemetry-collector,146596::budget run +-payment,280::budget run +-postgresql,2529::budget run +-prometheus-server,5994::budget run +-quote,45::budget run +-recommendation,5328::budget run +-valkey-cart,15775::budget run +-wait-for-kafka,5299::budget run +-wait-for-valkey-cart,529::budget run +-*,2555::budget run ++accounting,3388::budget run ++ad,9581::budget run ++aws-eks-nodeagent,64::budget run ++aws-node,308::budget run ++aws-vpc-cni-init,392::budget run ++cart,6624::budget run ++coredns,2469::budget run ++currency,92::budget run ++email,269::budget run ++flagd,870::budget run ++flagd-ui,154::budget run ++fluentd-10x,8553::budget run ++fraud-detection,230::budget run ++frontend,33734::budget run ++frontend-proxy,10138::budget run ++grafana,122725::budget run ++grafana-sc-alerts,673::budget run ++grafana-sc-dashboard,952::budget run ++grafana-sc-datasources,683::budget run ++image-provider,1712::budget run ++init-config,566::budget run ++jaeger,6919::budget run ++kafka,78110::budget run ++kube-proxy,14525::budget run ++load-generator,779::budget run ++metrics-server,2440::budget run ++opensearch,25525::budget run ++opentelemetry-collector,133965::budget run ++payment,265::budget run ++postgresql,2390::budget run ++prometheus-server,5813::budget run ++quote,42::budget run ++recommendation,3245::budget run ++valkey-cart,15762::budget run ++wait-for-kafka,3217::budget run ++wait-for-valkey-cart,500::budget run ++*,2415::budget run +diff --git a/reports/latest.md b/reports/latest.md +index 1625f7d..e35ef5c 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,52 +1,52 @@ +-# Hot-tier run: 2026-10-26T00:30:08.000Z ++# Hot-tier run: 2026-10-26T00:31:12.000Z + +-Hot tier: 12 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.5 GB a day of counted lines moved to the frozen tier. ++Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.7 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 3 MB in 2 indices; its tier_down lines not yet moved: 8 MB in 1. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +-- Each counted byte takes 1.39 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 2.7 GB a day; 207 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 165 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.59 bytes of store kept, 0.53 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 2.9 GB a day; 179 MB a day fits the size, so each statement keeps up to 1 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 160 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 3585 | +-| ad | the frozen tier | 1 | 91 MB | 6 MB | 5154 | 15821 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 68 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 325 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 414 | +-| cart | the frozen tier | 1 | 225 MB | 4 MB | 5154 | 10798 | +-| coredns | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 2613 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 97 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 285 | +-| flagd | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 921 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 162 | +-| fluentd-10x | the frozen tier | 1 | 18 MB | 4 MB | 5154 | 10944 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 243 | +-| frontend | the frozen tier | 1 | 53 MB | 12 MB | 5154 | 33767 | +-| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 5154 | 10726 | +-| grafana | the frozen tier | 1 | 53 MB | 47 MB | 5154 | 130285 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 712 | +-| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 1008 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 722 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 1812 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 598 | +-| jaeger | the frozen tier | 1 | 3 MB | 3 MB | 5154 | 7320 | +-| kafka | the frozen tier | 1 | 423 MB | 41 MB | 5154 | 113697 | +-| kube-proxy | the frozen tier | 1 | 6 MB | 6 MB | 5154 | 16151 | +-| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 825 | +-| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 2582 | +-| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 5154 | 25572 | +-| opentelemetry-collector | the frozen tier | 1 | 1.7 GB | 53 MB | 5154 | 146596 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 280 | +-| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 5154 | 2529 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 5154 | 5994 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 45 | +-| recommendation | the frozen tier | 1 | 59 MB | 2 MB | 5154 | 5328 | +-| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 5154 | 15775 | +-| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 5154 | 5299 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 5154 | 529 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 5154 | 2555 | ++| accounting | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 3388 | ++| ad | the frozen tier | 1 | 100 MB | 3 MB | 3081 | 9581 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 64 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 308 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 392 | ++| cart | the frozen tier | 1 | 238 MB | 2 MB | 3081 | 6624 | ++| coredns | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 2469 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 92 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 269 | ++| flagd | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 870 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 154 | ++| fluentd-10x | the frozen tier | 1 | 19 MB | 3 MB | 3081 | 8553 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 230 | ++| frontend | the frozen tier | 1 | 57 MB | 12 MB | 3081 | 33734 | ++| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 3081 | 10138 | ++| grafana | the frozen tier | 1 | 52 MB | 44 MB | 3081 | 122725 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 673 | ++| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 952 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 683 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 1712 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 566 | ++| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 3081 | 6919 | ++| kafka | the frozen tier | 1 | 455 MB | 28 MB | 3081 | 78110 | ++| kube-proxy | the frozen tier | 1 | 6 MB | 5 MB | 3081 | 14525 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 779 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 2440 | ++| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 3081 | 25525 | ++| opentelemetry-collector | the frozen tier | 1 | 1.9 GB | 48 MB | 3081 | 133965 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 265 | ++| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 2390 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 3081 | 5813 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 42 | ++| recommendation | the frozen tier | 1 | 62 MB | 1 MB | 3081 | 3245 | ++| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 3081 | 15762 | ++| wait-for-kafka | the frozen tier | 1 | 2 MB | 1 MB | 3081 | 3217 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 500 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 3081 | 2415 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/commit.json new file mode 100644 index 0000000..b50ce34 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/commit.json @@ -0,0 +1 @@ +{"before":"0cf7d84d09c22a6a34f532f78e9bfaeb4fb311c5","after":"74d3d473d13dfd1093ff84a85d1dcd948e2bbdaf","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/config-generation.csv new file mode 100644 index 0000000..417d31c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,53f9121356c1 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/latest.md new file mode 100644 index 0000000..e35ef5c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/latest.md @@ -0,0 +1,52 @@ +# Hot-tier run: 2026-10-26T00:31:12.000Z + +Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.7 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.59 bytes of store kept, 0.53 as tier_down (measured on the indices on hot). +- The Receiver's statements send 2.9 GB a day; 179 MB a day fits the size, so each statement keeps up to 1 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 160 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 3388 | +| ad | the frozen tier | 1 | 100 MB | 3 MB | 3081 | 9581 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 64 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 308 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 392 | +| cart | the frozen tier | 1 | 238 MB | 2 MB | 3081 | 6624 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 2469 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 92 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 269 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 870 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 154 | +| fluentd-10x | the frozen tier | 1 | 19 MB | 3 MB | 3081 | 8553 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 230 | +| frontend | the frozen tier | 1 | 57 MB | 12 MB | 3081 | 33734 | +| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 3081 | 10138 | +| grafana | the frozen tier | 1 | 52 MB | 44 MB | 3081 | 122725 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 673 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 952 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 683 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 1712 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 566 | +| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 3081 | 6919 | +| kafka | the frozen tier | 1 | 455 MB | 28 MB | 3081 | 78110 | +| kube-proxy | the frozen tier | 1 | 6 MB | 5 MB | 3081 | 14525 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 779 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 2440 | +| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 3081 | 25525 | +| opentelemetry-collector | the frozen tier | 1 | 1.9 GB | 48 MB | 3081 | 133965 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 265 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 2390 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 3081 | 5813 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 42 | +| recommendation | the frozen tier | 1 | 62 MB | 1 MB | 3081 | 3245 | +| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 3081 | 15762 | +| wait-for-kafka | the frozen tier | 1 | 2 MB | 1 MB | 3081 | 3217 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 500 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 3081 | 2415 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/report.txt new file mode 100644 index 0000000..3ec5d50 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/report.txt @@ -0,0 +1,61 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.7 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +[tenx-recur] Each counted byte takes 1.59 bytes of store kept, 0.53 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 2.9 GB a day; 179 MB a day fits the size, so each statement keeps up to 1 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 160 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:31:12.000Z + +Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.7 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.59 bytes of store kept, 0.53 as tier_down (measured on the indices on hot). +- The Receiver's statements send 2.9 GB a day; 179 MB a day fits the size, so each statement keeps up to 1 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 160 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 3388 | +| ad | the frozen tier | 1 | 100 MB | 3 MB | 3081 | 9581 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 64 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 308 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 392 | +| cart | the frozen tier | 1 | 238 MB | 2 MB | 3081 | 6624 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 2469 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 92 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 269 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 870 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 154 | +| fluentd-10x | the frozen tier | 1 | 19 MB | 3 MB | 3081 | 8553 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 230 | +| frontend | the frozen tier | 1 | 57 MB | 12 MB | 3081 | 33734 | +| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 3081 | 10138 | +| grafana | the frozen tier | 1 | 52 MB | 44 MB | 3081 | 122725 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 673 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 952 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 683 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 1712 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 566 | +| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 3081 | 6919 | +| kafka | the frozen tier | 1 | 455 MB | 28 MB | 3081 | 78110 | +| kube-proxy | the frozen tier | 1 | 6 MB | 5 MB | 3081 | 14525 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 779 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 2440 | +| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 3081 | 25525 | +| opentelemetry-collector | the frozen tier | 1 | 1.9 GB | 48 MB | 3081 | 133965 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 265 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 2390 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 3081 | 5813 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 42 | +| recommendation | the frozen tier | 1 | 62 MB | 1 MB | 3081 | 3245 | +| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 3081 | 15762 | +| wait-for-kafka | the frozen tier | 1 | 2 MB | 1 MB | 3081 | 3217 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 500 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 3081 | 2415 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/run.json new file mode 100644 index 0000000..a4fc627 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:31:15.358Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.7 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/totals.csv new file mode 100644 index 0000000..969299a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003112Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,3388::budget run +ad,9581::budget run +aws-eks-nodeagent,64::budget run +aws-node,308::budget run +aws-vpc-cni-init,392::budget run +cart,6624::budget run +coredns,2469::budget run +currency,92::budget run +email,269::budget run +flagd,870::budget run +flagd-ui,154::budget run +fluentd-10x,8553::budget run +fraud-detection,230::budget run +frontend,33734::budget run +frontend-proxy,10138::budget run +grafana,122725::budget run +grafana-sc-alerts,673::budget run +grafana-sc-dashboard,952::budget run +grafana-sc-datasources,683::budget run +image-provider,1712::budget run +init-config,566::budget run +jaeger,6919::budget run +kafka,78110::budget run +kube-proxy,14525::budget run +load-generator,779::budget run +metrics-server,2440::budget run +opensearch,25525::budget run +opentelemetry-collector,133965::budget run +payment,265::budget run +postgresql,2390::budget run +prometheus-server,5813::budget run +quote,42::budget run +recommendation,3245::budget run +valkey-cart,15762::budget run +wait-for-kafka,3217::budget run +wait-for-valkey-cart,500::budget run +*,2415::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/caps.csv new file mode 100644 index 0000000..ede54d2 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,4051::budget run +ad,4051::budget run +aws-eks-nodeagent,4051::budget run +aws-node,4051::budget run +aws-vpc-cni-init,4051::budget run +cart,4051::budget run +coredns,4051::budget run +currency,4051::budget run +email,4051::budget run +flagd,4051::budget run +flagd-ui,4051::budget run +fluentd-10x,4051::budget run +fraud-detection,4051::budget run +frontend,4051::budget run +frontend-proxy,4051::budget run +grafana,4051::budget run +grafana-sc-alerts,4051::budget run +grafana-sc-dashboard,4051::budget run +grafana-sc-datasources,4051::budget run +image-provider,4051::budget run +init-config,4051::budget run +jaeger,4051::budget run +kafka,4051::budget run +kube-proxy,4051::budget run +load-generator,4051::budget run +metrics-server,4051::budget run +opensearch,4051::budget run +opentelemetry-collector,4051::budget run +payment,4051::budget run +postgresql,4051::budget run +prometheus-server,4051::budget run +quote,4051::budget run +recommendation,4051::budget run +valkey-cart,4051::budget run +wait-for-kafka,4051::budget run +wait-for-valkey-cart,4051::budget run +*,4051::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/commit.diff new file mode 100644 index 0000000..2f63702 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/commit.diff @@ -0,0 +1,268 @@ +diff --git a/log10x/policy/caps.csv b/log10x/policy/caps.csv +index eb318b1..ede54d2 100644 +--- a/log10x/policy/caps.csv ++++ b/log10x/policy/caps.csv +@@ -1,38 +1,38 @@ + container,cap +-accounting,3081::budget run +-ad,3081::budget run +-aws-eks-nodeagent,3081::budget run +-aws-node,3081::budget run +-aws-vpc-cni-init,3081::budget run +-cart,3081::budget run +-coredns,3081::budget run +-currency,3081::budget run +-email,3081::budget run +-flagd,3081::budget run +-flagd-ui,3081::budget run +-fluentd-10x,3081::budget run +-fraud-detection,3081::budget run +-frontend,3081::budget run +-frontend-proxy,3081::budget run +-grafana,3081::budget run +-grafana-sc-alerts,3081::budget run +-grafana-sc-dashboard,3081::budget run +-grafana-sc-datasources,3081::budget run +-image-provider,3081::budget run +-init-config,3081::budget run +-jaeger,3081::budget run +-kafka,3081::budget run +-kube-proxy,3081::budget run +-load-generator,3081::budget run +-metrics-server,3081::budget run +-opensearch,3081::budget run +-opentelemetry-collector,3081::budget run +-payment,3081::budget run +-postgresql,3081::budget run +-prometheus-server,3081::budget run +-quote,3081::budget run +-recommendation,3081::budget run +-valkey-cart,3081::budget run +-wait-for-kafka,3081::budget run +-wait-for-valkey-cart,3081::budget run +-*,3081::budget run ++accounting,4051::budget run ++ad,4051::budget run ++aws-eks-nodeagent,4051::budget run ++aws-node,4051::budget run ++aws-vpc-cni-init,4051::budget run ++cart,4051::budget run ++coredns,4051::budget run ++currency,4051::budget run ++email,4051::budget run ++flagd,4051::budget run ++flagd-ui,4051::budget run ++fluentd-10x,4051::budget run ++fraud-detection,4051::budget run ++frontend,4051::budget run ++frontend-proxy,4051::budget run ++grafana,4051::budget run ++grafana-sc-alerts,4051::budget run ++grafana-sc-dashboard,4051::budget run ++grafana-sc-datasources,4051::budget run ++image-provider,4051::budget run ++init-config,4051::budget run ++jaeger,4051::budget run ++kafka,4051::budget run ++kube-proxy,4051::budget run ++load-generator,4051::budget run ++metrics-server,4051::budget run ++opensearch,4051::budget run ++opentelemetry-collector,4051::budget run ++payment,4051::budget run ++postgresql,4051::budget run ++prometheus-server,4051::budget run ++quote,4051::budget run ++recommendation,4051::budget run ++valkey-cart,4051::budget run ++wait-for-kafka,4051::budget run ++wait-for-valkey-cart,4051::budget run ++*,4051::budget run +diff --git a/log10x/policy/config-generation.csv b/log10x/policy/config-generation.csv +index 417d31c..ce5c893 100644 +--- a/log10x/policy/config-generation.csv ++++ b/log10x/policy/config-generation.csv +@@ -1,2 +1,2 @@ + key,value +-generation,53f9121356c1 ++generation,946754def770 +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 969299a..e1c921f 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,3388::budget run +-ad,9581::budget run +-aws-eks-nodeagent,64::budget run +-aws-node,308::budget run +-aws-vpc-cni-init,392::budget run +-cart,6624::budget run +-coredns,2469::budget run +-currency,92::budget run +-email,269::budget run +-flagd,870::budget run +-flagd-ui,154::budget run +-fluentd-10x,8553::budget run +-fraud-detection,230::budget run +-frontend,33734::budget run +-frontend-proxy,10138::budget run +-grafana,122725::budget run +-grafana-sc-alerts,673::budget run +-grafana-sc-dashboard,952::budget run +-grafana-sc-datasources,683::budget run +-image-provider,1712::budget run +-init-config,566::budget run +-jaeger,6919::budget run +-kafka,78110::budget run +-kube-proxy,14525::budget run +-load-generator,779::budget run +-metrics-server,2440::budget run +-opensearch,25525::budget run +-opentelemetry-collector,133965::budget run +-payment,265::budget run +-postgresql,2390::budget run +-prometheus-server,5813::budget run +-quote,42::budget run +-recommendation,3245::budget run +-valkey-cart,15762::budget run +-wait-for-kafka,3217::budget run +-wait-for-valkey-cart,500::budget run +-*,2415::budget run ++accounting,3225::budget run ++ad,12477::budget run ++aws-eks-nodeagent,61::budget run ++aws-node,293::budget run ++aws-vpc-cni-init,373::budget run ++cart,8544::budget run ++coredns,2351::budget run ++currency,88::budget run ++email,256::budget run ++flagd,828::budget run ++flagd-ui,146::budget run ++fluentd-10x,9261::budget run ++fraud-detection,219::budget run ++frontend,33706::budget run ++frontend-proxy,9650::budget run ++grafana,119257::budget run ++grafana-sc-alerts,641::budget run ++grafana-sc-dashboard,907::budget run ++grafana-sc-datasources,650::budget run ++image-provider,1630::budget run ++init-config,538::budget run ++jaeger,6586::budget run ++kafka,94276::budget run ++kube-proxy,14903::budget run ++load-generator,742::budget run ++metrics-server,2323::budget run ++opensearch,25247::budget run ++opentelemetry-collector,132004::budget run ++payment,252::budget run ++postgresql,2275::budget run ++prometheus-server,5894::budget run ++quote,40::budget run ++recommendation,4208::budget run ++valkey-cart,17153::budget run ++wait-for-kafka,4182::budget run ++wait-for-valkey-cart,476::budget run ++*,2299::budget run +diff --git a/reports/latest.md b/reports/latest.md +index e35ef5c..5b383cb 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,52 +1,52 @@ +-# Hot-tier run: 2026-10-26T00:31:12.000Z ++# Hot-tier run: 2026-10-26T00:32:11.000Z + +-Hot tier: 13 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.7 GB a day of counted lines moved to the frozen tier. ++Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.8 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 2 indices; its tier_down lines not yet moved: 9 MB in 1. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +-- Each counted byte takes 1.59 bytes of store kept, 0.53 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 2.9 GB a day; 179 MB a day fits the size, so each statement keeps up to 1 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 160 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.41 bytes of store kept, 0.57 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 3 GB a day; 186 MB a day fits the size, so each statement keeps up to 1 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 155 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 3388 | +-| ad | the frozen tier | 1 | 100 MB | 3 MB | 3081 | 9581 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 64 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 308 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 392 | +-| cart | the frozen tier | 1 | 238 MB | 2 MB | 3081 | 6624 | +-| coredns | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 2469 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 92 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 269 | +-| flagd | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 870 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 154 | +-| fluentd-10x | the frozen tier | 1 | 19 MB | 3 MB | 3081 | 8553 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 230 | +-| frontend | the frozen tier | 1 | 57 MB | 12 MB | 3081 | 33734 | +-| frontend-proxy | the frozen tier | 1 | 4 MB | 4 MB | 3081 | 10138 | +-| grafana | the frozen tier | 1 | 52 MB | 44 MB | 3081 | 122725 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 673 | +-| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 952 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 683 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 1712 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 566 | +-| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 3081 | 6919 | +-| kafka | the frozen tier | 1 | 455 MB | 28 MB | 3081 | 78110 | +-| kube-proxy | the frozen tier | 1 | 6 MB | 5 MB | 3081 | 14525 | +-| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 779 | +-| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 2440 | +-| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 3081 | 25525 | +-| opentelemetry-collector | the frozen tier | 1 | 1.9 GB | 48 MB | 3081 | 133965 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 265 | +-| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 3081 | 2390 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 3081 | 5813 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 42 | +-| recommendation | the frozen tier | 1 | 62 MB | 1 MB | 3081 | 3245 | +-| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 3081 | 15762 | +-| wait-for-kafka | the frozen tier | 1 | 2 MB | 1 MB | 3081 | 3217 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 3081 | 500 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 3081 | 2415 | ++| accounting | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 3225 | ++| ad | the frozen tier | 1 | 106 MB | 4 MB | 4051 | 12477 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 61 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 293 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 373 | ++| cart | the frozen tier | 1 | 251 MB | 3 MB | 4051 | 8544 | ++| coredns | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 2351 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 88 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 256 | ++| flagd | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 828 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 146 | ++| fluentd-10x | the frozen tier | 1 | 20 MB | 3 MB | 4051 | 9261 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 219 | ++| frontend | the frozen tier | 1 | 58 MB | 12 MB | 4051 | 33706 | ++| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 4051 | 9650 | ++| grafana | the frozen tier | 1 | 50 MB | 43 MB | 4051 | 119257 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 641 | ++| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 907 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 650 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 1630 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 538 | ++| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 4051 | 6586 | ++| kafka | the frozen tier | 1 | 476 MB | 34 MB | 4051 | 94276 | ++| kube-proxy | the frozen tier | 1 | 5 MB | 5 MB | 4051 | 14903 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 742 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 2323 | ++| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 4051 | 25247 | ++| opentelemetry-collector | the frozen tier | 1 | 2 GB | 48 MB | 4051 | 132004 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 252 | ++| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 2275 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 4051 | 5894 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 40 | ++| recommendation | the frozen tier | 1 | 67 MB | 2 MB | 4051 | 4208 | ++| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 4051 | 17153 | ++| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 4051 | 4182 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 476 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 4051 | 2299 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/commit.json new file mode 100644 index 0000000..e9fdafe --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/commit.json @@ -0,0 +1 @@ +{"before":"74d3d473d13dfd1093ff84a85d1dcd948e2bbdaf","after":"3aee0d6bca3d40f7ee6c384766b051e592608944","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/config-generation.csv new file mode 100644 index 0000000..ce5c893 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,946754def770 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/latest.md new file mode 100644 index 0000000..5b383cb --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/latest.md @@ -0,0 +1,52 @@ +# Hot-tier run: 2026-10-26T00:32:11.000Z + +Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.8 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.41 bytes of store kept, 0.57 as tier_down (measured on the indices on hot). +- The Receiver's statements send 3 GB a day; 186 MB a day fits the size, so each statement keeps up to 1 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 155 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 3225 | +| ad | the frozen tier | 1 | 106 MB | 4 MB | 4051 | 12477 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 61 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 293 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 373 | +| cart | the frozen tier | 1 | 251 MB | 3 MB | 4051 | 8544 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 2351 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 88 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 256 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 828 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 146 | +| fluentd-10x | the frozen tier | 1 | 20 MB | 3 MB | 4051 | 9261 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 219 | +| frontend | the frozen tier | 1 | 58 MB | 12 MB | 4051 | 33706 | +| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 4051 | 9650 | +| grafana | the frozen tier | 1 | 50 MB | 43 MB | 4051 | 119257 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 641 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 907 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 650 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 1630 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 538 | +| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 4051 | 6586 | +| kafka | the frozen tier | 1 | 476 MB | 34 MB | 4051 | 94276 | +| kube-proxy | the frozen tier | 1 | 5 MB | 5 MB | 4051 | 14903 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 742 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 2323 | +| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 4051 | 25247 | +| opentelemetry-collector | the frozen tier | 1 | 2 GB | 48 MB | 4051 | 132004 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 252 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 2275 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 4051 | 5894 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 40 | +| recommendation | the frozen tier | 1 | 67 MB | 2 MB | 4051 | 4208 | +| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 4051 | 17153 | +| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 4051 | 4182 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 476 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 4051 | 2299 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/report.txt new file mode 100644 index 0000000..f7d2bb9 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/report.txt @@ -0,0 +1,61 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.8 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +[tenx-recur] Each counted byte takes 1.41 bytes of store kept, 0.57 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 3 GB a day; 186 MB a day fits the size, so each statement keeps up to 1 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 155 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:32:11.000Z + +Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.8 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.41 bytes of store kept, 0.57 as tier_down (measured on the indices on hot). +- The Receiver's statements send 3 GB a day; 186 MB a day fits the size, so each statement keeps up to 1 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 155 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 3225 | +| ad | the frozen tier | 1 | 106 MB | 4 MB | 4051 | 12477 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 61 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 293 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 373 | +| cart | the frozen tier | 1 | 251 MB | 3 MB | 4051 | 8544 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 2351 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 88 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 256 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 828 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 146 | +| fluentd-10x | the frozen tier | 1 | 20 MB | 3 MB | 4051 | 9261 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 219 | +| frontend | the frozen tier | 1 | 58 MB | 12 MB | 4051 | 33706 | +| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 4051 | 9650 | +| grafana | the frozen tier | 1 | 50 MB | 43 MB | 4051 | 119257 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 641 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 907 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 650 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 1630 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 538 | +| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 4051 | 6586 | +| kafka | the frozen tier | 1 | 476 MB | 34 MB | 4051 | 94276 | +| kube-proxy | the frozen tier | 1 | 5 MB | 5 MB | 4051 | 14903 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 742 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 2323 | +| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 4051 | 25247 | +| opentelemetry-collector | the frozen tier | 1 | 2 GB | 48 MB | 4051 | 132004 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 252 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 2275 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 4051 | 5894 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 40 | +| recommendation | the frozen tier | 1 | 67 MB | 2 MB | 4051 | 4208 | +| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 4051 | 17153 | +| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 4051 | 4182 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 476 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 4051 | 2299 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/run.json new file mode 100644 index 0000000..c37e1fb --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:32:13.932Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.8 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/totals.csv new file mode 100644 index 0000000..e1c921f --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003210Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,3225::budget run +ad,12477::budget run +aws-eks-nodeagent,61::budget run +aws-node,293::budget run +aws-vpc-cni-init,373::budget run +cart,8544::budget run +coredns,2351::budget run +currency,88::budget run +email,256::budget run +flagd,828::budget run +flagd-ui,146::budget run +fluentd-10x,9261::budget run +fraud-detection,219::budget run +frontend,33706::budget run +frontend-proxy,9650::budget run +grafana,119257::budget run +grafana-sc-alerts,641::budget run +grafana-sc-dashboard,907::budget run +grafana-sc-datasources,650::budget run +image-provider,1630::budget run +init-config,538::budget run +jaeger,6586::budget run +kafka,94276::budget run +kube-proxy,14903::budget run +load-generator,742::budget run +metrics-server,2323::budget run +opensearch,25247::budget run +opentelemetry-collector,132004::budget run +payment,252::budget run +postgresql,2275::budget run +prometheus-server,5894::budget run +quote,40::budget run +recommendation,4208::budget run +valkey-cart,17153::budget run +wait-for-kafka,4182::budget run +wait-for-valkey-cart,476::budget run +*,2299::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/caps.csv new file mode 100644 index 0000000..08efb8b --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,1194::budget run +ad,1194::budget run +aws-eks-nodeagent,1194::budget run +aws-node,1194::budget run +aws-vpc-cni-init,1194::budget run +cart,1194::budget run +coredns,1194::budget run +currency,1194::budget run +email,1194::budget run +flagd,1194::budget run +flagd-ui,1194::budget run +fluentd-10x,1194::budget run +fraud-detection,1194::budget run +frontend,1194::budget run +frontend-proxy,1194::budget run +grafana,1194::budget run +grafana-sc-alerts,1194::budget run +grafana-sc-dashboard,1194::budget run +grafana-sc-datasources,1194::budget run +image-provider,1194::budget run +init-config,1194::budget run +jaeger,1194::budget run +kafka,1194::budget run +kube-proxy,1194::budget run +load-generator,1194::budget run +metrics-server,1194::budget run +opensearch,1194::budget run +opentelemetry-collector,1194::budget run +payment,1194::budget run +postgresql,1194::budget run +prometheus-server,1194::budget run +quote,1194::budget run +recommendation,1194::budget run +valkey-cart,1194::budget run +wait-for-kafka,1194::budget run +wait-for-valkey-cart,1194::budget run +*,1194::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/commit.diff new file mode 100644 index 0000000..60390a6 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/commit.diff @@ -0,0 +1,268 @@ +diff --git a/log10x/policy/caps.csv b/log10x/policy/caps.csv +index ede54d2..08efb8b 100644 +--- a/log10x/policy/caps.csv ++++ b/log10x/policy/caps.csv +@@ -1,38 +1,38 @@ + container,cap +-accounting,4051::budget run +-ad,4051::budget run +-aws-eks-nodeagent,4051::budget run +-aws-node,4051::budget run +-aws-vpc-cni-init,4051::budget run +-cart,4051::budget run +-coredns,4051::budget run +-currency,4051::budget run +-email,4051::budget run +-flagd,4051::budget run +-flagd-ui,4051::budget run +-fluentd-10x,4051::budget run +-fraud-detection,4051::budget run +-frontend,4051::budget run +-frontend-proxy,4051::budget run +-grafana,4051::budget run +-grafana-sc-alerts,4051::budget run +-grafana-sc-dashboard,4051::budget run +-grafana-sc-datasources,4051::budget run +-image-provider,4051::budget run +-init-config,4051::budget run +-jaeger,4051::budget run +-kafka,4051::budget run +-kube-proxy,4051::budget run +-load-generator,4051::budget run +-metrics-server,4051::budget run +-opensearch,4051::budget run +-opentelemetry-collector,4051::budget run +-payment,4051::budget run +-postgresql,4051::budget run +-prometheus-server,4051::budget run +-quote,4051::budget run +-recommendation,4051::budget run +-valkey-cart,4051::budget run +-wait-for-kafka,4051::budget run +-wait-for-valkey-cart,4051::budget run +-*,4051::budget run ++accounting,1194::budget run ++ad,1194::budget run ++aws-eks-nodeagent,1194::budget run ++aws-node,1194::budget run ++aws-vpc-cni-init,1194::budget run ++cart,1194::budget run ++coredns,1194::budget run ++currency,1194::budget run ++email,1194::budget run ++flagd,1194::budget run ++flagd-ui,1194::budget run ++fluentd-10x,1194::budget run ++fraud-detection,1194::budget run ++frontend,1194::budget run ++frontend-proxy,1194::budget run ++grafana,1194::budget run ++grafana-sc-alerts,1194::budget run ++grafana-sc-dashboard,1194::budget run ++grafana-sc-datasources,1194::budget run ++image-provider,1194::budget run ++init-config,1194::budget run ++jaeger,1194::budget run ++kafka,1194::budget run ++kube-proxy,1194::budget run ++load-generator,1194::budget run ++metrics-server,1194::budget run ++opensearch,1194::budget run ++opentelemetry-collector,1194::budget run ++payment,1194::budget run ++postgresql,1194::budget run ++prometheus-server,1194::budget run ++quote,1194::budget run ++recommendation,1194::budget run ++valkey-cart,1194::budget run ++wait-for-kafka,1194::budget run ++wait-for-valkey-cart,1194::budget run ++*,1194::budget run +diff --git a/log10x/policy/config-generation.csv b/log10x/policy/config-generation.csv +index ce5c893..52b5199 100644 +--- a/log10x/policy/config-generation.csv ++++ b/log10x/policy/config-generation.csv +@@ -1,2 +1,2 @@ + key,value +-generation,946754def770 ++generation,f2c3ec363524 +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index e1c921f..a2ed0c8 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,3225::budget run +-ad,12477::budget run +-aws-eks-nodeagent,61::budget run +-aws-node,293::budget run +-aws-vpc-cni-init,373::budget run +-cart,8544::budget run +-coredns,2351::budget run +-currency,88::budget run +-email,256::budget run +-flagd,828::budget run +-flagd-ui,146::budget run +-fluentd-10x,9261::budget run +-fraud-detection,219::budget run +-frontend,33706::budget run +-frontend-proxy,9650::budget run +-grafana,119257::budget run +-grafana-sc-alerts,641::budget run +-grafana-sc-dashboard,907::budget run +-grafana-sc-datasources,650::budget run +-image-provider,1630::budget run +-init-config,538::budget run +-jaeger,6586::budget run +-kafka,94276::budget run +-kube-proxy,14903::budget run +-load-generator,742::budget run +-metrics-server,2323::budget run +-opensearch,25247::budget run +-opentelemetry-collector,132004::budget run +-payment,252::budget run +-postgresql,2275::budget run +-prometheus-server,5894::budget run +-quote,40::budget run +-recommendation,4208::budget run +-valkey-cart,17153::budget run +-wait-for-kafka,4182::budget run +-wait-for-valkey-cart,476::budget run +-*,2299::budget run ++accounting,3079::budget run ++ad,5563::budget run ++aws-eks-nodeagent,59::budget run ++aws-node,279::budget run ++aws-vpc-cni-init,356::budget run ++cart,3503::budget run ++coredns,2244::budget run ++currency,84::budget run ++email,245::budget run ++flagd,791::budget run ++flagd-ui,139::budget run ++fluentd-10x,6709::budget run ++fraud-detection,209::budget run ++frontend,33682::budget run ++frontend-proxy,9214::budget run ++grafana,112124::budget run ++grafana-sc-alerts,612::budget run ++grafana-sc-dashboard,866::budget run ++grafana-sc-datasources,620::budget run ++image-provider,1556::budget run ++init-config,514::budget run ++jaeger,6288::budget run ++kafka,53701::budget run ++kube-proxy,9742::budget run ++load-generator,708::budget run ++metrics-server,2218::budget run ++opensearch,22484::budget run ++opentelemetry-collector,123327::budget run ++payment,240::budget run ++postgresql,2173::budget run ++prometheus-server,4382::budget run ++quote,39::budget run ++recommendation,2649::budget run ++valkey-cart,6817::budget run ++wait-for-kafka,1318::budget run ++wait-for-valkey-cart,455::budget run ++*,1864::budget run +diff --git a/reports/latest.md b/reports/latest.md +index 5b383cb..e3ce607 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,52 +1,52 @@ +-# Hot-tier run: 2026-10-26T00:32:11.000Z ++# Hot-tier run: 2026-10-26T00:33:09.000Z + +-Hot tier: 15 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.8 GB a day of counted lines moved to the frozen tier. ++Hot tier: 17 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 3 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 2 indices; its tier_down lines not yet moved: 12 MB in 1. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 2 indices; its tier_down lines not yet moved: 13 MB in 1. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +-- Each counted byte takes 1.41 bytes of store kept, 0.57 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 3 GB a day; 186 MB a day fits the size, so each statement keeps up to 1 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 155 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.74 bytes of store kept, 0.53 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 3.2 GB a day; 151 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 151 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 3225 | +-| ad | the frozen tier | 1 | 106 MB | 4 MB | 4051 | 12477 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 61 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 293 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 373 | +-| cart | the frozen tier | 1 | 251 MB | 3 MB | 4051 | 8544 | +-| coredns | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 2351 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 88 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 256 | +-| flagd | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 828 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 146 | +-| fluentd-10x | the frozen tier | 1 | 20 MB | 3 MB | 4051 | 9261 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 219 | +-| frontend | the frozen tier | 1 | 58 MB | 12 MB | 4051 | 33706 | +-| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 4051 | 9650 | +-| grafana | the frozen tier | 1 | 50 MB | 43 MB | 4051 | 119257 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 641 | +-| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 907 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 650 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 1630 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 538 | +-| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 4051 | 6586 | +-| kafka | the frozen tier | 1 | 476 MB | 34 MB | 4051 | 94276 | +-| kube-proxy | the frozen tier | 1 | 5 MB | 5 MB | 4051 | 14903 | +-| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 742 | +-| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 2323 | +-| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 4051 | 25247 | +-| opentelemetry-collector | the frozen tier | 1 | 2 GB | 48 MB | 4051 | 132004 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 252 | +-| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 4051 | 2275 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 4051 | 5894 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 40 | +-| recommendation | the frozen tier | 1 | 67 MB | 2 MB | 4051 | 4208 | +-| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 4051 | 17153 | +-| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 4051 | 4182 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 4051 | 476 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 4051 | 2299 | ++| accounting | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 3079 | ++| ad | the frozen tier | 1 | 110 MB | 2 MB | 1194 | 5563 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 59 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 279 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 356 | ++| cart | the frozen tier | 1 | 263 MB | 1 MB | 1194 | 3503 | ++| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 2244 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 84 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 245 | ++| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 791 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 139 | ++| fluentd-10x | the frozen tier | 1 | 20 MB | 2 MB | 1194 | 6709 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 209 | ++| frontend | the frozen tier | 1 | 62 MB | 12 MB | 1194 | 33682 | ++| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 1194 | 9214 | ++| grafana | the frozen tier | 1 | 49 MB | 40 MB | 1194 | 112124 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 612 | ++| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 866 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 620 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 1556 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 514 | ++| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 1194 | 6288 | ++| kafka | the frozen tier | 1 | 497 MB | 19 MB | 1194 | 53701 | ++| kube-proxy | the frozen tier | 1 | 5 MB | 4 MB | 1194 | 9742 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 708 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 2218 | ++| opensearch | the frozen tier | 1 | 9 MB | 8 MB | 1194 | 22484 | ++| opentelemetry-collector | the frozen tier | 1 | 2 GB | 44 MB | 1194 | 123327 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 240 | ++| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 2173 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1194 | 4382 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 39 | ++| recommendation | the frozen tier | 1 | 71 MB | 1 MB | 1194 | 2649 | ++| valkey-cart | the frozen tier | 1 | 6 MB | 2 MB | 1194 | 6817 | ++| wait-for-kafka | the frozen tier | 1 | 2 MB | 0 MB | 1194 | 1318 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 455 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1194 | 1864 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/commit.json new file mode 100644 index 0000000..687f040 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/commit.json @@ -0,0 +1 @@ +{"before":"3aee0d6bca3d40f7ee6c384766b051e592608944","after":"5b00b5892c77762542b5dac9d49f134eba096e59","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/config-generation.csv new file mode 100644 index 0000000..52b5199 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,f2c3ec363524 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/latest.md new file mode 100644 index 0000000..e3ce607 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/latest.md @@ -0,0 +1,52 @@ +# Hot-tier run: 2026-10-26T00:33:09.000Z + +Hot tier: 17 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 3 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 2 indices; its tier_down lines not yet moved: 13 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.74 bytes of store kept, 0.53 as tier_down (measured on the indices on hot). +- The Receiver's statements send 3.2 GB a day; 151 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 151 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 3079 | +| ad | the frozen tier | 1 | 110 MB | 2 MB | 1194 | 5563 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 59 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 279 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 356 | +| cart | the frozen tier | 1 | 263 MB | 1 MB | 1194 | 3503 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 2244 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 84 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 245 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 791 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 139 | +| fluentd-10x | the frozen tier | 1 | 20 MB | 2 MB | 1194 | 6709 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 209 | +| frontend | the frozen tier | 1 | 62 MB | 12 MB | 1194 | 33682 | +| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 1194 | 9214 | +| grafana | the frozen tier | 1 | 49 MB | 40 MB | 1194 | 112124 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 612 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 866 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 620 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 1556 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 514 | +| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 1194 | 6288 | +| kafka | the frozen tier | 1 | 497 MB | 19 MB | 1194 | 53701 | +| kube-proxy | the frozen tier | 1 | 5 MB | 4 MB | 1194 | 9742 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 708 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 2218 | +| opensearch | the frozen tier | 1 | 9 MB | 8 MB | 1194 | 22484 | +| opentelemetry-collector | the frozen tier | 1 | 2 GB | 44 MB | 1194 | 123327 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 240 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 2173 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1194 | 4382 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 39 | +| recommendation | the frozen tier | 1 | 71 MB | 1 MB | 1194 | 2649 | +| valkey-cart | the frozen tier | 1 | 6 MB | 2 MB | 1194 | 6817 | +| wait-for-kafka | the frozen tier | 1 | 2 MB | 0 MB | 1194 | 1318 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 455 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1194 | 1864 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/report.txt new file mode 100644 index 0000000..aebb0bb --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/report.txt @@ -0,0 +1,61 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 17 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 3 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 2 indices; its tier_down lines not yet moved: 13 MB in 1. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +[tenx-recur] Each counted byte takes 1.74 bytes of store kept, 0.53 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 3.2 GB a day; 151 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 151 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:33:09.000Z + +Hot tier: 17 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 3 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 2 indices; its tier_down lines not yet moved: 13 MB in 1. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.74 bytes of store kept, 0.53 as tier_down (measured on the indices on hot). +- The Receiver's statements send 3.2 GB a day; 151 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 151 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 3079 | +| ad | the frozen tier | 1 | 110 MB | 2 MB | 1194 | 5563 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 59 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 279 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 356 | +| cart | the frozen tier | 1 | 263 MB | 1 MB | 1194 | 3503 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 2244 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 84 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 245 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 791 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 139 | +| fluentd-10x | the frozen tier | 1 | 20 MB | 2 MB | 1194 | 6709 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 209 | +| frontend | the frozen tier | 1 | 62 MB | 12 MB | 1194 | 33682 | +| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 1194 | 9214 | +| grafana | the frozen tier | 1 | 49 MB | 40 MB | 1194 | 112124 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 612 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 866 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 620 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 1556 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 514 | +| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 1194 | 6288 | +| kafka | the frozen tier | 1 | 497 MB | 19 MB | 1194 | 53701 | +| kube-proxy | the frozen tier | 1 | 5 MB | 4 MB | 1194 | 9742 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 708 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 2218 | +| opensearch | the frozen tier | 1 | 9 MB | 8 MB | 1194 | 22484 | +| opentelemetry-collector | the frozen tier | 1 | 2 GB | 44 MB | 1194 | 123327 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 240 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 2173 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1194 | 4382 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 39 | +| recommendation | the frozen tier | 1 | 71 MB | 1 MB | 1194 | 2649 | +| valkey-cart | the frozen tier | 1 | 6 MB | 2 MB | 1194 | 6817 | +| wait-for-kafka | the frozen tier | 1 | 2 MB | 0 MB | 1194 | 1318 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 455 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1194 | 1864 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/run.json new file mode 100644 index 0000000..339f87f --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:33:12.300Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 17 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 3 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/totals.csv new file mode 100644 index 0000000..a2ed0c8 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003309Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,3079::budget run +ad,5563::budget run +aws-eks-nodeagent,59::budget run +aws-node,279::budget run +aws-vpc-cni-init,356::budget run +cart,3503::budget run +coredns,2244::budget run +currency,84::budget run +email,245::budget run +flagd,791::budget run +flagd-ui,139::budget run +fluentd-10x,6709::budget run +fraud-detection,209::budget run +frontend,33682::budget run +frontend-proxy,9214::budget run +grafana,112124::budget run +grafana-sc-alerts,612::budget run +grafana-sc-dashboard,866::budget run +grafana-sc-datasources,620::budget run +image-provider,1556::budget run +init-config,514::budget run +jaeger,6288::budget run +kafka,53701::budget run +kube-proxy,9742::budget run +load-generator,708::budget run +metrics-server,2218::budget run +opensearch,22484::budget run +opentelemetry-collector,123327::budget run +payment,240::budget run +postgresql,2173::budget run +prometheus-server,4382::budget run +quote,39::budget run +recommendation,2649::budget run +valkey-cart,6817::budget run +wait-for-kafka,1318::budget run +wait-for-valkey-cart,455::budget run +*,1864::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/caps.csv new file mode 100644 index 0000000..8b572bc --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,4768::budget run +ad,4768::budget run +aws-eks-nodeagent,4768::budget run +aws-node,4768::budget run +aws-vpc-cni-init,4768::budget run +cart,4768::budget run +coredns,4768::budget run +currency,4768::budget run +email,4768::budget run +flagd,4768::budget run +flagd-ui,4768::budget run +fluentd-10x,4768::budget run +fraud-detection,4768::budget run +frontend,4768::budget run +frontend-proxy,4768::budget run +grafana,4768::budget run +grafana-sc-alerts,4768::budget run +grafana-sc-dashboard,4768::budget run +grafana-sc-datasources,4768::budget run +image-provider,4768::budget run +init-config,4768::budget run +jaeger,4768::budget run +kafka,4768::budget run +kube-proxy,4768::budget run +load-generator,4768::budget run +metrics-server,4768::budget run +opensearch,4768::budget run +opentelemetry-collector,4768::budget run +payment,4768::budget run +postgresql,4768::budget run +prometheus-server,4768::budget run +quote,4768::budget run +recommendation,4768::budget run +valkey-cart,4768::budget run +wait-for-kafka,4768::budget run +wait-for-valkey-cart,4768::budget run +*,4768::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/commit.diff new file mode 100644 index 0000000..a680492 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/commit.diff @@ -0,0 +1,268 @@ +diff --git a/log10x/policy/caps.csv b/log10x/policy/caps.csv +index 08efb8b..8b572bc 100644 +--- a/log10x/policy/caps.csv ++++ b/log10x/policy/caps.csv +@@ -1,38 +1,38 @@ + container,cap +-accounting,1194::budget run +-ad,1194::budget run +-aws-eks-nodeagent,1194::budget run +-aws-node,1194::budget run +-aws-vpc-cni-init,1194::budget run +-cart,1194::budget run +-coredns,1194::budget run +-currency,1194::budget run +-email,1194::budget run +-flagd,1194::budget run +-flagd-ui,1194::budget run +-fluentd-10x,1194::budget run +-fraud-detection,1194::budget run +-frontend,1194::budget run +-frontend-proxy,1194::budget run +-grafana,1194::budget run +-grafana-sc-alerts,1194::budget run +-grafana-sc-dashboard,1194::budget run +-grafana-sc-datasources,1194::budget run +-image-provider,1194::budget run +-init-config,1194::budget run +-jaeger,1194::budget run +-kafka,1194::budget run +-kube-proxy,1194::budget run +-load-generator,1194::budget run +-metrics-server,1194::budget run +-opensearch,1194::budget run +-opentelemetry-collector,1194::budget run +-payment,1194::budget run +-postgresql,1194::budget run +-prometheus-server,1194::budget run +-quote,1194::budget run +-recommendation,1194::budget run +-valkey-cart,1194::budget run +-wait-for-kafka,1194::budget run +-wait-for-valkey-cart,1194::budget run +-*,1194::budget run ++accounting,4768::budget run ++ad,4768::budget run ++aws-eks-nodeagent,4768::budget run ++aws-node,4768::budget run ++aws-vpc-cni-init,4768::budget run ++cart,4768::budget run ++coredns,4768::budget run ++currency,4768::budget run ++email,4768::budget run ++flagd,4768::budget run ++flagd-ui,4768::budget run ++fluentd-10x,4768::budget run ++fraud-detection,4768::budget run ++frontend,4768::budget run ++frontend-proxy,4768::budget run ++grafana,4768::budget run ++grafana-sc-alerts,4768::budget run ++grafana-sc-dashboard,4768::budget run ++grafana-sc-datasources,4768::budget run ++image-provider,4768::budget run ++init-config,4768::budget run ++jaeger,4768::budget run ++kafka,4768::budget run ++kube-proxy,4768::budget run ++load-generator,4768::budget run ++metrics-server,4768::budget run ++opensearch,4768::budget run ++opentelemetry-collector,4768::budget run ++payment,4768::budget run ++postgresql,4768::budget run ++prometheus-server,4768::budget run ++quote,4768::budget run ++recommendation,4768::budget run ++valkey-cart,4768::budget run ++wait-for-kafka,4768::budget run ++wait-for-valkey-cart,4768::budget run ++*,4768::budget run +diff --git a/log10x/policy/config-generation.csv b/log10x/policy/config-generation.csv +index 52b5199..b34cbf8 100644 +--- a/log10x/policy/config-generation.csv ++++ b/log10x/policy/config-generation.csv +@@ -1,2 +1,2 @@ + key,value +-generation,f2c3ec363524 ++generation,bbddfb473fb5 +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index a2ed0c8..7bd44df 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,3079::budget run +-ad,5563::budget run +-aws-eks-nodeagent,59::budget run +-aws-node,279::budget run +-aws-vpc-cni-init,356::budget run +-cart,3503::budget run +-coredns,2244::budget run +-currency,84::budget run +-email,245::budget run +-flagd,791::budget run +-flagd-ui,139::budget run +-fluentd-10x,6709::budget run +-fraud-detection,209::budget run +-frontend,33682::budget run +-frontend-proxy,9214::budget run +-grafana,112124::budget run +-grafana-sc-alerts,612::budget run +-grafana-sc-dashboard,866::budget run +-grafana-sc-datasources,620::budget run +-image-provider,1556::budget run +-init-config,514::budget run +-jaeger,6288::budget run +-kafka,53701::budget run +-kube-proxy,9742::budget run +-load-generator,708::budget run +-metrics-server,2218::budget run +-opensearch,22484::budget run +-opentelemetry-collector,123327::budget run +-payment,240::budget run +-postgresql,2173::budget run +-prometheus-server,4382::budget run +-quote,39::budget run +-recommendation,2649::budget run +-valkey-cart,6817::budget run +-wait-for-kafka,1318::budget run +-wait-for-valkey-cart,455::budget run +-*,1864::budget run ++accounting,2937::budget run ++ad,14597::budget run ++aws-eks-nodeagent,56::budget run ++aws-node,267::budget run ++aws-vpc-cni-init,340::budget run ++cart,9977::budget run ++coredns,2141::budget run ++currency,80::budget run ++email,233::budget run ++flagd,754::budget run ++flagd-ui,133::budget run ++fluentd-10x,9513::budget run ++fraud-detection,199::budget run ++frontend,33660::budget run ++frontend-proxy,8790::budget run ++grafana,111334::budget run ++grafana-sc-alerts,584::budget run ++grafana-sc-dashboard,826::budget run ++grafana-sc-datasources,592::budget run ++image-provider,1485::budget run ++init-config,490::budget run ++jaeger,5999::budget run ++kafka,100246::budget run ++kube-proxy,13914::budget run ++load-generator,676::budget run ++metrics-server,2115::budget run ++opensearch,23928::budget run ++opentelemetry-collector,125706::budget run ++payment,229::budget run ++postgresql,2072::budget run ++prometheus-server,6079::budget run ++quote,37::budget run ++recommendation,4911::budget run ++valkey-cart,16973::budget run ++wait-for-kafka,4886::budget run ++wait-for-valkey-cart,434::budget run ++*,2094::budget run +diff --git a/reports/latest.md b/reports/latest.md +index e3ce607..8d1f8bf 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,52 +1,52 @@ +-# Hot-tier run: 2026-10-26T00:33:09.000Z ++# Hot-tier run: 2026-10-26T00:34:11.000Z + +-Hot tier: 17 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 3 GB a day of counted lines moved to the frozen tier. ++Hot tier: 18 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.9 GB a day of counted lines moved to the frozen tier. + +-- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 5 MB in 2 indices; its tier_down lines not yet moved: 13 MB in 1. ++- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 3 indices; its tier_down lines not yet moved: 15 MB in 2. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +-- Each counted byte takes 1.74 bytes of store kept, 0.53 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 3.2 GB a day; 151 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 151 MB a day on hot whatever the limits; the size holds down to that floor. ++- Each counted byte takes 1.37 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). ++- The Receiver's statements send 3.1 GB a day; 183 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 146 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 3079 | +-| ad | the frozen tier | 1 | 110 MB | 2 MB | 1194 | 5563 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 59 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 279 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 356 | +-| cart | the frozen tier | 1 | 263 MB | 1 MB | 1194 | 3503 | +-| coredns | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 2244 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 84 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 245 | +-| flagd | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 791 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 139 | +-| fluentd-10x | the frozen tier | 1 | 20 MB | 2 MB | 1194 | 6709 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 209 | +-| frontend | the frozen tier | 1 | 62 MB | 12 MB | 1194 | 33682 | +-| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 1194 | 9214 | +-| grafana | the frozen tier | 1 | 49 MB | 40 MB | 1194 | 112124 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 612 | +-| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 866 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 620 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 1556 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 514 | +-| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 1194 | 6288 | +-| kafka | the frozen tier | 1 | 497 MB | 19 MB | 1194 | 53701 | +-| kube-proxy | the frozen tier | 1 | 5 MB | 4 MB | 1194 | 9742 | +-| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 708 | +-| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 2218 | +-| opensearch | the frozen tier | 1 | 9 MB | 8 MB | 1194 | 22484 | +-| opentelemetry-collector | the frozen tier | 1 | 2 GB | 44 MB | 1194 | 123327 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 240 | +-| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1194 | 2173 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1194 | 4382 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 39 | +-| recommendation | the frozen tier | 1 | 71 MB | 1 MB | 1194 | 2649 | +-| valkey-cart | the frozen tier | 1 | 6 MB | 2 MB | 1194 | 6817 | +-| wait-for-kafka | the frozen tier | 1 | 2 MB | 0 MB | 1194 | 1318 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 1194 | 455 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 1194 | 1864 | ++| accounting | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2937 | ++| ad | the frozen tier | 1 | 107 MB | 5 MB | 4768 | 14597 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 56 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 267 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 340 | ++| cart | the frozen tier | 1 | 260 MB | 4 MB | 4768 | 9977 | ++| coredns | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2141 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 80 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 233 | ++| flagd | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 754 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 133 | ++| fluentd-10x | the frozen tier | 1 | 20 MB | 3 MB | 4768 | 9513 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 199 | ++| frontend | the frozen tier | 1 | 62 MB | 12 MB | 4768 | 33660 | ++| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 4768 | 8790 | ++| grafana | the frozen tier | 1 | 47 MB | 40 MB | 4768 | 111334 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 584 | ++| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 826 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 592 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 1485 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 490 | ++| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 4768 | 5999 | ++| kafka | the frozen tier | 1 | 490 MB | 36 MB | 4768 | 100246 | ++| kube-proxy | the frozen tier | 1 | 5 MB | 5 MB | 4768 | 13914 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 676 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2115 | ++| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 4768 | 23928 | ++| opentelemetry-collector | the frozen tier | 1 | 2 GB | 45 MB | 4768 | 125706 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 229 | ++| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2072 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 4768 | 6079 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 37 | ++| recommendation | the frozen tier | 1 | 70 MB | 2 MB | 4768 | 4911 | ++| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 4768 | 16973 | ++| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 4768 | 4886 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 434 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 4768 | 2094 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/commit.json new file mode 100644 index 0000000..c9ad6a1 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/commit.json @@ -0,0 +1 @@ +{"before":"5b00b5892c77762542b5dac9d49f134eba096e59","after":"11c9221f78ca19363f57e2bbed1752c4ae6105bc","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/config-generation.csv new file mode 100644 index 0000000..b34cbf8 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,bbddfb473fb5 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/latest.md new file mode 100644 index 0000000..8d1f8bf --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/latest.md @@ -0,0 +1,52 @@ +# Hot-tier run: 2026-10-26T00:34:11.000Z + +Hot tier: 18 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.9 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 3 indices; its tier_down lines not yet moved: 15 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.37 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +- The Receiver's statements send 3.1 GB a day; 183 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 146 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2937 | +| ad | the frozen tier | 1 | 107 MB | 5 MB | 4768 | 14597 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 56 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 267 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 340 | +| cart | the frozen tier | 1 | 260 MB | 4 MB | 4768 | 9977 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2141 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 80 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 233 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 754 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 133 | +| fluentd-10x | the frozen tier | 1 | 20 MB | 3 MB | 4768 | 9513 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 199 | +| frontend | the frozen tier | 1 | 62 MB | 12 MB | 4768 | 33660 | +| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 4768 | 8790 | +| grafana | the frozen tier | 1 | 47 MB | 40 MB | 4768 | 111334 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 584 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 826 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 592 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 1485 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 490 | +| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 4768 | 5999 | +| kafka | the frozen tier | 1 | 490 MB | 36 MB | 4768 | 100246 | +| kube-proxy | the frozen tier | 1 | 5 MB | 5 MB | 4768 | 13914 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 676 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2115 | +| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 4768 | 23928 | +| opentelemetry-collector | the frozen tier | 1 | 2 GB | 45 MB | 4768 | 125706 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 229 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2072 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 4768 | 6079 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 37 | +| recommendation | the frozen tier | 1 | 70 MB | 2 MB | 4768 | 4911 | +| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 4768 | 16973 | +| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 4768 | 4886 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 434 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 4768 | 2094 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/report.txt new file mode 100644 index 0000000..55c92ef --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/report.txt @@ -0,0 +1,61 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 18 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.9 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 3 indices; its tier_down lines not yet moved: 15 MB in 2. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +[tenx-recur] Each counted byte takes 1.37 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 3.1 GB a day; 183 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 146 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:34:11.000Z + +Hot tier: 18 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.9 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 3 indices; its tier_down lines not yet moved: 15 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.37 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +- The Receiver's statements send 3.1 GB a day; 183 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 146 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2937 | +| ad | the frozen tier | 1 | 107 MB | 5 MB | 4768 | 14597 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 56 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 267 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 340 | +| cart | the frozen tier | 1 | 260 MB | 4 MB | 4768 | 9977 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2141 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 80 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 233 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 754 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 133 | +| fluentd-10x | the frozen tier | 1 | 20 MB | 3 MB | 4768 | 9513 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 199 | +| frontend | the frozen tier | 1 | 62 MB | 12 MB | 4768 | 33660 | +| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 4768 | 8790 | +| grafana | the frozen tier | 1 | 47 MB | 40 MB | 4768 | 111334 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 584 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 826 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 592 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 1485 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 490 | +| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 4768 | 5999 | +| kafka | the frozen tier | 1 | 490 MB | 36 MB | 4768 | 100246 | +| kube-proxy | the frozen tier | 1 | 5 MB | 5 MB | 4768 | 13914 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 676 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2115 | +| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 4768 | 23928 | +| opentelemetry-collector | the frozen tier | 1 | 2 GB | 45 MB | 4768 | 125706 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 229 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2072 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 4768 | 6079 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 37 | +| recommendation | the frozen tier | 1 | 70 MB | 2 MB | 4768 | 4911 | +| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 4768 | 16973 | +| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 4768 | 4886 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 434 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 4768 | 2094 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/run.json new file mode 100644 index 0000000..8e11360 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:34:14.149Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 18 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.9 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/totals.csv new file mode 100644 index 0000000..7bd44df --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003410Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,2937::budget run +ad,14597::budget run +aws-eks-nodeagent,56::budget run +aws-node,267::budget run +aws-vpc-cni-init,340::budget run +cart,9977::budget run +coredns,2141::budget run +currency,80::budget run +email,233::budget run +flagd,754::budget run +flagd-ui,133::budget run +fluentd-10x,9513::budget run +fraud-detection,199::budget run +frontend,33660::budget run +frontend-proxy,8790::budget run +grafana,111334::budget run +grafana-sc-alerts,584::budget run +grafana-sc-dashboard,826::budget run +grafana-sc-datasources,592::budget run +image-provider,1485::budget run +init-config,490::budget run +jaeger,5999::budget run +kafka,100246::budget run +kube-proxy,13914::budget run +load-generator,676::budget run +metrics-server,2115::budget run +opensearch,23928::budget run +opentelemetry-collector,125706::budget run +payment,229::budget run +postgresql,2072::budget run +prometheus-server,6079::budget run +quote,37::budget run +recommendation,4911::budget run +valkey-cart,16973::budget run +wait-for-kafka,4886::budget run +wait-for-valkey-cart,434::budget run +*,2094::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/actions.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/actions.csv new file mode 100644 index 0000000..ca15017 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/actions.csv @@ -0,0 +1,38 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +image-provider,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +valkey-cart,tier_down::budget run +wait-for-kafka,tier_down::budget run +wait-for-valkey-cart,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/caps.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/caps.csv new file mode 100644 index 0000000..81671a1 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/caps.csv @@ -0,0 +1,38 @@ +container,cap +accounting,6855::budget run +ad,6855::budget run +aws-eks-nodeagent,6855::budget run +aws-node,6855::budget run +aws-vpc-cni-init,6855::budget run +cart,6855::budget run +coredns,6855::budget run +currency,6855::budget run +email,6855::budget run +flagd,6855::budget run +flagd-ui,6855::budget run +fluentd-10x,6855::budget run +fraud-detection,6855::budget run +frontend,6855::budget run +frontend-proxy,6855::budget run +grafana,6855::budget run +grafana-sc-alerts,6855::budget run +grafana-sc-dashboard,6855::budget run +grafana-sc-datasources,6855::budget run +image-provider,6855::budget run +init-config,6855::budget run +jaeger,6855::budget run +kafka,6855::budget run +kube-proxy,6855::budget run +load-generator,6855::budget run +metrics-server,6855::budget run +opensearch,6855::budget run +opentelemetry-collector,6855::budget run +payment,6855::budget run +postgresql,6855::budget run +prometheus-server,6855::budget run +quote,6855::budget run +recommendation,6855::budget run +valkey-cart,6855::budget run +wait-for-kafka,6855::budget run +wait-for-valkey-cart,6855::budget run +*,6855::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/commit.diff b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/commit.diff new file mode 100644 index 0000000..fb02e7a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/commit.diff @@ -0,0 +1,266 @@ +diff --git a/log10x/policy/caps.csv b/log10x/policy/caps.csv +index 8b572bc..81671a1 100644 +--- a/log10x/policy/caps.csv ++++ b/log10x/policy/caps.csv +@@ -1,38 +1,38 @@ + container,cap +-accounting,4768::budget run +-ad,4768::budget run +-aws-eks-nodeagent,4768::budget run +-aws-node,4768::budget run +-aws-vpc-cni-init,4768::budget run +-cart,4768::budget run +-coredns,4768::budget run +-currency,4768::budget run +-email,4768::budget run +-flagd,4768::budget run +-flagd-ui,4768::budget run +-fluentd-10x,4768::budget run +-fraud-detection,4768::budget run +-frontend,4768::budget run +-frontend-proxy,4768::budget run +-grafana,4768::budget run +-grafana-sc-alerts,4768::budget run +-grafana-sc-dashboard,4768::budget run +-grafana-sc-datasources,4768::budget run +-image-provider,4768::budget run +-init-config,4768::budget run +-jaeger,4768::budget run +-kafka,4768::budget run +-kube-proxy,4768::budget run +-load-generator,4768::budget run +-metrics-server,4768::budget run +-opensearch,4768::budget run +-opentelemetry-collector,4768::budget run +-payment,4768::budget run +-postgresql,4768::budget run +-prometheus-server,4768::budget run +-quote,4768::budget run +-recommendation,4768::budget run +-valkey-cart,4768::budget run +-wait-for-kafka,4768::budget run +-wait-for-valkey-cart,4768::budget run +-*,4768::budget run ++accounting,6855::budget run ++ad,6855::budget run ++aws-eks-nodeagent,6855::budget run ++aws-node,6855::budget run ++aws-vpc-cni-init,6855::budget run ++cart,6855::budget run ++coredns,6855::budget run ++currency,6855::budget run ++email,6855::budget run ++flagd,6855::budget run ++flagd-ui,6855::budget run ++fluentd-10x,6855::budget run ++fraud-detection,6855::budget run ++frontend,6855::budget run ++frontend-proxy,6855::budget run ++grafana,6855::budget run ++grafana-sc-alerts,6855::budget run ++grafana-sc-dashboard,6855::budget run ++grafana-sc-datasources,6855::budget run ++image-provider,6855::budget run ++init-config,6855::budget run ++jaeger,6855::budget run ++kafka,6855::budget run ++kube-proxy,6855::budget run ++load-generator,6855::budget run ++metrics-server,6855::budget run ++opensearch,6855::budget run ++opentelemetry-collector,6855::budget run ++payment,6855::budget run ++postgresql,6855::budget run ++prometheus-server,6855::budget run ++quote,6855::budget run ++recommendation,6855::budget run ++valkey-cart,6855::budget run ++wait-for-kafka,6855::budget run ++wait-for-valkey-cart,6855::budget run ++*,6855::budget run +diff --git a/log10x/policy/config-generation.csv b/log10x/policy/config-generation.csv +index b34cbf8..1b43ed9 100644 +--- a/log10x/policy/config-generation.csv ++++ b/log10x/policy/config-generation.csv +@@ -1,2 +1,2 @@ + key,value +-generation,bbddfb473fb5 ++generation,7f322a3fa798 +diff --git a/log10x/policy/totals.csv b/log10x/policy/totals.csv +index 7bd44df..3dfefe2 100644 +--- a/log10x/policy/totals.csv ++++ b/log10x/policy/totals.csv +@@ -1,38 +1,38 @@ + container,total +-accounting,2937::budget run +-ad,14597::budget run +-aws-eks-nodeagent,56::budget run +-aws-node,267::budget run +-aws-vpc-cni-init,340::budget run +-cart,9977::budget run +-coredns,2141::budget run +-currency,80::budget run +-email,233::budget run +-flagd,754::budget run +-flagd-ui,133::budget run +-fluentd-10x,9513::budget run +-fraud-detection,199::budget run +-frontend,33660::budget run +-frontend-proxy,8790::budget run +-grafana,111334::budget run +-grafana-sc-alerts,584::budget run +-grafana-sc-dashboard,826::budget run +-grafana-sc-datasources,592::budget run +-image-provider,1485::budget run +-init-config,490::budget run +-jaeger,5999::budget run +-kafka,100246::budget run +-kube-proxy,13914::budget run +-load-generator,676::budget run +-metrics-server,2115::budget run +-opensearch,23928::budget run +-opentelemetry-collector,125706::budget run +-payment,229::budget run +-postgresql,2072::budget run +-prometheus-server,6079::budget run +-quote,37::budget run +-recommendation,4911::budget run +-valkey-cart,16973::budget run +-wait-for-kafka,4886::budget run +-wait-for-valkey-cart,434::budget run +-*,2094::budget run ++accounting,2814::budget run ++ad,20847::budget run ++aws-eks-nodeagent,53::budget run ++aws-node,255::budget run ++aws-vpc-cni-init,325::budget run ++cart,14133::budget run ++coredns,2051::budget run ++currency,76::budget run ++email,224::budget run ++flagd,723::budget run ++flagd-ui,127::budget run ++fluentd-10x,11401::budget run ++fraud-detection,191::budget run ++frontend,33639::budget run ++frontend-proxy,8421::budget run ++grafana,108935::budget run ++grafana-sc-alerts,559::budget run ++grafana-sc-dashboard,791::budget run ++grafana-sc-datasources,567::budget run ++image-provider,1422::budget run ++init-config,470::budget run ++jaeger,5747::budget run ++kafka,125775::budget run ++kube-proxy,13330::budget run ++load-generator,647::budget run ++metrics-server,2027::budget run ++opensearch,22872::budget run ++opentelemetry-collector,130368::budget run ++payment,220::budget run ++postgresql,1985::budget run ++prometheus-server,5773::budget run ++quote,35::budget run ++recommendation,6992::budget run ++valkey-cart,15981::budget run ++wait-for-kafka,5332::budget run ++wait-for-valkey-cart,415::budget run ++*,2006::budget run +diff --git a/reports/latest.md b/reports/latest.md +index 8d1f8bf..dd33df9 100644 +--- a/reports/latest.md ++++ b/reports/latest.md +@@ -1,52 +1,52 @@ +-# Hot-tier run: 2026-10-26T00:34:11.000Z ++# Hot-tier run: 2026-10-26T00:35:10.000Z + +-Hot tier: 18 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.9 GB a day of counted lines moved to the frozen tier. ++Hot tier: 18 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.7 GB a day of counted lines moved to the frozen tier. + + - Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 3 indices; its tier_down lines not yet moved: 15 MB in 2. + - A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). + - Each counted byte takes 1.37 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +-- The Receiver's statements send 3.1 GB a day; 183 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +-- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 146 MB a day on hot whatever the limits; the size holds down to that floor. ++- The Receiver's statements send 2.9 GB a day; 196 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. ++- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 142 MB a day on hot whatever the limits; the size holds down to that floor. + - Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + + | Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | + |---|---|---|---|---|---|---| +-| accounting | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2937 | +-| ad | the frozen tier | 1 | 107 MB | 5 MB | 4768 | 14597 | +-| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 56 | +-| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 267 | +-| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 340 | +-| cart | the frozen tier | 1 | 260 MB | 4 MB | 4768 | 9977 | +-| coredns | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2141 | +-| currency | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 80 | +-| email | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 233 | +-| flagd | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 754 | +-| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 133 | +-| fluentd-10x | the frozen tier | 1 | 20 MB | 3 MB | 4768 | 9513 | +-| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 199 | +-| frontend | the frozen tier | 1 | 62 MB | 12 MB | 4768 | 33660 | +-| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 4768 | 8790 | +-| grafana | the frozen tier | 1 | 47 MB | 40 MB | 4768 | 111334 | +-| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 584 | +-| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 826 | +-| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 592 | +-| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 1485 | +-| init-config | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 490 | +-| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 4768 | 5999 | +-| kafka | the frozen tier | 1 | 490 MB | 36 MB | 4768 | 100246 | +-| kube-proxy | the frozen tier | 1 | 5 MB | 5 MB | 4768 | 13914 | +-| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 676 | +-| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2115 | +-| opensearch | the frozen tier | 1 | 9 MB | 9 MB | 4768 | 23928 | +-| opentelemetry-collector | the frozen tier | 1 | 2 GB | 45 MB | 4768 | 125706 | +-| payment | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 229 | +-| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 4768 | 2072 | +-| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 4768 | 6079 | +-| quote | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 37 | +-| recommendation | the frozen tier | 1 | 70 MB | 2 MB | 4768 | 4911 | +-| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 4768 | 16973 | +-| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 4768 | 4886 | +-| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 4768 | 434 | +-| `*` (any other service) | the frozen tier | 1 | | 1 MB | 4768 | 2094 | ++| accounting | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 2814 | ++| ad | the frozen tier | 1 | 99 MB | 8 MB | 6855 | 20847 | ++| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 53 | ++| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 255 | ++| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 325 | ++| cart | the frozen tier | 1 | 241 MB | 5 MB | 6855 | 14133 | ++| coredns | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 2051 | ++| currency | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 76 | ++| email | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 224 | ++| flagd | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 723 | ++| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 127 | ++| fluentd-10x | the frozen tier | 1 | 19 MB | 4 MB | 6855 | 11401 | ++| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 191 | ++| frontend | the frozen tier | 1 | 58 MB | 12 MB | 6855 | 33639 | ++| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 6855 | 8421 | ++| grafana | the frozen tier | 1 | 45 MB | 39 MB | 6855 | 108935 | ++| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 559 | ++| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 791 | ++| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 567 | ++| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 1422 | ++| init-config | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 470 | ++| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 6855 | 5747 | ++| kafka | the frozen tier | 1 | 455 MB | 45 MB | 6855 | 125775 | ++| kube-proxy | the frozen tier | 1 | 5 MB | 5 MB | 6855 | 13330 | ++| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 647 | ++| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 2027 | ++| opensearch | the frozen tier | 1 | 8 MB | 8 MB | 6855 | 22872 | ++| opentelemetry-collector | the frozen tier | 1 | 1.9 GB | 47 MB | 6855 | 130368 | ++| payment | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 220 | ++| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 1985 | ++| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 6855 | 5773 | ++| quote | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 35 | ++| recommendation | the frozen tier | 1 | 65 MB | 3 MB | 6855 | 6992 | ++| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 6855 | 15981 | ++| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 6855 | 5332 | ++| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 415 | ++| `*` (any other service) | the frozen tier | 1 | | 1 MB | 6855 | 2006 | + + Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/commit.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/commit.json new file mode 100644 index 0000000..0a1d9b4 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/commit.json @@ -0,0 +1 @@ +{"before":"11c9221f78ca19363f57e2bbed1752c4ae6105bc","after":"dca4133ef2fbbfc884b3e95aeabdd6af9ad1054f","exit":0} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/config-generation.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/config-generation.csv new file mode 100644 index 0000000..1b43ed9 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/config-generation.csv @@ -0,0 +1,2 @@ +key,value +generation,7f322a3fa798 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/exit b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/latest.md b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/latest.md new file mode 100644 index 0000000..dd33df9 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/latest.md @@ -0,0 +1,52 @@ +# Hot-tier run: 2026-10-26T00:35:10.000Z + +Hot tier: 18 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.7 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 3 indices; its tier_down lines not yet moved: 15 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.37 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +- The Receiver's statements send 2.9 GB a day; 196 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 142 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 2814 | +| ad | the frozen tier | 1 | 99 MB | 8 MB | 6855 | 20847 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 53 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 255 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 325 | +| cart | the frozen tier | 1 | 241 MB | 5 MB | 6855 | 14133 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 2051 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 76 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 224 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 723 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 127 | +| fluentd-10x | the frozen tier | 1 | 19 MB | 4 MB | 6855 | 11401 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 191 | +| frontend | the frozen tier | 1 | 58 MB | 12 MB | 6855 | 33639 | +| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 6855 | 8421 | +| grafana | the frozen tier | 1 | 45 MB | 39 MB | 6855 | 108935 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 559 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 791 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 567 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 1422 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 470 | +| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 6855 | 5747 | +| kafka | the frozen tier | 1 | 455 MB | 45 MB | 6855 | 125775 | +| kube-proxy | the frozen tier | 1 | 5 MB | 5 MB | 6855 | 13330 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 647 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 2027 | +| opensearch | the frozen tier | 1 | 8 MB | 8 MB | 6855 | 22872 | +| opentelemetry-collector | the frozen tier | 1 | 1.9 GB | 47 MB | 6855 | 130368 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 220 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 1985 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 6855 | 5773 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 35 | +| recommendation | the frozen tier | 1 | 65 MB | 3 MB | 6855 | 6992 | +| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 6855 | 15981 | +| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 6855 | 5332 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 415 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 6855 | 2006 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/report.txt b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/report.txt new file mode 100644 index 0000000..d608d9d --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/report.txt @@ -0,0 +1,61 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 18 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.7 GB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 3 indices; its tier_down lines not yet moved: 15 MB in 2. +[tenx-recur] A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +[tenx-recur] Each counted byte takes 1.37 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 2.9 GB a day; 196 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 142 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-26T00:35:10.000Z + +Hot tier: 18 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.7 GB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 3 indices; its tier_down lines not yet moved: 15 MB in 2. +- A kept line stays on hot 2 hours (ILM policy tenx-keep (rollover max_age 10m + frozen min_age 110m)); a tier_down index 0.3 hours (rollover max_age 10m + a median 7 minutes from rollover to mount over 1 moved indices). +- Each counted byte takes 1.37 bytes of store kept, 0.58 as tier_down (measured on the indices on hot). +- The Receiver's statements send 2.9 GB a day; 196 MB a day fits the size, so each statement keeps up to 2 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 142 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 2814 | +| ad | the frozen tier | 1 | 99 MB | 8 MB | 6855 | 20847 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 53 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 255 | +| aws-vpc-cni-init | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 325 | +| cart | the frozen tier | 1 | 241 MB | 5 MB | 6855 | 14133 | +| coredns | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 2051 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 76 | +| email | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 224 | +| flagd | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 723 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 127 | +| fluentd-10x | the frozen tier | 1 | 19 MB | 4 MB | 6855 | 11401 | +| fraud-detection | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 191 | +| frontend | the frozen tier | 1 | 58 MB | 12 MB | 6855 | 33639 | +| frontend-proxy | the frozen tier | 1 | 3 MB | 3 MB | 6855 | 8421 | +| grafana | the frozen tier | 1 | 45 MB | 39 MB | 6855 | 108935 | +| grafana-sc-alerts | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 559 | +| grafana-sc-dashboard | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 791 | +| grafana-sc-datasources | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 567 | +| image-provider | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 1422 | +| init-config | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 470 | +| jaeger | the frozen tier | 1 | 2 MB | 2 MB | 6855 | 5747 | +| kafka | the frozen tier | 1 | 455 MB | 45 MB | 6855 | 125775 | +| kube-proxy | the frozen tier | 1 | 5 MB | 5 MB | 6855 | 13330 | +| load-generator | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 647 | +| metrics-server | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 2027 | +| opensearch | the frozen tier | 1 | 8 MB | 8 MB | 6855 | 22872 | +| opentelemetry-collector | the frozen tier | 1 | 1.9 GB | 47 MB | 6855 | 130368 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 220 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 6855 | 1985 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 6855 | 5773 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 35 | +| recommendation | the frozen tier | 1 | 65 MB | 3 MB | 6855 | 6992 | +| valkey-cart | the frozen tier | 1 | 6 MB | 6 MB | 6855 | 15981 | +| wait-for-kafka | the frozen tier | 1 | 2 MB | 2 MB | 6855 | 5332 | +| wait-for-valkey-cart | the frozen tier | 1 | 0 MB | 0 MB | 6855 | 415 | +| `*` (any other service) | the frozen tier | 1 | | 1 MB | 6855 | 2006 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/run.json b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/run.json new file mode 100644 index 0000000..7df8109 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-26T00:35:12.597Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 18 MB of 41 MB (0.0406 GB set in the policy) on 1 hot node; at these limits it settles at about 41 MB within 2 hours, with 2.7 GB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/totals.csv b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/totals.csv new file mode 100644 index 0000000..3dfefe2 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/budget-runs/20261026T003509Z/totals.csv @@ -0,0 +1,38 @@ +container,total +accounting,2814::budget run +ad,20847::budget run +aws-eks-nodeagent,53::budget run +aws-node,255::budget run +aws-vpc-cni-init,325::budget run +cart,14133::budget run +coredns,2051::budget run +currency,76::budget run +email,224::budget run +flagd,723::budget run +flagd-ui,127::budget run +fluentd-10x,11401::budget run +fraud-detection,191::budget run +frontend,33639::budget run +frontend-proxy,8421::budget run +grafana,108935::budget run +grafana-sc-alerts,559::budget run +grafana-sc-dashboard,791::budget run +grafana-sc-datasources,567::budget run +image-provider,1422::budget run +init-config,470::budget run +jaeger,5747::budget run +kafka,125775::budget run +kube-proxy,13330::budget run +load-generator,647::budget run +metrics-server,2027::budget run +opensearch,22872::budget run +opentelemetry-collector,130368::budget run +payment,220::budget run +postgresql,1985::budget run +prometheus-server,5773::budget run +quote,35::budget run +recommendation,6992::budget run +valkey-cart,15981::budget run +wait-for-kafka,5332::budget run +wait-for-valkey-cart,415::budget run +*,2006::budget run diff --git a/elastic-hot-tier-budget/results-e103-smoke/build.json b/elastic-hot-tier-budget/results-e103-smoke/build.json new file mode 100644 index 0000000..78639bb --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/build.json @@ -0,0 +1,52 @@ +{ + "images": { + "docker.elastic.co/elasticsearch/elasticsearch:9.5.5": { + "id": "sha256:13b4a40bb095774f5fda635e45fd5677cc6a26565e42b24997b0c72b88e75984", + "repo_digests": [ + "docker.elastic.co/elasticsearch/elasticsearch@sha256:13b4a40bb095774f5fda635e45fd5677cc6a26565e42b24997b0c72b88e75984" + ], + "created": "2026-09-30T22:20:56.268968505Z" + }, + "fluent/fluent-bit:4.2": { + "id": "sha256:6806769c10acf11b41d14083b3101a24433b4ca113a1ab7a6b1a1b7cad86a7e8", + "repo_digests": [ + "fluent/fluent-bit@sha256:6806769c10acf11b41d14083b3101a24433b4ca113a1ab7a6b1a1b7cad86a7e8" + ], + "created": "2026-08-04T04:31:13.800466265Z" + }, + "local/edge-10x:e103": { + "id": "sha256:8f74b8bbec2ef115fb83eb365172872afa5aec2cea4ae5fd065b49c1d77fe361", + "repo_digests": [ + "local/edge-10x@sha256:8f74b8bbec2ef115fb83eb365172872afa5aec2cea4ae5fd065b49c1d77fe361" + ], + "created": "2026-10-25T23:56:14.606292327Z" + }, + "python:3.12-slim": { + "id": "sha256:a0f57064cbdcd6285fa45f51237ab0b35ea8ebaed06a1f5973e89e19dfb86704", + "repo_digests": [ + "python@sha256:a0f57064cbdcd6285fa45f51237ab0b35ea8ebaed06a1f5973e89e19dfb86704" + ], + "created": "2026-10-21T01:10:54.051521261Z" + }, + "alpine:latest": { + "id": "sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6", + "repo_digests": [ + "alpine@sha256:294b683cb724975bec92580e1e685676bd4b50bda910ddb8c51d4cabeaec77e6" + ], + "created": "2026-09-17T20:37:20.889221879Z" + } + }, + "log10x_mcp": { + "path": "/Users/talweiss/git/l1x-co/wt-mcp-main2", + "commit": "d6fa2b1129a4b780db3338b7f327186864ce0907", + "branch": "HEAD", + "describe": "d6fa2b1 Doctor reads the config generation from its own series; 1.30.84 (#297)", + "version": "1.30.84", + "dirty": false + }, + "benchmarks": { + "commit": "61ce4dfa98d4f3dc419fef67be583f3390dc68a7", + "branch": "bench/e103-hot-tier-smoke", + "rig_changes_uncommitted": [] + } +} \ No newline at end of file diff --git a/elastic-hot-tier-budget/results-e103-smoke/data/returned.jsonl.gz b/elastic-hot-tier-budget/results-e103-smoke/data/returned.jsonl.gz new file mode 100644 index 0000000..73a9749 Binary files /dev/null and b/elastic-hot-tier-budget/results-e103-smoke/data/returned.jsonl.gz differ diff --git a/elastic-hot-tier-budget/results-e103-smoke/delivery/a.json b/elastic-hot-tier-budget/results-e103-smoke/delivery/a.json new file mode 100644 index 0000000..b753160 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/delivery/a.json @@ -0,0 +1 @@ +{"aliases": "tenx-app,tenx-tierdown", "documents_read": 79416, "count_api": 79416, "indices": 6, "by_index": {"partial-tenx-tierdown-000001": 17334, "tenx-app-000001": 4715, "tenx-app-000002": 1225, "tenx-app-000003": 20, "tenx-tierdown-000002": 53929, "tenx-tierdown-000003": 2193}} diff --git a/elastic-hot-tier-budget/results-e103-smoke/delivery/a.tsv.gz b/elastic-hot-tier-budget/results-e103-smoke/delivery/a.tsv.gz new file mode 100644 index 0000000..9b11ece Binary files /dev/null and b/elastic-hot-tier-budget/results-e103-smoke/delivery/a.tsv.gz differ diff --git a/elastic-hot-tier-budget/results-e103-smoke/delivery/aliases-9200.json b/elastic-hot-tier-budget/results-e103-smoke/delivery/aliases-9200.json new file mode 100644 index 0000000..8ad530d --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/delivery/aliases-9200.json @@ -0,0 +1 @@ +{"tenx-tierdown-000002":{"aliases":{"tenx-tierdown":{"is_write_index":false}}},"tenx-tierdown-000003":{"aliases":{"tenx-tierdown":{"is_write_index":true}}},"tenx-app-000002":{"aliases":{"tenx-app":{"is_write_index":false}}},"tenx-app-000003":{"aliases":{"tenx-app":{"is_write_index":true}}},"partial-tenx-tierdown-000001":{"aliases":{"tenx-tierdown":{},"tenx-tierdown-000001":{}}},"tenx-app-000001":{"aliases":{"tenx-app":{"is_write_index":false}}},".snapshot-blob-cache":{"aliases":{}}} \ No newline at end of file diff --git a/elastic-hot-tier-budget/results-e103-smoke/delivery/cat-indices-9200.json b/elastic-hot-tier-budget/results-e103-smoke/delivery/cat-indices-9200.json new file mode 100644 index 0000000..24e8a78 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/delivery/cat-indices-9200.json @@ -0,0 +1 @@ +[{"index":"tenx-tierdown-000002","health":"green","status":"open","docs.count":"53929","store.size":"13556314","creation.date":"1792974105524"},{"index":"tenx-tierdown-000003","health":"green","status":"open","docs.count":"2193","store.size":"1056132","creation.date":"1792974805587"},{"index":"tenx-app-000002","health":"green","status":"open","docs.count":"1225","store.size":"976927","creation.date":"1792974105540"},{"index":"tenx-app-000003","health":"green","status":"open","docs.count":"20","store.size":"132981","creation.date":"1792974805502"},{"index":"partial-tenx-tierdown-000001","health":"green","status":"open","docs.count":"17334","store.size":"0","creation.date":"1792973504568"},{"index":".snapshot-blob-cache","health":"green","status":"open","docs.count":"32","store.size":"39857","creation.date":"1792974507194"},{"index":"tenx-app-000001","health":"green","status":"open","docs.count":"4715","store.size":"2502800","creation.date":"1792973504890"},{"index":".ds-ilm-history-7-2026.10.26-000001","health":"green","status":"open","docs.count":"54","store.size":"100021","creation.date":"1792973509031"},{"index":".ds-.logs-elasticsearch.deprecation-default-2026.10.26-000001","health":"green","status":"open","docs.count":"1","store.size":"11919","creation.date":"1792973503438"}] \ No newline at end of file diff --git a/elastic-hot-tier-budget/results-e103-smoke/es-setup/es-setup.json b/elastic-hot-tier-budget/results-e103-smoke/es-setup/es-setup.json new file mode 100644 index 0000000..10eaa8d --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/es-setup/es-setup.json @@ -0,0 +1,79 @@ +{ + "es": "http://127.0.0.1:9200", + "requests": 6, + "td_placement": null, + "log": [ + { + "method": "POST", + "path": "/_license/start_trial?acknowledge=true", + "status": 200, + "body": "{\"acknowledged\":true,\"trial_was_started\":true,\"type\":\"trial\"}" + }, + { + "method": "GET", + "path": "/_license", + "status": 200, + "body": "{\n \"license\" : {\n \"status\" : \"active\",\n \"uid\" : \"0d3b36fc-d30c-4c87-b127-fe3d25f2c9d0\",\n \"type\" : \"trial\",\n \"issue_date\" : \"2026-10-26T00:11:42.625Z\",\n \"issue_date_in_millis\" : 1792973502625,\n \"expiry_date\" : \"2026-11-25T00:11:42.625Z\",\n \"expiry_date_in_millis\" : 1795565502625,\n \"max_nodes\" : 1000,\n \"max_resource_units\" : null,\n \"issued_to\" : \"tenx-rig\",\n \"issuer\" : \"elasticsearch\",\n \"start_date_in_millis\" : -1\n }\n}\n" + }, + { + "method": "PUT", + "path": "/_snapshot/rig-snapshots", + "status": 200, + "body": "{\"acknowledged\":true}" + }, + { + "method": "PUT", + "path": "_ilm/policy/tenx-tier-down", + "status": 200, + "body": "{\"acknowledged\":true}" + }, + { + "method": "PUT", + "path": "_ilm/policy/tenx-keep", + "status": 200, + "body": "{\"acknowledged\":true}" + }, + { + "method": "PUT", + "path": "_index_template/tenx-tierdown-tpl", + "status": 200, + "body": "{\"acknowledged\":true}" + }, + { + "method": "PUT", + "path": "_index_template/tenx-app-tpl", + "status": 200, + "body": "{\"acknowledged\":true}" + }, + { + "method": "PUT", + "path": "tenx-tierdown-000001", + "status": 200, + "body": "{\"acknowledged\":true,\"shards_acknowledged\":true,\"index\":\"tenx-tierdown-000001\"}" + }, + { + "method": "PUT", + "path": "tenx-app-000001", + "status": 200, + "body": "{\"acknowledged\":true,\"shards_acknowledged\":true,\"index\":\"tenx-app-000001\"}" + }, + { + "method": "PUT", + "path": "/_ilm/policy/tenx-keep", + "status": 200, + "body": "{\"acknowledged\":true}" + }, + { + "method": "PUT", + "path": "/_ilm/policy/tenx-tier-down", + "status": 200, + "body": "{\"acknowledged\":true}" + }, + { + "method": "PUT", + "path": "/_cluster/settings", + "status": 200, + "body": "{\"acknowledged\":true,\"persistent\":{\"indices\":{\"lifecycle\":{\"poll_interval\":\"100s\"}}},\"transient\":{}}" + } + ] +} diff --git a/elastic-hot-tier-budget/results-e103-smoke/es-setup/recipe.txt b/elastic-hot-tier-budget/results-e103-smoke/es-setup/recipe.txt new file mode 100644 index 0000000..1006d8a --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/es-setup/recipe.txt @@ -0,0 +1,81 @@ +# 1) Policy for the tier_down slice: roll over hourly, then mount the rolled +# index as a PARTIALLY-MOUNTED searchable snapshot in the frozen tier as soon +# as ILM sees it. Every route into frozen lands in hot first. Measured with +# the default 10-minute poll: rollover at the first poll past the hour, the +# snapshot and mount three polls later, about 110 minutes per index on hot. +PUT _ilm/policy/tenx-tier-down +{ + "policy": { "phases": { + "hot": { "min_age": "0ms", "actions": { "rollover": { "max_primary_shard_size": "50gb", "max_age": "1h" } } }, + "frozen": { "min_age": "1m", "actions": { "searchable_snapshot": { "snapshot_repository": "rig-snapshots" } } } + }} +} + +# 2) Policy for everything we keep. Same rollover, never leaves hot. This is the +# control: without it you cannot tell "the slice moved" from "everything moved". +PUT _ilm/policy/tenx-keep +{ + "policy": { "phases": { + "hot": { "min_age": "0ms", "actions": { "rollover": { "max_primary_shard_size": "50gb", "max_age": "30d" } } } + }} +} + +# 3) Templates. The marker decides WHICH INDEX an event lands in; the index +# decides which policy governs it. tenx_hash and routeState are mapped as +# keyword so both survive as queryable fields in frozen (verified). +PUT _index_template/tenx-tierdown-tpl +{ + "index_patterns": ["tenx-tierdown-*"], + "template": { + "settings": { + "index.lifecycle.name": "tenx-tier-down", + "index.lifecycle.rollover_alias": "tenx-tierdown", + "number_of_replicas": 0 + }, + "mappings": { "properties": { + "tenx_hash": { "type": "keyword" }, + "routeState": { "type": "keyword" }, + "@timestamp": { "type": "date" } + }} + } +} + +PUT _index_template/tenx-app-tpl +{ + "index_patterns": ["tenx-app-*"], + "template": { + "settings": { + "index.lifecycle.name": "tenx-keep", + "index.lifecycle.rollover_alias": "tenx-app", + "number_of_replicas": 0 + }, + "mappings": { "properties": { + "tenx_hash": { "type": "keyword" }, + "routeState": { "type": "keyword" }, + "@timestamp": { "type": "date" } + }} + } +} + +# 4) Bootstrap the write indices. Both policies use rollover, so each alias +# needs an initial backing index flagged is_write_index. Skipping this is the +# usual reason ILM sits in check-rollover-ready forever. +PUT tenx-tierdown-000001 +{ "aliases": { "tenx-tierdown": { "is_write_index": true } } } + +PUT tenx-app-000001 +{ "aliases": { "tenx-app": { "is_write_index": true } } } + +# 5) Verify, once data has rolled over and aged past min_age. The frozen index +# is RENAMED with a partial- prefix, which is how you know it mounted: +# +# GET _cat/indices/*tenx-tierdown*?h=index,docs.count,store.size +# partial-tenx-tierdown-000001 0b <- data now in rig-snapshots +# +# GET tenx-tierdown-000001/_ilm/explain -> phase: frozen, step: complete +# POST partial-tenx-tierdown-000001/_search +# { "query": { "term": { "tenx_hash": "" } } } +# ...must still return the documents. If it does not, STOP: the slice is +# cheap but unretrievable and the down-tier is not worth doing. + +NOTE: Elasticsearch tiering is INDEX-level, so the shipper routes the marked slice to its own index and ILM does the rest. Verified live: the frozen index reported store=0b locally while the hot control stayed at 19.2kb, and a term query on `tenx_hash` still returned 400/400 with `routeState` aggregatable, so the down-tiered slice remains retrievable by stamped identity. PREREQUISITE: searchable snapshots need an Enterprise licence (Elastic documents no lower tier for them), plus a registered snapshot repository (Elastic Cloud provides `found-snapshots` by default). RESIDENCY: every route into the frozen tier lands in hot first, this one included. The tier_down index rolls over hourly and mounts right after, so with the default 10-minute ILM poll each index lived about 110 minutes on hot (rollover at the first poll past the hour, snapshot and mount three polls later), so a demoted line leaves hot 30 to 110 minutes after it arrives. Hourly rollover means one frozen index per hour per alias; add a delete phase at the retention the account wants so the shard count on frozen stays bounded. ON ELASTIC CLOUD HOSTED THE SAVING IS NOT AUTOMATIC: Hosted is priced by provisioned resources, so moving data out of hot creates headroom and the deployment must then be RESIZED to bank it. Do not tell a Hosted customer their bill drops on its own; tell them their hot tier can shrink. Self-managed differs: the saving is disk you stop buying. OPERATIONAL NOTE: ILM polls every 10 minutes by default (`indices.lifecycle.poll_interval`), so a transition will not be visible immediately after rollover; that is not a failure. diff --git a/elastic-hot-tier-budget/results-e103-smoke/fluent-bit/fluent-bit.conf b/elastic-hot-tier-budget/results-e103-smoke/fluent-bit/fluent-bit.conf new file mode 100644 index 0000000..1666226 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/fluent-bit/fluent-bit.conf @@ -0,0 +1,138 @@ +# Two-arm variant of the slim config. Arm A as in fluent-bit-slim: every line to the Receiver, returned +# records to tenx-app / tenx-tierdown on cluster A. Arm B: a copy of every line, before the Receiver, to +# cluster B by a rule fixed before the run (render-fb-two-arm.py fills opentelemetry\-collector): the services that made +# the top half of bytes in the first 20 minutes go to native-overflow, every other service to native-app. +# No raw copy (lines in = Fluent Bit's input counter); the local copies of returned records keep only +# container, times, hash, route, utf8Size, level, name, lines. +# Rig Fluent Bit pipeline (Elastic hot-tier run). +# replay (Forward, tag app.replay) -> Receiver (tenx:24224) +# Receiver (:24225, Tag_Prefix tenx.) -> split by routeState -> pass / offload / tier_down / other +# tier_down goes to the tenx-tierdown alias, pass and other to tenx-app, as the MCP's +# Elastic recipe says ("Point the forwarder at the tenx-tierdown alias for events where +# routeState == tier_down, and at tenx-app for everything else"); offload stays local. +# routeState is kept on the record: the recipe maps it as keyword in both templates. + +[SERVICE] + Flush 1 + Log_Level info + Grace 10 + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + +[INPUT] + Name forward + Listen 0.0.0.0 + Port 24224 + Buffer_Max_Size 16M + Buffer_Chunk_Size 1M + +[FILTER] + Name lua + Match app.* + script /fluent-bit/etc/rig/rig.lua + call prep + +[OUTPUT] + Name forward + Match app.* + Host tenx + Port 24224 + Retry_Limit False + Workers 1 + net.max_worker_connections 1 + net.keepalive on + net.keepalive_max_recycle 100000000 + net.keepalive_idle_timeout 300 + +[INPUT] + Name forward + Listen 0.0.0.0 + Port 24225 + Tag_Prefix tenx. + Buffer_Max_Size 16M + Buffer_Chunk_Size 1M + +[FILTER] + Name lua + Match tenx.* + script /fluent-bit/etc/rig/rig.lua + call route + +[FILTER] + Name rewrite_tag + Match tenx.* + Rule $_route ^offload$ routed.offload true + Rule $_route ^tier_down$ routed.tier_down true + Rule $_route ^pass$ routed.pass true + Rule $_route ^other$ routed.other true + +[FILTER] + Name record_modifier + Match routed.* + Remove_key _route + +[OUTPUT] + Name file + Match routed.pass + Path /data + File pass.txt + Format template + Template {service}|{fb_ts}|{rb_ts}|{tenx_hash}|{routeState}|{utf8Size}|{severity_level}|{message_pattern}|{_lines} + +[OUTPUT] + Name file + Match routed.offload + Path /data + File offload.txt + Format template + Template {service}|{fb_ts}|{rb_ts}|{tenx_hash}|{routeState}|{utf8Size}|{severity_level}|{message_pattern}|{_lines} + +[OUTPUT] + Name file + Match routed.tier_down + Path /data + File tier_down.txt + Format template + Template {service}|{fb_ts}|{rb_ts}|{tenx_hash}|{routeState}|{utf8Size}|{severity_level}|{message_pattern}|{_lines} + +[OUTPUT] + Name file + Match routed.other + Path /data + File other.txt + Format template + Template {service}|{fb_ts}|{rb_ts}|{tenx_hash}|{routeState}|{utf8Size}|{severity_level}|{message_pattern}|{_lines} + +[OUTPUT] + Name es + Match routed.tier_down + Host es-hot + Port 9200 + Index tenx-tierdown + Logstash_Format Off + Suppress_Type_Name On + Replace_Dots On + Trace_Error On + Buffer_Size 2MB + Retry_Limit False + +[OUTPUT] + Name es + Match_Regex ^routed\.(pass|other)$ + Host es-hot + Port 9200 + Index tenx-app + Logstash_Format Off + Suppress_Type_Name On + Replace_Dots On + Trace_Error On + Buffer_Size 2MB + Retry_Limit False + +[OUTPUT] + Name file + Match routed.* + Path /data + File returned.jsonl + Format plain diff --git a/elastic-hot-tier-budget/results-e103-smoke/fluent-bit/rig.lua b/elastic-hot-tier-budget/results-e103-smoke/fluent-bit/rig.lua new file mode 100644 index 0000000..0cb7a5d --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/fluent-bit/rig.lua @@ -0,0 +1,46 @@ +-- Rig filters for Fluent Bit. +-- prep: ingest from the Docker fluentd driver. The container moves into a Kubernetes-style +-- `kubernetes` map (container_name, docker_id), the shape Fluent Bit's kubernetes +-- filter gives a pod's records, where both the Receiver's fluentK8s extractor and the +-- MCP's file mode read the container. fb_ts is the driver's event time. +-- route: return path from the Receiver. rb_ts is the wall-clock time the record came back; +-- _route picks the file by routeState. service is the container (Datadog's service +-- tag, read by service_budgets); status is the Receiver's level, lower case. +local ok, ffi = pcall(require, "ffi") +local now +if ok then + ffi.cdef[[ typedef struct { long tv_sec; long tv_usec; } rig_timeval; int gettimeofday(rig_timeval *tv, void *tz); ]] + local tv = ffi.new("rig_timeval") + now = function() ffi.C.gettimeofday(tv, nil); return tonumber(tv.tv_sec) + tonumber(tv.tv_usec) / 1e6 end +else + now = function() return os.time() end +end + +function prep(tag, ts, rec) + local c = rec["container_name"] + if c then c = (string.gsub(c, "^/", "")) end + local k = { container_name = c } + if rec["container_id"] then k["docker_id"] = rec["container_id"] end + rec["container_name"] = nil + rec["container_id"] = nil + rec["kubernetes"] = k + rec["fb_ts"] = string.format("%.3f", ts) + return 2, ts, rec +end + +function route(tag, ts, rec) + local r = rec["routeState"] + if r == nil or r == "pass" then rec["_route"] = "pass" + elseif r == "offload" or r == "tier_down" then rec["_route"] = r + else rec["_route"] = "other" end + rec["rb_ts"] = string.format("%.3f", now()) + local k = rec["kubernetes"] + if type(k) == "table" and k["container_name"] then rec["service"] = k["container_name"] end + if rec["severity_level"] then rec["status"] = string.lower(rec["severity_level"]) end + local n = 0 + for l in string.gmatch(rec["log"] or "", "[^\n]+") do + if string.find(l, "%S") then n = n + 1 end + end + rec["_lines"] = n + return 2, ts, rec +end diff --git a/elastic-hot-tier-budget/results-e103-smoke/fluentbit-metrics.json b/elastic-hot-tier-budget/results-e103-smoke/fluentbit-metrics.json new file mode 100644 index 0000000..e399a80 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/fluentbit-metrics.json @@ -0,0 +1 @@ +{"input":{"forward.0":{"records":101663,"bytes":37129568},"forward.1":{"records":79416,"bytes":44653070},"emitter_for_rewrite_tag.2":{"records":79416,"bytes":51957301}},"filter":{"lua.0":{"drop_records":0,"add_records":0,"records":101663,"bytes":39671143,"drop_bytes":0},"lua.1":{"drop_records":0,"add_records":0,"records":79416,"bytes":51957301,"drop_bytes":0},"rewrite_tag.2":{"drop_records":0,"add_records":0,"records":79416,"bytes":51957301,"drop_bytes":0,"emit_records":79416},"record_modifier.3":{"drop_records":0,"add_records":0,"records":79416,"bytes":50954693,"drop_bytes":0}},"output":{"forward.0":{"proc_records":101663,"proc_bytes":39671143,"errors":0,"retries":0,"retries_failed":0,"dropped_records":0,"retried_records":0},"file.1":{"proc_records":5960,"proc_bytes":3859243,"errors":0,"retries":0,"retries_failed":0,"dropped_records":0,"retried_records":0},"file.2":{"proc_records":0,"proc_bytes":0,"errors":0,"retries":0,"retries_failed":0,"dropped_records":0,"retried_records":0},"file.3":{"proc_records":73456,"proc_bytes":47095450,"errors":0,"retries":0,"retries_failed":0,"dropped_records":0,"retried_records":0},"file.4":{"proc_records":0,"proc_bytes":0,"errors":0,"retries":0,"retries_failed":0,"dropped_records":0,"retried_records":0},"es.5":{"proc_records":73456,"proc_bytes":47095450,"errors":0,"retries":0,"retries_failed":0,"dropped_records":0,"retried_records":0},"es.6":{"proc_records":5960,"proc_bytes":3859243,"errors":0,"retries":0,"retries_failed":0,"dropped_records":0,"retried_records":0},"file.7":{"proc_records":79416,"proc_bytes":50954693,"errors":0,"retries":0,"retries_failed":0,"dropped_records":0,"retried_records":0}}} diff --git a/elastic-hot-tier-budget/results-e103-smoke/hb-tenx.log b/elastic-hot-tier-budget/results-e103-smoke/hb-tenx.log new file mode 100644 index 0000000..cbafa4f --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/hb-tenx.log @@ -0,0 +1,51 @@ +[INFO ] 2026-10-26 00:12:10.111 [main] PipelineLauncher - launching 10x engine v1.1.151, flavor: 'runtime', pipeline: 'run' +[INFO ] 2026-10-26 00:12:10.112 [main] PipelineLauncher - log4j2 config file loaded from: /etc/tenx/config/log4j2.yaml +[WARN ] 2026-10-26 00:12:11.503 [main] PipelineLauncher - No license configured; running on the built-in free tier: up to 10 nodes, airgapped. +[WARN ] 2026-10-26 00:12:11.504 [main] PipelineLauncher - Above 10 nodes, a paid license token is required: set licenseFile / licenseKey, the TENX_LICENSE_FILE / TENX_LICENSE_KEY env var, or a licenseKey/licenseFile entry in your bootstrap .yaml. See https://doc.log10x.com/manage/license/ +[INFO ] 2026-10-26 00:12:13.011 [main] PipelineLauncher - pipelineConfigLoader took: 1505ms +[INFO ] 2026-10-26 00:12:13.098 [main] ExecutionPipeline - executing /opt/tenx-edge/lib/app/modules/pipelines/run/pipeline.yaml (myReceiver) +[INFO ] 2026-10-26 00:12:13.306 [main] ExecutionPipeline - pipeline units: configLoader => symbolLoader(shared/symbols, tenx/symbols) => templateLoader => readStream(stream:fluentbit) => parallelize(0.5 / 1) => transform => group => aggregateSingle(emitted_events: 10000/2s) => aggregateSingle(all_events: 10000/2s) => aggregateSingle(config_generation: 10000/2s) => logOutput(run/modules/output/event/forward) => printProgress => monitor => tokenizeStats => publishEnvMetrics +πŸͺ΅ Initializing 10x engine. Logging to SYS_OUT +πŸš€ Launching 10x Engine: Receiver app +πŸ“š Loading symbol libraries from: /etc/tenx/config/data/shared/symbols,/etc/tenx/symbols +πŸ“‚ Loaded symbol files: /etc/tenx/symbols/tenx-symbols-1.1.150.10x.tar +πŸ“‚ Loading TenXTemplates from: data/templates/*.json,data/sample/output/*.json +πŸ“₯ Reading events from Fluent Bit via Forward protocol on tcp://0.0.0.0:24224 +πŸ’° Loading rate cap file: /tmp/tenx/github/log10x/policy/caps.csv +🚦 Applying rate regulator to: fluentbit +[INFO ] 2026-10-26 00:12:24.216 [main] ForwardProtocolInputStream - initializing Forward protocol input stream on TCP port: 24224 +[INFO ] 2026-10-26 00:12:24.294 [main] ForwardProtocolInputStream - Forward protocol server listening on TCP port: 24224 +βš™οΈ Transforming events into TenXObjects using 50% of available cores +ℹ️ Enriching TenXObjects with severity field: 'severity_level' +πŸ”² Consolidating multi-line events into logical TenXObject groups +πŸ’¬ Enriching TenXObjects with message field: 'message_pattern' +🌐 Enriching TenXObjects with HTTP code field: 'http_code' +πŸ“‹ Enriching TenXObjects with lookup field: 'http_message' from file: /etc/tenx/config/pipelines/run/initialize/lookup/http.csv +☸️ Enriching TenXObjects with k8s container context using fluentK8s +[INFO ] 2026-10-26 00:12:26.201 [main] EventLogPublisherConfigBuilder - Added new loggers to log4j: [tenxForward__fluent-bit_24225] +πŸ“ Writing TenXObject field: 'fullText("tenx_hash","routeState","message_pattern","severity_level","utf8Size")' β†’ Fluentd: fluent-bit:24225 +[INFO ] 2026-10-26 00:12:26.709 [main] ResourceReloadUnit - resource reload: starting with 4 watched path(s); pollInterval=10s +[INFO ] 2026-10-26 00:12:31.849 [AsyncWorkerTaskInput: readStream 00:12:26] ForwardProtocolInputStream - client connected to Forward protocol input: tcp://0.0.0.0:24224 +[INFO ] 2026-10-26 00:15:08.547 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/config-generation.csv, /tmp/tenx/github/log10x/policy/caps.csv, /tmp/tenx/github/log10x/policy/actions.csv, /tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:16:08.667 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:17:08.736 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/config-generation.csv, /tmp/tenx/github/log10x/policy/caps.csv, /tmp/tenx/github/log10x/policy/actions.csv, /tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:18:08.747 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:19:08.764 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:20:08.773 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:21:08.780 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:22:08.787 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:23:08.794 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:24:08.803 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:25:08.810 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:26:08.817 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:27:08.824 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:28:08.831 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:29:08.837 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:30:08.844 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/config-generation.csv, /tmp/tenx/github/log10x/policy/caps.csv, /tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:31:08.853 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/config-generation.csv, /tmp/tenx/github/log10x/policy/caps.csv, /tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:32:08.860 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/config-generation.csv, /tmp/tenx/github/log10x/policy/caps.csv, /tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:33:08.870 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/config-generation.csv, /tmp/tenx/github/log10x/policy/caps.csv, /tmp/tenx/github/log10x/policy/totals.csv] +[INFO ] 2026-10-26 00:34:08.877 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/config-generation.csv, /tmp/tenx/github/log10x/policy/caps.csv, /tmp/tenx/github/log10x/policy/totals.csv] +[WARN ] 2026-10-26 00:34:54.258 [forward-input-watchdog] ForwardProtocolInputStream - socket idle for 70452ms with no decoded messages, forcing reconnection +[INFO ] 2026-10-26 00:34:54.271 [AsyncWorkerTaskInput: readStream 00:12:26] ForwardProtocolInputStream - client connection lost (null), waiting for new connection +[INFO ] 2026-10-26 00:35:08.887 [AsyncWorkerTaskInput: configLoader 00:12:26] ResourceReloadUnit - resource reload: reloaded pipeline unit lookup, changed resources: [/tmp/tenx/github/log10x/policy/config-generation.csv, /tmp/tenx/github/log10x/policy/caps.csv, /tmp/tenx/github/log10x/policy/totals.csv] diff --git a/elastic-hot-tier-budget/results-e103-smoke/meter.jsonl.gz b/elastic-hot-tier-budget/results-e103-smoke/meter.jsonl.gz new file mode 100644 index 0000000..4257f1c Binary files /dev/null and b/elastic-hot-tier-budget/results-e103-smoke/meter.jsonl.gz differ diff --git a/elastic-hot-tier-budget/results-e103-smoke/policy.yaml b/elastic-hot-tier-budget/results-e103-smoke/policy.yaml new file mode 100644 index 0000000..9d22a08 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/policy.yaml @@ -0,0 +1,71 @@ +# Log10x recurring cost-reduction policy +# Generated by log10x_setup_recurring: edit and re-commit as needed. +# The CLI reads this file at the start of each tick. +schema_version: "1.0" + +reduction: + # empty = all services + target_services: [] + # Elasticsearch hot tier held under a size: GB of index store on the hot nodes, + # or auto for 80% of their disk (Elasticsearch stops allocating at 85%). + # Each run rewrites the byte limits so growth goes to the frozen tier. + hot_tier_gb: 0.0406 + # hours a kept line stays on hot: read from the ILM policy on pass_alias unless set here + # hot_retention_hours: 168 + pass_alias: tenx-app + tier_down_alias: tenx-tierdown + container_field: kubernetes.container_name + # error lines over a limit that pass per statement per node per window + keep_count: 50 + default_action: tier_down + nodes_by_container: + accounting: 1 + ad: 1 + aws-eks-nodeagent: 1 + aws-node: 1 + aws-vpc-cni-init: 1 + cart: 1 + coredns: 1 + currency: 1 + email: 1 + flagd: 1 + flagd-ui: 1 + fluentd-10x: 1 + fraud-detection: 1 + frontend: 1 + frontend-proxy: 1 + grafana: 1 + grafana-sc-alerts: 1 + grafana-sc-dashboard: 1 + grafana-sc-datasources: 1 + image-provider: 1 + init-config: 1 + jaeger: 1 + kafka: 1 + kube-proxy: 1 + load-generator: 1 + metrics-server: 1 + opensearch: 1 + opentelemetry-collector: 1 + payment: 1 + postgresql: 1 + prometheus-server: 1 + quote: 1 + recommendation: 1 + valkey-cart: 1 + wait-for-kafka: 1 + wait-for-valkey-cart: 1 + exceptions: [] + min_delta_pp: 2 + +schedule: + preset: hourly + cron_utc: "0 * * * *" + scheduler: github_actions + +config_plane: + repo: /Users/talweiss/git/l1x-co/wt-e103-hottier/elastic-hot-tier-budget/results-e103-smoke/policy-clone + delivery: gitops + # the run commits straight to this branch; the engine's gitops pull reads it (GH_BRANCH) + policy_branch: test/elastic-hot-tier-policy + diff --git a/elastic-hot-tier-budget/results-e103-smoke/score-e103.json b/elastic-hot-tier-budget/results-e103-smoke/score-e103.json new file mode 100644 index 0000000..16e4c3e --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/score-e103.json @@ -0,0 +1,20 @@ +{ + "feed_final_seq": 101663, + "records": 79415, + "receiver_own_messages": 1, + "cross_container_records": 0, + "nonblank_lines_sent": 101637, + "lines_matched": 101637, + "lines_missing": 0, + "lines_unmatched": 0, + "records_out_of_feed_order": 0, + "readme_run_based_count": 0, + "readme_runs_mismatched": 2, + "elasticsearch": { + "documents": 79416, + "numbered_documents": 79415, + "summed_lines": 101637 + }, + "examples": [], + "verdict": "PASS" +} \ No newline at end of file diff --git a/elastic-hot-tier-budget/results-e103-smoke/timeline.jsonl b/elastic-hot-tier-budget/results-e103-smoke/timeline.jsonl new file mode 100644 index 0000000..68a3af8 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/timeline.jsonl @@ -0,0 +1,45 @@ +{"event":"run_start","at":"2026-10-26T00:10:39Z","epoch":1792973439,"scale":6,"M":6,"arm_b_top_half":["opentelemetry-collector"]} +{"event":"policy_branch_reset_to_seed","at":"2026-10-26T00:10:45Z","epoch":1792973445,"commit":"78c5eea4ed9fd3c0b1fef9757f4904e9284207d1"} +{"event":"policy_branch_seeded","at":"2026-10-26T00:10:50Z","epoch":1792973450,"branch":"test/elastic-hot-tier-policy"} +{"event":"es_ready","at":"2026-10-26T00:11:40Z","epoch":1792973500,"version":"9.5.5"} +{"event":"es_setup_done","at":"2026-10-26T00:11:45Z","epoch":1792973505} +{"event":"receiver_ready","at":"2026-10-26T00:12:25Z","epoch":1792973545} +{"event":"meter_started","at":"2026-10-26T00:12:26Z","epoch":1792973546} +{"event":"feed_start","at":"2026-10-26T00:12:27Z","epoch":1792973547,"speed":8,"lines_per_s":56} +{"event":"phase_baseline","at":"2026-10-26T00:12:27Z","epoch":1792973547} +{"event":"budget_run","at":"2026-10-26T00:14:51Z","epoch":1792973691,"dir":"budget-runs/20261026T001446Z","exit":0,"commit":"78c5eea4ed9fd3c0b1fef9757f4904e9284207d1","dry_run":true} +{"event":"size_set","at":"2026-10-26T00:14:52Z","epoch":1792973692,"uncapped_settles_bytes":58000000.0,"hot_tier_gb":0.0406} +{"event":"budget_run","at":"2026-10-26T00:14:58Z","epoch":1792973698,"dir":"budget-runs/20261026T001452Z","exit":0,"commit":"afa4fe44ce9771c9dc07d8b99ae01c3444ab6d24","dry_run":false} +{"event":"phase_held","at":"2026-10-26T00:14:58Z","epoch":1792973698} +{"event":"budget_run","at":"2026-10-26T00:15:04Z","epoch":1792973704,"dir":"budget-runs/20261026T001458Z","exit":0,"commit":"fc28e9434dc230ca97fc10af9173ef1c22d14180","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:15:15Z","epoch":1792973715,"dir":"budget-runs/20261026T001509Z","exit":0,"commit":"09309365d88d13c5d5b6f39a888e2cbd434ba3cb","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:15:27Z","epoch":1792973727,"dir":"budget-runs/20261026T001520Z","exit":0,"commit":"c72d0a2be4f4c292919e02a324ca752708d2274b","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:16:18Z","epoch":1792973778,"dir":"budget-runs/20261026T001609Z","exit":0,"commit":"79cc5655b373e5323862134669c10bd6fc64c335","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:17:16Z","epoch":1792973836,"dir":"budget-runs/20261026T001709Z","exit":0,"commit":"89bb297cf1b8d141aada6b3138632a1f990a404d","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:18:12Z","epoch":1792973892,"dir":"budget-runs/20261026T001808Z","exit":0,"commit":"ee058f9c351a20cf6521dbb4ebf7f387177c95a7","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:19:13Z","epoch":1792973953,"dir":"budget-runs/20261026T001908Z","exit":0,"commit":"1142b2983785d73525518e634d446e2336d4e0e1","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:20:12Z","epoch":1792974012,"dir":"budget-runs/20261026T002009Z","exit":0,"commit":"9a2ac1b0feb8702e9586825b0ce1a52f4808dc31","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:21:12Z","epoch":1792974072,"dir":"budget-runs/20261026T002108Z","exit":0,"commit":"bdd584edda39a3913bf210dd96714051a38b304b","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:22:12Z","epoch":1792974132,"dir":"budget-runs/20261026T002208Z","exit":0,"commit":"9f9729f27515d4a4921495cafd93544e476a69e0","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:23:16Z","epoch":1792974196,"dir":"budget-runs/20261026T002313Z","exit":0,"commit":"f2bec2d3ce3661d12bbf371e2342fc6f93e6b593","dry_run":false} +{"event":"s_binds","at":"2026-10-26T00:23:41Z","epoch":1792974221,"rule":"R after the first applied run","hot_store":0,"S":40600000} +{"event":"budget_run","at":"2026-10-26T00:24:15Z","epoch":1792974255,"dir":"budget-runs/20261026T002412Z","exit":0,"commit":"2ce259a177b753a0651d30d40ad14706d29f4f71","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:25:13Z","epoch":1792974313,"dir":"budget-runs/20261026T002510Z","exit":0,"commit":"c50b3ed2068a410b712650c9b6d1b372f6eb9361","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:26:12Z","epoch":1792974372,"dir":"budget-runs/20261026T002609Z","exit":0,"commit":"20a17566f950916eec1fab37d48e0d4b52d32cba","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:27:16Z","epoch":1792974436,"dir":"budget-runs/20261026T002712Z","exit":0,"commit":"82bd4fb4729c3b7ca6442d2d1811919257a19633","dry_run":false} +{"event":"feed_speed","at":"2026-10-26T00:27:41Z","epoch":1792974461,"speed":20,"lines_per_s":140} +{"event":"phase_step","at":"2026-10-26T00:27:41Z","epoch":1792974461,"speed":20} +{"event":"budget_run","at":"2026-10-26T00:28:14Z","epoch":1792974494,"dir":"budget-runs/20261026T002811Z","exit":0,"commit":"b38595b6f85a4836e82df549be73476fb17d161f","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:29:13Z","epoch":1792974553,"dir":"budget-runs/20261026T002909Z","exit":0,"commit":"8750a9199bc0d9c79f8dbab55ee4b043473189ec","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:30:12Z","epoch":1792974612,"dir":"budget-runs/20261026T003008Z","exit":0,"commit":"0cf7d84d09c22a6a34f532f78e9bfaeb4fb311c5","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:31:15Z","epoch":1792974675,"dir":"budget-runs/20261026T003112Z","exit":0,"commit":"74d3d473d13dfd1093ff84a85d1dcd948e2bbdaf","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:32:14Z","epoch":1792974734,"dir":"budget-runs/20261026T003210Z","exit":0,"commit":"3aee0d6bca3d40f7ee6c384766b051e592608944","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:33:12Z","epoch":1792974792,"dir":"budget-runs/20261026T003309Z","exit":0,"commit":"5b00b5892c77762542b5dac9d49f134eba096e59","dry_run":false} +{"event":"feed_stop","at":"2026-10-26T00:33:45Z","epoch":1792974825,"final_seq":101663} +{"event":"phase_drain","at":"2026-10-26T00:33:45Z","epoch":1792974825} +{"event":"budget_run","at":"2026-10-26T00:34:14Z","epoch":1792974854,"dir":"budget-runs/20261026T003410Z","exit":0,"commit":"11c9221f78ca19363f57e2bbed1752c4ae6105bc","dry_run":false} +{"event":"budget_run","at":"2026-10-26T00:35:12Z","epoch":1792974912,"dir":"budget-runs/20261026T003509Z","exit":0,"commit":"dca4133ef2fbbfc884b3e95aeabdd6af9ad1054f","dry_run":false} +{"event":"ilm_stopped","at":"2026-10-26T00:35:50Z","epoch":1792974950} +{"event":"delivery_read","at":"2026-10-26T00:35:53Z","epoch":1792974953} +{"event":"run_end","at":"2026-10-26T00:36:00Z","epoch":1792974960} +{"event":"clean","at":"2026-10-26T00:36:15Z","epoch":1792974975,"volumes_removed":"elastic-budget_es-frozen-data elastic-budget_es-hot-data elastic-budget_es-snapshots"} diff --git a/elastic-hot-tier-budget/results-e103-smoke/values/compose.receiver.yaml b/elastic-hot-tier-budget/results-e103-smoke/values/compose.receiver.yaml new file mode 100644 index 0000000..f6a196f --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/values/compose.receiver.yaml @@ -0,0 +1,28 @@ +# Generated by rig/gen-receiver-values.mjs from values.yaml in this directory. Do not edit. +services: + tenx: + image: "local/edge-10x:e103" + container_name: hb-tenx + environment: + - "TENX_AIRGAPPED=true" + - "GH_ENABLED=true" + - "GH_REPO=log-10x/config" + - "GH_BRANCH=test/elastic-hot-tier-policy" + - "GH_TOKEN" + - "TENX_FLUENTBIT_OUTPUT_HOST=fluent-bit" + - "TENX_JAVA_HEAP=320M" + tmpfs: + - /tmp + command: + - "@run/input/forwarder/fluentbit" + - "@apps/receiver" + - "symbolMessageHashField" + - "tenx_hash" + - "outputEncodeField" + - "fullText(\"tenx_hash\",\"routeState\",\"message_pattern\",\"severity_level\",\"utf8Size\")" + deploy: + resources: + limits: + cpus: "0.5" + memory: "512M" + restart: unless-stopped diff --git a/elastic-hot-tier-budget/results-e103-smoke/values/launch.json b/elastic-hot-tier-budget/results-e103-smoke/values/launch.json new file mode 100644 index 0000000..26236f2 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/values/launch.json @@ -0,0 +1,58 @@ +{ + "mcp_version": "1.30.84", + "cap_bytes_per_window_per_node": null, + "image": "local/edge-10x:e103", + "image_rendered": "log10x/edge-10x:1.1.147@sha256:a043ccd828742c4eccee09abb654ce6a7357b6123f73300ed04f8605e874b8c4", + "args_from_values": [ + "@run/input/forwarder/fluentbit", + "@apps/receiver", + "symbolMessageHashField", + "tenx_hash", + "outputEncodeField", + "fullText(\"tenx_hash\",\"routeState\",\"message_pattern\",\"severity_level\",\"utf8Size\")" + ], + "env_applied": [ + { + "name": "TENX_AIRGAPPED", + "value": "true" + }, + { + "name": "GH_ENABLED", + "value": "true" + }, + { + "name": "GH_REPO", + "value": "log-10x/config" + }, + { + "name": "GH_BRANCH", + "value": "test/elastic-hot-tier-policy" + } + ], + "env_substituted": [], + "env_left_out": [], + "env_from_shell_by_name": [ + "GH_TOKEN" + ], + "policy_git": { + "repo": "log-10x/config", + "branch": "test/elastic-hot-tier-policy", + "tokenSecretName": "log10x-policy-git" + }, + "rig_env": [ + [ + "TENX_FLUENTBIT_OUTPUT_HOST", + "fluent-bit" + ], + [ + "TENX_JAVA_HEAP", + "320M" + ] + ], + "measurement_args": [], + "resources_limits": { + "cpus": "0.5", + "memory": "512M" + }, + "notes": [] +} diff --git a/elastic-hot-tier-budget/results-e103-smoke/values/receiver-values.json b/elastic-hot-tier-budget/results-e103-smoke/values/receiver-values.json new file mode 100644 index 0000000..43a298c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/values/receiver-values.json @@ -0,0 +1,20 @@ +{ + "options": [ + [ + "symbolMessageHashField", + "tenx_hash" + ], + [ + "outputEncodeField", + "fullText(\"tenx_hash\",\"routeState\",\"message_pattern\",\"severity_level\",\"utf8Size\")" + ] + ], + "env": [], + "fleetDefaultWritten": false, + "teamLabelWritten": false, + "regulatorValuesWritten": false, + "keepCountsWritten": false, + "eventsUnitWritten": false, + "engineVersion": "1.1.147", + "notes": [] +} diff --git a/elastic-hot-tier-budget/results-e103-smoke/values/render.out b/elastic-hot-tier-budget/results-e103-smoke/values/render.out new file mode 100644 index 0000000..3948b51 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/values/render.out @@ -0,0 +1,27 @@ +{ + "image": "local/edge-10x:e103", + "args": [ + "@run/input/forwarder/fluentbit", + "@apps/receiver", + "symbolMessageHashField", + "tenx_hash", + "outputEncodeField", + "fullText(\"tenx_hash\",\"routeState\",\"message_pattern\",\"severity_level\",\"utf8Size\")" + ], + "env_applied": [ + "TENX_AIRGAPPED", + "GH_ENABLED", + "GH_REPO", + "GH_BRANCH" + ], + "env_left_out": [], + "env_from_shell_by_name": [ + "GH_TOKEN" + ], + "policy_git": { + "repo": "log-10x/config", + "branch": "test/elastic-hot-tier-policy", + "tokenSecretName": "log10x-policy-git" + }, + "notes": [] +} diff --git a/elastic-hot-tier-budget/results-e103-smoke/values/seed.sh b/elastic-hot-tier-budget/results-e103-smoke/values/seed.sh new file mode 100644 index 0000000..ac19ea8 --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/values/seed.sh @@ -0,0 +1,17 @@ +# needs GH_TOKEN exported: a PAT with Contents write on log-10x/config +set -euo pipefail +AUTH="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n')" +SEED=$(mktemp -d) +trap 'rm -rf "$SEED"' EXIT +git -c "http.https://github.com/.extraheader=$AUTH" clone --quiet "https://github.com/log-10x/config.git" "$SEED" +cd "$SEED" +if git -c "http.https://github.com/.extraheader=$AUTH" ls-remote --exit-code --heads origin test/elastic-hot-tier-policy >/dev/null; then git -c "http.https://github.com/.extraheader=$AUTH" fetch --quiet origin test/elastic-hot-tier-policy && git checkout --quiet -B test/elastic-hot-tier-policy FETCH_HEAD; else git checkout --quiet -b test/elastic-hot-tier-policy; fi +mkdir -p log10x/policy +[ -f log10x/policy/caps.csv ] || printf 'container,cap\ntenx-seed,1::install seed\n' > log10x/policy/caps.csv +[ -f log10x/policy/actions.csv ] || printf 'container,action\ntenx-seed,pass::install seed\n' > log10x/policy/actions.csv +[ -f log10x/policy/totals.csv ] || printf 'container,total\ntenx-seed,1::install seed\n' > log10x/policy/totals.csv +[ -f log10x/policy/config-generation.csv ] || printf 'key,value\ngeneration,seed\n' > log10x/policy/config-generation.csv +printf 'bytes\n' > log10x/policy/regulator-unit +git add log10x/policy +git -c user.name=log10x-install -c user.email=install@log10x.com commit --quiet -m "log10x: seed the policy files" || true +git -c "http.https://github.com/.extraheader=$AUTH" push --quiet origin test/elastic-hot-tier-policy diff --git a/elastic-hot-tier-budget/results-e103-smoke/values/values.yaml b/elastic-hot-tier-budget/results-e103-smoke/values/values.yaml new file mode 100644 index 0000000..b42387c --- /dev/null +++ b/elastic-hot-tier-budget/results-e103-smoke/values/values.yaml @@ -0,0 +1,109 @@ +# Receiver overlay for Fluent Bit (upstream fluent/fluent-bit chart). +# Layer on top of your existing values: +# helm upgrade --install fluent/fluent-bit \ +# -f your-existing-fluent-bit-values.yaml \ +# -f my-receiver.yaml --namespace + +extraContainers: + - name: log10x + image: log10x/edge-10x:1.1.147@sha256:a043ccd828742c4eccee09abb654ce6a7357b6123f73300ed04f8605e874b8c4 + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + args: + - "@run/input/forwarder/fluentbit" + - "@apps/receiver" + - "symbolMessageHashField" + - "tenx_hash" + - "outputEncodeField" + - "fullText("tenx_hash","routeState","message_pattern","severity_level","utf8Size")" + env: + - name: TENX_AIRGAPPED + value: "true" + - name: GH_ENABLED + value: "true" + - name: GH_REPO + value: "log-10x/config" + - name: GH_BRANCH + value: "test/elastic-hot-tier-policy" + - name: GH_TOKEN + valueFrom: + secretKeyRef: + name: log10x-policy-git + key: GH_TOKEN + volumeMounts: + - name: tenx-tmp + mountPath: /tmp + resources: + requests: { cpu: 100m, memory: 256Mi } + limits: { cpu: 500m, memory: 512Mi } + +extraVolumes: + - name: tenx-tmp + emptyDir: {} + +# Replace the chart's default config with the sidecar bypass pattern. +# Inputs: tail your sources + receive processed events back from the +# sidecar on :24225 with Tag_Prefix tenx. (so filters and the ingest +# handoff Match kube.* and skip the returning tenx.*). +# Filters: enrichment (kubernetes metadata) on kube.* only. +# Outputs: handoff to the sidecar on 127.0.0.1:24224 + destination +# for the returning tenx.* events. +config: + service: | + [SERVICE] + Daemon Off + Flush {{ .Values.flush }} + Log_Level {{ .Values.logLevel }} + Parsers_File /fluent-bit/etc/parsers.conf + Parsers_File /fluent-bit/etc/conf/custom_parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port {{ .Values.metricsPort }} + Health_Check On + + inputs: | + [INPUT] + Name tail + Path /var/log/containers/*.log + Exclude_Path /var/log/containers/*fluent-bit*.log,/var/log/containers/*fluentd*.log,/var/log/containers/*filebeat*.log,/var/log/containers/*logstash*.log,/var/log/containers/*otel-collector*.log,/var/log/containers/*opentelemetry-collector*.log + multiline.parser docker, cri + Tag kube.* + Mem_Buf_Limit 5MB + Skip_Long_Lines On + + # Egress: returning events from the 10x sidecar. Tag_Prefix tenx. + # prevents enrichment filters and the sidecar-handoff output from + # re-firing (they Match kube.* only). + [INPUT] + Name forward + Listen 0.0.0.0 + Port 24225 + Tag_Prefix tenx. + + filters: | + [FILTER] + Name kubernetes + Match kube.* + Merge_Log On + Keep_Log Off + K8S-Logging.Parser On + K8S-Logging.Exclude On + + outputs: | + # Hand off to the Log10x sidecar. + [OUTPUT] + Name forward + Match kube.* + Host 127.0.0.1 + Port 24224 + Retry_Limit False + + [OUTPUT] + Name es + Match tenx.* + Host elasticsearch-master + Logstash_Format On diff --git a/elastic-hot-tier-budget/rig/gen-receiver-values.mjs b/elastic-hot-tier-budget/rig/gen-receiver-values.mjs index 8ea074b..ee83810 100644 --- a/elastic-hot-tier-budget/rig/gen-receiver-values.mjs +++ b/elastic-hot-tier-budget/rig/gen-receiver-values.mjs @@ -88,6 +88,8 @@ for (const e of env) { else left.push(e); } const rigEnv = [['TENX_FLUENTBIT_OUTPUT_HOST', 'fluent-bit']]; +// An engine image that runs its jar on a JVM: TENX_JAVA_HEAP keeps the heap inside the memory limit. +if (process.env.TENX_JAVA_HEAP) rigEnv.push(['TENX_JAVA_HEAP', process.env.TENX_JAVA_HEAP]); // The MCP renders symbolMessageHashField and the utf8Size field list itself (elasticHotTier), so nothing is added. const measure = []; diff --git a/elastic-hot-tier-budget/rig/phases-two-arm.sh b/elastic-hot-tier-budget/rig/phases-two-arm.sh index 1f32153..3830f5a 100755 --- a/elastic-hot-tier-budget/rig/phases-two-arm.sh +++ b/elastic-hot-tier-budget/rig/phases-two-arm.sh @@ -12,7 +12,8 @@ set -euo pipefail HERE=$(cd "$(dirname "$0")" && pwd) RESULTS=${RESULTS:?results dir} RUN="$HERE/run.sh" -export ARM_B=1 ES_URL_B=http://127.0.0.1:9201 +# ARM_A_ONLY=1 runs arm A alone: no cluster B and no copy to it; Fluent Bit also keeps each returned record whole +if [ -z "${ARM_A_ONLY:-}" ]; then export ARM_B=1 ES_URL_B=http://127.0.0.1:9201; fi export POLICY_BRANCH=${POLICY_BRANCH:-test/elastic-hot-tier-policy} export FB_DIR=$RESULTS/fluent-bit KEEP_ROLLOVER=10m KEEP_FROZEN_AFTER=110m TD_ROLLOVER=10m ILM_POLL=100s METER_S=${METER_S:-100} export FROZEN_CACHE=${FROZEN_CACHE:-128mb} @@ -88,12 +89,12 @@ while :; do done "$RUN" logs -curl -s -m 30 -X POST http://127.0.0.1:9200/_ilm/stop > /dev/null; curl -s -m 30 -X POST http://127.0.0.1:9201/_ilm/stop > /dev/null +curl -s -m 30 -X POST http://127.0.0.1:9200/_ilm/stop > /dev/null; [ -z "${ARM_B:-}" ] || curl -s -m 30 -X POST http://127.0.0.1:9201/_ilm/stop > /dev/null "$RUN" mark ilm_stopped mkdir -p "$RESULTS/delivery" python3 "$HERE/dump-seq.py" http://127.0.0.1:9200 tenx-app,tenx-tierdown "$RESULTS/delivery/a.tsv.gz" > "$RESULTS/delivery/a.json" || true -python3 "$HERE/dump-seq.py" http://127.0.0.1:9201 native-app,native-overflow "$RESULTS/delivery/b.tsv.gz" > "$RESULTS/delivery/b.json" || true -for p in 9200 9201; do +[ -z "${ARM_B:-}" ] || python3 "$HERE/dump-seq.py" http://127.0.0.1:9201 native-app,native-overflow "$RESULTS/delivery/b.tsv.gz" > "$RESULTS/delivery/b.json" || true +for p in 9200 ${ARM_B:+9201}; do curl -s -m 60 "http://127.0.0.1:$p/_cat/indices?format=json&bytes=b&expand_wildcards=all&h=index,health,status,docs.count,store.size,creation.date" > "$RESULTS/delivery/cat-indices-$p.json" || true curl -s -m 60 "http://127.0.0.1:$p/_alias?filter_path=*.aliases" > "$RESULTS/delivery/aliases-$p.json" || true done diff --git a/elastic-hot-tier-budget/rig/render-fb-two-arm.py b/elastic-hot-tier-budget/rig/render-fb-two-arm.py index b6d08e5..b5a1cfe 100755 --- a/elastic-hot-tier-budget/rig/render-fb-two-arm.py +++ b/elastic-hot-tier-budget/rig/render-fb-two-arm.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Writes the two-arm Fluent Bit config with arm B's service list from top-half.py. +"""Writes the two-arm Fluent Bit config with arm B's service list from top-half.py. With ARM_A_ONLY set, +arm B's blocks are left out and every returned record is also written whole to returned.jsonl. render-fb-two-arm.py """ import json, os, re, shutil, sys @@ -8,6 +9,17 @@ os.makedirs(out, exist_ok=True) alt = '|'.join(re.escape(c) for c in rule['top_half']) conf = open(os.path.join(HERE, 'fluent-bit-two-arm', 'fluent-bit.conf.tmpl')).read().replace('@TOP_HALF@', alt) +if os.environ.get('ARM_A_ONLY'): + blocks = re.split(r'\n(?=\[)', conf) + conf = '\n'.join(b for b in blocks if 'armb' not in b).rstrip('\n') + ''' + +[OUTPUT] + Name file + Match routed.* + Path /data + File returned.jsonl + Format plain +''' open(os.path.join(out, 'fluent-bit.conf'), 'w').write(conf) shutil.copy(os.path.join(HERE, 'fluent-bit-two-arm', 'rig.lua'), os.path.join(out, 'rig.lua')) print(alt) diff --git a/elastic-hot-tier-budget/rig/score-e103.py b/elastic-hot-tier-budget/rig/score-e103.py new file mode 100644 index 0000000..6e80ab6 --- /dev/null +++ b/elastic-hot-tier-budget/rig/score-e103.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Arm A's records by the lines they hold (engine ask E103). Reads the feed (replay/source.jsonl: seq s is +row (s - 1) mod rows), the feed's last seq (timeline feed_stop), and every record the Receiver returned, kept +whole by Fluent Bit (data/returned.jsonl, ARM_A_ONLY). Each record's non-blank lines are matched to the +non-blank lines sent: the first at or after the record's seq, each next one the nearest unused sent line with +the same text after the line before it (or, failing that, the nearest unused one before it). A record whose +matched lines came from more than one container is a cross-container record. +Reports: records, cross-container records, non-blank lines sent, matched, missing (sent, in no record) and +unmatched (returned, no unused sent line: a duplicate or a changed line), records whose lines are out of feed +order; beside them, the count the 2026-10-24 README used (records sorted by seq, each the run of seqs up to +the next record's, spanning containers), which assumes lines are grouped in arrival order; and the documents +in Elasticsearch (delivery/a.tsv.gz) with their summed line counts. Exit 1 on any cross-container record, +missing or unmatched line. + score-e103.py + score-e103.py --self-test groups the run's feed with one head rule, in arrival order and per + container: the first must show cross-container records (by lines as + many as by the run-based count), the second none""" +import bisect, collections, gzip, json, os, re, sys + +NB = re.compile(r'[^ \t\n\v\f\r]') +WINDOW = 200000 + + +def opener(p): + if os.path.exists(p): + return open(p) + return gzip.open(p + '.gz', 'rt') + + +def load(R): + src = [json.loads(l) for l in open(os.path.join(R, 'replay', 'source.jsonl'))] + final = None + for l in open(os.path.join(R, 'timeline.jsonl')): + e = json.loads(l) + if e.get('event') == 'feed_stop': + final = e.get('final_seq') + recs, own = [], 0 + with opener(os.path.join(R, 'data', 'returned.jsonl')) as f: + for l in f: + if not l.strip(): + continue + r = json.loads(l) + if r.get('seq') is None: + own += 1 + continue + recs.append((int(r['seq']), (r.get('kubernetes') or {}).get('container_name'), r.get('log') or '', r.get('_lines'))) + return src, final, recs, own + + +def sent_lines(src, N): + n = len(src) + ent = [] # (seq, text, container) per non-blank line, in feed order + for s in range(1, N + 1): + r = src[(s - 1) % n] + c = r['container_name'].lstrip('/') + for t in r['log'].split('\n'): + if NB.search(t): + ent.append((s, t.rstrip(), c)) + return ent + + +def score(ent, recs, src, N): + pos = collections.defaultdict(list) + for i, (_, t, _) in enumerate(ent): + pos[t].append(i) + seqs = [e[0] for e in ent] + used = bytearray(len(ent)) + + def take(t, prev): + p = pos.get(t) + if not p: + return None + k = bisect.bisect_right(p, prev) + j = k + while j < len(p) and p[j] - prev <= WINDOW: + if not used[p[j]]: + return p[j] + j += 1 + j = k - 1 + while j >= 0 and prev - p[j] <= WINDOW: + if not used[p[j]]: + return p[j] + j -= 1 + return None + + cross, unmatched, out_of_order, examples = 0, 0, 0, [] + for seq, c, log, _ in sorted(recs, key=lambda x: x[0]): + prev = bisect.bisect_left(seqs, seq) - 1 + got = [] + for t in log.split('\n'): + if not NB.search(t): + continue + i = take(t.rstrip(), prev) + if i is None: + unmatched += 1 + continue + used[i] = 1 + got.append(i) + prev = i + out_of_order += any(got[k] > got[k + 1] for k in range(len(got) - 1)) + if len({ent[i][2] for i in got}) > 1: + cross += 1 + if len(examples) < 5: + examples.append({'record_seq': seq, 'filed_under': c, 'lines': [{'seq': ent[i][0], 'container': ent[i][2]} for i in got]}) + return cross, unmatched, out_of_order, examples, used + + +def run_based(recs, src, N): + n = len(src) + nonblank = [sum(1 for l in r['log'].split('\n') if NB.search(l)) for r in src] + cont = [r['container_name'].lstrip('/') for r in src] + rs = sorted((s, (ln or 0)) for s, _, _, ln in recs) + spanning, mismatched = 0, 0 + for k, (s, ln) in enumerate(rs): + nxt = rs[k + 1][0] if k + 1 < len(rs) else N + 1 + run = range(s, nxt) + if sum(nonblank[(x - 1) % n] for x in run) != ln: + mismatched += 1 + if len({cont[(x - 1) % n] for x in run if nonblank[(x - 1) % n]}) > 1: + spanning += 1 + return spanning, mismatched + + +def es_side(R): + p = os.path.join(R, 'delivery', 'a.tsv.gz') + if not os.path.exists(p): + return None + docs = numbered = lines = 0 + for l in gzip.open(p, 'rt'): + seq, c, ln, route, index = l.rstrip('\n').split('\t') + docs += 1 + if seq: + numbered += 1 + lines += int(ln) if ln not in ('', 'None') else 0 + return {'documents': docs, 'numbered_documents': numbered, 'summed_lines': lines} + + +def main(): + self_test = sys.argv[1] == '--self-test' + R = sys.argv[2] if self_test else sys.argv[1] + src, final, recs, own = load(R) + N = final or max(s for s, *_ in recs) + ent = sent_lines(src, N) + if self_test: + # both groupings built from the feed with one head rule: in arrival order across containers (the + # defect) and per container (the fix); the first must show cross-container records, counted alike by + # lines and by the run-based method, the second none + head = re.compile(r'^(\{|\[|\d|[A-Z]{4,5}\b|level=|time=|ts=|[IWEF]\d{4})') + n = len(src) + def group(key): + out, cur = [], {} + for s in range(1, N + 1): + r = src[(s - 1) % n] + if not NB.search(r['log']): + continue + c = r['container_name'].lstrip('/') + k = key(c) + if head.match(r['log']) or k not in cur: + if k in cur: + out.append(cur.pop(k)) + cur[k] = [s, c, [], 0] + cur[k][2].append(r['log']) + cur[k][3] += sum(1 for t in r['log'].split('\n') if NB.search(t)) + out += cur.values() + return [(s, c, '\n'.join(lines), ln) for s, c, lines, ln in out] + res = {} + for name, key in (('arrival_order', lambda c: 0), ('per_container', lambda c: c)): + fake = group(key) + cross, unmatched, ooo, ex, used = score(ent, fake, src, N) + spanning, mism = run_based(fake, src, N) + res[name] = {'records': len(fake), 'cross_container_by_lines': cross, 'run_based_count': spanning, + 'unmatched': unmatched, 'missing': len(ent) - sum(used)} + a, b = res['arrival_order'], res['per_container'] + ok = (a['cross_container_by_lines'] > 0 and a['cross_container_by_lines'] == a['run_based_count'] and b['cross_container_by_lines'] == 0 + and not (a['unmatched'] or a['missing'] or b['unmatched'] or b['missing'])) + print(json.dumps({'self_test': 'PASS' if ok else 'FAIL', **res})) + sys.exit(0 if ok else 1) + cross, unmatched, ooo, examples, used = score(ent, recs, src, N) + spanning, mism = run_based(recs, src, N) + missing = len(ent) - sum(used) + out = {'feed_final_seq': final, 'records': len(recs), 'receiver_own_messages': own, 'cross_container_records': cross, + 'nonblank_lines_sent': len(ent), 'lines_matched': sum(used), 'lines_missing': missing, 'lines_unmatched': unmatched, + 'records_out_of_feed_order': ooo, + 'readme_run_based_count': spanning, 'readme_runs_mismatched': mism, + 'elasticsearch': es_side(R), 'examples': examples} + out['verdict'] = 'PASS' if not (cross or missing or unmatched) else 'FAIL' + json.dump(out, open(os.path.join(R, 'score-e103.json'), 'w'), indent=1) + print(json.dumps({k: v for k, v in out.items() if k != 'examples'}, indent=1)) + sys.exit(0 if out['verdict'] == 'PASS' else 1) + + +if __name__ == '__main__': + main()