From 422addcc9ae6bd2bee0642c74a647e30e6d26ba8 Mon Sep 17 00:00:00 2001 From: Tal Weiss <73443663+talwgx@users.noreply.github.com> Date: Sun, 25 Oct 2026 19:54:09 -0400 Subject: [PATCH] elastic-hot-tier-budget: install proof, recipe and Receiver from log10x-mcp 1.30.90 tools alone, four hot-tier runs exit 0 Co-Authored-By: Claude Opus 5.5 --- .../install-e2e/.gitignore | 3 + elastic-hot-tier-budget/install-e2e/README.md | 30 ++ .../install-e2e/compose.yaml | 74 +++ .../install-e2e/render.mjs | 172 ++++++ .../install-e2e/results/apply-instructions.md | 9 + .../install-e2e/results/compose.receiver.yaml | 28 + .../install-e2e/results/e100-fluent-bit.log | 54 ++ .../install-e2e/results/e100-tenx.log | 63 +++ .../install-e2e/results/es-setup.json | 82 +++ .../install-e2e/results/final-state.txt | 49 ++ .../install-e2e/results/install-plan.md | 358 ++++++++++++ .../install-e2e/results/launch.json | 60 +++ .../install-e2e/results/policy.yaml | 72 +++ .../install-e2e/results/policy/actions.csv | 34 ++ .../install-e2e/results/policy/caps.csv | 34 ++ .../install-e2e/results/policy/totals.csv | 34 ++ .../install-e2e/results/receiver.json | 69 +++ .../install-e2e/results/recipe-requests.json | 137 +++++ .../install-e2e/results/recipe.txt | 89 +++ .../install-e2e/results/render.out | 36 ++ .../install-e2e/results/retriever-plan.md | 508 ++++++++++++++++++ .../install-e2e/results/setup-es.out | 9 + .../ticks/20261025T235021Z/branch-log.txt | 2 + .../results/ticks/20261025T235021Z/exit | 1 + .../results/ticks/20261025T235021Z/report.txt | 52 ++ .../results/ticks/20261025T235021Z/run.json | 1 + .../ticks/20261025T235028Z/branch-log.txt | 3 + .../results/ticks/20261025T235028Z/exit | 1 + .../results/ticks/20261025T235028Z/report.txt | 58 ++ .../results/ticks/20261025T235028Z/run.json | 1 + .../ticks/20261025T235318Z/branch-log.txt | 3 + .../results/ticks/20261025T235318Z/exit | 1 + .../results/ticks/20261025T235318Z/report.txt | 61 +++ .../results/ticks/20261025T235318Z/run.json | 1 + .../ticks/20261025T235320Z/branch-log.txt | 4 + .../results/ticks/20261025T235320Z/exit | 1 + .../results/ticks/20261025T235320Z/report.txt | 61 +++ .../results/ticks/20261025T235320Z/run.json | 1 + .../install-e2e/results/timeline.jsonl | 13 + elastic-hot-tier-budget/install-e2e/run.sh | 106 ++++ .../install-e2e/setup-es.mjs | 45 ++ 41 files changed, 2420 insertions(+) create mode 100644 elastic-hot-tier-budget/install-e2e/.gitignore create mode 100644 elastic-hot-tier-budget/install-e2e/README.md create mode 100644 elastic-hot-tier-budget/install-e2e/compose.yaml create mode 100644 elastic-hot-tier-budget/install-e2e/render.mjs create mode 100644 elastic-hot-tier-budget/install-e2e/results/apply-instructions.md create mode 100644 elastic-hot-tier-budget/install-e2e/results/compose.receiver.yaml create mode 100644 elastic-hot-tier-budget/install-e2e/results/e100-fluent-bit.log create mode 100644 elastic-hot-tier-budget/install-e2e/results/e100-tenx.log create mode 100644 elastic-hot-tier-budget/install-e2e/results/es-setup.json create mode 100644 elastic-hot-tier-budget/install-e2e/results/final-state.txt create mode 100644 elastic-hot-tier-budget/install-e2e/results/install-plan.md create mode 100644 elastic-hot-tier-budget/install-e2e/results/launch.json create mode 100644 elastic-hot-tier-budget/install-e2e/results/policy.yaml create mode 100644 elastic-hot-tier-budget/install-e2e/results/policy/actions.csv create mode 100644 elastic-hot-tier-budget/install-e2e/results/policy/caps.csv create mode 100644 elastic-hot-tier-budget/install-e2e/results/policy/totals.csv create mode 100644 elastic-hot-tier-budget/install-e2e/results/receiver.json create mode 100644 elastic-hot-tier-budget/install-e2e/results/recipe-requests.json create mode 100644 elastic-hot-tier-budget/install-e2e/results/recipe.txt create mode 100644 elastic-hot-tier-budget/install-e2e/results/render.out create mode 100644 elastic-hot-tier-budget/install-e2e/results/retriever-plan.md create mode 100644 elastic-hot-tier-budget/install-e2e/results/setup-es.out create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/branch-log.txt create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/exit create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/report.txt create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/run.json create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/branch-log.txt create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/exit create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/report.txt create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/run.json create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/branch-log.txt create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/exit create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/report.txt create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/run.json create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/branch-log.txt create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/exit create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/report.txt create mode 100644 elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/run.json create mode 100644 elastic-hot-tier-budget/install-e2e/results/timeline.jsonl create mode 100755 elastic-hot-tier-budget/install-e2e/run.sh create mode 100644 elastic-hot-tier-budget/install-e2e/setup-es.mjs diff --git a/elastic-hot-tier-budget/install-e2e/.gitignore b/elastic-hot-tier-budget/install-e2e/.gitignore new file mode 100644 index 0000000..2fae44f --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/.gitignore @@ -0,0 +1,3 @@ +results/data/ +results/replay/ +results/policy-git/ diff --git a/elastic-hot-tier-budget/install-e2e/README.md b/elastic-hot-tier-budget/install-e2e/README.md new file mode 100644 index 0000000..33369e9 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/README.md @@ -0,0 +1,30 @@ +# Install proof: the hot-tier run from the tools alone + +Does `log10x_setup_recurring hot_tier_gb` run against a cluster set up only by the frozen-tier recipe from `log10x_advise_retriever` and a Receiver installed only from `log10x_advise_install` with `log_destination: "elasticsearch"`, with no hand-set engine option and no hand-added ILM phase? + +**Result: yes.** Four `tenx-recur` runs, all exit 0 at their reads: a dry run and an applied run with only kept lines on hot, then, once the applied limits sent lines to `tenx-tierdown`, a dry run and an applied run with documents in both aliases. Each run read the kept lines' hours on hot from the recipe's own policy: `A kept line stays on hot 168 hours (ILM policy tenx-keep (rollover max_age 1d + frozen min_age 6d))`. + +log10x-mcp 1.30.90, engine image `ghcr.io/log-10x/edge-10x:1.1.150` (the values render `log10x/edge-10x:1.1.147@sha256:a043ccd828742c4eccee09abb654ce6a7357b6123f73300ed04f8605e874b8c4`; `launch.json` records both), Elasticsearch 9.5.5, one hot node and one warm and frozen node, 2026-10-25 23:46 to 23:53 UTC. No time scaling. + +## What ran + +`MCP= ./run.sh `, in this order: + +1. `render 0.002`: `render.mjs` calls the three tools' handlers from the build. `advise_retriever` with `destination: "elasticsearch"`: the frozen-tier block is `results/recipe.txt`, its six `PUT` requests with the comment lines dropped are `results/recipe-requests.json`. `advise_install` with app `receiver`, forwarder `fluentbit`, `log_destination: "elasticsearch"`, builtin licence, airgapped, metrics backend `elastic`: the plan is `results/install-plan.md`, the sidecar and the policy seed are `results/receiver.json`. `setup_recurring` with `hot_tier_gb: 0.002` on `github_actions`, every replay container in `nodes_by_container`: `results/policy.yaml` and `results/apply-instructions.md`. The size is 2 MB so that the run holds lines on the first pass. +2. `policy-init`: a local bare repo stands in for the policy repo: `policy.yaml` on `main`, the seed files from advise_install's ConfigMap (`caps.csv`, `actions.csv`, `totals.csv`, `regulator-unit: bytes`) under `log10x/policy/` on `log10x-policy`. +3. `up`: the two Elasticsearch nodes, then `setup-es.mjs`: the trial licence and a filesystem repository registered as `found-snapshots` (cluster preparation, so that the recipe applies as rendered), then the six requests, all 200; `tenx-tierdown-000001` on the warm node, `tenx-app-000001` on hot. Then Fluent Bit (`../rig/fluent-bit`, which sends `tier_down` lines to `tenx-tierdown` and the rest to `tenx-app`) and the Receiver from `results/compose.receiver.yaml`: the args exactly as the values render them, including `symbolMessageHashField tenx_hash` and `outputEncodeField fullText("tenx_hash","routeState","message_pattern","severity_level","utf8Size")`. +4. `feed start 8`: the OTel demo sample at 56 lines a second (`../rig/forward_send.py`, the first 12,000 rows of the replay source). +5. `tick --dry-run`, then `tick`: exit 0 both; the applied run committed the four files to `log10x-policy`. +6. `sync`: the committed `caps.csv`, `actions.csv` and `totals.csv` copied into the Receiver's policy mount, Receiver restarted. +7. About three minutes later, `tenx-app` held 7,440 documents and `tenx-tierdown` 2,340, every `tier_down` document with `utf8Size`. `tick --dry-run`, then `tick`: exit 0 both. +8. `feed stop`, `down`: every e100 container, volume and network removed. + +Summary line of the last run (`results/ticks/20261025T235320Z/run.json`): `applied: Hot tier: 4 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 4.6 GB within 168 hours, with 442 MB a day of counted lines moved to the frozen tier.` It settles above the size because the always-pass sample alone keeps about 592 MB a day, which the report states. + +## Rig substitutions (no cluster) + +The values read the policy files through the Kubernetes pull lane (`K8S_ENABLED`, `K8S_CONFIGMAP`, `K8S_NAMESPACE`); here `K8S_ENABLED=false` and the three lookup files are mounted at `/policy`. `CONFIG_GENERATION_FILE` is left out, since the seed has no `config-generation.csv`. `TENX_FLUENTBIT_OUTPUT_HOST=fluent-bit`, because Fluent Bit is a separate container. The applied files reach the Receiver by `sync` and a restart, in place of the ConfigMap or git pull. No engine option was added or changed. `launch.json` lists each substitution. + +## Not shown + +Kept lines reaching frozen after 6 days; a tier_down mount; the GitHub Actions workflow or a Kubernetes CronJob; the git pull lane (`policy_git`); an Elastic Cloud deployment. diff --git a/elastic-hot-tier-budget/install-e2e/compose.yaml b/elastic-hot-tier-budget/install-e2e/compose.yaml new file mode 100644 index 0000000..97ab921 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/compose.yaml @@ -0,0 +1,74 @@ +# E100/E101 install proof: one hot node and one warm+frozen node (Elasticsearch 9.5.5, the +# same images, heaps and limits as ../rig/compose.yaml with ../rig/compose.warm.yaml), Fluent Bit +# with ../rig/fluent-bit, and the Receiver from compose.receiver.yaml (written by render.mjs from +# log10x_advise_install's elasticsearch values). Compose project e100, containers e100-*. +# docker compose -p e100 -f compose.yaml -f results/compose.receiver.yaml ... + +volumes: + es-hot-data: {} + es-wf-data: {} + es-snapshots: {} + +x-es-common: &es-common + image: docker.elastic.co/elasticsearch/elasticsearch:9.5.5 + ulimits: + memlock: { soft: -1, hard: -1 } + nofile: { soft: 65536, hard: 65536 } + +services: + + es-hot: + <<: *es-common + container_name: e100-es-hot + environment: + - node.name=es-hot + - cluster.name=e100 + - node.roles=master,data_hot,data_content,ingest + - cluster.initial_master_nodes=es-hot + - discovery.seed_hosts=es-hot + - path.repo=/snapshots + - xpack.security.enabled=false + - xpack.ml.enabled=false + - ES_JAVA_OPTS=-Xms1g -Xmx1g + volumes: + - es-hot-data:/usr/share/elasticsearch/data + - es-snapshots:/snapshots + ports: + - "127.0.0.1:9200:9200" + deploy: + resources: + limits: + memory: 2G + + es-warm-frozen: + <<: *es-common + container_name: e100-es-warm-frozen + environment: + - node.name=es-warm-frozen + - cluster.name=e100 + - node.roles=data_warm,data_frozen + - discovery.seed_hosts=es-hot + - path.repo=/snapshots + - xpack.security.enabled=false + - xpack.ml.enabled=false + - xpack.searchable.snapshot.shared_cache.size=512mb + - ES_JAVA_OPTS=-Xms512m -Xmx512m + volumes: + - es-wf-data:/usr/share/elasticsearch/data + - es-snapshots:/snapshots + deploy: + resources: + limits: + memory: 1200M + + fluent-bit: + image: fluent/fluent-bit:4.2 + container_name: e100-fluent-bit + command: ["/fluent-bit/bin/fluent-bit", "-c", "/fluent-bit/etc/rig/fluent-bit.conf"] + volumes: + - ${RIG_DIR}/fluent-bit:/fluent-bit/etc/rig:ro + - ${RIG_DATA}:/data + deploy: + resources: + limits: + memory: 300M diff --git a/elastic-hot-tier-budget/install-e2e/render.mjs b/elastic-hot-tier-budget/install-e2e/render.mjs new file mode 100644 index 0000000..fd8e62e --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/render.mjs @@ -0,0 +1,172 @@ +// Renders what the E100/E101 install proof applies, from a log10x-mcp build, through the three +// tools' own handlers, and adds nothing to what they return: +// MCP= node render.mjs +// 1. log10x_advise_retriever (destination elasticsearch): the plan's Elasticsearch frozen-tier +// block, verbatim to recipe.txt; its requests with the comment lines dropped to recipe-requests.json. +// 2. log10x_advise_install (app receiver, forwarder fluentbit, log_destination elasticsearch, +// builtin licence, airgapped, metrics backend elastic): the plan to install-plan.md; from its values file the sidecar's +// image, args, env and limits, and from its seed step the policy ConfigMap's data, to receiver.json. +// 3. log10x_setup_recurring (hot_tier_gb on github_actions, the checkout as the config plane): +// policy.yaml and apply-instructions.md. +import { writeFileSync, mkdirSync, readFileSync } from 'fs'; +import { join, resolve } from 'path'; +import { createRequire } from 'module'; + +const MCP = process.env.MCP; +if (!MCP) throw new Error('set MCP to a log10x-mcp package'); +const [outArg, gbArg, checkout, containersArg] = process.argv.slice(2); +if (!outArg || !gbArg || !checkout || !containersArg) throw new Error('usage: render.mjs '); +const out = resolve(outArg); +mkdirSync(out, { recursive: true }); +const YAML = createRequire(`${MCP}/package.json`)('yaml'); +const mcpVersion = JSON.parse(readFileSync(`${MCP}/package.json`, 'utf8')).version; + +const { putSnapshot } = await import(`${MCP}/build/lib/discovery/snapshot-store.js`); +const { SNAPSHOT_SCHEMA_VERSION } = await import(`${MCP}/build/lib/discovery/types.js`); +const { executeAdviseRetriever } = await import(`${MCP}/build/tools/advise-retriever.js`); +const { executeAdviseInstall } = await import(`${MCP}/build/tools/advise-install.js`); +const { executeSetupRecurring } = await import(`${MCP}/build/tools/setup-recurring.js`); + +// A discovery snapshot of a cluster with Fluent Bit in `logging`; the AWS fields give the +// Retriever plan a bucket and region so it renders its offload section. +const snapshotId = `e100-${Date.now()}`; +putSnapshot({ + schemaVersion: SNAPSHOT_SCHEMA_VERSION, + snapshotId, + startedAt: new Date().toISOString(), + finishedAt: new Date().toISOString(), + kubectl: { + available: true, + namespaces: ['logging'], + probedNamespaces: ['logging'], + forwarders: [{ kind: 'fluentbit', namespace: 'logging', workloadKind: 'DaemonSet', workloadName: 'fluent-bit', image: 'cr.fluentbit.io/fluent/fluent-bit:4.2', containerName: 'fluent-bit', labels: {}, readyReplicas: 1 }], + helmReleases: [], + log10xApps: [], + storageClasses: [], + ingressClasses: [], + serviceAccountIrsa: [], + backendAgents: [], + }, + aws: { available: true, region: 'us-east-1', s3Buckets: [], sqsQueues: [], cwLogGroups: [] }, + recommendations: { suggestedNamespace: 'logging', alreadyInstalled: {} }, + probeLog: [], +}); + +// 1. advise_retriever +const q = (n) => `https://sqs.us-east-1.amazonaws.com/000000000000/e100-${n}`; +const ret = await executeAdviseRetriever({ + snapshot_id: snapshotId, + infra_mode: 'existing', + index_source_bucket: 'e100-retriever-input', + iam_role_arn: 'arn:aws:iam::000000000000:role/e100-retriever', + index_queue_url: q('index'), + query_queue_url: q('query'), + subquery_queue_url: q('subquery'), + stream_queue_url: q('stream'), + license_source: 'builtin', + destination: 'elasticsearch', +}); +const retMd = String(ret.data?.markdown ?? JSON.stringify(ret.data)); +writeFileSync(join(out, 'retriever-plan.md'), retMd); +const at = retMd.indexOf('_Elasticsearch frozen tier_'); +if (at < 0) throw new Error('advise_retriever rendered no Elasticsearch frozen-tier block; see retriever-plan.md'); +const fence = retMd.indexOf('```text\n', at); +const recipe = retMd.slice(fence + '```text\n'.length, retMd.indexOf('\n```', fence + 8)); +writeFileSync(join(out, 'recipe.txt'), recipe + '\n'); +const requests = []; +const lines = recipe.split('\n').filter((l) => !l.startsWith('#')); +for (let i = 0; i < lines.length; i++) { + const m = /^(PUT|POST) (\S+)\s*$/.exec(lines[i]); + if (!m) continue; + const buf = []; + let depth = 0; + for (let j = i + 1; j < lines.length; j++) { + buf.push(lines[j]); + for (const ch of lines[j]) depth += ch === '{' ? 1 : ch === '}' ? -1 : 0; + if (depth === 0) { i = j; break; } + } + requests.push({ method: m[1], path: '/' + m[2], body: JSON.parse(buf.join('\n')) }); +} +writeFileSync(join(out, 'recipe-requests.json'), JSON.stringify(requests, null, 1) + '\n'); + +// 2. advise_install +const inst = await executeAdviseInstall({ + snapshot_id: snapshotId, + app: 'receiver', + forwarder: 'fluentbit', + backends: ['elastic'], + backend_credentials: { elastic: { secretName: 'elastic-credentials', plainValues: { ELASTICSEARCH_HOST: 'http://es-hot:9200' } } }, + airgapped: true, + license_source: 'builtin', + log_destination: 'elasticsearch', +}, {}); +const md = String(inst.data?.markdown ?? ''); +writeFileSync(join(out, 'install-plan.md'), md); +const vAt = md.indexOf('Write `my-receiver-values.yaml`'); +if (vAt < 0) throw new Error(`advise_install rendered no values file (mode ${inst.data?.mode}); see install-plan.md`); +const vStart = md.indexOf('```yaml\n', vAt) + '```yaml\n'.length; +const values = YAML.parse(md.slice(vStart, md.indexOf('\n```', vStart))); +const sidecar = values.extraContainers.find((c) => c.name === 'log10x'); +const seedAt = md.indexOf("cat <<'POLICY_EOF' | kubectl apply -f -\n"); +const seedText = md.slice(seedAt).split('\n').slice(1).join('\n').split('\nPOLICY_EOF')[0]; +const cm = YAML.parseAllDocuments(seedText).map((d) => d.toJS()).find((d) => d?.kind === 'ConfigMap'); +writeFileSync(join(out, 'receiver.json'), JSON.stringify({ mcp_version: mcpVersion, image: sidecar.image, args: sidecar.args, env: sidecar.env, resources: sidecar.resources, policy_configmap: { name: cm.metadata.name, data: cm.data } }, null, 1) + '\n'); + +// 3. setup_recurring +const rec = await executeSetupRecurring({ + target_services: [], + hot_tier_gb: gbArg === 'auto' ? 'auto' : Number(gbArg), + scheduler: 'github_actions', + config_plane: checkout, + nodes_by_container: Object.fromEntries(containersArg.split(',').filter(Boolean).map((c) => [c, 1])), + confirm: true, +}); +if (rec.data?.mode !== 'emit') throw new Error(`setup_recurring did not emit: ${rec.data?.markdown}`); +writeFileSync(join(out, 'policy.yaml'), String(rec.data.policy_yaml)); +writeFileSync(join(out, 'apply-instructions.md'), String(rec.data.apply_instructions) + '\n'); + +// The Receiver as a compose service: the image (ENGINE_IMAGE replaces the rendered one, both +// recorded), every arg in order, every env var with a plain value, the limits. Substituted, +// because there is no cluster: the values read the policy files through the Kubernetes pull +// lane (K8S_ENABLED, K8S_CONFIGMAP, K8S_NAMESPACE); here the seed files are mounted at /policy. +// CONFIG_GENERATION_FILE is left out: the seed carries no config-generation.csv. +// Rig plumbing: TENX_FLUENTBIT_OUTPUT_HOST=fluent-bit (Fluent Bit is a separate container). +const SUBSTITUTE = { K8S_ENABLED: 'false', CAP_LOOKUP_FILE: '/policy/caps.csv', ACTION_LOOKUP_FILE: '/policy/actions.csv', TOTAL_LOOKUP_FILE: '/policy/totals.csv' }; +const LEAVE_OUT = new Set(['K8S_NAMESPACE', 'K8S_CONFIGMAP', 'CONFIG_GENERATION_FILE']); +const applied = []; +const substituted = []; +const left = []; +for (const e of sidecar.env) { + if (LEAVE_OUT.has(e.name)) left.push(e.name); + else if (e.name in SUBSTITUTE) { substituted.push({ name: e.name, rendered: e.value, rig: SUBSTITUTE[e.name] }); applied.push([e.name, SUBSTITUTE[e.name]]); } + else if (e.value !== undefined) applied.push([e.name, String(e.value)]); + else left.push(e.name); +} +const image = process.env.ENGINE_IMAGE || sidecar.image; +const lim = sidecar.resources?.limits ?? {}; +const cpus = lim.cpu ? (String(lim.cpu).endsWith('m') ? String(Number(String(lim.cpu).slice(0, -1)) / 1000) : String(lim.cpu)) : undefined; +const mem = lim.memory ? String(lim.memory).replace(/Mi$/, 'M').replace(/Gi$/, 'G') : undefined; +const jq = (v) => JSON.stringify(String(v)).replace(/\$/g, '$$$$'); +writeFileSync(join(out, 'compose.receiver.yaml'), [ + '# Generated by install-e2e/render.mjs from log10x_advise_install\'s values (receiver.json). Do not edit.', + 'services:', + ' tenx:', + ` image: ${jq(image)}`, + ' container_name: e100-tenx', + ' environment:', + ...applied.map(([k, v]) => ` - ${jq(`${k}=${v}`)}`), + ` - ${jq('TENX_FLUENTBIT_OUTPUT_HOST=fluent-bit')}`, + ' tmpfs:', + ' - /tmp', + ' volumes:', + ' - ${RESULTS}/policy:/policy:ro', + ' command:', + ...sidecar.args.map((a) => ` - ${jq(a)}`), + ...(cpus || mem ? [' deploy:', ' resources:', ' limits:', ...(cpus ? [` cpus: ${jq(cpus)}`] : []), ...(mem ? [` memory: ${jq(mem)}`] : [])] : []), + '', +].join('\n')); +mkdirSync(join(out, 'policy'), { recursive: true }); +for (const f of ['caps.csv', 'actions.csv', 'totals.csv']) writeFileSync(join(out, 'policy', f), cm.data[f]); +writeFileSync(join(out, 'launch.json'), JSON.stringify({ mcp_version: mcpVersion, image, image_rendered: sidecar.image, args_from_values: sidecar.args, env_applied: applied.map(([k]) => k), env_substituted: substituted, env_left_out: left, rig_env: ['TENX_FLUENTBIT_OUTPUT_HOST=fluent-bit'], resources_limits: { cpus, memory: mem }, policy_seed_from_configmap: Object.keys(cm.data) }, null, 1) + '\n'); + +console.log(JSON.stringify({ mcp_version: mcpVersion, recipe_requests: requests.map((r) => `${r.method} ${r.path}`), image: sidecar.image, args: sidecar.args, env: sidecar.env.map((e) => e.name), seed_keys: Object.keys(cm.data) }, null, 1)); diff --git a/elastic-hot-tier-budget/install-e2e/results/apply-instructions.md b/elastic-hot-tier-budget/install-e2e/results/apply-instructions.md new file mode 100644 index 0000000..9ae48f9 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/apply-instructions.md @@ -0,0 +1,9 @@ +### Apply instructions: hot-tier run, GitHub Actions (gitops pull) + +1. Install (or re-install) the Receiver with `log10x_advise_install` and `log_destination: "elasticsearch"`: it counts bytes, the default, stamps `utf8Size` (the size the limits count) and `tenx_hash` on every record it returns, and records `bytes` in the policy files (`regulator-unit`). The run writes limits in bytes and refuses, writing nothing, a Receiver recorded as counting events. Pass `policy_git: { repo: "" }` so the Receiver pulls `log10x/policy/*.csv` from the `log10x-policy` branch (GH_ENABLED, GH_REPO, GH_BRANCH, GH_TOKEN). +2. Apply the Elasticsearch frozen-tier recipe (`log10x_advise_retriever` with `destination: "elasticsearch"`): kept lines go to the `tenx-app` alias and `routeState: tier_down` lines to `tenx-tierdown`, which ILM mounts on the frozen tier within hours. Its `tenx-keep` policy rolls kept lines over daily and mounts them on frozen 6 days later, 7 days on hot at most, and the run reads that from the ILM policy on `tenx-app`: its rollover `max_age` plus the `min_age` of the first phase after hot. A policy of the account's own on `tenx-app` needs a phase after hot at the age a kept line leaves hot (`warm`, `cold`, `frozen` or `delete`), or `hot_retention_hours` in policy.yaml. +3. **Commit `policy.yaml`** to the root of the repo, with `nodes_by_container` set for every service. +4. **Commit `log10x-recurring.yml`** to `.github/workflows/`. +5. **Add the secrets** `LOG10X_ES_URL` and `LOG10X_ES_API_KEY` (an encoded API key). The run reads the cluster (node stats, ILM and the Receiver's indices) and writes nothing to it. +6. Each run commits `log10x/policy/{caps,actions,totals,config-generation}.csv` and `reports/latest.md` directly to the `log10x-policy` branch (schedule `0 * * * *` UTC). +7. Hourly is about 720 runs a month on the repo's Actions minutes; for fewer, pass `schedule: "every-6h"` or a cron expression such as `"0 */3 * * *"`. diff --git a/elastic-hot-tier-budget/install-e2e/results/compose.receiver.yaml b/elastic-hot-tier-budget/install-e2e/results/compose.receiver.yaml new file mode 100644 index 0000000..1739b11 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/compose.receiver.yaml @@ -0,0 +1,28 @@ +# Generated by install-e2e/render.mjs from log10x_advise_install's values (receiver.json). Do not edit. +services: + tenx: + image: "ghcr.io/log-10x/edge-10x:1.1.150" + container_name: e100-tenx + environment: + - "TENX_AIRGAPPED=true" + - "K8S_ENABLED=false" + - "CAP_LOOKUP_FILE=/policy/caps.csv" + - "ACTION_LOOKUP_FILE=/policy/actions.csv" + - "TOTAL_LOOKUP_FILE=/policy/totals.csv" + - "TENX_FLUENTBIT_OUTPUT_HOST=fluent-bit" + tmpfs: + - /tmp + volumes: + - ${RESULTS}/policy:/policy:ro + command: + - "@run/input/forwarder/fluentbit" + - "@apps/receiver" + - "symbolMessageHashField" + - "tenx_hash" + - "outputEncodeField" + - "fullText(\"tenx_hash\",\"routeState\",\"message_pattern\",\"severity_level\",\"utf8Size\")" + deploy: + resources: + limits: + cpus: "0.5" + memory: "512M" diff --git a/elastic-hot-tier-budget/install-e2e/results/e100-fluent-bit.log b/elastic-hot-tier-budget/install-e2e/results/e100-fluent-bit.log new file mode 100644 index 0000000..02b81d2 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/e100-fluent-bit.log @@ -0,0 +1,54 @@ +Fluent Bit v4.2.8 +* Copyright (C) 2015-2026 The Fluent Bit Authors +* Fluent Bit is a CNCF graduated project under the Fluent organization +* https://fluentbit.io + +______ _ _ ______ _ _ ___ _____ +| ___| | | | | ___ (_) | / | / __ \ +| |_ | |_ _ ___ _ __ | |_ | |_/ /_| |_ __ __/ /| | `' / /' +| _| | | | | |/ _ \ '_ \| __| | ___ \ | __| \ \ / / /_| | / / +| | | | |_| | __/ | | | |_ | |_/ / | |_ \ V /\___ |_./ /___ +\_| |_|\__,_|\___|_| |_|\__| \____/|_|\__| \_/ |_(_)_____/ + + Fluent Bit v4.2 – Direct Routes Ahead + Celebrating 10 Years of Open, Fluent Innovation! + +[2026/10/25 23:47:10.188821716] [ info] [fluent bit] version=4.2.8, commit=419540c428, pid=1 +[2026/10/25 23:47:10.189095330] [ info] [storage] ver=1.5.4, type=memory, sync=normal, checksum=off, max_chunks_up=128 +[2026/10/25 23:47:10.189105784] [ info] [simd ] SSE2 +[2026/10/25 23:47:10.189110220] [ info] [cmetrics] version=1.0.6 +[2026/10/25 23:47:10.189114485] [ info] [ctraces ] version=0.6.6 +[2026/10/25 23:47:10.189191711] [ info] [input:forward:forward.0] initializing +[2026/10/25 23:47:10.189200045] [ info] [input:forward:forward.0] storage_strategy='memory' (memory only) +[2026/10/25 23:47:10.189329271] [ info] [input:forward:forward.0] listening on 0.0.0.0:24224 +[2026/10/25 23:47:10.189685830] [ info] [input:forward:forward.1] initializing +[2026/10/25 23:47:10.189694791] [ info] [input:forward:forward.1] storage_strategy='memory' (memory only) +[2026/10/25 23:47:10.189731812] [ info] [input:forward:forward.1] listening on 0.0.0.0:24225 +[2026/10/25 23:47:10.209294707] [ info] [input:emitter:emitter_for_rewrite_tag.2] initializing +[2026/10/25 23:47:10.209340989] [ info] [input:emitter:emitter_for_rewrite_tag.2] storage_strategy='memory' (memory only) +[2026/10/25 23:47:10.237837035] [ info] [output:file:file.0] worker #0 started +[2026/10/25 23:47:10.238042512] [ info] [output:forward:forward.1] worker #0 started +[2026/10/25 23:47:10.246788622] [ info] [output:file:file.2] worker #0 started +[2026/10/25 23:47:10.247577068] [ info] [http_server] listen iface=0.0.0.0 tcp_port=2020 +[2026/10/25 23:47:10.247594857] [ info] [sp] stream processor started +[2026/10/25 23:47:10.247660017] [ info] [engine] Shutdown Grace Period=10, Shutdown Input Grace Period=5 +[2026/10/25 23:47:10.257011267] [ info] [output:file:file.3] worker #0 started +[2026/10/25 23:47:10.265855564] [ info] [output:file:file.5] worker #0 started +[2026/10/25 23:47:10.266474245] [ info] [output:file:file.4] worker #0 started +[2026/10/25 23:47:10.266936627] [ info] [output:es:es.6] worker #0 started +[2026/10/25 23:47:10.275794059] [ info] [output:es:es.6] worker #1 started +[2026/10/25 23:47:10.281459214] [ info] [output:es:es.7] worker #0 started +[2026/10/25 23:47:10.287652752] [ info] [output:es:es.7] worker #1 started +[2026/10/25 23:50:30.861083466] [error] [output:forward:forward.1] no upstream connections available +[2026/10/25 23:50:30.864087798] [ warn] [engine] failed to flush chunk '1-1792972229.856755232.flb', retry in 11 seconds: task_id=0, input=forward.0 > output=forward.1 (out_id=1) +[2026/10/25 23:50:32.855732054] [error] [output:forward:forward.1] no upstream connections available +[2026/10/25 23:50:32.856255716] [ warn] [engine] failed to flush chunk '1-1792972231.856495373.flb', retry in 6 seconds: task_id=2, input=forward.0 > output=forward.1 (out_id=1) +[2026/10/25 23:50:32.879883567] [error] [net] TCP connection failed: tenx:24224 (Connection refused) +[2026/10/25 23:50:32.879980966] [error] [output:forward:forward.1] no upstream connections available +[2026/10/25 23:50:32.880271193] [ warn] [engine] failed to flush chunk '1-1792972230.856470537.flb', retry in 11 seconds: task_id=1, input=forward.0 > output=forward.1 (out_id=1) +[2026/10/25 23:50:33.857053996] [error] [output:forward:forward.1] no upstream connections available +[2026/10/25 23:50:33.857219193] [ warn] [engine] failed to flush chunk '1-1792972232.856170193.flb', retry in 6 seconds: task_id=3, input=forward.0 > output=forward.1 (out_id=1) +[2026/10/25 23:50:38.868537750] [ info] [engine] flush chunk '1-1792972231.856495373.flb' succeeded at retry 1: task_id=2, input=forward.0 > output=forward.1 (out_id=1) +[2026/10/25 23:50:39.857913747] [ info] [engine] flush chunk '1-1792972232.856170193.flb' succeeded at retry 1: task_id=3, input=forward.0 > output=forward.1 (out_id=1) +[2026/10/25 23:50:41.855386356] [ info] [engine] flush chunk '1-1792972229.856755232.flb' succeeded at retry 1: task_id=0, input=forward.0 > output=forward.1 (out_id=1) +[2026/10/25 23:50:43.855471680] [ info] [engine] flush chunk '1-1792972230.856470537.flb' succeeded at retry 1: task_id=1, input=forward.0 > output=forward.1 (out_id=1) diff --git a/elastic-hot-tier-budget/install-e2e/results/e100-tenx.log b/elastic-hot-tier-budget/install-e2e/results/e100-tenx.log new file mode 100644 index 0000000..f5a5117 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/e100-tenx.log @@ -0,0 +1,63 @@ +[INFO ] 2026-10-25 23:47:11.106 [main] PipelineLauncher - launching 10x engine v1.1.150, flavor: 'runtime', pipeline: 'run' +[INFO ] 2026-10-25 23:47:11.106 [main] PipelineLauncher - log4j2 config file loaded from: /etc/tenx/config/log4j2.yaml +[WARN ] 2026-10-25 23:47:11.138 [main] PipelineLauncher - No license configured; running on the built-in free tier: up to 10 nodes, airgapped. +[WARN ] 2026-10-25 23:47:11.138 [main] PipelineLauncher - Above 10 nodes, a paid license token is required: set licenseFile / licenseKey, the TENX_LICENSE_FILE / TENX_LICENSE_KEY env var, or a licenseKey/licenseFile entry in your bootstrap .yaml. See https://doc.log10x.com/manage/license/ +[INFO ] 2026-10-25 23:47:11.242 [main] PipelineLauncher - pipelineConfigLoader took: 104ms +[INFO ] 2026-10-25 23:47:11.243 [main] ExecutionPipeline - executing /opt/tenx-edge/lib/app/modules/pipelines/run/pipeline.yaml (myReceiver) +[INFO ] 2026-10-25 23:47:11.243 [main] ExecutionPipeline - pipeline units: configLoader => symbolLoader(shared/symbols, tenx/symbols) => templateLoader => readStream(stream:fluentbit) => parallelize(0.5 / 1) => transform => group => aggregateSingle(emitted_events: 10000/2s) => aggregateSingle(all_events: 10000/2s) => logOutput(run/modules/output/event/forward) => printProgress => monitor => tokenizeStats => publishEnvMetrics +πŸͺ΅ Initializing 10x engine. Logging to SYS_OUT +πŸš€ Launching 10x Engine: Receiver app +πŸ“š Loading symbol libraries from: /etc/tenx/config/data/shared/symbols,/etc/tenx/symbols +πŸ“‚ Loaded symbol files: /etc/tenx/symbols/tenx-symbols-1.1.150.10x.tar +πŸ“‚ Loading TenXTemplates from: data/templates/*.json,data/sample/output/*.json +πŸ“₯ Reading events from Fluent Bit via Forward protocol on tcp://0.0.0.0:24224 +πŸ’° Loading rate cap file: /policy/caps.csv +🚦 Applying rate regulator to: fluentbit +[INFO ] 2026-10-25 23:47:13.313 [main] ForwardProtocolInputStream - initializing Forward protocol input stream on TCP port: 24224 +[INFO ] 2026-10-25 23:47:13.314 [main] ForwardProtocolInputStream - Forward protocol server listening on TCP port: 24224 +βš™οΈ Transforming events into TenXObjects using 50% of available cores +ℹ️ Enriching TenXObjects with severity field: 'severity_level' +πŸ”² Consolidating multi-line events into logical TenXObject groups +πŸ’¬ Enriching TenXObjects with message field: 'message_pattern' +🌐 Enriching TenXObjects with HTTP code field: 'http_code' +πŸ“‹ Enriching TenXObjects with lookup field: 'http_message' from file: /etc/tenx/config/pipelines/run/initialize/lookup/http.csv +☸️ Enriching TenXObjects with k8s container context using fluentK8s +[INFO ] 2026-10-25 23:47:13.529 [main] EventLogPublisherConfigBuilder - Added new loggers to log4j: [tenxForward__fluent-bit_24225] +πŸ“ Writing TenXObject field: 'fullText("tenx_hash","routeState","message_pattern","severity_level","utf8Size")' β†’ Fluentd: fluent-bit:24225 +[INFO ] 2026-10-25 23:47:13.607 [main] ResourceReloadUnit - resource reload: starting with 4 watched path(s); pollInterval=10s +[INFO ] 2026-10-25 23:47:31.864 [AsyncWorkerTaskInput: readStream 23:47:13] ForwardProtocolInputStream - client connected to Forward protocol input: tcp://0.0.0.0:24224 +[INFO ] 2026-10-25 23:50:30.309 [tenx-drain-hook] PipelineShutdownDrain - draining 1 executing pipeline(s), budget: 10000ms +[INFO ] 2026-10-25 23:50:30.310 [tenx-drain-hook] ForwardProtocolInputStream - closing Forward protocol input stream: null +[INFO ] 2026-10-25 23:50:30.312 [main] ExecutionPipeline - shutting down +[INFO ] 2026-10-25 23:50:30.514 [main] ExecutionPipeline - shutdown scripts phase took 0ms +[INFO ] 2026-10-25 23:50:30.519 [main] ExecutionPipeline - shutdown phase took 5ms +[INFO ] 2026-10-25 23:50:30.537 [main] ExecutionPipeline - terminate phase took 0ms +[INFO ] 2026-10-25 23:50:30.538 [main] ExecutionPipeline - execution of: /opt/tenx-edge/lib/app/modules/pipelines/run/pipeline.yaml (myReceiver) completed in: 199sec +[INFO ] 2026-10-25 23:50:32.489 [main] PipelineLauncher - launching 10x engine v1.1.150, flavor: 'runtime', pipeline: 'run' +[INFO ] 2026-10-25 23:50:32.489 [main] PipelineLauncher - log4j2 config file loaded from: /etc/tenx/config/log4j2.yaml +[WARN ] 2026-10-25 23:50:32.571 [main] PipelineLauncher - No license configured; running on the built-in free tier: up to 10 nodes, airgapped. +[WARN ] 2026-10-25 23:50:32.571 [main] PipelineLauncher - Above 10 nodes, a paid license token is required: set licenseFile / licenseKey, the TENX_LICENSE_FILE / TENX_LICENSE_KEY env var, or a licenseKey/licenseFile entry in your bootstrap .yaml. See https://doc.log10x.com/manage/license/ +[INFO ] 2026-10-25 23:50:32.668 [main] PipelineLauncher - pipelineConfigLoader took: 97ms +[INFO ] 2026-10-25 23:50:32.668 [main] ExecutionPipeline - executing /opt/tenx-edge/lib/app/modules/pipelines/run/pipeline.yaml (myReceiver) +[INFO ] 2026-10-25 23:50:32.669 [main] ExecutionPipeline - pipeline units: configLoader => symbolLoader(shared/symbols, tenx/symbols) => templateLoader => readStream(stream:fluentbit) => parallelize(0.5 / 1) => transform => group => aggregateSingle(emitted_events: 10000/2s) => aggregateSingle(all_events: 10000/2s) => logOutput(run/modules/output/event/forward) => printProgress => monitor => tokenizeStats => publishEnvMetrics +πŸͺ΅ Initializing 10x engine. Logging to SYS_OUT +πŸš€ Launching 10x Engine: Receiver app +πŸ“š Loading symbol libraries from: /etc/tenx/config/data/shared/symbols,/etc/tenx/symbols +πŸ“‚ Loaded symbol files: /etc/tenx/symbols/tenx-symbols-1.1.150.10x.tar +πŸ“‚ Loading TenXTemplates from: data/templates/*.json,data/sample/output/*.json +πŸ“₯ Reading events from Fluent Bit via Forward protocol on tcp://0.0.0.0:24224 +πŸ’° Loading rate cap file: /policy/caps.csv +🚦 Applying rate regulator to: fluentbit +[INFO ] 2026-10-25 23:50:34.273 [main] ForwardProtocolInputStream - initializing Forward protocol input stream on TCP port: 24224 +[INFO ] 2026-10-25 23:50:34.273 [main] ForwardProtocolInputStream - Forward protocol server listening on TCP port: 24224 +βš™οΈ Transforming events into TenXObjects using 50% of available cores +ℹ️ Enriching TenXObjects with severity field: 'severity_level' +πŸ”² Consolidating multi-line events into logical TenXObject groups +πŸ’¬ Enriching TenXObjects with message field: 'message_pattern' +🌐 Enriching TenXObjects with HTTP code field: 'http_code' +πŸ“‹ Enriching TenXObjects with lookup field: 'http_message' from file: /etc/tenx/config/pipelines/run/initialize/lookup/http.csv +☸️ Enriching TenXObjects with k8s container context using fluentK8s +[INFO ] 2026-10-25 23:50:34.460 [main] EventLogPublisherConfigBuilder - Added new loggers to log4j: [tenxForward__fluent-bit_24225] +πŸ“ Writing TenXObject field: 'fullText("tenx_hash","routeState","message_pattern","severity_level","utf8Size")' β†’ Fluentd: fluent-bit:24225 +[INFO ] 2026-10-25 23:50:34.489 [main] ResourceReloadUnit - resource reload: starting with 4 watched path(s); pollInterval=10s +[INFO ] 2026-10-25 23:50:34.857 [AsyncWorkerTaskInput: readStream 23:50:34] ForwardProtocolInputStream - client connected to Forward protocol input: tcp://0.0.0.0:24224 diff --git a/elastic-hot-tier-budget/install-e2e/results/es-setup.json b/elastic-hot-tier-budget/install-e2e/results/es-setup.json new file mode 100644 index 0000000..1c0931a --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/es-setup.json @@ -0,0 +1,82 @@ +{ + "es": "http://127.0.0.1:9200", + "prep": [ + { + "method": "POST", + "path": "/_license/start_trial?acknowledge=true", + "status": 200, + "body": "{\"acknowledged\":true,\"trial_was_started\":true,\"type\":\"trial\"}" + }, + { + "method": "PUT", + "path": "/_snapshot/found-snapshots", + "status": 200, + "body": "{\"acknowledged\":true}" + } + ], + "applied": [ + { + "method": "PUT", + "path": "/_ilm/policy/tenx-tier-down", + "status": 200, + "body": "{\"acknowledged\":true}" + }, + { + "method": "PUT", + "path": "/_ilm/policy/tenx-keep", + "status": 200, + "body": "{\"acknowledged\":true}" + }, + { + "method": "PUT", + "path": "/_index_template/tenx-tierdown-tpl", + "status": 200, + "body": "{\"acknowledged\":true}" + }, + { + "method": "PUT", + "path": "/_index_template/tenx-app-tpl", + "status": 200, + "body": "{\"acknowledged\":true}" + }, + { + "method": "PUT", + "path": "/tenx-tierdown-000001", + "status": 200, + "body": "{\"acknowledged\":true,\"shards_acknowledged\":true,\"index\":\"tenx-tierdown-000001\"}" + }, + { + "method": "PUT", + "path": "/tenx-app-000001", + "status": 200, + "body": "{\"acknowledged\":true,\"shards_acknowledged\":true,\"index\":\"tenx-app-000001\"}" + } + ], + "placement": [ + { + "index": "tenx-tierdown-000001", + "shard": "0", + "prirep": "p", + "state": "STARTED", + "node": "es-warm-frozen", + "roles": [ + "data_frozen", + "data_warm" + ] + }, + { + "index": "tenx-app-000001", + "shard": "0", + "prirep": "p", + "state": "STARTED", + "node": "es-hot", + "roles": [ + "data_content", + "data_hot", + "ingest", + "master" + ] + } + ], + "tier_down_write_index_on_warm": true +} diff --git a/elastic-hot-tier-budget/install-e2e/results/final-state.txt b/elastic-hot-tier-budget/install-e2e/results/final-state.txt new file mode 100644 index 0000000..6f836e4 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/final-state.txt @@ -0,0 +1,49 @@ +alias index is_write_index +tenx-tierdown tenx-tierdown-000001 true +tenx-app tenx-app-000001 true +index docs.count store.size +tenx-app-000001 7472 3.6mb +tenx-tierdown-000001 2387 1mb +index prirep state node +tenx-app-000001 p STARTED es-hot +tenx-tierdown-000001 p STARTED es-warm-frozen +{ + "indices" : { + "tenx-app-000001" : { + "policy" : "tenx-keep", + "phase" : "hot", + "action" : "rollover" + }, + "tenx-tierdown-000001" : { + "policy" : "tenx-tier-down", + "phase" : "hot", + "action" : "rollover" + } + } +} +{ + "tenx-keep" : { + "policy" : { + "phases" : { + "frozen" : { + "min_age" : "6d", + "actions" : { + "searchable_snapshot" : { + "snapshot_repository" : "found-snapshots", + "force_merge_index" : true + } + } + }, + "hot" : { + "min_age" : "0ms", + "actions" : { + "rollover" : { + "max_age" : "1d", + "max_primary_shard_size" : "50gb" + } + } + } + } + } + } +} diff --git a/elastic-hot-tier-budget/install-e2e/results/install-plan.md b/elastic-hot-tier-budget/install-e2e/results/install-plan.md new file mode 100644 index 0000000..ed372a0 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/install-plan.md @@ -0,0 +1,358 @@ +# Install plan: Receiver sidecar + +- **App**: Receiver (sidecar) +- **Forwarder**: `fluentbit` +- **Metrics backends**: `elastic` +- **Airgapped**: yes (engine emits only to user-owned backends) +- **License**: none in the plan. The engine runs its built-in Free tier: up to 10 nodes, no expiry, airgapped. Nothing is minted or stored and no licence Secret is created. To license it later, re-run with `license_source: "signin"` or `license_source: "paste"`; the plan then adds the step that creates the licence Secret. + +# Receiver advisor: install + verify + teardown + +## Target +- **app**: receiver +- **forwarder**: fluentbit +- **release**: `my-receiver` +- **namespace**: `logging` +- **snapshot**: `e100-1792971977960` + +## Notes +- Destination is `mock` (events written to forwarder stdout). Ideal for dogfooding or smoke tests; switch to `elasticsearch`, `splunk`, `datadog`, or `cloudwatch` for production. +- The Receiver counts bytes and stamps `tenx_hash` and `utf8Size` (the size its byte limits count) on every record it returns, which the hot-tier run (log10x_setup_recurring `hot_tier_gb`) reads from its indices. Send kept lines to the `tenx-app` alias and `routeState: tier_down` lines to `tenx-tierdown`, with the Elasticsearch frozen-tier recipe from log10x_advise_retriever (`destination: "elasticsearch"`). + +## Preflight +| Check | Status | Detail | +|---|---|---| +| kubectl | `ok` | context `undefined` | +| namespace | `ok` | `logging` exists | +| forwarder alignment | **warn** | no existing forwarder detected; the plan installs one from scratch | +| release collision | `ok` | no `my-receiver` release in `logging`: clear to install | +| license | `ok` | no licence configured: the engine runs its built-in Free tier (up to 10 nodes, no expiry, airgapped) and logs that it does at startup. Nothing to mint, store or mount. To license it later, re-run the wizard with `license_source: "signin"` or `"paste"`; the plan then adds a step that creates the licence Secret. | +| destination-expander | `?` | Compaction is only lossless end to end when the destination can expand the encoded events again at read time, and that expander is a separate install on the destination, not part of this release: the [10x Splunk app](https://doc.log10x.com/apps/receiver/compact/splunk/), or the [l1es plugin](https://doc.log10x.com/apps/receiver/compact/elasticsearch/) for self-managed Elasticsearch 8.17.0 / OpenSearch 2.19.0. Install it before you compact anything the destination is searched on, or those searches return compacted lines. Managed and serverless platforms (Datadog, CloudWatch, Elastic Cloud Serverless) have nowhere to install one. Compaction is not offered on ClickHouse, where it measured about 7% of table bytes and the bill is compute. Use tier_down or offload on all of these instead of compaction. | + +## Install +### 1. Add Fluent Bit Helm repo +_Makes the fluent/fluent-bit chart available to `helm install`._ + +```bash +helm repo add fluent https://fluent.github.io/helm-charts +helm repo update +helm search repo fluent/fluent-bit +``` + +### 2. Create target namespace +_The Reporter installs into `logging`. `--create-namespace` on helm install will also do this, but a separate step makes retries idempotent._ + +```bash +kubectl create namespace logging --dry-run=client -o yaml | kubectl apply -f - +``` + +### 3. Seed the engine-policy ConfigMap +_The receiver reads its byte caps (caps.csv) and per-service actions (actions.csv) from the `log10x-action-intent` ConfigMap via the engine's kubernetes pull lane. This seeds it with a no-op policy so the pod boots policy-ready (a missing ConfigMap aborts the engine launch by design), and grants every ServiceAccount in `logging` read access to this one ConfigMap. `log10x_configure_engine` writes the real policy here later._ + +```bash +cat <<'POLICY_EOF' | kubectl apply -f - +apiVersion: v1 +kind: ConfigMap +metadata: + name: log10x-action-intent + namespace: logging + labels: + app.kubernetes.io/managed-by: log10x-mcp + app.kubernetes.io/component: engine-policy +data: + caps.csv: | + container,cap + tenx-seed,1::install seed; replaced by log10x_configure_engine + actions.csv: | + container,action + tenx-seed,pass::install seed; replaced by log10x_configure_engine + totals.csv: | + container,total + tenx-seed,1::install seed; replaced by the budget run + regulator-unit: bytes +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: log10x-policy-reader + namespace: logging +rules: + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["log10x-action-intent"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: log10x-policy-reader + namespace: logging +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: log10x-policy-reader +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts:logging +POLICY_EOF +``` + +### 4. Write Helm values +_Tenx config + Fluent Bit-specific output destination (`mock`)._ + +Write `my-receiver-values.yaml`: + +```yaml +# Receiver overlay for Fluent Bit (upstream fluent/fluent-bit chart). +# Layer on top of your existing values: +# helm upgrade --install fluent/fluent-bit \ +# -f your-existing-fluent-bit-values.yaml \ +# -f my-receiver.yaml --namespace + +extraContainers: + - name: log10x + image: log10x/edge-10x:1.1.147@sha256:a043ccd828742c4eccee09abb654ce6a7357b6123f73300ed04f8605e874b8c4 + imagePullPolicy: IfNotPresent + securityContext: + runAsNonRoot: true + runAsUser: 1000 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + args: + - "@run/input/forwarder/fluentbit" + - "@apps/receiver" + - "symbolMessageHashField" + - "tenx_hash" + - "outputEncodeField" + - "fullText(\"tenx_hash\",\"routeState\",\"message_pattern\",\"severity_level\",\"utf8Size\")" + env: + - name: TENX_AIRGAPPED + value: "true" + - name: K8S_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: K8S_ENABLED + value: "true" + - name: K8S_CONFIGMAP + value: log10x-action-intent + - name: CAP_LOOKUP_FILE + value: /tmp/tenx/kubernetes/$(K8S_NAMESPACE)/log10x-action-intent/caps.csv + - name: ACTION_LOOKUP_FILE + value: /tmp/tenx/kubernetes/$(K8S_NAMESPACE)/log10x-action-intent/actions.csv + - name: CONFIG_GENERATION_FILE + value: /tmp/tenx/kubernetes/$(K8S_NAMESPACE)/log10x-action-intent/config-generation.csv + - name: TOTAL_LOOKUP_FILE + value: /tmp/tenx/kubernetes/$(K8S_NAMESPACE)/log10x-action-intent/totals.csv + volumeMounts: + - name: tenx-tmp + mountPath: /tmp + resources: + requests: { cpu: 100m, memory: 256Mi } + limits: { cpu: 500m, memory: 512Mi } + +extraVolumes: + - name: tenx-tmp + emptyDir: {} + +# Replace the chart's default config with the sidecar bypass pattern. +# Inputs: tail your sources + receive processed events back from the +# sidecar on :24225 with Tag_Prefix tenx. (so filters and the ingest +# handoff Match kube.* and skip the returning tenx.*). +# Filters: enrichment (kubernetes metadata) on kube.* only. +# Outputs: handoff to the sidecar on 127.0.0.1:24224 + destination +# for the returning tenx.* events. +config: + service: | + [SERVICE] + Daemon Off + Flush {{ .Values.flush }} + Log_Level {{ .Values.logLevel }} + Parsers_File /fluent-bit/etc/parsers.conf + Parsers_File /fluent-bit/etc/conf/custom_parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port {{ .Values.metricsPort }} + Health_Check On + + inputs: | + [INPUT] + Name tail + Path /var/log/containers/*.log + Exclude_Path /var/log/containers/*fluent-bit*.log,/var/log/containers/*fluentd*.log,/var/log/containers/*filebeat*.log,/var/log/containers/*logstash*.log,/var/log/containers/*otel-collector*.log,/var/log/containers/*opentelemetry-collector*.log + multiline.parser docker, cri + Tag kube.* + Mem_Buf_Limit 5MB + Skip_Long_Lines On + + # Egress: returning events from the 10x sidecar. Tag_Prefix tenx. + # prevents enrichment filters and the sidecar-handoff output from + # re-firing (they Match kube.* only). + [INPUT] + Name forward + Listen 0.0.0.0 + Port 24225 + Tag_Prefix tenx. + + filters: | + [FILTER] + Name kubernetes + Match kube.* + Merge_Log On + Keep_Log Off + K8S-Logging.Parser On + K8S-Logging.Exclude On + + outputs: | + # Hand off to the Log10x sidecar. + [OUTPUT] + Name forward + Match kube.* + Host 127.0.0.1 + Port 24224 + Retry_Limit False + + # Destination for processed tenx.* events. Replace with your real + # destination (es, splunk, kafka, s3, ...): this stdout block is + # safe for dogfooding. + [OUTPUT] + Name stdout + Match tenx.* + Format json_lines +``` + +### 5. Install via Helm +_Deploys the Fluent Bit chart with the 10x Receiver sidecar enabled._ + +```bash +helm upgrade --install my-receiver fluent/fluent-bit \ + -n logging --create-namespace \ + -f my-receiver-values.yaml +``` + +### 6. Wait for rollout +_Blocks until the workload reports Ready. Uses the chart's own label selector (legacy Helm for elastic charts, k8s-recommended elsewhere): a mismatched selector silently returns "No resources found" and exits 0._ + +```bash +kubectl -n logging rollout status daemonset -l app.kubernetes.io/instance=my-receiver --timeout=5m || true +kubectl -n logging rollout status deployment -l app.kubernetes.io/instance=my-receiver --timeout=5m || true +kubectl -n logging rollout status statefulset -l app.kubernetes.io/instance=my-receiver --timeout=10m || true +``` + +## GitOps: MCP-managed runtime config (optional) + +Compaction is a per-source switch: the chart `optimize` flag makes the sidecar emit `encode()` for every event it sees, smaller on a destination whose 10x expander is installed (see the prerequisite below; run log10x_measure_compaction for the ratio on the real stream). The compactReceiver cap file is the exception list under that switch: with compactReceiverDefault true, a CSV keyed by the pattern name lists the patterns to keep as full text. Wire GitOps once and edit the list as PRs; the engine hot-reloads the CSV without a pod restart. + +**Enable this if:** +- You compact the source and want to keep specific patterns in full text (audit, compliance, debug). +- You want decisions to evolve over time without redeploying the receiver. +- You want the MCP to manage the cap-file via PRs (review-able, reversible). + +**Skip this if:** +- You will turn on the chart `optimize` feature flag later to compact every event of this source. Install the destination expander first (see the prerequisite below). +- You will not be using the receiver app at all (this section is receiver-only). + +### Repo layout + +Mirror this in your GitOps config repo (the engine pulls it on each poll): + +``` +pipelines/run/receive/compact/compact-cap.csv # the exception list: one row per pattern, keyed by the pattern name (message_pattern, the first column of log10x_top_patterns), boolean true|false value; in-place writes hot-reload (no restart) +pipelines/run/receive/compact/compact-object-cap.js # predicate logic: JS change β†’ pipeline restart (rarely needed; CSV covers most cases) +``` + +### Pod env vars + +Set these on the receiver pod (helm `--set env.=` or the chart's env block): + +| Env | Required | Default | Note | +|---|---|---|---| +| `GH_FILES_ENABLED` | **yes** | `true` | switch for the GitHub puller of the files `paths` lists (on this engine GH_ENABLED pulls the regulator policy folder `log10x/policy/` instead, which this repo does not carry) | +| `GH_REPO` | **yes** | `your-org/your-config-repo` | owner/name of the GitOps config repo (forked from log-10x/config recommended) | +| `GH_TOKEN` | **yes** | `` | PAT with Contents: Read scope; store as a k8s Secret + reference via valueFrom | +| `GH_BRANCH` | optional | `main` | branch to pull from | +| `GH_SYNC_INTERVAL` | optional | `30s` | engine re-fetches the repo this often | +| `compactReceiverLookupFile` | **yes** | `pipelines/run/receive/compact/compact-cap.csv` | must match the path inside your GitOps repo; one row per pattern, keyed by the pattern name (message_pattern), boolean true\|false value | +| `compactReceiverDefault` | optional | `true` | `true`: a listed pattern with value false keeps its full text, every other pattern compacts (the exception list under the chart `optimize` flag). `false`: only listed patterns with value true compact. | + +### Once wired, author entries via the MCP + +Run `log10x_configure_engine` to compose the per-pattern compact decisions and emit a literal `gh` PR command. Example: + +``` +log10x_configure_engine \ + gitops_repo=your-org/your-config-repo \ + service=payment-service \ + target_percent=30 \ + destination=datadog +``` + +### Caveats +- Compaction is only lossless end to end when the destination can expand the encoded events again at read time, and that expander is a separate install on the destination, not part of this release: the [10x Splunk app](https://doc.log10x.com/apps/receiver/compact/splunk/), or the [l1es plugin](https://doc.log10x.com/apps/receiver/compact/elasticsearch/) for self-managed Elasticsearch 8.17.0 / OpenSearch 2.19.0. Install it before you compact anything the destination is searched on, or those searches return compacted lines. Managed and serverless platforms (Datadog, CloudWatch, Elastic Cloud Serverless) have nowhere to install one. Compaction is not offered on ClickHouse, where it measured about 7% of table bytes and the bill is compute. Use tier_down or offload on all of these instead of compaction. +- The `paths` glob in `pipelines/gitops/config.yaml` is `test/*.csv`, and it is a YAML list, which does not read environment variables, so `GH_PATH` does not change it. Fork the config repo and edit `paths` to cover `pipelines/run/receive/compact/*`. +- Customers running multiple receiver pods all watching the same GitOps repo will see fan-out: a single PR triggers reload on every pod within a poll window. That is the intended behavior: kept here as a heads-up for capacity planning. +- Hot-reload requires in-place writes (the gitops pattern). Do not source the cap-file via a Kubernetes ConfigMap mount: CM swaps the file via a symlink rename, which the engine's stat-based watcher will not see. + +## Verify +Each probe below answers one specific question: run them in order. A probe passes when its commands exit 0 and (optionally) match the `expectOutput` regex. + +### 1. pods-ready: Are all Fluent Bit pods Ready (with the log10x sidecar)? +_Expect to pass within 300s._ + +```bash +kubectl -n logging wait --for=condition=Ready pod -l app.kubernetes.io/instance=my-receiver --timeout=5m +``` + +Expect output matching: `condition met` + +### 2. sidecar-alive: Is the log10x sidecar running and processing events? + +```bash +kubectl -n logging logs -l app.kubernetes.io/instance=my-receiver -c log10x --tail=400 | grep -E 'tenx|pattern|receiver|template' | head -20 +``` + +### 3. forwarder-handoff: Is Fluent Bit forwarding kube.* events to the sidecar on 127.0.0.1:24224? + +```bash +kubectl -n logging logs -l app.kubernetes.io/instance=my-receiver -c fluent-bit --tail=400 | grep -E '\\[OUTPUT\\].*forward|127\\.0\\.0\\.1:24224' | head -10 +``` + +### 4. tenx-mock-events: Are returning tenx.* events reaching the stdout destination? +_Expect to pass within 120s._ + +```bash +kubectl -n logging logs -l app.kubernetes.io/instance=my-receiver -c fluent-bit --tail=200 | grep -E '"tag":"tenx\\.' | head -5 +``` + +Expect output matching: `tenx.` + +## Teardown +### 1. Uninstall the Helm release +_Removes the forwarder DaemonSet/Deployment/StatefulSet, its ServiceAccount, ConfigMaps, and the 10x Reporter sidecar._ + +```bash +helm -n logging uninstall my-receiver +``` + +### 2. Clean up derived resources +_Helm does not reap PVCs or Secrets that were created outside of the release. Selector honors the chart family's label convention (k8s-recommended vs legacy Helm)._ + +```bash +kubectl -n logging delete pvc -l app.kubernetes.io/instance=my-receiver --ignore-not-found +kubectl -n logging delete secret reporter-credentials --ignore-not-found +``` + +### 3. Verify nothing remains +_Confirm no pods, services, or configmaps are lingering under the release label._ + +```bash +kubectl -n logging get all,configmap,secret,pvc -l app.kubernetes.io/instance=my-receiver +``` + +--- +_Plan built from snapshot `e100-1792971977960`._ + +**Next actions** (after running the plan above): + - Verify the install is healthy: `log10x_doctor`, runs all freshness/auth/config checks across reporter, receiver, retriever. + - Confirm the config snapshot picked up the change: `log10x_discover_env`, shows what the engine sees as the active config. \ No newline at end of file diff --git a/elastic-hot-tier-budget/install-e2e/results/launch.json b/elastic-hot-tier-budget/install-e2e/results/launch.json new file mode 100644 index 0000000..193e494 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/launch.json @@ -0,0 +1,60 @@ +{ + "mcp_version": "1.30.90", + "image": "ghcr.io/log-10x/edge-10x:1.1.150", + "image_rendered": "log10x/edge-10x:1.1.147@sha256:a043ccd828742c4eccee09abb654ce6a7357b6123f73300ed04f8605e874b8c4", + "args_from_values": [ + "@run/input/forwarder/fluentbit", + "@apps/receiver", + "symbolMessageHashField", + "tenx_hash", + "outputEncodeField", + "fullText(\"tenx_hash\",\"routeState\",\"message_pattern\",\"severity_level\",\"utf8Size\")" + ], + "env_applied": [ + "TENX_AIRGAPPED", + "K8S_ENABLED", + "CAP_LOOKUP_FILE", + "ACTION_LOOKUP_FILE", + "TOTAL_LOOKUP_FILE" + ], + "env_substituted": [ + { + "name": "K8S_ENABLED", + "rendered": "true", + "rig": "false" + }, + { + "name": "CAP_LOOKUP_FILE", + "rendered": "/tmp/tenx/kubernetes/$(K8S_NAMESPACE)/log10x-action-intent/caps.csv", + "rig": "/policy/caps.csv" + }, + { + "name": "ACTION_LOOKUP_FILE", + "rendered": "/tmp/tenx/kubernetes/$(K8S_NAMESPACE)/log10x-action-intent/actions.csv", + "rig": "/policy/actions.csv" + }, + { + "name": "TOTAL_LOOKUP_FILE", + "rendered": "/tmp/tenx/kubernetes/$(K8S_NAMESPACE)/log10x-action-intent/totals.csv", + "rig": "/policy/totals.csv" + } + ], + "env_left_out": [ + "K8S_NAMESPACE", + "K8S_CONFIGMAP", + "CONFIG_GENERATION_FILE" + ], + "rig_env": [ + "TENX_FLUENTBIT_OUTPUT_HOST=fluent-bit" + ], + "resources_limits": { + "cpus": "0.5", + "memory": "512M" + }, + "policy_seed_from_configmap": [ + "caps.csv", + "actions.csv", + "totals.csv", + "regulator-unit" + ] +} diff --git a/elastic-hot-tier-budget/install-e2e/results/policy.yaml b/elastic-hot-tier-budget/install-e2e/results/policy.yaml new file mode 100644 index 0000000..e327313 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/policy.yaml @@ -0,0 +1,72 @@ +# Log10x recurring cost-reduction policy +# Generated by log10x_setup_recurring: edit and re-commit as needed. +# The CLI reads this file at the start of each tick. +schema_version: "1.0" + +reduction: + # empty = all services + target_services: [] + # Elasticsearch hot tier held under a size: GB of index store on the hot nodes, + # or auto for 80% of their disk, which needs the tier_down index on warm + # (the run refuses auto while that index sits on hot). + # Each run rewrites the byte limits so growth goes to the frozen tier. + hot_tier_gb: 0.002 + # hours a kept line stays on hot: read from the ILM policy on pass_alias unless set here + # hot_retention_hours: 168 + pass_alias: tenx-app + tier_down_alias: tenx-tierdown + container_field: kubernetes.container_name + # error lines over a limit that pass per statement per node per window + keep_count: 50 + default_action: tier_down + nodes_by_container: + accounting: 1 + ad: 1 + aws-eks-nodeagent: 1 + aws-node: 1 + aws-vpc-cni-init: 1 + cart: 1 + coredns: 1 + currency: 1 + email: 1 + flagd: 1 + flagd-ui: 1 + fluentd-10x: 1 + fraud-detection: 1 + frontend: 1 + frontend-proxy: 1 + grafana: 1 + grafana-sc-alerts: 1 + grafana-sc-dashboard: 1 + grafana-sc-datasources: 1 + image-provider: 1 + init-config: 1 + jaeger: 1 + kafka: 1 + kube-proxy: 1 + load-generator: 1 + metrics-server: 1 + opensearch: 1 + opentelemetry-collector: 1 + payment: 1 + postgresql: 1 + prometheus-server: 1 + quote: 1 + recommendation: 1 + valkey-cart: 1 + wait-for-kafka: 1 + wait-for-valkey-cart: 1 + exceptions: [] + min_delta_pp: 2 + +schedule: + preset: hourly + cron_utc: "0 * * * *" + scheduler: github_actions + +config_plane: + repo: /Users/talweiss/git/l1x-co/wt-e100-bench/elastic-hot-tier-budget/install-e2e/results/policy-git/checkout + delivery: gitops + # the run commits straight to this branch; the engine's gitops pull reads it (GH_BRANCH) + policy_branch: log10x-policy + diff --git a/elastic-hot-tier-budget/install-e2e/results/policy/actions.csv b/elastic-hot-tier-budget/install-e2e/results/policy/actions.csv new file mode 100644 index 0000000..030ad0a --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/policy/actions.csv @@ -0,0 +1,34 @@ +container,action +accounting,tier_down::budget run +ad,tier_down::budget run +aws-eks-nodeagent,tier_down::budget run +aws-node,tier_down::budget run +aws-vpc-cni-init,tier_down::budget run +cart,tier_down::budget run +coredns,tier_down::budget run +currency,tier_down::budget run +email,tier_down::budget run +flagd,tier_down::budget run +flagd-ui,tier_down::budget run +fluentd-10x,tier_down::budget run +fraud-detection,tier_down::budget run +frontend,tier_down::budget run +frontend-proxy,tier_down::budget run +grafana,tier_down::budget run +grafana-sc-alerts,tier_down::budget run +grafana-sc-dashboard,tier_down::budget run +grafana-sc-datasources,tier_down::budget run +init-config,tier_down::budget run +jaeger,tier_down::budget run +kafka,tier_down::budget run +kube-proxy,tier_down::budget run +load-generator,tier_down::budget run +metrics-server,tier_down::budget run +opensearch,tier_down::budget run +opentelemetry-collector,tier_down::budget run +payment,tier_down::budget run +postgresql,tier_down::budget run +prometheus-server,tier_down::budget run +quote,tier_down::budget run +recommendation,tier_down::budget run +*,tier_down::budget run diff --git a/elastic-hot-tier-budget/install-e2e/results/policy/caps.csv b/elastic-hot-tier-budget/install-e2e/results/policy/caps.csv new file mode 100644 index 0000000..8cbc2d1 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/policy/caps.csv @@ -0,0 +1,34 @@ +container,cap +accounting,1::budget run +ad,1::budget run +aws-eks-nodeagent,1::budget run +aws-node,1::budget run +aws-vpc-cni-init,1::budget run +cart,1::budget run +coredns,1::budget run +currency,1::budget run +email,1::budget run +flagd,1::budget run +flagd-ui,1::budget run +fluentd-10x,1::budget run +fraud-detection,1::budget run +frontend,1::budget run +frontend-proxy,1::budget run +grafana,1::budget run +grafana-sc-alerts,1::budget run +grafana-sc-dashboard,1::budget run +grafana-sc-datasources,1::budget run +init-config,1::budget run +jaeger,1::budget run +kafka,1::budget run +kube-proxy,1::budget run +load-generator,1::budget run +metrics-server,1::budget run +opensearch,1::budget run +opentelemetry-collector,1::budget run +payment,1::budget run +postgresql,1::budget run +prometheus-server,1::budget run +quote,1::budget run +recommendation,1::budget run +*,1::budget run diff --git a/elastic-hot-tier-budget/install-e2e/results/policy/totals.csv b/elastic-hot-tier-budget/install-e2e/results/policy/totals.csv new file mode 100644 index 0000000..7230c74 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/policy/totals.csv @@ -0,0 +1,34 @@ +container,total +accounting,17753::budget run +ad,6631::budget run +aws-eks-nodeagent,455::budget run +aws-node,1220::budget run +aws-vpc-cni-init,2209::budget run +cart,5612::budget run +coredns,7558::budget run +currency,529::budget run +email,2005::budget run +flagd,2575::budget run +flagd-ui,864::budget run +fluentd-10x,5505::budget run +fraud-detection,2296::budget run +frontend,10985::budget run +frontend-proxy,57554::budget run +grafana,614280::budget run +grafana-sc-alerts,3809::budget run +grafana-sc-dashboard,5388::budget run +grafana-sc-datasources,4645::budget run +init-config,3395::budget run +jaeger,34034::budget run +kafka,125755::budget run +kube-proxy,35929::budget run +load-generator,4113::budget run +metrics-server,14031::budget run +opensearch,96000::budget run +opentelemetry-collector,459948::budget run +payment,2464::budget run +postgresql,7592::budget run +prometheus-server,10963::budget run +quote,238::budget run +recommendation,3378::budget run +*,5446::budget run diff --git a/elastic-hot-tier-budget/install-e2e/results/receiver.json b/elastic-hot-tier-budget/install-e2e/results/receiver.json new file mode 100644 index 0000000..11503be --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/receiver.json @@ -0,0 +1,69 @@ +{ + "mcp_version": "1.30.90", + "image": "log10x/edge-10x:1.1.147@sha256:a043ccd828742c4eccee09abb654ce6a7357b6123f73300ed04f8605e874b8c4", + "args": [ + "@run/input/forwarder/fluentbit", + "@apps/receiver", + "symbolMessageHashField", + "tenx_hash", + "outputEncodeField", + "fullText(\"tenx_hash\",\"routeState\",\"message_pattern\",\"severity_level\",\"utf8Size\")" + ], + "env": [ + { + "name": "TENX_AIRGAPPED", + "value": "true" + }, + { + "name": "K8S_NAMESPACE", + "valueFrom": { + "fieldRef": { + "fieldPath": "metadata.namespace" + } + } + }, + { + "name": "K8S_ENABLED", + "value": "true" + }, + { + "name": "K8S_CONFIGMAP", + "value": "log10x-action-intent" + }, + { + "name": "CAP_LOOKUP_FILE", + "value": "/tmp/tenx/kubernetes/$(K8S_NAMESPACE)/log10x-action-intent/caps.csv" + }, + { + "name": "ACTION_LOOKUP_FILE", + "value": "/tmp/tenx/kubernetes/$(K8S_NAMESPACE)/log10x-action-intent/actions.csv" + }, + { + "name": "CONFIG_GENERATION_FILE", + "value": "/tmp/tenx/kubernetes/$(K8S_NAMESPACE)/log10x-action-intent/config-generation.csv" + }, + { + "name": "TOTAL_LOOKUP_FILE", + "value": "/tmp/tenx/kubernetes/$(K8S_NAMESPACE)/log10x-action-intent/totals.csv" + } + ], + "resources": { + "requests": { + "cpu": "100m", + "memory": "256Mi" + }, + "limits": { + "cpu": "500m", + "memory": "512Mi" + } + }, + "policy_configmap": { + "name": "log10x-action-intent", + "data": { + "caps.csv": "container,cap\ntenx-seed,1::install seed; replaced by log10x_configure_engine\n", + "actions.csv": "container,action\ntenx-seed,pass::install seed; replaced by log10x_configure_engine\n", + "totals.csv": "container,total\ntenx-seed,1::install seed; replaced by the budget run\n", + "regulator-unit": "bytes" + } + } +} diff --git a/elastic-hot-tier-budget/install-e2e/results/recipe-requests.json b/elastic-hot-tier-budget/install-e2e/results/recipe-requests.json new file mode 100644 index 0000000..8251a8c --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/recipe-requests.json @@ -0,0 +1,137 @@ +[ + { + "method": "PUT", + "path": "/_ilm/policy/tenx-tier-down", + "body": { + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_primary_shard_size": "50gb", + "max_age": "1h" + } + } + }, + "frozen": { + "min_age": "1m", + "actions": { + "searchable_snapshot": { + "snapshot_repository": "found-snapshots" + } + } + } + } + } + } + }, + { + "method": "PUT", + "path": "/_ilm/policy/tenx-keep", + "body": { + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_primary_shard_size": "50gb", + "max_age": "1d" + } + } + }, + "frozen": { + "min_age": "6d", + "actions": { + "searchable_snapshot": { + "snapshot_repository": "found-snapshots" + } + } + } + } + } + } + }, + { + "method": "PUT", + "path": "/_index_template/tenx-tierdown-tpl", + "body": { + "index_patterns": [ + "tenx-tierdown-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "tenx-tier-down", + "index.lifecycle.rollover_alias": "tenx-tierdown", + "index.routing.allocation.include._tier_preference": "data_warm,data_hot", + "number_of_replicas": 0 + }, + "mappings": { + "properties": { + "tenx_hash": { + "type": "keyword" + }, + "routeState": { + "type": "keyword" + }, + "@timestamp": { + "type": "date" + } + } + } + } + } + }, + { + "method": "PUT", + "path": "/_index_template/tenx-app-tpl", + "body": { + "index_patterns": [ + "tenx-app-*" + ], + "template": { + "settings": { + "index.lifecycle.name": "tenx-keep", + "index.lifecycle.rollover_alias": "tenx-app", + "number_of_replicas": 0 + }, + "mappings": { + "properties": { + "tenx_hash": { + "type": "keyword" + }, + "routeState": { + "type": "keyword" + }, + "@timestamp": { + "type": "date" + } + } + } + } + } + }, + { + "method": "PUT", + "path": "/tenx-tierdown-000001", + "body": { + "aliases": { + "tenx-tierdown": { + "is_write_index": true + } + } + } + }, + { + "method": "PUT", + "path": "/tenx-app-000001", + "body": { + "aliases": { + "tenx-app": { + "is_write_index": true + } + } + } + } +] diff --git a/elastic-hot-tier-budget/install-e2e/results/recipe.txt b/elastic-hot-tier-budget/install-e2e/results/recipe.txt new file mode 100644 index 0000000..06ba1b8 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/recipe.txt @@ -0,0 +1,89 @@ +# 1) Policy for the tier_down slice: roll over hourly, then mount the rolled +# index as a PARTIALLY-MOUNTED searchable snapshot in the frozen tier as soon +# as ILM sees it. The template in 3) writes the slice on warm, so it waits +# there for its mount and takes no room on hot. With the default 10-minute +# poll: rollover at the first poll past the hour, the snapshot and mount +# three polls later. +PUT _ilm/policy/tenx-tier-down +{ + "policy": { "phases": { + "hot": { "min_age": "0ms", "actions": { "rollover": { "max_primary_shard_size": "50gb", "max_age": "1h" } } }, + "frozen": { "min_age": "1m", "actions": { "searchable_snapshot": { "snapshot_repository": "found-snapshots" } } } + }} +} + +# 2) Policy for everything we keep: roll over daily, then mount the rolled index +# on the frozen tier in the same repository. The frozen min_age, counted from +# rollover, is how long kept lines stay on hot: 6 days after the daily +# rollover, 7 days at most. The hot-tier run (log10x_setup_recurring +# hot_tier_gb) reads it, max_age plus min_age, as a kept line's hours on hot. +# No delete phase: applying the recipe removes no data. Still the control: +# tier_down lines reach frozen within about two hours of arriving, kept lines +# only after this retention, so "the slice moved" stays distinct from +# "everything moved". +PUT _ilm/policy/tenx-keep +{ + "policy": { "phases": { + "hot": { "min_age": "0ms", "actions": { "rollover": { "max_primary_shard_size": "50gb", "max_age": "1d" } } }, + "frozen": { "min_age": "6d", "actions": { "searchable_snapshot": { "snapshot_repository": "found-snapshots" } } } + }} +} + +# 3) Templates. The marker decides WHICH INDEX an event lands in; the index +# decides which policy governs it. tenx_hash and routeState are mapped as +# keyword so both survive as queryable fields in frozen (verified). +PUT _index_template/tenx-tierdown-tpl +{ + "index_patterns": ["tenx-tierdown-*"], + "template": { + "settings": { + "index.lifecycle.name": "tenx-tier-down", + "index.lifecycle.rollover_alias": "tenx-tierdown", + "index.routing.allocation.include._tier_preference": "data_warm,data_hot", + "number_of_replicas": 0 + }, + "mappings": { "properties": { + "tenx_hash": { "type": "keyword" }, + "routeState": { "type": "keyword" }, + "@timestamp": { "type": "date" } + }} + } +} + +PUT _index_template/tenx-app-tpl +{ + "index_patterns": ["tenx-app-*"], + "template": { + "settings": { + "index.lifecycle.name": "tenx-keep", + "index.lifecycle.rollover_alias": "tenx-app", + "number_of_replicas": 0 + }, + "mappings": { "properties": { + "tenx_hash": { "type": "keyword" }, + "routeState": { "type": "keyword" }, + "@timestamp": { "type": "date" } + }} + } +} + +# 4) Bootstrap the write indices. Both policies use rollover, so each alias +# needs an initial backing index flagged is_write_index. Skipping this is the +# usual reason ILM sits in check-rollover-ready forever. +PUT tenx-tierdown-000001 +{ "aliases": { "tenx-tierdown": { "is_write_index": true } } } + +PUT tenx-app-000001 +{ "aliases": { "tenx-app": { "is_write_index": true } } } + +# 5) Verify, once data has rolled over and aged past min_age. The frozen index +# is RENAMED with a partial- prefix, which is how you know it mounted: +# +# GET _cat/indices/*tenx-tierdown*?h=index,docs.count,store.size +# partial-tenx-tierdown-000001 0b <- data now in found-snapshots +# +# GET tenx-tierdown-000001/_ilm/explain -> phase: frozen, step: complete +# POST partial-tenx-tierdown-000001/_search +# { "query": { "term": { "tenx_hash": "" } } } +# ...must still return the documents. If it does not, STOP: the slice is +# cheap but unretrievable and the down-tier is not worth doing. diff --git a/elastic-hot-tier-budget/install-e2e/results/render.out b/elastic-hot-tier-budget/install-e2e/results/render.out new file mode 100644 index 0000000..f336524 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/render.out @@ -0,0 +1,36 @@ +{ + "mcp_version": "1.30.90", + "recipe_requests": [ + "PUT /_ilm/policy/tenx-tier-down", + "PUT /_ilm/policy/tenx-keep", + "PUT /_index_template/tenx-tierdown-tpl", + "PUT /_index_template/tenx-app-tpl", + "PUT /tenx-tierdown-000001", + "PUT /tenx-app-000001" + ], + "image": "log10x/edge-10x:1.1.147@sha256:a043ccd828742c4eccee09abb654ce6a7357b6123f73300ed04f8605e874b8c4", + "args": [ + "@run/input/forwarder/fluentbit", + "@apps/receiver", + "symbolMessageHashField", + "tenx_hash", + "outputEncodeField", + "fullText(\"tenx_hash\",\"routeState\",\"message_pattern\",\"severity_level\",\"utf8Size\")" + ], + "env": [ + "TENX_AIRGAPPED", + "K8S_NAMESPACE", + "K8S_ENABLED", + "K8S_CONFIGMAP", + "CAP_LOOKUP_FILE", + "ACTION_LOOKUP_FILE", + "CONFIG_GENERATION_FILE", + "TOTAL_LOOKUP_FILE" + ], + "seed_keys": [ + "caps.csv", + "actions.csv", + "totals.csv", + "regulator-unit" + ] +} diff --git a/elastic-hot-tier-budget/install-e2e/results/retriever-plan.md b/elastic-hot-tier-budget/install-e2e/results/retriever-plan.md new file mode 100644 index 0000000..c2224ae --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/retriever-plan.md @@ -0,0 +1,508 @@ +# Retriever advisor: install + verify + teardown + +## Target +- **app**: retriever +- **release**: `my-retriever` +- **namespace**: `logging` +- **snapshot**: `e100-1792971977960` + +## Notes +- Retriever infra (S3 buckets, SQS queues, IAM role + IRSA binding, CloudWatch log groups) is provisioned via the Terraform module, NOT by this advisor. The plan below assumes infra already exists. +- No licence key is written into the values file. The engine runs on its built-in Free tier (up to 10 nodes, no expiry, airgapped) until a key is supplied, and a key is supplied at install time with `--set-string tenx.apiKey="$LOG10X_API_KEY"` so it stays out of any file on disk. + +## Preflight +| Check | Status | Detail | +|---|---|---| +| kubectl | `ok` | context `undefined` | +| namespace | `ok` | `logging` exists | +| release collision | `ok` | no `my-retriever` release in `logging`: clear to install | +| AWS access | `ok` | account `?`, region `us-east-1` | +| input S3 bucket | `ok` | `e100-retriever-input` | +| index S3 prefix | `ok` | e100-retriever-input/indexing-results/ | +| IRSA role | `ok` | `arn:aws:iam::000000000000:role/e100-retriever` | +| SQS index queue | `ok` | `https://sqs.us-east-1.amazonaws.com/000000000000/e100-index` | +| SQS query queue | `ok` | `https://sqs.us-east-1.amazonaws.com/000000000000/e100-query` | +| SQS subquery queue | `ok` | `https://sqs.us-east-1.amazonaws.com/000000000000/e100-subquery` | +| SQS stream queue | `ok` | `https://sqs.us-east-1.amazonaws.com/000000000000/e100-stream` | +| queryLogGroup configured | **warn** | queryLogGroup is empty in helm values. Per-query CloudWatch observability disabled; queryEventLog calls early-return without writing. +To enable, set: + queryLogGroup: log10x-retriever-query-events # CW log group (pre-create via Terraform or aws logs create-log-group) +AND ensure the retriever IRSA role (arn:aws:iam::000000000000:role/e100-retriever) has logs:CreateLogStream and logs:PutLogEvents on arn:aws:logs:{region}:{account}:log-group:{logGroup}:* | + +## Install +### 1. Add Retriever Helm repo +_Makes the log10x/retriever-10x chart available to `helm install`._ + +```bash +helm repo add log10x https://log-10x.github.io/helm-charts +helm repo update +helm search repo log10x/retriever-10x +``` + +### 2. Create target namespace +_The Retriever installs into `logging`._ + +```bash +kubectl create namespace logging --dry-run=client -o yaml | kubectl apply -f - +``` + +### 3. Write Helm values +_Wires the tenx block, IRSA ServiceAccount, S3 buckets, and all four SQS queue URLs into the chart._ + +Write `my-retriever-values.yaml`: + +```yaml +tenx: + enabled: true + # apiKey: pass at install time with --set-string tenx.apiKey="$LOG10X_API_KEY" + runtimeName: "my-retriever" + gitToken: "public-repo-no-token-needed" + config: + git: + enabled: true + url: "https://github.com/log-10x/config.git" + +serviceAccount: + create: true + annotations: + eks.amazonaws.com/role-arn: "arn:aws:iam::000000000000:role/e100-retriever" + +inputBucket: "e100-retriever-input" +indexBucket: "e100-retriever-input/indexing-results/" + +indexQueueUrl: "https://sqs.us-east-1.amazonaws.com/000000000000/e100-index" +queryQueueUrl: "https://sqs.us-east-1.amazonaws.com/000000000000/e100-query" +subQueryQueueUrl: "https://sqs.us-east-1.amazonaws.com/000000000000/e100-subquery" +streamQueueUrl: "https://sqs.us-east-1.amazonaws.com/000000000000/e100-stream" +``` + +### 4. Install via Helm +_Deploys the indexer + query-handler + stream-worker + filter CronJobs._ + +```bash +helm upgrade --install my-retriever log10x/retriever-10x \ + -n logging --create-namespace \ + -f my-retriever-values.yaml +``` + +### 5. Wait for rollout +_Blocks until indexer + query-handler + stream-worker report Ready._ + +```bash +kubectl -n logging rollout status deployment -l app.kubernetes.io/instance=my-retriever --timeout=10m || true +``` + +## Forwarder offload (cost reduction that keeps every line) +Route the slice 10x marks for offload (`routeState == "offload"`) to the customer's own S3 before the SIEM bills it; the Retriever indexes that bucket and fetches it back by stamped identity. Nothing is deleted, it is relocated: every line is kept. + +Target: `s3://e100-retriever-input/app/` (region `us-east-1`), newline-delimited JSON. +Prerequisite on the engine side: run the receiver with `outputOffload true` (full-text events plus the `routeState` marker, every event flowing back). + +**fluent-bit offload recipe** + +```ini +[SERVICE] + Grace 5 # let the re-emitted chunks flush before shutdown + +# 1) map the routeState marker to a dedicated routing key for the rewrite_tag +# Rules below. offload/tier_down/drop get their own tags; pass, compact and +# sample stay on tenx.app for the SIEM. (Keeping rec["routeState"]=="drop" +# explicit so the noise slice is unambiguous.) +[FILTER] + Name lua + Match tenx.* + call tag_route + code function tag_route(tag,ts,rec) local r=rec["routeState"] if r=="offload" then rec["_route"]="offload" elseif r=="tier_down" then rec["_route"]="tier_down" elseif r=="drop" then rec["_route"]="drop" else rec["_route"]="siem" end return 2,ts,rec end + +# 2) route each non-SIEM action to its own tag. KEEP=true (4th field): KEEP=false +# drops the re-emitted record entirely in fluent-bit. The original copy stays +# on tenx.app and the routed slices are excluded from it in step 3. +[FILTER] + Name rewrite_tag + Match tenx.* + Rule $_route ^offload$ tenx.offload true + Rule $_route ^tier_down$ tenx.tier_down true + Rule $_route ^drop$ tenx.drop true + +# 3) keep the routed slices OUT of the SIEM path (the KEEP=true originals on +# tenx.app). What remains on tenx.app is pass/compact/sample == _route siem. +[FILTER] + Name grep + Match tenx.app + Regex _route ^siem$ + +# 4) strip the markers (tenx_hash kept). +# tenx.* spans the retagged tags and the kept tenx.app (the wildcard +# crosses dots). Safe here because the split is forwarder-side: the routing +# decision is already carried by the tag, not by a field the destination reads. +[FILTER] + Name record_modifier + Match tenx.* + Remove_key routeState + Remove_key _route + +# 5) offload slice -> customer-owned S3 as JSONL +[OUTPUT] + Name s3 + Match tenx.offload + bucket e100-retriever-input + region us-east-1 + s3_key_format /app/$UUID.jsonl + use_put_object On + json_date_format iso8601 + +# 6) tier_down slice -> your cheaper in-platform SIEM tier. +# PLACEHOLDER: replace with the OUTPUT for your cheap tier, e.g. a second +# [OUTPUT] Name cloudwatch_logs pointed at an Infrequent-Access log group, +# a datadog output tagged to a Flex index, or an azure_logs_ingestion output +# to an Azure Basic/Auxiliary table. See cloudwatchIaRecipe() / +# datadogFlexRecipe() / azureLogsTierRecipe() for the destination-side TF. +# [OUTPUT] +# Name +# Match tenx.tier_down + +# 7) drop slice -> SUPPRESSED. The null output discards the noise slice. +[OUTPUT] + Name null + Match tenx.drop + +# 8) pass / compact / sample -> your existing SIEM output, Match tenx.app. +# compact already carries the engine's encoded bytes on the wire, so no +# special handling beyond routing it to the SIEM. +``` + +_Placement: all FILTERs sit on the 10x return path (`Match tenx.*`); `routeState` only exists on post-sidecar records. The lua filter maps the marker to a routing key `_route`, the `rewrite_tag` Rules send offload/tier_down/drop to their own tags (S3 / cheap-tier / `null`), and pass/compact/sample stay on `tenx.app` for the SIEM._ + +Prerequisites: +- Engine: the receiver runs with `outputOffload true` (full-text events + `routeState` marker, all events flow back to the forwarder). +- IAM: the forwarder's identity can `s3:PutObject` to `e100-retriever-input/app/*`: see `forwarderWriteIamPolicy()` (only the `offload` branch needs this grant). +- Match the route-state name as a STRING (`routeState == "offload"`, `"tier_down"`, `"drop"`, ...), never a boolean test (the engine writes the action name as a JSON string). +- Encoding: KEEP the shipped default `fluentbitOutputEncodeType: delimited`, which delivers `routeState` as its own record field. Do NOT set it to `json`: that collapses the record into a single string field, the routing lua stops matching, and the offload/drop slices silently fall through to the SIEM with no error. +- The lua filter (marker -> routing key) and `KEEP=true` are both mandatory in this shape: the routes are keyed off `_route`, and KEEP=false drops the re-emitted record (verified live on fluent-bit v5). + +Other supported forwarders: vector, fluentd, otel-collector, logstash, cribl. + +**Forwarder write access** (the one new IAM grant: the Retriever's own role only READS the source bucket). Ready-to-apply Terraform, EKS IRSA, non-EKS noted at the bottom: + +```hcl +# Forwarder-write IAM for the offload loop. The forwarder PutObjects the +# routeState=="offload" slice to the Retriever input bucket; the Retriever's own role only +# READS it, so this is a SEPARATE, additive grant. + +variable "bucket" { + type = string + description = "Retriever input bucket. Objects land at //..." +} +variable "prefix" { + type = string + default = "app" + description = "Key prefix == Retriever target. PutObject is scoped to //*." +} +variable "oidc_provider_arn" { + type = string + description = "Cluster IAM OIDC provider ARN (arn:aws:iam:::oidc-provider/oidc.eks..amazonaws.com/id/)." +} +variable "namespace" { type = string } # forwarder ServiceAccount namespace +variable "service_account" { type = string } # forwarder ServiceAccount name +variable "name_prefix" { + type = string + default = "tenx-forwarder-offload" +} + +locals { + # IRSA conditions key on the issuer URL (no scheme): the part after oidc-provider/. + oidc_issuer = split("oidc-provider/", var.oidc_provider_arn)[1] +} + +# PutObject scoped to //* +data "aws_iam_policy_document" "write" { + statement { + sid = "TenxForwarderOffloadWrite" + effect = "Allow" + actions = ["s3:PutObject"] + resources = ["arn:aws:s3:::${var.bucket}/${var.prefix}/*"] + } +} + +resource "aws_iam_policy" "write" { + name = "${var.name_prefix}-write" + policy = data.aws_iam_policy_document.write.json +} + +# IRSA trust: OIDC-federated assume-role pinned to ONE ServiceAccount. +data "aws_iam_policy_document" "trust" { + statement { + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + principals { + type = "Federated" + identifiers = [var.oidc_provider_arn] + } + condition { + test = "StringEquals" + variable = "${local.oidc_issuer}:sub" + values = ["system:serviceaccount:${var.namespace}:${var.service_account}"] + } + condition { + test = "StringEquals" + variable = "${local.oidc_issuer}:aud" + values = ["sts.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "forwarder_offload" { + name = var.name_prefix + assume_role_policy = data.aws_iam_policy_document.trust.json +} + +resource "aws_iam_role_policy_attachment" "write" { + role = aws_iam_role.forwarder_offload.name + policy_arn = aws_iam_policy.write.arn +} + +# Annotate the forwarder ServiceAccount: eks.amazonaws.com/role-arn = +output "forwarder_offload_role_arn" { + value = aws_iam_role.forwarder_offload.arn +} + +# Non-EKS: reuse aws_iam_policy.write unchanged; only the identity differs. +# EC2/self-managed: attach it to the node instance-profile role. +# on-prem/outside AWS: attach it to an aws_iam_user + aws_iam_access_key, +# feed the key into the forwarder's S3 output creds. +``` + +**Or down-tier in the SIEM instead of offloading** (keep events in-platform at a cheaper tier, same `routeState` marker, no second attribute): + +_Elasticsearch frozen tier_: Elasticsearch tiering is INDEX-level, so the shipper routes the marked slice to its own index and ILM does the rest. Verified live: the frozen index reported store=0b locally while the hot control stayed at 19.2kb, and a term query on `tenx_hash` still returned 400/400 with `routeState` aggregatable, so the down-tiered slice remains retrievable by stamped identity. PREREQUISITE: searchable snapshots need an Enterprise licence (Elastic documents no lower tier for them), plus a registered snapshot repository (Elastic Cloud provides `found-snapshots` by default). RESIDENCY: the tier_down template prefers data_warm, so the slice is written on warm and mounts on frozen from there, taking no room on hot. A cluster with no data_warm node places it on hot instead (the preference falls back to data_hot), and the hot-tier run counts it there. KEPT LINES: tenx-keep rolls over daily and mounts the rolled index on frozen 6 days later, so a kept line stays on hot 7 days at most; the hot-tier run reads that from the policy. The tier_down alias rolls over hourly and the kept alias daily, one frozen index per rollover. The recipe has no delete phase, so applying it removes no data; a delete phase at the retention the account wants keeps the shard count on frozen bounded. ON ELASTIC CLOUD HOSTED THE SAVING IS NOT AUTOMATIC: Hosted is priced by provisioned resources, so moving data out of hot creates headroom and the deployment must then be RESIZED to bank it. Do not tell a Hosted customer their bill drops on its own; tell them their hot tier can shrink. Self-managed differs: the saving is disk you stop buying. OPERATIONAL NOTE: ILM polls every 10 minutes by default (`indices.lifecycle.poll_interval`), so a transition will not be visible immediately after rollover; that is not a failure. + +```text +# 1) Policy for the tier_down slice: roll over hourly, then mount the rolled +# index as a PARTIALLY-MOUNTED searchable snapshot in the frozen tier as soon +# as ILM sees it. The template in 3) writes the slice on warm, so it waits +# there for its mount and takes no room on hot. With the default 10-minute +# poll: rollover at the first poll past the hour, the snapshot and mount +# three polls later. +PUT _ilm/policy/tenx-tier-down +{ + "policy": { "phases": { + "hot": { "min_age": "0ms", "actions": { "rollover": { "max_primary_shard_size": "50gb", "max_age": "1h" } } }, + "frozen": { "min_age": "1m", "actions": { "searchable_snapshot": { "snapshot_repository": "found-snapshots" } } } + }} +} + +# 2) Policy for everything we keep: roll over daily, then mount the rolled index +# on the frozen tier in the same repository. The frozen min_age, counted from +# rollover, is how long kept lines stay on hot: 6 days after the daily +# rollover, 7 days at most. The hot-tier run (log10x_setup_recurring +# hot_tier_gb) reads it, max_age plus min_age, as a kept line's hours on hot. +# No delete phase: applying the recipe removes no data. Still the control: +# tier_down lines reach frozen within about two hours of arriving, kept lines +# only after this retention, so "the slice moved" stays distinct from +# "everything moved". +PUT _ilm/policy/tenx-keep +{ + "policy": { "phases": { + "hot": { "min_age": "0ms", "actions": { "rollover": { "max_primary_shard_size": "50gb", "max_age": "1d" } } }, + "frozen": { "min_age": "6d", "actions": { "searchable_snapshot": { "snapshot_repository": "found-snapshots" } } } + }} +} + +# 3) Templates. The marker decides WHICH INDEX an event lands in; the index +# decides which policy governs it. tenx_hash and routeState are mapped as +# keyword so both survive as queryable fields in frozen (verified). +PUT _index_template/tenx-tierdown-tpl +{ + "index_patterns": ["tenx-tierdown-*"], + "template": { + "settings": { + "index.lifecycle.name": "tenx-tier-down", + "index.lifecycle.rollover_alias": "tenx-tierdown", + "index.routing.allocation.include._tier_preference": "data_warm,data_hot", + "number_of_replicas": 0 + }, + "mappings": { "properties": { + "tenx_hash": { "type": "keyword" }, + "routeState": { "type": "keyword" }, + "@timestamp": { "type": "date" } + }} + } +} + +PUT _index_template/tenx-app-tpl +{ + "index_patterns": ["tenx-app-*"], + "template": { + "settings": { + "index.lifecycle.name": "tenx-keep", + "index.lifecycle.rollover_alias": "tenx-app", + "number_of_replicas": 0 + }, + "mappings": { "properties": { + "tenx_hash": { "type": "keyword" }, + "routeState": { "type": "keyword" }, + "@timestamp": { "type": "date" } + }} + } +} + +# 4) Bootstrap the write indices. Both policies use rollover, so each alias +# needs an initial backing index flagged is_write_index. Skipping this is the +# usual reason ILM sits in check-rollover-ready forever. +PUT tenx-tierdown-000001 +{ "aliases": { "tenx-tierdown": { "is_write_index": true } } } + +PUT tenx-app-000001 +{ "aliases": { "tenx-app": { "is_write_index": true } } } + +# 5) Verify, once data has rolled over and aged past min_age. The frozen index +# is RENAMED with a partial- prefix, which is how you know it mounted: +# +# GET _cat/indices/*tenx-tierdown*?h=index,docs.count,store.size +# partial-tenx-tierdown-000001 0b <- data now in found-snapshots +# +# GET tenx-tierdown-000001/_ilm/explain -> phase: frozen, step: complete +# POST partial-tenx-tierdown-000001/_search +# { "query": { "term": { "tenx_hash": "" } } } +# ...must still return the documents. If it does not, STOP: the slice is +# cheap but unretrievable and the down-tier is not worth doing. +``` + +Point the forwarder at the `tenx-tierdown` alias for events where `routeState == "tier_down"`, and at `tenx-app` for everything else. There is no separate cheap-tier endpoint to configure: on Elasticsearch the index IS the tier selector. + +Fetch back: `log10x_retriever_query` by pattern identity returns the offloaded events from S3. + +## Verify +Each probe below answers one specific question: run them in order. A probe passes when its commands exit 0 and (optionally) match the `expectOutput` regex. + +### 1. pods-ready: Are indexer + query-handler + stream-worker pods Ready? +_Expect to pass within 600s._ + +```bash +kubectl -n logging wait --for=condition=Ready pod -l app.kubernetes.io/instance=my-retriever --timeout=10m +``` + +Expect output matching: `condition met` + +### 2. indexer-healthy: Is the indexer processing messages from the index queue? +_Expect to pass within 120s._ + +```bash +kubectl -n logging logs -l app.kubernetes.io/instance=my-retriever,app.kubernetes.io/component=indexer --tail=200 | grep -iE 'index|processed|bloom' | head -20 +``` + +### 3. query-endpoint-healthy: Is the query endpoint responding? + +```bash +kubectl -n logging get ingress,svc -l app.kubernetes.io/instance=my-retriever +``` + +### 4. retriever-service-accessibility: Is the Retriever Service reachable from outside the cluster? If type=ClusterIP, use port-forward, LoadBalancer, or deploy the MCP server inside the cluster. + +```bash +kubectl -n logging get svc -l app.kubernetes.io/instance=my-retriever -o json 2>/dev/null | jq -r '.items[] | "Service \(.metadata.name): type=\(.spec.type) port=\(.spec.ports[0].port // "?")"' 2>/dev/null || kubectl -n logging get svc -l app.kubernetes.io/instance=my-retriever -o wide +``` + +### 5. s3-offload-input: Is the forwarder offloading the dropped slice into the source bucket? + +```bash +aws s3 ls s3://e100-retriever-input/app/ --recursive --summarize 2>/dev/null | tail -5 || echo "no objects under app/ yet: if this stays empty the forwarder offload is NOT wired. See the 'Forwarder offload' section: the receiver needs outputOffload=true, the per-forwarder recipe applied, and the forwarder-write IRSA (s3:PutObject to this bucket/app/)." +``` + +### 6. s3-indexing-results: Is the indexer writing to the index bucket? + +```bash +aws s3 ls s3://e100-retriever-input/indexing-results/ --summarize 2>/dev/null | tail -5 || echo "no index results yet (may take a few minutes after first index run). If s3-offload-input is also empty, fix the forwarder offload first: the indexer has nothing to index." +``` + +### 7. sqs-drainage: Is the index queue being drained (messages not piling up)? + +```bash +aws sqs get-queue-attributes --queue-url "https://sqs.us-east-1.amazonaws.com/000000000000/e100-index" --attribute-names ApproximateNumberOfMessages --output json +``` + +## Teardown +### 1. Uninstall the Helm release +_Removes indexer, query-handler, stream-worker deployments, filter CronJobs, ConfigMaps, and the chart-created ServiceAccount. LEAVES AWS infra (S3, SQS, IAM role) intact: that lifecycle belongs to Terraform._ + +```bash +helm -n logging uninstall my-retriever +``` + +### 2. Clean up derived resources +_Helm does not reap PVCs or Secrets created outside the release._ + +```bash +kubectl -n logging delete pvc -l app.kubernetes.io/instance=my-retriever --ignore-not-found +``` + +### 3. Verify nothing remains +_Confirm no workloads are lingering under the release label._ + +```bash +kubectl -n logging get all,configmap,secret,pvc -l app.kubernetes.io/instance=my-retriever +``` + +### 4. (Optional) teardown AWS infra +_If you're fully removing the Retriever, tear down the Terraform module that created the S3 buckets, SQS queues, and IAM role. Skipping this leaves empty AWS infra behind (zero-cost for SQS idle, pennies for S3 storage)._ + +```bash +# From your terraform directory: +# terraform destroy -target=module.retriever_aws_infra +``` + +## How to query the Retriever from outside the cluster + +The Retriever Service defaults to **ClusterIP** (cluster-internal only). `log10x_retriever_query` and `log10x_retriever_series` send HTTP requests to the Retriever: from your laptop the cluster-internal URL will fail with ENOTFOUND or ECONNREFUSED. + +Pick one option: + +### Option A: kubectl port-forward (dev/test, ephemeral) + +```bash +kubectl port-forward -n logging svc/my-retriever-retriever-10x-all-in-one 18080:80 +``` + +Then set the Retriever URL so the MCP server can reach it: + +```bash +export __SAVE_LOG10X_RETRIEVER_URL__=http://localhost:18080 +``` + +The port-forward is ephemeral: it stops when the terminal session ends. Restart it whenever you need to run retriever queries. + +### Option B: Migrate the Service to LoadBalancer (persistent) + +Add to your Terraform module call or values file: + +```yaml +# retriever-values-lb.yaml +retriever: + service: + type: LoadBalancer + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: "nlb" +``` + +Apply with: + +```bash +helm upgrade my-retriever log10x/retriever-10x -n logging -f retriever-values-lb.yaml +``` + +After the LoadBalancer is provisioned, run `kubectl -n logging get svc my-retriever-retriever-10x-all-in-one` to get the external hostname, then: + +```bash +export __SAVE_LOG10X_RETRIEVER_URL__=http:// +``` + +### Option C: Deploy the MCP server inside the cluster + +When the MCP server runs as a pod in the same cluster, the chart-default ClusterIP URL (`http://my-retriever-retriever-10x-all-in-one.logging.svc.cluster.local:80`) resolves correctly with no additional configuration. This is the cleanest path for shared teams and CI pipelines. + +--- +_Plan built from snapshot `e100-1792971977960`._ + +**Next actions** (after running the plan above): + - Verify the install is healthy: `log10x_doctor`, runs all freshness/auth/config checks across reporter, receiver, retriever. + - Confirm the config snapshot picked up the change: `log10x_discover_env`, shows what the engine sees as the active config. \ No newline at end of file diff --git a/elastic-hot-tier-budget/install-e2e/results/setup-es.out b/elastic-hot-tier-budget/install-e2e/results/setup-es.out new file mode 100644 index 0000000..100a33d --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/setup-es.out @@ -0,0 +1,9 @@ +200 POST /_license/start_trial?acknowledge=true +200 PUT /_snapshot/found-snapshots +200 PUT /_ilm/policy/tenx-tier-down +200 PUT /_ilm/policy/tenx-keep +200 PUT /_index_template/tenx-tierdown-tpl +200 PUT /_index_template/tenx-app-tpl +200 PUT /tenx-tierdown-000001 +200 PUT /tenx-app-000001 +placement ["tenx-tierdown-000001@es-warm-frozen(data_frozen+data_warm)","tenx-app-000001@es-hot(data_content+data_hot+ingest+master)"] diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/branch-log.txt b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/branch-log.txt new file mode 100644 index 0000000..ae6af45 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/branch-log.txt @@ -0,0 +1,2 @@ +89f7913 seed: the policy files from log10x_advise_install +ddd815e policy.yaml from log10x_setup_recurring diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/exit b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/report.txt b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/report.txt new file mode 100644 index 0000000..250bbb6 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/report.txt @@ -0,0 +1,52 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp [DRY RUN] +[tenx-recur] Hot tier: 2 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 3.8 GB within 168 hours, with 348 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 0 on hot, 0 MB in 1 off hot, outside the size. +[tenx-recur] A kept line stays on hot 168 hours (ILM policy tenx-keep (rollover max_age 1d + frozen min_age 6d)); the tier_down write index is off hot, so moved lines take no room on hot while they wait for their mount. +[tenx-recur] Each counted byte takes 0.97 bytes of store kept, 0.97 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 908 MB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 560 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-25T23:50:21.000Z + +Hot tier: 2 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 3.8 GB within 168 hours, with 348 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 0 on hot, 0 MB in 1 off hot, outside the size. +- A kept line stays on hot 168 hours (ILM policy tenx-keep (rollover max_age 1d + frozen min_age 6d)); the tier_down write index is off hot, so moved lines take no room on hot while they wait for their mount. +- Each counted byte takes 0.97 bytes of store kept, 0.97 as tier_down (measured on the indices on hot). +- The Receiver's statements send 908 MB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 560 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 560 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 2 MB at about 3.8 GB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 8 MB | 7 MB | 1 | 18236 | +| ad | the frozen tier | 1 | 9 MB | 2 MB | 1 | 6752 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 472 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1265 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2289 | +| cart | the frozen tier | 1 | 18 MB | 2 MB | 1 | 5707 | +| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7672 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 548 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2077 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2668 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 896 | +| fraud-detection | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2379 | +| frontend | the frozen tier | 1 | 4 MB | 4 MB | 1 | 11384 | +| frontend-proxy | the frozen tier | 1 | 21 MB | 21 MB | 1 | 59646 | +| grafana | the frozen tier | 1 | 230 MB | 229 MB | 1 | 636386 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3947 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5584 | +| grafana-sc-datasources | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4814 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3518 | +| jaeger | the frozen tier | 1 | 13 MB | 13 MB | 1 | 35272 | +| kafka | the frozen tier | 1 | 115 MB | 46 MB | 1 | 128723 | +| kube-proxy | the frozen tier | 1 | 30 MB | 13 MB | 1 | 37068 | +| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4263 | +| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14542 | +| opensearch | the frozen tier | 1 | 39 MB | 36 MB | 1 | 99195 | +| opentelemetry-collector | the frozen tier | 1 | 400 MB | 166 MB | 1 | 460398 | +| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5646 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/run.json b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/run.json new file mode 100644 index 0000000..37dbb37 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235021Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-25T23:50:22.848Z","status":"dry_run","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"dry-run: Hot tier: 2 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 3.8 GB within 168 hours, with 348 MB a day of counted lines moved to the frozen tier."} diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/branch-log.txt b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/branch-log.txt new file mode 100644 index 0000000..7d9d283 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/branch-log.txt @@ -0,0 +1,3 @@ +22756de chore(log10x): hot-tier run, hot held under 2 MB +89f7913 seed: the policy files from log10x_advise_install +ddd815e policy.yaml from log10x_setup_recurring diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/exit b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/report.txt b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/report.txt new file mode 100644 index 0000000..b737f63 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/report.txt @@ -0,0 +1,58 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 2 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 3.6 GB within 168 hours, with 359 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 0 on hot, 0 MB in 1 off hot, outside the size. +[tenx-recur] A kept line stays on hot 168 hours (ILM policy tenx-keep (rollover max_age 1d + frozen min_age 6d)); the tier_down write index is off hot, so moved lines take no room on hot while they wait for their mount. +[tenx-recur] Each counted byte takes 0.93 bytes of store kept, 0.93 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 917 MB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 558 MB a day on hot whatever the limits; the size holds down to that floor. +# Hot-tier run: 2026-10-25T23:50:28.000Z + +Hot tier: 2 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 3.6 GB within 168 hours, with 359 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 2 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 0 on hot, 0 MB in 1 off hot, outside the size. +- A kept line stays on hot 168 hours (ILM policy tenx-keep (rollover max_age 1d + frozen min_age 6d)); the tier_down write index is off hot, so moved lines take no room on hot while they wait for their mount. +- Each counted byte takes 0.93 bytes of store kept, 0.93 as tier_down (measured on the indices on hot). +- The Receiver's statements send 917 MB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 558 MB a day on hot whatever the limits; the size holds down to that floor. +- Warning: 1 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 558 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 2 MB at about 3.6 GB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 7 MB | 6 MB | 1 | 17753 | +| ad | the frozen tier | 1 | 9 MB | 2 MB | 1 | 6631 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 455 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1220 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2209 | +| cart | the frozen tier | 1 | 17 MB | 2 MB | 1 | 5612 | +| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7558 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 529 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2005 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2575 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 864 | +| fluentd-10x | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5505 | +| fraud-detection | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2296 | +| frontend | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10985 | +| frontend-proxy | the frozen tier | 1 | 21 MB | 21 MB | 1 | 57554 | +| grafana | the frozen tier | 1 | 222 MB | 221 MB | 1 | 614280 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3809 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5388 | +| grafana-sc-datasources | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4645 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3395 | +| jaeger | the frozen tier | 1 | 12 MB | 12 MB | 1 | 34034 | +| kafka | the frozen tier | 1 | 111 MB | 45 MB | 1 | 125755 | +| kube-proxy | the frozen tier | 1 | 29 MB | 13 MB | 1 | 35929 | +| load-generator | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4113 | +| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 14031 | +| opensearch | the frozen tier | 1 | 38 MB | 35 MB | 1 | 96000 | +| opentelemetry-collector | the frozen tier | 1 | 410 MB | 166 MB | 1 | 459948 | +| payment | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2464 | +| postgresql | the frozen tier | 1 | 3 MB | 3 MB | 1 | 7592 | +| prometheus-server | the frozen tier | 1 | 4 MB | 4 MB | 1 | 10963 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 238 | +| recommendation | the frozen tier | 1 | 6 MB | 1 MB | 1 | 3378 | +| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5446 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/run.json b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/run.json new file mode 100644 index 0000000..189fdcc --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235028Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-25T23:50:30.016Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 2 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 3.6 GB within 168 hours, with 359 MB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/branch-log.txt b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/branch-log.txt new file mode 100644 index 0000000..7d9d283 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/branch-log.txt @@ -0,0 +1,3 @@ +22756de chore(log10x): hot-tier run, hot held under 2 MB +89f7913 seed: the policy files from log10x_advise_install +ddd815e policy.yaml from log10x_setup_recurring diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/exit b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/report.txt b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/report.txt new file mode 100644 index 0000000..7083573 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/report.txt @@ -0,0 +1,61 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp [DRY RUN] +[tenx-recur] Hot tier: 4 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 4.6 GB within 168 hours, with 441 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 0 on hot, 1 MB in 1 off hot, outside the size. +[tenx-recur] A kept line stays on hot 168 hours (ILM policy tenx-keep (rollover max_age 1d + frozen min_age 6d)); the tier_down write index is off hot, so moved lines take no room on hot while they wait for their mount. +[tenx-recur] Each counted byte takes 1.11 bytes of store kept, 1.11 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 592 MB a day on hot whatever the limits; the size holds down to that floor. +[tenx-recur] Hot is above the size now; the allowance is held at 52% until it drains, which takes up to 168 hours as the indices on hot age out. +# Hot-tier run: 2026-10-25T23:53:18.000Z + +Hot tier: 4 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 4.6 GB within 168 hours, with 441 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 0 on hot, 1 MB in 1 off hot, outside the size. +- A kept line stays on hot 168 hours (ILM policy tenx-keep (rollover max_age 1d + frozen min_age 6d)); the tier_down write index is off hot, so moved lines take no room on hot while they wait for their mount. +- Each counted byte takes 1.11 bytes of store kept, 1.11 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 592 MB a day on hot whatever the limits; the size holds down to that floor. +- Hot is above the size now; the allowance is held at 52% until it drains, which takes up to 168 hours as the indices on hot age out. +- Warning: 2 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 592 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 2 MB at about 4.6 GB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 4 MB | 4 MB | 1 | 12065 | +| ad | the frozen tier | 1 | 15 MB | 2 MB | 1 | 6829 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 492 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1318 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2386 | +| cart | the frozen tier | 1 | 33 MB | 2 MB | 1 | 5855 | +| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7808 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 571 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2165 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2781 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 933 | +| fluentd-10x | the frozen tier | 1 | 10 MB | 6 MB | 1 | 17395 | +| fraud-detection | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2479 | +| frontend | the frozen tier | 1 | 8 MB | 8 MB | 1 | 21373 | +| frontend-proxy | the frozen tier | 1 | 22 MB | 22 MB | 1 | 62154 | +| grafana | the frozen tier | 1 | 240 MB | 239 MB | 1 | 662935 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4113 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5819 | +| grafana-sc-datasources | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5016 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3666 | +| jaeger | the frozen tier | 1 | 13 MB | 13 MB | 1 | 36754 | +| kafka | the frozen tier | 1 | 148 MB | 48 MB | 1 | 132610 | +| kube-proxy | the frozen tier | 1 | 31 MB | 14 MB | 1 | 38371 | +| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4442 | +| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 15153 | +| opensearch | the frozen tier | 1 | 41 MB | 37 MB | 1 | 103444 | +| opentelemetry-collector | the frozen tier | 1 | 430 MB | 168 MB | 1 | 467047 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1330 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4099 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5919 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 128 | +| recommendation | the frozen tier | 1 | 9 MB | 1 MB | 1 | 2946 | +| valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2794 | +| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 5016 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/run.json b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/run.json new file mode 100644 index 0000000..3404742 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235318Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-25T23:53:20.582Z","status":"dry_run","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"dry-run: Hot tier: 4 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 4.6 GB within 168 hours, with 441 MB a day of counted lines moved to the frozen tier."} diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/branch-log.txt b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/branch-log.txt new file mode 100644 index 0000000..80387fc --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/branch-log.txt @@ -0,0 +1,4 @@ +0faf7e2 chore(log10x): hot-tier run, hot held under 2 MB +22756de chore(log10x): hot-tier run, hot held under 2 MB +89f7913 seed: the policy files from log10x_advise_install +ddd815e policy.yaml from log10x_setup_recurring diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/exit b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/exit new file mode 100644 index 0000000..573541a --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/exit @@ -0,0 +1 @@ +0 diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/report.txt b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/report.txt new file mode 100644 index 0000000..c27aa83 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/report.txt @@ -0,0 +1,61 @@ +[tenx-recur] policy loaded: target=30% services=all lookback=24h min_delta=2pp +[tenx-recur] Hot tier: 4 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 4.6 GB within 168 hours, with 442 MB a day of counted lines moved to the frozen tier. +[tenx-recur] Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 0 on hot, 1 MB in 1 off hot, outside the size. +[tenx-recur] A kept line stays on hot 168 hours (ILM policy tenx-keep (rollover max_age 1d + frozen min_age 6d)); the tier_down write index is off hot, so moved lines take no room on hot while they wait for their mount. +[tenx-recur] Each counted byte takes 1.11 bytes of store kept, 1.11 as tier_down (measured on the indices on hot). +[tenx-recur] The Receiver's statements send 1 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +[tenx-recur] The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 589 MB a day on hot whatever the limits; the size holds down to that floor. +[tenx-recur] Hot is above the size now; the allowance is held at 52% until it drains, which takes up to 168 hours as the indices on hot age out. +# Hot-tier run: 2026-10-25T23:53:20.000Z + +Hot tier: 4 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 4.6 GB within 168 hours, with 442 MB a day of counted lines moved to the frozen tier. + +- Outside the Receiver: 0 MB on hot. The Receiver's kept lines: 4 MB in 1 indices; its tier_down lines not yet moved: 0 MB in 0 on hot, 1 MB in 1 off hot, outside the size. +- A kept line stays on hot 168 hours (ILM policy tenx-keep (rollover max_age 1d + frozen min_age 6d)); the tier_down write index is off hot, so moved lines take no room on hot while they wait for their mount. +- Each counted byte takes 1.11 bytes of store kept, 1.11 as tier_down (measured on the indices on hot). +- The Receiver's statements send 1 GB a day; 0 MB a day fits the size, so each statement keeps up to 0 MB a day and the noisiest give way first. +- The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) keeps about 589 MB a day on hot whatever the limits; the size holds down to that floor. +- Hot is above the size now; the allowance is held at 52% until it drains, which takes up to 168 hours as the indices on hot age out. +- Warning: 2 documents on hot carry no utf8Size, so their statements count as 0 bytes. The Receiver writes it when installed for the hot-tier run. +- Warning: The always-pass sample (5 lines a statement a window on every node, 50 more for an error statement over its limit) alone keeps about 589 MB a day on hot, more than the 0 MB a day that fits, so hot settles above 2 MB at about 4.6 GB. Raise the size, move the noisiest services to offload, or lower the sample. + +| Service | Destination | Nodes | Sent a day | Kept a day | Limit per statement | Total | +|---|---|---|---|---|---|---| +| accounting | the frozen tier | 1 | 4 MB | 4 MB | 1 | 12000 | +| ad | the frozen tier | 1 | 15 MB | 2 MB | 1 | 6819 | +| aws-eks-nodeagent | the frozen tier | 1 | 0 MB | 0 MB | 1 | 489 | +| aws-node | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1311 | +| aws-vpc-cni-init | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2373 | +| cart | the frozen tier | 1 | 33 MB | 2 MB | 1 | 5840 | +| coredns | the frozen tier | 1 | 5 MB | 3 MB | 1 | 7790 | +| currency | the frozen tier | 1 | 0 MB | 0 MB | 1 | 568 | +| email | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2153 | +| flagd | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2766 | +| flagd-ui | the frozen tier | 1 | 0 MB | 0 MB | 1 | 928 | +| fluentd-10x | the frozen tier | 1 | 10 MB | 6 MB | 1 | 17323 | +| fraud-detection | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2466 | +| frontend | the frozen tier | 1 | 8 MB | 8 MB | 1 | 21258 | +| frontend-proxy | the frozen tier | 1 | 22 MB | 22 MB | 1 | 61819 | +| grafana | the frozen tier | 1 | 239 MB | 237 MB | 1 | 659486 | +| grafana-sc-alerts | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4091 | +| grafana-sc-dashboard | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5787 | +| grafana-sc-datasources | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4989 | +| init-config | the frozen tier | 1 | 1 MB | 1 MB | 1 | 3647 | +| jaeger | the frozen tier | 1 | 13 MB | 13 MB | 1 | 36556 | +| kafka | the frozen tier | 1 | 147 MB | 48 MB | 1 | 132143 | +| kube-proxy | the frozen tier | 1 | 31 MB | 14 MB | 1 | 38198 | +| load-generator | the frozen tier | 1 | 2 MB | 2 MB | 1 | 4418 | +| metrics-server | the frozen tier | 1 | 5 MB | 5 MB | 1 | 15071 | +| opensearch | the frozen tier | 1 | 41 MB | 37 MB | 1 | 102948 | +| opentelemetry-collector | the frozen tier | 1 | 432 MB | 168 MB | 1 | 466624 | +| payment | the frozen tier | 1 | 0 MB | 0 MB | 1 | 1323 | +| postgresql | the frozen tier | 1 | 1 MB | 1 MB | 1 | 4077 | +| prometheus-server | the frozen tier | 1 | 2 MB | 2 MB | 1 | 5888 | +| quote | the frozen tier | 1 | 0 MB | 0 MB | 1 | 127 | +| recommendation | the frozen tier | 1 | 9 MB | 1 MB | 1 | 2943 | +| valkey-cart | the frozen tier | 1 | 1 MB | 1 MB | 1 | 2779 | +| `*` (any other service) | the frozen tier | 1 | | 2 MB | 1 | 4989 | + +Limits and totals are bytes per 4-minute window per node, counted the way the Receiver counts them (utf8Size). Error lines pass 50 over a limit per statement per node per window. + +[tenx-recur] actions: diff --git a/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/run.json b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/run.json new file mode 100644 index 0000000..358b14f --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/ticks/20261025T235320Z/run.json @@ -0,0 +1 @@ +{"ts":"2026-10-25T23:53:22.251Z","status":"applied","projected_savings_pct":0,"delta_patterns":0,"delta_pp":0,"message":"applied: Hot tier: 4 MB of 2 MB (0.002 GB set in the policy) on 1 hot node; at these limits it settles at about 4.6 GB within 168 hours, with 442 MB a day of counted lines moved to the frozen tier.","history_path":"/var/folders/45/5yzkj5nn28g5p4qv4b_pg81m0000gn/T/log10x-recur-history.jsonl"} diff --git a/elastic-hot-tier-budget/install-e2e/results/timeline.jsonl b/elastic-hot-tier-budget/install-e2e/results/timeline.jsonl new file mode 100644 index 0000000..597f497 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/results/timeline.jsonl @@ -0,0 +1,13 @@ +{"event":"rendered","at":"2026-10-25T23:46:21Z","epoch":1792971981,"hot_tier_gb":"0.002"} +{"event":"policy_init","at":"2026-10-25T23:46:21Z","epoch":1792971981} +{"event":"es_ready","at":"2026-10-25T23:47:05Z","epoch":1792972025,"version":"9.5.5"} +{"event":"es_setup_done","at":"2026-10-25T23:47:09Z","epoch":1792972029} +{"event":"receiver_ready","at":"2026-10-25T23:47:13Z","epoch":1792972033} +{"event":"feed_start","at":"2026-10-25T23:47:31Z","epoch":1792972051,"speed":8,"lines_per_s":56} +{"event":"tick","at":"2026-10-25T23:50:22Z","epoch":1792972222,"dir":"ticks/20261025T235021Z","exit":0,"dry_run":true} +{"event":"tick","at":"2026-10-25T23:50:30Z","epoch":1792972230,"dir":"ticks/20261025T235028Z","exit":0,"dry_run":false} +{"event":"policy_synced","at":"2026-10-25T23:50:35Z","epoch":1792972235,"commit":"22756de"} +{"event":"tick","at":"2026-10-25T23:53:20Z","epoch":1792972400,"dir":"ticks/20261025T235318Z","exit":0,"dry_run":true} +{"event":"tick","at":"2026-10-25T23:53:22Z","epoch":1792972402,"dir":"ticks/20261025T235320Z","exit":0,"dry_run":false} +{"event":"feed_stop","at":"2026-10-25T23:53:28Z","epoch":1792972408} +{"event":"down","at":"2026-10-25T23:53:32Z","epoch":1792972412} diff --git a/elastic-hot-tier-budget/install-e2e/run.sh b/elastic-hot-tier-budget/install-e2e/run.sh new file mode 100755 index 0000000..eea6f72 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/run.sh @@ -0,0 +1,106 @@ +#!/bin/bash +# E100/E101 install proof. Subcommands, in the order the README ran them: +# render render.mjs: the recipe (advise_retriever), the Receiver (advise_install), +# policy.yaml (setup_recurring, the local policy checkout as its config plane) +# policy-init a local bare repo as the policy remote: policy.yaml on main, the seed files +# from advise_install's ConfigMap under log10x/policy/ on log10x-policy +# up Elasticsearch hot + warm/frozen, the snapshot volume owned by uid 1000, +# setup-es.mjs (licence, repository, the recipe), then Fluent Bit and the Receiver +# feed start | stop the OTel sample replay at 7 lines a second x speed (forward_send.py) +# tick [--dry-run] tenx-recur on policy.yaml against the cluster and the local policy repo +# sync copy the four files from log10x-policy into the mounted policy dir, restart the Receiver +# state aliases, indices, doc counts and shard nodes +# down remove every e100 container and volume (docker compose -p e100 down -v) +set -euo pipefail +HERE=$(cd "$(dirname "$0")" && pwd) +export RIG_DIR=$HERE/../rig +export RESULTS=${RESULTS:-$HERE/results} +export RIG_DATA=$RESULTS/data +export MCP=${MCP:?log10x-mcp package} +export ENGINE_IMAGE=${ENGINE_IMAGE:-ghcr.io/log-10x/edge-10x:1.1.150} +POLICY=$RESULTS/policy-git +T=$RESULTS/timeline.jsonl +mkdir -p "$RESULTS" "$RIG_DATA" +mark() { echo "{\"event\":\"$1\",\"at\":\"$(date -u +%FT%TZ)\",\"epoch\":$(date +%s)${2:+,$2}}" >> "$T"; } +compose() { docker compose -p e100 -f "$HERE/compose.yaml" -f "$RESULTS/compose.receiver.yaml" "$@"; } +es() { curl -s -m 30 "http://127.0.0.1:9200$1"; } +git_local() { git -c user.name='log10x hot-tier run (e100 rig)' -c user.email=install@log10x.com "$@"; } +case "${1:-}" in + render) + gb=${2:?hot_tier_gb} + containers=$(python3 -c "import json;print(','.join(sorted({json.loads(l)['container_name'].lstrip('/') for l in open('$RESULTS/replay/source.jsonl')})))") + node "$HERE/render.mjs" "$RESULTS" "$gb" "$POLICY/checkout" "$containers" > "$RESULTS/render.out" + mark rendered "\"hot_tier_gb\":\"$gb\"" + ;; + policy-init) + rm -rf "$POLICY" && mkdir -p "$POLICY" + git init -q --bare "$POLICY/origin.git" + git clone -q "$POLICY/origin.git" "$POLICY/checkout" 2>/dev/null + cd "$POLICY/checkout" + git checkout -q -b main + cp "$RESULTS/policy.yaml" policy.yaml && git add policy.yaml && git_local commit -q -m "policy.yaml from log10x_setup_recurring" && git push -q origin main + git checkout -q -b log10x-policy + mkdir -p log10x/policy + python3 -c " +import json +d=json.load(open('$RESULTS/receiver.json'))['policy_configmap']['data'] +for k,v in d.items(): open('log10x/policy/'+k,'w').write(v if v.endswith('\n') else v+'\n')" + git add log10x/policy && git_local commit -q -m "seed: the policy files from log10x_advise_install" && git push -q origin log10x-policy + git checkout -q main + mark policy_init + ;; + up) + compose up -d es-hot es-warm-frozen + for i in $(seq 1 90); do [ "$( (es '/_cat/nodes?h=name' || true) | grep -c . || true)" = 2 ] && break; sleep 4; done + docker run -d --name e100-chown -v e100_es-snapshots:/s alpine:latest chown 1000:0 /s >/dev/null + docker wait e100-chown >/dev/null; docker rm -f -v e100-chown >/dev/null + mark es_ready "\"version\":\"$(es / | python3 -c 'import json,sys;print(json.load(sys.stdin)["version"]["number"])')\"" + node "$HERE/setup-es.mjs" "$RESULTS" | tee "$RESULTS/setup-es.out" + mark es_setup_done + compose up -d fluent-bit tenx + for i in $(seq 1 120); do docker logs e100-tenx 2>&1 | grep -q -E 'Forward protocol server listening|Writing TenXObject field' && break; sleep 3; done + mark receiver_ready + ;; + feed) + case "${2:-}" in + start) + docker run -d --name e100-feed --network e100_default -v "$RIG_DIR:/rig:ro" -v "$RESULTS/replay:/replay:ro" python:3.12-slim \ + python3 -u /rig/forward_send.py replay /replay/source.jsonl "${3:?speed}" fluent-bit 24224 "${4:-0}" >/dev/null + mark feed_start "\"speed\":$3,\"lines_per_s\":$(python3 -c "print(7*$3)")" ;; + stop) docker rm -f -v e100-feed >/dev/null 2>&1 || true; mark feed_stop ;; + esac + ;; + tick) + stamp=$(date -u +%Y%m%dT%H%M%SZ); out=$RESULTS/ticks/$stamp; mkdir -p "$out" + set +e + LOG10X_ES_URL=http://127.0.0.1:9200 LOG10X_GITOPS_REPO_PATH=$POLICY/checkout \ + GIT_CONFIG_COUNT=2 GIT_CONFIG_KEY_0=user.name GIT_CONFIG_VALUE_0='log10x hot-tier run (e100 rig)' GIT_CONFIG_KEY_1=user.email GIT_CONFIG_VALUE_1=install@log10x.com \ + node "$MCP/build/bin/tenx-recur.js" --policy "$RESULTS/policy.yaml" --verbose "${@:2}" > "$out/run.json" 2> "$out/report.txt" + rc=$?; set -e + echo "$rc" > "$out/exit" + git -C "$POLICY/checkout" fetch -q origin '+refs/heads/log10x-policy:refs/remotes/origin/log10x-policy' + git -C "$POLICY/checkout" log --format='%h %s' origin/log10x-policy > "$out/branch-log.txt" + mark tick "\"dir\":\"ticks/$stamp\",\"exit\":$rc,\"dry_run\":$([ "${2:-}" = --dry-run ] && echo true || echo false)" + echo "exit $rc ($out)" + ;; + sync) + git -C "$POLICY/checkout" fetch -q origin '+refs/heads/log10x-policy:refs/remotes/origin/log10x-policy' + for f in caps.csv actions.csv totals.csv; do git -C "$POLICY/checkout" show "origin/log10x-policy:log10x/policy/$f" > "$RESULTS/policy/$f"; done + docker restart e100-tenx >/dev/null + for i in $(seq 1 120); do docker logs --since 30s e100-tenx 2>&1 | grep -q -E 'Forward protocol server listening|Writing TenXObject field' && break; sleep 3; done + mark policy_synced "\"commit\":\"$(git -C "$POLICY/checkout" rev-parse --short origin/log10x-policy)\"" + ;; + state) + es '/_cat/aliases/tenx-*?v&h=alias,index,is_write_index' + es '/_cat/indices/*tenx-*?v&h=index,docs.count,store.size&s=index' + es '/_cat/shards/*tenx-*?v&h=index,prirep,state,node&s=index' + ;; + down) + docker rm -f -v e100-feed e100-chown >/dev/null 2>&1 || true + docker logs e100-tenx > "$RESULTS/e100-tenx.log" 2>&1 || true + docker logs e100-fluent-bit > "$RESULTS/e100-fluent-bit.log" 2>&1 || true + compose down -v + mark down + ;; + *) sed -n '2,15p' "$0"; exit 2;; +esac diff --git a/elastic-hot-tier-budget/install-e2e/setup-es.mjs b/elastic-hot-tier-budget/install-e2e/setup-es.mjs new file mode 100644 index 0000000..be65a35 --- /dev/null +++ b/elastic-hot-tier-budget/install-e2e/setup-es.mjs @@ -0,0 +1,45 @@ +// Prepares the cluster and applies the recipe's requests as render.mjs extracted them, unchanged. +// node setup-es.mjs +// Cluster preparation, not recipe: the self-managed trial licence (searchable snapshots need +// Enterprise features) and a filesystem snapshot repository registered under the name the +// recipe renders (`found-snapshots`, the repository Elastic Cloud provides). Then every request +// in recipe-requests.json in order, and a check that the tier_down write index sits on a warm node. +import { readFileSync, writeFileSync } from 'fs'; +import { join, resolve } from 'path'; + +const ES = process.env.ES_URL ?? 'http://127.0.0.1:9200'; +const dir = resolve(process.argv[2] ?? '.'); +const requests = JSON.parse(readFileSync(join(dir, 'recipe-requests.json'), 'utf8')); + +async function call(method, path, body) { + const r = await fetch(ES + path, { method, headers: { 'content-type': 'application/json' }, body: body === undefined ? undefined : JSON.stringify(body) }); + const text = await r.text(); + return { method, path, status: r.status, body: text.slice(0, 2000) }; +} +const json = async (path) => (await fetch(ES + path)).json(); + +const prep = [ + await call('POST', '/_license/start_trial?acknowledge=true'), + await call('PUT', '/_snapshot/found-snapshots', { type: 'fs', settings: { location: '/snapshots' } }), +]; +const applied = []; +for (const q of requests) applied.push(await call(q.method, q.path, q.body)); + +const roles = await json('/_nodes?filter_path=nodes.*.name,nodes.*.roles'); +const rolesOf = Object.fromEntries(Object.values(roles.nodes).map((n) => [n.name, n.roles])); +let placement = []; +for (let i = 0; i < 60; i++) { + placement = (await json('/_cat/shards/tenx-*?format=json&h=index,shard,prirep,state,node')).map((x) => ({ ...x, roles: rolesOf[x.node] ?? null })); + if (placement.length >= 2 && placement.every((x) => x.state === 'STARTED')) break; + await new Promise((r) => setTimeout(r, 2000)); +} +const td = placement.find((x) => x.index === 'tenx-tierdown-000001'); +const tdOnWarm = !!td && (td.roles ?? []).includes('data_warm'); +writeFileSync(join(dir, 'es-setup.json'), JSON.stringify({ es: ES, prep, applied, placement, tier_down_write_index_on_warm: tdOnWarm }, null, 1) + '\n'); +for (const x of [...prep, ...applied]) console.log(x.status, x.method, x.path); +console.log('placement', JSON.stringify(placement.map((x) => `${x.index}@${x.node}(${(x.roles ?? []).join('+')})`))); +const bad = applied.filter((x) => x.status >= 300).concat(prep.filter((x) => x.status >= 300 && !(x.path.includes('start_trial') && x.status === 403))); +if (bad.length || !tdOnWarm) { + console.error('FAILED:', bad.map((x) => `${x.status} ${x.method} ${x.path} ${x.body.slice(0, 300)}`).join('\n'), tdOnWarm ? '' : 'tier_down write index not on a warm node'); + process.exit(1); +}