From 4d25d552c4ec28a384cfbf197c30450280102216 Mon Sep 17 00:00:00 2001 From: "D. Dessy" Date: Thu, 29 Oct 2026 20:06:04 +0100 Subject: [PATCH] Slow Eventgen, set the host in Eventgen, stop installing the CIM app (SA-S4R 2.0.1) Eventgen: each stanza now emits its batch every 10 s instead of every 1 s, so the baseline is about 300 events/min (was ~3,000) and the attack stream adds about 150 events/min (was ~1,500). Batch sizes are unchanged, so the product/cart ratio and the attack-to-baseline ratio stay the same. Host: the shop host is set in eventgen.conf [global] and inherited by every sample stanza. This replaces the index-time host transform from #38, which was built on a wrong conclusion: the earlier "Eventgen ignores host" test copied the file into the running app without reloading Eventgen's config in splunkd, so the old value kept being used. With the reload, host works per stanza and from [global]. The transform, its props stanzas and the claim in the docs are gone, and a host change no longer needs a Splunk restart. The docs now say that a hand-edited eventgen.conf needs POST .../conf-eventgen/_reload. CIM: compose.yml no longer installs the Splunk CIM app (Splunkbase 1621), which the workshop does not need. SA-S4R keeps what makes the data CIM-compliant: the CIM Web field names and aliases, the s4r_web_access eventtype with its web tag, and the default.meta exports. tests/s4r-cim-web.sh fails if the Splunkbase pin comes back. Version: neither the rate nor the host adds an interface, so SA-S4R goes 2.0.0 -> 2.0.1. Checked live: after a conf reload and a modinput restart the rate is ~55 events per 10 s; with the old transform unloaded the events still carry host=www.buttercupenterprises.com from Eventgen alone; with the CIM app disabled and so1 restarted, the S4R tools, the host and tag, the CIM-named fields and the tool policies all still work. Co-Authored-By: Claude Sonnet 5.5 --- SA-S4R/default/app.conf | 6 +++--- SA-S4R/default/eventgen.conf | 13 ++++++++++--- SA-S4R/default/props.conf | 14 +++----------- SA-S4R/default/transforms.conf | 7 ------- compose.yml | 3 +-- docs/poc/ARCHITECTURE.md | 2 +- docs/poc/CONFIGURATION.md | 1 - docs/poc/SPECS.md | 1 - docs/s4r/SA-S4R-APP.md | 21 +++++++++------------ docs/s4r/SPL-CATALOG.md | 4 ++-- tests/s4r-cim-web.sh | 28 +++++++++++++++------------- 11 files changed, 44 insertions(+), 56 deletions(-) diff --git a/SA-S4R/default/app.conf b/SA-S4R/default/app.conf index 13a8244..59343a7 100644 --- a/SA-S4R/default/app.conf +++ b/SA-S4R/default/app.conf @@ -3,12 +3,12 @@ [ui] is_visible = true label = Splunk4Rookies -description = Buttercup Enterprises workshop sample data (Eventgen access_combined traffic, lookups, dashboard assets). +description = Buttercup Enterprises workshop sample data (Eventgen access_combined traffic, lookups, workshop MCP tools, dashboard assets). [launcher] author = splunk-mcp contributors -description = Splunk4Rookies PoC: synthetic Buttercup web access logs via Eventgen, field extractions, and product_codes lookup for workshop Labs 3–7. -version = 2.0.0 +description = Splunk4Rookies PoC: synthetic Buttercup web access logs via Eventgen, field extractions, CIM-named fields, the product_codes lookup and the SA-S4R_* MCP tools for workshop Labs 3–7. +version = 2.0.1 [package] id = SA-S4R diff --git a/SA-S4R/default/eventgen.conf b/SA-S4R/default/eventgen.conf index 10c3eb1..197fefc 100644 --- a/SA-S4R/default/eventgen.conf +++ b/SA-S4R/default/eventgen.conf @@ -5,9 +5,16 @@ [global] debug = false +# The shop's virtual host: every sample stanza inherits it, so events get host=www.buttercupenterprises.com +# (without it Eventgen stamps its own 127.0.0.1). CIM Web dest, site and url_domain are aliases of host. +host = www.buttercupenterprises.com + +# Rate: every stanza emits `count` events each `interval` seconds, so the baseline is +# (34 + 16) / 10 s = ~300 events/min and the attack stream adds ~150 events/min. + # Product page views (~67% of events) — uid present, no action= [product.screen.sample] -interval = 1 +interval = 10 count = 34 randomizeCount = 0.8 index = main @@ -59,7 +66,7 @@ token.10.replacement = integer[150:960] # Cart actions (~33%) — action=view|addtocart|purchase|remove|changequantity [cart.do.sample] -interval = 1 +interval = 10 count = 16 randomizeCount = 0.8 index = main @@ -115,7 +122,7 @@ token.10.replacement = integer[150:960] # Saved search: S4R Validate NK Attack Traffic (last 15m) [attack.nk.purchase.sample] disabled = true -interval = 1 +interval = 10 count = 25 randomizeCount = 0.5 index = main diff --git a/SA-S4R/default/props.conf b/SA-S4R/default/props.conf index 0c882c3..bfd20e1 100644 --- a/SA-S4R/default/props.conf +++ b/SA-S4R/default/props.conf @@ -1,12 +1,3 @@ -[source::product.screen.sample] -TRANSFORMS-s4r_host = s4r_set_host - -[source::cart.do.sample] -TRANSFORMS-s4r_host = s4r_set_host - -[source::attack.nk.purchase.sample] -TRANSFORMS-s4r_host = s4r_set_host - [access_combined] SHOULD_LINEMERGE = false EXTRACT-action = action=(?[^&\s"]+) @@ -14,8 +5,9 @@ EXTRACT-product_id = product_id=(?[^&\s"]+) EXTRACT-uid = uid=(?[^&\s"]+) EXTRACT-JSESSIONID = JSESSIONID=(?[^&\s"]+) -# CIM Web mapping (tags.conf tags the events `web`). The s4r_set_host transform makes host the -# shop's virtual host, which is the site and destination. `action` stays the cart verb (view, purchase, ...): +# CIM-compliant field names for the Web model (tags.conf tags the events `web`). The CIM app itself is +# not installed by this repo; these make the data usable with it. Eventgen sets host to the shop's virtual +# host (eventgen.conf), which is the site and destination. `action` stays the cart verb (view, purchase, ...): # the workshop searches depend on it, so Web.action is not CIM-normalized. FIELDALIAS-cim_src = clientip AS src FIELDALIAS-cim_dest = host AS dest host AS site host AS url_domain diff --git a/SA-S4R/default/transforms.conf b/SA-S4R/default/transforms.conf index 0996d73..78bebbc 100644 --- a/SA-S4R/default/transforms.conf +++ b/SA-S4R/default/transforms.conf @@ -1,10 +1,3 @@ [product_codes] filename = product_codes.csv case_sensitive_match = false - -# Eventgen ignores its `host` setting with the default modinput output, so every event would -# carry host=127.0.0.1. Set the shop's virtual host at index time, only for the S4R samples. -[s4r_set_host] -REGEX = . -DEST_KEY = MetaData:Host -FORMAT = host::www.buttercupenterprises.com diff --git a/compose.yml b/compose.yml index af76e17..864e63c 100644 --- a/compose.yml +++ b/compose.yml @@ -17,13 +17,12 @@ services: # app/4353 = Config Explorer (optional UI utility) # app/7931 = Splunk MCP Server (required for /services/mcp) # app/7245 = Splunk AI Assistant (SPL / saia_* MCP tools) - # app/1621 = Splunk Common Information Model (Web data model for SA-S4R) # If a download fails (404, etc.), replace the /release/VERSION/ segment with a current # release for that app from https://splunkbase.splunk.com/app/ . # SA-S4R is this repo's package from the moving GitHub Release `latest` # (package-s4r.yml). The image downloads SPLUNK_APPS_URL only on the first # start of an empty so1-etc volume. - SPLUNK_APPS_URL: https://splunkbase.splunk.com/app/1924/release/8.2.2/download/,https://splunkbase.splunk.com/app/4353/release/1.8.26/download/,https://splunkbase.splunk.com/app/7931/release/2.1.0/download/,https://splunkbase.splunk.com/app/7245/release/2.3.6/download/,https://splunkbase.splunk.com/app/1621/release/8.8.0/download/,https://github.com/dd-Splunk/splunk-mcp/releases/download/latest/SA-S4R.spl + SPLUNK_APPS_URL: https://splunkbase.splunk.com/app/1924/release/8.2.2/download/,https://splunkbase.splunk.com/app/4353/release/1.8.26/download/,https://splunkbase.splunk.com/app/7931/release/2.1.0/download/,https://splunkbase.splunk.com/app/7245/release/2.3.6/download/,https://github.com/dd-Splunk/splunk-mcp/releases/download/latest/SA-S4R.spl TZ: ${TZ:?set TZ in config.env} volumes: - so1-var:/opt/splunk/var diff --git a/docs/poc/ARCHITECTURE.md b/docs/poc/ARCHITECTURE.md index bab567b..4b4527e 100644 --- a/docs/poc/ARCHITECTURE.md +++ b/docs/poc/ARCHITECTURE.md @@ -20,7 +20,7 @@ The [Model Context Protocol](https://modelcontextprotocol.io/) lets a client (Cl ## Splunkbase applications -`compose.yml` sets `SPLUNK_APPS_URL` to a comma-separated list of package download URLs. Installed from Splunkbase: **SA-Eventgen** (1924), **Config Explorer** (4353), **Splunk MCP Server** (7931), **Splunk AI Assistant** (7245), and the **Splunk Common Information Model** (1621, provides the `Web` data model SA-S4R maps to). **SA-S4R** is installed from this repo’s GitHub Release **`latest`** (`SA-S4R.spl`). Pin details and the full table live in [CONFIGURATION.md](CONFIGURATION.md#composeyml). +`compose.yml` sets `SPLUNK_APPS_URL` to a comma-separated list of package download URLs. Installed from Splunkbase: **SA-Eventgen** (1924), **Config Explorer** (4353), **Splunk MCP Server** (7931), and **Splunk AI Assistant** (7245). **SA-S4R** is installed from this repo’s GitHub Release **`latest`** (`SA-S4R.spl`). Pin details and the full table live in [CONFIGURATION.md](CONFIGURATION.md#composeyml). You need valid **Splunkbase** credentials in **`.env`** (or the process environment): **`SPLUNKBASE_USER`** and **`SPLUNKBASE_PASS`**. diff --git a/docs/poc/CONFIGURATION.md b/docs/poc/CONFIGURATION.md index d937a0b..f5ea4fc 100644 --- a/docs/poc/CONFIGURATION.md +++ b/docs/poc/CONFIGURATION.md @@ -8,7 +8,6 @@ | 4353 | Config Explorer (optional UI utility) | 1.8.26 (Splunkbase lists Splunk up to 10.4; runs on 10.6 but not declared) | | 7931 | Splunk MCP Server (required for `/services/mcp`) | 2.1.0 (token API changed in 2.1; see [Encrypted bearer token](#encrypted-bearer-token-this-repo)) | | 7245 | Splunk AI Assistant | 2.3.6 | -| 1621 | Splunk Common Information Model (provides the `Web` data model for SA-S4R) | 8.8.0 | | — | SA-S4R (this repo) | GitHub Release [`latest`](https://github.com/dd-Splunk/splunk-mcp/releases/download/latest/SA-S4R.spl) | Check current Splunkbase releases: `https://splunkbase.splunk.com/api/v1/app//release/` (first entry is latest). Update the `/release/VERSION/` segment in **`compose.yml`** when bumping. diff --git a/docs/poc/SPECS.md b/docs/poc/SPECS.md index d953ab9..47ebf03 100644 --- a/docs/poc/SPECS.md +++ b/docs/poc/SPECS.md @@ -36,7 +36,6 @@ MCP user must **not** be `admin`. The **`ai`** SPL command is available to all u - SA-Eventgen (1924) - Config Explorer (4353) - Splunk AI Assistant (7245) -- Splunk Common Information Model (1621, `Web` data model for SA-S4R) Splunk AI Toolkit (2890) and Python for Scientific Computing (2882) are **out of scope** unless installed manually. diff --git a/docs/s4r/SA-S4R-APP.md b/docs/s4r/SA-S4R-APP.md index b18a9ec..1712d9c 100644 --- a/docs/s4r/SA-S4R-APP.md +++ b/docs/s4r/SA-S4R-APP.md @@ -28,7 +28,7 @@ SA-S4R/ # tracked in git │ ├── restmap.conf # /s4r_workshop_mode for API MCP tools │ ├── savedsearches.conf # Governed SPL backing SPL MCP tools │ ├── s4r_mcp_tools.json # Batch-replace payload for POST /services/mcp_tools -│ ├── tags.conf # web tag → CIM Web data model +│ ├── tags.conf # web tag (CIM Web) │ └── transforms.conf # product_codes lookup (file: lookups/product_codes.csv) ├── lookups/ │ └── product_codes.csv # Demo lookup for Lab 5 @@ -65,7 +65,7 @@ Splunk apps split **shipped baseline** (`default/`) from **instance-specific ove **Rules (Splunk and this repo):** 1. **Splunk Web, Settings → Knowledge, nav editor, field extractor, Dashboard Studio saves** — must land under **`SA-S4R/local/`** only. **Never** save customizations into **`default/`** (Splunk will overwrite shipped objects on upgrade/reinstall). -2. **Agents and contributors** — do not add workshop dashboards, nav tabs, or Lab 4 field extractions under **`SA-S4R/default/`** in git. **Exceptions** (maintainer-owned in **`default/`**): MCP packaging (`savedsearches.conf` for tool backing, `s4r_mcp_tools.json`, REST handler) — [MCP-TOOLS.md](MCP-TOOLS.md) — and the sourcetype baseline: field extractions, the CIM Web mapping (`props.conf`, `eventtypes.conf`, `tags.conf`), the host transform and the `s4r_user` role. Workshop UI setup is in [Workshop objects in `local/`](#workshop-objects-in-local) and [DASHBOARD.md](DASHBOARD.md). +2. **Agents and contributors** — do not add workshop dashboards, nav tabs, or Lab 4 field extractions under **`SA-S4R/default/`** in git. **Exceptions** (maintainer-owned in **`default/`**): MCP packaging (`savedsearches.conf` for tool backing, `s4r_mcp_tools.json`, REST handler) — [MCP-TOOLS.md](MCP-TOOLS.md) — and the sourcetype baseline: field extractions, the CIM-compliant field names and `web` tag (`props.conf`, `eventtypes.conf`, `tags.conf`), the Eventgen host and rate, and the `s4r_user` role. Workshop UI setup is in [Workshop objects in `local/`](#workshop-objects-in-local) and [DASHBOARD.md](DASHBOARD.md). 3. **Packaging** — **`package-s4r.yml`** excludes **`local/`** (entire directory) so instance-specific content is not published in **`SA-S4R.spl`**. Git ignores **`SA-S4R/local/**`** (see **`.gitignore`**). 4. **Install** — Compose does not bind-mount the app tree. The package lands in **`so1-etc`**. Workshop UI and Eventgen overrides still belong in that installed app’s **`local/`**. @@ -139,9 +139,9 @@ The **Buttercup Enterprises** workshop tab and Dashboard Studio view live under **`default/props.conf`** extracts `action`, `product_id`, `uid`, and `JSESSIONID` from the request line so workshop SPL such as `action=purchase` works without manual field extraction. -### CIM Web mapping +### CIM-compliant fields (CIM Web) -**`default/props.conf`**, **`eventtypes.conf`** and **`tags.conf`** map `access_combined` to the CIM **Web** data model. The CIM app (Splunkbase 1621, pinned in **`compose.yml`**) provides the model, and **`default.meta`** exports the eventtype and tag to `system` so it can see them. The app ships the mapping but no acceleration. +**`default/props.conf`**, **`eventtypes.conf`** and **`tags.conf`** give `access_combined` the field names and the `web` tag of the CIM **Web** data model, so any Splunk with the Splunk Common Information Model app (Splunkbase 1621) can use the data as is, and **`default.meta`** exports the eventtype and tag to `system` for that. **This repo does not install the CIM app**: the workshop does not need it, and the S4R searches use plain SPL. Install it yourself if you want the CIM `Web` model or CIM-based apps. | Web field | Source | | --------- | ------ | @@ -153,15 +153,11 @@ The **Buttercup Enterprises** workshop tab and Dashboard Studio view live under | `vendor_product` | constant `Buttercup Enterprises Web Shop` | | `status`, `uri_path`, `uri_query`, `bytes`, `user` | already extracted by the stock sourcetype | -**Host.** Eventgen ignores its own `host` setting with the default modinput output (its spec limits it to `splunkstream` and `syslogout`), so every event would carry `host=127.0.0.1`. **`transforms.conf`** therefore sets `host` to `www.buttercupenterprises.com` at index time, and **`props.conf`** applies it only to the three S4R sample sources (`product.screen.sample`, `cart.do.sample`, `attack.nk.purchase.sample`), never to other `access_combined` data. `host` is a real indexed field, so `tstats ... where host=...` works. Changing it needs a Splunk restart, and events indexed earlier keep their old host. +**Host.** **`eventgen.conf`** sets `host = www.buttercupenterprises.com` in `[global]`, so every sample stanza (including the attack stream) inherits it and events no longer carry Eventgen's own `127.0.0.1`. `host` is a real indexed field, so `tstats ... where host=...` works, and `dest`, `site` and `url_domain` are aliases of it. Events indexed before a change keep their old host. -**Known deviation:** `action` stays the cart verb (`view`, `addtocart`, `purchase`, `remove`, `changequantity`) because the workshop searches filter on it. `Web.action` therefore holds those verbs instead of CIM values such as `allowed` and `blocked`. `product_id` is a custom field, not part of the model. +**Known deviation:** `action` stays the cart verb (`view`, `addtocart`, `purchase`, `remove`, `changequantity`) because the workshop searches filter on it. With the CIM app installed, `Web.action` would hold those verbs instead of CIM values such as `allowed` and `blocked`. `product_id` is a custom field, not part of CIM. -Query it with: - -```spl -| tstats count from datamodel=Web where Web.status>=400 by Web.src, Web.status -``` +With the CIM app installed, `| tstats count from datamodel=Web by Web.dest, Web.status` works on this data. **`platform`** (Lab 4) belongs in the installed app’s **`local/props.conf`** — see [Workshop objects in `local/`](#workshop-objects-in-local). Agents/MCP still use inline `rex` per [SPL-CATALOG.md](SPL-CATALOG.md). @@ -187,7 +183,8 @@ The stanza **`[product_codes]`** in **`default/transforms.conf`** points at the **Splunk Web:** save all knowledge objects, nav changes, field extractions, and dashboards under **`local/`** only — never **`default/`** (see **`default/` vs `local/`** above). - Edit **`samples/action.txt`**, **`status.txt`**, or **`useragent.txt`** to change categorical choices. -- Tune **`interval`**, **`count`**, and **`randomizeCount`** per stanza in `eventgen.conf`. +- Tune **`interval`**, **`count`**, and **`randomizeCount`** per stanza in `eventgen.conf`. Each stanza emits `count` events every `interval` seconds: the default is one batch every 10 s, so about 300 events/min for the baseline (`product.screen` 34 + `cart.do` 16 per batch) and about 150 events/min more while the attack stream is on. +- After editing `eventgen.conf` by hand on a running stack, run `POST /servicesNS/nobody/SA-S4R/configs/conf-eventgen/_reload`, then disable and enable the `SA-Eventgen` modinput. Copying the file alone is not enough, because splunkd keeps the old config in memory. - Adjust the **`product.screen`** / **`cart.do`** ratio via each stanza’s **`count`**. ### Workshop modes: infrastructure vs NK attack diff --git a/docs/s4r/SPL-CATALOG.md b/docs/s4r/SPL-CATALOG.md index 690b292..157f475 100644 --- a/docs/s4r/SPL-CATALOG.md +++ b/docs/s4r/SPL-CATALOG.md @@ -30,8 +30,8 @@ index=main sourcetype=access_combined | Cart `action` values | `view`, `addtocart`, `purchase`, `remove`, `changequantity` | | Extracted fields | `action`, `product_id`, `uid`, `JSESSIONID` (`SA-S4R/default/props.conf`) | | Geo IP field | `clientip` (confirm casing in Search) | -| Host | `host=www.buttercupenterprises.com` on every S4R event (set at index time; Eventgen's own `host` setting has no effect with modinput output) | -| CIM Web model | Events are tagged `web` and mapped to the CIM `Web` data model, for example `\| tstats count from datamodel=Web by Web.dest, Web.status` ([SA-S4R-APP.md § CIM Web mapping](SA-S4R-APP.md#cim-web-mapping)). `Web.action` holds the cart verb, not `allowed` or `blocked` | +| Host | `host=www.buttercupenterprises.com` on every S4R event (set in `eventgen.conf` `[global]`) | +| CIM fields | Field names and the `web` tag follow CIM Web (`src`, `dest`, `status`, `http_method`, …); the CIM app is not installed ([SA-S4R-APP.md § CIM-compliant fields](SA-S4R-APP.md#cim-compliant-fields-cim-web)) | | Success | `status` 2xx | | Failure (workshop) | `status>=400` | | Failed purchase | `action=purchase` AND `status>=400` | diff --git a/tests/s4r-cim-web.sh b/tests/s4r-cim-web.sh index 4d2979b..01c447f 100755 --- a/tests/s4r-cim-web.sh +++ b/tests/s4r-cim-web.sh @@ -28,15 +28,16 @@ grep -q '^EXTRACT-action = ' "${APP}/default/props.conf" || die "props.conf lost if grep -Eq '^EVAL-(dest|site|url_domain) = ' "${APP}/default/props.conf"; then die "props.conf hard-codes dest, site or url_domain; they come from host" fi -# Eventgen ignores `host` with modinput output, so host is set at index time for each sample source -grep -Eq '^FORMAT = host::www\.buttercupenterprises\.com$' "${APP}/default/transforms.conf" \ - || die "transforms.conf does not set the shop host" -while IFS= read -r src; do - awk -v s="[source::${src}]" '$0 == s {f=1; next} /^\[/ {f=0} f && /^TRANSFORMS-s4r_host = s4r_set_host$/ {ok=1} END {exit !ok}' \ - "${APP}/default/props.conf" || die "props.conf does not set host for source ${src}" -done < <(sed -n 's/^\[\(.*\.sample\)\]$/\1/p' "${APP}/default/eventgen.conf") - -# the tag that puts the events in the Web data model, and the exports that let Splunk_SA_CIM see it +# host comes from Eventgen ([global] reaches every sample stanza), so no index-time override is needed +grep -Eq '^host = www\.buttercupenterprises\.com$' "${APP}/default/eventgen.conf" \ + || die "eventgen.conf [global] does not set the shop host" +awk '$0 == "[global]" {f=1; next} /^\[/ {f=0} f && /^host = /{ok=1} END {exit !ok}' "${APP}/default/eventgen.conf" \ + || die "host is not set in eventgen.conf [global]" +if grep -q 's4r_set_host' "${APP}/default/props.conf" "${APP}/default/transforms.conf"; then + die "the index-time host override is back; set host in eventgen.conf instead" +fi + +# the tag that marks the events as CIM Web, and the exports that let a CIM install see it grep -Eq '^search = .*sourcetype=access_combined' "${APP}/default/eventtypes.conf" \ || die "eventtypes.conf does not match access_combined" etype="$(sed -n 's/^\[\(s4r_[a-z_]*\)\]$/\1/p' "${APP}/default/eventtypes.conf" | head -1)" @@ -48,8 +49,9 @@ for stanza in eventtypes tags; do "${APP}/metadata/default.meta" || die "default.meta does not export ${stanza} to system" done -# the Web data model ships in the CIM app, so compose must install it -grep -Eq 'SPLUNK_APPS_URL:.*splunkbase\.splunk\.com/app/1621/release/[0-9.]+/download/' compose.yml \ - || die "compose.yml does not install the Splunk CIM app (Splunkbase 1621)" +# this repo does not install the CIM app: the mapping stays, the Splunkbase pin must not come back +if grep -Eq 'splunkbase\.splunk\.com/app/1621/' compose.yml; then + die "compose.yml installs the CIM app (Splunkbase 1621); the S4R stack does not need it" +fi -echo "OK: SA-S4R maps access_combined to CIM Web and compose installs the CIM app" +echo "OK: SA-S4R keeps CIM Web field names and the web tag, and the stack does not install the CIM app"