Slow Eventgen, set the host in Eventgen, stop installing the CIM app (SA-S4R 2.0.1)

SA-S4R/default/app.conf

@@ -3,12 +3,12 @@
 [ui]
 is_visible = true
 label = Splunk4Rookies
-description = Buttercup Enterprises workshop sample data (Eventgen access_combined traffic, lookups, dashboard assets).
+description = Buttercup Enterprises workshop sample data (Eventgen access_combined traffic, lookups, workshop MCP tools, dashboard assets).
 
 [launcher]
 author = splunk-mcp contributors
-description = Splunk4Rookies PoC: synthetic Buttercup web access logs via Eventgen, field extractions, and product_codes lookup for workshop Labs 3–7.
-version = 2.0.0
+description = Splunk4Rookies PoC: synthetic Buttercup web access logs via Eventgen, field extractions, CIM-named fields, the product_codes lookup and the SA-S4R_* MCP tools for workshop Labs 3–7.
+version = 2.0.1
 
 [package]
 id = SA-S4R

SA-S4R/default/eventgen.conf

@@ -5,9 +5,16 @@
 [global]
 debug = false
 
+# The shop's virtual host: every sample stanza inherits it, so events get host=www.buttercupenterprises.com
+# (without it Eventgen stamps its own 127.0.0.1). CIM Web dest, site and url_domain are aliases of host.
+host = www.buttercupenterprises.com
+
+# Rate: every stanza emits `count` events each `interval` seconds, so the baseline is
+# (34 + 16) / 10 s = ~300 events/min and the attack stream adds ~150 events/min.
+
 # Product page views (~67% of events) — uid present, no action=
 [product.screen.sample]
-interval = 1
+interval = 10
 count = 34
 randomizeCount = 0.8
 index = main
@@ -59,7 +66,7 @@ token.10.replacement = integer[150:960]
 
 # Cart actions (~33%) — action=view|addtocart|purchase|remove|changequantity
 [cart.do.sample]
-interval = 1
+interval = 10
 count = 16
 randomizeCount = 0.8
 index = main
@@ -115,7 +122,7 @@ token.10.replacement = integer[150:960]
 # Saved search: S4R Validate NK Attack Traffic (last 15m)
 [attack.nk.purchase.sample]
 disabled = true
-interval = 1
+interval = 10
 count = 25
 randomizeCount = 0.5
 index = main

SA-S4R/default/props.conf

@@ -1,21 +1,13 @@
-[source::product.screen.sample]
-TRANSFORMS-s4r_host = s4r_set_host
-
-[source::cart.do.sample]
-TRANSFORMS-s4r_host = s4r_set_host
-
-[source::attack.nk.purchase.sample]
-TRANSFORMS-s4r_host = s4r_set_host
-
 [access_combined]
 SHOULD_LINEMERGE = false
 EXTRACT-action = action=(?<action>[^&\s"]+)
 EXTRACT-product_id = product_id=(?<product_id>[^&\s"]+)
 EXTRACT-uid = uid=(?<uid>[^&\s"]+)
 EXTRACT-JSESSIONID = JSESSIONID=(?<JSESSIONID>[^&\s"]+)
 
-# CIM Web mapping (tags.conf tags the events `web`). The s4r_set_host transform makes host the
-# shop's virtual host, which is the site and destination. `action` stays the cart verb (view, purchase, ...):
+# CIM-compliant field names for the Web model (tags.conf tags the events `web`). The CIM app itself is
+# not installed by this repo; these make the data usable with it. Eventgen sets host to the shop's virtual
+# host (eventgen.conf), which is the site and destination. `action` stays the cart verb (view, purchase, ...):
 # the workshop searches depend on it, so Web.action is not CIM-normalized.
 FIELDALIAS-cim_src = clientip AS src
 FIELDALIAS-cim_dest = host AS dest host AS site host AS url_domain

SA-S4R/default/transforms.conf

@@ -1,10 +1,3 @@
 [product_codes]
 filename = product_codes.csv
 case_sensitive_match = false
-
-# Eventgen ignores its `host` setting with the default modinput output, so every event would
-# carry host=127.0.0.1. Set the shop's virtual host at index time, only for the S4R samples.
-[s4r_set_host]
-REGEX = .
-DEST_KEY = MetaData:Host
-FORMAT = host::www.buttercupenterprises.com

compose.yml

@@ -17,13 +17,12 @@ services:
       #   app/4353  = Config Explorer (optional UI utility)
       #   app/7931  = Splunk MCP Server (required for /services/mcp)
       #   app/7245  = Splunk AI Assistant (SPL / saia_* MCP tools)
-      #   app/1621  = Splunk Common Information Model (Web data model for SA-S4R)
       # If a download fails (404, etc.), replace the /release/VERSION/ segment with a current
       # release for that app from https://splunkbase.splunk.com/app/<id> .
       # SA-S4R is this repo's package from the moving GitHub Release `latest`
       # (package-s4r.yml). The image downloads SPLUNK_APPS_URL only on the first
       # start of an empty so1-etc volume.
-      SPLUNK_APPS_URL: https://splunkbase.splunk.com/app/1924/release/8.2.2/download/,https://splunkbase.splunk.com/app/4353/release/1.8.26/download/,https://splunkbase.splunk.com/app/7931/release/2.1.0/download/,https://splunkbase.splunk.com/app/7245/release/2.3.6/download/,https://splunkbase.splunk.com/app/1621/release/8.8.0/download/,https://github.com/dd-Splunk/splunk-mcp/releases/download/latest/SA-S4R.spl
+      SPLUNK_APPS_URL: https://splunkbase.splunk.com/app/1924/release/8.2.2/download/,https://splunkbase.splunk.com/app/4353/release/1.8.26/download/,https://splunkbase.splunk.com/app/7931/release/2.1.0/download/,https://splunkbase.splunk.com/app/7245/release/2.3.6/download/,https://github.com/dd-Splunk/splunk-mcp/releases/download/latest/SA-S4R.spl
       TZ: ${TZ:?set TZ in config.env}
     volumes:
       - so1-var:/opt/splunk/var

docs/poc/ARCHITECTURE.md

@@ -20,7 +20,7 @@ The [Model Context Protocol](https://modelcontextprotocol.io/) lets a client (Cl
 
 ## Splunkbase applications
 
-`compose.yml` sets `SPLUNK_APPS_URL` to a comma-separated list of package download URLs. Installed from Splunkbase: **SA-Eventgen** (1924), **Config Explorer** (4353), **Splunk MCP Server** (7931), **Splunk AI Assistant** (7245), and the **Splunk Common Information Model** (1621, provides the `Web` data model SA-S4R maps to). **SA-S4R** is installed from this repo’s GitHub Release **`latest`** (`SA-S4R.spl`). Pin details and the full table live in [CONFIGURATION.md](CONFIGURATION.md#composeyml).
+`compose.yml` sets `SPLUNK_APPS_URL` to a comma-separated list of package download URLs. Installed from Splunkbase: **SA-Eventgen** (1924), **Config Explorer** (4353), **Splunk MCP Server** (7931), and **Splunk AI Assistant** (7245). **SA-S4R** is installed from this repo’s GitHub Release **`latest`** (`SA-S4R.spl`). Pin details and the full table live in [CONFIGURATION.md](CONFIGURATION.md#composeyml).
 
 You need valid **Splunkbase** credentials in **`.env`** (or the process environment): **`SPLUNKBASE_USER`** and **`SPLUNKBASE_PASS`**.
 

docs/poc/CONFIGURATION.md

@@ -8,7 +8,6 @@
 | 4353 | Config Explorer (optional UI utility) | 1.8.26 (Splunkbase lists Splunk up to 10.4; runs on 10.6 but not declared) |
 | 7931 | Splunk MCP Server (required for `/services/mcp`) | 2.1.0 (token API changed in 2.1; see [Encrypted bearer token](#encrypted-bearer-token-this-repo)) |
 | 7245 | Splunk AI Assistant | 2.3.6 |
-| 1621 | Splunk Common Information Model (provides the `Web` data model for SA-S4R) | 8.8.0 |
 | — | SA-S4R (this repo) | GitHub Release [`latest`](https://github.com/dd-Splunk/splunk-mcp/releases/download/latest/SA-S4R.spl) |
 
 Check current Splunkbase releases: `https://splunkbase.splunk.com/api/v1/app/<id>/release/` (first entry is latest). Update the `/release/VERSION/` segment in **`compose.yml`** when bumping.

docs/poc/SPECS.md

@@ -36,7 +36,6 @@ MCP user must **not** be `admin`. The **`ai`** SPL command is available to all u
 - SA-Eventgen (1924)
 - Config Explorer (4353)
 - Splunk AI Assistant (7245)
-- Splunk Common Information Model (1621, `Web` data model for SA-S4R)
 
 Splunk AI Toolkit (2890) and Python for Scientific Computing (2882) are **out of scope** unless installed manually.
 

docs/s4r/SA-S4R-APP.md

@@ -28,7 +28,7 @@ SA-S4R/                         # tracked in git
 │   ├── restmap.conf            # /s4r_workshop_mode for API MCP tools
 │   ├── savedsearches.conf      # Governed SPL backing SPL MCP tools
 │   ├── s4r_mcp_tools.json      # Batch-replace payload for POST /services/mcp_tools
-│   ├── tags.conf               # web tag → CIM Web data model
+│   ├── tags.conf               # web tag (CIM Web)
 │   └── transforms.conf         # product_codes lookup (file: lookups/product_codes.csv)
 ├── lookups/
 │   └── product_codes.csv       # Demo lookup for Lab 5
@@ -65,7 +65,7 @@ Splunk apps split **shipped baseline** (`default/`) from **instance-specific ove
 **Rules (Splunk and this repo):**
 
 1. **Splunk Web, Settings → Knowledge, nav editor, field extractor, Dashboard Studio saves** — must land under **`SA-S4R/local/`** only. **Never** save customizations into **`default/`** (Splunk will overwrite shipped objects on upgrade/reinstall).
-2. **Agents and contributors** — do not add workshop dashboards, nav tabs, or Lab 4 field extractions under **`SA-S4R/default/`** in git. **Exceptions** (maintainer-owned in **`default/`**): MCP packaging (`savedsearches.conf` for tool backing, `s4r_mcp_tools.json`, REST handler) — [MCP-TOOLS.md](MCP-TOOLS.md) — and the sourcetype baseline: field extractions, the CIM Web mapping (`props.conf`, `eventtypes.conf`, `tags.conf`), the host transform and the `s4r_user` role. Workshop UI setup is in [Workshop objects in `local/`](#workshop-objects-in-local) and [DASHBOARD.md](DASHBOARD.md).
+2. **Agents and contributors** — do not add workshop dashboards, nav tabs, or Lab 4 field extractions under **`SA-S4R/default/`** in git. **Exceptions** (maintainer-owned in **`default/`**): MCP packaging (`savedsearches.conf` for tool backing, `s4r_mcp_tools.json`, REST handler) — [MCP-TOOLS.md](MCP-TOOLS.md) — and the sourcetype baseline: field extractions, the CIM-compliant field names and `web` tag (`props.conf`, `eventtypes.conf`, `tags.conf`), the Eventgen host and rate, and the `s4r_user` role. Workshop UI setup is in [Workshop objects in `local/`](#workshop-objects-in-local) and [DASHBOARD.md](DASHBOARD.md).
 3. **Packaging** — **`package-s4r.yml`** excludes **`local/`** (entire directory) so instance-specific content is not published in **`SA-S4R.spl`**. Git ignores **`SA-S4R/local/**`** (see **`.gitignore`**).
 4. **Install** — Compose does not bind-mount the app tree. The package lands in **`so1-etc`**. Workshop UI and Eventgen overrides still belong in that installed app’s **`local/`**.
 
@@ -139,9 +139,9 @@ The **Buttercup Enterprises** workshop tab and Dashboard Studio view live under
 
 **`default/props.conf`** extracts `action`, `product_id`, `uid`, and `JSESSIONID` from the request line so workshop SPL such as `action=purchase` works without manual field extraction.
 
-### CIM Web mapping
+### CIM-compliant fields (CIM Web)
 
-**`default/props.conf`**, **`eventtypes.conf`** and **`tags.conf`** map `access_combined` to the CIM **Web** data model. The CIM app (Splunkbase 1621, pinned in **`compose.yml`**) provides the model, and **`default.meta`** exports the eventtype and tag to `system` so it can see them. The app ships the mapping but no acceleration.
+**`default/props.conf`**, **`eventtypes.conf`** and **`tags.conf`** give `access_combined` the field names and the `web` tag of the CIM **Web** data model, so any Splunk with the Splunk Common Information Model app (Splunkbase 1621) can use the data as is, and **`default.meta`** exports the eventtype and tag to `system` for that. **This repo does not install the CIM app**: the workshop does not need it, and the S4R searches use plain SPL. Install it yourself if you want the CIM `Web` model or CIM-based apps.
 
 | Web field | Source |
 | --------- | ------ |
@@ -153,15 +153,11 @@ The **Buttercup Enterprises** workshop tab and Dashboard Studio view live under
 | `vendor_product` | constant `Buttercup Enterprises Web Shop` |
 | `status`, `uri_path`, `uri_query`, `bytes`, `user` | already extracted by the stock sourcetype |
 
-**Host.** Eventgen ignores its own `host` setting with the default modinput output (its spec limits it to `splunkstream` and `syslogout`), so every event would carry `host=127.0.0.1`. **`transforms.conf`** therefore sets `host` to `www.buttercupenterprises.com` at index time, and **`props.conf`** applies it only to the three S4R sample sources (`product.screen.sample`, `cart.do.sample`, `attack.nk.purchase.sample`), never to other `access_combined` data. `host` is a real indexed field, so `tstats ... where host=...` works. Changing it needs a Splunk restart, and events indexed earlier keep their old host.
+**Host.** **`eventgen.conf`** sets `host = www.buttercupenterprises.com` in `[global]`, so every sample stanza (including the attack stream) inherits it and events no longer carry Eventgen's own `127.0.0.1`. `host` is a real indexed field, so `tstats ... where host=...` works, and `dest`, `site` and `url_domain` are aliases of it. Events indexed before a change keep their old host.
 
-**Known deviation:** `action` stays the cart verb (`view`, `addtocart`, `purchase`, `remove`, `changequantity`) because the workshop searches filter on it. `Web.action` therefore holds those verbs instead of CIM values such as `allowed` and `blocked`. `product_id` is a custom field, not part of the model.
+**Known deviation:** `action` stays the cart verb (`view`, `addtocart`, `purchase`, `remove`, `changequantity`) because the workshop searches filter on it. With the CIM app installed, `Web.action` would hold those verbs instead of CIM values such as `allowed` and `blocked`. `product_id` is a custom field, not part of CIM.
 
-Query it with:
-
-```spl
-| tstats count from datamodel=Web where Web.status>=400 by Web.src, Web.status
-```
+With the CIM app installed, `| tstats count from datamodel=Web by Web.dest, Web.status` works on this data.
 
 **`platform`** (Lab 4) belongs in the installed app’s **`local/props.conf`** — see [Workshop objects in `local/`](#workshop-objects-in-local). Agents/MCP still use inline `rex` per [SPL-CATALOG.md](SPL-CATALOG.md).
 
@@ -187,7 +183,8 @@ The stanza **`[product_codes]`** in **`default/transforms.conf`** points at the
 **Splunk Web:** save all knowledge objects, nav changes, field extractions, and dashboards under **`local/`** only — never **`default/`** (see **`default/` vs `local/`** above).
 
 - Edit **`samples/action.txt`**, **`status.txt`**, or **`useragent.txt`** to change categorical choices.
-- Tune **`interval`**, **`count`**, and **`randomizeCount`** per stanza in `eventgen.conf`.
+- Tune **`interval`**, **`count`**, and **`randomizeCount`** per stanza in `eventgen.conf`. Each stanza emits `count` events every `interval` seconds: the default is one batch every 10 s, so about 300 events/min for the baseline (`product.screen` 34 + `cart.do` 16 per batch) and about 150 events/min more while the attack stream is on.
+- After editing `eventgen.conf` by hand on a running stack, run `POST /servicesNS/nobody/SA-S4R/configs/conf-eventgen/_reload`, then disable and enable the `SA-Eventgen` modinput. Copying the file alone is not enough, because splunkd keeps the old config in memory.
 - Adjust the **`product.screen`** / **`cart.do`** ratio via each stanza’s **`count`**.
 
 ### Workshop modes: infrastructure vs NK attack

docs/s4r/SPL-CATALOG.md

@@ -30,8 +30,8 @@ index=main sourcetype=access_combined
 | Cart `action` values | `view`, `addtocart`, `purchase`, `remove`, `changequantity` |
 | Extracted fields | `action`, `product_id`, `uid`, `JSESSIONID` (`SA-S4R/default/props.conf`) |
 | Geo IP field | `clientip` (confirm casing in Search) |
-| Host | `host=www.buttercupenterprises.com` on every S4R event (set at index time; Eventgen's own `host` setting has no effect with modinput output) |
-| CIM Web model | Events are tagged `web` and mapped to the CIM `Web` data model, for example `\| tstats count from datamodel=Web by Web.dest, Web.status` ([SA-S4R-APP.md § CIM Web mapping](SA-S4R-APP.md#cim-web-mapping)). `Web.action` holds the cart verb, not `allowed` or `blocked` |
+| Host | `host=www.buttercupenterprises.com` on every S4R event (set in `eventgen.conf` `[global]`) |
+| CIM fields | Field names and the `web` tag follow CIM Web (`src`, `dest`, `status`, `http_method`, …); the CIM app is not installed ([SA-S4R-APP.md § CIM-compliant fields](SA-S4R-APP.md#cim-compliant-fields-cim-web)) |
 | Success | `status` 2xx |
 | Failure (workshop) | `status>=400` |
 | Failed purchase | `action=purchase` AND `status>=400` |

tests/s4r-cim-web.sh

@@ -28,15 +28,16 @@ grep -q '^EXTRACT-action = ' "${APP}/default/props.conf" || die "props.conf lost
 if grep -Eq '^EVAL-(dest|site|url_domain) = ' "${APP}/default/props.conf"; then
   die "props.conf hard-codes dest, site or url_domain; they come from host"
 fi
-# Eventgen ignores `host` with modinput output, so host is set at index time for each sample source
-grep -Eq '^FORMAT = host::www\.buttercupenterprises\.com$' "${APP}/default/transforms.conf" \
-  || die "transforms.conf does not set the shop host"
-while IFS= read -r src; do
-  awk -v s="[source::${src}]" '$0 == s {f=1; next} /^\[/ {f=0} f && /^TRANSFORMS-s4r_host = s4r_set_host$/ {ok=1} END {exit !ok}' \
-    "${APP}/default/props.conf" || die "props.conf does not set host for source ${src}"
-done < <(sed -n 's/^\[\(.*\.sample\)\]$/\1/p' "${APP}/default/eventgen.conf")
-
-# the tag that puts the events in the Web data model, and the exports that let Splunk_SA_CIM see it
+# host comes from Eventgen ([global] reaches every sample stanza), so no index-time override is needed
+grep -Eq '^host = www\.buttercupenterprises\.com$' "${APP}/default/eventgen.conf" \
+  || die "eventgen.conf [global] does not set the shop host"
+awk '$0 == "[global]" {f=1; next} /^\[/ {f=0} f && /^host = /{ok=1} END {exit !ok}' "${APP}/default/eventgen.conf" \
+  || die "host is not set in eventgen.conf [global]"
+if grep -q 's4r_set_host' "${APP}/default/props.conf" "${APP}/default/transforms.conf"; then
+  die "the index-time host override is back; set host in eventgen.conf instead"
+fi
+
+# the tag that marks the events as CIM Web, and the exports that let a CIM install see it
 grep -Eq '^search = .*sourcetype=access_combined' "${APP}/default/eventtypes.conf" \
   || die "eventtypes.conf does not match access_combined"
 etype="$(sed -n 's/^\[\(s4r_[a-z_]*\)\]$/\1/p' "${APP}/default/eventtypes.conf" | head -1)"
@@ -48,8 +49,9 @@ for stanza in eventtypes tags; do
     "${APP}/metadata/default.meta" || die "default.meta does not export ${stanza} to system"
 done
 
-# the Web data model ships in the CIM app, so compose must install it
-grep -Eq 'SPLUNK_APPS_URL:.*splunkbase\.splunk\.com/app/1621/release/[0-9.]+/download/' compose.yml \
-  || die "compose.yml does not install the Splunk CIM app (Splunkbase 1621)"
+# this repo does not install the CIM app: the mapping stays, the Splunkbase pin must not come back
+if grep -Eq 'splunkbase\.splunk\.com/app/1621/' compose.yml; then
+  die "compose.yml installs the CIM app (Splunkbase 1621); the S4R stack does not need it"
+fi
 
-echo "OK: SA-S4R maps access_combined to CIM Web and compose installs the CIM app"
+echo "OK: SA-S4R keeps CIM Web field names and the web tag, and the stack does not install the CIM app"