diff --git a/SA-S4R/default/app.conf b/SA-S4R/default/app.conf index 13a8244..59343a7 100644 --- a/SA-S4R/default/app.conf +++ b/SA-S4R/default/app.conf @@ -3,12 +3,12 @@ [ui] is_visible = true label = Splunk4Rookies -description = Buttercup Enterprises workshop sample data (Eventgen access_combined traffic, lookups, dashboard assets). +description = Buttercup Enterprises workshop sample data (Eventgen access_combined traffic, lookups, workshop MCP tools, dashboard assets). [launcher] author = splunk-mcp contributors -description = Splunk4Rookies PoC: synthetic Buttercup web access logs via Eventgen, field extractions, and product_codes lookup for workshop Labs 3–7. -version = 2.0.0 +description = Splunk4Rookies PoC: synthetic Buttercup web access logs via Eventgen, field extractions, CIM-named fields, the product_codes lookup and the SA-S4R_* MCP tools for workshop Labs 3–7. +version = 2.0.1 [package] id = SA-S4R diff --git a/SA-S4R/default/eventgen.conf b/SA-S4R/default/eventgen.conf index 10c3eb1..197fefc 100644 --- a/SA-S4R/default/eventgen.conf +++ b/SA-S4R/default/eventgen.conf @@ -5,9 +5,16 @@ [global] debug = false +# The shop's virtual host: every sample stanza inherits it, so events get host=www.buttercupenterprises.com +# (without it Eventgen stamps its own 127.0.0.1). CIM Web dest, site and url_domain are aliases of host. +host = www.buttercupenterprises.com + +# Rate: every stanza emits `count` events each `interval` seconds, so the baseline is +# (34 + 16) / 10 s = ~300 events/min and the attack stream adds ~150 events/min. + # Product page views (~67% of events) — uid present, no action= [product.screen.sample] -interval = 1 +interval = 10 count = 34 randomizeCount = 0.8 index = main @@ -59,7 +66,7 @@ token.10.replacement = integer[150:960] # Cart actions (~33%) — action=view|addtocart|purchase|remove|changequantity [cart.do.sample] -interval = 1 +interval = 10 count = 16 randomizeCount = 0.8 index = main @@ -115,7 +122,7 @@ token.10.replacement = integer[150:960] # Saved search: S4R Validate NK Attack Traffic (last 15m) [attack.nk.purchase.sample] disabled = true -interval = 1 +interval = 10 count = 25 randomizeCount = 0.5 index = main diff --git a/SA-S4R/default/props.conf b/SA-S4R/default/props.conf index 0c882c3..bfd20e1 100644 --- a/SA-S4R/default/props.conf +++ b/SA-S4R/default/props.conf @@ -1,12 +1,3 @@ -[source::product.screen.sample] -TRANSFORMS-s4r_host = s4r_set_host - -[source::cart.do.sample] -TRANSFORMS-s4r_host = s4r_set_host - -[source::attack.nk.purchase.sample] -TRANSFORMS-s4r_host = s4r_set_host - [access_combined] SHOULD_LINEMERGE = false EXTRACT-action = action=(?[^&\s"]+) @@ -14,8 +5,9 @@ EXTRACT-product_id = product_id=(?[^&\s"]+) EXTRACT-uid = uid=(?[^&\s"]+) EXTRACT-JSESSIONID = JSESSIONID=(?[^&\s"]+) -# CIM Web mapping (tags.conf tags the events `web`). The s4r_set_host transform makes host the -# shop's virtual host, which is the site and destination. `action` stays the cart verb (view, purchase, ...): +# CIM-compliant field names for the Web model (tags.conf tags the events `web`). The CIM app itself is +# not installed by this repo; these make the data usable with it. Eventgen sets host to the shop's virtual +# host (eventgen.conf), which is the site and destination. `action` stays the cart verb (view, purchase, ...): # the workshop searches depend on it, so Web.action is not CIM-normalized. FIELDALIAS-cim_src = clientip AS src FIELDALIAS-cim_dest = host AS dest host AS site host AS url_domain diff --git a/SA-S4R/default/transforms.conf b/SA-S4R/default/transforms.conf index 0996d73..78bebbc 100644 --- a/SA-S4R/default/transforms.conf +++ b/SA-S4R/default/transforms.conf @@ -1,10 +1,3 @@ [product_codes] filename = product_codes.csv case_sensitive_match = false - -# Eventgen ignores its `host` setting with the default modinput output, so every event would -# carry host=127.0.0.1. Set the shop's virtual host at index time, only for the S4R samples. -[s4r_set_host] -REGEX = . -DEST_KEY = MetaData:Host -FORMAT = host::www.buttercupenterprises.com diff --git a/compose.yml b/compose.yml index af76e17..864e63c 100644 --- a/compose.yml +++ b/compose.yml @@ -17,13 +17,12 @@ services: # app/4353 = Config Explorer (optional UI utility) # app/7931 = Splunk MCP Server (required for /services/mcp) # app/7245 = Splunk AI Assistant (SPL / saia_* MCP tools) - # app/1621 = Splunk Common Information Model (Web data model for SA-S4R) # If a download fails (404, etc.), replace the /release/VERSION/ segment with a current # release for that app from https://splunkbase.splunk.com/app/ . # SA-S4R is this repo's package from the moving GitHub Release `latest` # (package-s4r.yml). The image downloads SPLUNK_APPS_URL only on the first # start of an empty so1-etc volume. - SPLUNK_APPS_URL: https://splunkbase.splunk.com/app/1924/release/8.2.2/download/,https://splunkbase.splunk.com/app/4353/release/1.8.26/download/,https://splunkbase.splunk.com/app/7931/release/2.1.0/download/,https://splunkbase.splunk.com/app/7245/release/2.3.6/download/,https://splunkbase.splunk.com/app/1621/release/8.8.0/download/,https://github.com/dd-Splunk/splunk-mcp/releases/download/latest/SA-S4R.spl + SPLUNK_APPS_URL: https://splunkbase.splunk.com/app/1924/release/8.2.2/download/,https://splunkbase.splunk.com/app/4353/release/1.8.26/download/,https://splunkbase.splunk.com/app/7931/release/2.1.0/download/,https://splunkbase.splunk.com/app/7245/release/2.3.6/download/,https://github.com/dd-Splunk/splunk-mcp/releases/download/latest/SA-S4R.spl TZ: ${TZ:?set TZ in config.env} volumes: - so1-var:/opt/splunk/var diff --git a/docs/poc/ARCHITECTURE.md b/docs/poc/ARCHITECTURE.md index bab567b..4b4527e 100644 --- a/docs/poc/ARCHITECTURE.md +++ b/docs/poc/ARCHITECTURE.md @@ -20,7 +20,7 @@ The [Model Context Protocol](https://modelcontextprotocol.io/) lets a client (Cl ## Splunkbase applications -`compose.yml` sets `SPLUNK_APPS_URL` to a comma-separated list of package download URLs. Installed from Splunkbase: **SA-Eventgen** (1924), **Config Explorer** (4353), **Splunk MCP Server** (7931), **Splunk AI Assistant** (7245), and the **Splunk Common Information Model** (1621, provides the `Web` data model SA-S4R maps to). **SA-S4R** is installed from this repo’s GitHub Release **`latest`** (`SA-S4R.spl`). Pin details and the full table live in [CONFIGURATION.md](CONFIGURATION.md#composeyml). +`compose.yml` sets `SPLUNK_APPS_URL` to a comma-separated list of package download URLs. Installed from Splunkbase: **SA-Eventgen** (1924), **Config Explorer** (4353), **Splunk MCP Server** (7931), and **Splunk AI Assistant** (7245). **SA-S4R** is installed from this repo’s GitHub Release **`latest`** (`SA-S4R.spl`). Pin details and the full table live in [CONFIGURATION.md](CONFIGURATION.md#composeyml). You need valid **Splunkbase** credentials in **`.env`** (or the process environment): **`SPLUNKBASE_USER`** and **`SPLUNKBASE_PASS`**. diff --git a/docs/poc/CONFIGURATION.md b/docs/poc/CONFIGURATION.md index d937a0b..f5ea4fc 100644 --- a/docs/poc/CONFIGURATION.md +++ b/docs/poc/CONFIGURATION.md @@ -8,7 +8,6 @@ | 4353 | Config Explorer (optional UI utility) | 1.8.26 (Splunkbase lists Splunk up to 10.4; runs on 10.6 but not declared) | | 7931 | Splunk MCP Server (required for `/services/mcp`) | 2.1.0 (token API changed in 2.1; see [Encrypted bearer token](#encrypted-bearer-token-this-repo)) | | 7245 | Splunk AI Assistant | 2.3.6 | -| 1621 | Splunk Common Information Model (provides the `Web` data model for SA-S4R) | 8.8.0 | | — | SA-S4R (this repo) | GitHub Release [`latest`](https://github.com/dd-Splunk/splunk-mcp/releases/download/latest/SA-S4R.spl) | Check current Splunkbase releases: `https://splunkbase.splunk.com/api/v1/app//release/` (first entry is latest). Update the `/release/VERSION/` segment in **`compose.yml`** when bumping. diff --git a/docs/poc/SPECS.md b/docs/poc/SPECS.md index d953ab9..47ebf03 100644 --- a/docs/poc/SPECS.md +++ b/docs/poc/SPECS.md @@ -36,7 +36,6 @@ MCP user must **not** be `admin`. The **`ai`** SPL command is available to all u - SA-Eventgen (1924) - Config Explorer (4353) - Splunk AI Assistant (7245) -- Splunk Common Information Model (1621, `Web` data model for SA-S4R) Splunk AI Toolkit (2890) and Python for Scientific Computing (2882) are **out of scope** unless installed manually. diff --git a/docs/s4r/SA-S4R-APP.md b/docs/s4r/SA-S4R-APP.md index b18a9ec..1712d9c 100644 --- a/docs/s4r/SA-S4R-APP.md +++ b/docs/s4r/SA-S4R-APP.md @@ -28,7 +28,7 @@ SA-S4R/ # tracked in git │ ├── restmap.conf # /s4r_workshop_mode for API MCP tools │ ├── savedsearches.conf # Governed SPL backing SPL MCP tools │ ├── s4r_mcp_tools.json # Batch-replace payload for POST /services/mcp_tools -│ ├── tags.conf # web tag → CIM Web data model +│ ├── tags.conf # web tag (CIM Web) │ └── transforms.conf # product_codes lookup (file: lookups/product_codes.csv) ├── lookups/ │ └── product_codes.csv # Demo lookup for Lab 5 @@ -65,7 +65,7 @@ Splunk apps split **shipped baseline** (`default/`) from **instance-specific ove **Rules (Splunk and this repo):** 1. **Splunk Web, Settings → Knowledge, nav editor, field extractor, Dashboard Studio saves** — must land under **`SA-S4R/local/`** only. **Never** save customizations into **`default/`** (Splunk will overwrite shipped objects on upgrade/reinstall). -2. **Agents and contributors** — do not add workshop dashboards, nav tabs, or Lab 4 field extractions under **`SA-S4R/default/`** in git. **Exceptions** (maintainer-owned in **`default/`**): MCP packaging (`savedsearches.conf` for tool backing, `s4r_mcp_tools.json`, REST handler) — [MCP-TOOLS.md](MCP-TOOLS.md) — and the sourcetype baseline: field extractions, the CIM Web mapping (`props.conf`, `eventtypes.conf`, `tags.conf`), the host transform and the `s4r_user` role. Workshop UI setup is in [Workshop objects in `local/`](#workshop-objects-in-local) and [DASHBOARD.md](DASHBOARD.md). +2. **Agents and contributors** — do not add workshop dashboards, nav tabs, or Lab 4 field extractions under **`SA-S4R/default/`** in git. **Exceptions** (maintainer-owned in **`default/`**): MCP packaging (`savedsearches.conf` for tool backing, `s4r_mcp_tools.json`, REST handler) — [MCP-TOOLS.md](MCP-TOOLS.md) — and the sourcetype baseline: field extractions, the CIM-compliant field names and `web` tag (`props.conf`, `eventtypes.conf`, `tags.conf`), the Eventgen host and rate, and the `s4r_user` role. Workshop UI setup is in [Workshop objects in `local/`](#workshop-objects-in-local) and [DASHBOARD.md](DASHBOARD.md). 3. **Packaging** — **`package-s4r.yml`** excludes **`local/`** (entire directory) so instance-specific content is not published in **`SA-S4R.spl`**. Git ignores **`SA-S4R/local/**`** (see **`.gitignore`**). 4. **Install** — Compose does not bind-mount the app tree. The package lands in **`so1-etc`**. Workshop UI and Eventgen overrides still belong in that installed app’s **`local/`**. @@ -139,9 +139,9 @@ The **Buttercup Enterprises** workshop tab and Dashboard Studio view live under **`default/props.conf`** extracts `action`, `product_id`, `uid`, and `JSESSIONID` from the request line so workshop SPL such as `action=purchase` works without manual field extraction. -### CIM Web mapping +### CIM-compliant fields (CIM Web) -**`default/props.conf`**, **`eventtypes.conf`** and **`tags.conf`** map `access_combined` to the CIM **Web** data model. The CIM app (Splunkbase 1621, pinned in **`compose.yml`**) provides the model, and **`default.meta`** exports the eventtype and tag to `system` so it can see them. The app ships the mapping but no acceleration. +**`default/props.conf`**, **`eventtypes.conf`** and **`tags.conf`** give `access_combined` the field names and the `web` tag of the CIM **Web** data model, so any Splunk with the Splunk Common Information Model app (Splunkbase 1621) can use the data as is, and **`default.meta`** exports the eventtype and tag to `system` for that. **This repo does not install the CIM app**: the workshop does not need it, and the S4R searches use plain SPL. Install it yourself if you want the CIM `Web` model or CIM-based apps. | Web field | Source | | --------- | ------ | @@ -153,15 +153,11 @@ The **Buttercup Enterprises** workshop tab and Dashboard Studio view live under | `vendor_product` | constant `Buttercup Enterprises Web Shop` | | `status`, `uri_path`, `uri_query`, `bytes`, `user` | already extracted by the stock sourcetype | -**Host.** Eventgen ignores its own `host` setting with the default modinput output (its spec limits it to `splunkstream` and `syslogout`), so every event would carry `host=127.0.0.1`. **`transforms.conf`** therefore sets `host` to `www.buttercupenterprises.com` at index time, and **`props.conf`** applies it only to the three S4R sample sources (`product.screen.sample`, `cart.do.sample`, `attack.nk.purchase.sample`), never to other `access_combined` data. `host` is a real indexed field, so `tstats ... where host=...` works. Changing it needs a Splunk restart, and events indexed earlier keep their old host. +**Host.** **`eventgen.conf`** sets `host = www.buttercupenterprises.com` in `[global]`, so every sample stanza (including the attack stream) inherits it and events no longer carry Eventgen's own `127.0.0.1`. `host` is a real indexed field, so `tstats ... where host=...` works, and `dest`, `site` and `url_domain` are aliases of it. Events indexed before a change keep their old host. -**Known deviation:** `action` stays the cart verb (`view`, `addtocart`, `purchase`, `remove`, `changequantity`) because the workshop searches filter on it. `Web.action` therefore holds those verbs instead of CIM values such as `allowed` and `blocked`. `product_id` is a custom field, not part of the model. +**Known deviation:** `action` stays the cart verb (`view`, `addtocart`, `purchase`, `remove`, `changequantity`) because the workshop searches filter on it. With the CIM app installed, `Web.action` would hold those verbs instead of CIM values such as `allowed` and `blocked`. `product_id` is a custom field, not part of CIM. -Query it with: - -```spl -| tstats count from datamodel=Web where Web.status>=400 by Web.src, Web.status -``` +With the CIM app installed, `| tstats count from datamodel=Web by Web.dest, Web.status` works on this data. **`platform`** (Lab 4) belongs in the installed app’s **`local/props.conf`** — see [Workshop objects in `local/`](#workshop-objects-in-local). Agents/MCP still use inline `rex` per [SPL-CATALOG.md](SPL-CATALOG.md). @@ -187,7 +183,8 @@ The stanza **`[product_codes]`** in **`default/transforms.conf`** points at the **Splunk Web:** save all knowledge objects, nav changes, field extractions, and dashboards under **`local/`** only — never **`default/`** (see **`default/` vs `local/`** above). - Edit **`samples/action.txt`**, **`status.txt`**, or **`useragent.txt`** to change categorical choices. -- Tune **`interval`**, **`count`**, and **`randomizeCount`** per stanza in `eventgen.conf`. +- Tune **`interval`**, **`count`**, and **`randomizeCount`** per stanza in `eventgen.conf`. Each stanza emits `count` events every `interval` seconds: the default is one batch every 10 s, so about 300 events/min for the baseline (`product.screen` 34 + `cart.do` 16 per batch) and about 150 events/min more while the attack stream is on. +- After editing `eventgen.conf` by hand on a running stack, run `POST /servicesNS/nobody/SA-S4R/configs/conf-eventgen/_reload`, then disable and enable the `SA-Eventgen` modinput. Copying the file alone is not enough, because splunkd keeps the old config in memory. - Adjust the **`product.screen`** / **`cart.do`** ratio via each stanza’s **`count`**. ### Workshop modes: infrastructure vs NK attack diff --git a/docs/s4r/SPL-CATALOG.md b/docs/s4r/SPL-CATALOG.md index 690b292..157f475 100644 --- a/docs/s4r/SPL-CATALOG.md +++ b/docs/s4r/SPL-CATALOG.md @@ -30,8 +30,8 @@ index=main sourcetype=access_combined | Cart `action` values | `view`, `addtocart`, `purchase`, `remove`, `changequantity` | | Extracted fields | `action`, `product_id`, `uid`, `JSESSIONID` (`SA-S4R/default/props.conf`) | | Geo IP field | `clientip` (confirm casing in Search) | -| Host | `host=www.buttercupenterprises.com` on every S4R event (set at index time; Eventgen's own `host` setting has no effect with modinput output) | -| CIM Web model | Events are tagged `web` and mapped to the CIM `Web` data model, for example `\| tstats count from datamodel=Web by Web.dest, Web.status` ([SA-S4R-APP.md § CIM Web mapping](SA-S4R-APP.md#cim-web-mapping)). `Web.action` holds the cart verb, not `allowed` or `blocked` | +| Host | `host=www.buttercupenterprises.com` on every S4R event (set in `eventgen.conf` `[global]`) | +| CIM fields | Field names and the `web` tag follow CIM Web (`src`, `dest`, `status`, `http_method`, …); the CIM app is not installed ([SA-S4R-APP.md § CIM-compliant fields](SA-S4R-APP.md#cim-compliant-fields-cim-web)) | | Success | `status` 2xx | | Failure (workshop) | `status>=400` | | Failed purchase | `action=purchase` AND `status>=400` | diff --git a/tests/s4r-cim-web.sh b/tests/s4r-cim-web.sh index 4d2979b..01c447f 100755 --- a/tests/s4r-cim-web.sh +++ b/tests/s4r-cim-web.sh @@ -28,15 +28,16 @@ grep -q '^EXTRACT-action = ' "${APP}/default/props.conf" || die "props.conf lost if grep -Eq '^EVAL-(dest|site|url_domain) = ' "${APP}/default/props.conf"; then die "props.conf hard-codes dest, site or url_domain; they come from host" fi -# Eventgen ignores `host` with modinput output, so host is set at index time for each sample source -grep -Eq '^FORMAT = host::www\.buttercupenterprises\.com$' "${APP}/default/transforms.conf" \ - || die "transforms.conf does not set the shop host" -while IFS= read -r src; do - awk -v s="[source::${src}]" '$0 == s {f=1; next} /^\[/ {f=0} f && /^TRANSFORMS-s4r_host = s4r_set_host$/ {ok=1} END {exit !ok}' \ - "${APP}/default/props.conf" || die "props.conf does not set host for source ${src}" -done < <(sed -n 's/^\[\(.*\.sample\)\]$/\1/p' "${APP}/default/eventgen.conf") - -# the tag that puts the events in the Web data model, and the exports that let Splunk_SA_CIM see it +# host comes from Eventgen ([global] reaches every sample stanza), so no index-time override is needed +grep -Eq '^host = www\.buttercupenterprises\.com$' "${APP}/default/eventgen.conf" \ + || die "eventgen.conf [global] does not set the shop host" +awk '$0 == "[global]" {f=1; next} /^\[/ {f=0} f && /^host = /{ok=1} END {exit !ok}' "${APP}/default/eventgen.conf" \ + || die "host is not set in eventgen.conf [global]" +if grep -q 's4r_set_host' "${APP}/default/props.conf" "${APP}/default/transforms.conf"; then + die "the index-time host override is back; set host in eventgen.conf instead" +fi + +# the tag that marks the events as CIM Web, and the exports that let a CIM install see it grep -Eq '^search = .*sourcetype=access_combined' "${APP}/default/eventtypes.conf" \ || die "eventtypes.conf does not match access_combined" etype="$(sed -n 's/^\[\(s4r_[a-z_]*\)\]$/\1/p' "${APP}/default/eventtypes.conf" | head -1)" @@ -48,8 +49,9 @@ for stanza in eventtypes tags; do "${APP}/metadata/default.meta" || die "default.meta does not export ${stanza} to system" done -# the Web data model ships in the CIM app, so compose must install it -grep -Eq 'SPLUNK_APPS_URL:.*splunkbase\.splunk\.com/app/1621/release/[0-9.]+/download/' compose.yml \ - || die "compose.yml does not install the Splunk CIM app (Splunkbase 1621)" +# this repo does not install the CIM app: the mapping stays, the Splunkbase pin must not come back +if grep -Eq 'splunkbase\.splunk\.com/app/1621/' compose.yml; then + die "compose.yml installs the CIM app (Splunkbase 1621); the S4R stack does not need it" +fi -echo "OK: SA-S4R maps access_combined to CIM Web and compose installs the CIM app" +echo "OK: SA-S4R keeps CIM Web field names and the web tag, and the stack does not install the CIM app"