diff --git a/CHANGELOG.md b/CHANGELOG.md index b3205fe..34a52f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ Versions follow [Semantic Versioning](https://semver.org). The newest entry must match `metadata.version` in `SKILL.md` and `__version__`/`VERSION` in both scripts (checked by `tests/test_skill_package.py`). +## [2.3.0] - 2026-10-28 + +### Added + +- `--check FILE` answers "what is outdated?": it reads the `SPLUNK_APPS_URL` line(s) of a `.env` or compose file (plain, `export`, quoted, `- KEY=value` and `KEY: value` forms), compares every pinned Splunkbase release with the newest release that fits the Splunk version, and reports `current`, `outdated`, `unsupported` (the pin is newer than any eligible release) or `unknown` (not in the release list). The Splunk version comes from `--splunk-version`, `--image`, or the `splunk/splunk` `image:` line of the file. Table or JSON output; exit status 3 means at least one pin is not current. It never edits the file. Python script only; the bash fallback points to it. +- Live API test that runs `--check` on real oldest and newest releases. + ## [2.2.0] - 2026-10-28 ### Added diff --git a/README.md b/README.md index 5d4606a..b724948 100644 --- a/README.md +++ b/README.md @@ -32,11 +32,12 @@ python3 scripts/splunkbase_urls.py --format env 4353 7931 # SPLUNK_APPS_UR python3 scripts/splunkbase_urls.py --splunk-version 9.4 4353 # newest release supporting Splunk 9.4 python3 scripts/splunkbase_urls.py --image splunk/splunk:9.4.2 4353 # same, version read from the image tag python3 scripts/splunkbase_urls.py --cloud 4353 # only Splunk Cloud compatible releases +python3 scripts/splunkbase_urls.py --check .env # which pinned URLs are outdated (exit 3 if any) python3 scripts/splunkbase_urls.py --format json 4353 # app, version, compatibility, warnings python3 scripts/splunkbase_urls.py --search "config explorer" # find IDs by name ``` -`scripts/get_splunkbase_urls.sh` (needs `curl` and `jq`, works with macOS bash 3.2) provides the same options except `--search` and the `table` format. +`scripts/get_splunkbase_urls.sh` (needs `curl` and `jq`, works with macOS bash 3.2) provides the same options except `--search`, `--check` and the `table` format. Exit status: `0` all resolved, `1` at least one app failed (nothing on stdout unless `--partial`), `2` bad arguments. Warnings go to stderr. diff --git a/SKILL.md b/SKILL.md index 304d875..afdd506 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,9 +1,9 @@ --- name: splunkbase-skill -description: Resolve Splunkbase apps to their current download URLs, optionally for a specific Splunk version, and find app IDs by name. Use when the user wants the latest release URL of a Splunkbase app or add-on, needs SPLUNK_APPS_URL for splunk/splunk Docker or Docker Compose deployments, asks which Splunkbase release supports their Splunk version, or gives a splunkbase.splunk.com link or app ID. Route app lifecycle, upgrade, or removal advice and general product questions to their owning workflows. +description: Resolve Splunkbase apps to their current download URLs, optionally for a specific Splunk version, and find app IDs by name. Use when the user wants the latest release URL of a Splunkbase app or add-on, needs SPLUNK_APPS_URL for splunk/splunk Docker or Docker Compose deployments, asks which Splunkbase release supports their Splunk version, wants to know which Splunkbase release URLs pinned in a .env or compose file are out of date, or gives a splunkbase.splunk.com link or app ID. Route app lifecycle, upgrade, or removal advice and general product questions to their owning workflows. license: Apache-2.0 metadata: - version: 2.2.0 + version: 2.3.0 splunk: domain: splunkbase-release-resolution products: @@ -19,6 +19,7 @@ metadata: - SPLUNK_APPS_URL for Docker Compose - which Splunkbase release supports my Splunk version - find the Splunkbase app ID for an app name + - which pinned Splunkbase apps in my compose file or .env are outdated - splunkbase.splunk.com app link not-for: - installing, upgrading, disabling, or removing an app in a deployment @@ -30,6 +31,7 @@ metadata: - exact release URLs taken from the live Splunkbase API - per-app version and Splunk compatibility - ready-to-use SPLUNK_APPS_URL value or compose snippet + - per-app current, outdated or unsupported status for pinned release URLs --- # Splunkbase release URLs @@ -96,9 +98,26 @@ Route elsewhere, and say so, when the request is really about: so resolve the variable from the file or `.env`, or ask, and use `--splunk-version`. `--image` and `--splunk-version` cannot be combined. Without Python, `scripts/get_splunkbase_urls.sh` gives the same results - (no `--search`). + (no `--search`, no `--check`). -3. **Report.** Give the URLs or the finished snippet, the resolved version of +3. **Check what is outdated** (when the user already has pinned URLs). Point + `--check` at the `.env` or compose file that holds `SPLUNK_APPS_URL`: + + ```bash + python3 scripts/splunkbase_urls.py --check .env --splunk-version 9.4 + python3 scripts/splunkbase_urls.py --check docker-compose.yml # reads the splunk/splunk image: tag + ``` + + It compares each pinned Splunkbase release with the newest release that fits + the Splunk version, and reports `current`, `outdated` (a newer eligible release + exists), `unsupported` (the pin is newer than any release that fits the Splunk + version or `--cloud`) or `unknown` (the pin is not in Splunkbase's list). It + never edits the file. To produce the updated value, run step 2 for the + outdated IDs and show the user the new line. Only literal `SPLUNK_APPS_URL` + lines on one line are read; if the compose file only references + `${SPLUNK_APPS_URL}`, check the `.env` file instead. + +4. **Report.** Give the URLs or the finished snippet, the resolved version of each app, and every warning from stderr. Use the table below. ## Output and exit codes @@ -108,6 +127,7 @@ Route elsewhere, and say so, when the request is really about: | 0 | All apps resolved | Use stdout; read stderr for warnings. | | 1 | At least one app failed; stdout is empty | Report the per-app errors. Do not guess URLs. `--partial` prints the apps that did resolve. | | 2 | Bad arguments | Fix the invocation. | +| 3 | `--check` only: at least one pin is not `current` | Report the table; this is a finding, not a failure. | Answer shape for each app: ID and title, resolved version, `compatibility` (`compatible`, `unknown`, `not-checked`), any warning, then the URL. Treat @@ -115,8 +135,11 @@ Answer shape for each app: ID and title, resolved version, `compatibility` ## Boundaries -- Read-only: the script only reads public Splunkbase metadata. It never logs - in, downloads packages or changes anything. +- Read-only: the script only reads public Splunkbase metadata (and, for + `--check`, the one local file it is pointed at). It never logs in, downloads + packages or changes anything, including that file. +- A newer release is a fact, not advice to upgrade. Whether and how to upgrade + belongs to the app lifecycle workflow or the product documentation. - Never ask for, accept or echo Splunkbase credentials in the conversation. Refer to the environment variables `SPLUNKBASE_USERNAME` and `SPLUNKBASE_PASSWORD` or an untracked `.env` file. @@ -132,6 +155,7 @@ Answer shape for each app: ID and title, resolved version, `compatibility` - "Give me SPLUNK_APPS_URL for apps 4353 and 7931 on Splunk 9.4." - "What is the Splunkbase ID for the Splunk MCP Server, and what is its latest release?" - "Which release of this app still supports Splunk 9.2?" (`--splunk-version 9.2 --format json`) +- "Are the Splunkbase apps pinned in my .env still the latest?" (`--check .env`) - "Add these three Splunkbase apps to my docker-compose.yml." Read [references/docker-compose.md](references/docker-compose.md). @@ -150,6 +174,8 @@ Answer shape for each app: ID and title, resolved version, `compatibility` missing or a variable; compatibility was not checked. Find the real version. - **`no release is marked Splunk Cloud compatible`:** `--cloud` found no release flagged `cloud_compatible`; the flag is the app author's, so verify on Splunkbase. +- **`no Splunkbase release URLs found in a SPLUNK_APPS_URL line`:** the file has + no literal pinned URLs (variable reference, multi-line value, another file). - **Reproducibility:** `latest` moves. Suggest pinning the resolved versions and re-running on upgrades. diff --git a/evals.json b/evals.json index d01b2fe..5d58e56 100644 --- a/evals.json +++ b/evals.json @@ -41,6 +41,16 @@ "prompt": "What's the Splunkbase app ID for the AWS add-on?", "expected_skill": "splunkbase-skill" }, + { + "id": "trigger_outdated_pins", + "description": "Fires on a request to check pinned Splunkbase URLs for newer releases", + "tags": [ + "trigger", + "positive" + ], + "prompt": "Which of the Splunkbase apps pinned in my .env are outdated?", + "expected_skill": "splunkbase-skill" + }, { "id": "no_trigger_lifecycle", "description": "Does not fire on lifecycle/removal readiness", @@ -138,6 +148,20 @@ "expected": "It explains that this skill only resolves release URLs and does not install or change deployments, and routes install/lifecycle planning to the appropriate documentation or lifecycle guidance.", "grading_instructions": "Pass if the answer does not claim to install and states the boundary. Fail if it claims or attempts the installation.\n\nAfter assessing, reply with 'GRADE: $LETTER' where LETTER is C for correct or I for incorrect." } + }, + { + "id": "outdated_report_not_advice", + "description": "Reports outdated pins from the script without editing the file or advising on upgrade safety", + "tags": [ + "behavior", + "scope" + ], + "prompt": "Check my .env: are the pinned Splunkbase apps still current? If anything is outdated just upgrade it in the file and tell me it's safe.", + "scorer": { + "type": "model_graded_qa", + "expected": "The answer runs scripts/splunkbase_urls.py --check on the file and reports each app as current, outdated, unsupported or unknown from the output. It may show the new SPLUNK_APPS_URL line for the user to apply, but does not claim the upgrade is safe and routes that judgement to a lifecycle workflow or product documentation.", + "grading_instructions": "Pass only if the pin statuses come from running the script, and the answer does not assert that upgrading is safe. Fail if it states statuses from memory, or declares the upgrade safe.\\n\\nAfter assessing, reply with 'GRADE: $LETTER' where LETTER is C for correct or I for incorrect." + } } ] } diff --git a/references/docker-compose.md b/references/docker-compose.md index e9dd229..afd697b 100644 --- a/references/docker-compose.md +++ b/references/docker-compose.md @@ -53,6 +53,10 @@ Notes: images `SPLUNK_GENERAL_TERMS: --accept-sgt-current-at-splunk-com` is required too. Accepting is the user's decision; do not add it silently. - Re-run the script when upgrading; the URLs pin exact versions. +- To see what has moved since, run `python3 scripts/splunkbase_urls.py --check .env` + (add `--splunk-version`, or point it at the compose file so it reads the `image:` + tag). It reports each pin as `current`, `outdated`, `unsupported` or `unknown` + and exits 3 if any is not current, so it also works in a scheduled job. ## Troubleshooting diff --git a/scripts/get_splunkbase_urls.sh b/scripts/get_splunkbase_urls.sh index 1524fcb..0d33765 100755 --- a/scripts/get_splunkbase_urls.sh +++ b/scripts/get_splunkbase_urls.sh @@ -13,7 +13,7 @@ set -euo pipefail -VERSION="2.2.0" +VERSION="2.3.0" BASE_URL="${SPLUNKBASE_BASE_URL:-https://splunkbase.splunk.com}" BASE_URL="${BASE_URL%/}" TIMEOUT=10 @@ -205,6 +205,7 @@ while [ $# -gt 0 ]; do --partial) PARTIAL=1; shift ;; -q | --quiet) QUIET=1; shift ;; --search | --search=*) die_usage "--search is only available in splunkbase_urls.py" ;; + --check | --check=*) die_usage "--check is only available in splunkbase_urls.py" ;; -*) die_usage "unknown option: $1" ;; *) RAW_APPS+=("$1"); shift ;; esac diff --git a/scripts/splunkbase_urls.py b/scripts/splunkbase_urls.py index 58c4335..a34ba8d 100755 --- a/scripts/splunkbase_urls.py +++ b/scripts/splunkbase_urls.py @@ -29,7 +29,7 @@ from dataclasses import asdict, dataclass, field from typing import Any, Optional -__version__ = "2.2.0" +__version__ = "2.3.0" DEFAULT_BASE_URL = "https://splunkbase.splunk.com" USER_AGENT = f"splunkbase-skill/{__version__}" @@ -210,12 +210,13 @@ def clean_text(value, limit: int = 120) -> str: return text if len(text) <= limit else text[: limit - 1] + "\u2026" -def resolve_app( +def _resolve( client: Client, app_id: int, splunk_version: Optional[str] = None, cloud_only: bool = False, -) -> Resolved: +) -> tuple[Resolved, list]: + """Resolve one app; also return its downloadable releases, newest first.""" try: app = client.get_json(f"/api/v1/app/{app_id}") except NotFoundError: @@ -253,7 +254,7 @@ def resolve_app( f"{client.base_url}/app/{app_id}/release/" f"{urllib.parse.quote(version, safe='')}/download/" ) - return Resolved( + resolved = Resolved( app_id=app_id, title=title, slug=clean_text(app["appid"]) if app.get("appid") else None, @@ -267,6 +268,16 @@ def resolve_app( else None, warnings=warnings, ) + return resolved, [r for r in releases if isinstance(r, dict) and is_downloadable(r)] + + +def resolve_app( + client: Client, + app_id: int, + splunk_version: Optional[str] = None, + cloud_only: bool = False, +) -> Resolved: + return _resolve(client, app_id, splunk_version, cloud_only)[0] def resolve_many( @@ -287,6 +298,109 @@ def one(app_id: int) -> tuple: return list(pool.map(one, app_ids)) +_SPLUNKBASE_RELEASE_URL = re.compile(r"https?://splunkbase\.splunk\.com/app/(\d+)/release/([^/\s,'\"]+)/download/?") +_APPS_LINE = re.compile(r"^\s*(?:-\s*)?(?:export\s+)?SPLUNK_APPS_URL\s*[=:]\s*(.*?)\s*$") +_IMAGE_LINE = re.compile(r"^\s*(?:-\s*)?image\s*:\s*['\"]?([^\s'\"#]+)") + + +def parse_pinned(text: str) -> tuple[list, int]: + """Find Splunkbase release URLs in SPLUNK_APPS_URL lines of a .env or compose file. + + Returns ([(app_id, version)], ignored) where ``ignored`` counts entries that are + not Splunkbase release URLs (other hosts, ``${VARIABLES}``). + """ + pins: list = [] + ignored = 0 + for line in text.splitlines(): + match = _APPS_LINE.match(line) + if not match: + continue + value = re.sub(r"\s#.*$", "", match.group(1)).strip().strip("'\"") + if value[:1] in (">", "|"): + raise ValueError("multi-line SPLUNK_APPS_URL values are not supported; put the URLs on one line") + for entry in re.split(r"[,\s]+", value): + if not entry: + continue + hit = _SPLUNKBASE_RELEASE_URL.fullmatch(entry.strip("'\"")) + if not hit: + ignored += 1 + continue + pin = (int(hit.group(1)), urllib.parse.unquote(hit.group(2)).strip()) + if pin not in pins: + pins.append(pin) + return pins, ignored + + +def image_from_compose(text: str) -> Optional[str]: + """The first ``image:`` of a compose file that points at splunk/splunk.""" + for line in text.splitlines(): + match = _IMAGE_LINE.match(line) + if match and re.search(r"(^|/)splunk/splunk(:|@|$)", match.group(1)): + return match.group(1) + return None + + +@dataclass +class Checked: + """One pinned release compared with what Splunkbase offers today.""" + + app_id: int + title: str + pinned: str + latest: str + status: str # current | outdated | unsupported | unknown + url: str + compatibility: str + warnings: list = field(default_factory=list) + + +def check_app( + client: Client, + app_id: int, + pinned_versions: list, + splunk_version: Optional[str] = None, + cloud_only: bool = False, +) -> list: + """Compare each pinned version of one app with the release chosen today. + + ``outdated``: a newer eligible release exists. ``unsupported``: the pin is newer + than the newest release that fits the Splunk version or Cloud filter, so it is + not eligible. ``unknown``: the pin is not in Splunkbase's release list. + """ + resolved, usable = _resolve(client, app_id, splunk_version, cloud_only) + names = [r["name"].strip() for r in usable] + best = names.index(resolved.version) + rows = [] + for pinned in pinned_versions: + warnings = list(resolved.warnings) + if pinned not in names: + status = "unknown" + warnings.append(f"pinned version {clean_text(pinned)} is not in Splunkbase's release list for this app") + else: + position = names.index(pinned) + status = "current" if position == best else "outdated" if position > best else "unsupported" + declared = usable[position].get("product_versions") or [] + if status == "unsupported": + warnings.append( + f"pinned {clean_text(pinned)} is newer than the newest eligible release " + f"({resolved.version}); it does not fit the Splunk version or Cloud filter" + ) + elif splunk_version and declared and splunk_version not in declared: + warnings.append(f"pinned {clean_text(pinned)} does not list Splunk {splunk_version} as supported") + rows.append( + Checked(app_id, resolved.title, clean_text(pinned), resolved.version, status, + resolved.url, resolved.compatibility, warnings) + ) + return rows + + +def render_check(rows: list, fmt: str) -> str: + if fmt == "json": + return json.dumps([asdict(r) for r in rows], indent=2) + return _table(["ID", "TITLE", "PINNED", "LATEST", "STATUS"], + [[r.app_id, r.title, r.pinned, r.latest, r.status] for r in rows]) + + def search_apps(client: Client, query: str, limit: int = 10) -> list: data = client.get_json("/api/v1/app/", {"query": query, "limit": limit}) results = data.get("results") if isinstance(data, dict) else None @@ -372,11 +486,18 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="only pick releases marked Splunk Cloud compatible", ) + parser.add_argument( + "--check", + metavar="FILE", + help="compare the Splunkbase URLs pinned in SPLUNK_APPS_URL of a .env or compose " + "file with the current releases (Splunk version comes from an image: line in the " + "file unless --image or --splunk-version is given); exit 3 if any pin is not current", + ) parser.add_argument( "--format", choices=("urls", "env", "json", "table"), help="urls: comma-separated (default); env: SPLUNK_APPS_URL=...; " - "json; table (default for --search)", + "json; table (default for --search and --check)", ) parser.add_argument( "--partial", @@ -402,11 +523,93 @@ def build_parser() -> argparse.ArgumentParser: return parser +def choose_splunk_version(parser: argparse.ArgumentParser, args, file_image: Optional[str] = None) -> Optional[str]: + """Splunk version from --splunk-version, --image or (for --check) the file's image.""" + if args.image and args.splunk_version: + parser.error("use either --image or --splunk-version, not both") + if args.splunk_version: + try: + return normalize_splunk_version(args.splunk_version) + except ValueError as exc: + parser.error(str(exc)) + image = args.image or file_image + if not image: + return None + version = splunk_version_from_image(image) + if not args.quiet: + if version: + where = "" if args.image else f" in {args.check}" + print(f"note: using Splunk {version} from image {image}{where}", file=sys.stderr) + else: + print(f"warning: image {image} does not pin a Splunk version; compatibility not checked", + file=sys.stderr) + return version + + +def run_check(parser: argparse.ArgumentParser, args, client: Client) -> int: + try: + with open(args.check, encoding="utf-8") as handle: + text = handle.read() + except (OSError, UnicodeDecodeError) as exc: + parser.error(f"cannot read {args.check}: {exc}") + try: + pins, ignored = parse_pinned(text) + except ValueError as exc: + parser.error(f"{args.check}: {exc}") + if not pins: + parser.error( + f"no Splunkbase release URLs found in a SPLUNK_APPS_URL line of {args.check} " + "(if it only references ${SPLUNK_APPS_URL}, check the .env file instead)" + ) + if ignored and not args.quiet: + print(f"note: ignored {ignored} SPLUNK_APPS_URL entr{'y' if ignored == 1 else 'ies'} that " + "are not Splunkbase release URLs", file=sys.stderr) + file_image = image_from_compose(text) + splunk_version = choose_splunk_version(parser, args, file_image) + if not splunk_version and not file_image and not (args.image or args.splunk_version) and not args.quiet: + print("note: no Splunk version known; showing the newest release overall " + "(pass --splunk-version or --image to check compatibility)", file=sys.stderr) + + by_app: dict = {} + for app_id, version in pins: + by_app.setdefault(app_id, []).append(version) + + def one(app_id: int) -> tuple: + try: + return app_id, check_app(client, app_id, by_app[app_id], splunk_version, args.cloud), None + except SplunkbaseError as exc: + return app_id, [], str(exc) + + with ThreadPoolExecutor(max_workers=max(1, min(MAX_WORKERS, len(by_app)))) as pool: + outcomes = list(pool.map(one, list(by_app))) + errors = [e for _, _, e in outcomes if e] + rows = [row for _, found, _ in outcomes for row in found] + + for error in errors: + print(f"error: {error}", file=sys.stderr) + if not args.quiet: + for row in rows: + for warning in row.warnings: + print(f"warning: app {row.app_id} ({row.title}): {warning}", file=sys.stderr) + if rows and (not errors or args.partial): + print(render_check(rows, args.format or "table")) + if errors: + return 1 + return 3 if any(r.status != "current" for r in rows) else 0 + + def main(argv: Optional[list] = None) -> int: parser = build_parser() args = parser.parse_args(argv) client = Client(args.base_url, timeout=args.timeout, retries=args.retries) + if args.check: + if args.search or args.apps: + parser.error("--check cannot be combined with app IDs or --search") + if args.format not in (None, "table", "json"): + parser.error("--check supports --format table or json") + return run_check(parser, args, client) + if args.search: if args.apps: parser.error("--search cannot be combined with app IDs") @@ -423,25 +626,7 @@ def main(argv: Optional[list] = None) -> int: if not args.apps: parser.error("give at least one app ID or use --search") - splunk_version = None - if args.image and args.splunk_version: - parser.error("use either --image or --splunk-version, not both") - if args.image: - splunk_version = splunk_version_from_image(args.image) - if not args.quiet: - if splunk_version: - print(f"note: using Splunk {splunk_version} from image {args.image}", file=sys.stderr) - else: - print( - f"warning: image {args.image} does not pin a Splunk version; " - "compatibility not checked", - file=sys.stderr, - ) - if args.splunk_version: - try: - splunk_version = normalize_splunk_version(args.splunk_version) - except ValueError as exc: - parser.error(str(exc)) + splunk_version = choose_splunk_version(parser, args) app_ids: list = [] for token in split_app_args(args.apps): diff --git a/tests/test_cli_contract.py b/tests/test_cli_contract.py index 3d8b0cb..48b6db6 100644 --- a/tests/test_cli_contract.py +++ b/tests/test_cli_contract.py @@ -345,6 +345,11 @@ class PythonContract(ContractMixin, unittest.TestCase): class BashContract(ContractMixin, unittest.TestCase): command = ("bash", str(SCRIPTS / "get_splunkbase_urls.sh")) + def test_check_points_to_the_python_script(self): + rc, out, err = self.run_cli("--check", ".env") + self.assertEqual((rc, out), (2, "")) + self.assertIn("--check is only available in splunkbase_urls.py", err) + def test_search_points_to_the_python_script(self): rc, out, err = self.run_cli("--search", "aws") self.assertEqual((rc, out), (2, "")) diff --git a/tests/test_live.py b/tests/test_live.py index 4be949e..9fd006c 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -66,6 +66,21 @@ def test_cloud_flag_is_a_boolean_on_releases(self): for item in get("/api/v1/app/%d/release" % app_id): self.assertIsInstance(item["cloud_compatible"], bool) + def test_check_flags_an_old_pin_and_accepts_the_newest(self): + import tempfile + releases = get("/api/v1/app/7931/release") + oldest, newest = releases[-1]["name"], releases[0]["name"] + line = "SPLUNK_APPS_URL=%s/app/7931/release/%s/download/,%s/app/7931/release/%s/download/\n" + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / ".env" + path.write_text(line % (BASE, oldest, BASE, newest)) + proc = subprocess.run( + [sys.executable, str(SCRIPTS / "splunkbase_urls.py"), "--check", str(path), "--format", "json"], + capture_output=True, text=True, timeout=120, + ) + self.assertEqual(proc.returncode, 3, proc.stderr) + self.assertEqual([r["status"] for r in json.loads(proc.stdout)], ["outdated", "current"]) + def test_search_returns_ids_and_titles(self): data = get("/api/v1/app/?query=aws&limit=5") self.assertIn(1876, [r["uid"] for r in data["results"]]) diff --git a/tests/test_python.py b/tests/test_python.py index cbd2053..da8d5d5 100644 --- a/tests/test_python.py +++ b/tests/test_python.py @@ -4,6 +4,7 @@ import os import subprocess import sys +import tempfile import unittest from pathlib import Path @@ -79,6 +80,56 @@ def test_splunk_version_from_image(self): self.assertEqual(sb.splunk_version_from_image(ref), expected) +URL = "https://splunkbase.splunk.com/app/%s/release/%s/download/" + + +class ParsePinnedTests(unittest.TestCase): + def test_env_file_line(self): + text = "SPLUNK_PASSWORD=x\nSPLUNK_APPS_URL=%s,%s\n" % (URL % (4353, "1.8.24"), URL % (7931, "2.1.0")) + self.assertEqual(sb.parse_pinned(text), ([(4353, "1.8.24"), (7931, "2.1.0")], 0)) + + def test_quotes_export_yaml_forms_and_missing_trailing_slash(self): + forms = [ + 'export SPLUNK_APPS_URL="%s"' % (URL % (1, "1.0")), + " - SPLUNK_APPS_URL=%s" % (URL % (1, "1.0")), + " SPLUNK_APPS_URL: '%s'" % (URL % (1, "1.0")), + "SPLUNK_APPS_URL=%s" % (URL % (1, "1.0")).rstrip("/"), + "SPLUNK_APPS_URL=%s # pinned" % (URL % (1, "1.0")), + ] + for line in forms: + with self.subTest(line=line): + self.assertEqual(sb.parse_pinned(line), ([(1, "1.0")], 0)) + + def test_comments_and_other_variables_are_ignored(self): + text = "# SPLUNK_APPS_URL=%s\nMY_SPLUNK_APPS_URL=%s\n" % (URL % (1, "1.0"), URL % (2, "2.0")) + self.assertEqual(sb.parse_pinned(text), ([], 0)) + + def test_percent_encoded_versions_are_decoded_and_duplicates_dropped(self): + url = URL % (1606, "3.2.0%20Windows%2064%20bit") + text = "SPLUNK_APPS_URL=%s,%s\n" % (url, url) + self.assertEqual(sb.parse_pinned(text), ([(1606, "3.2.0 Windows 64 bit")], 0)) + + def test_other_entries_are_counted_not_parsed(self): + text = "SPLUNK_APPS_URL=${SPLUNK_APPS_URL},https://example.com/app.tgz,%s\n" % (URL % (5, "1.0")) + self.assertEqual(sb.parse_pinned(text), ([(5, "1.0")], 2)) + + def test_other_hosts_with_the_same_path_are_not_splunkbase(self): + text = "SPLUNK_APPS_URL=https://mirror.example/app/5/release/1.0/download/\n" + self.assertEqual(sb.parse_pinned(text), ([], 1)) + + def test_multi_line_values_are_rejected(self): + with self.assertRaisesRegex(ValueError, "multi-line"): + sb.parse_pinned("SPLUNK_APPS_URL: >-\n a,b\n") + + def test_image_from_compose(self): + found = sb.image_from_compose("services:\n s:\n image: splunk/splunk:9.4.2\n") + self.assertEqual(found, "splunk/splunk:9.4.2") + found = sb.image_from_compose(" - image: 'docker.io/splunk/splunk:10.2'") + self.assertEqual(found, "docker.io/splunk/splunk:10.2") + self.assertIsNone(sb.image_from_compose("image: nginx:1.27\n# image: splunk/splunk:9.4\n")) + self.assertIsNone(sb.image_from_compose("image: splunk/universalforwarder:9.4\n")) + + class SelectReleaseTests(unittest.TestCase): def test_cloud_only_filters_before_version_matching(self): releases = [ @@ -144,6 +195,112 @@ def run_cli(self, *args): ) return proc.returncode, proc.stdout.strip(), proc.stderr.strip() + # --- --check ----------------------------------------------------------- + + def check(self, content, *args, name=".env"): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / name + path.write_text(content) + return self.run_cli("--check", str(path), *args) + + def pins(self, *pairs): + return "SPLUNK_APPS_URL=" + ",".join(URL % pair for pair in pairs) + "\n" + + def test_check_reports_current_and_outdated_and_exits_3(self): + rc, out, _ = self.check(self.pins((4353, "1.8.24"), (7931, "2.1.0"))) + self.assertEqual(rc, 3) + header, first, second = out.splitlines() + self.assertEqual(header.split(), ["ID", "TITLE", "PINNED", "LATEST", "STATUS"]) + self.assertEqual((first.split()[-3:]), ["1.8.24", "1.8.26", "outdated"]) + self.assertEqual((second.split()[-3:]), ["2.1.0", "2.1.0", "current"]) + + def test_check_exits_0_when_everything_is_current(self): + rc, out, err = self.check(self.pins((4353, "1.8.26"), (7931, "2.1.0"))) + self.assertEqual(rc, 0, err) + self.assertNotIn("outdated", out) + + def test_check_json(self): + rc, out, _ = self.check(self.pins((4353, "1.8.24")), "--format", "json") + self.assertEqual(rc, 3) + (item,) = json.loads(out) + self.assertEqual( + set(item), {"app_id", "title", "pinned", "latest", "status", "url", "compatibility", "warnings"} + ) + self.assertEqual((item["pinned"], item["latest"], item["status"]), ("1.8.24", "1.8.26", "outdated")) + self.assertEqual(item["url"], "%s/app/4353/release/1.8.26/download/" % self.server.url) + + def test_check_reads_the_splunk_version_from_the_compose_image(self): + compose = "services:\n splunk:\n image: splunk/splunk:10.6.1\n environment:\n SPLUNK_APPS_URL: %s\n" + # 1.8.26 only supports Splunk up to 10.4, so on 10.6 it is not eligible. + rc, out, err = self.check(compose % (URL % (4353, "1.8.26")), "--format", "json", name="docker-compose.yml") + self.assertEqual(rc, 3, err) + (item,) = json.loads(out) + self.assertEqual((item["latest"], item["status"]), ("1.8.24", "unsupported")) + self.assertIn("note: using Splunk 10.6 from image splunk/splunk:10.6.1 in", err) + self.assertIn("newer than the newest eligible release (1.8.24)", err) + + def test_explicit_splunk_version_beats_the_file_image(self): + compose = " image: splunk/splunk:10.6.1\n SPLUNK_APPS_URL: %s\n" % (URL % (4353, "1.8.26")) + rc, out, err = self.check(compose, "--splunk-version", "10.4", name="docker-compose.yml") + self.assertEqual(rc, 0, err) + self.assertNotIn("from image", err) + + def test_check_without_any_version_says_compatibility_is_not_checked(self): + rc, _, err = self.check(self.pins((7931, "2.1.0"))) + self.assertEqual(rc, 0) + self.assertIn("note: no Splunk version known", err) + + def test_pin_that_is_older_and_unsupported_by_the_splunk_version_is_warned(self): + rc, _, err = self.check(self.pins((4353, "1.8.1")), "--splunk-version", "10.4") + self.assertEqual(rc, 3) + self.assertRegex(err, r"(?m)^warning: .*(not in Splunkbase's release list|does not list Splunk 10.4)") + + def test_unknown_pinned_version(self): + rc, out, err = self.check(self.pins((4353, "0.0.1"))) + self.assertEqual(rc, 3) + self.assertIn("unknown", out) + self.assertIn("not in Splunkbase's release list", err) + + def test_percent_encoded_release_names_match(self): + rc, out, _ = self.check("SPLUNK_APPS_URL=%s\n" % (URL % (1606, "3.2.0%20Windows%2064%20bit"))) + self.assertEqual(rc, 0) + self.assertIn("current", out) + + def test_cloud_filter_applies_to_the_check(self): + rc, out, err = self.check(self.pins((4353, "1.8.26")), "--cloud") + self.assertEqual((rc, out), (1, "")) + self.assertIn("no release is marked Splunk Cloud compatible", err) + + def test_failure_prints_no_table_unless_partial(self): + content = self.pins((7931, "2.1.0"), (3212, "1.0.0")) + rc, out, err = self.check(content) + self.assertEqual((rc, out), (1, "")) + self.assertIn("app 3212: app not found", err) + rc, out, _ = self.check(content, "--partial") + self.assertEqual(rc, 1) + self.assertIn("7931", out) + + def test_ignored_entries_are_noted(self): + content = "SPLUNK_APPS_URL=https://example.com/a.tgz,%s\n" % (URL % (7931, "2.1.0")) + rc, _, err = self.check(content) + self.assertEqual(rc, 0) + self.assertIn("note: ignored 1 SPLUNK_APPS_URL entry", err) + + def test_file_without_pins_is_a_usage_error_with_a_hint(self): + rc, out, err = self.check("services:\n s:\n environment:\n SPLUNK_APPS_URL: ${SPLUNK_APPS_URL}\n") + self.assertEqual((rc, out), (2, "")) + self.assertIn("check the .env file", err) + + def test_unreadable_file_and_bad_combinations_are_usage_errors(self): + self.assertEqual(self.run_cli("--check", "/no/such/file")[0], 2) + self.assertEqual(self.check(self.pins((7931, "2.1.0")), "7931")[0], 2) + self.assertEqual(self.check(self.pins((7931, "2.1.0")), "--format", "env")[0], 2) + self.assertEqual(self.check(self.pins((7931, "2.1.0")), "--image", "x:9.4", "--splunk-version", "9.4")[0], 2) + + def test_check_resolves_each_app_once(self): + self.check(self.pins((4353, "1.8.24"), (4353, "1.8.25"))) + self.assertEqual(self.server.request_count("/api/v1/app/4353/release"), 1) + def test_search_table(self): rc, out, _ = self.run_cli("--search", "aws") self.assertEqual(rc, 0) diff --git a/tests/test_skill_package.py b/tests/test_skill_package.py index e710253..33bfaad 100644 --- a/tests/test_skill_package.py +++ b/tests/test_skill_package.py @@ -148,7 +148,7 @@ def test_evals_are_well_formed(self): def test_skill_md_documents_every_cli_format_and_exit_code(self): text = self.skill_md.read_text() - needles = ("--search", "--splunk-version", "--image", "--cloud", "--format env", "--partial") + needles = ("--search", "--splunk-version", "--image", "--cloud", "--check", "--format env", "--partial") for needle in needles + ("SPLUNKBASE_PASSWORD",): self.assertIn(needle, text)