From f750dd6bc22eb4bb5c28aaf06fc623f935edba09 Mon Sep 17 00:00:00 2001 From: ZhuchkaTriplesix Date: Fri, 16 Oct 2026 21:28:00 +0300 Subject: [PATCH] feat(ext-tester): --strict fails the run on warnings, and a strict input for the action (#1487) A missing OS sandbox is only a warning, so --sandbox could pass green without isolating anything. --strict turns warnings into exit code 1. --- .github/actions/test-extension/action.yml | 9 ++++++++ CHANGELOG.md | 4 ++++ bin/querya_ext_tester.dart | 15 ++++++++++-- docs/extension-testing.md | 10 ++++---- .../harness/extension_harness_report.dart | 5 ++++ .../extension_harness_report_test.dart | 23 +++++++++++++++++++ 6 files changed, 60 insertions(+), 6 deletions(-) diff --git a/.github/actions/test-extension/action.yml b/.github/actions/test-extension/action.yml index 57a2bb3f..bb42d49f 100644 --- a/.github/actions/test-extension/action.yml +++ b/.github/actions/test-extension/action.yml @@ -34,6 +34,13 @@ inputs: description: Run the plugin inside the OS sandbox when available ("true" / "false"). required: false default: 'false' + strict: + description: >- + Fail on warnings too ("true" / "false"), for example when the OS sandbox + could not be applied. Needs a tester newer than 0.4.19; with an older + one the run fails because it does not know --strict. + required: false + default: 'false' junit-path: description: Where to write the JUnit XML report. required: false @@ -111,6 +118,7 @@ runs: QUERY: ${{ inputs.query }} TABLE: ${{ inputs.table }} SANDBOX: ${{ inputs.sandbox }} + STRICT: ${{ inputs.strict }} JUNIT_PATH: ${{ inputs.junit-path }} TAP_PATH: ${{ inputs.tap-path }} run: | @@ -122,6 +130,7 @@ runs: if [ -n "$CONNECT" ]; then ARGS+=(--connect "$CONNECT"); fi if [ -n "$TABLE" ]; then ARGS+=(--table "$TABLE"); fi if [ "$SANDBOX" = "true" ]; then ARGS+=(--sandbox); fi + if [ "$STRICT" = "true" ]; then ARGS+=(--strict); fi if [ -n "$JUNIT_PATH" ]; then ARGS+=(--junit "$JUNIT_PATH"); fi if [ -n "$TAP_PATH" ]; then ARGS+=(--tap "$TAP_PATH"); fi diff --git a/CHANGELOG.md b/CHANGELOG.md index aca93b66..80546183 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **`querya-ext-tester --strict`** — warnings fail the run (exit 1), for example a recommended method that errors or an OS sandbox that could not be applied; with `--sandbox` in CI a green run now means the plugin really ran isolated. The `test-extension` action has a matching `strict` input. + ### Fixed - **In-app update of a package install (AUR, `.deb`, `.rpm`)** — the updater no longer downloads a zip and quits when the app lives in `/opt/querya-desktop` (owned by root): the update dialog says how to update it (`yay -Syu querya-desktop`, `apt`, `dnf`) with a *Copy update command* button, or offers the release page for any other folder the user cannot write. If an in-place replace fails anyway, the app is started again and the reason is logged to `~/.local/state/querya-update.log`, instead of staying closed. diff --git a/bin/querya_ext_tester.dart b/bin/querya_ext_tester.dart index 6956ab20..7fd0ddc3 100644 --- a/bin/querya_ext_tester.dart +++ b/bin/querya_ext_tester.dart @@ -4,7 +4,8 @@ // // dart run bin/querya_ext_tester.dart [options] // -// Exit codes: 0 all checks passed, 1 a check failed, 2 bad usage. +// Exit codes: 0 all checks passed, 1 a check failed (with --strict, or warned), +// 2 bad usage. import 'dart:convert'; import 'dart:io'; @@ -24,6 +25,9 @@ Options: --table also check db.getTableSchema for this table --database database passed to db.connect --sandbox run the plugin in the OS sandbox when available + --strict treat warnings as failures (exit 1), for example a + recommended method that errors or a sandbox that + could not be applied --timeout-ms per-request timeout (default 5000) --junit write a JUnit XML report --tap write a TAP report @@ -46,6 +50,7 @@ Future _run(List args) async { String? table; String? database; var sandbox = false; + var strict = false; var timeoutMs = 5000; String? junit; String? tap; @@ -68,6 +73,8 @@ Future _run(List args) async { return 0; case '--sandbox': sandbox = true; + case '--strict': + strict = true; case '--no-color': color = false; case '--target-version' || @@ -168,5 +175,9 @@ Future _run(List args) async { File(trace).writeAsStringSync(report.toTraceJson()); stdout.writeln('RPC trace of ${report.rpc.length} calls written to $trace'); } - return report.isSuccess ? 0 : 1; + if (report.isSuccess && strict && report.warningCount > 0) { + stdout.writeln('--strict: ${report.warningCount} warning(s) count as ' + 'failures'); + } + return report.passes(strict: strict) ? 0 : 1; } diff --git a/docs/extension-testing.md b/docs/extension-testing.md index e7922504..899ead4b 100644 --- a/docs/extension-testing.md +++ b/docs/extension-testing.md @@ -69,13 +69,14 @@ querya-ext-tester ./my-extension \ | `--table ` | also check `db.getTableSchema` | | `--database ` | database passed to `db.connect` | | `--sandbox` | run the plugin in the OS sandbox (`bwrap` on Linux, `sandbox-exec` on macOS) when available | +| `--strict` | warnings fail the run too (exit `1`): a recommended method that errors, or a sandbox that could not be applied. Not in 0.4.19 or earlier | | `--timeout-ms ` | per-request timeout (default 5000) | | `--junit ` / `--tap ` | write a JUnit XML / TAP report | | `--trace ` | write every RPC round trip as JSON: method, time, request and response size and a bounded view of the data (see *Following the data*) | | `--no-color` | plain output | -Exit codes: `0` all checks passed (warnings and skips do not fail), `1` a -check failed, `2` bad command line. +Exit codes: `0` all checks passed (warnings and skips do not fail, unless +`--strict`), `1` a check failed, `2` bad command line. Credentials only travel inside the `db.connect` payload over the plugin's stdio; they are never put on the command line of the plugin or into its @@ -346,8 +347,9 @@ test-extension: the plugin in the same OS sandbox. 5. Capture machine-readable results with `--junit report.xml` or `--tap report.tap`. -There is no `--strict` mode: recommended methods only warn, and exit code `1` -means a required check failed. +Recommended methods only warn, and exit code `1` means a required check failed. +Add `--strict` (with `--sandbox` in CI) so a warning, in particular "OS sandbox +is not available", fails the run instead of passing as green. ## Running from source diff --git a/lib/core/extensions/harness/extension_harness_report.dart b/lib/core/extensions/harness/extension_harness_report.dart index 82613739..224d18d7 100644 --- a/lib/core/extensions/harness/extension_harness_report.dart +++ b/lib/core/extensions/harness/extension_harness_report.dart @@ -151,6 +151,11 @@ final class HarnessReport { /// `true` when nothing failed (warnings and skips do not fail a run). bool get isSuccess => failedCount == 0; + /// Like [isSuccess], and with [strict] a warning fails the run as well + /// (a recommended method that errors, a sandbox that could not be applied). + bool passes({bool strict = false}) => + isSuccess && (!strict || warningCount == 0); + /// Human-readable summary, one line per check plus indented details. String toConsole({bool color = false}) { String paint(String code, String text) => diff --git a/test/core/extensions/harness/extension_harness_report_test.dart b/test/core/extensions/harness/extension_harness_report_test.dart index c7f2b534..236032d9 100644 --- a/test/core/extensions/harness/extension_harness_report_test.dart +++ b/test/core/extensions/harness/extension_harness_report_test.dart @@ -41,6 +41,29 @@ void main() { expect(r.isSuccess, isFalse); }); + test('strict turns warnings into failures, never the other way', () { + HarnessReport with_(List statuses) => HarnessReport( + extensionId: 'acme.db', + targetVersion: const HostVersion(0, 4, 18), + totalDuration: Duration.zero, + checks: [ + for (var i = 0; i < statuses.length; i++) + HarnessCheck(name: 'c$i', status: statuses[i]), + ], + ); + + final clean = with_([HarnessCheckStatus.passed, HarnessCheckStatus.skipped]); + final warned = with_([HarnessCheckStatus.passed, HarnessCheckStatus.warning]); + final failed = with_([HarnessCheckStatus.failed]); + + expect(clean.passes(), isTrue); + expect(clean.passes(strict: true), isTrue); + expect(warned.passes(), isTrue); + expect(warned.passes(strict: true), isFalse); + expect(failed.passes(), isFalse); + expect(failed.passes(strict: true), isFalse); + }); + test('console output lists every check and a summary', () { final text = _report().toConsole(); expect(text, contains('PASS system.handshake (12 ms)'));