feat(ext-tester): --strict and the action's strict input (#1487)

.github/actions/test-extension/action.yml

@@ -34,6 +34,13 @@ inputs:
     description: Run the plugin inside the OS sandbox when available ("true" / "false").
     required: false
     default: 'false'
+  strict:
+    description: >-
+      Fail on warnings too ("true" / "false"), for example when the OS sandbox
+      could not be applied. Needs a tester newer than 0.4.19; with an older
+      one the run fails because it does not know --strict.
+    required: false
+    default: 'false'
   junit-path:
     description: Where to write the JUnit XML report.
     required: false
@@ -111,6 +118,7 @@ runs:
         QUERY: ${{ inputs.query }}
         TABLE: ${{ inputs.table }}
         SANDBOX: ${{ inputs.sandbox }}
+        STRICT: ${{ inputs.strict }}
         JUNIT_PATH: ${{ inputs.junit-path }}
         TAP_PATH: ${{ inputs.tap-path }}
       run: |
@@ -122,6 +130,7 @@ runs:
         if [ -n "$CONNECT" ]; then ARGS+=(--connect "$CONNECT"); fi
         if [ -n "$TABLE" ]; then ARGS+=(--table "$TABLE"); fi
         if [ "$SANDBOX" = "true" ]; then ARGS+=(--sandbox); fi
+        if [ "$STRICT" = "true" ]; then ARGS+=(--strict); fi
         if [ -n "$JUNIT_PATH" ]; then ARGS+=(--junit "$JUNIT_PATH"); fi
         if [ -n "$TAP_PATH" ]; then ARGS+=(--tap "$TAP_PATH"); fi
 

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- **`querya-ext-tester --strict`** — warnings fail the run (exit 1), for example a recommended method that errors or an OS sandbox that could not be applied; with `--sandbox` in CI a green run now means the plugin really ran isolated. The `test-extension` action has a matching `strict` input.
+
 ### Fixed
 
 - **In-app update of a package install (AUR, `.deb`, `.rpm`)** — the updater no longer downloads a zip and quits when the app lives in `/opt/querya-desktop` (owned by root): the update dialog says how to update it (`yay -Syu querya-desktop`, `apt`, `dnf`) with a *Copy update command* button, or offers the release page for any other folder the user cannot write. If an in-place replace fails anyway, the app is started again and the reason is logged to `~/.local/state/querya-update.log`, instead of staying closed.

bin/querya_ext_tester.dart

@@ -4,7 +4,8 @@
 //
 //   dart run bin/querya_ext_tester.dart <extension-dir> [options]
 //
-// Exit codes: 0 all checks passed, 1 a check failed, 2 bad usage.
+// Exit codes: 0 all checks passed, 1 a check failed (with --strict, or warned),
+// 2 bad usage.
 import 'dart:convert';
 import 'dart:io';
 
@@ -24,6 +25,9 @@ Options:
   --table <name>            also check db.getTableSchema for this table
   --database <name>         database passed to db.connect
   --sandbox                 run the plugin in the OS sandbox when available
+  --strict                  treat warnings as failures (exit 1), for example a
+                            recommended method that errors or a sandbox that
+                            could not be applied
   --timeout-ms <n>          per-request timeout (default 5000)
   --junit <path>            write a JUnit XML report
   --tap <path>              write a TAP report
@@ -46,6 +50,7 @@ Future<int> _run(List<String> args) async {
   String? table;
   String? database;
   var sandbox = false;
+  var strict = false;
   var timeoutMs = 5000;
   String? junit;
   String? tap;
@@ -68,6 +73,8 @@ Future<int> _run(List<String> args) async {
         return 0;
       case '--sandbox':
         sandbox = true;
+      case '--strict':
+        strict = true;
       case '--no-color':
         color = false;
       case '--target-version' ||
@@ -168,5 +175,9 @@ Future<int> _run(List<String> args) async {
     File(trace).writeAsStringSync(report.toTraceJson());
     stdout.writeln('RPC trace of ${report.rpc.length} calls written to $trace');
   }
-  return report.isSuccess ? 0 : 1;
+  if (report.isSuccess && strict && report.warningCount > 0) {
+    stdout.writeln('--strict: ${report.warningCount} warning(s) count as '
+        'failures');
+  }
+  return report.passes(strict: strict) ? 0 : 1;
 }

docs/extension-testing.md

@@ -69,13 +69,14 @@ querya-ext-tester ./my-extension \
 | `--table <name>` | also check `db.getTableSchema` |
 | `--database <name>` | database passed to `db.connect` |
 | `--sandbox` | run the plugin in the OS sandbox (`bwrap` on Linux, `sandbox-exec` on macOS) when available |
+| `--strict` | warnings fail the run too (exit `1`): a recommended method that errors, or a sandbox that could not be applied. Not in 0.4.19 or earlier |
 | `--timeout-ms <n>` | per-request timeout (default 5000) |
 | `--junit <path>` / `--tap <path>` | write a JUnit XML / TAP report |
 | `--trace <path>` | write every RPC round trip as JSON: method, time, request and response size and a bounded view of the data (see *Following the data*) |
 | `--no-color` | plain output |
 
-Exit codes: `0` all checks passed (warnings and skips do not fail), `1` a
-check failed, `2` bad command line.
+Exit codes: `0` all checks passed (warnings and skips do not fail, unless
+`--strict`), `1` a check failed, `2` bad command line.
 
 Credentials only travel inside the `db.connect` payload over the plugin's
 stdio; they are never put on the command line of the plugin or into its
@@ -346,8 +347,9 @@ test-extension:
    the plugin in the same OS sandbox.
 5. Capture machine-readable results with `--junit report.xml` or `--tap report.tap`.
 
-There is no `--strict` mode: recommended methods only warn, and exit code `1`
-means a required check failed.
+Recommended methods only warn, and exit code `1` means a required check failed.
+Add `--strict` (with `--sandbox` in CI) so a warning, in particular "OS sandbox
+is not available", fails the run instead of passing as green.
 
 ## Running from source
 

lib/core/extensions/harness/extension_harness_report.dart

@@ -151,6 +151,11 @@ final class HarnessReport {
   /// `true` when nothing failed (warnings and skips do not fail a run).
   bool get isSuccess => failedCount == 0;
 
+  /// Like [isSuccess], and with [strict] a warning fails the run as well
+  /// (a recommended method that errors, a sandbox that could not be applied).
+  bool passes({bool strict = false}) =>
+      isSuccess && (!strict || warningCount == 0);
+
   /// Human-readable summary, one line per check plus indented details.
   String toConsole({bool color = false}) {
     String paint(String code, String text) =>

test/core/extensions/harness/extension_harness_report_test.dart

@@ -41,6 +41,29 @@ void main() {
     expect(r.isSuccess, isFalse);
   });
 
+  test('strict turns warnings into failures, never the other way', () {
+    HarnessReport with_(List<HarnessCheckStatus> statuses) => HarnessReport(
+          extensionId: 'acme.db',
+          targetVersion: const HostVersion(0, 4, 18),
+          totalDuration: Duration.zero,
+          checks: [
+            for (var i = 0; i < statuses.length; i++)
+              HarnessCheck(name: 'c$i', status: statuses[i]),
+          ],
+        );
+
+    final clean = with_([HarnessCheckStatus.passed, HarnessCheckStatus.skipped]);
+    final warned = with_([HarnessCheckStatus.passed, HarnessCheckStatus.warning]);
+    final failed = with_([HarnessCheckStatus.failed]);
+
+    expect(clean.passes(), isTrue);
+    expect(clean.passes(strict: true), isTrue);
+    expect(warned.passes(), isTrue);
+    expect(warned.passes(strict: true), isFalse);
+    expect(failed.passes(), isFalse);
+    expect(failed.passes(strict: true), isFalse);
+  });
+
   test('console output lists every check and a summary', () {
     final text = _report().toConsole();
     expect(text, contains('PASS system.handshake (12 ms)'));