ci: nightly device integration tests (Android emulator + iOS simulator)

.github/workflows/device-tests.yml

@@ -0,0 +1,92 @@
+name: Device tests
+
+# The normal PR CI never boots a device, so bugs that only show on a real Android/iOS runtime
+# (see issues #30, #66, and lib_audit_5 broken by #84) stay invisible until someone runs
+# example/integration_test by hand. This runs them on GitHub-hosted emulator/simulator.
+# Not a merge gate (emulators are slower and occasionally flaky): nightly + on demand, and on
+# PRs that touch this workflow itself so it can be validated.
+on:
+  schedule:
+    - cron: '0 20 * * *'   # 03:00 ICT
+  workflow_dispatch:
+    inputs:
+      groups:
+        description: 'Comma-separated group names (empty = all)'
+        required: false
+        default: ''
+  pull_request:
+    paths:
+      - '.github/workflows/device-tests.yml'
+      - 'scripts/run_device_groups.py'
+
+concurrency:
+  group: device-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  android:
+    name: Android emulator (API 34)
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: subosito/flutter-action@v2
+        with:
+          flutter-version-file: .fvmrc
+          channel: ''
+          cache: true
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+
+      - name: Enable KVM
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      - name: Install dependencies
+        run: cd example && flutter pub get
+
+      - name: Run device groups
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: 34
+          arch: x86_64
+          target: google_apis
+          profile: pixel_6
+          disable-animations: true
+          script: python3 scripts/run_device_groups.py emulator-5554 --only "${{ github.event.inputs.groups }}"
+
+  ios:
+    name: iOS simulator
+    runs-on: macos-latest
+    timeout-minutes: 120
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: subosito/flutter-action@v2
+        with:
+          flutter-version-file: .fvmrc
+          channel: ''
+          cache: true
+
+      - name: Boot simulator
+        id: sim
+        run: |
+          UDID=$(xcrun simctl list devices available -j | python3 -c "
+          import json,sys
+          d=json.load(sys.stdin)['devices']
+          c=[x for rt,v in d.items() if 'iOS' in rt for x in v if 'iPhone' in x['name'] and x['isAvailable']]
+          print(c[-1]['udid'])")
+          xcrun simctl boot "$UDID"
+          echo "udid=$UDID" >> "$GITHUB_OUTPUT"
+
+      - name: Install dependencies
+        run: cd example && flutter pub get
+
+      - name: Run device groups
+        run: python3 scripts/run_device_groups.py "${{ steps.sim.outputs.udid }}" --only "${{ github.event.inputs.groups }}"

scripts/run_device_groups.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""Run example/integration_test/device_integration_test.dart group by group on one device.
+
+The full file hangs when run in one go (see CONTRIBUTING / project notes), so each group is a
+separate `flutter test --plain-name` with its own timeout. A failed group is retried once; a
+group that passes on retry is reported as FLAKY (green but visible).
+
+Usage: scripts/run_device_groups.py <flutter-device-id> [--only "Group A,Group B"] [--timeout 900]
+Exit code is non-zero if any group failed twice.
+"""
+import argparse, os, subprocess, sys, time
+
+# Android-only/iOS-only groups simply report "no tests" on the other platform; that is not a failure.
+GROUPS = ['issue_84:', 'Trigger Types', 'ExistingPolicy', 'Constraints', 'All Workers',
+          'TLS Certificate Pinning', 'Task Chains', 'Tags', 'Cancellation',
+          'Events and Progress Streams', 'DartWorker Constraint and Delay Enforcement',
+          'Custom Native Workers', 'New v1.1 Features', 'Tag utilities', 'Error resilience',
+          'Task status lifecycle', 'Trigger type acceptance', 'Battery restriction diagnostics',
+          'Issue #62', 'Issue #36', 'Concurrent tasks', 'Chain data flow', 'Issue #38/#39',
+          'Issue #57', 'v1_5_0']
+
+
+def run(group, device, timeout, cwd):
+    try:
+        r = subprocess.run(
+            ['flutter', 'test', 'integration_test/device_integration_test.dart', '-d', device,
+             '--plain-name', group], cwd=cwd, capture_output=True, text=True, timeout=timeout)
+    except subprocess.TimeoutExpired:
+        return 'timeout', f'timeout after {timeout}s'
+    out = r.stdout + r.stderr
+    if 'All tests passed' in out:
+        return 'pass', ''
+    if 'No tests ran' in out or ('+0' in out and 'Some tests failed' not in out and '[E]' not in out):
+        return 'none', 'no tests for this platform'
+    fails = [l.strip()[-160:] for l in out.splitlines() if '[E]' in l or 'Some tests failed' in l]
+    return 'fail', ' | '.join(fails[-2:]) or 'unknown failure'
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument('device')
+    ap.add_argument('--only', default='')
+    ap.add_argument('--timeout', type=int, default=900)
+    a = ap.parse_args()
+    groups = [g for g in a.only.split(',') if g] or GROUPS
+    cwd = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'example')
+    rows, failed = [], False
+    for g in groups:
+        t = time.time()
+        status, note = run(g, a.device, a.timeout, cwd)
+        if status in ('fail', 'timeout'):
+            status2, note2 = run(g, a.device, a.timeout, cwd)
+            if status2 == 'pass':
+                status, note = 'FLAKY', f'failed once ({note}), passed on retry'
+            else:
+                status, note, failed = 'FAIL', note2 or note, True
+        rows.append((g, status, int(time.time() - t), note))
+        print(f'{g}: {status} ({int(time.time() - t)}s) {note}', flush=True)
+    summary = os.environ.get('GITHUB_STEP_SUMMARY')
+    if summary:
+        with open(summary, 'a') as f:
+            f.write(f'### Device tests on `{a.device}`\n\n| group | result | s | note |\n|---|---|---|---|\n')
+            for g, s, secs, n in rows:
+                f.write(f'| {g} | {s} | {secs} | {n} |\n')
+    sys.exit(1 if failed else 0)
+
+
+if __name__ == '__main__':
+    main()