diff --git a/.github/workflows/device-tests.yml b/.github/workflows/device-tests.yml new file mode 100644 index 0000000..7e32e5d --- /dev/null +++ b/.github/workflows/device-tests.yml @@ -0,0 +1,92 @@ +name: Device tests + +# The normal PR CI never boots a device, so bugs that only show on a real Android/iOS runtime +# (see issues #30, #66, and lib_audit_5 broken by #84) stay invisible until someone runs +# example/integration_test by hand. This runs them on GitHub-hosted emulator/simulator. +# Not a merge gate (emulators are slower and occasionally flaky): nightly + on demand, and on +# PRs that touch this workflow itself so it can be validated. +on: + schedule: + - cron: '0 20 * * *' # 03:00 ICT + workflow_dispatch: + inputs: + groups: + description: 'Comma-separated group names (empty = all)' + required: false + default: '' + pull_request: + paths: + - '.github/workflows/device-tests.yml' + - 'scripts/run_device_groups.py' + +concurrency: + group: device-tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + android: + name: Android emulator (API 34) + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + + - uses: subosito/flutter-action@v2 + with: + flutter-version-file: .fvmrc + channel: '' + cache: true + + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: Install dependencies + run: cd example && flutter pub get + + - name: Run device groups + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 34 + arch: x86_64 + target: google_apis + profile: pixel_6 + disable-animations: true + script: python3 scripts/run_device_groups.py emulator-5554 --only "${{ github.event.inputs.groups }}" + + ios: + name: iOS simulator + runs-on: macos-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + + - uses: subosito/flutter-action@v2 + with: + flutter-version-file: .fvmrc + channel: '' + cache: true + + - name: Boot simulator + id: sim + run: | + UDID=$(xcrun simctl list devices available -j | python3 -c " + import json,sys + d=json.load(sys.stdin)['devices'] + c=[x for rt,v in d.items() if 'iOS' in rt for x in v if 'iPhone' in x['name'] and x['isAvailable']] + print(c[-1]['udid'])") + xcrun simctl boot "$UDID" + echo "udid=$UDID" >> "$GITHUB_OUTPUT" + + - name: Install dependencies + run: cd example && flutter pub get + + - name: Run device groups + run: python3 scripts/run_device_groups.py "${{ steps.sim.outputs.udid }}" --only "${{ github.event.inputs.groups }}" diff --git a/scripts/run_device_groups.py b/scripts/run_device_groups.py new file mode 100755 index 0000000..c771dc6 --- /dev/null +++ b/scripts/run_device_groups.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Run example/integration_test/device_integration_test.dart group by group on one device. + +The full file hangs when run in one go (see CONTRIBUTING / project notes), so each group is a +separate `flutter test --plain-name` with its own timeout. A failed group is retried once; a +group that passes on retry is reported as FLAKY (green but visible). + +Usage: scripts/run_device_groups.py [--only "Group A,Group B"] [--timeout 900] +Exit code is non-zero if any group failed twice. +""" +import argparse, os, subprocess, sys, time + +# Android-only/iOS-only groups simply report "no tests" on the other platform; that is not a failure. +GROUPS = ['issue_84:', 'Trigger Types', 'ExistingPolicy', 'Constraints', 'All Workers', + 'TLS Certificate Pinning', 'Task Chains', 'Tags', 'Cancellation', + 'Events and Progress Streams', 'DartWorker Constraint and Delay Enforcement', + 'Custom Native Workers', 'New v1.1 Features', 'Tag utilities', 'Error resilience', + 'Task status lifecycle', 'Trigger type acceptance', 'Battery restriction diagnostics', + 'Issue #62', 'Issue #36', 'Concurrent tasks', 'Chain data flow', 'Issue #38/#39', + 'Issue #57', 'v1_5_0'] + + +def run(group, device, timeout, cwd): + try: + r = subprocess.run( + ['flutter', 'test', 'integration_test/device_integration_test.dart', '-d', device, + '--plain-name', group], cwd=cwd, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired: + return 'timeout', f'timeout after {timeout}s' + out = r.stdout + r.stderr + if 'All tests passed' in out: + return 'pass', '' + if 'No tests ran' in out or ('+0' in out and 'Some tests failed' not in out and '[E]' not in out): + return 'none', 'no tests for this platform' + fails = [l.strip()[-160:] for l in out.splitlines() if '[E]' in l or 'Some tests failed' in l] + return 'fail', ' | '.join(fails[-2:]) or 'unknown failure' + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('device') + ap.add_argument('--only', default='') + ap.add_argument('--timeout', type=int, default=900) + a = ap.parse_args() + groups = [g for g in a.only.split(',') if g] or GROUPS + cwd = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'example') + rows, failed = [], False + for g in groups: + t = time.time() + status, note = run(g, a.device, a.timeout, cwd) + if status in ('fail', 'timeout'): + status2, note2 = run(g, a.device, a.timeout, cwd) + if status2 == 'pass': + status, note = 'FLAKY', f'failed once ({note}), passed on retry' + else: + status, note, failed = 'FAIL', note2 or note, True + rows.append((g, status, int(time.time() - t), note)) + print(f'{g}: {status} ({int(time.time() - t)}s) {note}', flush=True) + summary = os.environ.get('GITHUB_STEP_SUMMARY') + if summary: + with open(summary, 'a') as f: + f.write(f'### Device tests on `{a.device}`\n\n| group | result | s | note |\n|---|---|---|---|\n') + for g, s, secs, n in rows: + f.write(f'| {g} | {s} | {secs} | {n} |\n') + sys.exit(1 if failed else 0) + + +if __name__ == '__main__': + main()