|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Run example/integration_test/device_integration_test.dart group by group on one device. |
| 3 | +
|
| 4 | +The full file hangs when run in one go (see CONTRIBUTING / project notes), so each group is a |
| 5 | +separate `flutter test --plain-name` with its own timeout. A failed group is retried once; a |
| 6 | +group that passes on retry is reported as FLAKY (green but visible). |
| 7 | +
|
| 8 | +Usage: scripts/run_device_groups.py <flutter-device-id> [--only "Group A,Group B"] [--timeout 900] |
| 9 | +Exit code is non-zero if any group failed twice. |
| 10 | +""" |
| 11 | +import argparse, os, subprocess, sys, time |
| 12 | + |
| 13 | +# Android-only/iOS-only groups simply report "no tests" on the other platform; that is not a failure. |
| 14 | +GROUPS = ['issue_84:', 'Trigger Types', 'ExistingPolicy', 'Constraints', 'All Workers', |
| 15 | + 'TLS Certificate Pinning', 'Task Chains', 'Tags', 'Cancellation', |
| 16 | + 'Events and Progress Streams', 'DartWorker Constraint and Delay Enforcement', |
| 17 | + 'Custom Native Workers', 'New v1.1 Features', 'Tag utilities', 'Error resilience', |
| 18 | + 'Task status lifecycle', 'Trigger type acceptance', 'Battery restriction diagnostics', |
| 19 | + 'Issue #62', 'Issue #36', 'Concurrent tasks', 'Chain data flow', 'Issue #38/#39', |
| 20 | + 'Issue #57', 'v1_5_0'] |
| 21 | + |
| 22 | + |
| 23 | +def run(group, device, timeout, cwd): |
| 24 | + try: |
| 25 | + r = subprocess.run( |
| 26 | + ['flutter', 'test', 'integration_test/device_integration_test.dart', '-d', device, |
| 27 | + '--plain-name', group], cwd=cwd, capture_output=True, text=True, timeout=timeout) |
| 28 | + except subprocess.TimeoutExpired: |
| 29 | + return 'timeout', f'timeout after {timeout}s' |
| 30 | + out = r.stdout + r.stderr |
| 31 | + if 'All tests passed' in out: |
| 32 | + return 'pass', '' |
| 33 | + if 'No tests ran' in out or ('+0' in out and 'Some tests failed' not in out and '[E]' not in out): |
| 34 | + return 'none', 'no tests for this platform' |
| 35 | + fails = [l.strip()[-160:] for l in out.splitlines() if '[E]' in l or 'Some tests failed' in l] |
| 36 | + return 'fail', ' | '.join(fails[-2:]) or 'unknown failure' |
| 37 | + |
| 38 | + |
| 39 | +def main(): |
| 40 | + ap = argparse.ArgumentParser() |
| 41 | + ap.add_argument('device') |
| 42 | + ap.add_argument('--only', default='') |
| 43 | + ap.add_argument('--timeout', type=int, default=900) |
| 44 | + a = ap.parse_args() |
| 45 | + groups = [g for g in a.only.split(',') if g] or GROUPS |
| 46 | + cwd = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'example') |
| 47 | + rows, failed = [], False |
| 48 | + for g in groups: |
| 49 | + t = time.time() |
| 50 | + status, note = run(g, a.device, a.timeout, cwd) |
| 51 | + if status in ('fail', 'timeout'): |
| 52 | + status2, note2 = run(g, a.device, a.timeout, cwd) |
| 53 | + if status2 == 'pass': |
| 54 | + status, note = 'FLAKY', f'failed once ({note}), passed on retry' |
| 55 | + else: |
| 56 | + status, note, failed = 'FAIL', note2 or note, True |
| 57 | + rows.append((g, status, int(time.time() - t), note)) |
| 58 | + print(f'{g}: {status} ({int(time.time() - t)}s) {note}', flush=True) |
| 59 | + summary = os.environ.get('GITHUB_STEP_SUMMARY') |
| 60 | + if summary: |
| 61 | + with open(summary, 'a') as f: |
| 62 | + f.write(f'### Device tests on `{a.device}`\n\n| group | result | s | note |\n|---|---|---|---|\n') |
| 63 | + for g, s, secs, n in rows: |
| 64 | + f.write(f'| {g} | {s} | {secs} | {n} |\n') |
| 65 | + sys.exit(1 if failed else 0) |
| 66 | + |
| 67 | + |
| 68 | +if __name__ == '__main__': |
| 69 | + main() |
0 commit comments