From 5272c79bbaf26560c0dca91745f1c19239cd24ae Mon Sep 17 00:00:00 2001 From: Maximilian Haupt Date: Tue, 6 Oct 2026 11:57:05 +0200 Subject: [PATCH 1/2] ci: make the iOS E2E job tolerate staging and simulator flakiness The flows run against live staging on hosted runners, and most failures so far came from there rather than from the code: a 502 from staging's OIDC discovery, its first-layer page loading past the layer's 8s timeout, and Maestro's iOS driver not starting in time. - A failed flow runs once more, and every flow does if Maestro stops before reporting any. Flows start from a clean slate, so a rerun is safe. A pass on retry still uploads the debug output, Metro's log included, and leaves a warning annotation naming the flows. - The default-config job gives the layer's page 30s to load, through a new EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS override in the example. - The simulator boots at the start of the job, so its first boot happens while the app builds, and Maestro's driver gets 300s instead of 240s to start. - Maestro (2.11.0) and the simulator (iPhone 16 Pro, iOS 18.5) are pinned, so a Maestro release or a runner image update can't change the job between runs. --- .github/workflows/e2e-ios.yml | 120 ++++++++++++++++++++++++++------ examples/onetrust/e2e/README.md | 18 +++++ examples/onetrust/src/Config.ts | 12 +++- 3 files changed, 127 insertions(+), 23 deletions(-) diff --git a/.github/workflows/e2e-ios.yml b/.github/workflows/e2e-ios.yml index cb79b65..b011a85 100644 --- a/.github/workflows/e2e-ios.yml +++ b/.github/workflows/e2e-ios.yml @@ -20,6 +20,13 @@ jobs: # macOS Sequoia+ and a full Xcode 26 install, neither available on 14. runs-on: macos-15 name: e2e-ios (${{ matrix.scenario }}) + env: + # Pinned rather than "newest" and "first iPhone the image lists", so a + # Maestro release or a runner image update can't change the job + # between two runs of the same commit. + MAESTRO_VERSION: '2.11.0' + SIM_DEVICE: 'iPhone 16 Pro' + SIM_RUNTIME: 'com.apple.CoreSimulator.SimRuntime.iOS-18-5' strategy: fail-fast: false matrix: @@ -27,23 +34,54 @@ jobs: # setup and build, so every flow that runs against the example's # default config shares the first one, and only flows that need Metro # started with other EXPO_PUBLIC_* values get a job of their own. + # + # layer_page_load_timeout_ms gives the layer's page longer to load + # from live staging than the SDK's 8s default; cmp-timeout-fail-open + # forces every timeout down instead, and the CMP-failure flow never + # gets as far as the layer. include: - scenario: default-config flows: happy-path cancelled-login deny-all consent-survives-relaunch onetrust_app_id: '' gate_timeout_ms: '' + layer_page_load_timeout_ms: '30000' - scenario: onetrust-adapter-failure flows: onetrust-adapter-failure onetrust_app_id: 'e2e-invalid-app-id' gate_timeout_ms: '' + layer_page_load_timeout_ms: '' - scenario: cmp-timeout-fail-open flows: cmp-timeout-fail-open onetrust_app_id: '' gate_timeout_ms: '1' + layer_page_load_timeout_ms: '' steps: - name: Checkout uses: actions/checkout@v4 + # macos-15 defaults xcode-select to 16.4; idb-companion needs a full + # Xcode 26 install (CLT alone isn't enough), which is present on the + # image but not selected by default. Selected first, since the + # simulator boots against it. + - name: Select Xcode 26 + run: sudo xcode-select -s /Applications/Xcode_26.2.app + + # Booted now, without waiting, so the simulator gets through its first + # boot (data migration included) while dependencies install and the + # app builds, rather than inside the flow step. + - name: Boot simulator + run: | + udid=$(xcrun simctl list devices available -j \ + | jq -r --arg rt "$SIM_RUNTIME" --arg name "$SIM_DEVICE" \ + '(.devices[$rt] // []) | map(select(.name == $name)) | .[0].udid // empty') + if [ -z "$udid" ]; then + echo "::error::No available $SIM_DEVICE simulator on $SIM_RUNTIME" + xcrun simctl list devices available + exit 1 + fi + echo "SIM_UDID=$udid" >> "$GITHUB_ENV" + xcrun simctl boot "$udid" + - name: Setup uses: ./.github/actions/setup @@ -53,12 +91,7 @@ jobs: distribution: 'zulu' java-version: '17' - # macos-15 defaults xcode-select to 16.4; idb-companion needs a full - # Xcode 26 install (CLT alone isn't enough), which is present on the - # image but not selected by default. - - name: Select Xcode 26 - run: sudo xcode-select -s /Applications/Xcode_26.2.app - + # Installs MAESTRO_VERSION from the job's env. - name: Install Maestro and idb-companion run: | curl -Ls "https://get.maestro.mobile.dev" | bash @@ -109,6 +142,7 @@ jobs: env: EXPO_PUBLIC_ONETRUST_APP_ID: ${{ matrix.onetrust_app_id }} EXPO_PUBLIC_GATE_TIMEOUT_MS: ${{ matrix.gate_timeout_ms }} + EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS: ${{ matrix.layer_page_load_timeout_ms }} # A Debug build fetches its JS bundle from Metro at app-launch time, # so the env vars that pick the per-flow scenario have to be set on # the Metro process, not on the (now scenario-independent) build @@ -138,35 +172,77 @@ jobs: # Maestro's own XCTest-runner driver bootstrap ("iOS driver not # ready in time") can take longer than its default on a # GitHub-hosted runner than on a local machine. - MAESTRO_DRIVER_STARTUP_TIMEOUT: '240000' + MAESTRO_DRIVER_STARTUP_TIMEOUT: '300000' FLOWS: ${{ matrix.flows }} + E2E_STAGING_EMAIL: ${{ secrets.E2E_STAGING_EMAIL }} + E2E_STAGING_PASSWORD: ${{ secrets.E2E_STAGING_PASSWORD }} run: | - DEVICE_UDID=$(xcrun simctl list devices available -j | jq -r '[.devices[] | .[] | select(.name | test("iPhone"))][0].udid') - xcrun simctl boot "$DEVICE_UDID" - xcrun simctl bootstatus "$DEVICE_UDID" -b - xcrun simctl install "$DEVICE_UDID" ios/build/Build/Products/Debug-iphonesimulator/ContentpassOnetrustExample.app - flow_files=() - for flow in $FLOWS; do - flow_files+=("e2e/flows/$flow.yaml") - done - # One invocation runs every flow even when an earlier one fails, - # and fails the step if any did. - maestro test "${flow_files[@]}" \ - -e E2E_STAGING_EMAIL="${{ secrets.E2E_STAGING_EMAIL }}" \ - -e E2E_STAGING_PASSWORD="${{ secrets.E2E_STAGING_PASSWORD }}" + xcrun simctl bootstatus "$SIM_UDID" -b + xcrun simctl install "$SIM_UDID" ios/build/Build/Products/Debug-iphonesimulator/ContentpassOnetrustExample.app + + # One invocation runs every flow it's given, even after one fails, + # and records in a JUnit report which ones did. + run_flows() { + local report=$1 + shift + local files=() + for flow in "$@"; do + files+=("e2e/flows/$flow.yaml") + done + maestro test --udid "$SIM_UDID" --format junit --output "$report" \ + "${files[@]}" \ + -e E2E_STAGING_EMAIL="$E2E_STAGING_EMAIL" \ + -e E2E_STAGING_PASSWORD="$E2E_STAGING_PASSWORD" + } + + failed_flows() { + python3 - "$1" <<'PY' + import sys + import xml.etree.ElementTree as ET + + for case in ET.parse(sys.argv[1]).getroot().iter("testcase"): + if case.find("failure") is not None or case.find("error") is not None: + print(case.get("name")) + PY + } + + if run_flows /tmp/maestro-attempt-1.xml $FLOWS; then + exit 0 + fi + + # Every flow starts from a clean slate, so a failed one can simply + # run again. Without a report, Maestro never got as far as a flow + # (most often its iOS driver not starting), so all of them do. + retry="" + if [ -s /tmp/maestro-attempt-1.xml ]; then + retry=$(failed_flows /tmp/maestro-attempt-1.xml | xargs) + fi + if [ -n "$retry" ]; then + echo "::warning title=E2E retry::Retrying failed flows: $retry" + else + retry=$FLOWS + echo "::warning title=E2E retry::Maestro stopped before reporting any flow; retrying all of them" + fi + echo "E2E_RETRIED=true" >> "$GITHUB_ENV" + + run_flows /tmp/maestro-attempt-2.xml $retry + echo "::warning title=Flaky E2E run::Passed on retry: $retry" # Metro's output is the app's JS console, including the SDK's # console.debug lines (every message from the layer's WebView, the # gate's and the CMP adapter's progress), none of which reach the # simulator log that Maestro collects. + # + # Also collected when a retry passed, so a flake still leaves its + # evidence behind instead of disappearing into a green run. - name: Add Metro log to debug output - if: failure() + if: failure() || env.E2E_RETRIED == 'true' run: | mkdir -p ~/.maestro/tests cp /tmp/metro.log ~/.maestro/tests/ 2>/dev/null || true - name: Upload Maestro debug output - if: failure() + if: failure() || env.E2E_RETRIED == 'true' uses: actions/upload-artifact@v4 with: name: maestro-debug-${{ matrix.scenario }} diff --git a/examples/onetrust/e2e/README.md b/examples/onetrust/e2e/README.md index d89ee2e..af5b930 100644 --- a/examples/onetrust/e2e/README.md +++ b/examples/onetrust/e2e/README.md @@ -69,6 +69,24 @@ Every flow starts with `clearKeychain` and a `clearState` launch, since a job runs its flows one after another on the same simulator, and the login tokens live in the keychain, which `clearState` leaves alone. +## In CI + +The flows run against live staging on hosted runners, so the job is built +to tolerate what fails there without the code being at fault: + +- A failed flow runs once more, and so does every flow if Maestro stops + before reporting any (its iOS driver not starting, most often). A pass on + retry turns the job green with a warning annotation naming the flows, and + still uploads the debug output, Metro's log included, so the flake can be + looked into. +- The `default-config` job starts Metro with + `EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS=30000`, so a slow page from + staging doesn't fail the gate open halfway through a flow; the SDK's + default is 8s. +- The simulator boots at the start of the job and settles while the app + builds. It is pinned (`SIM_DEVICE`, `SIM_RUNTIME`), as is Maestro + (`MAESTRO_VERSION`), in the workflow's job `env`. + ## Running a flow ```sh diff --git a/examples/onetrust/src/Config.ts b/examples/onetrust/src/Config.ts index d46975c..21da959 100644 --- a/examples/onetrust/src/Config.ts +++ b/examples/onetrust/src/Config.ts @@ -25,6 +25,14 @@ const forcedTimeoutMs = process.env.EXPO_PUBLIC_GATE_TIMEOUT_MS ? Number(process.env.EXPO_PUBLIC_GATE_TIMEOUT_MS) : undefined; +// Lets an E2E run against live staging give the layer's page longer to load +// than the SDK's default, so a slow response doesn't fail the gate open +// halfway through a flow that is testing something else. +const layerPageLoadTimeoutMs = process.env + .EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS + ? Number(process.env.EXPO_PUBLIC_LAYER_PAGE_LOAD_TIMEOUT_MS) + : undefined; + export const GATE_TIMEOUTS: ContentpassGateTimeouts | undefined = forcedTimeoutMs ? { @@ -37,4 +45,6 @@ export const GATE_TIMEOUTS: ContentpassGateTimeouts | undefined = layerPageLoadTimeoutMs: forcedTimeoutMs, layerReadyTimeoutMs: forcedTimeoutMs, } - : undefined; + : layerPageLoadTimeoutMs + ? { layerPageLoadTimeoutMs } + : undefined; From 1ad41eda0d4f1acfa2a3245dc3e5bc1e7121a2b9 Mon Sep 17 00:00:00 2001 From: Maximilian Haupt Date: Tue, 6 Oct 2026 12:16:33 +0200 Subject: [PATCH 2/2] ci: redact the staging credentials from uploaded E2E debug output Maestro writes the variables passed with -e, and the text it types, in plain text into its debug output: commands.json and maestro.log carry the staging account's email and password. GitHub masks secrets in job logs but not inside uploaded artifacts, and this repository's artifacts are public. Before the upload, both values are replaced with [redacted] in every non-image file, and any file that still contains one is dropped. --- .github/workflows/e2e-ios.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/e2e-ios.yml b/.github/workflows/e2e-ios.yml index b011a85..5119e93 100644 --- a/.github/workflows/e2e-ios.yml +++ b/.github/workflows/e2e-ios.yml @@ -241,6 +241,31 @@ jobs: mkdir -p ~/.maestro/tests cp /tmp/metro.log ~/.maestro/tests/ 2>/dev/null || true + # Maestro writes the staging account's credentials in plain text into + # its debug output (commands.json, maestro.log). GitHub masks secrets + # in job logs but not inside uploaded files, and this public repo's + # artifacts can be downloaded by anyone, so both are scrubbed before + # the upload. A file that still holds one afterwards (a screenshot, + # say) is dropped rather than uploaded. + - name: Redact credentials from debug output + if: failure() || env.E2E_RETRIED == 'true' + env: + E2E_STAGING_EMAIL: ${{ secrets.E2E_STAGING_EMAIL }} + E2E_STAGING_PASSWORD: ${{ secrets.E2E_STAGING_PASSWORD }} + run: | + dir=~/.maestro/tests + [ -d "$dir" ] || exit 0 + for var in E2E_STAGING_EMAIL E2E_STAGING_PASSWORD; do + secret=${!var} + [ -n "$secret" ] || continue + SECRET=$secret find "$dir" -type f ! -name '*.png' \ + -exec perl -pi -e 's/\Q$ENV{SECRET}\E/[redacted]/g' {} + + grep -rlF -- "$secret" "$dir" | while read -r f; do + echo "Dropping $f, which still contains $var" + rm -f "$f" + done + done + - name: Upload Maestro debug output if: failure() || env.E2E_RETRIED == 'true' uses: actions/upload-artifact@v4