From 6f185ca836f7505e65f5825baa685b31a93cdb25 Mon Sep 17 00:00:00 2001 From: dprevost-perso Date: Fri, 9 Oct 2026 07:38:47 -0400 Subject: [PATCH 1/7] test: let the local Android configs take the AVD and version from env vars ANDROID_AVD and ANDROID_PLATFORM_VERSION replace the fixed AVD names (Pixel_8_Pro_Android_15_API_35, Pixel_7_Pro_Android_14_API_34), which exist only on one machine. The defaults do not change. Co-Authored-By: Claude Opus 5.5 --- tests/configs/wdio.local.android.emus.app.conf.ts | 6 +++++- tests/configs/wdio.local.android.emus.web.conf.ts | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/configs/wdio.local.android.emus.app.conf.ts b/tests/configs/wdio.local.android.emus.app.conf.ts index 73c77f6fb..8a0fc90d5 100644 --- a/tests/configs/wdio.local.android.emus.app.conf.ts +++ b/tests/configs/wdio.local.android.emus.app.conf.ts @@ -1,6 +1,10 @@ import { join } from 'node:path' import { config as sharedConfig } from './wdio.local.appium.shared.conf.ts' +// The AVD name and Android version, for example in CI: ANDROID_AVD=ci_android_35 ANDROID_PLATFORM_VERSION=15.0 +const androidDeviceName = process.env.ANDROID_AVD ?? 'Pixel_7_Pro_Android_14_API_34' +const androidPlatformVersion = process.env.ANDROID_PLATFORM_VERSION ?? '14.0' + export const config: WebdriverIO.Config = { ...sharedConfig, // ================== @@ -22,7 +26,7 @@ export const config: WebdriverIO.Config = { // androidCaps('Pixel_5_Android_12_API_32', 'PORTRAIT', '12.0', true), // androidCaps('Pixel_6_Pro_Android_13_API_33', 'PORTRAIT', '13.0'), // androidCaps('Pixel_6_Pro_Android_13_API_33', 'PORTRAIT', '13.0', true), - androidCaps('Pixel_7_Pro_Android_14_API_34', 'PORTRAIT', '14.0'), + androidCaps(androidDeviceName, 'PORTRAIT', androidPlatformVersion), // androidCaps('Pixel_7_Pro_Android_14_API_34', 'PORTRAIT', '14.0', true), ], } diff --git a/tests/configs/wdio.local.android.emus.web.conf.ts b/tests/configs/wdio.local.android.emus.web.conf.ts index 68c4a3a57..fe2a0e5bc 100644 --- a/tests/configs/wdio.local.android.emus.web.conf.ts +++ b/tests/configs/wdio.local.android.emus.web.conf.ts @@ -1,6 +1,10 @@ import { join } from 'node:path' import { config as sharedConfig } from './wdio.local.appium.shared.conf.ts' +// The AVD name and Android version, for example in CI: ANDROID_AVD=ci_android_35 ANDROID_PLATFORM_VERSION=15.0 +const androidDeviceName = process.env.ANDROID_AVD ?? 'Pixel_8_Pro_Android_15_API_35' +const androidPlatformVersion = process.env.ANDROID_PLATFORM_VERSION ?? '15.0' + export const config: WebdriverIO.Config = { ...sharedConfig, // ================== @@ -12,7 +16,7 @@ export const config: WebdriverIO.Config = { // Capabilities // ============ capabilities: [ - androidCaps('Pixel_8_Pro_Android_15_API_35', 'PORTRAIT', '15.0', true), + androidCaps(androidDeviceName, 'PORTRAIT', androidPlatformVersion, true), // androidCaps('Pixel_8_Pro_Android_15_API_35', 'LANDSCAPE', '15.0', true), ], } From 37b66b6acba0702c835766167c3f685f90655775 Mon Sep 17 00:00:00 2001 From: dprevost-perso Date: Fri, 9 Oct 2026 08:39:29 -0400 Subject: [PATCH 2/7] ci: run the Android mobile web e2e tests on an emulator A new workflow starts an Android 15 emulator (google_apis, x86_64, KVM) in the GitHub runner with reactivecircus/android-emulator-runner, starts Appium 3 with the UiAutomator2 driver, and runs the mobile web suite twice: the first run saves the baselines, the second compares. It runs for pull requests that change the core, the service, the mobile web spec or its configs, and by hand. The LambdaTest Android jobs stay. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/android-emulator-e2e.sh | 28 ++++++++ .github/workflows/android-emulator.yml | 92 +++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100755 .github/scripts/android-emulator-e2e.sh create mode 100644 .github/workflows/android-emulator.yml diff --git a/.github/scripts/android-emulator-e2e.sh b/.github/scripts/android-emulator-e2e.sh new file mode 100755 index 000000000..438873dc3 --- /dev/null +++ b/.github/scripts/android-emulator-e2e.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Runs the Android mobile web e2e tests on the emulator that reactivecircus/android-emulator-runner started. +# The baselines are not in git: the first run saves them, the second run compares with them. +set -euo pipefail + +adb devices +adb shell getprop ro.build.version.release +if ! adb shell pm list packages | grep -q 'com.android.chrome'; then + echo "Chrome is not installed on the emulator, use a system image with Chrome (google_apis)" + exit 1 +fi + +mkdir -p logs +# Appium downloads the chromedriver that matches the Chrome version of the emulator +appium --port 4723 --allow-insecure 'uiautomator2:chromedriver_autodownload' --log logs/appium.log & +for _ in $(seq 1 60); do + curl -sf http://127.0.0.1:4723/status > /dev/null && break + sleep 1 +done +curl -sf http://127.0.0.1:4723/status > /dev/null || { echo "Appium did not start"; cat logs/appium.log; exit 1; } + +echo "::group::Save the baselines" +pnpm test.local.emus.web +echo "::endgroup::" + +echo "::group::Compare with the baselines" +pnpm test.local.emus.web +echo "::endgroup::" diff --git a/.github/workflows/android-emulator.yml b/.github/workflows/android-emulator.yml new file mode 100644 index 000000000..fd3aa80c8 --- /dev/null +++ b/.github/workflows/android-emulator.yml @@ -0,0 +1,92 @@ +name: android emulator + +# The Android mobile web e2e tests on an emulator in the GitHub runner. No cloud credentials are needed, so this +# also runs for pull requests from forks. It runs only when a change can affect the mobile screenshots. +# The LambdaTest Android jobs (e2e workflow) stay: they test real cloud devices with committed baselines. +on: + pull_request: + paths: + - "packages/image-comparison-core/src/**" + - "packages/visual-service/src/**" + - "tests/specs/mobile.web.spec.ts" + - "tests/configs/wdio.shared.conf.ts" + - "tests/configs/wdio.local.appium.shared.conf.ts" + - "tests/configs/wdio.local.android.emus.web.conf.ts" + - ".github/scripts/android-emulator-e2e.sh" + - ".github/workflows/android-emulator.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }} + cancel-in-progress: true + +jobs: + android-emulator-web: + name: 🤖 Run the Android mobile web e2e tests on an emulator + runs-on: ubuntu-latest + # Without a limit, a hung job runs until the GitHub default of 6 hours + timeout-minutes: 45 + env: + # Used by tests/configs/wdio.local.android.emus.web.conf.ts + ANDROID_AVD: ci_android_35 + ANDROID_PLATFORM_VERSION: "15" + steps: + - name: ⬇️ Checkout Repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + + - name: 📦 Setup pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - name: 🟢 Setup Node.js + uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 + with: + node-version-file: ".nvmrc" + cache: pnpm + + - name: 🧩 Install Dependencies + run: pnpm install --frozen-lockfile + + - name: 🏗️ Build + run: pnpm build + + - name: 📱 Install Appium and the UiAutomator2 driver + run: | + npm install --global appium@3.8.0 + appium driver install uiautomator2@8.7.0 + + # The emulator needs hardware acceleration + - name: ⚙️ Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + # A new AVD with a `google_apis` image: it has Chrome, and without a Play Store account Chrome + # can not update itself during the run + - name: 🤖 Run the e2e tests on the Android emulator + uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0 + with: + api-level: 35 + target: google_apis + arch: x86_64 + profile: pixel_8_pro + avd-name: ci_android_35 + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -no-snapshot + disable-animations: true + script: bash .github/scripts/android-emulator-e2e.sh + + - name: 📤 Upload artifacts + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + if: failure() + with: + name: android-emulator-logs + include-hidden-files: true + path: | + logs/ + .tmp/ + tests/localBaseline/ From 03c106bdab9dafd38ec05c14712af20bdfb6b0c7 Mon Sep 17 00:00:00 2001 From: dprevost-perso Date: Fri, 9 Oct 2026 08:43:33 -0400 Subject: [PATCH 3/7] ci: use the pixel_6 profile for the Android emulator, document the workflow The cmdline-tools of the runner image do not know pixel_8_pro. Co-Authored-By: Claude Opus 5.5 --- .agents/skills/verify-visual-testing/SKILL.md | 2 +- .github/workflows/android-emulator.yml | 4 +++- AGENTS.md | 6 ++++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.agents/skills/verify-visual-testing/SKILL.md b/.agents/skills/verify-visual-testing/SKILL.md index 4abeafd48..1efb9f74d 100644 --- a/.agents/skills/verify-visual-testing/SKILL.md +++ b/.agents/skills/verify-visual-testing/SKILL.md @@ -24,7 +24,7 @@ Always run `pnpm build` first: the e2e configs use `packages/*/dist`. | Web commands, screenshots, compare logic, matchers | The local Chrome suites: `pnpm test.local.chrome.v10` (Mocha), `.jasmine`, `.emulation`, `pnpm test.local.desktop.multi` | A unit test | | The desktop specs (`basics`, `desktop*`, `matcher`, check/save folders) | `BASELINE_SETUP=true pnpm test.local.desktop`, then `pnpm test.local.desktop` | The real run without the setup run (the baselines are missing) | | OCR | `pnpm test.ocr.local.desktop` (a local page with a committed font, also in `checks`), and the cloud OCR job for the website spec | A unit test with a mocked tesseract | -| Mobile web or native app | A local emulator or simulator suite (`test.local.emus.web`, `test.local.emus.app`, `test.local.sims.web`, `test.local.sims.app`, `test.local.multi.web.app`), or a cloud run | A desktop browser with mobile emulation | +| Mobile web or native app | A local emulator or simulator suite (`test.local.emus.web`, `test.local.emus.app`, `test.local.sims.web`, `test.local.sims.app`, `test.local.multi.web.app`; the Android configs read `ANDROID_AVD` and `ANDROID_PLATFORM_VERSION`), or a cloud run. The `android emulator` workflow runs `test.local.emus.web` in CI | A desktop browser with mobile emulation | | A failure that happens only in CI on Linux | The Docker check below | A macOS run | | Cloud configs, cloud baselines, a device or OS version | A cloud run (below) | A local run | diff --git a/.github/workflows/android-emulator.yml b/.github/workflows/android-emulator.yml index fd3aa80c8..3df53f6f5 100644 --- a/.github/workflows/android-emulator.yml +++ b/.github/workflows/android-emulator.yml @@ -74,7 +74,9 @@ jobs: api-level: 35 target: google_apis arch: x86_64 - profile: pixel_8_pro + # The cmdline-tools of the runner image do not know the newest Pixel profiles + profile: pixel_6 + pre-emulator-launch-script: avdmanager list device -c | tr '\n' ' ' avd-name: ci_android_35 emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -no-snapshot disable-animations: true diff --git a/AGENTS.md b/AGENTS.md index e12acf8cf..e57cb6bc0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,6 +94,12 @@ Mock the browser in unit tests; do not start a browser. every push to `main` and to the maintenance branches. Lint, types and unit tests; the local headless Chrome suites (v10 Mocha, Jasmine, multi-remote, emulation) and the local desktop suite (setup run, then the real run). +- [`android emulator`](.github/workflows/android-emulator.yml): the Android + mobile web suite on an emulator in the runner (Appium, same-run baselines). + It runs for PRs (also from forks) that change the core, the service, the + mobile web spec or its configs. Locally: start an emulator and Appium, then + `ANDROID_AVD= ANDROID_PLATFORM_VERSION= pnpm test.local.emus.web` + (see [.github/scripts/android-emulator-e2e.sh](.github/scripts/android-emulator-e2e.sh)). - [`e2e`](.github/workflows/e2e.yml): the LambdaTest and Sauce Labs jobs. They need the cloud credentials, so they run only for branches in this repository, not for forks or Dependabot. From 19c946f635cbd4decc7c710ea28a1681d9722a4b Mon Sep 17 00:00:00 2001 From: dprevost-perso Date: Fri, 9 Oct 2026 09:50:40 -0400 Subject: [PATCH 4/7] ci: fail on a missing baseline in the Android compare run, run on dependency changes - In CI only the setup run (BASELINE_SETUP=true) saves baselines with the local Appium configs, so the compare run fails on a missing baseline. - The workflow also runs when the root or package manifests, the lockfile, the workspace file or the pnpm patches change. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/android-emulator-e2e.sh | 2 +- .github/workflows/android-emulator.yml | 7 +++++++ tests/configs/wdio.local.appium.shared.conf.ts | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/scripts/android-emulator-e2e.sh b/.github/scripts/android-emulator-e2e.sh index 438873dc3..c66eabc24 100755 --- a/.github/scripts/android-emulator-e2e.sh +++ b/.github/scripts/android-emulator-e2e.sh @@ -20,7 +20,7 @@ done curl -sf http://127.0.0.1:4723/status > /dev/null || { echo "Appium did not start"; cat logs/appium.log; exit 1; } echo "::group::Save the baselines" -pnpm test.local.emus.web +BASELINE_SETUP=true pnpm test.local.emus.web echo "::endgroup::" echo "::group::Compare with the baselines" diff --git a/.github/workflows/android-emulator.yml b/.github/workflows/android-emulator.yml index 3df53f6f5..d8c069188 100644 --- a/.github/workflows/android-emulator.yml +++ b/.github/workflows/android-emulator.yml @@ -6,6 +6,13 @@ name: android emulator on: pull_request: paths: + # Dependency updates (for example webdriverio, @wdio/*, Appium-related packages) can change the screenshots + - "package.json" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + - "patches/**" + - "packages/image-comparison-core/package.json" + - "packages/visual-service/package.json" - "packages/image-comparison-core/src/**" - "packages/visual-service/src/**" - "tests/specs/mobile.web.spec.ts" diff --git a/tests/configs/wdio.local.appium.shared.conf.ts b/tests/configs/wdio.local.appium.shared.conf.ts index b75f3d7e3..b5c4e980c 100644 --- a/tests/configs/wdio.local.appium.shared.conf.ts +++ b/tests/configs/wdio.local.appium.shared.conf.ts @@ -36,6 +36,9 @@ export const config: Omit = { blockOutToolBar: true, blockOutSideBar: true, enableLayoutTesting: true, + // In CI only the setup run (BASELINE_SETUP=true) saves baselines: in the compare run a missing + // baseline must fail, for example when the viewport in the file name changed between the runs + autoSaveBaseline: !process.env.CI || process.env.BASELINE_SETUP === 'true', } satisfies VisualServiceOptions, ], ], From 3a5c7d03d56d636d28c044e8510736211826facd Mon Sep 17 00:00:00 2001 From: dprevost-perso Date: Fri, 9 Oct 2026 09:52:18 -0400 Subject: [PATCH 5/7] ci: stop the Android emulator job from hanging, keep Appium out of the job log The job hung for 20+ minutes after the tests passed: when the action stops the emulator, it waits for the crashpad_handler processes that the emulator leaves running (ReactiveCircus/android-emulator-runner#385). The script now stops Appium and kills those processes when it ends. The Appium output (6000 lines) goes to logs/appium.log instead of the job log. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/android-emulator-e2e.sh | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/.github/scripts/android-emulator-e2e.sh b/.github/scripts/android-emulator-e2e.sh index c66eabc24..3adb21ee7 100755 --- a/.github/scripts/android-emulator-e2e.sh +++ b/.github/scripts/android-emulator-e2e.sh @@ -11,8 +11,27 @@ if ! adb shell pm list packages | grep -q 'com.android.chrome'; then fi mkdir -p logs -# Appium downloads the chromedriver that matches the Chrome version of the emulator -appium --port 4723 --allow-insecure 'uiautomator2:chromedriver_autodownload' --log logs/appium.log & +# Appium downloads the chromedriver that matches the Chrome version of the emulator. +# Its output goes to a file (uploaded when the job fails), not to the job log. +appium --port 4723 --allow-insecure 'uiautomator2:chromedriver_autodownload' > logs/appium.log 2>&1 & +APPIUM_PID=$! + +cleanup() { + kill "$APPIUM_PID" 2> /dev/null || true + # When the action stops the emulator, it waits for all emulator processes, but the emulator leaves its + # crashpad_handler processes running, so the job hangs (ReactiveCircus/android-emulator-runner#385). + # Kill them after the emulator stopped. + ( + sleep 30 + for _ in $(seq 1 12); do + pgrep -x crashpad_handler > /dev/null || break + pkill -9 -x crashpad_handler || true + sleep 5 + done + ) > /dev/null 2>&1 & +} +trap cleanup EXIT + for _ in $(seq 1 60); do curl -sf http://127.0.0.1:4723/status > /dev/null && break sleep 1 From cfd084f8fd6b2ac92678bfbcef364dd5bca1a7b4 Mon Sep 17 00:00:00 2001 From: dprevost-perso Date: Fri, 9 Oct 2026 10:52:04 -0400 Subject: [PATCH 6/7] ci: warm up the Android emulator before the setup run In the first minutes after the boot the CI emulator is busy, and a native screenshot can still show the frame before a scroll, so the full page baseline of the setup run was stitched wrong (the scroll positions were exact). In an experiment with 6 CI jobs, 2 of 3 failed without a warm-up and 0 of 3 with one. The warm-up runs the full page test once and does not keep its files. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/android-emulator-e2e.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/scripts/android-emulator-e2e.sh b/.github/scripts/android-emulator-e2e.sh index 3adb21ee7..a9003a588 100755 --- a/.github/scripts/android-emulator-e2e.sh +++ b/.github/scripts/android-emulator-e2e.sh @@ -38,6 +38,13 @@ for _ in $(seq 1 60); do done curl -sf http://127.0.0.1:4723/status > /dev/null || { echo "Appium did not start"; cat logs/appium.log; exit 1; } +# Warm-up: in the first minutes after the boot the emulator is busy, and a screenshot can still show the frame before +# a scroll (wrong full page stitch: 2 of 3 CI jobs without a warm-up, 0 of 3 with it). Its files are not kept. +echo "::group::Warm up the emulator" +BASELINE_SETUP=true pnpm test.local.emus.web --mochaOpts.grep "full page screenshot successful" || true +rm -rf tests/localBaseline .tmp +echo "::endgroup::" + echo "::group::Save the baselines" BASELINE_SETUP=true pnpm test.local.emus.web echo "::endgroup::" From 67ef5f4a7747983d690f3a13941929f7241a4650 Mon Sep 17 00:00:00 2001 From: dprevost-perso Date: Fri, 9 Oct 2026 11:37:55 -0400 Subject: [PATCH 7/7] ci: match crashpad_handler by its command line in the emulator cleanup Linux cuts process names to 15 characters ("crashpad_handle"), so `pgrep -x crashpad_handler` never matched and the cleanup did not kill those processes. Use -f with the path of the executable. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/android-emulator-e2e.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/scripts/android-emulator-e2e.sh b/.github/scripts/android-emulator-e2e.sh index a9003a588..05eb7a228 100755 --- a/.github/scripts/android-emulator-e2e.sh +++ b/.github/scripts/android-emulator-e2e.sh @@ -20,12 +20,13 @@ cleanup() { kill "$APPIUM_PID" 2> /dev/null || true # When the action stops the emulator, it waits for all emulator processes, but the emulator leaves its # crashpad_handler processes running, so the job hangs (ReactiveCircus/android-emulator-runner#385). - # Kill them after the emulator stopped. + # Kill them after the emulator stopped. Linux cuts process names to 15 characters ("crashpad_handle"), + # so match the command line (-f) with the path of the executable. ( sleep 30 for _ in $(seq 1 12); do - pgrep -x crashpad_handler > /dev/null || break - pkill -9 -x crashpad_handler || true + pgrep -f '/crashpad_handler' > /dev/null || break + pkill -9 -f '/crashpad_handler' || true sleep 5 done ) > /dev/null 2>&1 &