ci: run the Android mobile web e2e tests on an emulator

.agents/skills/verify-visual-testing/SKILL.md

@@ -24,7 +24,7 @@ Always run `pnpm build` first: the e2e configs use `packages/*/dist`.
 | Web commands, screenshots, compare logic, matchers | The local Chrome suites: `pnpm test.local.chrome.v10` (Mocha), `.jasmine`, `.emulation`, `pnpm test.local.desktop.multi` | A unit test |
 | The desktop specs (`basics`, `desktop*`, `matcher`, check/save folders) | `BASELINE_SETUP=true pnpm test.local.desktop`, then `pnpm test.local.desktop` | The real run without the setup run (the baselines are missing) |
 | OCR | `pnpm test.ocr.local.desktop` (a local page with a committed font, also in `checks`), and the cloud OCR job for the website spec | A unit test with a mocked tesseract |
-| Mobile web or native app | A local emulator or simulator suite (`test.local.emus.web`, `test.local.emus.app`, `test.local.sims.web`, `test.local.sims.app`, `test.local.multi.web.app`), or a cloud run | A desktop browser with mobile emulation |
+| Mobile web or native app | A local emulator or simulator suite (`test.local.emus.web`, `test.local.emus.app`, `test.local.sims.web`, `test.local.sims.app`, `test.local.multi.web.app`; the Android configs read `ANDROID_AVD` and `ANDROID_PLATFORM_VERSION`), or a cloud run. The `android emulator` workflow runs `test.local.emus.web` in CI | A desktop browser with mobile emulation |
 | A failure that happens only in CI on Linux | The Docker check below | A macOS run |
 | Cloud configs, cloud baselines, a device or OS version | A cloud run (below) | A local run |
 

.github/scripts/android-emulator-e2e.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Runs the Android mobile web e2e tests on the emulator that reactivecircus/android-emulator-runner started.
+# The baselines are not in git: the first run saves them, the second run compares with them.
+set -euo pipefail
+
+adb devices
+adb shell getprop ro.build.version.release
+if ! adb shell pm list packages | grep -q 'com.android.chrome'; then
+    echo "Chrome is not installed on the emulator, use a system image with Chrome (google_apis)"
+    exit 1
+fi
+
+mkdir -p logs
+# Appium downloads the chromedriver that matches the Chrome version of the emulator.
+# Its output goes to a file (uploaded when the job fails), not to the job log.
+appium --port 4723 --allow-insecure 'uiautomator2:chromedriver_autodownload' > logs/appium.log 2>&1 &
+APPIUM_PID=$!
+
+cleanup() {
+    kill "$APPIUM_PID" 2> /dev/null || true
+    # When the action stops the emulator, it waits for all emulator processes, but the emulator leaves its
+    # crashpad_handler processes running, so the job hangs (ReactiveCircus/android-emulator-runner#385).
+    # Kill them after the emulator stopped. Linux cuts process names to 15 characters ("crashpad_handle"),
+    # so match the command line (-f) with the path of the executable.
+    (
+        sleep 30
+        for _ in $(seq 1 12); do
+            pgrep -f '/crashpad_handler' > /dev/null || break
+            pkill -9 -f '/crashpad_handler' || true
+            sleep 5
+        done
+    ) > /dev/null 2>&1 &
+}
+trap cleanup EXIT
+
+for _ in $(seq 1 60); do
+    curl -sf http://127.0.0.1:4723/status > /dev/null && break
+    sleep 1
+done
+curl -sf http://127.0.0.1:4723/status > /dev/null || { echo "Appium did not start"; cat logs/appium.log; exit 1; }
+
+# Warm-up: in the first minutes after the boot the emulator is busy, and a screenshot can still show the frame before
+# a scroll (wrong full page stitch: 2 of 3 CI jobs without a warm-up, 0 of 3 with it). Its files are not kept.
+echo "::group::Warm up the emulator"
+BASELINE_SETUP=true pnpm test.local.emus.web --mochaOpts.grep "full page screenshot successful" || true
+rm -rf tests/localBaseline .tmp
+echo "::endgroup::"
+
+echo "::group::Save the baselines"
+BASELINE_SETUP=true pnpm test.local.emus.web
+echo "::endgroup::"
+
+echo "::group::Compare with the baselines"
+pnpm test.local.emus.web
+echo "::endgroup::"

.github/workflows/android-emulator.yml

@@ -0,0 +1,101 @@
+name: android emulator
+
+# The Android mobile web e2e tests on an emulator in the GitHub runner. No cloud credentials are needed, so this
+# also runs for pull requests from forks. It runs only when a change can affect the mobile screenshots.
+# The LambdaTest Android jobs (e2e workflow) stay: they test real cloud devices with committed baselines.
+on:
+  pull_request:
+    paths:
+      # Dependency updates (for example webdriverio, @wdio/*, Appium-related packages) can change the screenshots
+      - "package.json"
+      - "pnpm-lock.yaml"
+      - "pnpm-workspace.yaml"
+      - "patches/**"
+      - "packages/image-comparison-core/package.json"
+      - "packages/visual-service/package.json"
+      - "packages/image-comparison-core/src/**"
+      - "packages/visual-service/src/**"
+      - "tests/specs/mobile.web.spec.ts"
+      - "tests/configs/wdio.shared.conf.ts"
+      - "tests/configs/wdio.local.appium.shared.conf.ts"
+      - "tests/configs/wdio.local.android.emus.web.conf.ts"
+      - ".github/scripts/android-emulator-e2e.sh"
+      - ".github/workflows/android-emulator.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  android-emulator-web:
+    name: 🤖 Run the Android mobile web e2e tests on an emulator
+    runs-on: ubuntu-latest
+    # Without a limit, a hung job runs until the GitHub default of 6 hours
+    timeout-minutes: 45
+    env:
+      # Used by tests/configs/wdio.local.android.emus.web.conf.ts
+      ANDROID_AVD: ci_android_35
+      ANDROID_PLATFORM_VERSION: "15"
+    steps:
+      - name: ⬇️ Checkout Repository
+        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        with:
+          persist-credentials: false
+
+      - name: 📦 Setup pnpm
+        uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
+
+      - name: 🟢 Setup Node.js
+        uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
+        with:
+          node-version-file: ".nvmrc"
+          cache: pnpm
+
+      - name: 🧩 Install Dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: 🏗️ Build
+        run: pnpm build
+
+      - name: 📱 Install Appium and the UiAutomator2 driver
+        run: |
+          npm install --global appium@3.8.0
+          appium driver install uiautomator2@8.7.0
+
+      # The emulator needs hardware acceleration
+      - name: ⚙️ Enable KVM
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      # A new AVD with a `google_apis` image: it has Chrome, and without a Play Store account Chrome
+      # can not update itself during the run
+      - name: 🤖 Run the e2e tests on the Android emulator
+        uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0
+        with:
+          api-level: 35
+          target: google_apis
+          arch: x86_64
+          # The cmdline-tools of the runner image do not know the newest Pixel profiles
+          profile: pixel_6
+          pre-emulator-launch-script: avdmanager list device -c | tr '\n' ' '
+          avd-name: ci_android_35
+          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -no-snapshot
+          disable-animations: true
+          script: bash .github/scripts/android-emulator-e2e.sh
+
+      - name: 📤 Upload artifacts
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        if: failure()
+        with:
+          name: android-emulator-logs
+          include-hidden-files: true
+          path: |
+            logs/
+            .tmp/
+            tests/localBaseline/

AGENTS.md

@@ -94,6 +94,12 @@ Mock the browser in unit tests; do not start a browser.
   every push to `main` and to the maintenance branches. Lint, types and unit
   tests; the local headless Chrome suites (v10 Mocha, Jasmine, multi-remote,
   emulation) and the local desktop suite (setup run, then the real run).
+- [`android emulator`](.github/workflows/android-emulator.yml): the Android
+  mobile web suite on an emulator in the runner (Appium, same-run baselines).
+  It runs for PRs (also from forks) that change the core, the service, the
+  mobile web spec or its configs. Locally: start an emulator and Appium, then
+  `ANDROID_AVD=<avd> ANDROID_PLATFORM_VERSION=<version> pnpm test.local.emus.web`
+  (see [.github/scripts/android-emulator-e2e.sh](.github/scripts/android-emulator-e2e.sh)).
 - [`e2e`](.github/workflows/e2e.yml): the LambdaTest and Sauce Labs jobs. They
   need the cloud credentials, so they run only for branches in this
   repository, not for forks or Dependabot.

tests/configs/wdio.local.android.emus.app.conf.ts

@@ -1,6 +1,10 @@
 import { join } from 'node:path'
 import { config as sharedConfig } from './wdio.local.appium.shared.conf.ts'
 
+// The AVD name and Android version, for example in CI: ANDROID_AVD=ci_android_35 ANDROID_PLATFORM_VERSION=15.0
+const androidDeviceName = process.env.ANDROID_AVD ?? 'Pixel_7_Pro_Android_14_API_34'
+const androidPlatformVersion = process.env.ANDROID_PLATFORM_VERSION ?? '14.0'
+
 export const config: WebdriverIO.Config  = {
     ...sharedConfig,
     // ==================
@@ -22,7 +26,7 @@ export const config: WebdriverIO.Config  = {
         // androidCaps('Pixel_5_Android_12_API_32', 'PORTRAIT', '12.0', true),
         // androidCaps('Pixel_6_Pro_Android_13_API_33', 'PORTRAIT', '13.0'),
         // androidCaps('Pixel_6_Pro_Android_13_API_33', 'PORTRAIT', '13.0', true),
-        androidCaps('Pixel_7_Pro_Android_14_API_34', 'PORTRAIT', '14.0'),
+        androidCaps(androidDeviceName, 'PORTRAIT', androidPlatformVersion),
         // androidCaps('Pixel_7_Pro_Android_14_API_34', 'PORTRAIT', '14.0', true),
     ],
 }

tests/configs/wdio.local.android.emus.web.conf.ts

@@ -1,6 +1,10 @@
 import { join } from 'node:path'
 import { config as sharedConfig } from './wdio.local.appium.shared.conf.ts'
 
+// The AVD name and Android version, for example in CI: ANDROID_AVD=ci_android_35 ANDROID_PLATFORM_VERSION=15.0
+const androidDeviceName = process.env.ANDROID_AVD ?? 'Pixel_8_Pro_Android_15_API_35'
+const androidPlatformVersion = process.env.ANDROID_PLATFORM_VERSION ?? '15.0'
+
 export const config: WebdriverIO.Config  = {
     ...sharedConfig,
     // ==================
@@ -12,7 +16,7 @@ export const config: WebdriverIO.Config  = {
     // Capabilities
     // ============
     capabilities: [
-        androidCaps('Pixel_8_Pro_Android_15_API_35', 'PORTRAIT', '15.0', true),
+        androidCaps(androidDeviceName, 'PORTRAIT', androidPlatformVersion, true),
         // androidCaps('Pixel_8_Pro_Android_15_API_35', 'LANDSCAPE', '15.0', true),
     ],
 }

tests/configs/wdio.local.appium.shared.conf.ts

@@ -36,6 +36,9 @@ export const config: Omit<WebdriverIO.Config, 'capabilities'>  = {
                 blockOutToolBar: true,
                 blockOutSideBar: true,
                 enableLayoutTesting: true,
+                // In CI only the setup run (BASELINE_SETUP=true) saves baselines: in the compare run a missing
+                // baseline must fail, for example when the viewport in the file name changed between the runs
+                autoSaveBaseline: !process.env.CI || process.env.BASELINE_SETUP === 'true',
             } satisfies VisualServiceOptions,
         ],
     ],