diff --git a/.agents/skills/verify-visual-testing/SKILL.md b/.agents/skills/verify-visual-testing/SKILL.md index 4abeafd4..1efb9f74 100644 --- a/.agents/skills/verify-visual-testing/SKILL.md +++ b/.agents/skills/verify-visual-testing/SKILL.md @@ -24,7 +24,7 @@ Always run `pnpm build` first: the e2e configs use `packages/*/dist`. | Web commands, screenshots, compare logic, matchers | The local Chrome suites: `pnpm test.local.chrome.v10` (Mocha), `.jasmine`, `.emulation`, `pnpm test.local.desktop.multi` | A unit test | | The desktop specs (`basics`, `desktop*`, `matcher`, check/save folders) | `BASELINE_SETUP=true pnpm test.local.desktop`, then `pnpm test.local.desktop` | The real run without the setup run (the baselines are missing) | | OCR | `pnpm test.ocr.local.desktop` (a local page with a committed font, also in `checks`), and the cloud OCR job for the website spec | A unit test with a mocked tesseract | -| Mobile web or native app | A local emulator or simulator suite (`test.local.emus.web`, `test.local.emus.app`, `test.local.sims.web`, `test.local.sims.app`, `test.local.multi.web.app`), or a cloud run | A desktop browser with mobile emulation | +| Mobile web or native app | A local emulator or simulator suite (`test.local.emus.web`, `test.local.emus.app`, `test.local.sims.web`, `test.local.sims.app`, `test.local.multi.web.app`; the Android configs read `ANDROID_AVD` and `ANDROID_PLATFORM_VERSION`), or a cloud run. The `android emulator` workflow runs `test.local.emus.web` in CI | A desktop browser with mobile emulation | | A failure that happens only in CI on Linux | The Docker check below | A macOS run | | Cloud configs, cloud baselines, a device or OS version | A cloud run (below) | A local run | diff --git a/.github/scripts/android-emulator-e2e.sh b/.github/scripts/android-emulator-e2e.sh new file mode 100755 index 00000000..05eb7a22 --- /dev/null +++ b/.github/scripts/android-emulator-e2e.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Runs the Android mobile web e2e tests on the emulator that reactivecircus/android-emulator-runner started. +# The baselines are not in git: the first run saves them, the second run compares with them. +set -euo pipefail + +adb devices +adb shell getprop ro.build.version.release +if ! adb shell pm list packages | grep -q 'com.android.chrome'; then + echo "Chrome is not installed on the emulator, use a system image with Chrome (google_apis)" + exit 1 +fi + +mkdir -p logs +# Appium downloads the chromedriver that matches the Chrome version of the emulator. +# Its output goes to a file (uploaded when the job fails), not to the job log. +appium --port 4723 --allow-insecure 'uiautomator2:chromedriver_autodownload' > logs/appium.log 2>&1 & +APPIUM_PID=$! + +cleanup() { + kill "$APPIUM_PID" 2> /dev/null || true + # When the action stops the emulator, it waits for all emulator processes, but the emulator leaves its + # crashpad_handler processes running, so the job hangs (ReactiveCircus/android-emulator-runner#385). + # Kill them after the emulator stopped. Linux cuts process names to 15 characters ("crashpad_handle"), + # so match the command line (-f) with the path of the executable. + ( + sleep 30 + for _ in $(seq 1 12); do + pgrep -f '/crashpad_handler' > /dev/null || break + pkill -9 -f '/crashpad_handler' || true + sleep 5 + done + ) > /dev/null 2>&1 & +} +trap cleanup EXIT + +for _ in $(seq 1 60); do + curl -sf http://127.0.0.1:4723/status > /dev/null && break + sleep 1 +done +curl -sf http://127.0.0.1:4723/status > /dev/null || { echo "Appium did not start"; cat logs/appium.log; exit 1; } + +# Warm-up: in the first minutes after the boot the emulator is busy, and a screenshot can still show the frame before +# a scroll (wrong full page stitch: 2 of 3 CI jobs without a warm-up, 0 of 3 with it). Its files are not kept. +echo "::group::Warm up the emulator" +BASELINE_SETUP=true pnpm test.local.emus.web --mochaOpts.grep "full page screenshot successful" || true +rm -rf tests/localBaseline .tmp +echo "::endgroup::" + +echo "::group::Save the baselines" +BASELINE_SETUP=true pnpm test.local.emus.web +echo "::endgroup::" + +echo "::group::Compare with the baselines" +pnpm test.local.emus.web +echo "::endgroup::" diff --git a/.github/workflows/android-emulator.yml b/.github/workflows/android-emulator.yml new file mode 100644 index 00000000..d8c06918 --- /dev/null +++ b/.github/workflows/android-emulator.yml @@ -0,0 +1,101 @@ +name: android emulator + +# The Android mobile web e2e tests on an emulator in the GitHub runner. No cloud credentials are needed, so this +# also runs for pull requests from forks. It runs only when a change can affect the mobile screenshots. +# The LambdaTest Android jobs (e2e workflow) stay: they test real cloud devices with committed baselines. +on: + pull_request: + paths: + # Dependency updates (for example webdriverio, @wdio/*, Appium-related packages) can change the screenshots + - "package.json" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + - "patches/**" + - "packages/image-comparison-core/package.json" + - "packages/visual-service/package.json" + - "packages/image-comparison-core/src/**" + - "packages/visual-service/src/**" + - "tests/specs/mobile.web.spec.ts" + - "tests/configs/wdio.shared.conf.ts" + - "tests/configs/wdio.local.appium.shared.conf.ts" + - "tests/configs/wdio.local.android.emus.web.conf.ts" + - ".github/scripts/android-emulator-e2e.sh" + - ".github/workflows/android-emulator.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.run_id }} + cancel-in-progress: true + +jobs: + android-emulator-web: + name: 🤖 Run the Android mobile web e2e tests on an emulator + runs-on: ubuntu-latest + # Without a limit, a hung job runs until the GitHub default of 6 hours + timeout-minutes: 45 + env: + # Used by tests/configs/wdio.local.android.emus.web.conf.ts + ANDROID_AVD: ci_android_35 + ANDROID_PLATFORM_VERSION: "15" + steps: + - name: ⬇️ Checkout Repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + + - name: 📦 Setup pnpm + uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0 + + - name: 🟢 Setup Node.js + uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0 + with: + node-version-file: ".nvmrc" + cache: pnpm + + - name: 🧩 Install Dependencies + run: pnpm install --frozen-lockfile + + - name: 🏗️ Build + run: pnpm build + + - name: 📱 Install Appium and the UiAutomator2 driver + run: | + npm install --global appium@3.8.0 + appium driver install uiautomator2@8.7.0 + + # The emulator needs hardware acceleration + - name: ⚙️ Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + # A new AVD with a `google_apis` image: it has Chrome, and without a Play Store account Chrome + # can not update itself during the run + - name: 🤖 Run the e2e tests on the Android emulator + uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0 + with: + api-level: 35 + target: google_apis + arch: x86_64 + # The cmdline-tools of the runner image do not know the newest Pixel profiles + profile: pixel_6 + pre-emulator-launch-script: avdmanager list device -c | tr '\n' ' ' + avd-name: ci_android_35 + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none -no-snapshot + disable-animations: true + script: bash .github/scripts/android-emulator-e2e.sh + + - name: 📤 Upload artifacts + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + if: failure() + with: + name: android-emulator-logs + include-hidden-files: true + path: | + logs/ + .tmp/ + tests/localBaseline/ diff --git a/AGENTS.md b/AGENTS.md index e12acf8c..e57cb6bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -94,6 +94,12 @@ Mock the browser in unit tests; do not start a browser. every push to `main` and to the maintenance branches. Lint, types and unit tests; the local headless Chrome suites (v10 Mocha, Jasmine, multi-remote, emulation) and the local desktop suite (setup run, then the real run). +- [`android emulator`](.github/workflows/android-emulator.yml): the Android + mobile web suite on an emulator in the runner (Appium, same-run baselines). + It runs for PRs (also from forks) that change the core, the service, the + mobile web spec or its configs. Locally: start an emulator and Appium, then + `ANDROID_AVD= ANDROID_PLATFORM_VERSION= pnpm test.local.emus.web` + (see [.github/scripts/android-emulator-e2e.sh](.github/scripts/android-emulator-e2e.sh)). - [`e2e`](.github/workflows/e2e.yml): the LambdaTest and Sauce Labs jobs. They need the cloud credentials, so they run only for branches in this repository, not for forks or Dependabot. diff --git a/tests/configs/wdio.local.android.emus.app.conf.ts b/tests/configs/wdio.local.android.emus.app.conf.ts index 73c77f6f..8a0fc90d 100644 --- a/tests/configs/wdio.local.android.emus.app.conf.ts +++ b/tests/configs/wdio.local.android.emus.app.conf.ts @@ -1,6 +1,10 @@ import { join } from 'node:path' import { config as sharedConfig } from './wdio.local.appium.shared.conf.ts' +// The AVD name and Android version, for example in CI: ANDROID_AVD=ci_android_35 ANDROID_PLATFORM_VERSION=15.0 +const androidDeviceName = process.env.ANDROID_AVD ?? 'Pixel_7_Pro_Android_14_API_34' +const androidPlatformVersion = process.env.ANDROID_PLATFORM_VERSION ?? '14.0' + export const config: WebdriverIO.Config = { ...sharedConfig, // ================== @@ -22,7 +26,7 @@ export const config: WebdriverIO.Config = { // androidCaps('Pixel_5_Android_12_API_32', 'PORTRAIT', '12.0', true), // androidCaps('Pixel_6_Pro_Android_13_API_33', 'PORTRAIT', '13.0'), // androidCaps('Pixel_6_Pro_Android_13_API_33', 'PORTRAIT', '13.0', true), - androidCaps('Pixel_7_Pro_Android_14_API_34', 'PORTRAIT', '14.0'), + androidCaps(androidDeviceName, 'PORTRAIT', androidPlatformVersion), // androidCaps('Pixel_7_Pro_Android_14_API_34', 'PORTRAIT', '14.0', true), ], } diff --git a/tests/configs/wdio.local.android.emus.web.conf.ts b/tests/configs/wdio.local.android.emus.web.conf.ts index 68c4a3a5..fe2a0e5b 100644 --- a/tests/configs/wdio.local.android.emus.web.conf.ts +++ b/tests/configs/wdio.local.android.emus.web.conf.ts @@ -1,6 +1,10 @@ import { join } from 'node:path' import { config as sharedConfig } from './wdio.local.appium.shared.conf.ts' +// The AVD name and Android version, for example in CI: ANDROID_AVD=ci_android_35 ANDROID_PLATFORM_VERSION=15.0 +const androidDeviceName = process.env.ANDROID_AVD ?? 'Pixel_8_Pro_Android_15_API_35' +const androidPlatformVersion = process.env.ANDROID_PLATFORM_VERSION ?? '15.0' + export const config: WebdriverIO.Config = { ...sharedConfig, // ================== @@ -12,7 +16,7 @@ export const config: WebdriverIO.Config = { // Capabilities // ============ capabilities: [ - androidCaps('Pixel_8_Pro_Android_15_API_35', 'PORTRAIT', '15.0', true), + androidCaps(androidDeviceName, 'PORTRAIT', androidPlatformVersion, true), // androidCaps('Pixel_8_Pro_Android_15_API_35', 'LANDSCAPE', '15.0', true), ], } diff --git a/tests/configs/wdio.local.appium.shared.conf.ts b/tests/configs/wdio.local.appium.shared.conf.ts index b75f3d7e..b5c4e980 100644 --- a/tests/configs/wdio.local.appium.shared.conf.ts +++ b/tests/configs/wdio.local.appium.shared.conf.ts @@ -36,6 +36,9 @@ export const config: Omit = { blockOutToolBar: true, blockOutSideBar: true, enableLayoutTesting: true, + // In CI only the setup run (BASELINE_SETUP=true) saves baselines: in the compare run a missing + // baseline must fail, for example when the viewport in the file name changed between the runs + autoSaveBaseline: !process.env.CI || process.env.BASELINE_SETUP === 'true', } satisfies VisualServiceOptions, ], ],