Skip to content

Commit 4c3c7db

Browse files
test(e2e): Switch from Appium to Maestro (#4210)
1 parent 0966e69 commit 4c3c7db

File tree

17 files changed

+178
-383
lines changed

17 files changed

+178
-383
lines changed

.github/workflows/e2e.yml

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ concurrency:
1414

1515
env:
1616
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
17+
MAESTRO_VERSION: '1.39.0'
18+
IOS_DEVICE: 'iPhone 14'
1719

1820
jobs:
1921
diff_check:
@@ -168,12 +170,10 @@ jobs:
168170
rn-version: '0.76.0'
169171
runs-on: macos-14 # uses m1 https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available/
170172
runtime: 'latest'
171-
device: 'iPhone 14'
172173
- platform: ios
173174
rn-version: '0.65.3'
174175
runs-on: macos-13
175176
runtime: 'latest'
176-
device: 'iPhone 14'
177177
- platform: android
178178
runs-on: ubuntu-latest
179179
exclude:
@@ -308,12 +308,10 @@ jobs:
308308
rn-version: '0.76.0'
309309
runs-on: macos-14 # uses m1 https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available/
310310
runtime: 'latest'
311-
device: 'iPhone 14'
312311
- platform: ios
313312
rn-version: '0.65.3'
314313
runs-on: macos-latest
315314
runtime: 'latest'
316-
device: 'iPhone 14'
317315
- platform: android
318316
runs-on: ubuntu-latest
319317
exclude:
@@ -329,12 +327,18 @@ jobs:
329327
- rn-version: '0.76.0'
330328
platform: 'ios'
331329
rn-architecture: 'new'
332-
env:
333-
PLATFORM: ${{ matrix.platform }}
334-
DEVICE: ${{ matrix.device }}
335330
steps:
336331
- uses: actions/checkout@v4
337332

333+
- name: Install Maestro
334+
uses: dniHze/maestro-test-action@bda8a93211c86d0a05b7a4597c5ad134566fbde4 # [email protected]
335+
with:
336+
version: ${{env.MAESTRO_VERSION}}
337+
338+
- name: Install iDB Companion
339+
if: ${{ matrix.platform == 'ios' }}
340+
run: brew tap facebook/fb && brew install facebook/fb/idb-companion
341+
338342
- uses: ./.github/actions/disk-cleanup
339343
if: ${{ matrix.platform == 'android' }}
340344

@@ -400,11 +404,10 @@ jobs:
400404
-timezone US/Pacific
401405
script: ./dev-packages/e2e-tests/cli.mjs ${{ matrix.platform }} --test
402406

403-
- uses: actions/cache@v4
407+
- uses: futureware-tech/simulator-action@bfa03d93ec9de6dacb0c5553bbf8da8afc6c2ee9 # pin@v3
404408
if: ${{ matrix.platform == 'ios' }}
405409
with:
406-
path: test/e2e/DerivedData/Build/Products/Debug-iphonesimulator/WebDriverAgentRunner-Runner.app
407-
key: appium-webdriveragent-${{ hashFiles('test/e2e/yarn.lock') }}
410+
model: ${{ env.IOS_DEVICE }}
408411

409412
- name: Run tests on iOS
410413
if: ${{ matrix.platform == 'ios' }}
@@ -415,6 +418,4 @@ jobs:
415418
uses: actions/upload-artifact@v4
416419
with:
417420
name: ${{ matrix.rn-version }}-${{ matrix.rn-architecture }}-${{ matrix.engine }}-${{ matrix.platform }}-${{ matrix.build-type }}-${{ matrix.ios-use-frameworks }}-logs
418-
path: |
419-
test/e2e/*.log
420-
test/e2e/*.png
421+
path: ./dev-packages/e2e-tests/maestro-logs

dev-packages/e2e-tests/.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@
33
*.app
44
*.apk
55

6-
react-native-versions
6+
/react-native-versions
7+
/maestro-logs

dev-packages/e2e-tests/cli.mjs

Lines changed: 29 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/usr/bin/env node
22
'use strict';
33

4-
import { execSync, spawn } from 'child_process';
4+
import { execSync, execFileSync, spawn } from 'child_process';
55
import * as fs from 'fs';
66
import * as path from 'path';
77
import { argv, env } from 'process';
@@ -64,8 +64,9 @@ const appRepoDir = `${e2eDir}/react-native-versions/${RNVersion}`;
6464
const appName = 'RnDiffApp';
6565
const appDir = `${appRepoDir}/${appName}`;
6666
const testAppName = `${appName}.${platform == 'ios' ? 'app' : 'apk'}`;
67-
const runtime = env.IOS_RUNTIME ? env.IOS_RUNTIME : 'latest';
68-
const device = env.IOS_DEVICE ? env.IOS_DEVICE : 'iPhone 15';
67+
const testApp = `${e2eDir}/${testAppName}`;
68+
const appId = platform === 'ios' ? 'org.reactjs.native.example.RnDiffApp' : 'com.rndiffapp';
69+
const sentryAuthToken = env.SENTRY_AUTH_TOKEN;
6970

7071
// Build and publish the SDK - we only need to do this once in CI.
7172
// Locally, we may want to get updates from the latest build so do it on every app build.
@@ -143,9 +144,6 @@ if (actions.includes('create')) {
143144
});
144145

145146
if (fs.existsSync(`${appDir}/Gemfile`)) {
146-
// TMP Fix for https://github.com/CocoaPods/Xcodeproj/issues/989
147-
fs.appendFileSync(`${appDir}/Gemfile`, "gem 'xcodeproj', '< 1.26.0'\n");
148-
149147
execSync(`bundle install`, { stdio: 'inherit', cwd: appDir, env: env });
150148
execSync('bundle exec pod install --repo-update', { stdio: 'inherit', cwd: `${appDir}/ios`, env: env });
151149
} else {
@@ -190,7 +188,8 @@ if (actions.includes('build')) {
190188
-workspace ${appName}.xcworkspace \
191189
-configuration ${buildType} \
192190
-scheme ${appName} \
193-
-destination 'platform=iOS Simulator,OS=${runtime},name=${device}' \
191+
-sdk 'iphonesimulator' \
192+
-destination 'generic/platform=iOS Simulator' \
194193
ONLY_ACTIVE_ARCH=yes \
195194
-derivedDataPath DerivedData \
196195
build | tee xcodebuild.log | xcbeautify`,
@@ -207,123 +206,40 @@ if (actions.includes('build')) {
207206
appProduct = `${appDir}/android/app/build/outputs/apk/release/app-release.apk`;
208207
}
209208

210-
var testApp = `${e2eDir}/${testAppName}`;
211209
console.log(`Moving ${appProduct} to ${testApp}`);
212210
if (fs.existsSync(testApp)) fs.rmSync(testApp, { recursive: true });
213211
fs.renameSync(appProduct, testApp);
214212
}
215213

216214
if (actions.includes('test')) {
217-
if (
218-
platform == 'ios' &&
219-
!fs.existsSync(`${e2eDir}/DerivedData/Build/Products/Debug-iphonesimulator/WebDriverAgentRunner-Runner.app`)
220-
) {
221-
// Build iOS WebDriverAgent
222-
execSync(
223-
`set -o pipefail && xcodebuild \
224-
-project node_modules/appium-webdriveragent/WebDriverAgent.xcodeproj \
225-
-scheme WebDriverAgentRunner \
226-
-destination 'platform=iOS Simulator,OS=${runtime},name=${device}' \
227-
GCC_TREAT_WARNINGS_AS_ERRORS=0 \
228-
COMPILER_INDEX_STORE_ENABLE=NO \
229-
ONLY_ACTIVE_ARCH=yes \
230-
-derivedDataPath DerivedData \
231-
build | tee xcodebuild-agent.log | xcbeautify`,
232-
{ stdio: 'inherit', cwd: e2eDir, env: env },
233-
);
234-
}
235-
236-
// Start the appium server.
237-
var processesToKill = {};
238-
async function newProcess(name, process) {
239-
await new Promise((resolve, reject) => {
240-
process.on('error', e => {
241-
console.error(`Failed to start process '${name}': ${e}`);
242-
reject(e);
243-
});
244-
process.on('spawn', () => {
245-
console.log(`Process '${name}' (${process.pid}) started`);
246-
resolve();
247-
});
248-
});
249-
250-
processesToKill[name] = {
251-
process: process,
252-
complete: new Promise((resolve, _reject) => {
253-
process.on('close', resolve);
254-
}),
255-
};
256-
}
257-
await newProcess(
258-
'appium',
259-
spawn('node_modules/.bin/appium', ['--log-timestamp', '--log-no-colors', '--log', `appium${platform}.log`], {
260-
stdio: 'inherit',
261-
cwd: e2eDir,
262-
env: env,
263-
shell: false,
264-
}),
265-
);
266-
267-
try {
268-
await waitForAppium();
269-
270-
// Run e2e tests
271-
const testEnv = env;
272-
testEnv.PLATFORM = platform;
273-
testEnv.APPIUM_APP = `./${testAppName}`;
274-
275-
if (platform == 'ios') {
276-
testEnv.APPIUM_DERIVED_DATA = 'DerivedData';
277-
} else if (platform == 'android') {
278-
execSync(`adb devices -l`, { stdio: 'inherit', cwd: e2eDir, env: env });
279-
280-
execSync(`adb logcat -c`, { stdio: 'inherit', cwd: e2eDir, env: env });
281-
282-
var adbLogStream = fs.createWriteStream(`${e2eDir}/adb.log`);
283-
const adbLogProcess = spawn('adb', ['logcat'], { cwd: e2eDir, env: env, shell: false });
284-
adbLogProcess.stdout.pipe(adbLogStream);
285-
adbLogProcess.stderr.pipe(adbLogStream);
286-
adbLogProcess.on('close', () => adbLogStream.close());
287-
await newProcess('adb logcat', adbLogProcess);
288-
}
289-
290-
execSync(`yarn test:e2e:runner --verbose`, { stdio: 'inherit', cwd: e2eDir, env: testEnv });
291-
} finally {
292-
for (const [name, info] of Object.entries(processesToKill)) {
293-
console.log(`Sending termination signal to process '${name}' (${info.process.pid})`);
294-
295-
// Send SIGTERM first to allow graceful shutdown.
296-
info.process.kill(15);
297-
298-
// Also send SIGKILL after 10 seconds.
299-
const killTimeout = setTimeout(() => process.kill(9), '10000');
300-
301-
// Wait for the process to exit (either via SIGTERM or SIGKILL).
302-
const code = await info.complete;
303-
304-
// Successfully exited now, no need to kill (if it hasn't run yet).
305-
clearTimeout(killTimeout);
306-
307-
console.log(`Process '${name}' (${info.process.pid}) exited with code ${code}`);
215+
// Run e2e tests
216+
if (platform == 'ios') {
217+
try {
218+
execSync('xcrun simctl list devices | grep -q "(Booted)"');
219+
} catch (error) {
220+
throw new Error('No simulator is currently booted. Please boot a simulator before running this script.');
308221
}
309-
}
310-
}
311222

312-
async function waitForAppium() {
313-
console.log('Waiting for Appium server to start...');
314-
for (let i = 0; i < 60; i++) {
223+
execFileSync('xcrun', ['simctl', 'install', 'booted', testApp]);
224+
} else if (platform == 'android') {
315225
try {
316-
await fetch('http://127.0.0.1:4723/sessions', { method: 'HEAD' });
317-
console.log('Appium server started');
318-
return;
226+
execSync('adb devices | grep -q "emulator"');
319227
} catch (error) {
320-
console.log(`Appium server hasn't started yet (${error})...`);
321-
await sleep(1000);
228+
throw new Error('No Android emulator is currently running. Please start an emulator before running this script.');
322229
}
230+
231+
execFileSync('adb', ['install', '-r', '-d', testApp]);
323232
}
324-
throw new Error('Appium server failed to start');
325-
}
326233

327-
async function sleep(millis) {
328-
return new Promise(resolve => setTimeout(resolve, millis));
234+
execSync(
235+
`maestro test maestro \
236+
--env=APP_ID="${appId}" \
237+
--env=SENTRY_AUTH_TOKEN="${sentryAuthToken}" \
238+
--debug-output maestro-logs \
239+
--flatten-debug-output`,
240+
{
241+
stdio: 'inherit',
242+
cwd: e2eDir,
243+
},
244+
);
329245
}
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
appId: ${APP_ID}
2+
---
3+
- runFlow: utils/launchTestAppClear.yml
4+
- tapOn: "Capture Exception"
5+
- runFlow: utils/assertEventIdVisible.yml
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
appId: ${APP_ID}
2+
---
3+
- runFlow: utils/launchTestAppClear.yml
4+
- tapOn: "Capture Message"
5+
- runFlow: utils/assertEventIdVisible.yml
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
appId: ${APP_ID}
2+
---
3+
- runFlow: utils/launchTestAppClear.yml
4+
- tapOn: "Unhandled Promise Rejection"
5+
- runFlow: utils/assertEventIdVisible.yml
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
appId: ${APP_ID}
2+
---
3+
- runFlow: utils/launchTestAppClear.yml
4+
- tapOn: "Close"
5+
6+
- assertNotVisible:
7+
id: "eventId"
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
appId: ${APP_ID}
2+
---
3+
- runFlow: utils/launchTestAppClear.yml
4+
- tapOn: "Crash"
5+
6+
- launchApp
7+
- assertVisible: "E2E Tests Ready"
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
appId: ${APP_ID}
2+
---
3+
- extendedWaitUntil:
4+
visible:
5+
id: "eventId"
6+
timeout: 600_000 # 10 minutes
7+
8+
- copyTextFrom:
9+
id: "eventId"
10+
- assertTrue: ${maestro.copiedText}
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
appId: ${APP_ID}
2+
---
3+
- launchApp:
4+
clearState: true
5+
arguments:
6+
sentryAuthToken: ${SENTRY_AUTH_TOKEN}
7+
8+
- extendedWaitUntil:
9+
visible: "E2E Tests Ready"
10+
timeout: 120_000 # 2 minutes

0 commit comments

Comments
 (0)