Skip to content

Commit a16de73

Browse files
matanlureymatanlurey
authored
Pin customer_testing to the SHA specified in tests.version (flutter#162048)
Closes flutter#162041. I also converted the `.sh` and `.bat` files (largely) to Dart, mostly because I don't know enough Windows XP command prompt in order to make the change safely. They should still run the same from the CI bootstrapping, but most of the logic is now Dart code. Will send a separate PR to update the `flutter/tests` PR template. /cc @johnmccutchan @Piinks
1 parent a3f0704 commit a16de73

File tree

7 files changed

+139
-163
lines changed

7 files changed

+139
-163
lines changed

‎dev/customer_testing/README.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# customer_testing
2+
3+
This tool checks out <https://github.com/flutter/tests> at the commit SHA
4+
specified in [`tests.version`](tests.version), and runs the tests registered to
5+
verify that end-user apps and libraries are working at the current tip-of-tree
6+
of Flutter.
7+
8+
To (locally) test a specific SHA, use `ci.dart`:
9+
10+
```sh
11+
cd dev/customer_testing
12+
dart ci.dart [sha]
13+
```
14+
15+
Or, to update the SHA for our CI, edit and send a PR for
16+
[`tests.version`](tests.version).

‎dev/customer_testing/ci.bat‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,16 @@ REM This should match the ci.sh file in this directory.
88
REM This is called from the LUCI recipes:
99
REM https://github.com/flutter/flutter/blob/main/dev/bots/suite_runners/run_customer_testing_tests.dart
1010

11+
REM This script does not assume that "flutter update-packages" has been
12+
REM run, to allow CIs to save time by skipping that steps since it's
13+
REM largely not needed to run the flutter/tests tests.
14+
REM
15+
REM However, we do need to update this directory.
16+
SETLOCAL
17+
cd /d %~dp0
1118
ECHO.
1219
ECHO Updating pub packages...
1320
CALL dart pub get
14-
CD ..\tools
15-
CALL dart pub get
16-
CD ..\customer_testing
17-
18-
ECHO.
19-
ECHO Finding correct version of customer tests...
20-
CMD /S /C "IF EXIST "..\..\bin\cache\pkg\tests\" RMDIR /S /Q ..\..\bin\cache\pkg\tests"
21-
git clone https://github.com/flutter/tests.git ..\..\bin\cache\pkg\tests
22-
FOR /F "usebackq tokens=*" %%a IN (`dart --enable-asserts ..\tools\bin\find_commit.dart . master ..\..\bin\cache\pkg\tests main`) DO git -C ..\..\bin\cache\pkg\tests checkout %%a
2321

24-
ECHO.
25-
ECHO Running tests...
26-
CD ..\..\bin\cache\pkg\tests
27-
CALL dart --enable-asserts ..\..\..\..\dev\customer_testing\run_tests.dart --verbose --skip-on-fetch-failure --skip-template registry/*.test
22+
REM Run the cross-platform script.
23+
CALL ..\..\bin\dart.bat ci.dart

‎dev/customer_testing/ci.dart‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
// Copyright 2014 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:io' as io;
6+
7+
import 'package:path/path.dart' as p;
8+
9+
/// To run this script, either:
10+
///
11+
/// ```sh
12+
/// cd dev/customer_testing
13+
/// dart ci.dart [sha]
14+
/// ```
15+
///
16+
/// Or:
17+
///
18+
/// ```sh
19+
/// ./dev/customer_testing/ci.sh
20+
/// ./dev/customer_testing/ci.bat
21+
/// ```
22+
void main(List<String> args) async {
23+
final String sha;
24+
if (args.isEmpty) {
25+
sha = io.File('tests.version').readAsStringSync().trim();
26+
} else if (args.length == 1) {
27+
sha = args.first;
28+
} else {
29+
io.stderr.writeln('Usage: dart ci.dart [sha]');
30+
io.exitCode = 1;
31+
return;
32+
}
33+
34+
final String flutterRootPath = p.canonicalize('../../');
35+
final io.Directory testsCacheDir = io.Directory(
36+
p.join(flutterRootPath, 'bin', 'cache', 'pkg', 'tests'),
37+
);
38+
39+
if (testsCacheDir.existsSync()) {
40+
io.stderr.writeln('Cleaning up existing repo: ${testsCacheDir.path}');
41+
testsCacheDir.deleteSync(recursive: true);
42+
}
43+
44+
io.stderr.writeln('Cloning flutter/tests');
45+
final io.Process clone = await io.Process.start('git', <String>[
46+
'clone',
47+
'--depth',
48+
'1',
49+
'https://github.com/flutter/tests.git',
50+
testsCacheDir.path,
51+
], mode: io.ProcessStartMode.inheritStdio);
52+
if ((await clone.exitCode) != 0) {
53+
io.exitCode = 1;
54+
return;
55+
}
56+
57+
io.stderr.writeln('Fetching/checking out $sha');
58+
final io.Process fetch = await io.Process.start(
59+
'git',
60+
<String>['fetch', 'origin', sha],
61+
mode: io.ProcessStartMode.inheritStdio,
62+
workingDirectory: testsCacheDir.path,
63+
);
64+
if ((await fetch.exitCode) != 0) {
65+
io.exitCode = 1;
66+
return;
67+
}
68+
final io.Process checkout = await io.Process.start(
69+
'git',
70+
<String>['checkout', sha],
71+
mode: io.ProcessStartMode.inheritStdio,
72+
workingDirectory: testsCacheDir.path,
73+
);
74+
if ((await checkout.exitCode) != 0) {
75+
io.exitCode = 1;
76+
return;
77+
}
78+
79+
io.stderr.writeln('Running tests...');
80+
final io.Process test = await io.Process.start('dart', <String>[
81+
'--enable-asserts',
82+
'run_tests.dart',
83+
'--skip-on-fetch-failure',
84+
'--skip-template',
85+
p.posix.joinAll(<String>[...p.split(testsCacheDir.path), 'registry', '*.test']),
86+
], mode: io.ProcessStartMode.inheritStdio);
87+
if ((await test.exitCode) != 0) {
88+
io.exitCode = 1;
89+
return;
90+
}
91+
}

‎dev/customer_testing/ci.sh‎

Lines changed: 18 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -8,30 +8,28 @@
88
# This is called from the LUCI recipes:
99
# https://github.com/flutter/flutter/blob/main/dev/bots/suite_runners/run_customer_testing_tests.dart
1010

11-
set -ex
11+
set -e
12+
13+
function script_location() {
14+
local script_location="${BASH_SOURCE[0]}"
15+
# Resolve symlinks
16+
while [[ -h "$script_location" ]]; do
17+
DIR="$(cd -P "$( dirname "$script_location")" >/dev/null && pwd)"
18+
script_location="$(readlink "$script_location")"
19+
[[ "$script_location" != /* ]] && script_location="$DIR/$script_location"
20+
done
21+
cd -P "$(dirname "$script_location")" >/dev/null && pwd
22+
}
23+
24+
# So that users can run this script from anywhere and it will work as expected.
25+
cd "$(script_location)"
1226

1327
# This script does not assume that "flutter update-packages" has been
1428
# run, to allow CIs to save time by skipping that steps since it's
1529
# largely not needed to run the flutter/tests tests.
1630
#
17-
# However, we do need to update this directory and the tools directory.
31+
# However, we do need to update this directory.
1832
dart pub get
19-
(cd ../tools; dart pub get) # used for find_commit.dart below
20-
21-
# Next we need to update the flutter/tests checkout.
22-
#
23-
# We use find_commit.dart so that we pull the version of flutter/tests
24-
# that was contemporary when the branch we are on was created. That
25-
# way, we can still run the tests on long-lived branches without being
26-
# affected by breaking changes on trunk causing changes to the tests
27-
# that wouldn't work on the long-lived branch.
28-
#
29-
# (This also prevents trunk from suddenly failing when tests are
30-
# revved on flutter/tests -- if you rerun a passing customer_tests
31-
# shard, it should still pass, even if we rolled one of the tests.)
32-
rm -rf ../../bin/cache/pkg/tests
33-
git clone https://github.com/flutter/tests.git ../../bin/cache/pkg/tests
34-
git -C ../../bin/cache/pkg/tests checkout `dart --enable-asserts ../tools/bin/find_commit.dart . master ../../bin/cache/pkg/tests main`
3533

36-
# Finally, run the tests.
37-
dart --enable-asserts run_tests.dart --skip-on-fetch-failure --skip-template ../../bin/cache/pkg/tests/registry/*.test
34+
# Run the cross-platform script.
35+
../../bin/dart run ci.dart

‎dev/customer_testing/run_tests.dart‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,10 @@ Future<bool> run(List<String> arguments) async {
7676
.where((File file) => !skipTemplate || path.basename(file.path) != 'template.test')
7777
.toList();
7878

79+
if (files.isEmpty && parsedArguments.rest.isNotEmpty) {
80+
print('No files resolved from glob(s): ${parsedArguments.rest}');
81+
}
82+
7983
if (help ||
8084
repeat == null ||
8185
files.isEmpty ||

‎dev/customer_testing/tests.version‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
b4cc097211814cfb0e1f08affb4574cf7de6b1cd

‎dev/tools/bin/find_commit.dart‎

Lines changed: 0 additions & 130 deletions
This file was deleted.

0 commit comments

Comments
 (0)