diff --git a/testing_app/README.md b/testing_app/README.md index dc147a2fe..ca53a8112 100644 --- a/testing_app/README.md +++ b/testing_app/README.md @@ -31,10 +31,14 @@ The Flutter SDK can run unit tests and widget tests in a virtual machine, withou - Run `flutter drive --target=test_driver/` - eg. `flutter drive --target=test_driver/app.dart` to run the test in `test_driver/app_test.dart` - Performance Tests: - - Run `flutter drive --target=test_driver/app.dart --driver test_driver/perf_test.dart --profile --trace-startup` + - Run `flutter drive --target=test_driver/app.dart --driver test_driver/perf_test.dart --profile --trace-startup` - Using a physical device and running performance tests in profile mode is recommended. - The `--trace-startup` option is used to avoid flushing older timeline events when the timeline gets long. -- State Management Tests: +- [E2E](https://pub.dev/packages/e2e) Tests: + - Run `flutter drive --target test/perf_test_e2e.dart --driver test_driver/e2e_test.dart --profile` + - Similar to the above but the test is driven on device. + - You may also reference [E2E manual](https://github.com/flutter/plugins/tree/master/packages/e2e#firebase-test-lab) for how to run such test on Firebase Test Lab. +- State Management Tests: - For testing state using Flutter Driver - Run `flutter drive --target=test_driver/` @@ -42,7 +46,7 @@ The Flutter SDK can run unit tests and widget tests in a virtual machine, withou - Refer [.travis.yml](../.travis.yml) and the [tool](../tool) directory to see how to test Flutter projects using Travis-CI. Note that we aren't performing Flutter Driver tests using the Travis tool in this repo. That is because it's recommended to use physical devices to run Driver tests. You can use [Firebase Test Lab](https://firebase.google.com/docs/test-lab), [Codemagic](https://codemagic.io/) or any platform of your choice to do that. - + ## Questions/issues If you have a general question about testing in Flutter, the best places to go are: diff --git a/testing_app/android/app/build.gradle b/testing_app/android/app/build.gradle index e840d21d4..7a0746804 100644 --- a/testing_app/android/app/build.gradle +++ b/testing_app/android/app/build.gradle @@ -43,6 +43,7 @@ android { targetSdkVersion 28 versionCode flutterVersionCode.toInteger() versionName flutterVersionName + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" } buildTypes { @@ -60,4 +61,10 @@ flutter { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" + + testImplementation 'junit:junit:4.12' + + // https://developer.android.com/jetpack/androidx/releases/test/#1.2.0 + androidTestImplementation 'androidx.test:runner:1.2.0' + androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' } diff --git a/testing_app/android/app/src/androidTest/java/dev/flutter/testing_app/MainActivityTest.java b/testing_app/android/app/src/androidTest/java/dev/flutter/testing_app/MainActivityTest.java new file mode 100644 index 000000000..bd19a0a12 --- /dev/null +++ b/testing_app/android/app/src/androidTest/java/dev/flutter/testing_app/MainActivityTest.java @@ -0,0 +1,12 @@ +package dev.flutter.testing_app; + +import androidx.test.rule.ActivityTestRule; +import dev.flutter.plugins.e2e.FlutterTestRunner; +import org.junit.Rule; +import org.junit.runner.RunWith; + +@RunWith(FlutterTestRunner.class) +public class MainActivityTest { + @Rule + public ActivityTestRule rule = new ActivityTestRule<>(MainActivity.class, true, false); +} diff --git a/testing_app/lib/screens/home.dart b/testing_app/lib/screens/home.dart index a20884131..a39aea341 100644 --- a/testing_app/lib/screens/home.dart +++ b/testing_app/lib/screens/home.dart @@ -29,6 +29,7 @@ class HomePage extends StatelessWidget { body: ListView.builder( itemCount: 100, cacheExtent: 20.0, + controller: ScrollController(), padding: const EdgeInsets.symmetric(vertical: 16), itemBuilder: (context, index) => ItemTile(index), ), diff --git a/testing_app/pubspec.lock b/testing_app/pubspec.lock index b47b953ce..ee373a189 100644 --- a/testing_app/pubspec.lock +++ b/testing_app/pubspec.lock @@ -7,14 +7,14 @@ packages: name: _fe_analyzer_shared url: "https://pub.dartlang.org" source: hosted - version: "4.0.0" + version: "7.0.0" analyzer: dependency: transitive description: name: analyzer url: "https://pub.dartlang.org" source: hosted - version: "0.39.10" + version: "0.39.17" archive: dependency: transitive description: @@ -50,6 +50,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.1.3" + cli_util: + dependency: transitive + description: + name: cli_util + url: "https://pub.dartlang.org" + source: hosted + version: "0.2.0" collection: dependency: transitive description: @@ -84,7 +91,7 @@ packages: name: csslib url: "https://pub.dartlang.org" source: hosted - version: "0.16.1" + version: "0.16.2" cupertino_icons: dependency: "direct main" description: @@ -92,6 +99,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "0.1.3" + e2e: + dependency: "direct dev" + description: + name: e2e + url: "https://pub.dartlang.org" + source: hosted + version: "0.7.0" file: dependency: transitive description: @@ -139,7 +153,7 @@ packages: name: http url: "https://pub.dartlang.org" source: hosted - version: "0.12.1" + version: "0.12.2" http_multi_server: dependency: transitive description: @@ -216,7 +230,7 @@ packages: name: mime url: "https://pub.dartlang.org" source: hosted - version: "0.9.6+3" + version: "0.9.7" multi_server_socket: dependency: transitive description: @@ -307,7 +321,7 @@ packages: name: provider url: "https://pub.dartlang.org" source: hosted - version: "4.1.3" + version: "4.3.2" pub_semver: dependency: transitive description: @@ -328,7 +342,7 @@ packages: name: shelf url: "https://pub.dartlang.org" source: hosted - version: "0.7.7" + version: "0.7.8" shelf_packages_handler: dependency: transitive description: @@ -452,7 +466,7 @@ packages: name: vm_service url: "https://pub.dartlang.org" source: hosted - version: "4.1.0" + version: "4.2.0" vm_service_client: dependency: transitive description: @@ -504,4 +518,4 @@ packages: version: "2.2.1" sdks: dart: ">=2.7.0 <3.0.0" - flutter: ">=1.16.0" + flutter: ">=1.16.0 <2.0.0" diff --git a/testing_app/pubspec.yaml b/testing_app/pubspec.yaml index a807c7142..8eaad5e66 100644 --- a/testing_app/pubspec.yaml +++ b/testing_app/pubspec.yaml @@ -19,6 +19,7 @@ dev_dependencies: flutter_driver: sdk: flutter test: ^1.14.4 + e2e: ^0.7.0 pedantic: ^1.9.0 flutter: diff --git a/testing_app/test/e2e_utils.dart b/testing_app/test/e2e_utils.dart new file mode 100644 index 000000000..418c5f49c --- /dev/null +++ b/testing_app/test/e2e_utils.dart @@ -0,0 +1,192 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// TODO(CareF): This file should be removed after the changes goes into flutter +// stable version. + +// ignore linter to keep the code consistent with its duplicate in the framework +// ignore_for_file: use_function_type_syntax_for_parameters, omit_local_variable_types, avoid_types_on_closure_parameters + +import 'dart:async'; +import 'dart:ui'; + +import 'package:e2e/e2e.dart'; +import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; + +/// The maximum amount of time considered safe to spend for a frame's build +/// phase. Anything past that is in the danger of missing the frame as 60FPS. +/// +/// Changing this doesn't re-evaluate existing summary. +Duration kBuildBudget = const Duration(milliseconds: 16); + +bool _firstRun = true; + +const String kDebugWarning = ''' +┏╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┓ +┇ ⚠ THIS BENCHMARK IS BEING RUN IN DEBUG MODE ⚠ ┇ +┡╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┦ +│ │ +│ Numbers obtained from a benchmark while asserts are │ +│ enabled will not accurately reflect the performance │ +│ that will be experienced by end users using release ╎ +│ builds. Benchmarks should be run using this command ╎ +│ line: "flutter run --profile test.dart" or ┊ +│ or "flutter drive --profile -t test.dart". ┊ +│ ┊ +└─────────────────────────────────────────────────╌┄┈ 🐢 +'''; + +/// watches the [FrameTiming] of `action` and report it to the e2e binding. +Future watchPerformance( + E2EWidgetsFlutterBinding binding, + Future action(), { + String reportKey = 'performance', +}) async { + assert(() { + if (_firstRun) { + debugPrint(kDebugWarning); + _firstRun = false; + } + return true; + }()); + final List frameTimings = []; + final TimingsCallback watcher = frameTimings.addAll; + binding.addTimingsCallback(watcher); + await action(); + binding.removeTimingsCallback(watcher); + final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings); + binding.reportData = {reportKey: frameTimes.summary}; +} + +/// This class and summarizes a list of [FrameTiming] for the performance +/// metrics. +class FrameTimingSummarizer { + factory FrameTimingSummarizer(List data) { + assert(data != null); + assert(data.isNotEmpty); + final List frameBuildTime = List.unmodifiable( + data.map((FrameTiming datum) => datum.buildDuration), + ); + final List frameBuildTimeSorted = + List.from(frameBuildTime)..sort(); + final List frameRasterizerTime = List.unmodifiable( + data.map((FrameTiming datum) => datum.rasterDuration), + ); + final List frameRasterizerTimeSorted = + List.from(frameRasterizerTime)..sort(); + final Duration Function(Duration, Duration) add = + (Duration a, Duration b) => a + b; + return FrameTimingSummarizer._( + frameBuildTime: frameBuildTime, + frameRasterizerTime: frameRasterizerTime, + // This avarage calculation is microsecond precision, which is fine + // because typical values of these times are milliseconds. + averageFrameBuildTime: frameBuildTime.reduce(add) ~/ data.length, + p90FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.90), + p99FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.99), + worstFrameBuildTime: frameBuildTimeSorted.last, + missedFrameBuildBudget: _countExceed(frameBuildTimeSorted, kBuildBudget), + averageFrameRasterizerTime: + frameRasterizerTime.reduce(add) ~/ data.length, + p90FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90), + p99FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90), + worstFrameRasterizerTime: frameRasterizerTimeSorted.last, + missedFrameRasterizerBudget: + _countExceed(frameRasterizerTimeSorted, kBuildBudget), + ); + } + + const FrameTimingSummarizer._( + {@required this.frameBuildTime, + @required this.frameRasterizerTime, + @required this.averageFrameBuildTime, + @required this.p90FrameBuildTime, + @required this.p99FrameBuildTime, + @required this.worstFrameBuildTime, + @required this.missedFrameBuildBudget, + @required this.averageFrameRasterizerTime, + @required this.p90FrameRasterizerTime, + @required this.p99FrameRasterizerTime, + @required this.worstFrameRasterizerTime, + @required this.missedFrameRasterizerBudget}); + + /// List of frame build time in microseconds + final List frameBuildTime; + + /// List of frame rasterizer time in microseconds + final List frameRasterizerTime; + + /// The average value of [frameBuildTime] in milliseconds. + final Duration averageFrameBuildTime; + + /// The 90-th percentile value of [frameBuildTime] in milliseconds + final Duration p90FrameBuildTime; + + /// The 99-th percentile value of [frameBuildTime] in milliseconds + final Duration p99FrameBuildTime; + + /// The largest value of [frameBuildTime] in milliseconds + final Duration worstFrameBuildTime; + + /// Number of items in [frameBuildTime] that's greater than [kBuildBudget] + final int missedFrameBuildBudget; + + /// The average value of [frameRasterizerTime] in milliseconds. + final Duration averageFrameRasterizerTime; + + /// The 90-th percentile value of [frameRasterizerTime] in milliseconds. + final Duration p90FrameRasterizerTime; + + /// The 99-th percentile value of [frameRasterizerTime] in milliseconds. + final Duration p99FrameRasterizerTime; + + /// The largest value of [frameRasterizerTime] in milliseconds. + final Duration worstFrameRasterizerTime; + + /// Number of items in [frameRasterizerTime] that's greater than [kBuildBudget] + final int missedFrameRasterizerBudget; + + Map get summary => { + 'average_frame_build_time_millis': + averageFrameBuildTime.inMicroseconds / 1E3, + '90th_percentile_frame_build_time_millis': + p90FrameBuildTime.inMicroseconds / 1E3, + '99th_percentile_frame_build_time_millis': + p99FrameBuildTime.inMicroseconds / 1E3, + 'worst_frame_build_time_millis': + worstFrameBuildTime.inMicroseconds / 1E3, + 'missed_frame_build_budget_count': missedFrameBuildBudget, + 'average_frame_rasterizer_time_millis': + averageFrameRasterizerTime.inMicroseconds / 1E3, + '90th_percentile_frame_rasterizer_time_millis': + p90FrameRasterizerTime.inMicroseconds / 1E3, + '99th_percentile_frame_rasterizer_time_millis': + p99FrameRasterizerTime.inMicroseconds / 1E3, + 'worst_frame_rasterizer_time_millis': + worstFrameRasterizerTime.inMicroseconds / 1E3, + 'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget, + 'frame_count': frameBuildTime.length, + 'frame_build_times': frameBuildTime + .map((Duration datum) => datum.inMicroseconds) + .toList(), + 'frame_rasterizer_times': frameRasterizerTime + .map((Duration datum) => datum.inMicroseconds) + .toList(), + }; +} + +// The following helper functions require data sorted + +// return the 100*p-th percentile of the data +T _findPercentile(List data, double p) { + assert(p >= 0 && p <= 1); + return data[((data.length - 1) * p).round()]; +} + +// return the number of items in data that > threshold +int _countExceed>(List data, T threshold) { + return data.length - + data.indexWhere((T datum) => datum.compareTo(threshold) > 0); +} diff --git a/testing_app/test/perf_test_e2e.dart b/testing_app/test/perf_test_e2e.dart new file mode 100644 index 000000000..de02992ad --- /dev/null +++ b/testing_app/test/perf_test_e2e.dart @@ -0,0 +1,87 @@ +// Copyright 2020 The Flutter team. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This file duplicates the behavior of test_driver/perf_test.dart, but uses +// the e2e package to implement a host-independent test. + +import 'dart:convert' show JsonEncoder; + +import 'package:e2e/e2e.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:testing_app/main.dart' as app; + +import 'e2e_utils.dart'; + +void main() { + final binding = + E2EWidgetsFlutterBinding.ensureInitialized() as E2EWidgetsFlutterBinding; + // The fullyLive frame policy simulates the way Flutter response to animations. + // See https://github.com/flutter/flutter/issues/60237 + binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive; + + group('Testing App Performance Tests on e2e', () { + testWidgets('Scrolling test', (tester) async { + app.main(); + await tester.pumpAndSettle(); + await watchPerformance(binding, () async { + final listFinder = find.byType(ListView); + final scroller = tester.widget(listFinder).controller; + await scroller.animateTo( + 7000, + duration: const Duration(seconds: 1), + curve: Curves.linear, + ); + await tester.pumpAndSettle(); + + await scroller.animateTo( + -7000, + duration: const Duration(seconds: 1), + curve: Curves.linear, + ); + await tester.pumpAndSettle(); + }, reportKey: 'scrolling'); + // The performance result is reported to `data['scrolling']`. + // See `e2e_test.dart` for detail. + print('scrolling performance test result:'); + print(JsonEncoder.withIndent(' ') + .convert(binding.reportData['scrolling'])); + }, semanticsEnabled: false); + + testWidgets('Favorites operations test', (tester) async { + app.main(); + await tester.pumpAndSettle(); + await watchPerformance(binding, () async { + final iconKeys = [ + 'icon_0', + 'icon_1', + 'icon_2', + ]; + for (var icon in iconKeys) { + await tester.tap(find.byKey(ValueKey(icon))); + await tester.pumpAndSettle(); + } + + await tester.tap(find.text('Favorites')); + await tester.pumpAndSettle(); + + final removeIconKeys = [ + 'remove_icon_0', + 'remove_icon_1', + 'remove_icon_2', + ]; + + for (final iconKey in removeIconKeys) { + await tester.tap(find.byKey(ValueKey(iconKey))); + await tester.pumpAndSettle(); + } + }, reportKey: 'favorites_operations'); + // The performance result is reported to `data['favorites_operations']`. + // See `e2e_test.dart` for detail. + print('favorites_operations performance test result:'); + print(JsonEncoder.withIndent(' ') + .convert(binding.reportData['favorites_operations'])); + }, semanticsEnabled: false); + }); +} diff --git a/testing_app/test_driver/e2e_test.dart b/testing_app/test_driver/e2e_test.dart new file mode 100644 index 000000000..90fc11606 --- /dev/null +++ b/testing_app/test_driver/e2e_test.dart @@ -0,0 +1,21 @@ +// Copyright 2020 The Flutter team. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:e2e/e2e_driver.dart' as driver; + +Future main() => driver.e2eDriver(responseDataCallback: (data) async { + await driver.writeResponseData( + data['scrolling'] as Map, + // This result is saved to `build/scrolling.json`. + testOutputFilename: 'scrolling', + ); + + await driver.writeResponseData( + data['favorites_operations'] as Map, + // This result is saved to `build/favorites_operations.json`. + testOutputFilename: 'favorites_operations', + ); + }); diff --git a/testing_app/test_driver/perf_test.dart b/testing_app/test_driver/perf_test.dart index 61b7b09b9..d3b7adbb1 100644 --- a/testing_app/test_driver/perf_test.dart +++ b/testing_app/test_driver/perf_test.dart @@ -42,12 +42,12 @@ void main() { final scrollingSummary = TimelineSummary.summarize(scrollingTimeline); // Then, save the summary to disk. - // Results will be stored in the file 'build/scrolling.timeline.json'. + // Results will be stored in + // the file 'build/scrolling.timeline_summary.json'. await scrollingSummary.writeSummaryToFile('scrolling', pretty: true); // Write the entire timeline to disk in a json format. - // Results will be stored in - // the file 'build/scrolling.timeline_summary.json'. + // Results will be stored in the file 'build/scrolling.timeline.json'. // This file can be opened in the Chrome browser's tracing tools // found by navigating to chrome://tracing. await scrollingSummary.writeTimelineToFile('scrolling', pretty: true); diff --git a/tool/travis_android_script.sh b/tool/travis_android_script.sh index 53fdcbec3..7bf2d059a 100755 --- a/tool/travis_android_script.sh +++ b/tool/travis_android_script.sh @@ -77,4 +77,21 @@ gcloud firebase test android run --type instrumentation \ --timeout 5m popd +echo "== Run e2e test for testing_app ==" +pushd "testing_app" +readonly APP_DIR=$(pwd) +"${LOCAL_SDK_PATH}/bin/flutter" packages get +"${LOCAL_SDK_PATH}/bin/flutter" build apk +pushd "android" +./gradlew app:assembleAndroidTest +./gradlew app:assembleRelease -Ptarget=${APP_DIR}/test/perf_test_e2e.dart +popd +gcloud auth activate-service-account --key-file=../svc-keyfile.json +gcloud --quiet config set project test-lab-project-ccbec +gcloud firebase test android run --type instrumentation \ + --app build/app/outputs/apk/release/app-release.apk \ + --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk \ + --timeout 5m +popd + echo "-- Success --"