Add an e2e (host-independent) test demo to testing_app (#515)

testing_app/README.md

@@ -34,6 +34,10 @@ The Flutter SDK can run unit tests and widget tests in a virtual machine, withou
   - Run `flutter drive --target=test_driver/app.dart --driver test_driver/perf_test.dart --profile --trace-startup`
     - Using a physical device and running performance tests in profile mode is recommended.
     - The `--trace-startup` option is used to avoid flushing older timeline events when the timeline gets long.
+- [E2E](https://pub.dev/packages/e2e) Tests:
+  - Run `flutter drive --target test/perf_test_e2e.dart --driver test_driver/e2e_test.dart --profile`
+  - Similar to the above but the test is driven on device.
+  - You may also reference [E2E manual](https://github.com/flutter/plugins/tree/master/packages/e2e#firebase-test-lab) for how to run such test on Firebase Test Lab.
 - State Management Tests:
   - For testing state using Flutter Driver
     - Run `flutter drive --target=test_driver/<file_path>`

testing_app/android/app/build.gradle

@@ -43,6 +43,7 @@ android {
         targetSdkVersion 28
         versionCode flutterVersionCode.toInteger()
         versionName flutterVersionName
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
     }
 
     buildTypes {
@@ -60,4 +61,10 @@ flutter {
 
 dependencies {
     implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
+
+    testImplementation 'junit:junit:4.12'
+
+    // https://developer.android.com/jetpack/androidx/releases/test/#1.2.0
+    androidTestImplementation 'androidx.test:runner:1.2.0'
+    androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0'
 }

testing_app/android/app/src/androidTest/java/dev/flutter/testing_app/MainActivityTest.java

@@ -0,0 +1,12 @@
+package dev.flutter.testing_app;
+
+import androidx.test.rule.ActivityTestRule;
+import dev.flutter.plugins.e2e.FlutterTestRunner;
+import org.junit.Rule;
+import org.junit.runner.RunWith;
+
+@RunWith(FlutterTestRunner.class)
+public class MainActivityTest {
+  @Rule
+  public ActivityTestRule<MainActivity> rule = new ActivityTestRule<>(MainActivity.class, true, false);
+}

testing_app/lib/screens/home.dart

@@ -29,6 +29,7 @@ class HomePage extends StatelessWidget {
       body: ListView.builder(
         itemCount: 100,
         cacheExtent: 20.0,
+        controller: ScrollController(),
         padding: const EdgeInsets.symmetric(vertical: 16),
         itemBuilder: (context, index) => ItemTile(index),
       ),

testing_app/pubspec.lock

@@ -7,14 +7,14 @@ packages:
       name: _fe_analyzer_shared
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "4.0.0"
+    version: "7.0.0"
   analyzer:
     dependency: transitive
     description:
       name: analyzer
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.39.10"
+    version: "0.39.17"
   archive:
     dependency: transitive
     description:
@@ -50,6 +50,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "1.1.3"
+  cli_util:
+    dependency: transitive
+    description:
+      name: cli_util
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.2.0"
   collection:
     dependency: transitive
     description:
@@ -84,7 +91,7 @@ packages:
       name: csslib
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.16.1"
+    version: "0.16.2"
   cupertino_icons:
     dependency: "direct main"
     description:
@@ -92,6 +99,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "0.1.3"
+  e2e:
+    dependency: "direct dev"
+    description:
+      name: e2e
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "0.7.0"
   file:
     dependency: transitive
     description:
@@ -139,7 +153,7 @@ packages:
       name: http
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.12.1"
+    version: "0.12.2"
   http_multi_server:
     dependency: transitive
     description:
@@ -216,7 +230,7 @@ packages:
       name: mime
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.9.6+3"
+    version: "0.9.7"
   multi_server_socket:
     dependency: transitive
     description:
@@ -307,7 +321,7 @@ packages:
       name: provider
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "4.1.3"
+    version: "4.3.2"
   pub_semver:
     dependency: transitive
     description:
@@ -328,7 +342,7 @@ packages:
       name: shelf
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "0.7.7"
+    version: "0.7.8"
   shelf_packages_handler:
     dependency: transitive
     description:
@@ -452,7 +466,7 @@ packages:
       name: vm_service
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "4.1.0"
+    version: "4.2.0"
   vm_service_client:
     dependency: transitive
     description:
@@ -504,4 +518,4 @@ packages:
     version: "2.2.1"
 sdks:
   dart: ">=2.7.0 <3.0.0"
-  flutter: ">=1.16.0"
+  flutter: ">=1.16.0 <2.0.0"

testing_app/pubspec.yaml

@@ -19,6 +19,7 @@ dev_dependencies:
   flutter_driver:
     sdk: flutter
   test: ^1.14.4
+  e2e: ^0.7.0
   pedantic: ^1.9.0
 
 flutter:

testing_app/test/e2e_utils.dart

@@ -0,0 +1,192 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// TODO(CareF): This file should be removed after the changes goes into flutter
+// stable version.
+
+// ignore linter to keep the code consistent with its duplicate in the framework
+// ignore_for_file: use_function_type_syntax_for_parameters, omit_local_variable_types, avoid_types_on_closure_parameters
+
+import 'dart:async';
+import 'dart:ui';
+
+import 'package:e2e/e2e.dart';
+import 'package:flutter/widgets.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+/// The maximum amount of time considered safe to spend for a frame's build
+/// phase. Anything past that is in the danger of missing the frame as 60FPS.
+///
+/// Changing this doesn't re-evaluate existing summary.
+Duration kBuildBudget = const Duration(milliseconds: 16);
+
+bool _firstRun = true;
+
+const String kDebugWarning = '''
+┏╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┓
+┇ ⚠    THIS BENCHMARK IS BEING RUN IN DEBUG MODE     ⚠  ┇
+┡╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┦
+│                                                       │
+│  Numbers obtained from a benchmark while asserts are  │
+│  enabled will not accurately reflect the performance  │
+│  that will be experienced by end users using release  ╎
+│  builds. Benchmarks should be run using this command  ╎
+│  line:  "flutter run --profile test.dart" or          ┊
+│  or "flutter drive --profile -t test.dart".           ┊
+│                                                       ┊
+└─────────────────────────────────────────────────╌┄┈  🐢
+''';
+
+/// watches the [FrameTiming] of `action` and report it to the e2e binding.
+Future<void> watchPerformance(
+  E2EWidgetsFlutterBinding binding,
+  Future<void> action(), {
+  String reportKey = 'performance',
+}) async {
+  assert(() {
+    if (_firstRun) {
+      debugPrint(kDebugWarning);
+      _firstRun = false;
+    }
+    return true;
+  }());
+  final List<FrameTiming> frameTimings = <FrameTiming>[];
+  final TimingsCallback watcher = frameTimings.addAll;
+  binding.addTimingsCallback(watcher);
+  await action();
+  binding.removeTimingsCallback(watcher);
+  final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings);
+  binding.reportData = <String, dynamic>{reportKey: frameTimes.summary};
+}
+
+/// This class and summarizes a list of [FrameTiming] for the performance
+/// metrics.
+class FrameTimingSummarizer {
+  factory FrameTimingSummarizer(List<FrameTiming> data) {
+    assert(data != null);
+    assert(data.isNotEmpty);
+    final List<Duration> frameBuildTime = List<Duration>.unmodifiable(
+      data.map<Duration>((FrameTiming datum) => datum.buildDuration),
+    );
+    final List<Duration> frameBuildTimeSorted =
+        List<Duration>.from(frameBuildTime)..sort();
+    final List<Duration> frameRasterizerTime = List<Duration>.unmodifiable(
+      data.map<Duration>((FrameTiming datum) => datum.rasterDuration),
+    );
+    final List<Duration> frameRasterizerTimeSorted =
+        List<Duration>.from(frameRasterizerTime)..sort();
+    final Duration Function(Duration, Duration) add =
+        (Duration a, Duration b) => a + b;
+    return FrameTimingSummarizer._(
+      frameBuildTime: frameBuildTime,
+      frameRasterizerTime: frameRasterizerTime,
+      // This avarage calculation is microsecond precision, which is fine
+      // because typical values of these times are milliseconds.
+      averageFrameBuildTime: frameBuildTime.reduce(add) ~/ data.length,
+      p90FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.90),
+      p99FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.99),
+      worstFrameBuildTime: frameBuildTimeSorted.last,
+      missedFrameBuildBudget: _countExceed(frameBuildTimeSorted, kBuildBudget),
+      averageFrameRasterizerTime:
+          frameRasterizerTime.reduce(add) ~/ data.length,
+      p90FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90),
+      p99FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90),
+      worstFrameRasterizerTime: frameRasterizerTimeSorted.last,
+      missedFrameRasterizerBudget:
+          _countExceed(frameRasterizerTimeSorted, kBuildBudget),
+    );
+  }
+
+  const FrameTimingSummarizer._(
+      {@required this.frameBuildTime,
+      @required this.frameRasterizerTime,
+      @required this.averageFrameBuildTime,
+      @required this.p90FrameBuildTime,
+      @required this.p99FrameBuildTime,
+      @required this.worstFrameBuildTime,
+      @required this.missedFrameBuildBudget,
+      @required this.averageFrameRasterizerTime,
+      @required this.p90FrameRasterizerTime,
+      @required this.p99FrameRasterizerTime,
+      @required this.worstFrameRasterizerTime,
+      @required this.missedFrameRasterizerBudget});
+
+  /// List of frame build time in microseconds
+  final List<Duration> frameBuildTime;
+
+  /// List of frame rasterizer time in microseconds
+  final List<Duration> frameRasterizerTime;
+
+  /// The average value of [frameBuildTime] in milliseconds.
+  final Duration averageFrameBuildTime;
+
+  /// The 90-th percentile value of [frameBuildTime] in milliseconds
+  final Duration p90FrameBuildTime;
+
+  /// The 99-th percentile value of [frameBuildTime] in milliseconds
+  final Duration p99FrameBuildTime;
+
+  /// The largest value of [frameBuildTime] in milliseconds
+  final Duration worstFrameBuildTime;
+
+  /// Number of items in [frameBuildTime] that's greater than [kBuildBudget]
+  final int missedFrameBuildBudget;
+
+  /// The average value of [frameRasterizerTime] in milliseconds.
+  final Duration averageFrameRasterizerTime;
+
+  /// The 90-th percentile value of [frameRasterizerTime] in milliseconds.
+  final Duration p90FrameRasterizerTime;
+
+  /// The 99-th percentile value of [frameRasterizerTime] in milliseconds.
+  final Duration p99FrameRasterizerTime;
+
+  /// The largest value of [frameRasterizerTime] in milliseconds.
+  final Duration worstFrameRasterizerTime;
+
+  /// Number of items in [frameRasterizerTime] that's greater than [kBuildBudget]
+  final int missedFrameRasterizerBudget;
+
+  Map<String, dynamic> get summary => <String, dynamic>{
+        'average_frame_build_time_millis':
+            averageFrameBuildTime.inMicroseconds / 1E3,
+        '90th_percentile_frame_build_time_millis':
+            p90FrameBuildTime.inMicroseconds / 1E3,
+        '99th_percentile_frame_build_time_millis':
+            p99FrameBuildTime.inMicroseconds / 1E3,
+        'worst_frame_build_time_millis':
+            worstFrameBuildTime.inMicroseconds / 1E3,
+        'missed_frame_build_budget_count': missedFrameBuildBudget,
+        'average_frame_rasterizer_time_millis':
+            averageFrameRasterizerTime.inMicroseconds / 1E3,
+        '90th_percentile_frame_rasterizer_time_millis':
+            p90FrameRasterizerTime.inMicroseconds / 1E3,
+        '99th_percentile_frame_rasterizer_time_millis':
+            p99FrameRasterizerTime.inMicroseconds / 1E3,
+        'worst_frame_rasterizer_time_millis':
+            worstFrameRasterizerTime.inMicroseconds / 1E3,
+        'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget,
+        'frame_count': frameBuildTime.length,
+        'frame_build_times': frameBuildTime
+            .map<int>((Duration datum) => datum.inMicroseconds)
+            .toList(),
+        'frame_rasterizer_times': frameRasterizerTime
+            .map<int>((Duration datum) => datum.inMicroseconds)
+            .toList(),
+      };
+}
+
+// The following helper functions require data sorted
+
+// return the 100*p-th percentile of the data
+T _findPercentile<T>(List<T> data, double p) {
+  assert(p >= 0 && p <= 1);
+  return data[((data.length - 1) * p).round()];
+}
+
+// return the number of items in data that > threshold
+int _countExceed<T extends Comparable<T>>(List<T> data, T threshold) {
+  return data.length -
+      data.indexWhere((T datum) => datum.compareTo(threshold) > 0);
+}

testing_app/test/perf_test_e2e.dart

@@ -0,0 +1,87 @@
+// Copyright 2020 The Flutter team. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file duplicates the behavior of test_driver/perf_test.dart, but uses
+// the e2e package to implement a host-independent test.
+
+import 'dart:convert' show JsonEncoder;
+
+import 'package:e2e/e2e.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:testing_app/main.dart' as app;
+
+import 'e2e_utils.dart';
+
+void main() {
+  final binding =
+      E2EWidgetsFlutterBinding.ensureInitialized() as E2EWidgetsFlutterBinding;
+  // The fullyLive frame policy simulates the way Flutter response to animations.
+  // See https://github.com/flutter/flutter/issues/60237
+  binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
+
+  group('Testing App Performance Tests on e2e', () {
+    testWidgets('Scrolling test', (tester) async {
+      app.main();
+      await tester.pumpAndSettle();
+      await watchPerformance(binding, () async {
+        final listFinder = find.byType(ListView);
+        final scroller = tester.widget<ListView>(listFinder).controller;
+        await scroller.animateTo(
+          7000,
+          duration: const Duration(seconds: 1),
+          curve: Curves.linear,
+        );
+        await tester.pumpAndSettle();
+
+        await scroller.animateTo(
+          -7000,
+          duration: const Duration(seconds: 1),
+          curve: Curves.linear,
+        );
+        await tester.pumpAndSettle();
+      }, reportKey: 'scrolling');
+      // The performance result is reported to `data['scrolling']`.
+      // See `e2e_test.dart` for detail.
+      print('scrolling performance test result:');
+      print(JsonEncoder.withIndent('  ')
+          .convert(binding.reportData['scrolling']));
+    }, semanticsEnabled: false);
+
+    testWidgets('Favorites operations test', (tester) async {
+      app.main();
+      await tester.pumpAndSettle();
+      await watchPerformance(binding, () async {
+        final iconKeys = [
+          'icon_0',
+          'icon_1',
+          'icon_2',
+        ];
+        for (var icon in iconKeys) {
+          await tester.tap(find.byKey(ValueKey<String>(icon)));
+          await tester.pumpAndSettle();
+        }
+
+        await tester.tap(find.text('Favorites'));
+        await tester.pumpAndSettle();
+
+        final removeIconKeys = [
+          'remove_icon_0',
+          'remove_icon_1',
+          'remove_icon_2',
+        ];
+
+        for (final iconKey in removeIconKeys) {
+          await tester.tap(find.byKey(ValueKey<String>(iconKey)));
+          await tester.pumpAndSettle();
+        }
+      }, reportKey: 'favorites_operations');
+      // The performance result is reported to `data['favorites_operations']`.
+      // See `e2e_test.dart` for detail.
+      print('favorites_operations performance test result:');
+      print(JsonEncoder.withIndent('  ')
+          .convert(binding.reportData['favorites_operations']));
+    }, semanticsEnabled: false);
+  });
+}

testing_app/test_driver/e2e_test.dart

@@ -0,0 +1,21 @@
+// Copyright 2020 The Flutter team. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:e2e/e2e_driver.dart' as driver;
+
+Future<void> main() => driver.e2eDriver(responseDataCallback: (data) async {
+      await driver.writeResponseData(
+        data['scrolling'] as Map<String, dynamic>,
+        // This result is saved to `build/scrolling.json`.
+        testOutputFilename: 'scrolling',
+      );
+
+      await driver.writeResponseData(
+        data['favorites_operations'] as Map<String, dynamic>,
+        // This result is saved to `build/favorites_operations.json`.
+        testOutputFilename: 'favorites_operations',
+      );
+    });

testing_app/test_driver/perf_test.dart

@@ -42,12 +42,12 @@ void main() {
       final scrollingSummary = TimelineSummary.summarize(scrollingTimeline);
 
       // Then, save the summary to disk.
-      // Results will be stored in the file 'build/scrolling.timeline.json'.
+      // Results will be stored in
+      // the file 'build/scrolling.timeline_summary.json'.
       await scrollingSummary.writeSummaryToFile('scrolling', pretty: true);
 
       // Write the entire timeline to disk in a json format.
-      // Results will be stored in
-      // the file 'build/scrolling.timeline_summary.json'.
+      // Results will be stored in the file 'build/scrolling.timeline.json'.
       // This file can be opened in the Chrome browser's tracing tools
       // found by navigating to chrome://tracing.
       await scrollingSummary.writeTimelineToFile('scrolling', pretty: true);

tool/travis_android_script.sh

@@ -77,4 +77,21 @@ gcloud firebase test android run --type instrumentation \
   --timeout 5m
 popd
 
+echo "== Run e2e test for testing_app =="
+pushd "testing_app"
+readonly APP_DIR=$(pwd)
+"${LOCAL_SDK_PATH}/bin/flutter" packages get
+"${LOCAL_SDK_PATH}/bin/flutter" build apk
+pushd "android"
+./gradlew app:assembleAndroidTest
+./gradlew app:assembleRelease -Ptarget=${APP_DIR}/test/perf_test_e2e.dart
+popd
+gcloud auth activate-service-account --key-file=../svc-keyfile.json
+gcloud --quiet config set project test-lab-project-ccbec
+gcloud firebase test android run --type instrumentation \
+  --app build/app/outputs/apk/release/app-release.apk \
+  --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk \
+  --timeout 5m
+popd
+
 echo "-- Success --"