[tools] Check integration tests for test (flutter#5936)

stuartmorgan-g · mvanbeusekom · commit ab2f4b1f0056 · 2022-07-19T15:05:12.000+02:00
diff --git a/packages/path_provider/path_provider/example/integration_test/path_provider_test.dart b/packages/path_provider/path_provider/example/integration_test/path_provider_test.dart
@@ -70,7 +70,8 @@ void main() {
   ];
 
   for (final StorageDirectory? type in _allDirs) {
-    test('getExternalStorageDirectories (type: $type)', () async {
+    testWidgets('getExternalStorageDirectories (type: $type)',
+        (WidgetTester tester) async {
       if (Platform.isIOS) {
         final Future<List<Directory>?> result =
             getExternalStorageDirectories(type: null);
diff --git a/packages/path_provider/path_provider_android/example/integration_test/path_provider_test.dart b/packages/path_provider/path_provider_android/example/integration_test/path_provider_test.dart
@@ -61,7 +61,8 @@ void main() {
   ];
 
   for (final StorageDirectory? type in _allDirs) {
-    test('getExternalStorageDirectories (type: $type)', () async {
+    testWidgets('getExternalStorageDirectories (type: $type)',
+        (WidgetTester tester) async {
       final PathProviderPlatform provider = PathProviderPlatform.instance;
 
       final List<String>? directories =
diff --git a/script/tool/CHANGELOG.md b/script/tool/CHANGELOG.md
@@ -1,6 +1,8 @@
 ## NEXT
 
 - Supports empty custom analysis allow list files.
+- `drive-examples` now validates files to ensure that they don't accidentally
+  use `test(...)`.
 
 ## 0.8.6
 
diff --git a/script/tool/lib/src/drive_examples_command.dart b/script/tool/lib/src/drive_examples_command.dart
@@ -182,7 +182,16 @@ class DriveExamplesCommand extends PackageLoopingCommand {
         if (legacyTestFile != null) {
           testTargets.add(legacyTestFile);
         } else {
-          (await _getIntegrationTests(example)).forEach(testTargets.add);
+          for (final File testFile in await _getIntegrationTests(example)) {
+            // Check files for known problematic patterns.
+            final bool passesValidation = _validateIntegrationTest(testFile);
+            if (!passesValidation) {
+              // Report the issue, but continue with the test as the validation
+              // errors don't prevent running.
+              errors.add('${testFile.basename} failed validation');
+            }
+            testTargets.add(testFile);
+          }
         }
 
         if (testTargets.isEmpty) {
@@ -310,6 +319,25 @@ class DriveExamplesCommand extends PackageLoopingCommand {
     return tests;
   }
 
+  /// Checks [testFile] for known bad patterns in integration tests, logging
+  /// any issues.
+  ///
+  /// Returns true if the file passes validation without issues.
+  bool _validateIntegrationTest(File testFile) {
+    final List<String> lines = testFile.readAsLinesSync();
+
+    final RegExp badTestPattern = RegExp(r'\s*test\(');
+    if (lines.any((String line) => line.startsWith(badTestPattern))) {
+      final String filename = testFile.basename;
+      printError(
+          '$filename uses "test", which will not report failures correctly. '
+          'Use testWidgets instead.');
+      return false;
+    }
+
+    return true;
+  }
+
   /// For each file in [targets], uses
   /// `flutter drive --driver [driver] --target <target>`
   /// to drive [example], returning a list of any failing test targets.
diff --git a/script/tool/test/drive_examples_command_test.dart b/script/tool/test/drive_examples_command_test.dart
@@ -307,6 +307,47 @@ void main() {
       );
     });
 
+    test('integration tests using test(...) fail validation', () async {
+      setMockFlutterDevicesOutput();
+      final RepositoryPackage package = createFakePlugin(
+        'plugin',
+        packagesDir,
+        extraFiles: <String>[
+          'example/test_driver/integration_test.dart',
+          'example/integration_test/foo_test.dart',
+          'example/android/android.java',
+        ],
+        platformSupport: <String, PlatformDetails>{
+          platformAndroid: const PlatformDetails(PlatformSupport.inline),
+          platformIOS: const PlatformDetails(PlatformSupport.inline),
+        },
+      );
+      package.directory
+          .childDirectory('example')
+          .childDirectory('integration_test')
+          .childFile('foo_test.dart')
+          .writeAsStringSync('''
+   test('this is the wrong kind of test!'), () {
+     ...
+   }
+''');
+
+      Error? commandError;
+      final List<String> output = await runCapturingPrint(
+          runner, <String>['drive-examples', '--android'],
+          errorHandler: (Error e) {
+        commandError = e;
+      });
+
+      expect(commandError, isA<ToolExit>());
+      expect(
+        output,
+        containsAllInOrder(<Matcher>[
+          contains('foo_test.dart failed validation'),
+        ]),
+      );
+    });
+
     test(
         'driving under folder "test_driver" when targets are under "integration_test"',
         () async {
