Add --coverage-package option (#2566)

Author: liamappelbe

pkgs/test/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 1.29.0-wip
+
+* Add `--coverage-package` flag, which filters the coverage report to specific
+  packages using RegExps.
+
 ## 1.28.0
 
 * Add `isSorted` and related matchers for iterables.

pkgs/test/pubspec.yaml

@@ -1,5 +1,5 @@
 name: test
-version: 1.28.0
+version: 1.29.0-wip
 description: >-
   A full featured library for writing and running Dart tests across platforms.
 repository: https://github.com/dart-lang/test/tree/master/pkgs/test
@@ -36,7 +36,7 @@ dependencies:
 
   # Use an exact version until the test_api and test_core package are stable.
   test_api: 0.7.8
-  test_core: 0.6.14
+  test_core: 0.6.15-wip
 
   typed_data: ^1.3.0
   web_socket_channel: '>=2.0.0 <4.0.0'

pkgs/test/test/io.dart

@@ -100,6 +100,7 @@ Future<TestProcess> runTest(
   bool forwardStdio = false,
   String? packageConfig,
   Iterable<String>? vmArgs,
+  String? workingDirectory,
 }) async {
   concurrency ??= 1;
   var testExecutablePath = _testExecutablePath;
@@ -127,6 +128,7 @@ Future<TestProcess> runTest(
     description: 'dart bin/test.dart',
     forwardStdio: forwardStdio,
     packageConfig: packageConfig,
+    workingDirectory: workingDirectory,
   );
 }
 
@@ -140,6 +142,7 @@ Future<TestProcess> runDart(
   String? description,
   bool forwardStdio = false,
   String? packageConfig,
+  String? workingDirectory,
 }) async {
   var allArgs = <String>[
     '--packages=${packageConfig ?? await Isolate.packageConfig}',
@@ -149,7 +152,7 @@ Future<TestProcess> runDart(
   return await TestProcess.start(
     p.absolute(Platform.resolvedExecutable),
     allArgs,
-    workingDirectory: d.sandbox,
+    workingDirectory: workingDirectory ?? d.sandbox,
     environment: environment,
     description: description,
     forwardStdio: forwardStdio,
@@ -160,11 +163,12 @@ Future<TestProcess> runDart(
 Future<TestProcess> runPub(
   Iterable<String> args, {
   Map<String, String>? environment,
+  String? workingDirectory,
 }) {
   return TestProcess.start(
     p.absolute(Platform.resolvedExecutable),
     ['pub', ...args],
-    workingDirectory: d.sandbox,
+    workingDirectory: workingDirectory ?? d.sandbox,
     environment: environment,
     description: 'pub ${args.first}',
   );

pkgs/test/test/runner/coverage_test.dart

@@ -21,7 +21,7 @@ void main() {
 
   group('with the --coverage flag,', () {
     late Directory coverageDirectory;
-    late d.DirectoryDescriptor packageDirectory;
+    late String pkgDir;
 
     Future<void> validateTest(TestProcess test) async {
       expect(test.stdout, emitsThrough(contains('+1: All tests passed!')));
@@ -47,9 +47,10 @@ void main() {
         'test_coverage',
       );
 
-      packageDirectory = d.dir(d.sandbox, [
-        d.dir('lib', [
-          d.file('calculate.dart', '''
+      final outerDirectory = d.dir(d.sandbox, [
+        d.dir('fake_package', [
+          d.dir('lib', [
+            d.file('calculate.dart', '''
             int calculate(int x) {
               if (x % 2 == 0) {
                 return x * 2;
@@ -58,9 +59,9 @@ void main() {
               }
             }
           '''),
-        ]),
-        d.dir('test', [
-          d.file('test.dart', '''
+          ]),
+          d.dir('test', [
+            d.file('test.dart', '''
             import 'package:fake_package/calculate.dart';
             import 'package:test/test.dart';
 
@@ -70,30 +71,66 @@ void main() {
               });
             }
           '''),
-        ]),
-        d.file('pubspec.yaml', '''
+          ]),
+          d.file('pubspec.yaml', '''
 name: fake_package
 version: 1.0.0
 environment:
   sdk: ^3.5.0
 dev_dependencies:
   test: ^1.26.2
         '''),
+        ]),
+        d.dir('fake_client', [
+          d.dir('lib', [
+            d.file('calculate.dart', '''
+            import 'package:fake_package/calculate.dart' as fake_package;
+
+            int calculate(int x) {
+              return fake_package.calculate(x);
+            }
+          '''),
+          ]),
+          d.dir('test', [
+            d.file('test.dart', '''
+            import 'package:fake_client/calculate.dart';
+            import 'package:test/test.dart';
+
+            void main() {
+              test('test 1', () {
+                expect(calculate(6), 12);
+              });
+            }
+          '''),
+          ]),
+          d.file('pubspec.yaml', '''
+name: fake_client
+version: 1.0.0
+environment:
+  sdk: ^3.5.0
+dependencies:
+  fake_package:
+    path: ../fake_package
+dev_dependencies:
+  test: ^1.26.2
+        '''),
+        ]),
       ]);
-      await packageDirectory.create();
+      await outerDirectory.create();
+      pkgDir = p.join(d.sandbox, 'fake_package');
     });
 
     tearDown(() async {
       await coverageDirectory.delete(recursive: true);
     });
 
     test('gathers coverage for VM tests', () async {
-      await (await runPub(['get'])).shouldExit(0);
-      var test = await runTest([
-        '--coverage',
-        coverageDirectory.path,
-        'test/test.dart',
-      ], packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'));
+      await (await runPub(['get'], workingDirectory: pkgDir)).shouldExit(0);
+      var test = await runTest(
+        ['--coverage', coverageDirectory.path, 'test/test.dart'],
+        packageConfig: p.join(pkgDir, '.dart_tool/package_config.json'),
+        workingDirectory: pkgDir,
+      );
       final coverage = await validateCoverage(test, 'test/test.dart.vm.json');
       final hitmap = coverage['package:fake_package/calculate.dart']!;
       expect(hitmap.lineHits, {1: 1, 2: 2, 3: 1, 5: 0});
@@ -102,7 +139,7 @@ dev_dependencies:
     });
 
     test('gathers branch coverage for VM tests', () async {
-      await (await runPub(['get'])).shouldExit(0);
+      await (await runPub(['get'], workingDirectory: pkgDir)).shouldExit(0);
       var test = await runTest(
         [
           '--coverage',
@@ -111,7 +148,8 @@ dev_dependencies:
           'test/test.dart',
         ],
         vmArgs: ['--branch-coverage'],
-        packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'),
+        packageConfig: p.join(pkgDir, '.dart_tool/package_config.json'),
+        workingDirectory: pkgDir,
       );
       final coverage = await validateCoverage(test, 'test/test.dart.vm.json');
       final hitmap = coverage['package:fake_package/calculate.dart']!;
@@ -121,16 +159,51 @@ dev_dependencies:
     });
 
     test('gathers lcov coverage for VM tests', () async {
-      await (await runPub(['get'])).shouldExit(0);
+      await (await runPub(['get'], workingDirectory: pkgDir)).shouldExit(0);
       final lcovFile = p.join(coverageDirectory.path, 'lcov.info');
-      var test = await runTest([
-        '--coverage-path',
-        lcovFile,
-        'test/test.dart',
-      ], packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'));
+      var test = await runTest(
+        ['--coverage-path', lcovFile, 'test/test.dart'],
+        packageConfig: p.join(pkgDir, '.dart_tool/package_config.json'),
+        workingDirectory: pkgDir,
+      );
       await validateTest(test);
       expect(File(lcovFile).readAsStringSync(), '''
-SF:${p.join(d.sandbox, 'lib', 'calculate.dart')}
+SF:${p.join(pkgDir, 'lib', 'calculate.dart')}
+DA:1,1
+DA:2,2
+DA:3,1
+DA:5,0
+LF:4
+LH:3
+end_of_record
+''');
+    });
+
+    test('gathers coverage for tests in multiple pacakges', () async {
+      final clientPkgDir = p.join(d.sandbox, 'fake_client');
+      await (await runPub([
+        'get',
+      ], workingDirectory: clientPkgDir)).shouldExit(0);
+      final lcovFile = p.join(coverageDirectory.path, 'lcov.info');
+      var test = await runTest(
+        [
+          '--coverage-path',
+          lcovFile,
+          '--coverage-package=fake_.*',
+          'test/test.dart',
+        ],
+        packageConfig: p.join(clientPkgDir, '.dart_tool/package_config.json'),
+        workingDirectory: clientPkgDir,
+      );
+      await validateTest(test);
+      expect(File(lcovFile).readAsStringSync(), '''
+SF:${p.join(clientPkgDir, 'lib', 'calculate.dart')}
+DA:3,1
+DA:4,1
+LF:2
+LH:2
+end_of_record
+SF:${p.join(pkgDir, 'lib', 'calculate.dart')}
 DA:1,1
 DA:2,2
 DA:3,1
@@ -142,14 +215,18 @@ end_of_record
     });
 
     test('gathers coverage for Chrome tests', () async {
-      await (await runPub(['get'])).shouldExit(0);
-      var test = await runTest([
-        '--coverage',
-        coverageDirectory.path,
-        'test/test.dart',
-        '-p',
-        'chrome',
-      ], packageConfig: p.join(d.sandbox, '.dart_tool/package_config.json'));
+      await (await runPub(['get'], workingDirectory: pkgDir)).shouldExit(0);
+      var test = await runTest(
+        [
+          '--coverage',
+          coverageDirectory.path,
+          'test/test.dart',
+          '-p',
+          'chrome',
+        ],
+        packageConfig: p.join(pkgDir, '.dart_tool/package_config.json'),
+        workingDirectory: pkgDir,
+      );
       await validateCoverage(test, 'test/test.dart.chrome.json');
     });
 

pkgs/test/test/runner/runner_test.dart

@@ -96,6 +96,9 @@ $_runtimeCompilers
                                       Implies --debug.
     --branch-coverage                 Include branch coverage information in the coverage report.
                                       Must be paired with --coverage or --coverage-path.
+    --coverage-package=<regexp>       A regular expression matching packages names to include in
+                                      the coverage report (if coverage is enabled). If unset,
+                                      matches the current package name.
     --[no-]chain-stack-traces         Use chained stack traces to provide greater exception details
                                       especially for asynchronous code. It may be useful to disable
                                       to provide improved test performance but at the cost of

pkgs/test/test/utils.dart

@@ -226,6 +226,7 @@ Configuration configuration({
   String? coverage,
   String? coverageLcov,
   bool? branchCoverage,
+  List<RegExp>? coveragePackages,
   int? concurrency,
   int? shardIndex,
   int? totalShards,
@@ -278,6 +279,7 @@ Configuration configuration({
   coverage: coverage,
   coverageLcov: coverageLcov,
   branchCoverage: branchCoverage,
+  coveragePackages: coveragePackages,
   concurrency: concurrency,
   shardIndex: shardIndex,
   totalShards: totalShards,

pkgs/test_core/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6.15-wip
+
+* Add `--coverage-package` flag, which filters the coverage report to specific
+  packages using RegExps.
+
 ## 0.6.14
 
 * Fix type cast when parsing a `null` hit map.