initial implementation of a github actions reporter

pkgs/test/test/runner/github_reporter_test.dart

@@ -0,0 +1,392 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn('vm')
+
+import 'dart:async';
+
+import 'package:test_descriptor/test_descriptor.dart' as d;
+
+import 'package:test/test.dart';
+
+import '../io.dart';
+
+void main() {
+  setUpAll(precompileTestExecutable);
+
+  test('reports when no tests are run', () async {
+    await d.file('test.dart', 'void main() {}').create();
+
+    var test = await runTest(['test.dart'], reporter: 'github');
+    expect(test.stdout, emitsThrough(contains('0 tests passed')));
+    await test.shouldExit(79);
+  });
+
+  test('runs several successful tests and reports when each completes', () {
+    return _expectReport('''
+        test('success 1', () {});
+        test('success 2', () {});
+        test('success 3', () {});''', '''
+        ::group::✅ success 1
+        ::endgroup::
+        ::group::✅ success 2
+        ::endgroup::
+        ::group::✅ success 3
+        ::endgroup::
+        🎉 3 tests passed.''');
+  });
+
+  test('runs several failing tests and reports when each fails', () {
+    return _expectReport('''
+        test('failure 1', () => throw TestFailure('oh no'));
+        test('failure 2', () => throw TestFailure('oh no'));
+        test('failure 3', () => throw TestFailure('oh no'));''', '''
+        ::group::❌ failure 1 (failed)
+        oh no
+        test.dart 6:33  main.<fn>
+        ::endgroup::
+        ::group::❌ failure 2 (failed)
+        oh no
+        test.dart 7:33  main.<fn>
+        ::endgroup::
+        ::group::❌ failure 3 (failed)
+        oh no
+        test.dart 8:33  main.<fn>
+        ::endgroup::
+        ::error::0 tests passed, 3 failed.''');
+  });
+
+  test('includes the full stack trace with --verbose-trace', () async {
+    await d.file('test.dart', '''
+import 'dart:async';
+
+import 'package:test/test.dart';
+
+void main() {
+  test("failure", () => throw "oh no");
+}
+''').create();
+
+    var test =
+        await runTest(['--verbose-trace', 'test.dart'], reporter: 'github');
+    expect(test.stdout, emitsThrough(contains('dart:async')));
+    await test.shouldExit(1);
+  });
+
+  test('runs failing tests along with successful tests', () {
+    return _expectReport('''
+        test('failure 1', () => throw TestFailure('oh no'));
+        test('success 1', () {});
+        test('failure 2', () => throw TestFailure('oh no'));
+        test('success 2', () {});''', '''
+        ::group::❌ failure 1 (failed)
+        oh no
+        test.dart 6:33  main.<fn>
+        ::endgroup::
+        ::group::✅ success 1
+        ::endgroup::
+        ::group::❌ failure 2 (failed)
+        oh no
+        test.dart 8:33  main.<fn>
+        ::endgroup::
+        ::group::✅ success 2
+        ::endgroup::
+        ::error::2 tests passed, 2 failed.''');
+  });
+
+  test('always prints the full test name', () {
+    return _expectReport(
+      '''
+        test(
+           'really gosh dang long test name. Even longer than that. No, yet '
+               'longer. A little more... okay, that should do it.',
+           () {});''',
+      '''
+        ::group::✅ really gosh dang long test name. Even longer than that. No, yet longer. A little more... okay, that should do it.
+        ::endgroup::''',
+      useContains: true,
+    );
+  });
+
+  test('gracefully handles multiple test failures in a row', () {
+    return _expectReport('''
+        // This completer ensures that the test isolate isn't killed until all
+        // errors have been thrown.
+        var completer = Completer();
+        test('failures', () {
+          Future.microtask(() => throw 'first error');
+          Future.microtask(() => throw 'second error');
+          Future.microtask(() => throw 'third error');
+          Future.microtask(completer.complete);
+        });
+        test('wait', () => completer.future);''', '''
+        ::group::❌ failures (failed)
+        first error
+        test.dart 10:34  main.<fn>.<fn>
+        second error
+        test.dart 11:34  main.<fn>.<fn>
+        third error
+        test.dart 12:34  main.<fn>.<fn>
+        ::endgroup::
+        ::group::✅ wait
+        ::endgroup::
+        ::error::1 test passed, 1 failed.''');
+  });
+
+  group('print:', () {
+    test('handles multiple prints', () {
+      return _expectReport(
+        '''
+        test('test', () {
+          print("one");
+          print("two");
+          print("three");
+          print("four");
+        });''',
+        '''
+        ::group::✅ test
+        one
+        two
+        three
+        four
+        ::endgroup::''',
+        useContains: true,
+      );
+    });
+
+    test('handles a print after the test completes', () {
+      return _expectReport('''
+        // This completer ensures that the test isolate isn't killed until all
+        // prints have happened.
+        var testDone = Completer();
+        var waitStarted = Completer();
+        test('test', () async {
+          waitStarted.future.then((_) {
+            Future(() => print("one"));
+            Future(() => print("two"));
+            Future(() => print("three"));
+            Future(() => print("four"));
+            Future(testDone.complete);
+          });
+        });
+
+        test('wait', () {
+          waitStarted.complete();
+          return testDone.future;
+        });''', '''
+        ::group::✅ test
+        ::endgroup::
+        one
+        two
+        three
+        four
+        ::group::✅ wait
+        ::endgroup::
+        🎉 2 tests passed.''');
+    });
+
+    // todo: fix this
+    //   test('interleaves prints and errors', () {
+    //     return _expectReport('''
+    //       // This completer ensures that the test isolate isn't killed until all
+    //       // prints have happened.
+    //       var completer = Completer();
+    //       test('test', () {
+    //         scheduleMicrotask(() {
+    //           print("three");
+    //           print("four");
+    //           throw "second error";
+    //         });
+
+    //         scheduleMicrotask(() {
+    //           print("five");
+    //           print("six");
+    //           completer.complete();
+    //         });
+
+    //         print("one");
+    //         print("two");
+    //         throw "first error";
+    //       });
+
+    //       test('wait', () => completer.future);''', '''
+    //       +0: test
+    //       one
+    //       two
+    //       +0 -1: test [E]
+    //         first error
+    //         test.dart 24:11  main.<fn>
+
+    //       three
+    //       four
+    //         second error
+    //         test.dart 13:13  main.<fn>.<fn>
+    //         ===== asynchronous gap ===========================
+    //         dart:async       scheduleMicrotask
+    //         test.dart 10:11  main.<fn>
+
+    //       five
+    //       six
+    //       +0 -1: wait
+    //       +1 -1: Some tests failed.''');
+    //   });
+  });
+
+  group('skip:', () {
+    test('displays skipped tests', () {
+      return _expectReport('''
+          test('skip 1', () {}, skip: true);
+          test('skip 2', () {}, skip: true);
+          test('skip 3', () {}, skip: true);''', '''
+          ::group::❎ skip 1 (skipped)
+          ::endgroup::
+          ::group::❎ skip 2 (skipped)
+          ::endgroup::
+          ::group::❎ skip 3 (skipped)
+          ::endgroup::
+          🎉 0 tests passed, 3 skipped.''');
+    });
+
+    test('displays a skipped group', () {
+      return _expectReport('''
+          group('skip', () {
+            test('test 1', () {});
+            test('test 2', () {});
+            test('test 3', () {});
+          }, skip: true);''', '''
+          ::group::❎ skip test 1 (skipped)
+          ::endgroup::
+          ::group::❎ skip test 2 (skipped)
+          ::endgroup::
+          ::group::❎ skip test 3 (skipped)
+          ::endgroup::
+          🎉 0 tests passed, 3 skipped.''');
+    });
+
+    test('runs skipped tests along with successful tests', () {
+      return _expectReport('''
+          test('skip 1', () {}, skip: true);
+          test('success 1', () {});
+          test('skip 2', () {}, skip: true);
+          test('success 2', () {});''', '''
+          ::group::❎ skip 1 (skipped)
+          ::endgroup::
+          ::group::✅ success 1
+          ::endgroup::
+          ::group::❎ skip 2 (skipped)
+          ::endgroup::
+          ::group::✅ success 2
+          ::endgroup::
+          🎉 2 tests passed, 2 skipped.''');
+    });
+
+    test('runs skipped tests along with successful and failing tests', () {
+      return _expectReport('''
+          test('failure 1', () => throw TestFailure('oh no'));
+          test('skip 1', () {}, skip: true);
+          test('success 1', () {});
+          test('failure 2', () => throw TestFailure('oh no'));
+          test('skip 2', () {}, skip: true);
+          test('success 2', () {});''', '''
+          ::group::❌ failure 1 (failed)
+          oh no
+          test.dart 6:35  main.<fn>
+          ::endgroup::
+          ::group::❎ skip 1 (skipped)
+          ::endgroup::
+          ::group::✅ success 1
+          ::endgroup::
+          ::group::❌ failure 2 (failed)
+          oh no
+          test.dart 9:35  main.<fn>
+          ::endgroup::
+          ::group::❎ skip 2 (skipped)
+          ::endgroup::
+          ::group::✅ success 2
+          ::endgroup::
+          ::error::2 tests passed, 2 failed, 2 skipped.''');
+    });
+
+    test('displays the skip reason if available', () {
+      return _expectReport('''
+          test('skip 1', () {}, skip: 'some reason');
+          test('skip 2', () {}, skip: 'or another');''', '''
+          ::group::❎ skip 1 (skipped)
+          Skip: some reason
+          ::endgroup::
+          ::group::❎ skip 2 (skipped)
+          Skip: or another
+          ::endgroup::
+          🎉 0 tests passed, 2 skipped.''');
+    });
+  });
+
+  test('loadSuite, setUpAll, and tearDownAll hidden if no content', () {
+    return _expectReport('''
+          group('one', () {
+            setUpAll(() {/* nothing to do here */});
+            tearDownAll(() {/* nothing to do here */});
+            test('test 1', () {});
+          });''', '''
+          ::group::✅ one test 1
+          ::endgroup::
+          🎉 1 test passed.''');
+  });
+
+  test('setUpAll and tearDownAll show when they have content', () {
+    return _expectReport('''
+          group('one', () {
+            setUpAll(() { print('one'); });
+            tearDownAll(() { print('two'); });
+            test('test 1', () {});
+          });''', '''
+          ::group::⏺ one (setUpAll)
+          one
+          ::endgroup::
+          ::group::✅ one test 1
+          ::endgroup::
+          ::group::⏺ one (tearDownAll)
+          two
+          ::endgroup::
+          🎉 1 test passed.''');
+  });
+}
+
+Future<void> _expectReport(
+  String tests,
+  String expected, {
+  List<String> args = const [],
+  bool useContains = false,
+}) async {
+  await d.file('test.dart', '''
+    import 'dart:async';
+
+    import 'package:test/test.dart';
+
+    void main() {
+$tests
+    }
+  ''').create();
+
+  var test = await runTest([
+    'test.dart',
+    ...args,
+  ], reporter: 'github');
+  await test.shouldExit();
+
+  var stdoutLines = await test.stdoutStream().toList();
+  var actual = stdoutLines
+      .map((line) => line.trim())
+      .where((line) => line.isNotEmpty)
+      .join('\n');
+
+  // Un-indent the expected string.
+  var indentation = expected.indexOf(RegExp('[^ ]'));
+  expected = expected.split('\n').map((line) {
+    if (line.isEmpty) return line;
+    return line.substring(indentation);
+  }).join('\n');
+
+  expect(actual, useContains ? contains(expected) : equals(expected));
+}