diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 2080d466..e87feff8 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -251,6 +251,7 @@ jobs: -e INPUT_SECONDS_BETWEEN_GITHUB_READS \ -e INPUT_SECONDS_BETWEEN_GITHUB_WRITES \ -e INPUT_JSON_THOUSANDS_SEPARATOR \ + -e INPUT_JSON_TEST_CASE_RESULTS \ -e HOME \ -e GITHUB_JOB \ -e GITHUB_REF \ diff --git a/README.md b/README.md index ced5d9bf..0a3e35ad 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,7 @@ The list of most notable options: |`check_run_annotations_branch`|`event.repository.default_branch` or `"main, master"`|Adds check run annotations only on given branches. If not given, this defaults to the default branch of your repository, e.g. `main` or `master`. Comma separated list of branch names allowed, asterisk `"*"` matches all branches. Example: `main, master, branch_one`.| |`json_file`|no file|Results are written to this JSON file.| |`json_thousands_separator`|`" "`|Formatted numbers in JSON use this character to separate groups of thousands. Common values are "," or ".". Defaults to punctuation space (\u2008).| +|`json_test_case_results`|`false`|Write out all individual test case results to the json output file. Setting this to true can greatly increase the size of the output. Defaults to false.| |`fail_on`|`"test failures"`|Configures the state of the created test result check run. With `"test failures"` it fails if any test fails or test errors occur. It never fails when set to `"nothing"`, and fails only on errors when set to `"errors"`.| Pull request comments highlight removal of tests or tests that the pull request moves into skip state. @@ -391,6 +392,9 @@ Compared to `"Access JSON via step outputs"` above, `errors` and `annotations` c ] } ``` + +Additionally, `json_test_case_results` can be enabled to write out the individual test case results into the JSON file. Enabling this may greatly increase the output size of the JSON file. + See [Create a badge from test results](#create-a-badge-from-test-results) for an example on how to create a badge from this JSON. diff --git a/action.yml b/action.yml index b9af055c..786dc912 100644 --- a/action.yml +++ b/action.yml @@ -101,6 +101,10 @@ inputs: description: 'Formatted numbers in JSON use this character to separate groups of thousands. Common values are "," or ".". Defaults to punctuation space (\u2008).' default: ' ' required: false + json_test_case_results: + description: 'Write out all individual test case results to JSON file. This may greatly increase the size of the output' + default: false + required: false outputs: json: diff --git a/composite/action.yml b/composite/action.yml index 9211f972..89060232 100644 --- a/composite/action.yml +++ b/composite/action.yml @@ -101,7 +101,10 @@ inputs: description: 'Formatted numbers in JSON use this character to separate groups of thousands. Common values are "," or ".". Defaults to punctuation space (\u2008).' default: ' ' required: false - + json_test_case_results: + description: 'Write out all individual test case results to JSON file. This may greatly increase the size of the output' + default: false + required: false outputs: json: description: "Test results as JSON" @@ -177,6 +180,7 @@ runs: SECONDS_BETWEEN_GITHUB_WRITES: ${{ inputs.seconds_between_github_writes }} JSON_FILE: ${{ inputs.json_file }} JSON_THOUSANDS_SEPARATOR: ${{ inputs.json_thousands_separator }} + JSON_TEST_CASE_RESULTS: ${{ inputs.json_test_case_results }} JOB_SUMMARY: ${{ inputs.job_summary }} # not documented ROOT_LOG_LEVEL: ${{ inputs.root_log_level }} diff --git a/python/publish/publisher.py b/python/publish/publisher.py index 08e1c730..42da94dc 100644 --- a/python/publish/publisher.py +++ b/python/publish/publisher.py @@ -40,6 +40,7 @@ class Settings: commit: str json_file: Optional[str] json_thousands_separator: str + json_test_case_results: bool fail_on_errors: bool fail_on_failures: bool # one of these *_files_glob must be set @@ -72,6 +73,7 @@ class PublishData: stats_with_delta: Optional[UnitTestRunDeltaResults] annotations: List[Annotation] check_url: str + cases: Optional[UnitTestCaseResults] @classmethod def _format_digit(cls, value: Union[int, Mapping[str, int], Any], thousands_separator: str) -> Union[str, Mapping[str, str], Any]: @@ -347,7 +349,8 @@ def publish_check(self, stats=stats, stats_with_delta=stats_with_delta if before_stats is not None else None, annotations=all_annotations, - check_url=check_run.html_url + check_url=check_run.html_url, + cases=cases if self._settings.json_test_case_results else None ) self.publish_json(data) diff --git a/python/publish_test_results.py b/python/publish_test_results.py index 15901d71..7fe79dd3 100644 --- a/python/publish_test_results.py +++ b/python/publish_test_results.py @@ -370,6 +370,7 @@ def get_settings(options: dict, gha: Optional[GithubAction] = None) -> Settings: commit=get_var('COMMIT', options) or get_commit_sha(event, event_name, options), json_file=get_var('JSON_FILE', options), json_thousands_separator=get_var('JSON_THOUSANDS_SEPARATOR', options) or punctuation_space, + json_test_case_results=get_bool_var('JSON_TEST_CASE_RESULTS', options, default=False), fail_on_errors=fail_on_errors, fail_on_failures=fail_on_failures, junit_files_glob=get_var('JUNIT_FILES', options) or default_junit_files_glob, diff --git a/python/test/test_action_script.py b/python/test/test_action_script.py index 13eaef96..96e38fb8 100644 --- a/python/test/test_action_script.py +++ b/python/test/test_action_script.py @@ -178,7 +178,8 @@ def get_settings(token='token', seconds_between_github_reads=1.5, seconds_between_github_writes=2.5, json_file=None, - json_thousands_separator=punctuation_space) -> Settings: + json_thousands_separator=punctuation_space, + json_test_case_results=False) -> Settings: return Settings( token=token, api_url=api_url, @@ -191,6 +192,7 @@ def get_settings(token='token', commit=commit, json_file=json_file, json_thousands_separator=json_thousands_separator, + json_test_case_results=json_test_case_results, fail_on_errors=fail_on_errors, fail_on_failures=fail_on_failures, junit_files_glob=junit_files_glob, @@ -822,7 +824,6 @@ def test_parse_files(self): self.assertTrue(any([call.args[0].startswith('reading TRX files [') for call in l.debug.call_args_list])) self.assertEqual([], gha.method_calls) - self.assertEqual(67, actual.files) if Version(sys.version.split(' ')[0]) >= Version('3.10.0') and sys.platform.startswith('darwin'): # on macOS and Python 3.10 we see one particular error diff --git a/python/test/test_publisher.py b/python/test/test_publisher.py index b21a5450..880a4a7e 100644 --- a/python/test/test_publisher.py +++ b/python/test/test_publisher.py @@ -87,6 +87,7 @@ def create_settings(comment_mode=comment_mode_always, event_name: str = 'event name', json_file: Optional[str] = None, json_thousands_separator: str = punctuation_space, + json_test_case_results: Optional[bool] = False, pull_request_build: str = pull_request_build_mode_merge, test_changes_limit: Optional[int] = 5): return Settings( @@ -101,6 +102,7 @@ def create_settings(comment_mode=comment_mode_always, commit='commit', json_file=json_file, json_thousands_separator=json_thousands_separator, + json_test_case_results=json_test_case_results, fail_on_errors=True, fail_on_failures=True, junit_files_glob='*.xml', @@ -1496,7 +1498,71 @@ def test_publish_check_with_multiple_annotation_pages(self): title=f'Error processing result file', raw_details='file' )], - check_url='http://check-run.url' + check_url='http://check-run.url', + cases=None + ) + + publish_data_with_cases = PublishData( + title='title', + summary='summary', + conclusion='conclusion', + stats=UnitTestRunResults( + files=12345, + errors=[ParseError('file', 'message', 1, 2, exception=ValueError("Invalid value"))], + suites=2, + duration=3456, + tests=4, tests_succ=5, tests_skip=6, tests_fail=7, tests_error=8901, + runs=9, runs_succ=10, runs_skip=11, runs_fail=12, runs_error=1345, + commit='commit' + ), + stats_with_delta=UnitTestRunDeltaResults( + files={'number': 1234, 'delta': -1234}, + errors=[ + ParseError('file', 'message', 1, 2, exception=ValueError("Invalid value")), + ParseError('file2', 'message2', 2, 4) + ], + suites={'number': 2, 'delta': -2}, + duration={'number': 3456, 'delta': -3456}, + tests={'number': 4, 'delta': -4}, tests_succ={'number': 5, 'delta': -5}, + tests_skip={'number': 6, 'delta': -6}, tests_fail={'number': 7, 'delta': -7}, + tests_error={'number': 8, 'delta': -8}, + runs={'number': 9, 'delta': -9}, runs_succ={'number': 10, 'delta': -10}, + runs_skip={'number': 11, 'delta': -11}, runs_fail={'number': 12, 'delta': -12}, + runs_error={'number': 1345, 'delta': -1345}, + commit='commit', + reference_type='type', reference_commit='ref' + ), + annotations=[Annotation( + path='path', + start_line=1, + end_line=2, + start_column=3, + end_column=4, + annotation_level='failure', + message='message', + title=f'Error processing result file', + raw_details='file' + )], + check_url='http://check-run.url', + cases= {"-keys": { + "-6689976583278291210": [None, "test.classpath.classname", "casename"] + }, + "-6689976583278291210": { + "success": { + "class_name": "test.classpath.classname", + "content": None, + "line": None, + "message": None, + "result": "success", + "result_file": "/path/to/test/test.classpath.classname", + "stderr": None, + "stdout": None, + "test_file": None, + "test_name": "casename", + "time": 0.1 + } + } + } ) def test_publish_data(self): @@ -1680,16 +1746,112 @@ def test_publish_json(self): with self.subTest(json_thousands_separator=separator): with tempfile.TemporaryDirectory() as path: filepath = os.path.join(path, 'file.json') - settings = self.create_settings(json_file=filepath, json_thousands_separator=separator) + settings = self.create_settings(json_file=filepath, json_thousands_separator=separator, json_test_case_results=False) gh, gha, req, repo, commit = self.create_mocks(digest=self.base_digest, check_names=[settings.check_name]) publisher = Publisher(settings, gh, gha) + publisher.publish_json(self.publish_data_with_cases) + gha.error.assert_not_called() + + # assert the file + with open(filepath, encoding='utf-8') as r: + actual = r.read() + self.maxDiff = None + self.assertEqual( + '{' + '"title": "title", ' + '"summary": "summary", ' + '"conclusion": "conclusion", ' + '"stats": {"files": 12345, "errors": [{"file": "file", "message": "message", "line": 1, "column": 2}], "suites": 2, "duration": 3456, "tests": 4, "tests_succ": 5, "tests_skip": 6, "tests_fail": 7, "tests_error": 8901, "runs": 9, "runs_succ": 10, "runs_skip": 11, "runs_fail": 12, "runs_error": 1345, "commit": "commit"}, ' + '"stats_with_delta": {"files": {"number": 1234, "delta": -1234}, "errors": [{"file": "file", "message": "message", "line": 1, "column": 2}, {"file": "file2", "message": "message2", "line": 2, "column": 4}], "suites": {"number": 2, "delta": -2}, "duration": {"number": 3456, "delta": -3456}, "tests": {"number": 4, "delta": -4}, "tests_succ": {"number": 5, "delta": -5}, "tests_skip": {"number": 6, "delta": -6}, "tests_fail": {"number": 7, "delta": -7}, "tests_error": {"number": 8, "delta": -8}, "runs": {"number": 9, "delta": -9}, "runs_succ": {"number": 10, "delta": -10}, "runs_skip": {"number": 11, "delta": -11}, "runs_fail": {"number": 12, "delta": -12}, "runs_error": {"number": 1345, "delta": -1345}, "commit": "commit", "reference_type": "type", "reference_commit": "ref"}, ' + '"annotations": [{"path": "path", "start_line": 1, "end_line": 2, "start_column": 3, "end_column": 4, "annotation_level": "failure", "message": "message", "title": "Error processing result file", "raw_details": "file"}], ' + '"check_url": "http://check-run.url", ' + '"cases": {"-keys": {"-6689976583278291210": [null, "test.classpath.classname", "casename"]}, "-6689976583278291210": {"success": {"class_name": "test.classpath.classname", "content": null, "line": null, "message": null, "result": "success", "result_file": "/path/to/test/test.classpath.classname", "stderr": null, "stdout": null, "test_file": null, "test_name": "casename", "time": 0.1}}}, ' + '"formatted": {' + '"stats": {"files": "12' + separator + '345", "errors": [{"file": "file", "message": "message", "line": 1, "column": 2}], "suites": "2", "duration": "3' + separator + '456", "tests": "4", "tests_succ": "5", "tests_skip": "6", "tests_fail": "7", "tests_error": "8' + separator + '901", "runs": "9", "runs_succ": "10", "runs_skip": "11", "runs_fail": "12", "runs_error": "1' + separator + '345", "commit": "commit"}, ' + '"stats_with_delta": {"files": {"number": "1' + separator + '234", "delta": "-1' + separator + '234"}, "errors": [{"file": "file", "message": "message", "line": 1, "column": 2}, {"file": "file2", "message": "message2", "line": 2, "column": 4}], "suites": {"number": "2", "delta": "-2"}, "duration": {"number": "3' + separator + '456", "delta": "-3' + separator + '456"}, "tests": {"number": "4", "delta": "-4"}, "tests_succ": {"number": "5", "delta": "-5"}, "tests_skip": {"number": "6", "delta": "-6"}, "tests_fail": {"number": "7", "delta": "-7"}, "tests_error": {"number": "8", "delta": "-8"}, "runs": {"number": "9", "delta": "-9"}, "runs_succ": {"number": "10", "delta": "-10"}, "runs_skip": {"number": "11", "delta": "-11"}, "runs_fail": {"number": "12", "delta": "-12"}, "runs_error": {"number": "1' + separator + '345", "delta": "-1' + separator + '345"}, "commit": "commit", "reference_type": "type", "reference_commit": "ref"}' + '}' + '}', + actual + ) + + # data is being sent to GH action output 'json' + # some list fields are replaced by their length + expected = { + "title": "title", + "summary": "summary", + "conclusion": "conclusion", + "stats": {"files": 12345, "errors": 1, "suites": 2, "duration": 3456, "tests": 4, "tests_succ": 5, + "tests_skip": 6, "tests_fail": 7, "tests_error": 8901, "runs": 9, "runs_succ": 10, + "runs_skip": 11, "runs_fail": 12, "runs_error": 1345, "commit": "commit"}, + "stats_with_delta": {"files": {"number": 1234, "delta": -1234}, "errors": 2, + "suites": {"number": 2, "delta": -2}, "duration": {"number": 3456, "delta": -3456}, + "tests": {"number": 4, "delta": -4}, "tests_succ": {"number": 5, "delta": -5}, + "tests_skip": {"number": 6, "delta": -6}, "tests_fail": {"number": 7, "delta": -7}, + "tests_error": {"number": 8, "delta": -8}, "runs": {"number": 9, "delta": -9}, + "runs_succ": {"number": 10, "delta": -10}, + "runs_skip": {"number": 11, "delta": -11}, + "runs_fail": {"number": 12, "delta": -12}, + "runs_error": {"number": 1345, "delta": -1345}, "commit": "commit", + "reference_type": "type", "reference_commit": "ref"}, + "annotations": 1, + "check_url": "http://check-run.url", + "cases": {"-keys": { + "-6689976583278291210": [None, "test.classpath.classname", "casename"] + }, + "-6689976583278291210": { + "success": { + "class_name": "test.classpath.classname", + "content": None, + "line": None, + "message": None, + "result": "success", + "result_file": "/path/to/test/test.classpath.classname", + "stderr": None, + "stdout": None, + "test_file": None, + "test_name": "casename", + "time": 0.1 + } + } + }, + "formatted": { + "stats": {"files": "12" + separator + "345", "errors": "1", "suites": "2", "duration": "3" + separator + "456", "tests": "4", "tests_succ": "5", + "tests_skip": "6", "tests_fail": "7", "tests_error": "8" + separator + "901", "runs": "9", "runs_succ": "10", + "runs_skip": "11", "runs_fail": "12", "runs_error": "1" + separator + "345", "commit": "commit"}, + "stats_with_delta": {"files": {"number": "1" + separator + "234", "delta": "-1" + separator + "234"}, "errors": "2", + "suites": {"number": "2", "delta": "-2"}, "duration": {"number": "3" + separator + "456", "delta": "-3" + separator + "456"}, + "tests": {"number": "4", "delta": "-4"}, "tests_succ": {"number": "5", "delta": "-5"}, + "tests_skip": {"number": "6", "delta": "-6"}, "tests_fail": {"number": "7", "delta": "-7"}, + "tests_error": {"number": "8", "delta": "-8"}, "runs": {"number": "9", "delta": "-9"}, + "runs_succ": {"number": "10", "delta": "-10"}, + "runs_skip": {"number": "11", "delta": "-11"}, + "runs_fail": {"number": "12", "delta": "-12"}, + "runs_error": {"number": "1" + separator + "345", "delta": "-1" + separator + "345"}, "commit": "commit", + "reference_type": "type", "reference_commit": "ref"} + } + } + gha.add_to_output.assert_called_once_with('json', json.dumps(expected, ensure_ascii=False)) + + + + def test_publish_json_with_test_cases(self): + for separator in ['.', ',', ' ', punctuation_space]: + with self.subTest(json_thousands_separator=separator): + with tempfile.TemporaryDirectory() as path: + filepath = os.path.join(path, 'file.json') + settings = self.create_settings(json_file=filepath, json_thousands_separator=separator, json_test_case_results=True) + + gh, gha, req, repo, commit = self.create_mocks(digest=self.base_digest, check_names=[settings.check_name]) + publisher = Publisher(settings, gh, gha) + publisher.publish_json(self.publish_data) gha.error.assert_not_called() # assert the file with open(filepath, encoding='utf-8') as r: + self.maxDiff = None actual = r.read() self.assertEqual( '{'