From 4ba3dbebd216ca7328a91dcb31710b76ebcc3ea2 Mon Sep 17 00:00:00 2001 From: cpdeethree Date: Sun, 23 Oct 2022 10:21:26 -0500 Subject: [PATCH] ci: update tests for writing cases to json feat --- .github/workflows/ci-cd.yml | 1 + README.md | 4 ++++ composite/action.yml | 1 + python/publish/publisher.py | 1 + python/publish_test_results.py | 1 + python/test/test_action_script.py | 6 ++++-- python/test/test_publisher.py | 4 +++- 7 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 2080d466..e87feff8 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -251,6 +251,7 @@ jobs: -e INPUT_SECONDS_BETWEEN_GITHUB_READS \ -e INPUT_SECONDS_BETWEEN_GITHUB_WRITES \ -e INPUT_JSON_THOUSANDS_SEPARATOR \ + -e INPUT_JSON_TEST_CASE_RESULTS \ -e HOME \ -e GITHUB_JOB \ -e GITHUB_REF \ diff --git a/README.md b/README.md index ced5d9bf..0a3e35ad 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,7 @@ The list of most notable options: |`check_run_annotations_branch`|`event.repository.default_branch` or `"main, master"`|Adds check run annotations only on given branches. If not given, this defaults to the default branch of your repository, e.g. `main` or `master`. Comma separated list of branch names allowed, asterisk `"*"` matches all branches. Example: `main, master, branch_one`.| |`json_file`|no file|Results are written to this JSON file.| |`json_thousands_separator`|`" "`|Formatted numbers in JSON use this character to separate groups of thousands. Common values are "," or ".". Defaults to punctuation space (\u2008).| +|`json_test_case_results`|`false`|Write out all individual test case results to the json output file. Setting this to true can greatly increase the size of the output. Defaults to false.| |`fail_on`|`"test failures"`|Configures the state of the created test result check run. With `"test failures"` it fails if any test fails or test errors occur. It never fails when set to `"nothing"`, and fails only on errors when set to `"errors"`.| Pull request comments highlight removal of tests or tests that the pull request moves into skip state. @@ -391,6 +392,9 @@ Compared to `"Access JSON via step outputs"` above, `errors` and `annotations` c ] } ``` + +Additionally, `json_test_case_results` can be enabled to write out the individual test case results into the JSON file. Enabling this may greatly increase the output size of the JSON file. + See [Create a badge from test results](#create-a-badge-from-test-results) for an example on how to create a badge from this JSON. diff --git a/composite/action.yml b/composite/action.yml index 7e61734e..89060232 100644 --- a/composite/action.yml +++ b/composite/action.yml @@ -180,6 +180,7 @@ runs: SECONDS_BETWEEN_GITHUB_WRITES: ${{ inputs.seconds_between_github_writes }} JSON_FILE: ${{ inputs.json_file }} JSON_THOUSANDS_SEPARATOR: ${{ inputs.json_thousands_separator }} + JSON_TEST_CASE_RESULTS: ${{ inputs.json_test_case_results }} JOB_SUMMARY: ${{ inputs.job_summary }} # not documented ROOT_LOG_LEVEL: ${{ inputs.root_log_level }} diff --git a/python/publish/publisher.py b/python/publish/publisher.py index 9226da28..83549d34 100644 --- a/python/publish/publisher.py +++ b/python/publish/publisher.py @@ -40,6 +40,7 @@ class Settings: commit: str json_file: Optional[str] json_thousands_separator: str + json_test_case_results: bool fail_on_errors: bool fail_on_failures: bool # one of these *_files_glob must be set diff --git a/python/publish_test_results.py b/python/publish_test_results.py index 15901d71..7fe79dd3 100644 --- a/python/publish_test_results.py +++ b/python/publish_test_results.py @@ -370,6 +370,7 @@ def get_settings(options: dict, gha: Optional[GithubAction] = None) -> Settings: commit=get_var('COMMIT', options) or get_commit_sha(event, event_name, options), json_file=get_var('JSON_FILE', options), json_thousands_separator=get_var('JSON_THOUSANDS_SEPARATOR', options) or punctuation_space, + json_test_case_results=get_bool_var('JSON_TEST_CASE_RESULTS', options, default=False), fail_on_errors=fail_on_errors, fail_on_failures=fail_on_failures, junit_files_glob=get_var('JUNIT_FILES', options) or default_junit_files_glob, diff --git a/python/test/test_action_script.py b/python/test/test_action_script.py index 13eaef96..409742b9 100644 --- a/python/test/test_action_script.py +++ b/python/test/test_action_script.py @@ -178,7 +178,8 @@ def get_settings(token='token', seconds_between_github_reads=1.5, seconds_between_github_writes=2.5, json_file=None, - json_thousands_separator=punctuation_space) -> Settings: + json_thousands_separator=punctuation_space, + json_test_case_results=False) -> Settings: return Settings( token=token, api_url=api_url, @@ -191,6 +192,7 @@ def get_settings(token='token', commit=commit, json_file=json_file, json_thousands_separator=json_thousands_separator, + json_test_case_results=json_test_case_results, fail_on_errors=fail_on_errors, fail_on_failures=fail_on_failures, junit_files_glob=junit_files_glob, @@ -822,7 +824,7 @@ def test_parse_files(self): self.assertTrue(any([call.args[0].startswith('reading TRX files [') for call in l.debug.call_args_list])) self.assertEqual([], gha.method_calls) - + print(actual.errors) self.assertEqual(67, actual.files) if Version(sys.version.split(' ')[0]) >= Version('3.10.0') and sys.platform.startswith('darwin'): # on macOS and Python 3.10 we see one particular error diff --git a/python/test/test_publisher.py b/python/test/test_publisher.py index b21a5450..bffb386c 100644 --- a/python/test/test_publisher.py +++ b/python/test/test_publisher.py @@ -87,6 +87,7 @@ def create_settings(comment_mode=comment_mode_always, event_name: str = 'event name', json_file: Optional[str] = None, json_thousands_separator: str = punctuation_space, + json_test_case_results=False, pull_request_build: str = pull_request_build_mode_merge, test_changes_limit: Optional[int] = 5): return Settings( @@ -101,6 +102,7 @@ def create_settings(comment_mode=comment_mode_always, commit='commit', json_file=json_file, json_thousands_separator=json_thousands_separator, + json_test_case_results=False, fail_on_errors=True, fail_on_failures=True, junit_files_glob='*.xml', @@ -1680,7 +1682,7 @@ def test_publish_json(self): with self.subTest(json_thousands_separator=separator): with tempfile.TemporaryDirectory() as path: filepath = os.path.join(path, 'file.json') - settings = self.create_settings(json_file=filepath, json_thousands_separator=separator) + settings = self.create_settings(json_file=filepath, json_thousands_separator=separator, json_test_case_results=False) gh, gha, req, repo, commit = self.create_mocks(digest=self.base_digest, check_names=[settings.check_name]) publisher = Publisher(settings, gh, gha)