ci: update tests for writing cases to json feat

.github/workflows/ci-cd.yml

@@ -251,6 +251,7 @@ jobs:
             -e INPUT_SECONDS_BETWEEN_GITHUB_READS \
             -e INPUT_SECONDS_BETWEEN_GITHUB_WRITES \
             -e INPUT_JSON_THOUSANDS_SEPARATOR \
+            -e INPUT_JSON_TEST_CASE_RESULTS \
             -e HOME \
             -e GITHUB_JOB \
             -e GITHUB_REF \

README.md

@@ -268,6 +268,7 @@ The list of most notable options:
 |`check_run_annotations_branch`|`event.repository.default_branch` or `"main, master"`|Adds check run annotations only on given branches. If not given, this defaults to the default branch of your repository, e.g. `main` or `master`. Comma separated list of branch names allowed, asterisk `"*"` matches all branches. Example: `main, master, branch_one`.|
 |`json_file`|no file|Results are written to this JSON file.|
 |`json_thousands_separator`|`" "`|Formatted numbers in JSON use this character to separate groups of thousands. Common values are "," or ".". Defaults to punctuation space (\u2008).|
+|`json_test_case_results`|`false`|Write out all individual test case results to the json output file. Setting this to true can greatly increase the size of the output. Defaults to false.|
 |`fail_on`|`"test failures"`|Configures the state of the created test result check run. With `"test failures"` it fails if any test fails or test errors occur. It never fails when set to `"nothing"`, and fails only on errors when set to `"errors"`.|
 
 Pull request comments highlight removal of tests or tests that the pull request moves into skip state.
@@ -391,6 +392,9 @@ Compared to `"Access JSON via step outputs"` above, `errors` and `annotations` c
    ]
 }
 ```
+
+Additionally, `json_test_case_results` can be enabled to write out the individual test case results into the JSON file. Enabling this may greatly increase the output size of the JSON file.
+
 </details>
 
 See [Create a badge from test results](#create-a-badge-from-test-results) for an example on how to create a badge from this JSON.

composite/action.yml

@@ -180,6 +180,7 @@ runs:
         SECONDS_BETWEEN_GITHUB_WRITES: ${{ inputs.seconds_between_github_writes }}
         JSON_FILE: ${{ inputs.json_file }}
         JSON_THOUSANDS_SEPARATOR: ${{ inputs.json_thousands_separator }}
+        JSON_TEST_CASE_RESULTS: ${{ inputs.json_test_case_results }}
         JOB_SUMMARY: ${{ inputs.job_summary }}
         # not documented
         ROOT_LOG_LEVEL: ${{ inputs.root_log_level }}

python/publish/publisher.py

@@ -40,6 +40,7 @@ class Settings:
     commit: str
     json_file: Optional[str]
     json_thousands_separator: str
+    json_test_case_results: bool
     fail_on_errors: bool
     fail_on_failures: bool
     # one of these *_files_glob must be set

python/publish_test_results.py

@@ -370,6 +370,7 @@ def get_settings(options: dict, gha: Optional[GithubAction] = None) -> Settings:
         commit=get_var('COMMIT', options) or get_commit_sha(event, event_name, options),
         json_file=get_var('JSON_FILE', options),
         json_thousands_separator=get_var('JSON_THOUSANDS_SEPARATOR', options) or punctuation_space,
+        json_test_case_results=get_bool_var('JSON_TEST_CASE_RESULTS', options, default=False),
         fail_on_errors=fail_on_errors,
         fail_on_failures=fail_on_failures,
         junit_files_glob=get_var('JUNIT_FILES', options) or default_junit_files_glob,

python/test/test_action_script.py

@@ -178,7 +178,8 @@ def get_settings(token='token',
                      seconds_between_github_reads=1.5,
                      seconds_between_github_writes=2.5,
                      json_file=None,
-                     json_thousands_separator=punctuation_space) -> Settings:
+                     json_thousands_separator=punctuation_space,
+                    json_test_case_results=False) -> Settings:
         return Settings(
             token=token,
             api_url=api_url,
@@ -191,6 +192,7 @@ def get_settings(token='token',
             commit=commit,
             json_file=json_file,
             json_thousands_separator=json_thousands_separator,
+            json_test_case_results=json_test_case_results,
             fail_on_errors=fail_on_errors,
             fail_on_failures=fail_on_failures,
             junit_files_glob=junit_files_glob,
@@ -822,7 +824,7 @@ def test_parse_files(self):
             self.assertTrue(any([call.args[0].startswith('reading TRX files [') for call in l.debug.call_args_list]))
 
         self.assertEqual([], gha.method_calls)
-
+        print(actual.errors)
         self.assertEqual(67, actual.files)
         if Version(sys.version.split(' ')[0]) >= Version('3.10.0') and sys.platform.startswith('darwin'):
             # on macOS and Python 3.10 we see one particular error

python/test/test_publisher.py

@@ -87,6 +87,7 @@ def create_settings(comment_mode=comment_mode_always,
                         event_name: str = 'event name',
                         json_file: Optional[str] = None,
                         json_thousands_separator: str = punctuation_space,
+                        json_test_case_results: False,
                         pull_request_build: str = pull_request_build_mode_merge,
                         test_changes_limit: Optional[int] = 5):
         return Settings(
@@ -101,6 +102,7 @@ def create_settings(comment_mode=comment_mode_always,
             commit='commit',
             json_file=json_file,
             json_thousands_separator=json_thousands_separator,
+            json_test_case_results=False,
             fail_on_errors=True,
             fail_on_failures=True,
             junit_files_glob='*.xml',
@@ -1680,7 +1682,7 @@ def test_publish_json(self):
             with self.subTest(json_thousands_separator=separator):
                 with tempfile.TemporaryDirectory() as path:
                     filepath = os.path.join(path, 'file.json')
-                    settings = self.create_settings(json_file=filepath, json_thousands_separator=separator)
+                    settings = self.create_settings(json_file=filepath, json_thousands_separator=separator, json_test_case_results=False)
 
                     gh, gha, req, repo, commit = self.create_mocks(digest=self.base_digest, check_names=[settings.check_name])
                     publisher = Publisher(settings, gh, gha)