feat: option to write out test cases to json

.github/workflows/ci-cd.yml

@@ -251,6 +251,7 @@ jobs:
             -e INPUT_SECONDS_BETWEEN_GITHUB_READS \
             -e INPUT_SECONDS_BETWEEN_GITHUB_WRITES \
             -e INPUT_JSON_THOUSANDS_SEPARATOR \
+            -e INPUT_JSON_TEST_CASE_RESULTS \
             -e HOME \
             -e GITHUB_JOB \
             -e GITHUB_REF \

README.md

@@ -268,6 +268,7 @@ The list of most notable options:
 |`check_run_annotations_branch`|`event.repository.default_branch` or `"main, master"`|Adds check run annotations only on given branches. If not given, this defaults to the default branch of your repository, e.g. `main` or `master`. Comma separated list of branch names allowed, asterisk `"*"` matches all branches. Example: `main, master, branch_one`.|
 |`json_file`|no file|Results are written to this JSON file.|
 |`json_thousands_separator`|`" "`|Formatted numbers in JSON use this character to separate groups of thousands. Common values are "," or ".". Defaults to punctuation space (\u2008).|
+|`json_test_case_results`|`false`|Write out all individual test case results to the json output file. Setting this to true can greatly increase the size of the output. Defaults to false.|
 |`fail_on`|`"test failures"`|Configures the state of the created test result check run. With `"test failures"` it fails if any test fails or test errors occur. It never fails when set to `"nothing"`, and fails only on errors when set to `"errors"`.|
 
 Pull request comments highlight removal of tests or tests that the pull request moves into skip state.
@@ -391,6 +392,9 @@ Compared to `"Access JSON via step outputs"` above, `errors` and `annotations` c
    ]
 }
 ```
+
+Additionally, `json_test_case_results` can be enabled to write out the individual test case results into the JSON file. Enabling this may greatly increase the output size of the JSON file.
+
 </details>
 
 See [Create a badge from test results](#create-a-badge-from-test-results) for an example on how to create a badge from this JSON.

action.yml

@@ -101,6 +101,10 @@ inputs:
     description: 'Formatted numbers in JSON use this character to separate groups of thousands. Common values are "," or ".". Defaults to punctuation space (\u2008).'
     default: ' '
     required: false
+  json_test_case_results:
+    description: 'Write out all individual test case results to JSON file. This may greatly increase the size of the output'
+    default: false
+    required: false
 
 outputs:
   json:

composite/action.yml

@@ -101,7 +101,10 @@ inputs:
     description: 'Formatted numbers in JSON use this character to separate groups of thousands. Common values are "," or ".". Defaults to punctuation space (\u2008).'
     default: ' '
     required: false
-
+  json_test_case_results:
+    description: 'Write out all individual test case results to JSON file. This may greatly increase the size of the output'
+    default: false
+    required: false
 outputs:
   json:
     description: "Test results as JSON"
@@ -177,6 +180,7 @@ runs:
         SECONDS_BETWEEN_GITHUB_WRITES: ${{ inputs.seconds_between_github_writes }}
         JSON_FILE: ${{ inputs.json_file }}
         JSON_THOUSANDS_SEPARATOR: ${{ inputs.json_thousands_separator }}
+        JSON_TEST_CASE_RESULTS: ${{ inputs.json_test_case_results }}
         JOB_SUMMARY: ${{ inputs.job_summary }}
         # not documented
         ROOT_LOG_LEVEL: ${{ inputs.root_log_level }}

python/publish/publisher.py

@@ -40,6 +40,7 @@ class Settings:
     commit: str
     json_file: Optional[str]
     json_thousands_separator: str
+    json_test_case_results: bool
     fail_on_errors: bool
     fail_on_failures: bool
     # one of these *_files_glob must be set
@@ -72,6 +73,7 @@ class PublishData:
     stats_with_delta: Optional[UnitTestRunDeltaResults]
     annotations: List[Annotation]
     check_url: str
+    cases: Optional[UnitTestCaseResults]
 
     @classmethod
     def _format_digit(cls, value: Union[int, Mapping[str, int], Any], thousands_separator: str) -> Union[str, Mapping[str, str], Any]:
@@ -347,7 +349,8 @@ def publish_check(self,
             stats=stats,
             stats_with_delta=stats_with_delta if before_stats is not None else None,
             annotations=all_annotations,
-            check_url=check_run.html_url
+            check_url=check_run.html_url,
+            cases=cases if self._settings.json_test_case_results else None
         )
         self.publish_json(data)
 

python/publish_test_results.py

@@ -370,6 +370,7 @@ def get_settings(options: dict, gha: Optional[GithubAction] = None) -> Settings:
         commit=get_var('COMMIT', options) or get_commit_sha(event, event_name, options),
         json_file=get_var('JSON_FILE', options),
         json_thousands_separator=get_var('JSON_THOUSANDS_SEPARATOR', options) or punctuation_space,
+        json_test_case_results=get_bool_var('JSON_TEST_CASE_RESULTS', options, default=False),
         fail_on_errors=fail_on_errors,
         fail_on_failures=fail_on_failures,
         junit_files_glob=get_var('JUNIT_FILES', options) or default_junit_files_glob,

python/test/test_action_script.py

@@ -178,7 +178,8 @@ def get_settings(token='token',
                      seconds_between_github_reads=1.5,
                      seconds_between_github_writes=2.5,
                      json_file=None,
-                     json_thousands_separator=punctuation_space) -> Settings:
+                     json_thousands_separator=punctuation_space,
+                    json_test_case_results=False) -> Settings:
         return Settings(
             token=token,
             api_url=api_url,
@@ -191,6 +192,7 @@ def get_settings(token='token',
             commit=commit,
             json_file=json_file,
             json_thousands_separator=json_thousands_separator,
+            json_test_case_results=json_test_case_results,
             fail_on_errors=fail_on_errors,
             fail_on_failures=fail_on_failures,
             junit_files_glob=junit_files_glob,
@@ -822,7 +824,6 @@ def test_parse_files(self):
             self.assertTrue(any([call.args[0].startswith('reading TRX files [') for call in l.debug.call_args_list]))
 
         self.assertEqual([], gha.method_calls)
-
         self.assertEqual(67, actual.files)
         if Version(sys.version.split(' ')[0]) >= Version('3.10.0') and sys.platform.startswith('darwin'):
             # on macOS and Python 3.10 we see one particular error