Introduce --report-log option

Fix pytest-dev#4488

changelog/4488.feature.rst

@@ -0,0 +1,9 @@
+New ``--report-log=FILE`` option that writes *report logs* into a file as the test session executes.
+
+Each line of the report log contains a self contained JSON object corresponding to a testing event,
+such as a collection or a test result report. The file is guaranteed to be flushed after writing
+each line, so systems can read and process events in real-time.
+
+This option is meant to the replace ``--resultlog``, which is deprecated and meant to be removed
+in a future release. If you use ``--resultlog``, please try out ``--report-log`` and
+provide feedback.

doc/en/report_log.rst

@@ -0,0 +1,46 @@
+.. _report_log:
+
+Report files
+============
+
+.. versionadded:: 5.3
+
+The ``--report-log=FILE`` option writes *report logs* into a file as the test session executes.
+
+Each line of the report log contains a self contained JSON object corresponding to a testing event,
+such as a collection or a test result report. The file is guaranteed to be flushed after writing
+each line, so systems can read and process events in real-time.
+
+Each JSON object contains a special key ``$report_type``, which contains a unique identifier for
+that kind of report object. For future compatibility, consumers of the file should ignore reports
+they don't recognize, as well as ignore unknown properties/keys in JSON objects that they do know,
+as future pytest versions might enrich the objects with more properties/keys.
+
+.. note::
+    This option is meant to the replace ``--resultlog``, which is deprecated and meant to be removed
+    in a future release. If you use ``--resultlog``, please try out ``--report-log`` and
+    provide feedback.
+
+Example
+-------
+
+Consider this file:
+
+.. code-block:: python
+
+    # content of test_report_example.py
+
+
+    def test_ok():
+        assert 5 + 5 == 10
+
+
+    def test_fail():
+        assert 4 + 4 == 1
+
+
+.. code-block:: pytest
+
+    $ pytest test_report_example.py -q --report-log=log.json
+
+The generated ``log.json`` will contain a JSON object per line.

src/_pytest/config/__init__.py

@@ -154,6 +154,7 @@ def directory_arg(path, optname):
     "assertion",
     "junitxml",
     "resultlog",
+    "report_log",
     "doctest",
     "cacheprovider",
     "freeze_support",

src/_pytest/report_log.py

@@ -0,0 +1,76 @@
+import json
+from pathlib import Path
+
+import pytest
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("terminal reporting", "report-log plugin options")
+    group.addoption(
+        "--report-log",
+        action="store",
+        metavar="path",
+        default=None,
+        help="Path to line-based json objects of test session events.",
+    )
+
+
+def pytest_configure(config):
+    report_log = config.option.report_log
+    if report_log and not hasattr(config, "slaveinput"):
+        config._report_log_plugin = ReportLogPlugin(config, Path(report_log))
+        config.pluginmanager.register(config._report_log_plugin)
+
+
+def pytest_unconfigure(config):
+    report_log_plugin = getattr(config, "_report_log_plugin", None)
+    if report_log_plugin:
+        report_log_plugin.close()
+        del config._report_log_plugin
+
+
+class ReportLogPlugin:
+    def __init__(self, config, log_path: Path):
+        self._config = config
+        self._log_path = log_path
+
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        self._file = log_path.open("w", buffering=1, encoding="UTF-8")
+
+    def close(self):
+        if self._file is not None:
+            self._file.close()
+            self._file = None
+
+    def _write_json_data(self, data):
+        self._file.write(json.dumps(data) + "\n")
+        self._file.flush()
+
+    def pytest_sessionstart(self):
+        data = {"pytest_version": pytest.__version__, "$report_type": "Header"}
+        self._write_json_data(data)
+
+    def pytest_runtest_logreport(self, report):
+        data = self._config.hook.pytest_report_to_serializable(
+            config=self._config, report=report
+        )
+        self._write_json_data(data)
+
+    def pytest_collectreport(self, report):
+        data = self._config.hook.pytest_report_to_serializable(
+            config=self._config, report=report
+        )
+        self._write_json_data(data)
+
+    def pytest_terminal_summary(self, terminalreporter):
+        terminalreporter.write_sep(
+            "-", "generated report log file: {}".format(self._log_path)
+        )
+
+    #
+    # def pytest_internalerror(self, excrepr):
+    #     reprcrash = getattr(excrepr, "reprcrash", None)
+    #     path = getattr(reprcrash, "path", None)
+    #     if path is None:
+    #         path = "cwd:%s" % py.path.local()
+    #     self.write_log_entry(path, "!", str(excrepr))

src/_pytest/reports.py

@@ -328,18 +328,18 @@ def toterminal(self, out):
 def pytest_report_to_serializable(report):
     if isinstance(report, (TestReport, CollectReport)):
         data = report._to_json()
-        data["_report_type"] = report.__class__.__name__
+        data["$report_type"] = report.__class__.__name__
         return data
 
 
 def pytest_report_from_serializable(data):
-    if "_report_type" in data:
-        if data["_report_type"] == "TestReport":
+    if "$report_type" in data:
+        if data["$report_type"] == "TestReport":
             return TestReport._from_json(data)
-        elif data["_report_type"] == "CollectReport":
+        elif data["$report_type"] == "CollectReport":
             return CollectReport._from_json(data)
         assert False, "Unknown report_type unserialize data: {}".format(
-            data["_report_type"]
+            data["$report_type"]
         )
 
 

testing/test_report_log.py

@@ -0,0 +1,44 @@
+import json
+
+import pytest
+from _pytest.reports import BaseReport
+
+
+def test_basics(testdir, tmp_path, pytestconfig):
+    """Basic testing of the report log functionality.
+
+    We don't test the test reports extensively because they have been
+    tested already in ``test_reports``.
+    """
+    testdir.makepyfile(
+        """
+        def test_ok():
+            pass
+
+        def test_fail():
+            assert 0
+    """
+    )
+
+    log_file = tmp_path / "log.json"
+
+    result = testdir.runpytest("--report-log", str(log_file))
+    assert result.ret == pytest.ExitCode.TESTS_FAILED
+    result.stdout.fnmatch_lines(["* generated report log file: {}*".format(log_file)])
+
+    json_objs = [json.loads(x) for x in log_file.read_text().splitlines()]
+    assert len(json_objs) == 9
+
+    # first line should be the header
+    header = json_objs[0]
+    assert header == {"pytest_version": pytest.__version__, "$report_type": "Header"}
+
+    # rest of the json objects should be unserialized into report objects; we don't test
+    # the actual report object extensively because it has been tested in ``test_reports``
+    # already.
+    pm = pytestconfig.pluginmanager
+    for json_obj in json_objs[1:]:
+        rep = pm.hook.pytest_report_from_serializable(
+            config=pytestconfig, data=json_obj
+        )
+        assert isinstance(rep, BaseReport)