From d1129cf96d0f3fd873cb7b466761cffe97b58e98 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 16 Oct 2019 19:24:41 -0300 Subject: [PATCH] Introduce --report-log option Fix #4488 --- changelog/4488.feature.rst | 9 ++++ doc/en/report_log.rst | 46 ++++++++++++++++++++ src/_pytest/config/__init__.py | 1 + src/_pytest/report_log.py | 76 ++++++++++++++++++++++++++++++++++ src/_pytest/reports.py | 10 ++--- testing/test_report_log.py | 44 ++++++++++++++++++++ 6 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 changelog/4488.feature.rst create mode 100644 doc/en/report_log.rst create mode 100644 src/_pytest/report_log.py create mode 100644 testing/test_report_log.py diff --git a/changelog/4488.feature.rst b/changelog/4488.feature.rst new file mode 100644 index 00000000000..ab504ceb35e --- /dev/null +++ b/changelog/4488.feature.rst @@ -0,0 +1,9 @@ +New ``--report-log=FILE`` option that writes *report logs* into a file as the test session executes. + +Each line of the report log contains a self contained JSON object corresponding to a testing event, +such as a collection or a test result report. The file is guaranteed to be flushed after writing +each line, so systems can read and process events in real-time. + +This option is meant to the replace ``--resultlog``, which is deprecated and meant to be removed +in a future release. If you use ``--resultlog``, please try out ``--report-log`` and +provide feedback. diff --git a/doc/en/report_log.rst b/doc/en/report_log.rst new file mode 100644 index 00000000000..f666613ed5d --- /dev/null +++ b/doc/en/report_log.rst @@ -0,0 +1,46 @@ +.. _report_log: + +Report files +============ + +.. versionadded:: 5.3 + +The ``--report-log=FILE`` option writes *report logs* into a file as the test session executes. + +Each line of the report log contains a self contained JSON object corresponding to a testing event, +such as a collection or a test result report. The file is guaranteed to be flushed after writing +each line, so systems can read and process events in real-time. + +Each JSON object contains a special key ``$report_type``, which contains a unique identifier for +that kind of report object. For future compatibility, consumers of the file should ignore reports +they don't recognize, as well as ignore unknown properties/keys in JSON objects that they do know, +as future pytest versions might enrich the objects with more properties/keys. + +.. note:: + This option is meant to the replace ``--resultlog``, which is deprecated and meant to be removed + in a future release. If you use ``--resultlog``, please try out ``--report-log`` and + provide feedback. + +Example +------- + +Consider this file: + +.. code-block:: python + + # content of test_report_example.py + + + def test_ok(): + assert 5 + 5 == 10 + + + def test_fail(): + assert 4 + 4 == 1 + + +.. code-block:: pytest + + $ pytest test_report_example.py -q --report-log=log.json + +The generated ``log.json`` will contain a JSON object per line. diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 4746fd6c7fc..2b0f48c07a7 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -154,6 +154,7 @@ def directory_arg(path, optname): "assertion", "junitxml", "resultlog", + "report_log", "doctest", "cacheprovider", "freeze_support", diff --git a/src/_pytest/report_log.py b/src/_pytest/report_log.py new file mode 100644 index 00000000000..af914e0ec46 --- /dev/null +++ b/src/_pytest/report_log.py @@ -0,0 +1,76 @@ +import json +from pathlib import Path + +import pytest + + +def pytest_addoption(parser): + group = parser.getgroup("terminal reporting", "report-log plugin options") + group.addoption( + "--report-log", + action="store", + metavar="path", + default=None, + help="Path to line-based json objects of test session events.", + ) + + +def pytest_configure(config): + report_log = config.option.report_log + if report_log and not hasattr(config, "slaveinput"): + config._report_log_plugin = ReportLogPlugin(config, Path(report_log)) + config.pluginmanager.register(config._report_log_plugin) + + +def pytest_unconfigure(config): + report_log_plugin = getattr(config, "_report_log_plugin", None) + if report_log_plugin: + report_log_plugin.close() + del config._report_log_plugin + + +class ReportLogPlugin: + def __init__(self, config, log_path: Path): + self._config = config + self._log_path = log_path + + log_path.parent.mkdir(parents=True, exist_ok=True) + self._file = log_path.open("w", buffering=1, encoding="UTF-8") + + def close(self): + if self._file is not None: + self._file.close() + self._file = None + + def _write_json_data(self, data): + self._file.write(json.dumps(data) + "\n") + self._file.flush() + + def pytest_sessionstart(self): + data = {"pytest_version": pytest.__version__, "$report_type": "Header"} + self._write_json_data(data) + + def pytest_runtest_logreport(self, report): + data = self._config.hook.pytest_report_to_serializable( + config=self._config, report=report + ) + self._write_json_data(data) + + def pytest_collectreport(self, report): + data = self._config.hook.pytest_report_to_serializable( + config=self._config, report=report + ) + self._write_json_data(data) + + def pytest_terminal_summary(self, terminalreporter): + terminalreporter.write_sep( + "-", "generated report log file: {}".format(self._log_path) + ) + + # + # def pytest_internalerror(self, excrepr): + # reprcrash = getattr(excrepr, "reprcrash", None) + # path = getattr(reprcrash, "path", None) + # if path is None: + # path = "cwd:%s" % py.path.local() + # self.write_log_entry(path, "!", str(excrepr)) diff --git a/src/_pytest/reports.py b/src/_pytest/reports.py index d8030b92649..ee755504169 100644 --- a/src/_pytest/reports.py +++ b/src/_pytest/reports.py @@ -328,18 +328,18 @@ def toterminal(self, out): def pytest_report_to_serializable(report): if isinstance(report, (TestReport, CollectReport)): data = report._to_json() - data["_report_type"] = report.__class__.__name__ + data["$report_type"] = report.__class__.__name__ return data def pytest_report_from_serializable(data): - if "_report_type" in data: - if data["_report_type"] == "TestReport": + if "$report_type" in data: + if data["$report_type"] == "TestReport": return TestReport._from_json(data) - elif data["_report_type"] == "CollectReport": + elif data["$report_type"] == "CollectReport": return CollectReport._from_json(data) assert False, "Unknown report_type unserialize data: {}".format( - data["_report_type"] + data["$report_type"] ) diff --git a/testing/test_report_log.py b/testing/test_report_log.py new file mode 100644 index 00000000000..7ce3488135c --- /dev/null +++ b/testing/test_report_log.py @@ -0,0 +1,44 @@ +import json + +import pytest +from _pytest.reports import BaseReport + + +def test_basics(testdir, tmp_path, pytestconfig): + """Basic testing of the report log functionality. + + We don't test the test reports extensively because they have been + tested already in ``test_reports``. + """ + testdir.makepyfile( + """ + def test_ok(): + pass + + def test_fail(): + assert 0 + """ + ) + + log_file = tmp_path / "log.json" + + result = testdir.runpytest("--report-log", str(log_file)) + assert result.ret == pytest.ExitCode.TESTS_FAILED + result.stdout.fnmatch_lines(["* generated report log file: {}*".format(log_file)]) + + json_objs = [json.loads(x) for x in log_file.read_text().splitlines()] + assert len(json_objs) == 9 + + # first line should be the header + header = json_objs[0] + assert header == {"pytest_version": pytest.__version__, "$report_type": "Header"} + + # rest of the json objects should be unserialized into report objects; we don't test + # the actual report object extensively because it has been tested in ``test_reports`` + # already. + pm = pytestconfig.pluginmanager + for json_obj in json_objs[1:]: + rep = pm.hook.pytest_report_from_serializable( + config=pytestconfig, data=json_obj + ) + assert isinstance(rep, BaseReport)