Keep track of <stderr> and <stdout> mix in CliRunner results.

Closes pallets#2522

CHANGES.rst

@@ -28,6 +28,11 @@ Unreleased
 
 -   Enable deferred evaluation of annotations with
     ``from __future__ import annotations``. :pr:`2270`
+-   Keep `<stdout>` and `<stderr>` streams independent in `CliRunner`. Always
+    collect `<stderr>` output and never raise an exception. Add a new
+    `<output>` stream to simulate what the user sees in its terminal. Only mix
+    `<stdout>` and `<stderr>` in `<output>` when ``mix_stderr=True``.
+    :issue:`2522` :pr:`2523`
 
 
 Version 8.1.7

src/click/testing.py

@@ -64,6 +64,48 @@ def _pause_echo(stream: EchoingStdin | None) -> cabc.Iterator[None]:
         stream._paused = False
 
 
+class BytesIOCopy(io.BytesIO):
+    """Patch ``io.BytesIO`` to let the written stream be copied to another.
+
+    .. versionadded:: 8.2
+    """
+
+    def __init__(self, copy_to: io.BytesIO) -> None:
+        super().__init__()
+        self.copy_to = copy_to
+
+    def flush(self) -> None:
+        super().flush()
+        self.copy_to.flush()
+
+    def write(self, b) -> int:  # type: ignore[no-untyped-def]
+        self.copy_to.write(b)
+        return super().write(b)
+
+
+class StreamMixer:
+    """Mixes `<stdout>` and `<stderr>` streams if ``mix_stderr=True``.
+
+    The result is available in the ``output`` attribute.
+
+    If ``mix_stderr=False``, the `<stdout>` and `<stderr>` streams are kept
+    independent and the ``output`` is the same as the `<stdout>` stream.
+
+    .. versionadded:: 8.2
+    """
+
+    def __init__(self, mix_stderr: bool) -> None:
+        if not mix_stderr:
+            self.stdout = io.BytesIO()
+            self.stderr = io.BytesIO()
+            self.output = self.stdout
+
+        else:
+            self.output = io.BytesIO()
+            self.stdout = BytesIOCopy(copy_to=self.output)
+            self.stderr = BytesIOCopy(copy_to=self.output)
+
+
 class _NamedTextIOWrapper(io.TextIOWrapper):
     def __init__(
         self, buffer: t.BinaryIO, name: str, mode: str, **kwargs: t.Any
@@ -108,7 +150,8 @@ def __init__(
         self,
         runner: CliRunner,
         stdout_bytes: bytes,
-        stderr_bytes: bytes | None,
+        stderr_bytes: bytes,
+        output_bytes: bytes,
         return_value: t.Any,
         exit_code: int,
         exception: BaseException | None,
@@ -119,8 +162,16 @@ def __init__(
         self.runner = runner
         #: The standard output as bytes.
         self.stdout_bytes = stdout_bytes
-        #: The standard error as bytes, or None if not available
+        #: The standard error as bytes.
+        #:
+        #: .. versionchanged:: 8.2
+        #:     No longer optional.
         self.stderr_bytes = stderr_bytes
+        #: A mix of `stdout_bytes` and `stderr_bytes``, as the user would see
+        # it in its terminal.
+        #:
+        #: .. versionadded:: 8.2
+        self.output_bytes = output_bytes
         #: The value returned from the invoked command.
         #:
         #: .. versionadded:: 8.0
@@ -134,8 +185,15 @@ def __init__(
 
     @property
     def output(self) -> str:
-        """The (standard) output as unicode string."""
-        return self.stdout
+        """The terminal output as unicode string, as the user would see it.
+
+        .. versionchanged:: 8.2
+            No longer a proxy for ``self.stdout``. Now has its own stream to mix
+            `<stdout>` and `<stderr>` depending on ``mix_stderr`` value.
+        """
+        return self.output_bytes.decode(self.runner.charset, "replace").replace(
+            "\r\n", "\n"
+        )
 
     @property
     def stdout(self) -> str:
@@ -146,9 +204,11 @@ def stdout(self) -> str:
 
     @property
     def stderr(self) -> str:
-        """The standard error as unicode string."""
-        if self.stderr_bytes is None:
-            raise ValueError("stderr not separately captured")
+        """The standard error as unicode string.
+
+        .. versionchanged:: 8.2
+            No longer raise an exception, always returns the `<stderr>` string.
+        """
         return self.stderr_bytes.decode(self.runner.charset, "replace").replace(
             "\r\n", "\n"
         )
@@ -166,15 +226,16 @@ class CliRunner:
 
     :param charset: the character set for the input and output data.
     :param env: a dictionary with environment variables for overriding.
-    :param echo_stdin: if this is set to `True`, then reading from stdin writes
-                       to stdout.  This is useful for showing examples in
+    :param echo_stdin: if this is set to `True`, then reading from `<stdin>` writes
+                       to `<stdout>`.  This is useful for showing examples in
                        some circumstances.  Note that regular prompts
                        will automatically echo the input.
-    :param mix_stderr: if this is set to `False`, then stdout and stderr are
-                       preserved as independent streams.  This is useful for
-                       Unix-philosophy apps that have predictable stdout and
-                       noisy stderr, such that each may be measured
-                       independently
+    :param mix_stderr: if this is set to `False`, then the output will be the
+                       same as the `<stdout>` stream.  If set to `True` (the
+                       default), then the output will feature both `<stdout>`
+                       and `<stderr>`, in the order they were written.  This is
+                       useful for testing the output of a command as the user
+                       would see it in its terminal.
     """
 
     def __init__(
@@ -211,22 +272,29 @@ def isolation(
         input: str | bytes | t.IO[t.Any] | None = None,
         env: cabc.Mapping[str, str | None] | None = None,
         color: bool = False,
-    ) -> cabc.Iterator[tuple[io.BytesIO, io.BytesIO | None]]:
+    ) -> cabc.Iterator[tuple[io.BytesIO, io.BytesIO, io.BytesIO]]:
         """A context manager that sets up the isolation for invoking of a
-        command line tool.  This sets up stdin with the given input data
+        command line tool.  This sets up `<stdin>` with the given input data
         and `os.environ` with the overrides from the given dictionary.
         This also rebinds some internals in Click to be mocked (like the
         prompt functionality).
 
         This is automatically done in the :meth:`invoke` method.
 
-        :param input: the input stream to put into sys.stdin.
+        :param input: the input stream to put into `sys.stdin`.
         :param env: the environment overrides as dictionary.
         :param color: whether the output should contain color codes. The
                       application can still override this explicitly.
 
+        .. versionadded:: 8.2
+            An additional output stream is returned, which is a mix of
+            `<stdout>` and `<stderr>` streams if ``mix_stderr=True``.
+
+        .. versionchanged:: 8.2
+            Always returns the `<stderr>` stream.
+
         .. versionchanged:: 8.0
-            ``stderr`` is opened with ``errors="backslashreplace"``
+            `<stderr>` is opened with ``errors="backslashreplace"``
             instead of the default ``"strict"``.
 
         .. versionchanged:: 4.0
@@ -243,11 +311,11 @@ def isolation(
 
         env = self.make_env(env)
 
-        bytes_output = io.BytesIO()
+        stream_mixer = StreamMixer(mix_stderr=self.mix_stderr)
 
         if self.echo_stdin:
             bytes_input = echo_input = t.cast(
-                t.BinaryIO, EchoingStdin(bytes_input, bytes_output)
+                t.BinaryIO, EchoingStdin(bytes_input, stream_mixer.stdout)
             )
 
         sys.stdin = text_input = _NamedTextIOWrapper(
@@ -260,21 +328,16 @@ def isolation(
             text_input._CHUNK_SIZE = 1  # type: ignore
 
         sys.stdout = _NamedTextIOWrapper(
-            bytes_output, encoding=self.charset, name="<stdout>", mode="w"
+            stream_mixer.stdout, encoding=self.charset, name="<stdout>", mode="w"
         )
 
-        bytes_error = None
-        if self.mix_stderr:
-            sys.stderr = sys.stdout
-        else:
-            bytes_error = io.BytesIO()
-            sys.stderr = _NamedTextIOWrapper(
-                bytes_error,
-                encoding=self.charset,
-                name="<stderr>",
-                mode="w",
-                errors="backslashreplace",
-            )
+        sys.stderr = _NamedTextIOWrapper(
+            stream_mixer.stderr,
+            encoding=self.charset,
+            name="<stderr>",
+            mode="w",
+            errors="backslashreplace",
+        )
 
         @_pause_echo(echo_input)  # type: ignore
         def visible_input(prompt: str | None = None) -> str:
@@ -329,7 +392,7 @@ def should_strip_ansi(
                         pass
                 else:
                     os.environ[key] = value
-            yield (bytes_output, bytes_error)
+            yield (stream_mixer.stdout, stream_mixer.stderr, stream_mixer.output)
         finally:
             for key, value in old_env.items():
                 if value is None:
@@ -378,6 +441,14 @@ def invoke(
         :param color: whether the output should contain color codes. The
                       application can still override this explicitly.
 
+        .. versionadded:: 8.2
+            The result object has the ``output_bytes`` attribute with
+            the mix of ``stdout_bytes`` and ``stderr_bytes``, as the user would
+            see it in its terminal.
+
+        .. versionchanged:: 8.2
+            The result object always returns the ``stderr_bytes`` stream.
+
         .. versionchanged:: 8.0
             The result object has the ``return_value`` attribute with
             the value returned from the invoked command.
@@ -434,15 +505,14 @@ def invoke(
             finally:
                 sys.stdout.flush()
                 stdout = outstreams[0].getvalue()
-                if self.mix_stderr:
-                    stderr = None
-                else:
-                    stderr = outstreams[1].getvalue()  # type: ignore
+                stderr = outstreams[1].getvalue()
+                output = outstreams[2].getvalue()
 
         return Result(
             runner=self,
             stdout_bytes=stdout,
             stderr_bytes=stderr,
+            output_bytes=output,
             return_value=return_value,
             exit_code=exit_code,
             exception=exception,

tests/test_termui.py

@@ -246,7 +246,7 @@ def test_secho(runner):
     ("value", "expect"), [(123, b"\x1b[45m123\x1b[0m"), (b"test", b"test")]
 )
 def test_secho_non_text(runner, value, expect):
-    with runner.isolation() as (out, _):
+    with runner.isolation() as (out, _, _):
         click.secho(value, nl=False, color=True, bg="magenta")
         result = out.getvalue()
         assert result == expect

tests/test_testing.py

@@ -303,25 +303,25 @@ def cli_env():
 def test_stderr():
     @click.command()
     def cli_stderr():
-        click.echo("stdout")
-        click.echo("stderr", err=True)
+        click.echo("1 - stdout")
+        click.echo("2 - stderr", err=True)
+        click.echo("3 - stdout")
+        click.echo("4 - stderr", err=True)
 
     runner = CliRunner(mix_stderr=False)
 
     result = runner.invoke(cli_stderr)
 
-    assert result.output == "stdout\n"
-    assert result.stdout == "stdout\n"
-    assert result.stderr == "stderr\n"
+    assert result.output == "1 - stdout\n3 - stdout\n"
+    assert result.stdout == "1 - stdout\n3 - stdout\n"
+    assert result.stderr == "2 - stderr\n4 - stderr\n"
 
     runner_mix = CliRunner(mix_stderr=True)
     result_mix = runner_mix.invoke(cli_stderr)
 
-    assert result_mix.output == "stdout\nstderr\n"
-    assert result_mix.stdout == "stdout\nstderr\n"
-
-    with pytest.raises(ValueError):
-        result_mix.stderr
+    assert result_mix.output == "1 - stdout\n2 - stderr\n3 - stdout\n4 - stderr\n"
+    assert result_mix.stdout == "1 - stdout\n3 - stdout\n"
+    assert result_mix.stderr == "2 - stderr\n4 - stderr\n"
 
     @click.command()
     def cli_empty_stderr():
@@ -414,7 +414,7 @@ def test_isolation_stderr_errors():
     """
     runner = CliRunner(mix_stderr=False)
 
-    with runner.isolation() as (_, err):
+    with runner.isolation() as (_, err, _):
         click.echo("\udce2", err=True, nl=False)
 
     assert err.getvalue() == b"\\udce2"