bootloader: use Py_UnbufferedStdioFlag to enable unbuffered output

bootloader/src/pyi_python.c

@@ -39,6 +39,7 @@ DECLVAR(Py_NoSiteFlag);
 DECLVAR(Py_NoUserSiteDirectory);
 DECLVAR(Py_OptimizeFlag);
 DECLVAR(Py_VerboseFlag);
+DECLVAR(Py_UnbufferedStdioFlag);
 
 /* functions with prefix `Py_` */
 DECLPROC(Py_BuildValue);
@@ -104,6 +105,7 @@ pyi_python_map_names(HMODULE dll, int pyvers)
     GETVAR(dll, Py_NoUserSiteDirectory);
     GETVAR(dll, Py_OptimizeFlag);
     GETVAR(dll, Py_VerboseFlag);
+    GETVAR(dll, Py_UnbufferedStdioFlag);
 
     /* functions with prefix `Py_` */
     GETPROC(dll, Py_BuildValue);

bootloader/src/pyi_python.h

@@ -89,6 +89,7 @@ EXTDECLVAR(int, Py_VerboseFlag);
 EXTDECLVAR(int, Py_IgnoreEnvironmentFlag);
 EXTDECLVAR(int, Py_DontWriteBytecodeFlag);
 EXTDECLVAR(int, Py_NoUserSiteDirectory);
+EXTDECLVAR(int, Py_UnbufferedStdioFlag);
 
 /* This initializes the table of loaded modules (sys.modules), and creates the fundamental modules builtins, __main__ and sys. It also initializes the module search path (sys.path). It does not set sys.argv; */
 EXTDECLPROC(int, Py_Initialize, (void));

bootloader/src/pyi_pythonlib.c

@@ -240,6 +240,9 @@ pyi_pylib_set_runtime_opts(ARCHIVE_STATUS *status)
         setbuf(stdin, (char *)NULL);
         setbuf(stdout, (char *)NULL);
         setbuf(stderr, (char *)NULL);
+
+        /* Enable unbuffered mode via Py_UnbufferedStdioFlag */
+        *PI_Py_UnbufferedStdioFlag = 1;
     }
     return 0;
 }

doc/spec-files.rst

@@ -359,7 +359,9 @@ For example modify the spec file this way::
           exclude_binaries=...
           )
 
-.. Warning:: The ``u`` option does not work on Windows. See this `GitHub issue <https://github.com/pyinstaller/pyinstaller/issues/1441>`_ for more details.
+.. Note:: The unbuffered stdio mode (the ``u`` option) enables unbuffered
+   binary layer of ``stdout`` and ``stderr`` streams on all supported Python
+   versions. The unbuffered text layer requires Python 3.7 or later.
 
 
 .. _spec file options for a mac os x bundle:

news/1441.feature.rst

@@ -0,0 +1,2 @@
+The `unbuffered stdio` mode (the ``u`` option) now sets the ``Py_UnbufferedStdioFlag``
+flag to enable unbuffered stdio mode in Python library.

tests/functional/scripts/pyi_unbuffered_output.py

@@ -0,0 +1,62 @@
+#-----------------------------------------------------------------------------
+# Copyright (c) 2021, PyInstaller Development Team.
+#
+# Distributed under the terms of the GNU General Public License (version 2
+# or later) with exception for distributing the bootloader.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#
+# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception)
+#-----------------------------------------------------------------------------
+
+# A simple program for testing unbuffered mode of python stdout and
+# stderr streams (both binary and text layers).
+#
+# The program periodically prints star ('*') character to a single line
+# on the specified output stream (stdout or stderr). Once the selected
+# number of stars have been printed, the end-of-transmission is signalled
+# by 'E' character.
+#
+# In the unbuffered mode, the caller should receive the characters
+# individually, and have enough time to process them. In the buffered
+# mode, all printed characters including the terminating E will be
+# received at once.
+#
+# NOTE: the unbuffered mode for text layers was introduced in Python 3.7.
+
+import argparse
+import time
+import sys
+
+# Argument parser
+parser = argparse.ArgumentParser(description="Unbuffered stdio test")
+parser.add_argument('--num-stars', type=int, default=5,
+                    help="Number of star characters to print.")
+parser.add_argument('--output-stream', type=str, default='stdout',
+                    help="Output stream ('stdout' or 'stderr')")
+parser.add_argument('--stream-mode', type=str, default='text',
+                    help="Output stream mode ('text' or 'binary')")
+args = parser.parse_args()
+
+# Select output stream and mode
+assert args.output_stream in {'stdout', 'stderr'}, \
+    f"Invalid output stream: {args.output_stream}!"
+assert args.stream_mode in {'text', 'binary'}, \
+    f"Invalid output stream mode: {args.stream_mode}!"
+
+stream = sys.stdout if args.output_stream == 'stdout' else sys.stderr
+if args.stream_mode == 'binary':
+    stream = stream.buffer  # Use binary layer
+    STAR = b'*'
+    EOT = b'E'
+else:
+    STAR = '*'
+    EOT = 'E'
+
+# Print the specified number of stars in a single line
+for i in range(args.num_stars):
+    stream.write(STAR)
+    time.sleep(1)
+
+# End-of-transmission
+stream.write(EOT)

tests/functional/specs/pyi_unbuffered_output.spec

@@ -0,0 +1,34 @@
+# -*- mode: python -*-
+#-----------------------------------------------------------------------------
+# Copyright (c) 2021, PyInstaller Development Team.
+#
+# Distributed under the terms of the GNU General Public License (version 2
+# or later) with exception for distributing the bootloader.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#
+# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception)
+#-----------------------------------------------------------------------------
+
+app_name = 'pyi_unbuffered_output'
+
+a = Analysis([os.path.join(os.path.dirname(SPECPATH), 'scripts/pyi_unbuffered_output.py')])
+pyz = PYZ(a.pure, a.zipped_data)
+exe = EXE(pyz,
+          a.scripts,
+          [('u', None, 'OPTION'), ],
+          exclude_binaries=True,
+          name=app_name,
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=False,
+          console=True)
+coll = COLLECT(exe,
+               a.binaries,
+               a.zipfiles,
+               a.datas,
+               strip=False,
+               upx=False,
+               upx_exclude=[],
+               name=app_name)

tests/functional/test_unbuffered_stdio.py

@@ -0,0 +1,89 @@
+#-----------------------------------------------------------------------------
+# Copyright (c) 2021, PyInstaller Development Team.
+#
+# Distributed under the terms of the GNU General Public License (version 2
+# or later) with exception for distributing the bootloader.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#
+# SPDX-License-Identifier: (GPL-2.0-or-later WITH Bootloader-exception)
+#-----------------------------------------------------------------------------
+
+"""
+Test for unbuffered stdio (stdout/stderr) mode.
+"""
+
+import os
+import asyncio
+
+import pytest
+
+from PyInstaller.compat import is_py37, is_win
+
+
+@pytest.mark.parametrize('stream_mode', ['binary', 'text'])
+@pytest.mark.parametrize('output_stream', ['stdout', 'stderr'])
+def test_unbuffered_stdio(tmp_path, output_stream, stream_mode,
+                          pyi_builder_spec):
+    # Unbuffered text layer was introduced in Python 3.7
+    if stream_mode == 'text' and not is_py37:
+        pytest.skip("Unbuffered text layer of stdout and stderr streams "
+                    "requires Python 3.7 or later.")
+
+    # Freeze the test program; test_spec() builds the app and runs it,
+    # so explicitly set the number of stars to 0 for this run.
+    pyi_builder_spec.test_spec('pyi_unbuffered_output.spec',
+                               app_args=['--num-stars', '0'])
+
+    # Path to the frozen executable
+    executable = os.path.join(tmp_path, 'dist',
+                              'pyi_unbuffered_output',
+                              'pyi_unbuffered_output')
+
+    # Expected number of stars
+    EXPECTED_STARS = 5
+
+    # Run the test program via asyncio.SubprocessProtocol and monitor
+    # the output
+    class SubprocessDotCounter(asyncio.SubprocessProtocol):
+        def __init__(self, loop, output='stdout'):
+            self.count = 0
+            self.loop = loop
+            # Select stdout vs stderr
+            assert output in {'stdout', 'stderr'}
+            self.out_fd = 1 if output == 'stdout' else 2
+
+        def pipe_data_received(self, fd, data):
+            if fd == self.out_fd:
+                # Treat any data batch that does not end with the *
+                # as irregularity
+                if not data.endswith(b'*'):
+                    return
+                self.count += data.count(b'*')
+
+        def connection_lost(self, exc):
+            self.loop.stop()  # end loop.run_forever()
+
+    # Create event loop
+    if is_win:
+        loop = asyncio.ProactorEventLoop()  # for subprocess' pipes on Windows
+    else:
+        loop = asyncio.SelectorEventLoop()
+    asyncio.set_event_loop(loop)
+
+    counter_proto = SubprocessDotCounter(loop, output=output_stream)
+
+    # Run
+    try:
+        proc = loop.subprocess_exec(lambda: counter_proto,
+                                    executable,
+                                    "--num-stars", str(EXPECTED_STARS),
+                                    "--output-stream", output_stream,
+                                    "--stream-mode", stream_mode)
+        loop.run_until_complete(proc)
+        loop.run_forever()
+    finally:
+        loop.close()
+
+    # Check the number of received stars
+    assert counter_proto.count == EXPECTED_STARS