From 5559f49095c969f34fb3c58d7f08e2a85abdb367 Mon Sep 17 00:00:00 2001 From: Allan Simon Date: Tue, 5 Nov 2019 21:35:41 +0100 Subject: [PATCH] permits black to run in AWS Lambda: AWS Lambda and some other virtualized environment may not permit access to /dev/shm on Linux and as such, trying to use ProcessPoolExecutor will fail. As using parallelism is only a 'nice to have' feature of black, if it fails we gracefully fallback to a monoprocess implementation, which permits black to finish normally. --- black.py | 14 +++++++++++--- tests/test_black.py | 21 +++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/black.py b/black.py index 68c0052fdfe..a0390755564 100644 --- a/black.py +++ b/black.py @@ -541,7 +541,14 @@ def reformat_many( if sys.platform == "win32": # Work around https://bugs.python.org/issue26903 worker_count = min(worker_count, 61) - executor = ProcessPoolExecutor(max_workers=worker_count) + try: + executor = ProcessPoolExecutor(max_workers=worker_count) + except OSError: + # we arrive here if the underlying system does not support multi-processing + # like in AWS Lambda, in which case we gracefully fallback to the default + # mono-process Executor by using None + executor = None + try: loop.run_until_complete( schedule_formatting( @@ -556,7 +563,8 @@ def reformat_many( ) finally: shutdown(loop) - executor.shutdown() + if executor is not None: + executor.shutdown() async def schedule_formatting( @@ -566,7 +574,7 @@ async def schedule_formatting( mode: FileMode, report: "Report", loop: asyncio.AbstractEventLoop, - executor: Executor, + executor: Optional[Executor], ) -> None: """Run formatting of `sources` in parallel using the provided `executor`. diff --git a/tests/test_black.py b/tests/test_black.py index 40bde361eeb..d9c44728449 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1159,6 +1159,27 @@ def test_cache_multiple_files(self) -> None: self.assertIn(one, cache) self.assertIn(two, cache) + @patch("black.ProcessPoolExecutor", autospec=True) + def test_works_in_mono_process_only_environment(self, mock_executor) -> None: + mock_executor.side_effect = OSError() + mode = black.FileMode() + with cache_dir() as workspace: + one = (workspace / "one.py").resolve() + with one.open("w") as fobj: + fobj.write("print('hello')") + two = (workspace / "two.py").resolve() + with two.open("w") as fobj: + fobj.write("print('hello')") + black.write_cache({}, [one], mode) + self.invokeBlack([str(workspace)]) + with one.open("r") as fobj: + self.assertEqual(fobj.read(), "print('hello')") + with two.open("r") as fobj: + self.assertEqual(fobj.read(), 'print("hello")\n') + cache = black.read_cache(mode) + self.assertIn(one, cache) + self.assertIn(two, cache) + def test_no_cache_when_writeback_diff(self) -> None: mode = black.FileMode() with cache_dir() as workspace: