Skip to content

Commit

Permalink
permits black to run in AWS Lambda:
Browse files Browse the repository at this point in the history
AWS Lambda and some other virtualized environment may not permit access
to /dev/shm on Linux and as such, trying to use ProcessPoolExecutor will
fail.

As using parallelism is only a 'nice to have' feature of black, if it fails
we gracefully fallback to a monoprocess implementation, which permits black
to finish normally.
  • Loading branch information
allanlegalstart committed Nov 5, 2019
1 parent 9b484d1 commit 5559f49
Show file tree
Hide file tree
Showing 2 changed files with 32 additions and 3 deletions.
14 changes: 11 additions & 3 deletions black.py
Expand Up @@ -541,7 +541,14 @@ def reformat_many(
if sys.platform == "win32":
# Work around https://bugs.python.org/issue26903
worker_count = min(worker_count, 61)
executor = ProcessPoolExecutor(max_workers=worker_count)
try:
executor = ProcessPoolExecutor(max_workers=worker_count)
except OSError:
# we arrive here if the underlying system does not support multi-processing
# like in AWS Lambda, in which case we gracefully fallback to the default
# mono-process Executor by using None
executor = None

try:
loop.run_until_complete(
schedule_formatting(
Expand All @@ -556,7 +563,8 @@ def reformat_many(
)
finally:
shutdown(loop)
executor.shutdown()
if executor is not None:
executor.shutdown()


async def schedule_formatting(
Expand All @@ -566,7 +574,7 @@ async def schedule_formatting(
mode: FileMode,
report: "Report",
loop: asyncio.AbstractEventLoop,
executor: Executor,
executor: Optional[Executor],
) -> None:
"""Run formatting of `sources` in parallel using the provided `executor`.
Expand Down
21 changes: 21 additions & 0 deletions tests/test_black.py
Expand Up @@ -1159,6 +1159,27 @@ def test_cache_multiple_files(self) -> None:
self.assertIn(one, cache)
self.assertIn(two, cache)

@patch("black.ProcessPoolExecutor", autospec=True)
def test_works_in_mono_process_only_environment(self, mock_executor) -> None:
mock_executor.side_effect = OSError()
mode = black.FileMode()
with cache_dir() as workspace:
one = (workspace / "one.py").resolve()
with one.open("w") as fobj:
fobj.write("print('hello')")
two = (workspace / "two.py").resolve()
with two.open("w") as fobj:
fobj.write("print('hello')")
black.write_cache({}, [one], mode)
self.invokeBlack([str(workspace)])
with one.open("r") as fobj:
self.assertEqual(fobj.read(), "print('hello')")
with two.open("r") as fobj:
self.assertEqual(fobj.read(), 'print("hello")\n')
cache = black.read_cache(mode)
self.assertIn(one, cache)
self.assertIn(two, cache)

def test_no_cache_when_writeback_diff(self) -> None:
mode = black.FileMode()
with cache_dir() as workspace:
Expand Down

0 comments on commit 5559f49

Please sign in to comment.