Release 1.1.1 (#1352)

Co-authored-by: Adrin Jalali <adrin.jalali@gmail.com>

CHANGES.rst

@@ -4,6 +4,12 @@ Latest changes
 Development version
 -------------------
 
+Release 1.1.1
+
+- Fix a security issue where ``eval(pre_dispatch)`` could potentially run
+  arbitrary code. Now only basic numerics are supported.
+  https://github.com/joblib/joblib/pull/1327
+
 Release 1.1.0
 --------------
 

azure-pipelines.yml

@@ -70,16 +70,16 @@ jobs:
         PYTHON_VERSION: "3.6"
 
       windows_py38:
-        imageName: "vs2017-win2016"
+        imageName: "windows-latest"
         PYTHON_VERSION: "3.8"
         EXTRA_CONDA_PACKAGES: "numpy=1.18"
 
       macos_py38:
-        imageName: "macos-10.14"
+        imageName: "macos-latest"
         PYTHON_VERSION: "3.8"
         EXTRA_CONDA_PACKAGES: "numpy=1.18"
       macos_py36_no_numpy:
-        imageName: "macos-10.14"
+        imageName: "macos-latest"
         PYTHON_VERSION: "3.6"
 
   variables:

joblib/__init__.py

@@ -106,7 +106,7 @@
 # Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
 # 'X.Y.dev0' is the canonical version of 'X.Y.dev'
 #
-__version__ = '1.1.0'
+__version__ = '1.1.1'
 
 
 import os

joblib/_utils.py

@@ -0,0 +1,44 @@
+# Adapted from https://stackoverflow.com/a/9558001/2536294
+
+import ast
+import operator as op
+
+# supported operators
+operators = {
+    ast.Add: op.add,
+    ast.Sub: op.sub,
+    ast.Mult: op.mul,
+    ast.Div: op.truediv,
+    ast.FloorDiv: op.floordiv,
+    ast.Mod: op.mod,
+    ast.Pow: op.pow,
+    ast.USub: op.neg,
+}
+
+
+def eval_expr(expr):
+    """
+    >>> eval_expr('2*6')
+    12
+    >>> eval_expr('2**6')
+    64
+    >>> eval_expr('1 + 2*3**(4) / (6 + -7)')
+    -161.0
+    """
+    try:
+        return eval_(ast.parse(expr, mode="eval").body)
+    except (TypeError, SyntaxError, KeyError) as e:
+        raise ValueError(
+            f"{expr!r} is not a valid or supported arithmetic expression."
+        ) from e
+
+
+def eval_(node):
+    if isinstance(node, ast.Num):  # <number>
+        return node.n
+    elif isinstance(node, ast.BinOp):  # <left> <operator> <right>
+        return operators[type(node.op)](eval_(node.left), eval_(node.right))
+    elif isinstance(node, ast.UnaryOp):  # <operator> <operand> e.g., -1
+        return operators[type(node.op)](eval_(node.operand))
+    else:
+        raise TypeError(node)

joblib/parallel.py

@@ -28,6 +28,7 @@
                                  LokyBackend)
 from .externals.cloudpickle import dumps, loads
 from .externals import loky
+from ._utils import eval_expr
 
 # Make sure that those two classes are part of the public joblib.parallel API
 # so that 3rd party backend implementers can import them from here.
@@ -477,7 +478,9 @@ class Parallel(Logger):
         pre_dispatch: {'all', integer, or expression, as in '3*n_jobs'}
             The number of batches (of tasks) to be pre-dispatched.
             Default is '2*n_jobs'. When batch_size="auto" this is reasonable
-            default and the workers should never starve.
+            default and the workers should never starve. Note that only basic
+            arithmetics are allowed here and no modules can be used in this
+            expression.
         batch_size: int or 'auto', default: 'auto'
             The number of atomic tasks to dispatch at once to each
             worker. When individual evaluations are very fast, dispatching
@@ -1012,7 +1015,9 @@ def _batched_calls_reducer_callback():
         else:
             self._original_iterator = iterator
             if hasattr(pre_dispatch, 'endswith'):
-                pre_dispatch = eval(pre_dispatch)
+                pre_dispatch = eval_expr(
+                    pre_dispatch.replace("n_jobs", str(n_jobs))
+                )
             self._pre_dispatch_amount = pre_dispatch = int(pre_dispatch)
 
             # The main thread will consume the first pre_dispatch items and

joblib/test/test_utils.py

@@ -0,0 +1,27 @@
+import pytest
+
+from joblib._utils import eval_expr
+
+
+@pytest.mark.parametrize(
+    "expr",
+    ["exec('import os')", "print(1)", "import os", "1+1; import os", "1^1"],
+)
+def test_eval_expr_invalid(expr):
+    with pytest.raises(
+        ValueError, match="is not a valid or supported arithmetic"
+    ):
+        eval_expr(expr)
+
+
+@pytest.mark.parametrize(
+    "expr, result",
+    [
+        ("2*6", 12),
+        ("2**6", 64),
+        ("1 + 2*3**(4) / (6 + -7)", -161.0),
+        ("(20 // 3) % 5", 1),
+    ],
+)
+def test_eval_expr_valid(expr, result):
+    assert eval_expr(expr) == result