From fb834764c072890097663ae3b869797402d035be Mon Sep 17 00:00:00 2001 From: mattip Date: Fri, 13 Sep 2019 12:36:27 +0300 Subject: [PATCH 1/4] MAINT: remove the entropy c-extension module --- numpy/random/__init__.py | 1 - numpy/random/entropy.pyx | 155 ----------------------------- numpy/random/mt19937.pyx | 7 +- numpy/random/setup.py | 12 --- numpy/random/src/entropy/entropy.c | 114 --------------------- numpy/random/src/entropy/entropy.h | 14 --- numpy/random/tests/test_smoke.py | 22 +--- 7 files changed, 5 insertions(+), 320 deletions(-) delete mode 100644 numpy/random/entropy.pyx delete mode 100644 numpy/random/src/entropy/entropy.c delete mode 100644 numpy/random/src/entropy/entropy.h diff --git a/numpy/random/__init__.py b/numpy/random/__init__.py index e7eecc5cd2a..f7c2484510f 100644 --- a/numpy/random/__init__.py +++ b/numpy/random/__init__.py @@ -181,7 +181,6 @@ from . import _pickle from . import common from . import bounded_integers -from . import entropy from .mtrand import * from .generator import Generator, default_rng diff --git a/numpy/random/entropy.pyx b/numpy/random/entropy.pyx deleted file mode 100644 index 95bf7c17747..00000000000 --- a/numpy/random/entropy.pyx +++ /dev/null @@ -1,155 +0,0 @@ -cimport numpy as np -import numpy as np - -from libc.stdint cimport uint32_t, uint64_t - -__all__ = ['random_entropy', 'seed_by_array'] - -np.import_array() - -cdef extern from "src/splitmix64/splitmix64.h": - cdef uint64_t splitmix64_next(uint64_t *state) nogil - -cdef extern from "src/entropy/entropy.h": - cdef bint entropy_getbytes(void* dest, size_t size) - cdef bint entropy_fallback_getbytes(void *dest, size_t size) - -cdef Py_ssize_t compute_numel(size): - cdef Py_ssize_t i, n = 1 - if isinstance(size, tuple): - for i in range(len(size)): - n *= size[i] - else: - n = size - return n - - -def seed_by_array(object seed, Py_ssize_t n): - """ - Transforms a seed array into an initial state - - Parameters - ---------- - seed: ndarray, 1d, uint64 - Array to use. If seed is a scalar, promote to array. - n : int - Number of 64-bit unsigned integers required - - Notes - ----- - Uses splitmix64 to perform the transformation - """ - cdef uint64_t seed_copy = 0 - cdef uint64_t[::1] seed_array - cdef uint64_t[::1] initial_state - cdef Py_ssize_t seed_size, iter_bound - cdef int i, loc = 0 - - if hasattr(seed, 'squeeze'): - seed = seed.squeeze() - arr = np.asarray(seed) - if arr.shape == (): - err_msg = 'Scalar seeds must be integers between 0 and 2**64 - 1' - if not np.isreal(arr): - raise TypeError(err_msg) - int_seed = int(seed) - if int_seed != seed: - raise TypeError(err_msg) - if int_seed < 0 or int_seed > 2**64 - 1: - raise ValueError(err_msg) - seed_array = np.array([int_seed], dtype=np.uint64) - elif issubclass(arr.dtype.type, np.inexact): - raise TypeError('seed array must be integers') - else: - err_msg = "Seed values must be integers between 0 and 2**64 - 1" - obj = np.asarray(seed).astype(np.object) - if obj.ndim != 1: - raise ValueError('Array-valued seeds must be 1-dimensional') - if not np.isreal(obj).all(): - raise TypeError(err_msg) - if ((obj > int(2**64 - 1)) | (obj < 0)).any(): - raise ValueError(err_msg) - try: - obj_int = obj.astype(np.uint64, casting='unsafe') - except ValueError: - raise ValueError(err_msg) - if not (obj == obj_int).all(): - raise TypeError(err_msg) - seed_array = obj_int - - seed_size = seed_array.shape[0] - iter_bound = n if n > seed_size else seed_size - - initial_state = np.empty(n, dtype=np.uint64) - for i in range(iter_bound): - if i < seed_size: - seed_copy ^= seed_array[i] - initial_state[loc] = splitmix64_next(&seed_copy) - loc += 1 - if loc == n: - loc = 0 - - return np.array(initial_state) - - -def random_entropy(size=None, source='system'): - """ - random_entropy(size=None, source='system') - - Read entropy from the system cryptographic provider - - Parameters - ---------- - size : int or tuple of ints, optional - Output shape. If the given shape is, e.g., ``(m, n, k)``, then - ``m * n * k`` samples are drawn. Default is None, in which case a - single value is returned. - source : str {'system', 'fallback'} - Source of entropy. 'system' uses system cryptographic pool. - 'fallback' uses a hash of the time and process id. - - Returns - ------- - entropy : scalar or array - Entropy bits in 32-bit unsigned integers. A scalar is returned if size - is `None`. - - Notes - ----- - On Unix-like machines, reads from ``/dev/urandom``. On Windows machines - reads from the RSA algorithm provided by the cryptographic service - provider. - - This function reads from the system entropy pool and so samples are - not reproducible. In particular, it does *NOT* make use of a - BitGenerator, and so ``seed`` and setting ``state`` have no - effect. - - Raises RuntimeError if the command fails. - """ - cdef bint success = True - cdef Py_ssize_t n = 0 - cdef uint32_t random = 0 - cdef uint32_t [:] randoms - - if source not in ('system', 'fallback'): - raise ValueError('Unknown value in source.') - - if size is None: - if source == 'system': - success = entropy_getbytes(&random, 4) - else: - success = entropy_fallback_getbytes(&random, 4) - else: - n = compute_numel(size) - randoms = np.zeros(n, dtype=np.uint32) - if source == 'system': - success = entropy_getbytes((&randoms[0]), 4 * n) - else: - success = entropy_fallback_getbytes((&randoms[0]), 4 * n) - if not success: - raise RuntimeError('Unable to read from system cryptographic provider') - - if n == 0: - return random - return np.asarray(randoms).reshape(size) diff --git a/numpy/random/mt19937.pyx b/numpy/random/mt19937.pyx index 49c3622f572..7d0f6cd2252 100644 --- a/numpy/random/mt19937.pyx +++ b/numpy/random/mt19937.pyx @@ -5,7 +5,6 @@ cimport numpy as np from .common cimport * from .bit_generator cimport BitGenerator, SeedSequence -from .entropy import random_entropy __all__ = ['MT19937'] @@ -156,7 +155,8 @@ cdef class MT19937(BitGenerator): Random seed initializing the pseudo-random number generator. Can be an integer in [0, 2**32-1], array of integers in [0, 2**32-1], a `SeedSequence, or ``None``. If `seed` - is ``None``, then sample entropy for a seed. + is ``None``, then fresh, unpredictable entropy will be pulled from + the OS. Raises ------ @@ -167,7 +167,8 @@ cdef class MT19937(BitGenerator): with self.lock: try: if seed is None: - val = random_entropy(RK_STATE_LEN) + seed = SeedSequence() + val = seed.generate_state(RK_STATE_LEN) # MSB is 1; assuring non-zero initial array self.rng_state.key[0] = 0x80000000UL for i in range(1, RK_STATE_LEN): diff --git a/numpy/random/setup.py b/numpy/random/setup.py index a820d326e0e..f0ebe331fa7 100644 --- a/numpy/random/setup.py +++ b/numpy/random/setup.py @@ -61,18 +61,6 @@ def generate_libraries(ext, build_dir): # One can force emulated 128-bit arithmetic if one wants. #PCG64_DEFS += [('PCG_FORCE_EMULATED_128BIT_MATH', '1')] - config.add_extension('entropy', - sources=['entropy.c', 'src/entropy/entropy.c'] + - [generate_libraries], - libraries=EXTRA_LIBRARIES, - extra_compile_args=EXTRA_COMPILE_ARGS, - extra_link_args=EXTRA_LINK_ARGS, - depends=[join('src', 'splitmix64', 'splitmix.h'), - join('src', 'entropy', 'entropy.h'), - 'entropy.pyx', - ], - define_macros=defs, - ) for gen in ['mt19937']: # gen.pyx, src/gen/gen.c, src/gen/gen-jump.c config.add_extension(gen, diff --git a/numpy/random/src/entropy/entropy.c b/numpy/random/src/entropy/entropy.c deleted file mode 100644 index eaca37a9ce5..00000000000 --- a/numpy/random/src/entropy/entropy.c +++ /dev/null @@ -1,114 +0,0 @@ -#include -#include -#include -#include - -#include "entropy.h" -#ifdef _WIN32 -/* Windows */ -#include -#include -#include - -#include -#else -/* Unix */ -#include -#include -#include -#include -#endif - -bool entropy_getbytes(void *dest, size_t size) { -#ifndef _WIN32 - - int fd = open("/dev/urandom", O_RDONLY); - if (fd < 0) - return false; - ssize_t sz = read(fd, dest, size); - if ((sz < 0) || ((size_t)sz < size)) - return false; - return close(fd) == 0; - -#else - - HCRYPTPROV hCryptProv; - BOOL done; - - if (!CryptAcquireContext(&hCryptProv, NULL, NULL, PROV_RSA_FULL, - CRYPT_VERIFYCONTEXT) || - !hCryptProv) { - return true; - } - done = CryptGenRandom(hCryptProv, (DWORD)size, (unsigned char *)dest); - CryptReleaseContext(hCryptProv, 0); - if (!done) { - return false; - } - - return true; -#endif -} - -/* Thomas Wang 32/64 bits integer hash function */ -uint32_t entropy_hash_32(uint32_t key) { - key += ~(key << 15); - key ^= (key >> 10); - key += (key << 3); - key ^= (key >> 6); - key += ~(key << 11); - key ^= (key >> 16); - return key; -} - -uint64_t entropy_hash_64(uint64_t key) { - key = (~key) + (key << 21); // key = (key << 21) - key - 1; - key = key ^ (key >> 24); - key = (key + (key << 3)) + (key << 8); // key * 265 - key = key ^ (key >> 14); - key = (key + (key << 2)) + (key << 4); // key * 21 - key = key ^ (key >> 28); - key = key + (key << 31); - return key; -} - -uint32_t entropy_randombytes(void) { - -#ifndef _WIN32 - struct timeval tv; - gettimeofday(&tv, NULL); - return entropy_hash_32(getpid()) ^ entropy_hash_32(tv.tv_sec) ^ - entropy_hash_32(tv.tv_usec) ^ entropy_hash_32(clock()); -#else - uint32_t out = 0; - int64_t counter; - struct _timeb tv; - _ftime_s(&tv); - out = entropy_hash_32(GetCurrentProcessId()) ^ - entropy_hash_32((uint32_t)tv.time) ^ entropy_hash_32(tv.millitm) ^ - entropy_hash_32(clock()); - if (QueryPerformanceCounter((LARGE_INTEGER *)&counter) != 0) - out ^= entropy_hash_32((uint32_t)(counter & 0xffffffff)); - return out; -#endif -} - -bool entropy_fallback_getbytes(void *dest, size_t size) { - int hashes = (int)size; - uint32_t *hash = malloc(hashes * sizeof(uint32_t)); - int i; - for (i = 0; i < hashes; i++) { - hash[i] = entropy_randombytes(); - } - memcpy(dest, (void *)hash, size); - free(hash); - return true; -} - -void entropy_fill(void *dest, size_t size) { - bool success; - success = entropy_getbytes(dest, size); - if (!success) { - entropy_fallback_getbytes(dest, size); - } -} diff --git a/numpy/random/src/entropy/entropy.h b/numpy/random/src/entropy/entropy.h deleted file mode 100644 index f00caf61d9d..00000000000 --- a/numpy/random/src/entropy/entropy.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef _RANDOMDGEN__ENTROPY_H_ -#define _RANDOMDGEN__ENTROPY_H_ - -#include -#include -#include - -extern void entropy_fill(void *dest, size_t size); - -extern bool entropy_getbytes(void *dest, size_t size); - -extern bool entropy_fallback_getbytes(void *dest, size_t size); - -#endif diff --git a/numpy/random/tests/test_smoke.py b/numpy/random/tests/test_smoke.py index 84d261e5e44..6e641b5f438 100644 --- a/numpy/random/tests/test_smoke.py +++ b/numpy/random/tests/test_smoke.py @@ -5,7 +5,7 @@ import numpy as np import pytest from numpy.testing import assert_equal, assert_, assert_array_equal -from numpy.random import (Generator, MT19937, PCG64, Philox, SFC64, entropy) +from numpy.random import (Generator, MT19937, PCG64, Philox, SFC64) @pytest.fixture(scope='module', params=(np.bool, np.int8, np.int16, np.int32, np.int64, @@ -806,23 +806,3 @@ def test_seed(self): np.random.default_rng(-1) with pytest.raises(ValueError): np.random.default_rng([12345, -1]) - - -class TestEntropy(object): - def test_entropy(self): - e1 = entropy.random_entropy() - e2 = entropy.random_entropy() - assert_((e1 != e2)) - e1 = entropy.random_entropy(10) - e2 = entropy.random_entropy(10) - assert_((e1 != e2).all()) - e1 = entropy.random_entropy(10, source='system') - e2 = entropy.random_entropy(10, source='system') - assert_((e1 != e2).all()) - - def test_fallback(self): - e1 = entropy.random_entropy(source='fallback') - time.sleep(0.1) - e2 = entropy.random_entropy(source='fallback') - assert_((e1 != e2)) - From 0bdbfa79c0eb237c773bc5fda8c80013a18347dd Mon Sep 17 00:00:00 2001 From: mattip Date: Fri, 13 Sep 2019 12:41:53 +0300 Subject: [PATCH 2/4] DOC: add release note --- doc/release/upcoming_changes/14498.changes.rst | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 doc/release/upcoming_changes/14498.changes.rst diff --git a/doc/release/upcoming_changes/14498.changes.rst b/doc/release/upcoming_changes/14498.changes.rst new file mode 100644 index 00000000000..fd784e28971 --- /dev/null +++ b/doc/release/upcoming_changes/14498.changes.rst @@ -0,0 +1,7 @@ +Remove ``numpy.random.entropy`` without a deprecation +----------------------------------------------------- + +``numpy.random.entropy`` was added to the `numpy.random` namespace in 1.17.0. +It was meant to be a private c-extension module, but was exposed as public. +It has been replaced by `numpy.random.SeedSequence` so the module was +completely removed. From 7d65ab4342ca17bf7771dde431830e7689980eef Mon Sep 17 00:00:00 2001 From: mattip Date: Sat, 14 Sep 2019 11:33:58 +0300 Subject: [PATCH 3/4] DOC: removed random.entropy from documentation --- doc/source/reference/random/entropy.rst | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 doc/source/reference/random/entropy.rst diff --git a/doc/source/reference/random/entropy.rst b/doc/source/reference/random/entropy.rst deleted file mode 100644 index 0664da6f9a7..00000000000 --- a/doc/source/reference/random/entropy.rst +++ /dev/null @@ -1,6 +0,0 @@ -System Entropy -============== - -.. module:: numpy.random.entropy - -.. autofunction:: random_entropy From 61deb98c40c553d49d5186c8c7079792d82633b2 Mon Sep 17 00:00:00 2001 From: mattip Date: Sat, 14 Sep 2019 13:20:34 +0300 Subject: [PATCH 4/4] DOC: remove random.entropy from docs --- doc/source/reference/random/index.rst | 4 ---- doc/source/reference/random/new-or-different.rst | 3 --- 2 files changed, 7 deletions(-) diff --git a/doc/source/reference/random/index.rst b/doc/source/reference/random/index.rst index 0b7a0bfad78..b0283f3a7d6 100644 --- a/doc/source/reference/random/index.rst +++ b/doc/source/reference/random/index.rst @@ -151,9 +151,6 @@ What's New or Different select distributions * Optional ``out`` argument that allows existing arrays to be filled for select distributions -* `~entropy.random_entropy` provides access to the system - source of randomness that is used in cryptographic applications (e.g., - ``/dev/urandom`` on Unix). * All BitGenerators can produce doubles, uint64s and uint32s via CTypes (`~.PCG64.ctypes`) and CFFI (`~.PCG64.cffi`). This allows the bit generators to be used in numba. @@ -203,7 +200,6 @@ Features new-or-different Comparing Performance extending - Reading System Entropy Original Source ~~~~~~~~~~~~~~~ diff --git a/doc/source/reference/random/new-or-different.rst b/doc/source/reference/random/new-or-different.rst index 5442f46c9e4..c8815f98f06 100644 --- a/doc/source/reference/random/new-or-different.rst +++ b/doc/source/reference/random/new-or-different.rst @@ -45,9 +45,6 @@ Feature Older Equivalent Notes And in more detail: -* `~.entropy.random_entropy` provides access to the system - source of randomness that is used in cryptographic applications (e.g., - ``/dev/urandom`` on Unix). * Simulate from the complex normal distribution (`~.Generator.complex_normal`) * The normal, exponential and gamma generators use 256-step Ziggurat