From 5fab50183a6e003eda7e8f2229fcd488efd77a87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20du=20Boisberranger?= <34657725+jeremiedbb@users.noreply.github.com> Date: Tue, 26 Nov 2019 17:37:26 +0100 Subject: [PATCH] MNT Include all pxd files in the package (#15626) * include all pxd files in package * _libsvm _liblinear to pxi + add a test * simplify stuff + use pathlib * cln * more cln --- maint_tools/check_pxd_in_installation.py | 59 +++++++++++++++++++ setup.py | 1 + .../svm/{_liblinear.pxd => _liblinear.pxi} | 5 +- sklearn/svm/_liblinear.pyx | 2 + sklearn/svm/{_libsvm.pxd => _libsvm.pxi} | 2 - sklearn/svm/_libsvm.pyx | 2 + sklearn/tree/setup.py | 4 -- sklearn/utils/_weight_vector.pxd | 7 --- 8 files changed, 66 insertions(+), 16 deletions(-) create mode 100644 maint_tools/check_pxd_in_installation.py rename sklearn/svm/{_liblinear.pxd => _liblinear.pxi} (98%) rename sklearn/svm/{_libsvm.pxd => _libsvm.pxi} (99%) diff --git a/maint_tools/check_pxd_in_installation.py b/maint_tools/check_pxd_in_installation.py new file mode 100644 index 0000000000000..83c4b706294ad --- /dev/null +++ b/maint_tools/check_pxd_in_installation.py @@ -0,0 +1,59 @@ +"""Utility for testing presence and usability of .pxd files in the installation + +Usage: +------ +python check_pxd_in_installation.py path/to/install_dir/of/scikit-learn +""" + +import os +import sys +import pathlib +import tempfile +import textwrap +import subprocess + + +sklearn_dir = pathlib.Path(sys.argv[1]) +pxd_files = list(sklearn_dir.glob("**/*.pxd")) + +print("> Found pxd files:") +for pxd_file in pxd_files: + print(' -', pxd_file) + +print("\n> Trying to compile a cython extension cimporting all corresponding " + "modules\n") +with tempfile.TemporaryDirectory() as tmpdir: + tmpdir = pathlib.Path(tmpdir) + # A cython test file which cimports all modules corresponding to found + # pxd files. + # e.g. sklearn/tree/_utils.pxd becomes `cimport sklearn.tree._utils` + with open(tmpdir / 'tst.pyx', 'w') as f: + for pxd_file in pxd_files: + to_import = str(pxd_file.relative_to(sklearn_dir)) + to_import = to_import.replace(os.path.sep, '.') + to_import = to_import.replace('.pxd', '') + f.write('cimport sklearn.' + to_import + '\n') + + # A basic setup file to build the test file. + # We set the language to c++ and we use numpy.get_include() because + # some modules require it. + with open(tmpdir / 'setup_tst.py', 'w') as f: + f.write(textwrap.dedent( + """ + from distutils.core import setup + from distutils.extension import Extension + from Cython.Build import cythonize + import numpy + + extensions = [Extension("tst", + sources=["tst.pyx"], + language="c++", + include_dirs=[numpy.get_include()])] + + setup(ext_modules=cythonize(extensions)) + """)) + + subprocess.run(["python", "setup_tst.py", "build_ext", "-i"], + check=True, cwd=tmpdir) + + print("\n> Compilation succeeded !") diff --git a/setup.py b/setup.py index 3ed5c786a17c3..80b1b8ecc9391 100755 --- a/setup.py +++ b/setup.py @@ -259,6 +259,7 @@ def setup_package(): 'scipy>={}'.format(SCIPY_MIN_VERSION), 'joblib>={}'.format(JOBLIB_MIN_VERSION) ], + package_data={'': ['*.pxd']}, **extra_setuptools_args) if len(sys.argv) == 1 or ( diff --git a/sklearn/svm/_liblinear.pxd b/sklearn/svm/_liblinear.pxi similarity index 98% rename from sklearn/svm/_liblinear.pxd rename to sklearn/svm/_liblinear.pxi index 0f10e54a532fe..148bf694dab4f 100644 --- a/sklearn/svm/_liblinear.pxd +++ b/sklearn/svm/_liblinear.pxi @@ -1,6 +1,3 @@ -cimport numpy as np - - cdef extern from "_cython_blas_helpers.h": ctypedef double (*dot_func)(int, double*, int, double*, int) ctypedef void (*axpy_func)(int, double, double*, int, double*, int) @@ -12,6 +9,7 @@ cdef extern from "_cython_blas_helpers.h": scal_func scal nrm2_func nrm2 + cdef extern from "linear.h": cdef struct feature_node cdef struct problem @@ -28,6 +26,7 @@ cdef extern from "linear.h": void free_and_destroy_model (model **) void destroy_param (parameter *) + cdef extern from "liblinear_helper.c": void copy_w(void *, model *, int) parameter *set_parameter(int, double, double, int, char *, char *, int, int, double) diff --git a/sklearn/svm/_liblinear.pyx b/sklearn/svm/_liblinear.pyx index 2f042748d94a0..9dd15e0716c7f 100644 --- a/sklearn/svm/_liblinear.pyx +++ b/sklearn/svm/_liblinear.pyx @@ -9,6 +9,8 @@ cimport numpy as np from ..utils._cython_blas cimport _dot, _axpy, _scal, _nrm2 +include "_liblinear.pxi" + np.import_array() diff --git a/sklearn/svm/_libsvm.pxd b/sklearn/svm/_libsvm.pxi similarity index 99% rename from sklearn/svm/_libsvm.pxd rename to sklearn/svm/_libsvm.pxi index 2664a335a372f..a3c8f1c33dd1e 100644 --- a/sklearn/svm/_libsvm.pxd +++ b/sklearn/svm/_libsvm.pxi @@ -1,5 +1,3 @@ -cimport numpy as np - ################################################################################ # Includes diff --git a/sklearn/svm/_libsvm.pyx b/sklearn/svm/_libsvm.pyx index 8f8e9f7465823..079a791fef3b6 100644 --- a/sklearn/svm/_libsvm.pyx +++ b/sklearn/svm/_libsvm.pyx @@ -35,6 +35,8 @@ import numpy as np cimport numpy as np from libc.stdlib cimport free +include "_libsvm.pxi" + cdef extern from *: ctypedef struct svm_parameter: pass diff --git a/sklearn/tree/setup.py b/sklearn/tree/setup.py index 2b9819795b74b..079ae9d869075 100644 --- a/sklearn/tree/setup.py +++ b/sklearn/tree/setup.py @@ -31,10 +31,6 @@ def configuration(parent_package="", top_path=None): extra_compile_args=["-O3"]) config.add_subpackage("tests") - config.add_data_files("_criterion.pxd") - config.add_data_files("_splitter.pxd") - config.add_data_files("_tree.pxd") - config.add_data_files("_utils.pxd") return config diff --git a/sklearn/utils/_weight_vector.pxd b/sklearn/utils/_weight_vector.pxd index 1f38bb7e0981f..fc1b47a50ef1f 100644 --- a/sklearn/utils/_weight_vector.pxd +++ b/sklearn/utils/_weight_vector.pxd @@ -1,12 +1,5 @@ """Efficient (dense) parameter vector implementation for linear models. """ -cimport numpy as np - - -cdef extern from "math.h": - cdef extern double sqrt(double x) - - cdef class WeightVector(object): cdef double *w_data_ptr cdef double *aw_data_ptr