From 589a6219d0c389418d2dc1f4f5fda78f0d67e1a1 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 19 Aug 2023 20:28:00 +0000 Subject: [PATCH 01/64] MAINT: Update to the new layout --- scipy/optimize/_highs/meson.build | 126 +++++++++++++++--------------- 1 file changed, 63 insertions(+), 63 deletions(-) diff --git a/scipy/optimize/_highs/meson.build b/scipy/optimize/_highs/meson.build index 8d701e5e3f67..39d8c16226e9 100644 --- a/scipy/optimize/_highs/meson.build +++ b/scipy/optimize/_highs/meson.build @@ -17,39 +17,39 @@ highs_define_macros = [ basiclu_lib = static_library('basiclu', [ - '../../_lib/highs/src/ipm/basiclu/src/basiclu_factorize.c', - '../../_lib/highs/src/ipm/basiclu/src/basiclu_get_factors.c', - '../../_lib/highs/src/ipm/basiclu/src/basiclu_initialize.c', - '../../_lib/highs/src/ipm/basiclu/src/basiclu_object.c', - '../../_lib/highs/src/ipm/basiclu/src/basiclu_solve_dense.c', - '../../_lib/highs/src/ipm/basiclu/src/basiclu_solve_for_update.c', - '../../_lib/highs/src/ipm/basiclu/src/basiclu_solve_sparse.c', - '../../_lib/highs/src/ipm/basiclu/src/basiclu_update.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_build_factors.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_condest.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_dfs.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_factorize_bump.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_file.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_garbage_perm.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_initialize.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_internal.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_markowitz.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_matrix_norm.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_pivot.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_residual_test.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_setup_bump.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_singletons.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_solve_dense.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_solve_for_update.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_solve_sparse.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_solve_symbolic.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_solve_triangular.c', - '../../_lib/highs/src/ipm/basiclu/src/lu_update.c' + '../../_lib/highs/src/ipm/basiclu/basiclu_factorize.c', + '../../_lib/highs/src/ipm/basiclu/basiclu_get_factors.c', + '../../_lib/highs/src/ipm/basiclu/basiclu_initialize.c', + '../../_lib/highs/src/ipm/basiclu/basiclu_object.c', + '../../_lib/highs/src/ipm/basiclu/basiclu_solve_dense.c', + '../../_lib/highs/src/ipm/basiclu/basiclu_solve_for_update.c', + '../../_lib/highs/src/ipm/basiclu/basiclu_solve_sparse.c', + '../../_lib/highs/src/ipm/basiclu/basiclu_update.c', + '../../_lib/highs/src/ipm/basiclu/lu_build_factors.c', + '../../_lib/highs/src/ipm/basiclu/lu_condest.c', + '../../_lib/highs/src/ipm/basiclu/lu_dfs.c', + '../../_lib/highs/src/ipm/basiclu/lu_factorize_bump.c', + '../../_lib/highs/src/ipm/basiclu/lu_file.c', + '../../_lib/highs/src/ipm/basiclu/lu_garbage_perm.c', + '../../_lib/highs/src/ipm/basiclu/lu_initialize.c', + '../../_lib/highs/src/ipm/basiclu/lu_internal.c', + '../../_lib/highs/src/ipm/basiclu/lu_markowitz.c', + '../../_lib/highs/src/ipm/basiclu/lu_matrix_norm.c', + '../../_lib/highs/src/ipm/basiclu/lu_pivot.c', + '../../_lib/highs/src/ipm/basiclu/lu_residual_test.c', + '../../_lib/highs/src/ipm/basiclu/lu_setup_bump.c', + '../../_lib/highs/src/ipm/basiclu/lu_singletons.c', + '../../_lib/highs/src/ipm/basiclu/lu_solve_dense.c', + '../../_lib/highs/src/ipm/basiclu/lu_solve_for_update.c', + '../../_lib/highs/src/ipm/basiclu/lu_solve_sparse.c', + '../../_lib/highs/src/ipm/basiclu/lu_solve_symbolic.c', + '../../_lib/highs/src/ipm/basiclu/lu_solve_triangular.c', + '../../_lib/highs/src/ipm/basiclu/lu_update.c' ], include_directories: [ 'src', '../../_lib/highs/src', - '../../_lib/highs/src/ipm/basiclu/include' + '../../_lib/highs/src/ipm/basiclu' ], c_args: [Wno_unused_variable, highs_define_macros] ) @@ -66,41 +66,41 @@ highs_flags = [ ipx_lib = static_library('ipx', [ - '../../_lib/highs/src/ipm/ipx/src/basiclu_kernel.cc', - '../../_lib/highs/src/ipm/ipx/src/basiclu_wrapper.cc', - '../../_lib/highs/src/ipm/ipx/src/basis.cc', - '../../_lib/highs/src/ipm/ipx/src/conjugate_residuals.cc', - '../../_lib/highs/src/ipm/ipx/src/control.cc', - '../../_lib/highs/src/ipm/ipx/src/crossover.cc', - '../../_lib/highs/src/ipm/ipx/src/diagonal_precond.cc', - '../../_lib/highs/src/ipm/ipx/src/forrest_tomlin.cc', - '../../_lib/highs/src/ipm/ipx/src/guess_basis.cc', - '../../_lib/highs/src/ipm/ipx/src/indexed_vector.cc', - '../../_lib/highs/src/ipm/ipx/src/info.cc', - '../../_lib/highs/src/ipm/ipx/src/ipm.cc', - '../../_lib/highs/src/ipm/ipx/src/ipx_c.cc', - '../../_lib/highs/src/ipm/ipx/src/iterate.cc', - '../../_lib/highs/src/ipm/ipx/src/kkt_solver.cc', - '../../_lib/highs/src/ipm/ipx/src/kkt_solver_basis.cc', - '../../_lib/highs/src/ipm/ipx/src/kkt_solver_diag.cc', - '../../_lib/highs/src/ipm/ipx/src/linear_operator.cc', - '../../_lib/highs/src/ipm/ipx/src/lp_solver.cc', - '../../_lib/highs/src/ipm/ipx/src/lu_factorization.cc', - '../../_lib/highs/src/ipm/ipx/src/lu_update.cc', - '../../_lib/highs/src/ipm/ipx/src/maxvolume.cc', - '../../_lib/highs/src/ipm/ipx/src/model.cc', - '../../_lib/highs/src/ipm/ipx/src/normal_matrix.cc', - '../../_lib/highs/src/ipm/ipx/src/sparse_matrix.cc', - '../../_lib/highs/src/ipm/ipx/src/sparse_utils.cc', - '../../_lib/highs/src/ipm/ipx/src/splitted_normal_matrix.cc', - '../../_lib/highs/src/ipm/ipx/src/starting_basis.cc', - '../../_lib/highs/src/ipm/ipx/src/symbolic_invert.cc', - '../../_lib/highs/src/ipm/ipx/src/timer.cc', - '../../_lib/highs/src/ipm/ipx/src/utils.cc' + '../../_lib/highs/src/ipm/ipx/basiclu_kernel.cc', + '../../_lib/highs/src/ipm/ipx/basiclu_wrapper.cc', + '../../_lib/highs/src/ipm/ipx/basis.cc', + '../../_lib/highs/src/ipm/ipx/conjugate_residuals.cc', + '../../_lib/highs/src/ipm/ipx/control.cc', + '../../_lib/highs/src/ipm/ipx/crossover.cc', + '../../_lib/highs/src/ipm/ipx/diagonal_precond.cc', + '../../_lib/highs/src/ipm/ipx/forrest_tomlin.cc', + '../../_lib/highs/src/ipm/ipx/guess_basis.cc', + '../../_lib/highs/src/ipm/ipx/indexed_vector.cc', + '../../_lib/highs/src/ipm/ipx/info.cc', + '../../_lib/highs/src/ipm/ipx/ipm.cc', + '../../_lib/highs/src/ipm/ipx/ipx_c.cc', + '../../_lib/highs/src/ipm/ipx/iterate.cc', + '../../_lib/highs/src/ipm/ipx/kkt_solver.cc', + '../../_lib/highs/src/ipm/ipx/kkt_solver_basis.cc', + '../../_lib/highs/src/ipm/ipx/kkt_solver_diag.cc', + '../../_lib/highs/src/ipm/ipx/linear_operator.cc', + '../../_lib/highs/src/ipm/ipx/lp_solver.cc', + '../../_lib/highs/src/ipm/ipx/lu_factorization.cc', + '../../_lib/highs/src/ipm/ipx/lu_update.cc', + '../../_lib/highs/src/ipm/ipx/maxvolume.cc', + '../../_lib/highs/src/ipm/ipx/model.cc', + '../../_lib/highs/src/ipm/ipx/normal_matrix.cc', + '../../_lib/highs/src/ipm/ipx/sparse_matrix.cc', + '../../_lib/highs/src/ipm/ipx/sparse_utils.cc', + '../../_lib/highs/src/ipm/ipx/splitted_normal_matrix.cc', + '../../_lib/highs/src/ipm/ipx/starting_basis.cc', + '../../_lib/highs/src/ipm/ipx/symbolic_invert.cc', + '../../_lib/highs/src/ipm/ipx/timer.cc', + '../../_lib/highs/src/ipm/ipx/utils.cc' ], include_directories: [ - '../../_lib/highs/src/ipm/ipx/include/', - '../../_lib/highs/src/ipm/basiclu/include/', + '../../_lib/highs/src/ipm/ipx/', + '../../_lib/highs/src/ipm/basiclu/', '../../_lib/highs/src/', '../../_lib/highs/extern/', 'cython/src/' @@ -218,7 +218,7 @@ highs_lib = static_library('highs', '../../_lib/highs/extern/', '../../_lib/highs/src/', '../../_lib/highs/src/io/', - '../../_lib/highs/src/ipm/ipx/include/', + '../../_lib/highs/src/ipm/ipx/', '../../_lib/highs/src/lp_data/', '../../_lib/highs/src/util/', ], From 5d5fe61b01f00ec0db624edf5dbeac998ef024b1 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 3 Sep 2023 15:12:15 +0000 Subject: [PATCH 02/64] MAINT: Delete all cython related cruft [highs] ENH: Rework _linprog_highs to use highspy Still needs a wrapper around Highs() MAINT: Refactor slightly and add commit credit Co-authored-by: mckib2 ENH: Use highspy exclusively TST: Almost pass everything MAINT,TMP: Use my highs until merge --- .gitmodules | 2 +- scipy/_lib/highs | 2 +- scipy/optimize/_highs/_highs_wrapper.py | 212 +++++ scipy/optimize/_highs/cython/__init__.py | 0 scipy/optimize/_highs/cython/src/HConfig.h | 0 scipy/optimize/_highs/cython/src/HConst.pxd | 106 --- scipy/optimize/_highs/cython/src/Highs.pxd | 56 -- scipy/optimize/_highs/cython/src/HighsIO.pxd | 20 - .../optimize/_highs/cython/src/HighsInfo.pxd | 22 - scipy/optimize/_highs/cython/src/HighsLp.pxd | 46 -- .../_highs/cython/src/HighsLpUtils.pxd | 9 - .../_highs/cython/src/HighsModelUtils.pxd | 10 - .../_highs/cython/src/HighsOptions.pxd | 110 --- .../_highs/cython/src/HighsRuntimeOptions.pxd | 9 - .../_highs/cython/src/HighsSparseMatrix.pxd | 15 - .../_highs/cython/src/HighsStatus.pxd | 12 - .../_highs/cython/src/SimplexConst.pxd | 95 --- scipy/optimize/_highs/cython/src/__init__.py | 0 .../_highs/cython/src/_highs_constants.pyx | 117 --- .../_highs/cython/src/_highs_wrapper.pyx | 736 ------------------ .../_highs/cython/src/highs_c_api.pxd | 7 - scipy/optimize/_highs/meson.build | 564 +++++++------- scipy/optimize/_highs/src/libhighs_export.h | 50 -- scipy/optimize/_linprog_highs.py | 184 ++--- scipy/optimize/_milp.py | 7 +- scipy/optimize/tests/test_linprog.py | 50 +- scipy/optimize/tests/test_milp.py | 26 +- 27 files changed, 649 insertions(+), 1818 deletions(-) create mode 100644 scipy/optimize/_highs/_highs_wrapper.py delete mode 100644 scipy/optimize/_highs/cython/__init__.py delete mode 100644 scipy/optimize/_highs/cython/src/HConfig.h delete mode 100644 scipy/optimize/_highs/cython/src/HConst.pxd delete mode 100644 scipy/optimize/_highs/cython/src/Highs.pxd delete mode 100644 scipy/optimize/_highs/cython/src/HighsIO.pxd delete mode 100644 scipy/optimize/_highs/cython/src/HighsInfo.pxd delete mode 100644 scipy/optimize/_highs/cython/src/HighsLp.pxd delete mode 100644 scipy/optimize/_highs/cython/src/HighsLpUtils.pxd delete mode 100644 scipy/optimize/_highs/cython/src/HighsModelUtils.pxd delete mode 100644 scipy/optimize/_highs/cython/src/HighsOptions.pxd delete mode 100644 scipy/optimize/_highs/cython/src/HighsRuntimeOptions.pxd delete mode 100644 scipy/optimize/_highs/cython/src/HighsSparseMatrix.pxd delete mode 100644 scipy/optimize/_highs/cython/src/HighsStatus.pxd delete mode 100644 scipy/optimize/_highs/cython/src/SimplexConst.pxd delete mode 100644 scipy/optimize/_highs/cython/src/__init__.py delete mode 100644 scipy/optimize/_highs/cython/src/_highs_constants.pyx delete mode 100644 scipy/optimize/_highs/cython/src/_highs_wrapper.pyx delete mode 100644 scipy/optimize/_highs/cython/src/highs_c_api.pxd delete mode 100644 scipy/optimize/_highs/src/libhighs_export.h diff --git a/.gitmodules b/.gitmodules index b6920b8714b2..6bcfddf6bc02 100644 --- a/.gitmodules +++ b/.gitmodules @@ -11,7 +11,7 @@ shallow = true [submodule "HiGHS"] path = scipy/_lib/highs - url = https://github.com/scipy/highs + url = https://github.com/HaoZeke/highs shallow = true [submodule "scipy/_lib/boost_math"] path = scipy/_lib/boost_math diff --git a/scipy/_lib/highs b/scipy/_lib/highs index 4a122958a82e..090453608adb 160000 --- a/scipy/_lib/highs +++ b/scipy/_lib/highs @@ -1 +1 @@ -Subproject commit 4a122958a82e67e725d08153e099efe4dad099a2 +Subproject commit 090453608adb6a1f14dbee01dfd117b80aa8938a diff --git a/scipy/optimize/_highs/_highs_wrapper.py b/scipy/optimize/_highs/_highs_wrapper.py new file mode 100644 index 000000000000..a67e124911e2 --- /dev/null +++ b/scipy/optimize/_highs/_highs_wrapper.py @@ -0,0 +1,212 @@ +from warnings import warn + +import numpy as np +from scipy.optimize._highs import highs_bindings as hspy # type: ignore[attr-defined] +from scipy.optimize._highs import _highs_options as hopt # type: ignore[attr-defined] +from scipy.optimize import OptimizeWarning + + +def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, options): + numcol = c.size + numrow = rhs.size + isMip = integrality is not None and np.sum(integrality) > 0 + + # default "null" return values + res = { + "x": None, + "fun": None, + } + + # Fill up a HighsLp object + lp = hspy.HighsLp() + lp.num_col_ = numcol + lp.num_row_ = numrow + lp.a_matrix_.num_col_ = numcol + lp.a_matrix_.num_row_ = numrow + lp.a_matrix_.format_ = hspy.MatrixFormat.kColwise + lp.col_cost_ = c + lp.col_lower_ = lb + lp.col_upper_ = ub + lp.row_lower_ = lhs + lp.row_upper_ = rhs + lp.a_matrix_.start_ = indptr + lp.a_matrix_.index_ = indices + lp.a_matrix_.value_ = data + if integrality.size > 0: + lp.integrality_ = [hspy.HighsVarType(i) for i in integrality] + + # Make a Highs object and pass it everything + highs = hspy.Highs() + highs_options = hspy.HighsOptions() + for key, val in options.items(): + # handle filtering of unsupported and default options + if val is None or key in ("sense",): + continue + + # ask for the option type + opt_type = hopt.get_option_type(key) + if -1 == opt_type: + warn(f"Unrecognized options detected: {dict({key: val})}", OptimizeWarning) + continue + else: + if key in ("presolve", "parallel"): + # handle fake bools (require bool -> str conversions) + if isinstance(val, bool): + val = "on" if val else "off" + else: + warn(f'Option f"{key}" is "{val}", but only True or False is ' + f'allowed. Using default.', OptimizeWarning) + continue + opt_type = hspy.HighsOptionType(opt_type) + status, msg = check_option(highs, key, val) + # { + # hspy.HighsOptionType.kBool: lambda _x, _y: (0, ""), + # hspy.HighsOptionType.kInt: hopt.check_int_option, + # hspy.HighsOptionType.kDouble: hopt.check_double_option, + # hspy.HighsOptionType.kString: hopt.check_string_option, + # }[opt_type](key, val) + + # have to do bool checking here because HiGHS doesn't have API + if opt_type == hspy.HighsOptionType.kBool: + if not isinstance(val, bool): + warn(f'Option f"{key}" is "{val}", but only True or False is ' + f'allowed. Using default.', OptimizeWarning) + continue + + # warn or set option + if status != 0: + warn(msg, OptimizeWarning) + else: + setattr(highs_options, key, val) + + opt_status = highs.passOptions(highs_options) + if opt_status == hspy.HighsStatus.kError: + res.update({ + "status": highs.getModelStatus(), + "message": highs.modelStatusToString(highs.getModelStatus()), + }) + return res + + init_status = highs.passModel(lp) + if init_status == hspy.HighsStatus.kError: + # if model fails to load, highs.getModelStatus() will be NOT_SET + err_model_status = hspy.HighsModelStatus.kModelError + res.update({ + "status": err_model_status, + "message": highs.modelStatusToString(err_model_status), + }) + return res + + # Solve the LP + run_status = highs.run() + if run_status == hspy.HighsStatus.kError: + res.update({ + "status": highs.getModelStatus(), + "message": highs.modelStatusToString(highs.getModelStatus()), + }) + return res + + # Extract what we need from the solution + model_status = highs.getModelStatus() + + # it should always be safe to get the info object + info = highs.getInfo() + + # Failure modes: + # LP: if we have anything other than an Optimal status, it + # is unsafe (and unhelpful) to read any results + # MIP: has a non-Optimal status or has timed out/reached max iterations + # 1) If not Optimal/TimedOut/MaxIter status, there is no solution + # 2) If TimedOut/MaxIter status, there may be a feasible solution. + # if the objective function value is not Infinity, then the + # current solution is feasible and can be returned. Else, there + # is no solution. + mipFailCondition = model_status not in ( + hspy.HighsModelStatus.kOptimal, + hspy.HighsModelStatus.kTimeLimit, + hspy.HighsModelStatus.kIterationLimit, + hspy.HighsModelStatus.kSolutionLimit, + ) or (model_status in { + hspy.HighsModelStatus.kTimeLimit, + hspy.HighsModelStatus.kIterationLimit, + hspy.HighsModelStatus.kSolutionLimit, + } and (info.objective_function_value == hspy.kHighsInf)) + lpFailCondition = model_status != hspy.HighsModelStatus.kOptimal + if (isMip and mipFailCondition) or (not isMip and lpFailCondition): + res.update({ + "status": model_status, + "message": f"model_status is {highs.modelStatusToString(model_status)}; " + f"primal_status is " + f"{highs.solutionStatusToString(info.primal_solution_status)}", + "simplex_nit": info.simplex_iteration_count, + "ipm_nit": info.ipm_iteration_count, + "crossover_nit": info.crossover_iteration_count, + }) + return res + + # Should be safe to read the solution: + solution = highs.getSolution() + basis = highs.getBasis() + + # Lagrangians for bounds based on column statuses + marg_bnds = np.zeros((2, numcol)) + for ii in range(numcol): + if basis.col_status[ii] == hspy.HighsBasisStatus.kLower: + marg_bnds[0, ii] = solution.col_dual[ii] + elif basis.col_status[ii] == hspy.HighsBasisStatus.kUpper: + marg_bnds[1, ii] = solution.col_dual[ii] + + res.update({ + "status": model_status, + "message": highs.modelStatusToString(model_status), + + # Primal solution + "x": np.array(solution.col_value), + + # Ax + s = b => Ax = b - s + # Note: this is for all constraints (A_ub and A_eq) + "slack": rhs - solution.row_value, + + # lambda are the lagrange multipliers associated with Ax=b + "lambda": np.array(solution.row_dual), + "marg_bnds": marg_bnds, + + "fun": info.objective_function_value, + "simplex_nit": info.simplex_iteration_count, + "ipm_nit": info.ipm_iteration_count, + "crossover_nit": info.crossover_iteration_count, + }) + + if isMip: + res.update({ + "mip_node_count": info.mip_node_count, + "mip_dual_bound": info.mip_dual_bound, + "mip_gap": info.mip_gap, + }) + + return res + +def check_option(highs_inst, option, value): + status, option_type = highs_inst.getOptionType(option) + + if status != HighsStatus.kOk: + return 1, "Invalid option name." + + valid_types = { + HighsOptionType.kBool: bool, + HighsOptionType.kInt: int, + HighsOptionType.kDouble: float, + HighsOptionType.kString: str + } + + expected_type = valid_types.get(option_type, None) + if expected_type is None: + return 3, "Unknown option type." + + if not isinstance(value, expected_type): + return 2, "Invalid option value." + + status, current_value = highs_inst.getOptionValue(option) + if status != HighsStatus.kOk: + return 4, "Failed to validate option value." + return 0, "Check option succeeded." diff --git a/scipy/optimize/_highs/cython/__init__.py b/scipy/optimize/_highs/cython/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/scipy/optimize/_highs/cython/src/HConfig.h b/scipy/optimize/_highs/cython/src/HConfig.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/scipy/optimize/_highs/cython/src/HConst.pxd b/scipy/optimize/_highs/cython/src/HConst.pxd deleted file mode 100644 index 503d9e74a263..000000000000 --- a/scipy/optimize/_highs/cython/src/HConst.pxd +++ /dev/null @@ -1,106 +0,0 @@ -# cython: language_level=3 - -from libcpp cimport bool -from libcpp.string cimport string - -cdef extern from "HConst.h" nogil: - - const int HIGHS_CONST_I_INF "kHighsIInf" - const double HIGHS_CONST_INF "kHighsInf" - const double kHighsTiny - const double kHighsZero - const int kHighsThreadLimit - - cdef enum HighsDebugLevel: - HighsDebugLevel_kHighsDebugLevelNone "kHighsDebugLevelNone" = 0 - HighsDebugLevel_kHighsDebugLevelCheap "kHighsDebugLevelCheap" - HighsDebugLevel_kHighsDebugLevelCostly "kHighsDebugLevelCostly" - HighsDebugLevel_kHighsDebugLevelExpensive "kHighsDebugLevelExpensive" - HighsDebugLevel_kHighsDebugLevelMin "kHighsDebugLevelMin" = HighsDebugLevel_kHighsDebugLevelNone - HighsDebugLevel_kHighsDebugLevelMax "kHighsDebugLevelMax" = HighsDebugLevel_kHighsDebugLevelExpensive - - ctypedef enum HighsModelStatus: - HighsModelStatusNOTSET "HighsModelStatus::kNotset" = 0 - HighsModelStatusLOAD_ERROR "HighsModelStatus::kLoadError" - HighsModelStatusMODEL_ERROR "HighsModelStatus::kModelError" - HighsModelStatusPRESOLVE_ERROR "HighsModelStatus::kPresolveError" - HighsModelStatusSOLVE_ERROR "HighsModelStatus::kSolveError" - HighsModelStatusPOSTSOLVE_ERROR "HighsModelStatus::kPostsolveError" - HighsModelStatusMODEL_EMPTY "HighsModelStatus::kModelEmpty" - HighsModelStatusOPTIMAL "HighsModelStatus::kOptimal" - HighsModelStatusINFEASIBLE "HighsModelStatus::kInfeasible" - HighsModelStatus_UNBOUNDED_OR_INFEASIBLE "HighsModelStatus::kUnboundedOrInfeasible" - HighsModelStatusUNBOUNDED "HighsModelStatus::kUnbounded" - HighsModelStatusREACHED_DUAL_OBJECTIVE_VALUE_UPPER_BOUND "HighsModelStatus::kObjectiveBound" - HighsModelStatusREACHED_OBJECTIVE_TARGET "HighsModelStatus::kObjectiveTarget" - HighsModelStatusREACHED_TIME_LIMIT "HighsModelStatus::kTimeLimit" - HighsModelStatusREACHED_ITERATION_LIMIT "HighsModelStatus::kIterationLimit" - HighsModelStatusUNKNOWN "HighsModelStatus::kUnknown" - HighsModelStatusHIGHS_MODEL_STATUS_MIN "HighsModelStatus::kMin" = HighsModelStatusNOTSET - HighsModelStatusHIGHS_MODEL_STATUS_MAX "HighsModelStatus::kMax" = HighsModelStatusUNKNOWN - - cdef enum HighsBasisStatus: - HighsBasisStatusLOWER "HighsBasisStatus::kLower" = 0, # (slack) variable is at its lower bound [including fixed variables] - HighsBasisStatusBASIC "HighsBasisStatus::kBasic" # (slack) variable is basic - HighsBasisStatusUPPER "HighsBasisStatus::kUpper" # (slack) variable is at its upper bound - HighsBasisStatusZERO "HighsBasisStatus::kZero" # free variable is non-basic and set to zero - HighsBasisStatusNONBASIC "HighsBasisStatus::kNonbasic" # nonbasic with no specific bound information - useful for users and postsolve - - cdef enum SolverOption: - SOLVER_OPTION_SIMPLEX "SolverOption::SOLVER_OPTION_SIMPLEX" = -1 - SOLVER_OPTION_CHOOSE "SolverOption::SOLVER_OPTION_CHOOSE" - SOLVER_OPTION_IPM "SolverOption::SOLVER_OPTION_IPM" - - cdef enum PrimalDualStatus: - PrimalDualStatusSTATUS_NOT_SET "PrimalDualStatus::STATUS_NOT_SET" = -1 - PrimalDualStatusSTATUS_MIN "PrimalDualStatus::STATUS_MIN" = PrimalDualStatusSTATUS_NOT_SET - PrimalDualStatusSTATUS_NO_SOLUTION "PrimalDualStatus::STATUS_NO_SOLUTION" - PrimalDualStatusSTATUS_UNKNOWN "PrimalDualStatus::STATUS_UNKNOWN" - PrimalDualStatusSTATUS_INFEASIBLE_POINT "PrimalDualStatus::STATUS_INFEASIBLE_POINT" - PrimalDualStatusSTATUS_FEASIBLE_POINT "PrimalDualStatus::STATUS_FEASIBLE_POINT" - PrimalDualStatusSTATUS_MAX "PrimalDualStatus::STATUS_MAX" = PrimalDualStatusSTATUS_FEASIBLE_POINT - - cdef enum HighsOptionType: - HighsOptionTypeBOOL "HighsOptionType::kBool" = 0 - HighsOptionTypeINT "HighsOptionType::kInt" - HighsOptionTypeDOUBLE "HighsOptionType::kDouble" - HighsOptionTypeSTRING "HighsOptionType::kString" - - # workaround for lack of enum class support in Cython < 3.x - # cdef enum class ObjSense(int): - # ObjSenseMINIMIZE "ObjSense::kMinimize" = 1 - # ObjSenseMAXIMIZE "ObjSense::kMaximize" = -1 - - cdef cppclass ObjSense: - pass - - cdef ObjSense ObjSenseMINIMIZE "ObjSense::kMinimize" - cdef ObjSense ObjSenseMAXIMIZE "ObjSense::kMaximize" - - # cdef enum class MatrixFormat(int): - # MatrixFormatkColwise "MatrixFormat::kColwise" = 1 - # MatrixFormatkRowwise "MatrixFormat::kRowwise" - # MatrixFormatkRowwisePartitioned "MatrixFormat::kRowwisePartitioned" - - cdef cppclass MatrixFormat: - pass - - cdef MatrixFormat MatrixFormatkColwise "MatrixFormat::kColwise" - cdef MatrixFormat MatrixFormatkRowwise "MatrixFormat::kRowwise" - cdef MatrixFormat MatrixFormatkRowwisePartitioned "MatrixFormat::kRowwisePartitioned" - - # cdef enum class HighsVarType(int): - # kContinuous "HighsVarType::kContinuous" - # kInteger "HighsVarType::kInteger" - # kSemiContinuous "HighsVarType::kSemiContinuous" - # kSemiInteger "HighsVarType::kSemiInteger" - # kImplicitInteger "HighsVarType::kImplicitInteger" - - cdef cppclass HighsVarType: - pass - - cdef HighsVarType kContinuous "HighsVarType::kContinuous" - cdef HighsVarType kInteger "HighsVarType::kInteger" - cdef HighsVarType kSemiContinuous "HighsVarType::kSemiContinuous" - cdef HighsVarType kSemiInteger "HighsVarType::kSemiInteger" - cdef HighsVarType kImplicitInteger "HighsVarType::kImplicitInteger" diff --git a/scipy/optimize/_highs/cython/src/Highs.pxd b/scipy/optimize/_highs/cython/src/Highs.pxd deleted file mode 100644 index 7139908d0341..000000000000 --- a/scipy/optimize/_highs/cython/src/Highs.pxd +++ /dev/null @@ -1,56 +0,0 @@ -# cython: language_level=3 - -from libc.stdio cimport FILE - -from libcpp cimport bool -from libcpp.string cimport string - -from .HighsStatus cimport HighsStatus -from .HighsOptions cimport HighsOptions -from .HighsInfo cimport HighsInfo -from .HighsLp cimport ( - HighsLp, - HighsSolution, - HighsBasis, - ObjSense, -) -from .HConst cimport HighsModelStatus - -cdef extern from "Highs.h": - # From HiGHS/src/Highs.h - cdef cppclass Highs: - HighsStatus passHighsOptions(const HighsOptions& options) - HighsStatus passModel(const HighsLp& lp) - HighsStatus run() - HighsStatus setHighsLogfile(FILE* logfile) - HighsStatus setHighsOutput(FILE* output) - HighsStatus writeHighsOptions(const string filename, const bool report_only_non_default_values = true) - - # split up for cython below - #const HighsModelStatus& getModelStatus(const bool scaled_model = False) const - const HighsModelStatus & getModelStatus() const - - const HighsInfo& getHighsInfo "getInfo" () const - string modelStatusToString(const HighsModelStatus model_status) const - #HighsStatus getHighsInfoValue(const string& info, int& value) - HighsStatus getHighsInfoValue(const string& info, double& value) const - const HighsOptions& getHighsOptions() const - - const HighsLp& getLp() const - - HighsStatus writeSolution(const string filename, const bool pretty) const - - HighsStatus setBasis() - const HighsSolution& getSolution() const - const HighsBasis& getBasis() const - - bool changeObjectiveSense(const ObjSense sense) - - HighsStatus setHighsOptionValueBool "setOptionValue" (const string & option, const bool value) - HighsStatus setHighsOptionValueInt "setOptionValue" (const string & option, const int value) - HighsStatus setHighsOptionValueStr "setOptionValue" (const string & option, const string & value) - HighsStatus setHighsOptionValueDbl "setOptionValue" (const string & option, const double value) - - string primalDualStatusToString(const int primal_dual_status) - - void resetGlobalScheduler(bool blocking) diff --git a/scipy/optimize/_highs/cython/src/HighsIO.pxd b/scipy/optimize/_highs/cython/src/HighsIO.pxd deleted file mode 100644 index 82b80ae643f1..000000000000 --- a/scipy/optimize/_highs/cython/src/HighsIO.pxd +++ /dev/null @@ -1,20 +0,0 @@ -# cython: language_level=3 - - -cdef extern from "HighsIO.h" nogil: - # workaround for lack of enum class support in Cython < 3.x - # cdef enum class HighsLogType(int): - # kInfo "HighsLogType::kInfo" = 1 - # kDetailed "HighsLogType::kDetailed" - # kVerbose "HighsLogType::kVerbose" - # kWarning "HighsLogType::kWarning" - # kError "HighsLogType::kError" - - cdef cppclass HighsLogType: - pass - - cdef HighsLogType kInfo "HighsLogType::kInfo" - cdef HighsLogType kDetailed "HighsLogType::kDetailed" - cdef HighsLogType kVerbose "HighsLogType::kVerbose" - cdef HighsLogType kWarning "HighsLogType::kWarning" - cdef HighsLogType kError "HighsLogType::kError" diff --git a/scipy/optimize/_highs/cython/src/HighsInfo.pxd b/scipy/optimize/_highs/cython/src/HighsInfo.pxd deleted file mode 100644 index 789b51089896..000000000000 --- a/scipy/optimize/_highs/cython/src/HighsInfo.pxd +++ /dev/null @@ -1,22 +0,0 @@ -# cython: language_level=3 - -cdef extern from "HighsInfo.h" nogil: - # From HiGHS/src/lp_data/HighsInfo.h - cdef cppclass HighsInfo: - # Inherited from HighsInfoStruct: - int mip_node_count - int simplex_iteration_count - int ipm_iteration_count - int crossover_iteration_count - int primal_solution_status - int dual_solution_status - int basis_validity - double objective_function_value - double mip_dual_bound - double mip_gap - int num_primal_infeasibilities - double max_primal_infeasibility - double sum_primal_infeasibilities - int num_dual_infeasibilities - double max_dual_infeasibility - double sum_dual_infeasibilities diff --git a/scipy/optimize/_highs/cython/src/HighsLp.pxd b/scipy/optimize/_highs/cython/src/HighsLp.pxd deleted file mode 100644 index 0944f083743f..000000000000 --- a/scipy/optimize/_highs/cython/src/HighsLp.pxd +++ /dev/null @@ -1,46 +0,0 @@ -# cython: language_level=3 - -from libcpp cimport bool -from libcpp.string cimport string -from libcpp.vector cimport vector - -from .HConst cimport HighsBasisStatus, ObjSense, HighsVarType -from .HighsSparseMatrix cimport HighsSparseMatrix - - -cdef extern from "HighsLp.h" nogil: - # From HiGHS/src/lp_data/HighsLp.h - cdef cppclass HighsLp: - int num_col_ - int num_row_ - - vector[double] col_cost_ - vector[double] col_lower_ - vector[double] col_upper_ - vector[double] row_lower_ - vector[double] row_upper_ - - HighsSparseMatrix a_matrix_ - - ObjSense sense_ - double offset_ - - string model_name_ - - vector[string] row_names_ - vector[string] col_names_ - - vector[HighsVarType] integrality_ - - bool isMip() const - - cdef cppclass HighsSolution: - vector[double] col_value - vector[double] col_dual - vector[double] row_value - vector[double] row_dual - - cdef cppclass HighsBasis: - bool valid_ - vector[HighsBasisStatus] col_status - vector[HighsBasisStatus] row_status diff --git a/scipy/optimize/_highs/cython/src/HighsLpUtils.pxd b/scipy/optimize/_highs/cython/src/HighsLpUtils.pxd deleted file mode 100644 index 18ede36c146a..000000000000 --- a/scipy/optimize/_highs/cython/src/HighsLpUtils.pxd +++ /dev/null @@ -1,9 +0,0 @@ -# cython: language_level=3 - -from .HighsStatus cimport HighsStatus -from .HighsLp cimport HighsLp -from .HighsOptions cimport HighsOptions - -cdef extern from "HighsLpUtils.h" nogil: - # From HiGHS/src/lp_data/HighsLpUtils.h - HighsStatus assessLp(HighsLp& lp, const HighsOptions& options) diff --git a/scipy/optimize/_highs/cython/src/HighsModelUtils.pxd b/scipy/optimize/_highs/cython/src/HighsModelUtils.pxd deleted file mode 100644 index 4fccc2e80046..000000000000 --- a/scipy/optimize/_highs/cython/src/HighsModelUtils.pxd +++ /dev/null @@ -1,10 +0,0 @@ -# cython: language_level=3 - -from libcpp.string cimport string - -from .HConst cimport HighsModelStatus - -cdef extern from "HighsModelUtils.h" nogil: - # From HiGHS/src/lp_data/HighsModelUtils.h - string utilHighsModelStatusToString(const HighsModelStatus model_status) - string utilBasisStatusToString(const int primal_dual_status) diff --git a/scipy/optimize/_highs/cython/src/HighsOptions.pxd b/scipy/optimize/_highs/cython/src/HighsOptions.pxd deleted file mode 100644 index 920c10c19e30..000000000000 --- a/scipy/optimize/_highs/cython/src/HighsOptions.pxd +++ /dev/null @@ -1,110 +0,0 @@ -# cython: language_level=3 - -from libc.stdio cimport FILE - -from libcpp cimport bool -from libcpp.string cimport string -from libcpp.vector cimport vector - -from .HConst cimport HighsOptionType - -cdef extern from "HighsOptions.h" nogil: - - cdef cppclass OptionRecord: - HighsOptionType type - string name - string description - bool advanced - - cdef cppclass OptionRecordBool(OptionRecord): - bool* value - bool default_value - - cdef cppclass OptionRecordInt(OptionRecord): - int* value - int lower_bound - int default_value - int upper_bound - - cdef cppclass OptionRecordDouble(OptionRecord): - double* value - double lower_bound - double default_value - double upper_bound - - cdef cppclass OptionRecordString(OptionRecord): - string* value - string default_value - - cdef cppclass HighsOptions: - # From HighsOptionsStruct: - - # Options read from the command line - string model_file - string presolve - string solver - string parallel - double time_limit - string options_file - - # Options read from the file - double infinite_cost - double infinite_bound - double small_matrix_value - double large_matrix_value - double primal_feasibility_tolerance - double dual_feasibility_tolerance - double ipm_optimality_tolerance - double dual_objective_value_upper_bound - int highs_debug_level - int simplex_strategy - int simplex_scale_strategy - int simplex_crash_strategy - int simplex_dual_edge_weight_strategy - int simplex_primal_edge_weight_strategy - int simplex_iteration_limit - int simplex_update_limit - int ipm_iteration_limit - int highs_min_threads - int highs_max_threads - int message_level - string solution_file - bool write_solution_to_file - bool write_solution_pretty - - # Advanced options - bool run_crossover - bool mps_parser_type_free - int keep_n_rows - int allowed_simplex_matrix_scale_factor - int allowed_simplex_cost_scale_factor - int simplex_dualise_strategy - int simplex_permute_strategy - int dual_simplex_cleanup_strategy - int simplex_price_strategy - int dual_chuzc_sort_strategy - bool simplex_initial_condition_check - double simplex_initial_condition_tolerance - double dual_steepest_edge_weight_log_error_threshhold - double dual_simplex_cost_perturbation_multiplier - double start_crossover_tolerance - bool less_infeasible_DSE_check - bool less_infeasible_DSE_choose_row - bool use_original_HFactor_logic - - # Options for MIP solver - int mip_max_nodes - int mip_report_level - - # Switch for MIP solver - bool mip - - # Options for HighsPrintMessage and HighsLogMessage - FILE* logfile - FILE* output - int message_level - string solution_file - bool write_solution_to_file - bool write_solution_pretty - - vector[OptionRecord*] records diff --git a/scipy/optimize/_highs/cython/src/HighsRuntimeOptions.pxd b/scipy/optimize/_highs/cython/src/HighsRuntimeOptions.pxd deleted file mode 100644 index 3e227b7a44f7..000000000000 --- a/scipy/optimize/_highs/cython/src/HighsRuntimeOptions.pxd +++ /dev/null @@ -1,9 +0,0 @@ -# cython: language_level=3 - -from libcpp cimport bool - -from .HighsOptions cimport HighsOptions - -cdef extern from "HighsRuntimeOptions.h" nogil: - # From HiGHS/src/lp_data/HighsRuntimeOptions.h - bool loadOptions(int argc, char** argv, HighsOptions& options) diff --git a/scipy/optimize/_highs/cython/src/HighsSparseMatrix.pxd b/scipy/optimize/_highs/cython/src/HighsSparseMatrix.pxd deleted file mode 100644 index 7eaa9ef79eee..000000000000 --- a/scipy/optimize/_highs/cython/src/HighsSparseMatrix.pxd +++ /dev/null @@ -1,15 +0,0 @@ -# cython: language_level=3 - -from libcpp.vector cimport vector - -from .HConst cimport MatrixFormat - - -cdef extern from "HighsSparseMatrix.h" nogil: - cdef cppclass HighsSparseMatrix: - MatrixFormat format_ - int num_col_ - int num_row_ - vector[int] start_ - vector[int] index_ - vector[double] value_ diff --git a/scipy/optimize/_highs/cython/src/HighsStatus.pxd b/scipy/optimize/_highs/cython/src/HighsStatus.pxd deleted file mode 100644 index b47813b5d391..000000000000 --- a/scipy/optimize/_highs/cython/src/HighsStatus.pxd +++ /dev/null @@ -1,12 +0,0 @@ -# cython: language_level=3 - -from libcpp.string cimport string - -cdef extern from "HighsStatus.h" nogil: - ctypedef enum HighsStatus: - HighsStatusError "HighsStatus::kError" = -1 - HighsStatusOK "HighsStatus::kOk" = 0 - HighsStatusWarning "HighsStatus::kWarning" = 1 - - - string highsStatusToString(HighsStatus status) diff --git a/scipy/optimize/_highs/cython/src/SimplexConst.pxd b/scipy/optimize/_highs/cython/src/SimplexConst.pxd deleted file mode 100644 index 77e7b96320d6..000000000000 --- a/scipy/optimize/_highs/cython/src/SimplexConst.pxd +++ /dev/null @@ -1,95 +0,0 @@ -# cython: language_level=3 - -from libcpp cimport bool - -cdef extern from "SimplexConst.h" nogil: - - cdef enum SimplexAlgorithm: - PRIMAL "SimplexAlgorithm::kPrimal" = 0 - DUAL "SimplexAlgorithm::kDual" - - cdef enum SimplexStrategy: - SIMPLEX_STRATEGY_MIN "SimplexStrategy::kSimplexStrategyMin" = 0 - SIMPLEX_STRATEGY_CHOOSE "SimplexStrategy::kSimplexStrategyChoose" = SIMPLEX_STRATEGY_MIN - SIMPLEX_STRATEGY_DUAL "SimplexStrategy::kSimplexStrategyDual" - SIMPLEX_STRATEGY_DUAL_PLAIN "SimplexStrategy::kSimplexStrategyDualPlain" = SIMPLEX_STRATEGY_DUAL - SIMPLEX_STRATEGY_DUAL_TASKS "SimplexStrategy::kSimplexStrategyDualTasks" - SIMPLEX_STRATEGY_DUAL_MULTI "SimplexStrategy::kSimplexStrategyDualMulti" - SIMPLEX_STRATEGY_PRIMAL "SimplexStrategy::kSimplexStrategyPrimal" - SIMPLEX_STRATEGY_MAX "SimplexStrategy::kSimplexStrategyMax" = SIMPLEX_STRATEGY_PRIMAL - SIMPLEX_STRATEGY_NUM "SimplexStrategy::kSimplexStrategyNum" - - cdef enum SimplexCrashStrategy: - SIMPLEX_CRASH_STRATEGY_MIN "SimplexCrashStrategy::kSimplexCrashStrategyMin" = 0 - SIMPLEX_CRASH_STRATEGY_OFF "SimplexCrashStrategy::kSimplexCrashStrategyOff" = SIMPLEX_CRASH_STRATEGY_MIN - SIMPLEX_CRASH_STRATEGY_LTSSF_K "SimplexCrashStrategy::kSimplexCrashStrategyLtssfK" - SIMPLEX_CRASH_STRATEGY_LTSSF "SimplexCrashStrategy::kSimplexCrashStrategyLtssf" = SIMPLEX_CRASH_STRATEGY_LTSSF_K - SIMPLEX_CRASH_STRATEGY_BIXBY "SimplexCrashStrategy::kSimplexCrashStrategyBixby" - SIMPLEX_CRASH_STRATEGY_LTSSF_PRI "SimplexCrashStrategy::kSimplexCrashStrategyLtssfPri" - SIMPLEX_CRASH_STRATEGY_LTSF_K "SimplexCrashStrategy::kSimplexCrashStrategyLtsfK" - SIMPLEX_CRASH_STRATEGY_LTSF_PRI "SimplexCrashStrategy::kSimplexCrashStrategyLtsfPri" - SIMPLEX_CRASH_STRATEGY_LTSF "SimplexCrashStrategy::kSimplexCrashStrategyLtsf" - SIMPLEX_CRASH_STRATEGY_BIXBY_NO_NONZERO_COL_COSTS "SimplexCrashStrategy::kSimplexCrashStrategyBixbyNoNonzeroColCosts" - SIMPLEX_CRASH_STRATEGY_BASIC "SimplexCrashStrategy::kSimplexCrashStrategyBasic" - SIMPLEX_CRASH_STRATEGY_TEST_SING "SimplexCrashStrategy::kSimplexCrashStrategyTestSing" - SIMPLEX_CRASH_STRATEGY_MAX "SimplexCrashStrategy::kSimplexCrashStrategyMax" = SIMPLEX_CRASH_STRATEGY_TEST_SING - - cdef enum SimplexEdgeWeightStrategy: - SIMPLEX_EDGE_WEIGHT_STRATEGY_MIN "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategyMin" = -1 - SIMPLEX_EDGE_WEIGHT_STRATEGY_CHOOSE "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategyChoose" = SIMPLEX_EDGE_WEIGHT_STRATEGY_MIN - SIMPLEX_EDGE_WEIGHT_STRATEGY_DANTZIG "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategyDantzig" - SIMPLEX_EDGE_WEIGHT_STRATEGY_DEVEX "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategyDevex" - SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategySteepestEdge" - SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE_UNIT_INITIAL "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategySteepestEdgeUnitInitial" - SIMPLEX_EDGE_WEIGHT_STRATEGY_MAX "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategyMax" = SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE_UNIT_INITIAL - - cdef enum SimplexPriceStrategy: - SIMPLEX_PRICE_STRATEGY_MIN = 0 - SIMPLEX_PRICE_STRATEGY_COL = SIMPLEX_PRICE_STRATEGY_MIN - SIMPLEX_PRICE_STRATEGY_ROW - SIMPLEX_PRICE_STRATEGY_ROW_SWITCH - SIMPLEX_PRICE_STRATEGY_ROW_SWITCH_COL_SWITCH - SIMPLEX_PRICE_STRATEGY_MAX = SIMPLEX_PRICE_STRATEGY_ROW_SWITCH_COL_SWITCH - - cdef enum SimplexDualChuzcStrategy: - SIMPLEX_DUAL_CHUZC_STRATEGY_MIN = 0 - SIMPLEX_DUAL_CHUZC_STRATEGY_CHOOSE = SIMPLEX_DUAL_CHUZC_STRATEGY_MIN - SIMPLEX_DUAL_CHUZC_STRATEGY_QUAD - SIMPLEX_DUAL_CHUZC_STRATEGY_HEAP - SIMPLEX_DUAL_CHUZC_STRATEGY_BOTH - SIMPLEX_DUAL_CHUZC_STRATEGY_MAX = SIMPLEX_DUAL_CHUZC_STRATEGY_BOTH - - cdef enum InvertHint: - INVERT_HINT_NO = 0 - INVERT_HINT_UPDATE_LIMIT_REACHED - INVERT_HINT_SYNTHETIC_CLOCK_SAYS_INVERT - INVERT_HINT_POSSIBLY_OPTIMAL - INVERT_HINT_POSSIBLY_PRIMAL_UNBOUNDED - INVERT_HINT_POSSIBLY_DUAL_UNBOUNDED - INVERT_HINT_POSSIBLY_SINGULAR_BASIS - INVERT_HINT_PRIMAL_INFEASIBLE_IN_PRIMAL_SIMPLEX - INVERT_HINT_CHOOSE_COLUMN_FAIL - INVERT_HINT_Count - - cdef enum DualEdgeWeightMode: - DANTZIG "DualEdgeWeightMode::DANTZIG" = 0 - DEVEX "DualEdgeWeightMode::DEVEX" - STEEPEST_EDGE "DualEdgeWeightMode::STEEPEST_EDGE" - Count "DualEdgeWeightMode::Count" - - cdef enum PriceMode: - ROW "PriceMode::ROW" = 0 - COL "PriceMode::COL" - - const int PARALLEL_THREADS_DEFAULT - const int DUAL_TASKS_MIN_THREADS - const int DUAL_MULTI_MIN_THREADS - - const bool invert_if_row_out_negative - - const int NONBASIC_FLAG_TRUE - const int NONBASIC_FLAG_FALSE - - const int NONBASIC_MOVE_UP - const int NONBASIC_MOVE_DN - const int NONBASIC_MOVE_ZE diff --git a/scipy/optimize/_highs/cython/src/__init__.py b/scipy/optimize/_highs/cython/src/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/scipy/optimize/_highs/cython/src/_highs_constants.pyx b/scipy/optimize/_highs/cython/src/_highs_constants.pyx deleted file mode 100644 index 815e7d79e9e1..000000000000 --- a/scipy/optimize/_highs/cython/src/_highs_constants.pyx +++ /dev/null @@ -1,117 +0,0 @@ -# cython: language_level=3 - -'''Export enum values and constants from HiGHS.''' - -from .HConst cimport ( - HIGHS_CONST_I_INF, - HIGHS_CONST_INF, - - HighsDebugLevel_kHighsDebugLevelNone, - HighsDebugLevel_kHighsDebugLevelCheap, - - HighsModelStatusNOTSET, - HighsModelStatusLOAD_ERROR, - HighsModelStatusMODEL_ERROR, - HighsModelStatusMODEL_EMPTY, - HighsModelStatusPRESOLVE_ERROR, - HighsModelStatusSOLVE_ERROR, - HighsModelStatusPOSTSOLVE_ERROR, - HighsModelStatusINFEASIBLE, - HighsModelStatus_UNBOUNDED_OR_INFEASIBLE, - HighsModelStatusUNBOUNDED, - HighsModelStatusOPTIMAL, - HighsModelStatusREACHED_DUAL_OBJECTIVE_VALUE_UPPER_BOUND, - HighsModelStatusREACHED_OBJECTIVE_TARGET, - HighsModelStatusREACHED_TIME_LIMIT, - HighsModelStatusREACHED_ITERATION_LIMIT, - - ObjSenseMINIMIZE, - kContinuous, - kInteger, - kSemiContinuous, - kSemiInteger, - kImplicitInteger, -) -from .HighsIO cimport ( - kInfo, - kDetailed, - kVerbose, - kWarning, - kError, -) -from .SimplexConst cimport ( - # Simplex strategy - SIMPLEX_STRATEGY_CHOOSE, - SIMPLEX_STRATEGY_DUAL, - SIMPLEX_STRATEGY_PRIMAL, - - # Crash strategy - SIMPLEX_CRASH_STRATEGY_OFF, - SIMPLEX_CRASH_STRATEGY_BIXBY, - SIMPLEX_CRASH_STRATEGY_LTSF, - - # Edge weight strategy - SIMPLEX_EDGE_WEIGHT_STRATEGY_CHOOSE, - SIMPLEX_EDGE_WEIGHT_STRATEGY_DANTZIG, - SIMPLEX_EDGE_WEIGHT_STRATEGY_DEVEX, - SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE, -) - -# HConst -CONST_I_INF = HIGHS_CONST_I_INF -CONST_INF = HIGHS_CONST_INF - -# Debug level -MESSAGE_LEVEL_NONE = HighsDebugLevel_kHighsDebugLevelNone -MESSAGE_LEVEL_MINIMAL = HighsDebugLevel_kHighsDebugLevelCheap - -# HighsIO -LOG_TYPE_INFO = kInfo -LOG_TYPE_DETAILED = kDetailed -LOG_TYPE_VERBOSE = kVerbose -LOG_TYPE_WARNING = kWarning -LOG_TYPE_ERROR = kError - -# HighsLp -MODEL_STATUS_NOTSET = HighsModelStatusNOTSET -MODEL_STATUS_LOAD_ERROR = HighsModelStatusLOAD_ERROR -MODEL_STATUS_MODEL_ERROR = HighsModelStatusMODEL_ERROR -MODEL_STATUS_PRESOLVE_ERROR = HighsModelStatusPRESOLVE_ERROR -MODEL_STATUS_SOLVE_ERROR = HighsModelStatusSOLVE_ERROR -MODEL_STATUS_POSTSOLVE_ERROR = HighsModelStatusPOSTSOLVE_ERROR -MODEL_STATUS_MODEL_EMPTY = HighsModelStatusMODEL_EMPTY -MODEL_STATUS_INFEASIBLE = HighsModelStatusINFEASIBLE -MODEL_STATUS_UNBOUNDED_OR_INFEASIBLE = HighsModelStatus_UNBOUNDED_OR_INFEASIBLE -MODEL_STATUS_UNBOUNDED = HighsModelStatusUNBOUNDED -MODEL_STATUS_OPTIMAL = HighsModelStatusOPTIMAL -MODEL_STATUS_REACHED_DUAL_OBJECTIVE_VALUE_UPPER_BOUND = HighsModelStatusREACHED_DUAL_OBJECTIVE_VALUE_UPPER_BOUND -MODEL_STATUS_REACHED_OBJECTIVE_TARGET = HighsModelStatusREACHED_OBJECTIVE_TARGET -MODEL_STATUS_REACHED_TIME_LIMIT = HighsModelStatusREACHED_TIME_LIMIT -MODEL_STATUS_REACHED_ITERATION_LIMIT = HighsModelStatusREACHED_ITERATION_LIMIT - -# Simplex strategy -HIGHS_SIMPLEX_STRATEGY_CHOOSE = SIMPLEX_STRATEGY_CHOOSE -HIGHS_SIMPLEX_STRATEGY_DUAL = SIMPLEX_STRATEGY_DUAL -HIGHS_SIMPLEX_STRATEGY_PRIMAL = SIMPLEX_STRATEGY_PRIMAL - -# Crash strategy -HIGHS_SIMPLEX_CRASH_STRATEGY_OFF = SIMPLEX_CRASH_STRATEGY_OFF -HIGHS_SIMPLEX_CRASH_STRATEGY_BIXBY = SIMPLEX_CRASH_STRATEGY_BIXBY -HIGHS_SIMPLEX_CRASH_STRATEGY_LTSF = SIMPLEX_CRASH_STRATEGY_LTSF - -# Edge weight strategy -HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_CHOOSE = SIMPLEX_EDGE_WEIGHT_STRATEGY_CHOOSE -HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_DANTZIG = SIMPLEX_EDGE_WEIGHT_STRATEGY_DANTZIG -HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_DEVEX = SIMPLEX_EDGE_WEIGHT_STRATEGY_DEVEX -HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE = SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE -# HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE_UNIT_INITIAL = SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE_UNIT_INITIAL - -# Objective sense -HIGHS_OBJECTIVE_SENSE_MINIMIZE = ObjSenseMINIMIZE - -# Variable types -HIGHS_VAR_TYPE_CONTINUOUS = kContinuous -HIGHS_VAR_TYPE_INTEGER = kInteger -HIGHS_VAR_TYPE_SEMI_CONTINUOUS = kSemiContinuous -HIGHS_VAR_TYPE_SEMI_INTEGER = kSemiInteger -HIGHS_VAR_TYPE_IMPLICIT_INTEGER = kImplicitInteger diff --git a/scipy/optimize/_highs/cython/src/_highs_wrapper.pyx b/scipy/optimize/_highs/cython/src/_highs_wrapper.pyx deleted file mode 100644 index d5da4e4fea25..000000000000 --- a/scipy/optimize/_highs/cython/src/_highs_wrapper.pyx +++ /dev/null @@ -1,736 +0,0 @@ -# cython: language_level=3 - -import numpy as np -cimport numpy as np -from scipy.optimize import OptimizeWarning -from warnings import warn -import numbers - -from libcpp.string cimport string -from libcpp.map cimport map as cppmap -from libcpp.cast cimport reinterpret_cast - -from .HConst cimport ( - HIGHS_CONST_INF, - - HighsModelStatus, - HighsModelStatusNOTSET, - HighsModelStatusMODEL_ERROR, - HighsModelStatusOPTIMAL, - HighsModelStatusREACHED_TIME_LIMIT, - HighsModelStatusREACHED_ITERATION_LIMIT, - - HighsOptionTypeBOOL, - HighsOptionTypeINT, - HighsOptionTypeDOUBLE, - HighsOptionTypeSTRING, - - HighsBasisStatus, - HighsBasisStatusLOWER, - HighsBasisStatusUPPER, - - MatrixFormatkColwise, - HighsVarType, -) -from .Highs cimport Highs -from .HighsStatus cimport ( - HighsStatus, - highsStatusToString, - HighsStatusError, - HighsStatusWarning, - HighsStatusOK, -) -from .HighsLp cimport ( - HighsLp, - HighsSolution, - HighsBasis, -) -from .HighsInfo cimport HighsInfo -from .HighsOptions cimport ( - HighsOptions, - OptionRecord, - OptionRecordBool, - OptionRecordInt, - OptionRecordDouble, - OptionRecordString, -) -from .HighsModelUtils cimport utilBasisStatusToString - -np.import_array() - -# options to reference for default values and bounds; -# make a map to quickly lookup -cdef HighsOptions _ref_opts -cdef cppmap[string, OptionRecord*] _ref_opt_lookup -cdef OptionRecord * _r = NULL -for _r in _ref_opts.records: - _ref_opt_lookup[_r.name] = _r - - -cdef str _opt_warning(string name, val, valid_set=None) noexcept: - cdef OptionRecord * r = _ref_opt_lookup[name] - - # BOOL - if r.type == HighsOptionTypeBOOL: - default_value = ( r).default_value - return ('Option "%s" is "%s", but only True or False is allowed. ' - 'Using default: %s.' % (name.decode(), str(val), default_value)) - - # INT - if r.type == HighsOptionTypeINT: - lower_bound = int(( r).lower_bound) - upper_bound = int(( r).upper_bound) - default_value = int(( r).default_value) - if upper_bound - lower_bound < 10: - int_range = str(set(range(lower_bound, upper_bound + 1))) - else: - int_range = '[%d, %d]' % (lower_bound, upper_bound) - return ('Option "%s" is "%s", but only values in %s are allowed. ' - 'Using default: %d.' % (name.decode(), str(val), int_range, default_value)) - - # DOUBLE - if r.type == HighsOptionTypeDOUBLE: - lower_bound = ( r).lower_bound - upper_bound = ( r).upper_bound - default_value = ( r).default_value - return ('Option "%s" is "%s", but only values in (%g, %g) are allowed. ' - 'Using default: %g.' % (name.decode(), str(val), lower_bound, upper_bound, default_value)) - - # STRING - if r.type == HighsOptionTypeSTRING: - if valid_set is not None: - descr = 'but only values in %s are allowed. ' % str(set(valid_set)) - else: - descr = 'but this is an invalid value. %s. ' % r.description.decode() - default_value = ( r).default_value.decode() - return ('Option "%s" is "%s", ' - '%s' - 'Using default: %s.' % (name.decode(), str(val), descr, default_value)) - - # We don't know what type (should be unreachable)? - return('Option "%s" is "%s", but this is not a valid value. ' - 'See documentation for valid options. ' - 'Using default.' % (name.decode(), str(val))) - -cdef apply_options(dict options, Highs & highs) noexcept: - '''Take options from dictionary and apply to HiGHS object.''' - - # Initialize for error checking - cdef HighsStatus opt_status = HighsStatusOK - - # Do all the ints - for opt in set([ - 'allowed_simplex_cost_scale_factor', - 'allowed_simplex_matrix_scale_factor', - 'dual_simplex_cleanup_strategy', - 'ipm_iteration_limit', - 'keep_n_rows', - 'threads', - 'mip_max_nodes', - 'highs_debug_level', - 'simplex_crash_strategy', - 'simplex_dual_edge_weight_strategy', - 'simplex_dualise_strategy', - 'simplex_iteration_limit', - 'simplex_permute_strategy', - 'simplex_price_strategy', - 'simplex_primal_edge_weight_strategy', - 'simplex_scale_strategy', - 'simplex_strategy', - 'simplex_update_limit', - 'small_matrix_value', - ]): - val = options.get(opt, None) - if val is not None: - if not isinstance(val, int): - warn(_opt_warning(opt.encode(), val), OptimizeWarning) - else: - opt_status = highs.setHighsOptionValueInt(opt.encode(), val) - if opt_status != HighsStatusOK: - warn(_opt_warning(opt.encode(), val), OptimizeWarning) - else: - if opt == "threads": - highs.resetGlobalScheduler(blocking=True) - - # Do all the doubles - for opt in set([ - 'dual_feasibility_tolerance', - 'dual_objective_value_upper_bound', - 'dual_simplex_cost_perturbation_multiplier', - 'dual_steepest_edge_weight_log_error_threshhold', - 'infinite_bound', - 'infinite_cost', - 'ipm_optimality_tolerance', - 'large_matrix_value', - 'primal_feasibility_tolerance', - 'simplex_initial_condition_tolerance', - 'small_matrix_value', - 'start_crossover_tolerance', - 'time_limit', - 'mip_rel_gap' - ]): - val = options.get(opt, None) - if val is not None: - if not isinstance(val, numbers.Number): - warn(_opt_warning(opt.encode(), val), OptimizeWarning) - else: - opt_status = highs.setHighsOptionValueDbl(opt.encode(), val) - if opt_status != HighsStatusOK: - warn(_opt_warning(opt.encode(), val), OptimizeWarning) - - - # Do all the strings - for opt in set(['solver']): - val = options.get(opt, None) - if val is not None: - if not isinstance(val, str): - warn(_opt_warning(opt.encode(), val), OptimizeWarning) - else: - opt_status = highs.setHighsOptionValueStr(opt.encode(), val.encode()) - if opt_status != HighsStatusOK: - warn(_opt_warning(opt.encode(), val), OptimizeWarning) - - - # Do all the bool to strings - for opt in set([ - 'parallel', - 'presolve', - ]): - val = options.get(opt, None) - if val is not None: - if isinstance(val, bool): - if val: - val0 = b'on' - else: - val0 = b'off' - opt_status = highs.setHighsOptionValueStr(opt.encode(), val0) - if opt_status != HighsStatusOK: - warn(_opt_warning(opt.encode(), val, valid_set=[True, False]), OptimizeWarning) - else: - warn(_opt_warning(opt.encode(), val, valid_set=[True, False]), OptimizeWarning) - - - # Do the actual bools - for opt in set([ - 'less_infeasible_DSE_check', - 'less_infeasible_DSE_choose_row', - 'log_to_console', - 'mps_parser_type_free', - 'output_flag', - 'run_as_hsol', - 'run_crossover', - 'simplex_initial_condition_check', - 'use_original_HFactor_logic', - ]): - val = options.get(opt, None) - if val is not None: - if val in [True, False]: - opt_status = highs.setHighsOptionValueBool(opt.encode(), val) - if opt_status != HighsStatusOK: - warn(_opt_warning(opt.encode(), val), OptimizeWarning) - else: - warn(_opt_warning(opt.encode(), val), OptimizeWarning) - - -ctypedef HighsVarType* HighsVarType_ptr - - -def _highs_wrapper( - double[::1] c, - int[::1] astart, - int[::1] aindex, - double[::1] avalue, - double[::1] lhs, - double[::1] rhs, - double[::1] lb, - double[::1] ub, - np.uint8_t[::1] integrality, - dict options): - '''Solve linear programs using HiGHS [1]_. - - Assume problems of the form: - - MIN c.T @ x - s.t. lhs <= A @ x <= rhs - lb <= x <= ub - - Parameters - ---------- - c : 1-D array, (n,) - Array of objective value coefficients. - astart : 1-D array - CSC format index array. - aindex : 1-D array - CSC format index array. - avalue : 1-D array - Data array of the matrix. - lhs : 1-D array (or None), (m,) - Array of left hand side values of the inequality constraints. - If ``lhs=None``, then an array of ``-inf`` is assumed. - rhs : 1-D array, (m,) - Array of right hand side values of the inequality constraints. - lb : 1-D array (or None), (n,) - Lower bounds on solution variables x. If ``lb=None``, then an - array of all `0` is assumed. - ub : 1-D array (or None), (n,) - Upper bounds on solution variables x. If ``ub=None``, then an - array of ``inf`` is assumed. - options : dict - A dictionary of solver options with the following fields: - - - allowed_simplex_cost_scale_factor : int - Undocumented advanced option. - - - allowed_simplex_matrix_scale_factor : int - Undocumented advanced option. - - - dual_feasibility_tolerance : double - Dual feasibility tolerance for simplex. - ``min(dual_feasibility_tolerance, - primal_feasibility_tolerance)`` will be used for - ipm feasibility tolerance. - - - dual_objective_value_upper_bound : double - Upper bound on objective value for dual simplex: - algorithm terminates if reached - - - dual_simplex_cleanup_strategy : int - Undocumented advanced option. - - - dual_simplex_cost_perturbation_multiplier : double - Undocumented advanced option. - - - dual_steepest_edge_weight_log_error_threshhold : double - Undocumented advanced option. - - - infinite_bound : double - Limit on abs(constraint bound): values larger than - this will be treated as infinite - - - infinite_cost : double - Limit on cost coefficient: values larger than this - will be treated as infinite. - - - ipm_iteration_limit : int - Iteration limit for interior-point solver. - - - ipm_optimality_tolerance : double - Optimality tolerance for IPM. - - - keep_n_rows : int {-1, 0, 1} - Undocumented advanced option. - - - ``-1``: ``KEEP_N_ROWS_DELETE_ROWS`` - - ``0``: ``KEEP_N_ROWS_DELETE_ENTRIES`` - - ``1``: ``KEEP_N_ROWS_KEEP_ROWS`` - - - large_matrix_value : double - Upper limit on abs(matrix entries): values larger than - this will be treated as infinite - - - less_infeasible_DSE_check : bool - Undocumented advanced option. - - - less_infeasible_DSE_choose_row : bool - Undocumented advanced option. - - - threads : int - Maximum number of threads in parallel execution. - - - message_level : int {0, 1, 2, 4, 7} - Verbosity level, corresponds to: - - - ``0``: ``ML_NONE`` - All messaging to stdout is suppressed. - - - ``1``: ``ML_VERBOSE`` - Includes a once-per-iteration report on simplex/ipm - progress and information about each nonzero row and - column. - - - ``2``: ``ML_DETAILED`` - Includes technical information about progress and - events in applying the simplex method. - - - ``4``: ``ML_MINIMAL`` - Once-per-solve information about progress as well as a - once-per-basis-matrix-reinversion report on progress in - simplex or a once-per-iteration report on progress in IPX. - - ``message_level`` behaves like a bitmask, i.e., any - combination of levels is possible using the bit-or - operator. - - - mps_parser_type_free : bool - Use free format MPS parsing. - - - parallel : bool - Run the solver in serial (False) or parallel (True). - - - presolve : bool - Run the presolve or not (or if ``None``, then choose). - - - primal_feasibility_tolerance : double - Primal feasibility tolerance. - ``min(dual_feasibility_tolerance, - primal_feasibility_tolerance)`` will be used for - ipm feasibility tolerance. - - - run_as_hsol : bool - Undocumented advanced option. - - - run_crossover : bool - Advanced option. Toggles running the crossover routine - for IPX. - - - sense : int {1, -1} - ``sense=1`` corresponds to the MIN problem, ``sense=-1`` - corresponds to the MAX problem. TODO: NOT IMPLEMENTED - - - simplex_crash_strategy : int {0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - Strategy for simplex crash: off / LTSSF / Bixby (0/1/2). - Default is ``0``. Corresponds to the following: - - - ``0``: ``SIMPLEX_CRASH_STRATEGY_OFF`` - - ``1``: ``SIMPLEX_CRASH_STRATEGY_LTSSF_K`` - - ``2``: ``SIMPLEX_CRASH_STRATEGY_BIXBY`` - - ``3``: ``SIMPLEX_CRASH_STRATEGY_LTSSF_PRI`` - - ``4``: ``SIMPLEX_CRASH_STRATEGY_LTSF_K`` - - ``5``: ``SIMPLEX_CRASH_STRATEGY_LTSF_PRI`` - - ``6``: ``SIMPLEX_CRASH_STRATEGY_LTSF`` - - ``7``: ``SIMPLEX_CRASH_STRATEGY_BIXBY_NO_NONZERO_COL_COSTS`` - - ``8``: ``SIMPLEX_CRASH_STRATEGY_BASIC`` - - ``9``: ``SIMPLE_CRASH_STRATEGY_TEST_SING`` - - - simplex_dualise_strategy : int - Undocumented advanced option. - - - simplex_dual_edge_weight_strategy : int {0, 1, 2, 3, 4} - Strategy for simplex dual edge weights: - Dantzig / Devex / Steepest Edge. Corresponds - to the following: - - - ``0``: ``SIMPLEX_DUAL_EDGE_WEIGHT_STRATEGY_DANTZIG`` - - ``1``: ``SIMPLEX_DUAL_EDGE_WEIGHT_STRATEGY_DEVEX`` - - ``2``: ``SIMPLEX_DUAL_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE_TO_DEVEX_SWITCH`` - - ``3``: ``SIMPLEX_DUAL_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE`` - - ``4``: ``SIMPLEX_DUAL_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE_UNIT_INITIAL`` - - - simplex_initial_condition_check : bool - Undocumented advanced option. - - - simplex_initial_condition_tolerance : double - Undocumented advanced option. - - - simplex_iteration_limit : int - Iteration limit for simplex solver. - - - simplex_permute_strategy : int - Undocumented advanced option. - - - simplex_price_strategy : int - Undocumented advanced option. - - - simplex_primal_edge_weight_strategy : int {0, 1} - Strategy for simplex primal edge weights: - Dantzig / Devex. Corresponds to the following: - - - ``0``: ``SIMPLEX_PRIMAL_EDGE_WEIGHT_STRATEGY_DANTZIG`` - - ``1``: ``SIMPLEX_PRIMAL_EDGE_WEIGHT_STRATEGY_DEVEX`` - - - simplex_scale_strategy : int {0, 1, 2, 3, 4, 5} - Strategy for scaling before simplex solver: - off / on (0/1) - - - ``0``: ``SIMPLEX_SCALE_STRATEGY_OFF`` - - ``1``: ``SIMPLEX_SCALE_STRATEGY_HIGHS`` - - ``2``: ``SIMPLEX_SCALE_STRATEGY_HIGHS_FORCED`` - - ``3``: ``SIMPLEX_SCALE_STRATEGY_HIGHS_015`` - - ``4``: ``SIMPLEX_SCALE_STRATEGY_HIGHS_0157`` - - ``5``: ``SIMPLEX_SCALE_STRATEGY_HSOL`` - - - simplex_strategy : int {0, 1, 2, 3, 4} - Strategy for simplex solver. Default: 1. Corresponds - to the following: - - - ``0``: ``SIMPLEX_STRATEGY_MIN`` - - ``1``: ``SIMPLEX_STRATEGY_DUAL`` - - ``2``: ``SIMPLEX_STRATEGY_DUAL_TASKS`` - - ``3``: ``SIMPLEX_STRATEGY_DUAL_MULTI`` - - ``4``: ``SIMPLEX_STRATEGY_PRIMAL`` - - - simplex_update_limit : int - Limit on the number of simplex UPDATE operations. - - - small_matrix_value : double - Lower limit on abs(matrix entries): values smaller - than this will be treated as zero. - - - solution_file : str - Solution file - - - solver : str {'simplex', 'ipm'} - Choose which solver to use. If ``solver='simplex'`` - and ``parallel=True`` then PAMI will be used. - - - start_crossover_tolerance : double - Tolerance to be satisfied before IPM crossover will - start. - - - time_limit : double - Max number of seconds to run the solver for. - - - use_original_HFactor_logic : bool - Undocumented advanced option. - - - write_solution_to_file : bool - Write the primal and dual solution to a file - - - write_solution_pretty : bool - Write the primal and dual solution in a pretty - (human-readable) format - - See [2]_ for a list of all non-advanced options. - - Returns - ------- - res : dict - - If model_status is one of OPTIMAL, - REACHED_DUAL_OBJECTIVE_VALUE_UPPER_BOUND, REACHED_TIME_LIMIT, - REACHED_ITERATION_LIMIT: - - - ``status`` : int - Model status code. - - - ``message`` : str - Message corresponding to model status code. - - - ``x`` : list - Solution variables. - - - ``slack`` : list - Slack variables. - - - ``lambda`` : list - Lagrange multipliers associated with the constraints - Ax = b. - - - ``s`` : list - Lagrange multipliers associated with the constraints - x >= 0. - - - ``fun`` - Final objective value. - - - ``simplex_nit`` : int - Number of iterations accomplished by the simplex - solver. - - - ``ipm_nit`` : int - Number of iterations accomplished by the interior- - point solver. - - If model_status is not one of the above: - - - ``status`` : int - Model status code. - - - ``message`` : str - Message corresponding to model status code. - - Notes - ----- - If ``options['write_solution_to_file']`` is ``True`` but - ``options['solution_file']`` is unset or ``''``, then the solution - will be printed to ``stdout``. - - If any iteration limit is reached, no solution will be - available. - - ``OptimizeWarning`` will be raised if any option value set by - the user is found to be incorrect. - - References - ---------- - .. [1] https://highs.dev/ - .. [2] https://www.maths.ed.ac.uk/hall/HiGHS/HighsOptions.html - ''' - - cdef int numcol = c.size - cdef int numrow = rhs.size - cdef int numnz = avalue.size - cdef int numintegrality = integrality.size - - # Fill up a HighsLp object - cdef HighsLp lp - lp.num_col_ = numcol - lp.num_row_ = numrow - lp.a_matrix_.num_col_ = numcol - lp.a_matrix_.num_row_ = numrow - lp.a_matrix_.format_ = MatrixFormatkColwise - - lp.col_cost_.resize(numcol) - lp.col_lower_.resize(numcol) - lp.col_upper_.resize(numcol) - - lp.row_lower_.resize(numrow) - lp.row_upper_.resize(numrow) - lp.a_matrix_.start_.resize(numcol + 1) - lp.a_matrix_.index_.resize(numnz) - lp.a_matrix_.value_.resize(numnz) - - # only need to set integrality if it's not's empty - cdef HighsVarType * integrality_ptr = NULL - if numintegrality > 0: - lp.integrality_.resize(numintegrality) - integrality_ptr = reinterpret_cast[HighsVarType_ptr](&integrality[0]) - lp.integrality_.assign(integrality_ptr, integrality_ptr + numcol) - - # Explicitly create pointers to pass to HiGHS C++ API; - # do checking to make sure null memory-views are not - # accessed (e.g., &lhs[0] raises exception when lhs is - # empty!) - cdef: - double * colcost_ptr = NULL - double * collower_ptr = NULL - double * colupper_ptr = NULL - double * rowlower_ptr = NULL - double * rowupper_ptr = NULL - int * astart_ptr = NULL - int * aindex_ptr = NULL - double * avalue_ptr = NULL - if numrow > 0: - rowlower_ptr = &lhs[0] - rowupper_ptr = &rhs[0] - lp.row_lower_.assign(rowlower_ptr, rowlower_ptr + numrow) - lp.row_upper_.assign(rowupper_ptr, rowupper_ptr + numrow) - else: - lp.row_lower_.empty() - lp.row_upper_.empty() - if numcol > 0: - colcost_ptr = &c[0] - collower_ptr = &lb[0] - colupper_ptr = &ub[0] - lp.col_cost_.assign(colcost_ptr, colcost_ptr + numcol) - lp.col_lower_.assign(collower_ptr, collower_ptr + numcol) - lp.col_upper_.assign(colupper_ptr, colupper_ptr + numcol) - else: - lp.col_cost_.empty() - lp.col_lower_.empty() - lp.col_upper_.empty() - lp.integrality_.empty() - if numnz > 0: - astart_ptr = &astart[0] - aindex_ptr = &aindex[0] - avalue_ptr = &avalue[0] - lp.a_matrix_.start_.assign(astart_ptr, astart_ptr + numcol + 1) - lp.a_matrix_.index_.assign(aindex_ptr, aindex_ptr + numnz) - lp.a_matrix_.value_.assign(avalue_ptr, avalue_ptr + numnz) - else: - lp.a_matrix_.start_.empty() - lp.a_matrix_.index_.empty() - lp.a_matrix_.value_.empty() - - # Create the options - cdef Highs highs - apply_options(options, highs) - - # Make a Highs object and pass it everything - cdef HighsModelStatus err_model_status = HighsModelStatusNOTSET - cdef HighsStatus init_status = highs.passModel(lp) - if init_status != HighsStatusOK: - if init_status != HighsStatusWarning: - err_model_status = HighsModelStatusMODEL_ERROR - return { - 'status': err_model_status, - 'message': highs.modelStatusToString(err_model_status).decode(), - } - - # Solve the LP - cdef HighsStatus run_status = highs.run() - if run_status == HighsStatusError: - return { - 'status': highs.getModelStatus(), - 'message': highsStatusToString(run_status).decode(), - } - - # Extract what we need from the solution - cdef HighsModelStatus model_status = highs.getModelStatus() - - # We might need an info object if we can look up the solution and a place to put solution - cdef HighsInfo info = highs.getHighsInfo() # it should always be safe to get the info object - cdef HighsSolution solution - cdef HighsBasis basis - cdef double[:, ::1] marg_bnds = np.zeros((2, numcol)) # marg_bnds[0, :]: lower - - # Failure modes: - # LP: if we have anything other than an Optimal status, it - # is unsafe (and unhelpful) to read any results - # MIP: has a non-Optimal status or has timed out/reached max iterations - # 1) If not Optimal/TimedOut/MaxIter status, there is no solution - # 2) If TimedOut/MaxIter status, there may be a feasible solution. - # if the objective function value is not Infinity, then the - # current solution is feasible and can be returned. Else, there - # is no solution. - mipFailCondition = model_status not in { - HighsModelStatusOPTIMAL, - HighsModelStatusREACHED_TIME_LIMIT, - HighsModelStatusREACHED_ITERATION_LIMIT, - } or (model_status in { - HighsModelStatusREACHED_TIME_LIMIT, - HighsModelStatusREACHED_ITERATION_LIMIT, - } and (info.objective_function_value == HIGHS_CONST_INF)) - lpFailCondition = model_status != HighsModelStatusOPTIMAL - if (highs.getLp().isMip() and mipFailCondition) or (not highs.getLp().isMip() and lpFailCondition): - return { - 'status': model_status, - 'message': f'model_status is {highs.modelStatusToString(model_status).decode()}; ' - f'primal_status is {utilBasisStatusToString( info.primal_solution_status).decode()}', - 'simplex_nit': info.simplex_iteration_count, - 'ipm_nit': info.ipm_iteration_count, - 'fun': None, - 'crossover_nit': info.crossover_iteration_count, - } - # If the model status is such that the solution can be read - else: - # Should be safe to read the solution: - solution = highs.getSolution() - basis = highs.getBasis() - - # lagrangians for bounds based on column statuses - for ii in range(numcol): - if HighsBasisStatusLOWER == basis.col_status[ii]: - marg_bnds[0, ii] = solution.col_dual[ii] - elif HighsBasisStatusUPPER == basis.col_status[ii]: - marg_bnds[1, ii] = solution.col_dual[ii] - - res = { - 'status': model_status, - 'message': highs.modelStatusToString(model_status).decode(), - - # Primal solution - 'x': [solution.col_value[ii] for ii in range(numcol)], - - # Ax + s = b => Ax = b - s - # Note: this is for all constraints (A_ub and A_eq) - 'slack': [rhs[ii] - solution.row_value[ii] for ii in range(numrow)], - - # lambda are the lagrange multipliers associated with Ax=b - 'lambda': [solution.row_dual[ii] for ii in range(numrow)], - 'marg_bnds': marg_bnds, - - 'fun': info.objective_function_value, - 'simplex_nit': info.simplex_iteration_count, - 'ipm_nit': info.ipm_iteration_count, - 'crossover_nit': info.crossover_iteration_count, - } - - if highs.getLp().isMip(): - res.update({ - 'mip_node_count': info.mip_node_count, - 'mip_dual_bound': info.mip_dual_bound, - 'mip_gap': info.mip_gap, - }) - - return res diff --git a/scipy/optimize/_highs/cython/src/highs_c_api.pxd b/scipy/optimize/_highs/cython/src/highs_c_api.pxd deleted file mode 100644 index b7097caf30bc..000000000000 --- a/scipy/optimize/_highs/cython/src/highs_c_api.pxd +++ /dev/null @@ -1,7 +0,0 @@ -# cython: language_level=3 - -cdef extern from "highs_c_api.h" nogil: - int Highs_passLp(void* highs, int numcol, int numrow, int numnz, - double* colcost, double* collower, double* colupper, - double* rowlower, double* rowupper, - int* astart, int* aindex, double* avalue) diff --git a/scipy/optimize/_highs/meson.build b/scipy/optimize/_highs/meson.build index 39d8c16226e9..9434512de9b8 100644 --- a/scipy/optimize/_highs/meson.build +++ b/scipy/optimize/_highs/meson.build @@ -1,284 +1,300 @@ -highs_define_macros = [ - '-DCMAKE_BUILD_TYPE="RELEASE"', - '-DFAST_BUILD=ON', - '-DHIGHS_GITHASH="n/a"', - '-DHIGHS_COMPILATION_DATE="2021-07-09"', # cannot generate dynamically - '-DHIGHS_VERSION_MAJOR=1', # don't care about this, look at CMakelists.txt - '-DHIGHS_VERSION_MINOR=2', - '-DHIGHS_VERSION_PATCH=0', - '-DHIGHS_DIR=' + meson.current_source_dir() / '..' / '..' / '_lib' / 'highs', - '-UOPENMP', - '-UEXT_PRESOLVE', - '-USCIP_DEV', - '-UHiGHSDEV', - '-UOSI_FOUND', - '-DNDEBUG' -] +# highs_define_macros = [ +# '-DCMAKE_BUILD_TYPE="RELEASE"', +# '-DFAST_BUILD=ON', +# '-DHIGHS_GITHASH="n/a"', +# '-DHIGHS_COMPILATION_DATE="2021-07-09"', # cannot generate dynamically +# '-DHIGHS_VERSION_MAJOR=1', # don't care about this, look at CMakelists.txt +# '-DHIGHS_VERSION_MINOR=2', +# '-DHIGHS_VERSION_PATCH=0', +# '-DHIGHS_DIR=' + meson.current_source_dir() / '..' / '..' / '_lib' / 'highs', +# '-UOPENMP', +# '-UEXT_PRESOLVE', +# '-USCIP_DEV', +# '-UHiGHSDEV', +# '-UOSI_FOUND', +# '-DNDEBUG' +# ] -basiclu_lib = static_library('basiclu', - [ - '../../_lib/highs/src/ipm/basiclu/basiclu_factorize.c', - '../../_lib/highs/src/ipm/basiclu/basiclu_get_factors.c', - '../../_lib/highs/src/ipm/basiclu/basiclu_initialize.c', - '../../_lib/highs/src/ipm/basiclu/basiclu_object.c', - '../../_lib/highs/src/ipm/basiclu/basiclu_solve_dense.c', - '../../_lib/highs/src/ipm/basiclu/basiclu_solve_for_update.c', - '../../_lib/highs/src/ipm/basiclu/basiclu_solve_sparse.c', - '../../_lib/highs/src/ipm/basiclu/basiclu_update.c', - '../../_lib/highs/src/ipm/basiclu/lu_build_factors.c', - '../../_lib/highs/src/ipm/basiclu/lu_condest.c', - '../../_lib/highs/src/ipm/basiclu/lu_dfs.c', - '../../_lib/highs/src/ipm/basiclu/lu_factorize_bump.c', - '../../_lib/highs/src/ipm/basiclu/lu_file.c', - '../../_lib/highs/src/ipm/basiclu/lu_garbage_perm.c', - '../../_lib/highs/src/ipm/basiclu/lu_initialize.c', - '../../_lib/highs/src/ipm/basiclu/lu_internal.c', - '../../_lib/highs/src/ipm/basiclu/lu_markowitz.c', - '../../_lib/highs/src/ipm/basiclu/lu_matrix_norm.c', - '../../_lib/highs/src/ipm/basiclu/lu_pivot.c', - '../../_lib/highs/src/ipm/basiclu/lu_residual_test.c', - '../../_lib/highs/src/ipm/basiclu/lu_setup_bump.c', - '../../_lib/highs/src/ipm/basiclu/lu_singletons.c', - '../../_lib/highs/src/ipm/basiclu/lu_solve_dense.c', - '../../_lib/highs/src/ipm/basiclu/lu_solve_for_update.c', - '../../_lib/highs/src/ipm/basiclu/lu_solve_sparse.c', - '../../_lib/highs/src/ipm/basiclu/lu_solve_symbolic.c', - '../../_lib/highs/src/ipm/basiclu/lu_solve_triangular.c', - '../../_lib/highs/src/ipm/basiclu/lu_update.c' - ], - include_directories: [ - 'src', - '../../_lib/highs/src', - '../../_lib/highs/src/ipm/basiclu' - ], - c_args: [Wno_unused_variable, highs_define_macros] -) +# basiclu_lib = static_library('basiclu', +# [ +# '../../_lib/highs/src/ipm/basiclu/basiclu_factorize.c', +# '../../_lib/highs/src/ipm/basiclu/basiclu_get_factors.c', +# '../../_lib/highs/src/ipm/basiclu/basiclu_initialize.c', +# '../../_lib/highs/src/ipm/basiclu/basiclu_object.c', +# '../../_lib/highs/src/ipm/basiclu/basiclu_solve_dense.c', +# '../../_lib/highs/src/ipm/basiclu/basiclu_solve_for_update.c', +# '../../_lib/highs/src/ipm/basiclu/basiclu_solve_sparse.c', +# '../../_lib/highs/src/ipm/basiclu/basiclu_update.c', +# '../../_lib/highs/src/ipm/basiclu/lu_build_factors.c', +# '../../_lib/highs/src/ipm/basiclu/lu_condest.c', +# '../../_lib/highs/src/ipm/basiclu/lu_dfs.c', +# '../../_lib/highs/src/ipm/basiclu/lu_factorize_bump.c', +# '../../_lib/highs/src/ipm/basiclu/lu_file.c', +# '../../_lib/highs/src/ipm/basiclu/lu_garbage_perm.c', +# '../../_lib/highs/src/ipm/basiclu/lu_initialize.c', +# '../../_lib/highs/src/ipm/basiclu/lu_internal.c', +# '../../_lib/highs/src/ipm/basiclu/lu_markowitz.c', +# '../../_lib/highs/src/ipm/basiclu/lu_matrix_norm.c', +# '../../_lib/highs/src/ipm/basiclu/lu_pivot.c', +# '../../_lib/highs/src/ipm/basiclu/lu_residual_test.c', +# '../../_lib/highs/src/ipm/basiclu/lu_setup_bump.c', +# '../../_lib/highs/src/ipm/basiclu/lu_singletons.c', +# '../../_lib/highs/src/ipm/basiclu/lu_solve_dense.c', +# '../../_lib/highs/src/ipm/basiclu/lu_solve_for_update.c', +# '../../_lib/highs/src/ipm/basiclu/lu_solve_sparse.c', +# '../../_lib/highs/src/ipm/basiclu/lu_solve_symbolic.c', +# '../../_lib/highs/src/ipm/basiclu/lu_solve_triangular.c', +# '../../_lib/highs/src/ipm/basiclu/lu_update.c' +# ], +# include_directories: [ +# '../../_lib/highs/src', +# '../../_lib/highs/src/ipm/basiclu', +# 'src/' +# ], +# c_args: [Wno_unused_variable, highs_define_macros] +# ) -highs_flags = [ - _cpp_Wno_class_memaccess, - _cpp_Wno_format_truncation, - _cpp_Wno_non_virtual_dtor, - _cpp_Wno_sign_compare, - _cpp_Wno_switch, - _cpp_Wno_unused_but_set_variable, - _cpp_Wno_unused_variable, -] +# highs_flags = [ +# _cpp_Wno_class_memaccess, +# _cpp_Wno_format_truncation, +# _cpp_Wno_non_virtual_dtor, +# _cpp_Wno_sign_compare, +# _cpp_Wno_switch, +# _cpp_Wno_unused_but_set_variable, +# _cpp_Wno_unused_variable, +# ] -ipx_lib = static_library('ipx', - [ - '../../_lib/highs/src/ipm/ipx/basiclu_kernel.cc', - '../../_lib/highs/src/ipm/ipx/basiclu_wrapper.cc', - '../../_lib/highs/src/ipm/ipx/basis.cc', - '../../_lib/highs/src/ipm/ipx/conjugate_residuals.cc', - '../../_lib/highs/src/ipm/ipx/control.cc', - '../../_lib/highs/src/ipm/ipx/crossover.cc', - '../../_lib/highs/src/ipm/ipx/diagonal_precond.cc', - '../../_lib/highs/src/ipm/ipx/forrest_tomlin.cc', - '../../_lib/highs/src/ipm/ipx/guess_basis.cc', - '../../_lib/highs/src/ipm/ipx/indexed_vector.cc', - '../../_lib/highs/src/ipm/ipx/info.cc', - '../../_lib/highs/src/ipm/ipx/ipm.cc', - '../../_lib/highs/src/ipm/ipx/ipx_c.cc', - '../../_lib/highs/src/ipm/ipx/iterate.cc', - '../../_lib/highs/src/ipm/ipx/kkt_solver.cc', - '../../_lib/highs/src/ipm/ipx/kkt_solver_basis.cc', - '../../_lib/highs/src/ipm/ipx/kkt_solver_diag.cc', - '../../_lib/highs/src/ipm/ipx/linear_operator.cc', - '../../_lib/highs/src/ipm/ipx/lp_solver.cc', - '../../_lib/highs/src/ipm/ipx/lu_factorization.cc', - '../../_lib/highs/src/ipm/ipx/lu_update.cc', - '../../_lib/highs/src/ipm/ipx/maxvolume.cc', - '../../_lib/highs/src/ipm/ipx/model.cc', - '../../_lib/highs/src/ipm/ipx/normal_matrix.cc', - '../../_lib/highs/src/ipm/ipx/sparse_matrix.cc', - '../../_lib/highs/src/ipm/ipx/sparse_utils.cc', - '../../_lib/highs/src/ipm/ipx/splitted_normal_matrix.cc', - '../../_lib/highs/src/ipm/ipx/starting_basis.cc', - '../../_lib/highs/src/ipm/ipx/symbolic_invert.cc', - '../../_lib/highs/src/ipm/ipx/timer.cc', - '../../_lib/highs/src/ipm/ipx/utils.cc' - ], - include_directories: [ - '../../_lib/highs/src/ipm/ipx/', - '../../_lib/highs/src/ipm/basiclu/', - '../../_lib/highs/src/', - '../../_lib/highs/extern/', - 'cython/src/' - ], - dependencies: thread_dep, - cpp_args: [highs_flags, highs_define_macros] -) +# ipx_lib = static_library('ipx', +# [ +# '../../_lib/highs/src/ipm/ipx/basiclu_kernel.cc', +# '../../_lib/highs/src/ipm/ipx/basiclu_wrapper.cc', +# '../../_lib/highs/src/ipm/ipx/basis.cc', +# '../../_lib/highs/src/ipm/ipx/conjugate_residuals.cc', +# '../../_lib/highs/src/ipm/ipx/control.cc', +# '../../_lib/highs/src/ipm/ipx/crossover.cc', +# '../../_lib/highs/src/ipm/ipx/diagonal_precond.cc', +# '../../_lib/highs/src/ipm/ipx/forrest_tomlin.cc', +# '../../_lib/highs/src/ipm/ipx/guess_basis.cc', +# '../../_lib/highs/src/ipm/ipx/indexed_vector.cc', +# '../../_lib/highs/src/ipm/ipx/info.cc', +# '../../_lib/highs/src/ipm/ipx/ipm.cc', +# '../../_lib/highs/src/ipm/ipx/ipx_c.cc', +# '../../_lib/highs/src/ipm/ipx/iterate.cc', +# '../../_lib/highs/src/ipm/ipx/kkt_solver.cc', +# '../../_lib/highs/src/ipm/ipx/kkt_solver_basis.cc', +# '../../_lib/highs/src/ipm/ipx/kkt_solver_diag.cc', +# '../../_lib/highs/src/ipm/ipx/linear_operator.cc', +# '../../_lib/highs/src/ipm/ipx/lp_solver.cc', +# '../../_lib/highs/src/ipm/ipx/lu_factorization.cc', +# '../../_lib/highs/src/ipm/ipx/lu_update.cc', +# '../../_lib/highs/src/ipm/ipx/maxvolume.cc', +# '../../_lib/highs/src/ipm/ipx/model.cc', +# '../../_lib/highs/src/ipm/ipx/normal_matrix.cc', +# '../../_lib/highs/src/ipm/ipx/sparse_matrix.cc', +# '../../_lib/highs/src/ipm/ipx/sparse_utils.cc', +# '../../_lib/highs/src/ipm/ipx/splitted_normal_matrix.cc', +# '../../_lib/highs/src/ipm/ipx/starting_basis.cc', +# '../../_lib/highs/src/ipm/ipx/symbolic_invert.cc', +# '../../_lib/highs/src/ipm/ipx/timer.cc', +# '../../_lib/highs/src/ipm/ipx/utils.cc' +# ], +# include_directories: [ +# '../../_lib/highs/src/ipm/ipx/', +# '../../_lib/highs/src/ipm/basiclu/', +# '../../_lib/highs/src/', +# '../../_lib/highs/extern/', +# 'src/' +# ], +# dependencies: thread_dep, +# cpp_args: [highs_flags, highs_define_macros] +# ) -highs_lib = static_library('highs', - [ - '../../_lib/highs/extern/filereaderlp/reader.cpp', - '../../_lib/highs/src/io/Filereader.cpp', - '../../_lib/highs/src/io/FilereaderLp.cpp', - '../../_lib/highs/src/io/FilereaderEms.cpp', - '../../_lib/highs/src/io/FilereaderMps.cpp', - '../../_lib/highs/src/io/HighsIO.cpp', - '../../_lib/highs/src/io/HMPSIO.cpp', - '../../_lib/highs/src/io/HMpsFF.cpp', - '../../_lib/highs/src/io/LoadOptions.cpp', - '../../_lib/highs/src/ipm/IpxWrapper.cpp', - '../../_lib/highs/src/lp_data/Highs.cpp', - '../../_lib/highs/src/lp_data/HighsDebug.cpp', - '../../_lib/highs/src/lp_data/HighsInfo.cpp', - '../../_lib/highs/src/lp_data/HighsInfoDebug.cpp', - '../../_lib/highs/src/lp_data/HighsDeprecated.cpp', - '../../_lib/highs/src/lp_data/HighsInterface.cpp', - '../../_lib/highs/src/lp_data/HighsLp.cpp', - '../../_lib/highs/src/lp_data/HighsLpUtils.cpp', - '../../_lib/highs/src/lp_data/HighsModelUtils.cpp', - '../../_lib/highs/src/lp_data/HighsRanging.cpp', - '../../_lib/highs/src/lp_data/HighsSolution.cpp', - '../../_lib/highs/src/lp_data/HighsSolutionDebug.cpp', - '../../_lib/highs/src/lp_data/HighsSolve.cpp', - '../../_lib/highs/src/lp_data/HighsStatus.cpp', - '../../_lib/highs/src/lp_data/HighsOptions.cpp', - '../../_lib/highs/src/mip/HighsMipSolver.cpp', - '../../_lib/highs/src/mip/HighsMipSolverData.cpp', - '../../_lib/highs/src/mip/HighsDomain.cpp', - '../../_lib/highs/src/mip/HighsDynamicRowMatrix.cpp', - '../../_lib/highs/src/mip/HighsLpRelaxation.cpp', - '../../_lib/highs/src/mip/HighsSeparation.cpp', - '../../_lib/highs/src/mip/HighsSeparator.cpp', - '../../_lib/highs/src/mip/HighsTableauSeparator.cpp', - '../../_lib/highs/src/mip/HighsModkSeparator.cpp', - '../../_lib/highs/src/mip/HighsPathSeparator.cpp', - '../../_lib/highs/src/mip/HighsCutGeneration.cpp', - '../../_lib/highs/src/mip/HighsSearch.cpp', - '../../_lib/highs/src/mip/HighsConflictPool.cpp', - '../../_lib/highs/src/mip/HighsCutPool.cpp', - '../../_lib/highs/src/mip/HighsCliqueTable.cpp', - '../../_lib/highs/src/mip/HighsGFkSolve.cpp', - '../../_lib/highs/src/mip/HighsTransformedLp.cpp', - '../../_lib/highs/src/mip/HighsLpAggregator.cpp', - '../../_lib/highs/src/mip/HighsDebugSol.cpp', - '../../_lib/highs/src/mip/HighsImplications.cpp', - '../../_lib/highs/src/mip/HighsPrimalHeuristics.cpp', - '../../_lib/highs/src/mip/HighsPseudocost.cpp', - '../../_lib/highs/src/mip/HighsRedcostFixing.cpp', - '../../_lib/highs/src/mip/HighsNodeQueue.cpp', - '../../_lib/highs/src/mip/HighsObjectiveFunction.cpp', - '../../_lib/highs/src/model/HighsHessian.cpp', - '../../_lib/highs/src/model/HighsHessianUtils.cpp', - '../../_lib/highs/src/model/HighsModel.cpp', - '../../_lib/highs/src/parallel/HighsTaskExecutor.cpp', - '../../_lib/highs/src/presolve/ICrash.cpp', - '../../_lib/highs/src/presolve/ICrashUtil.cpp', - '../../_lib/highs/src/presolve/ICrashX.cpp', - '../../_lib/highs/src/presolve/HighsPostsolveStack.cpp', - '../../_lib/highs/src/presolve/HighsSymmetry.cpp', - '../../_lib/highs/src/presolve/HPresolve.cpp', - '../../_lib/highs/src/presolve/PresolveComponent.cpp', - '../../_lib/highs/src/qpsolver/basis.cpp', - '../../_lib/highs/src/qpsolver/quass.cpp', - '../../_lib/highs/src/qpsolver/ratiotest.cpp', - '../../_lib/highs/src/qpsolver/scaling.cpp', - '../../_lib/highs/src/qpsolver/perturbation.cpp', - '../../_lib/highs/src/simplex/HEkk.cpp', - '../../_lib/highs/src/simplex/HEkkControl.cpp', - '../../_lib/highs/src/simplex/HEkkDebug.cpp', - '../../_lib/highs/src/simplex/HEkkPrimal.cpp', - '../../_lib/highs/src/simplex/HEkkDual.cpp', - '../../_lib/highs/src/simplex/HEkkDualRHS.cpp', - '../../_lib/highs/src/simplex/HEkkDualRow.cpp', - '../../_lib/highs/src/simplex/HEkkDualMulti.cpp', - '../../_lib/highs/src/simplex/HEkkInterface.cpp', - '../../_lib/highs/src/simplex/HighsSimplexAnalysis.cpp', - '../../_lib/highs/src/simplex/HSimplex.cpp', - '../../_lib/highs/src/simplex/HSimplexDebug.cpp', - '../../_lib/highs/src/simplex/HSimplexNla.cpp', - '../../_lib/highs/src/simplex/HSimplexNlaDebug.cpp', - '../../_lib/highs/src/simplex/HSimplexNlaFreeze.cpp', - '../../_lib/highs/src/simplex/HSimplexNlaProductForm.cpp', - '../../_lib/highs/src/simplex/HSimplexReport.cpp', - '../../_lib/highs/src/test/DevKkt.cpp', - '../../_lib/highs/src/test/KktCh2.cpp', - '../../_lib/highs/src/util/HFactor.cpp', - '../../_lib/highs/src/util/HFactorDebug.cpp', - '../../_lib/highs/src/util/HFactorExtend.cpp', - '../../_lib/highs/src/util/HFactorRefactor.cpp', - '../../_lib/highs/src/util/HFactorUtils.cpp', - '../../_lib/highs/src/util/HighsHash.cpp', - '../../_lib/highs/src/util/HighsLinearSumBounds.cpp', - '../../_lib/highs/src/util/HighsMatrixPic.cpp', - '../../_lib/highs/src/util/HighsMatrixUtils.cpp', - '../../_lib/highs/src/util/HighsSort.cpp', - '../../_lib/highs/src/util/HighsSparseMatrix.cpp', - '../../_lib/highs/src/util/HighsUtils.cpp', - '../../_lib/highs/src/util/HSet.cpp', - '../../_lib/highs/src/util/HVectorBase.cpp', - '../../_lib/highs/src/util/stringutil.cpp', - '../../_lib/highs/src/interfaces/highs_c_api.cpp' - ], - include_directories: [ - 'src/', - '../../_lib/highs/extern/', - '../../_lib/highs/src/', - '../../_lib/highs/src/io/', - '../../_lib/highs/src/ipm/ipx/', - '../../_lib/highs/src/lp_data/', - '../../_lib/highs/src/util/', - ], - dependencies: thread_dep, - cpp_args: [highs_flags, highs_define_macros] -) +# highs_lib = static_library('highs', +# [ +# '../../_lib/highs/extern/filereaderlp/reader.cpp', +# '../../_lib/highs/src/io/Filereader.cpp', +# '../../_lib/highs/src/io/FilereaderLp.cpp', +# '../../_lib/highs/src/io/FilereaderEms.cpp', +# '../../_lib/highs/src/io/FilereaderMps.cpp', +# '../../_lib/highs/src/io/HighsIO.cpp', +# '../../_lib/highs/src/io/HMPSIO.cpp', +# '../../_lib/highs/src/io/HMpsFF.cpp', +# '../../_lib/highs/src/io/LoadOptions.cpp', +# '../../_lib/highs/src/ipm/IpxWrapper.cpp', +# '../../_lib/highs/src/lp_data/Highs.cpp', +# '../../_lib/highs/src/lp_data/HighsDebug.cpp', +# '../../_lib/highs/src/lp_data/HighsInfo.cpp', +# '../../_lib/highs/src/lp_data/HighsInfoDebug.cpp', +# '../../_lib/highs/src/lp_data/HighsDeprecated.cpp', +# '../../_lib/highs/src/lp_data/HighsInterface.cpp', +# '../../_lib/highs/src/lp_data/HighsLp.cpp', +# '../../_lib/highs/src/lp_data/HighsLpUtils.cpp', +# '../../_lib/highs/src/lp_data/HighsModelUtils.cpp', +# '../../_lib/highs/src/lp_data/HighsRanging.cpp', +# '../../_lib/highs/src/lp_data/HighsSolution.cpp', +# '../../_lib/highs/src/lp_data/HighsSolutionDebug.cpp', +# '../../_lib/highs/src/lp_data/HighsSolve.cpp', +# '../../_lib/highs/src/lp_data/HighsStatus.cpp', +# '../../_lib/highs/src/lp_data/HighsOptions.cpp', +# '../../_lib/highs/src/mip/HighsMipSolver.cpp', +# '../../_lib/highs/src/mip/HighsMipSolverData.cpp', +# '../../_lib/highs/src/mip/HighsDomain.cpp', +# '../../_lib/highs/src/mip/HighsDynamicRowMatrix.cpp', +# '../../_lib/highs/src/mip/HighsLpRelaxation.cpp', +# '../../_lib/highs/src/mip/HighsSeparation.cpp', +# '../../_lib/highs/src/mip/HighsSeparator.cpp', +# '../../_lib/highs/src/mip/HighsTableauSeparator.cpp', +# '../../_lib/highs/src/mip/HighsModkSeparator.cpp', +# '../../_lib/highs/src/mip/HighsPathSeparator.cpp', +# '../../_lib/highs/src/mip/HighsCutGeneration.cpp', +# '../../_lib/highs/src/mip/HighsSearch.cpp', +# '../../_lib/highs/src/mip/HighsConflictPool.cpp', +# '../../_lib/highs/src/mip/HighsCutPool.cpp', +# '../../_lib/highs/src/mip/HighsCliqueTable.cpp', +# '../../_lib/highs/src/mip/HighsGFkSolve.cpp', +# '../../_lib/highs/src/mip/HighsTransformedLp.cpp', +# '../../_lib/highs/src/mip/HighsLpAggregator.cpp', +# '../../_lib/highs/src/mip/HighsDebugSol.cpp', +# '../../_lib/highs/src/mip/HighsImplications.cpp', +# '../../_lib/highs/src/mip/HighsPrimalHeuristics.cpp', +# '../../_lib/highs/src/mip/HighsPseudocost.cpp', +# '../../_lib/highs/src/mip/HighsRedcostFixing.cpp', +# '../../_lib/highs/src/mip/HighsNodeQueue.cpp', +# '../../_lib/highs/src/mip/HighsObjectiveFunction.cpp', +# '../../_lib/highs/src/model/HighsHessian.cpp', +# '../../_lib/highs/src/model/HighsHessianUtils.cpp', +# '../../_lib/highs/src/model/HighsModel.cpp', +# '../../_lib/highs/src/parallel/HighsTaskExecutor.cpp', +# '../../_lib/highs/src/presolve/ICrash.cpp', +# '../../_lib/highs/src/presolve/ICrashUtil.cpp', +# '../../_lib/highs/src/presolve/ICrashX.cpp', +# '../../_lib/highs/src/presolve/HighsPostsolveStack.cpp', +# '../../_lib/highs/src/presolve/HighsSymmetry.cpp', +# '../../_lib/highs/src/presolve/HPresolve.cpp', +# '../../_lib/highs/src/presolve/PresolveComponent.cpp', +# '../../_lib/highs/src/qpsolver/basis.cpp', +# '../../_lib/highs/src/qpsolver/quass.cpp', +# '../../_lib/highs/src/qpsolver/ratiotest.cpp', +# '../../_lib/highs/src/qpsolver/scaling.cpp', +# '../../_lib/highs/src/qpsolver/perturbation.cpp', +# '../../_lib/highs/src/simplex/HEkk.cpp', +# '../../_lib/highs/src/simplex/HEkkControl.cpp', +# '../../_lib/highs/src/simplex/HEkkDebug.cpp', +# '../../_lib/highs/src/simplex/HEkkPrimal.cpp', +# '../../_lib/highs/src/simplex/HEkkDual.cpp', +# '../../_lib/highs/src/simplex/HEkkDualRHS.cpp', +# '../../_lib/highs/src/simplex/HEkkDualRow.cpp', +# '../../_lib/highs/src/simplex/HEkkDualMulti.cpp', +# '../../_lib/highs/src/simplex/HEkkInterface.cpp', +# '../../_lib/highs/src/simplex/HighsSimplexAnalysis.cpp', +# '../../_lib/highs/src/simplex/HSimplex.cpp', +# '../../_lib/highs/src/simplex/HSimplexDebug.cpp', +# '../../_lib/highs/src/simplex/HSimplexNla.cpp', +# '../../_lib/highs/src/simplex/HSimplexNlaDebug.cpp', +# '../../_lib/highs/src/simplex/HSimplexNlaFreeze.cpp', +# '../../_lib/highs/src/simplex/HSimplexNlaProductForm.cpp', +# '../../_lib/highs/src/simplex/HSimplexReport.cpp', +# '../../_lib/highs/src/test/DevKkt.cpp', +# '../../_lib/highs/src/test/KktCh2.cpp', +# '../../_lib/highs/src/util/HFactor.cpp', +# '../../_lib/highs/src/util/HFactorDebug.cpp', +# '../../_lib/highs/src/util/HFactorExtend.cpp', +# '../../_lib/highs/src/util/HFactorRefactor.cpp', +# '../../_lib/highs/src/util/HFactorUtils.cpp', +# '../../_lib/highs/src/util/HighsHash.cpp', +# '../../_lib/highs/src/util/HighsLinearSumBounds.cpp', +# '../../_lib/highs/src/util/HighsMatrixPic.cpp', +# '../../_lib/highs/src/util/HighsMatrixUtils.cpp', +# '../../_lib/highs/src/util/HighsSort.cpp', +# '../../_lib/highs/src/util/HighsSparseMatrix.cpp', +# '../../_lib/highs/src/util/HighsUtils.cpp', +# '../../_lib/highs/src/util/HSet.cpp', +# '../../_lib/highs/src/util/HVectorBase.cpp', +# '../../_lib/highs/src/util/stringutil.cpp', +# '../../_lib/highs/src/interfaces/highs_c_api.cpp' +# ], +# include_directories: [ +# '../../_lib/highs/extern/', +# '../../_lib/highs/src/', +# '../../_lib/highs/src/io/', +# '../../_lib/highs/src/ipm/ipx/', +# '../../_lib/highs/src/lp_data/', +# '../../_lib/highs/src/util/', +# 'src/' +# ], +# dependencies: thread_dep, +# cpp_args: [highs_flags, highs_define_macros] +# ) -_highs_wrapper = py3.extension_module('_highs_wrapper', - cython_gen_cpp.process('cython/src/_highs_wrapper.pyx'), - include_directories: [ - 'cython/src/', - 'src/', - '../../_lib/highs/src/', - '../../_lib/highs/src/io/', - '../../_lib/highs/src/lp_data/', - '../../_lib/highs/src/util/' - ], - dependencies: [np_dep, thread_dep, atomic_dep], - link_args: version_link_args, - link_with: [highs_lib, ipx_lib, basiclu_lib], - cpp_args: [highs_flags, highs_define_macros, cython_c_args], - install: true, - subdir: 'scipy/optimize/_highs' -) +# _highs_wrapper = py3.extension_module('_highs_wrapper', +# # cython_gen_cpp.process('cython/src/_highs_wrapper.pyx'), +# # include_directories: [ +# # 'cython/src/', +# # 'src/', +# # '../../_lib/highs/src/', +# # '../../_lib/highs/src/io/', +# # '../../_lib/highs/src/lp_data/', +# # '../../_lib/highs/src/util/' +# # ], +# dependencies: [np_dep, thread_dep, atomic_dep], +# link_args: version_link_args, +# link_with: [highs_lib, ipx_lib, basiclu_lib], +# cpp_args: [highs_flags, highs_define_macros, cython_c_args], +# install: true, +# subdir: 'scipy/optimize/_highs' +# ) -_highs_constants = py3.extension_module('_highs_constants', - cython_gen_cpp.process('cython/src/_highs_constants.pyx'), - c_args: cython_c_args, - include_directories: [ - 'cython/src/', - 'src', - '../../_lib/highs/src/', - '../../_lib/highs/src/io/', - '../../_lib/highs/src/lp_data/', - '../../_lib/highs/src/simplex/' - ], - dependencies: thread_dep, - link_args: version_link_args, - install: true, - subdir: 'scipy/optimize/_highs' -) +# _highs_constants = py3.extension_module('_highs_constants', +# cython_gen_cpp.process('cython/src/_highs_constants.pyx'), +# c_args: cython_c_args, +# include_directories: [ +# 'cython/src/', +# 'src', +# '../../_lib/highs/src/', +# '../../_lib/highs/src/io/', +# '../../_lib/highs/src/lp_data/', +# '../../_lib/highs/src/simplex/' +# ], +# dependencies: thread_dep, +# link_args: version_link_args, +# install: true, +# subdir: 'scipy/optimize/_highs' +# ) -py3.install_sources([ - 'cython/src/HConst.pxd', - 'cython/src/Highs.pxd', - 'cython/src/HighsIO.pxd', - 'cython/src/HighsInfo.pxd', - 'cython/src/HighsLp.pxd', - 'cython/src/HighsLpUtils.pxd', - 'cython/src/HighsModelUtils.pxd', - 'cython/src/HighsOptions.pxd', - 'cython/src/HighsRuntimeOptions.pxd', - 'cython/src/HighsStatus.pxd', - 'cython/src/SimplexConst.pxd', - 'cython/src/highs_c_api.pxd' - ], - subdir: 'scipy/optimize/_highs/src/cython' -) +# subdir('../_lib/highs/highspy') -py3.install_sources( - ['__init__.py'], +# highspyext = py.extension_module( +# 'highspy', +# sources : [ +# 'highs_bindings.cpp', +# ], +# dependencies: [pyb11_dep, highs_dep], +# cpp_args: _args, +# install: true, +# subdir: 'scipy/optimize/_highs' +# ) + +# _highs_wrapper = py3.extension_module('_highs_wrapper', +# # cython_gen_cpp.process('cython/src/_highs_wrapper.pyx'), +# # include_directories: [ +# # 'cython/src/', +# # 'src/', +# # '../../_lib/highs/src/', +# # '../../_lib/highs/src/io/', +# # '../../_lib/highs/src/lp_data/', +# # '../../_lib/highs/src/util/' +# # ], +# dependencies: [np_dep, thread_dep, atomic_dep], +# link_args: version_link_args, +# link_with: [highs_lib, ipx_lib, basiclu_lib], +# cpp_args: [highs_flags, highs_define_macros, cython_c_args], +# install: true, +# subdir: 'scipy/optimize/_highs' +# ) + +py3.install_sources([ + '__init__.py', + '_highs_wrapper.py', +], subdir: 'scipy/optimize/_highs' ) diff --git a/scipy/optimize/_highs/src/libhighs_export.h b/scipy/optimize/_highs/src/libhighs_export.h deleted file mode 100644 index 095eab54c196..000000000000 --- a/scipy/optimize/_highs/src/libhighs_export.h +++ /dev/null @@ -1,50 +0,0 @@ - -#ifndef LIBHIGHS_EXPORT_H -#define LIBHIGHS_EXPORT_H - -#ifdef LIBHIGHS_STATIC_DEFINE -# define LIBHIGHS_EXPORT -# define LIBHIGHS_NO_EXPORT -#else -# ifndef LIBHIGHS_EXPORT -# ifdef libhighs_EXPORTS - /* We are building this library */ -# if defined(_MSC_VER) -# define LIBHIGHS_EXPORT __declspec(dllexport) -# else -# define LIBHIGHS_EXPORT __attribute__((visibility("default"))) -# endif -# else - /* We are using this library */ -# if defined(_MSC_VER) -# define LIBHIGHS_EXPORT __declspec(dllexport) -# else -# define LIBHIGHS_EXPORT __attribute__((visibility("default"))) -# endif -# endif -# endif - -# ifndef LIBHIGHS_NO_EXPORT -# define LIBHIGHS_NO_EXPORT __attribute__((visibility("hidden"))) -# endif -#endif - -#ifndef LIBHIGHS_DEPRECATED -# define LIBHIGHS_DEPRECATED __attribute__ ((__deprecated__)) -#endif - -#ifndef LIBHIGHS_DEPRECATED_EXPORT -# define LIBHIGHS_DEPRECATED_EXPORT LIBHIGHS_EXPORT LIBHIGHS_DEPRECATED -#endif - -#ifndef LIBHIGHS_DEPRECATED_NO_EXPORT -# define LIBHIGHS_DEPRECATED_NO_EXPORT LIBHIGHS_NO_EXPORT LIBHIGHS_DEPRECATED -#endif - -#if 0 /* DEFINE_NO_DEPRECATED */ -# ifndef LIBHIGHS_NO_DEPRECATED -# define LIBHIGHS_NO_DEPRECATED -# endif -#endif - -#endif /* LIBHIGHS_EXPORT_H */ diff --git a/scipy/optimize/_linprog_highs.py b/scipy/optimize/_linprog_highs.py index eb07443bb255..460c79fbc5a5 100644 --- a/scipy/optimize/_linprog_highs.py +++ b/scipy/optimize/_linprog_highs.py @@ -13,97 +13,107 @@ """ -import inspect +from enum import Enum import numpy as np from ._optimize import OptimizeWarning, OptimizeResult from warnings import warn from ._highs._highs_wrapper import _highs_wrapper -from ._highs._highs_constants import ( - CONST_INF, - MESSAGE_LEVEL_NONE, - HIGHS_OBJECTIVE_SENSE_MINIMIZE, - - MODEL_STATUS_NOTSET, - MODEL_STATUS_LOAD_ERROR, - MODEL_STATUS_MODEL_ERROR, - MODEL_STATUS_PRESOLVE_ERROR, - MODEL_STATUS_SOLVE_ERROR, - MODEL_STATUS_POSTSOLVE_ERROR, - MODEL_STATUS_MODEL_EMPTY, - MODEL_STATUS_OPTIMAL, - MODEL_STATUS_INFEASIBLE, - MODEL_STATUS_UNBOUNDED_OR_INFEASIBLE, - MODEL_STATUS_UNBOUNDED, - MODEL_STATUS_REACHED_DUAL_OBJECTIVE_VALUE_UPPER_BOUND - as MODEL_STATUS_RDOVUB, - MODEL_STATUS_REACHED_OBJECTIVE_TARGET, - MODEL_STATUS_REACHED_TIME_LIMIT, - MODEL_STATUS_REACHED_ITERATION_LIMIT, - - HIGHS_SIMPLEX_STRATEGY_DUAL, - - HIGHS_SIMPLEX_CRASH_STRATEGY_OFF, - - HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_CHOOSE, - HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_DANTZIG, - HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_DEVEX, - HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE, -) from scipy.sparse import csc_matrix, vstack, issparse - -def _highs_to_scipy_status_message(highs_status, highs_message): - """Converts HiGHS status number/message to SciPy status number/message""" - - scipy_statuses_messages = { - None: (4, "HiGHS did not provide a status code. "), - MODEL_STATUS_NOTSET: (4, ""), - MODEL_STATUS_LOAD_ERROR: (4, ""), - MODEL_STATUS_MODEL_ERROR: (2, ""), - MODEL_STATUS_PRESOLVE_ERROR: (4, ""), - MODEL_STATUS_SOLVE_ERROR: (4, ""), - MODEL_STATUS_POSTSOLVE_ERROR: (4, ""), - MODEL_STATUS_MODEL_EMPTY: (4, ""), - MODEL_STATUS_RDOVUB: (4, ""), - MODEL_STATUS_REACHED_OBJECTIVE_TARGET: (4, ""), - MODEL_STATUS_OPTIMAL: (0, "Optimization terminated successfully. "), - MODEL_STATUS_REACHED_TIME_LIMIT: (1, "Time limit reached. "), - MODEL_STATUS_REACHED_ITERATION_LIMIT: (1, "Iteration limit reached. "), - MODEL_STATUS_INFEASIBLE: (2, "The problem is infeasible. "), - MODEL_STATUS_UNBOUNDED: (3, "The problem is unbounded. "), - MODEL_STATUS_UNBOUNDED_OR_INFEASIBLE: (4, "The problem is unbounded " - "or infeasible. ")} - unrecognized = (4, "The HiGHS status code was not recognized. ") - scipy_status, scipy_message = ( - scipy_statuses_messages.get(highs_status, unrecognized)) - scipy_message = (f"{scipy_message}" - f"(HiGHS Status {highs_status}: {highs_message})") - return scipy_status, scipy_message - +from highspy import HighsModelStatus as hms +from highspy import simplex_constants as simpc +import highspy as hspy + +class SciPyRC(Enum): + """Return codes for SciPy solvers""" + OPTIMAL = 0 + ITERATION_LIMIT = 1 + INFEASIBLE = 2 + UNBOUNDED = 3 + NUMERICAL = 4 + + def to_string(self): + if self == SciPyRC.OPTIMAL: + return "Optimization terminated successfully. " + elif self == SciPyRC.ITERATION_LIMIT: + return "Iteration limit reached. " + elif self == SciPyRC.INFEASIBLE: + return "The problem is infeasible. " + elif self == SciPyRC.UNBOUNDED: + return "The problem is unbounded. " + elif self == SciPyRC.NUMERICAL: + return "Serious numerical difficulties encountered. " + else: + return "" + +class HighsStatusMapping: + """Class to map HiGHS statuses to SciPy Return Codes""" + def __init__(self): + # Custom mapping from HiGHS status and errors to SciPy status + self.highs_to_scipy = { + hms.kNotset: (SciPyRC.NUMERICAL, "Not set"), + hms.kLoadError: (SciPyRC.NUMERICAL, "Load Error"), + hms.kModelError: (SciPyRC.INFEASIBLE, "Model Error"), + hms.kPresolveError: (SciPyRC.NUMERICAL, "Presolve Error"), + hms.kSolveError: (SciPyRC.NUMERICAL, "Solve Error"), + hms.kPostsolveError: (SciPyRC.NUMERICAL, "Postsolve Error"), + hms.kModelEmpty: (SciPyRC.NUMERICAL, "Model Empty"), + hms.kOptimal: (SciPyRC.OPTIMAL, "Optimal"), + hms.kInfeasible: (SciPyRC.INFEASIBLE, "Infeasible"), + hms.kUnboundedOrInfeasible: (SciPyRC.NUMERICAL, "Unbounded or Infeasible"), + hms.kUnbounded: (SciPyRC.UNBOUNDED, "Unbounded"), + hms.kObjectiveBound: (SciPyRC.NUMERICAL, "Objective Bound"), + hms.kObjectiveTarget: (SciPyRC.NUMERICAL, "Objective Target"), + hms.kTimeLimit: (SciPyRC.ITERATION_LIMIT, "Time Limit"), + hms.kIterationLimit: (SciPyRC.ITERATION_LIMIT, "Iteration Limit"), + hms.kUnknown: (SciPyRC.NUMERICAL, "Unknown"), + hms.kSolutionLimit: (SciPyRC.NUMERICAL, "Solution Limit"), + } + + + def get_scipy_status(self, highs_status, highs_message): + """Converts HiGHS status and message to SciPy status and message""" + scipy_status, message_prefix = self.highs_to_scipy.get(hspy.HighsModelStatus(highs_status), + (SciPyRC.NUMERICAL, "Unknown HiGHS Status")) + scip = SciPyRC(scipy_status) + scipy_message = f"{scip.to_string()} (HiGHS Status {highs_status}: {highs_message})" + return scipy_status.value, scipy_message def _replace_inf(x): - # Replace `np.inf` with CONST_INF + # Replace `np.inf` with kHighsInf infs = np.isinf(x) with np.errstate(invalid="ignore"): - x[infs] = np.sign(x[infs])*CONST_INF + x[infs] = np.sign(x[infs])*hspy.kHighsInf return x -def _convert_to_highs_enum(option, option_str, choices): - # If option is in the choices we can look it up, if not use - # the default value taken from function signature and warn: +class SimplexStrategy(Enum): + DANTZIG = 'dantzig' + DEVEX = 'devex' + STEEPEST_DEVEX = 'steepest-devex' # highs min, choose + STEEPEST = 'steepest' # highs max + + def to_highs_enum(self): + mapping = { + SimplexStrategy.DANTZIG: simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategyDantzig.value, + SimplexStrategy.DEVEX: simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategyDevex.value, + SimplexStrategy.STEEPEST_DEVEX: simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategyChoose.value, + SimplexStrategy.STEEPEST: simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategySteepestEdge.value, + } + return mapping.get(self) + +def convert_to_highs_enum(option, option_str, choices_enum, default_value): + if option is None: + return choices_enum[default_value.upper()].to_highs_enum() try: - return choices[option.lower()] - except AttributeError: - return choices[option] + enum_value = choices_enum[option.upper()] except KeyError: - sig = inspect.signature(_linprog_highs) - default_str = sig.parameters[option_str].default warn(f"Option {option_str} is {option}, but only values in " - f"{set(choices.keys())} are allowed. Using default: " - f"{default_str}.", + f"{[e.value for e in choices_enum]} are allowed. Using default: " + f"{default_value}.", OptimizeWarning, stacklevel=3) - return choices[default_str] + enum_value = choices_enum[default_value.upper()] + return enum_value.to_highs_enum() def _linprog_highs(lp, solver, time_limit=None, presolve=True, @@ -309,15 +319,12 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, warn(message, OptimizeWarning, stacklevel=3) # Map options to HiGHS enum values - simplex_dual_edge_weight_strategy_enum = _convert_to_highs_enum( + simplex_dual_edge_weight_strategy_enum = convert_to_highs_enum( simplex_dual_edge_weight_strategy, - 'simplex_dual_edge_weight_strategy', - choices={'dantzig': HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_DANTZIG, - 'devex': HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_DEVEX, - 'steepest-devex': HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_CHOOSE, - 'steepest': - HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE, - None: None}) + "simplex_dual_edge_weight_strategy", + choices_enum=SimplexStrategy, + default_value="dantzig", + ) c, A_ub, b_ub, A_eq, b_eq, bounds, x0, integrality = lp @@ -339,10 +346,10 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, options = { 'presolve': presolve, - 'sense': HIGHS_OBJECTIVE_SENSE_MINIMIZE, + 'sense': hspy.ObjSense.kMinimize, 'solver': solver, 'time_limit': time_limit, - 'highs_debug_level': MESSAGE_LEVEL_NONE, + # 'highs_debug_level': hspy.kHighs, # TODO 'dual_feasibility_tolerance': dual_feasibility_tolerance, 'ipm_optimality_tolerance': ipm_optimality_tolerance, 'log_to_console': disp, @@ -351,8 +358,8 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, 'primal_feasibility_tolerance': primal_feasibility_tolerance, 'simplex_dual_edge_weight_strategy': simplex_dual_edge_weight_strategy_enum, - 'simplex_strategy': HIGHS_SIMPLEX_STRATEGY_DUAL, - 'simplex_crash_strategy': HIGHS_SIMPLEX_CRASH_STRATEGY_OFF, + 'simplex_strategy': simpc.kSimplexStrategyDual.value, + # 'simplex_crash_strategy': simpc.SimplexCrashStrategy.kSimplexCrashStrategyOff, 'ipm_iteration_limit': maxiter, 'simplex_iteration_limit': maxiter, 'mip_rel_gap': mip_rel_gap, @@ -397,9 +404,10 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, # this needs to be updated if we start choosing the solver intelligently # Convert to scipy-style status and message + highs_mapper = HighsStatusMapping() highs_status = res.get('status', None) highs_message = res.get('message', None) - status, message = _highs_to_scipy_status_message(highs_status, + status, message = highs_mapper.get_scipy_status(highs_status, highs_message) x = np.array(res['x']) if 'x' in res else None @@ -424,7 +432,7 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, }), 'fun': res.get('fun'), 'status': status, - 'success': res['status'] == MODEL_STATUS_OPTIMAL, + 'success': res['status'] == hms.kOptimal, 'message': message, 'nit': res.get('simplex_nit', 0) or res.get('ipm_nit', 0), 'crossover_nit': res.get('crossover_nit'), diff --git a/scipy/optimize/_milp.py b/scipy/optimize/_milp.py index fd9ecf52083f..bc4f192b4992 100644 --- a/scipy/optimize/_milp.py +++ b/scipy/optimize/_milp.py @@ -5,7 +5,7 @@ from ._highs._highs_wrapper import _highs_wrapper # type: ignore[import] from ._constraints import LinearConstraint, Bounds from ._optimize import OptimizeResult -from ._linprog_highs import _highs_to_scipy_status_message +from ._linprog_highs import HighsStatusMapping def _constraints_to_components(constraints): @@ -377,8 +377,9 @@ def milp(c, *, integrality=None, bounds=None, constraints=None, options=None): # Convert to scipy-style status and message highs_status = highs_res.get('status', None) highs_message = highs_res.get('message', None) - status, message = _highs_to_scipy_status_message(highs_status, - highs_message) + hstat = HighsStatusMapping() + status, message = hstat.get_scipy_status(highs_status, + highs_message) res['status'] = status res['message'] = message res['success'] = (status == 0) diff --git a/scipy/optimize/tests/test_linprog.py b/scipy/optimize/tests/test_linprog.py index 49a0f8de5a20..c2026ae09656 100644 --- a/scipy/optimize/tests/test_linprog.py +++ b/scipy/optimize/tests/test_linprog.py @@ -303,9 +303,10 @@ def test_deprecation(): def test_highs_status_message(): res = linprog(1, method='highs') - msg = "Optimization terminated successfully. (HiGHS Status 7:" + msg = "Optimization terminated successfully." assert res.status == 0 assert res.message.startswith(msg) + assert 'HiGHS Status 7' in res.message A, b, c, numbers, M = magic_square(6) bounds = [(0, 1)] * len(c) @@ -313,37 +314,46 @@ def test_highs_status_message(): options = {"time_limit": 0.1} res = linprog(c=c, A_eq=A, b_eq=b, bounds=bounds, method='highs', options=options, integrality=integrality) - msg = "Time limit reached. (HiGHS Status 13:" + msg = "Time limit reached" assert res.status == 1 - assert res.message.startswith(msg) + assert msg in res.message + assert 'HiGHS Status 13' in res.message options = {"maxiter": 10} res = linprog(c=c, A_eq=A, b_eq=b, bounds=bounds, method='highs-ds', options=options) - msg = "Iteration limit reached. (HiGHS Status 14:" + msg = "Iteration limit reached" assert res.status == 1 assert res.message.startswith(msg) + assert 'HiGHS Status 14' in res.message res = linprog(1, bounds=(1, -1), method='highs') - msg = "The problem is infeasible. (HiGHS Status 8:" + msg = "The problem is infeasible" assert res.status == 2 assert res.message.startswith(msg) - - res = linprog(-1, method='highs') - msg = "The problem is unbounded. (HiGHS Status 10:" - assert res.status == 3 - assert res.message.startswith(msg) - - from scipy.optimize._linprog_highs import _highs_to_scipy_status_message - status, message = _highs_to_scipy_status_message(58, "Hello!") - msg = "The HiGHS status code was not recognized. (HiGHS Status 58:" - assert status == 4 - assert message.startswith(msg) - - status, message = _highs_to_scipy_status_message(None, None) - msg = "HiGHS did not provide a status code. (HiGHS Status None: None)" + assert 'HiGHS Status 8' in res.message + + # TODO: Fix this + # res = linprog(-1, method='highs') + # msg = "The problem is unbounded." + # assert res.status == 3 + # assert res.message.startswith(msg) + # assert 'HiGHS Status 10' in res.message + + from scipy.optimize._linprog_highs import HighsStatusMapping + highs_mapper = HighsStatusMapping() + status, message = highs_mapper.get_scipy_status(58, "Hello!") + # msg = "The HiGHS status code was not recognized. (HiGHS Status 58:" assert status == 4 - assert message.startswith(msg) + # TODO: Fix this + # assert message.startswith(msg) + assert "HiGHS Status 58" in message + + # TODO: Fix this + # status, message = highs_mapper.get_scipy_status(None, None) + # msg = "HiGHS did not provide a status code. (HiGHS Status None: None)" + # assert status == 4 + # assert message.startswith(msg) def test_bug_17380(): diff --git a/scipy/optimize/tests/test_milp.py b/scipy/optimize/tests/test_milp.py index 0970a15a8bcc..c36ba5e19fb9 100644 --- a/scipy/optimize/tests/test_milp.py +++ b/scipy/optimize/tests/test_milp.py @@ -97,8 +97,9 @@ def test_result(): res = milp(c=c, constraints=(A, b, b), bounds=(0, 1), integrality=1) assert res.status == 0 assert res.success - msg = "Optimization terminated successfully. (HiGHS Status 7:" + msg = "Optimization terminated successfully." assert res.message.startswith(msg) + assert 'HiGHS Status 7' in res.message assert isinstance(res.x, np.ndarray) assert isinstance(res.fun, float) assert isinstance(res.mip_node_count, int) @@ -110,26 +111,29 @@ def test_result(): options={'time_limit': 0.05}) assert res.status == 1 assert not res.success - msg = "Time limit reached. (HiGHS Status 13:" - assert res.message.startswith(msg) + msg = "Time limit reached" + assert 'HiGHS Status 13' in res.message + assert msg in res.message assert (res.fun is res.mip_dual_bound is res.mip_gap is res.mip_node_count is res.x is None) res = milp(1, bounds=(1, -1)) assert res.status == 2 assert not res.success - msg = "The problem is infeasible. (HiGHS Status 8:" + msg = "The problem is infeasible" + assert 'HiGHS Status 8' in res.message assert res.message.startswith(msg) assert (res.fun is res.mip_dual_bound is res.mip_gap is res.mip_node_count is res.x is None) - res = milp(-1) - assert res.status == 3 - assert not res.success - msg = "The problem is unbounded. (HiGHS Status 10:" - assert res.message.startswith(msg) - assert (res.fun is res.mip_dual_bound is res.mip_gap - is res.mip_node_count is res.x is None) + # TODO: Fix this + # res = milp(-1) + # assert res.status == 3 + # assert not res.success + # msg = "The problem is unbounded. (HiGHS Status 10:" + # assert res.message.startswith(msg) + # assert (res.fun is res.mip_dual_bound is res.mip_gap + # is res.mip_node_count is res.x is None) def test_milp_optional_args(): From d41e540a09bd701d627eb29339b5aa5d724f0261 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 30 Sep 2023 10:10:10 +0000 Subject: [PATCH 03/64] MAINT: Clean out old meson build MAINT: Minor renaming for the modeling API BLD: Rework to use highs as a subproject MAINT: Remove highs submodule and rework --- .gitignore | 4 +- .gitmodules | 4 - LICENSES_bundled.txt | 4 +- mypy.ini | 9 - scipy/_lib/highs | 1 - scipy/_lib/meson.build | 29 ++- scipy/optimize/_highs/meson.build | 294 ------------------------------ scipy/optimize/_linprog_highs.py | 6 +- subprojects/highs.wrap | 3 + 9 files changed, 36 insertions(+), 318 deletions(-) delete mode 160000 scipy/_lib/highs create mode 100644 subprojects/highs.wrap diff --git a/.gitignore b/.gitignore index cb5a15536fe2..9e4e9b14b9b8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +# Wrap directories +subprojects/highs # Editor temporary/working/backup files # ######################################### .#* @@ -324,5 +326,3 @@ scipy/optimize/_group_columns.c scipy/optimize/cython_optimize/_zeros.c scipy/optimize/cython_optimize/_zeros.pyx scipy/optimize/lbfgsb/_lbfgsbmodule.c -scipy/optimize/_highs/cython/src/_highs_wrapper.cxx -scipy/optimize/_highs/cython/src/_highs_constants.cxx diff --git a/.gitmodules b/.gitmodules index 6bcfddf6bc02..39cd9bfe50a3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,10 +9,6 @@ path = scipy/_lib/unuran url = https://github.com/scipy/unuran.git shallow = true -[submodule "HiGHS"] - path = scipy/_lib/highs - url = https://github.com/HaoZeke/highs - shallow = true [submodule "scipy/_lib/boost_math"] path = scipy/_lib/boost_math url = https://github.com/boostorg/math.git diff --git a/LICENSES_bundled.txt b/LICENSES_bundled.txt index 9a3943898822..0c867c19c7b2 100644 --- a/LICENSES_bundled.txt +++ b/LICENSES_bundled.txt @@ -252,9 +252,9 @@ License: OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Name: HiGHS -Files: scipy/optimize/_highs/* +Files: subprojects/highs/* License: MIT - For details, see scipy/optimize/_highs/LICENCE + For details, see subprojects/highs/LICENCE Name: Boost Files: scipy/_lib/boost_math/* diff --git a/mypy.ini b/mypy.ini index aa840b19701e..3cf9d47a9aac 100644 --- a/mypy.ini +++ b/mypy.ini @@ -280,15 +280,6 @@ ignore_errors = True [mypy-scipy.optimize._linprog_util] ignore_errors = True -[mypy-scipy.optimize._linprog_highs] -ignore_errors = True - -[mypy-scipy.optimize._highs.highs_wrapper] -ignore_errors = True - -[mypy-scipy.optimize._highs.constants] -ignore_errors = True - [mypy-scipy.optimize._trustregion] ignore_errors = True diff --git a/scipy/_lib/highs b/scipy/_lib/highs deleted file mode 160000 index 090453608adb..000000000000 --- a/scipy/_lib/highs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 090453608adb6a1f14dbee01dfd117b80aa8938a diff --git a/scipy/_lib/meson.build b/scipy/_lib/meson.build index f57e48f078fe..eab13731a701 100644 --- a/scipy/_lib/meson.build +++ b/scipy/_lib/meson.build @@ -2,9 +2,6 @@ fs = import('fs') if not fs.exists('boost_math/README.md') error('Missing the `boost` submodule! Run `git submodule update --init` to fix this.') endif -if not fs.exists('highs/README.md') - error('Missing the `highs` submodule! Run `git submodule update --init` to fix this.') -endif if not fs.exists('unuran/README.md') error('Missing the `unuran` submodule! Run `git submodule update --init` to fix this.') endif @@ -181,3 +178,29 @@ py3.install_sources( subdir('_uarray') subdir('tests') + +# Setup the highs library +highs_proj = subproject('highs', + default_options : ['default_library=static', + 'with_pybind11=True']) +highs_dep = highs_proj.get_variable('highs_dep') +highspy_py = highs_proj.get_variable('highspy_py') +highspy_cpp = highs_proj.get_variable('highspy_cpp') + +pyb11_dep = [ + py3.dependency(), + dependency('pybind11') +] + +highspyext = py3.extension_module( + '_highs', + sources : highspy_cpp, + dependencies: [pyb11_dep, highs_dep], + install: true, + subdir: 'highspy', +) + +py3.install_sources( + highspy_py, + subdir: 'highspy', +) diff --git a/scipy/optimize/_highs/meson.build b/scipy/optimize/_highs/meson.build index 9434512de9b8..2e4ace80dda1 100644 --- a/scipy/optimize/_highs/meson.build +++ b/scipy/optimize/_highs/meson.build @@ -1,297 +1,3 @@ -# highs_define_macros = [ -# '-DCMAKE_BUILD_TYPE="RELEASE"', -# '-DFAST_BUILD=ON', -# '-DHIGHS_GITHASH="n/a"', -# '-DHIGHS_COMPILATION_DATE="2021-07-09"', # cannot generate dynamically -# '-DHIGHS_VERSION_MAJOR=1', # don't care about this, look at CMakelists.txt -# '-DHIGHS_VERSION_MINOR=2', -# '-DHIGHS_VERSION_PATCH=0', -# '-DHIGHS_DIR=' + meson.current_source_dir() / '..' / '..' / '_lib' / 'highs', -# '-UOPENMP', -# '-UEXT_PRESOLVE', -# '-USCIP_DEV', -# '-UHiGHSDEV', -# '-UOSI_FOUND', -# '-DNDEBUG' -# ] - -# basiclu_lib = static_library('basiclu', -# [ -# '../../_lib/highs/src/ipm/basiclu/basiclu_factorize.c', -# '../../_lib/highs/src/ipm/basiclu/basiclu_get_factors.c', -# '../../_lib/highs/src/ipm/basiclu/basiclu_initialize.c', -# '../../_lib/highs/src/ipm/basiclu/basiclu_object.c', -# '../../_lib/highs/src/ipm/basiclu/basiclu_solve_dense.c', -# '../../_lib/highs/src/ipm/basiclu/basiclu_solve_for_update.c', -# '../../_lib/highs/src/ipm/basiclu/basiclu_solve_sparse.c', -# '../../_lib/highs/src/ipm/basiclu/basiclu_update.c', -# '../../_lib/highs/src/ipm/basiclu/lu_build_factors.c', -# '../../_lib/highs/src/ipm/basiclu/lu_condest.c', -# '../../_lib/highs/src/ipm/basiclu/lu_dfs.c', -# '../../_lib/highs/src/ipm/basiclu/lu_factorize_bump.c', -# '../../_lib/highs/src/ipm/basiclu/lu_file.c', -# '../../_lib/highs/src/ipm/basiclu/lu_garbage_perm.c', -# '../../_lib/highs/src/ipm/basiclu/lu_initialize.c', -# '../../_lib/highs/src/ipm/basiclu/lu_internal.c', -# '../../_lib/highs/src/ipm/basiclu/lu_markowitz.c', -# '../../_lib/highs/src/ipm/basiclu/lu_matrix_norm.c', -# '../../_lib/highs/src/ipm/basiclu/lu_pivot.c', -# '../../_lib/highs/src/ipm/basiclu/lu_residual_test.c', -# '../../_lib/highs/src/ipm/basiclu/lu_setup_bump.c', -# '../../_lib/highs/src/ipm/basiclu/lu_singletons.c', -# '../../_lib/highs/src/ipm/basiclu/lu_solve_dense.c', -# '../../_lib/highs/src/ipm/basiclu/lu_solve_for_update.c', -# '../../_lib/highs/src/ipm/basiclu/lu_solve_sparse.c', -# '../../_lib/highs/src/ipm/basiclu/lu_solve_symbolic.c', -# '../../_lib/highs/src/ipm/basiclu/lu_solve_triangular.c', -# '../../_lib/highs/src/ipm/basiclu/lu_update.c' -# ], -# include_directories: [ -# '../../_lib/highs/src', -# '../../_lib/highs/src/ipm/basiclu', -# 'src/' -# ], -# c_args: [Wno_unused_variable, highs_define_macros] -# ) - -# highs_flags = [ -# _cpp_Wno_class_memaccess, -# _cpp_Wno_format_truncation, -# _cpp_Wno_non_virtual_dtor, -# _cpp_Wno_sign_compare, -# _cpp_Wno_switch, -# _cpp_Wno_unused_but_set_variable, -# _cpp_Wno_unused_variable, -# ] - -# ipx_lib = static_library('ipx', -# [ -# '../../_lib/highs/src/ipm/ipx/basiclu_kernel.cc', -# '../../_lib/highs/src/ipm/ipx/basiclu_wrapper.cc', -# '../../_lib/highs/src/ipm/ipx/basis.cc', -# '../../_lib/highs/src/ipm/ipx/conjugate_residuals.cc', -# '../../_lib/highs/src/ipm/ipx/control.cc', -# '../../_lib/highs/src/ipm/ipx/crossover.cc', -# '../../_lib/highs/src/ipm/ipx/diagonal_precond.cc', -# '../../_lib/highs/src/ipm/ipx/forrest_tomlin.cc', -# '../../_lib/highs/src/ipm/ipx/guess_basis.cc', -# '../../_lib/highs/src/ipm/ipx/indexed_vector.cc', -# '../../_lib/highs/src/ipm/ipx/info.cc', -# '../../_lib/highs/src/ipm/ipx/ipm.cc', -# '../../_lib/highs/src/ipm/ipx/ipx_c.cc', -# '../../_lib/highs/src/ipm/ipx/iterate.cc', -# '../../_lib/highs/src/ipm/ipx/kkt_solver.cc', -# '../../_lib/highs/src/ipm/ipx/kkt_solver_basis.cc', -# '../../_lib/highs/src/ipm/ipx/kkt_solver_diag.cc', -# '../../_lib/highs/src/ipm/ipx/linear_operator.cc', -# '../../_lib/highs/src/ipm/ipx/lp_solver.cc', -# '../../_lib/highs/src/ipm/ipx/lu_factorization.cc', -# '../../_lib/highs/src/ipm/ipx/lu_update.cc', -# '../../_lib/highs/src/ipm/ipx/maxvolume.cc', -# '../../_lib/highs/src/ipm/ipx/model.cc', -# '../../_lib/highs/src/ipm/ipx/normal_matrix.cc', -# '../../_lib/highs/src/ipm/ipx/sparse_matrix.cc', -# '../../_lib/highs/src/ipm/ipx/sparse_utils.cc', -# '../../_lib/highs/src/ipm/ipx/splitted_normal_matrix.cc', -# '../../_lib/highs/src/ipm/ipx/starting_basis.cc', -# '../../_lib/highs/src/ipm/ipx/symbolic_invert.cc', -# '../../_lib/highs/src/ipm/ipx/timer.cc', -# '../../_lib/highs/src/ipm/ipx/utils.cc' -# ], -# include_directories: [ -# '../../_lib/highs/src/ipm/ipx/', -# '../../_lib/highs/src/ipm/basiclu/', -# '../../_lib/highs/src/', -# '../../_lib/highs/extern/', -# 'src/' -# ], -# dependencies: thread_dep, -# cpp_args: [highs_flags, highs_define_macros] -# ) - -# highs_lib = static_library('highs', -# [ -# '../../_lib/highs/extern/filereaderlp/reader.cpp', -# '../../_lib/highs/src/io/Filereader.cpp', -# '../../_lib/highs/src/io/FilereaderLp.cpp', -# '../../_lib/highs/src/io/FilereaderEms.cpp', -# '../../_lib/highs/src/io/FilereaderMps.cpp', -# '../../_lib/highs/src/io/HighsIO.cpp', -# '../../_lib/highs/src/io/HMPSIO.cpp', -# '../../_lib/highs/src/io/HMpsFF.cpp', -# '../../_lib/highs/src/io/LoadOptions.cpp', -# '../../_lib/highs/src/ipm/IpxWrapper.cpp', -# '../../_lib/highs/src/lp_data/Highs.cpp', -# '../../_lib/highs/src/lp_data/HighsDebug.cpp', -# '../../_lib/highs/src/lp_data/HighsInfo.cpp', -# '../../_lib/highs/src/lp_data/HighsInfoDebug.cpp', -# '../../_lib/highs/src/lp_data/HighsDeprecated.cpp', -# '../../_lib/highs/src/lp_data/HighsInterface.cpp', -# '../../_lib/highs/src/lp_data/HighsLp.cpp', -# '../../_lib/highs/src/lp_data/HighsLpUtils.cpp', -# '../../_lib/highs/src/lp_data/HighsModelUtils.cpp', -# '../../_lib/highs/src/lp_data/HighsRanging.cpp', -# '../../_lib/highs/src/lp_data/HighsSolution.cpp', -# '../../_lib/highs/src/lp_data/HighsSolutionDebug.cpp', -# '../../_lib/highs/src/lp_data/HighsSolve.cpp', -# '../../_lib/highs/src/lp_data/HighsStatus.cpp', -# '../../_lib/highs/src/lp_data/HighsOptions.cpp', -# '../../_lib/highs/src/mip/HighsMipSolver.cpp', -# '../../_lib/highs/src/mip/HighsMipSolverData.cpp', -# '../../_lib/highs/src/mip/HighsDomain.cpp', -# '../../_lib/highs/src/mip/HighsDynamicRowMatrix.cpp', -# '../../_lib/highs/src/mip/HighsLpRelaxation.cpp', -# '../../_lib/highs/src/mip/HighsSeparation.cpp', -# '../../_lib/highs/src/mip/HighsSeparator.cpp', -# '../../_lib/highs/src/mip/HighsTableauSeparator.cpp', -# '../../_lib/highs/src/mip/HighsModkSeparator.cpp', -# '../../_lib/highs/src/mip/HighsPathSeparator.cpp', -# '../../_lib/highs/src/mip/HighsCutGeneration.cpp', -# '../../_lib/highs/src/mip/HighsSearch.cpp', -# '../../_lib/highs/src/mip/HighsConflictPool.cpp', -# '../../_lib/highs/src/mip/HighsCutPool.cpp', -# '../../_lib/highs/src/mip/HighsCliqueTable.cpp', -# '../../_lib/highs/src/mip/HighsGFkSolve.cpp', -# '../../_lib/highs/src/mip/HighsTransformedLp.cpp', -# '../../_lib/highs/src/mip/HighsLpAggregator.cpp', -# '../../_lib/highs/src/mip/HighsDebugSol.cpp', -# '../../_lib/highs/src/mip/HighsImplications.cpp', -# '../../_lib/highs/src/mip/HighsPrimalHeuristics.cpp', -# '../../_lib/highs/src/mip/HighsPseudocost.cpp', -# '../../_lib/highs/src/mip/HighsRedcostFixing.cpp', -# '../../_lib/highs/src/mip/HighsNodeQueue.cpp', -# '../../_lib/highs/src/mip/HighsObjectiveFunction.cpp', -# '../../_lib/highs/src/model/HighsHessian.cpp', -# '../../_lib/highs/src/model/HighsHessianUtils.cpp', -# '../../_lib/highs/src/model/HighsModel.cpp', -# '../../_lib/highs/src/parallel/HighsTaskExecutor.cpp', -# '../../_lib/highs/src/presolve/ICrash.cpp', -# '../../_lib/highs/src/presolve/ICrashUtil.cpp', -# '../../_lib/highs/src/presolve/ICrashX.cpp', -# '../../_lib/highs/src/presolve/HighsPostsolveStack.cpp', -# '../../_lib/highs/src/presolve/HighsSymmetry.cpp', -# '../../_lib/highs/src/presolve/HPresolve.cpp', -# '../../_lib/highs/src/presolve/PresolveComponent.cpp', -# '../../_lib/highs/src/qpsolver/basis.cpp', -# '../../_lib/highs/src/qpsolver/quass.cpp', -# '../../_lib/highs/src/qpsolver/ratiotest.cpp', -# '../../_lib/highs/src/qpsolver/scaling.cpp', -# '../../_lib/highs/src/qpsolver/perturbation.cpp', -# '../../_lib/highs/src/simplex/HEkk.cpp', -# '../../_lib/highs/src/simplex/HEkkControl.cpp', -# '../../_lib/highs/src/simplex/HEkkDebug.cpp', -# '../../_lib/highs/src/simplex/HEkkPrimal.cpp', -# '../../_lib/highs/src/simplex/HEkkDual.cpp', -# '../../_lib/highs/src/simplex/HEkkDualRHS.cpp', -# '../../_lib/highs/src/simplex/HEkkDualRow.cpp', -# '../../_lib/highs/src/simplex/HEkkDualMulti.cpp', -# '../../_lib/highs/src/simplex/HEkkInterface.cpp', -# '../../_lib/highs/src/simplex/HighsSimplexAnalysis.cpp', -# '../../_lib/highs/src/simplex/HSimplex.cpp', -# '../../_lib/highs/src/simplex/HSimplexDebug.cpp', -# '../../_lib/highs/src/simplex/HSimplexNla.cpp', -# '../../_lib/highs/src/simplex/HSimplexNlaDebug.cpp', -# '../../_lib/highs/src/simplex/HSimplexNlaFreeze.cpp', -# '../../_lib/highs/src/simplex/HSimplexNlaProductForm.cpp', -# '../../_lib/highs/src/simplex/HSimplexReport.cpp', -# '../../_lib/highs/src/test/DevKkt.cpp', -# '../../_lib/highs/src/test/KktCh2.cpp', -# '../../_lib/highs/src/util/HFactor.cpp', -# '../../_lib/highs/src/util/HFactorDebug.cpp', -# '../../_lib/highs/src/util/HFactorExtend.cpp', -# '../../_lib/highs/src/util/HFactorRefactor.cpp', -# '../../_lib/highs/src/util/HFactorUtils.cpp', -# '../../_lib/highs/src/util/HighsHash.cpp', -# '../../_lib/highs/src/util/HighsLinearSumBounds.cpp', -# '../../_lib/highs/src/util/HighsMatrixPic.cpp', -# '../../_lib/highs/src/util/HighsMatrixUtils.cpp', -# '../../_lib/highs/src/util/HighsSort.cpp', -# '../../_lib/highs/src/util/HighsSparseMatrix.cpp', -# '../../_lib/highs/src/util/HighsUtils.cpp', -# '../../_lib/highs/src/util/HSet.cpp', -# '../../_lib/highs/src/util/HVectorBase.cpp', -# '../../_lib/highs/src/util/stringutil.cpp', -# '../../_lib/highs/src/interfaces/highs_c_api.cpp' -# ], -# include_directories: [ -# '../../_lib/highs/extern/', -# '../../_lib/highs/src/', -# '../../_lib/highs/src/io/', -# '../../_lib/highs/src/ipm/ipx/', -# '../../_lib/highs/src/lp_data/', -# '../../_lib/highs/src/util/', -# 'src/' -# ], -# dependencies: thread_dep, -# cpp_args: [highs_flags, highs_define_macros] -# ) - -# _highs_wrapper = py3.extension_module('_highs_wrapper', -# # cython_gen_cpp.process('cython/src/_highs_wrapper.pyx'), -# # include_directories: [ -# # 'cython/src/', -# # 'src/', -# # '../../_lib/highs/src/', -# # '../../_lib/highs/src/io/', -# # '../../_lib/highs/src/lp_data/', -# # '../../_lib/highs/src/util/' -# # ], -# dependencies: [np_dep, thread_dep, atomic_dep], -# link_args: version_link_args, -# link_with: [highs_lib, ipx_lib, basiclu_lib], -# cpp_args: [highs_flags, highs_define_macros, cython_c_args], -# install: true, -# subdir: 'scipy/optimize/_highs' -# ) - -# _highs_constants = py3.extension_module('_highs_constants', -# cython_gen_cpp.process('cython/src/_highs_constants.pyx'), -# c_args: cython_c_args, -# include_directories: [ -# 'cython/src/', -# 'src', -# '../../_lib/highs/src/', -# '../../_lib/highs/src/io/', -# '../../_lib/highs/src/lp_data/', -# '../../_lib/highs/src/simplex/' -# ], -# dependencies: thread_dep, -# link_args: version_link_args, -# install: true, -# subdir: 'scipy/optimize/_highs' -# ) - -# subdir('../_lib/highs/highspy') - -# highspyext = py.extension_module( -# 'highspy', -# sources : [ -# 'highs_bindings.cpp', -# ], -# dependencies: [pyb11_dep, highs_dep], -# cpp_args: _args, -# install: true, -# subdir: 'scipy/optimize/_highs' -# ) - -# _highs_wrapper = py3.extension_module('_highs_wrapper', -# # cython_gen_cpp.process('cython/src/_highs_wrapper.pyx'), -# # include_directories: [ -# # 'cython/src/', -# # 'src/', -# # '../../_lib/highs/src/', -# # '../../_lib/highs/src/io/', -# # '../../_lib/highs/src/lp_data/', -# # '../../_lib/highs/src/util/' -# # ], -# dependencies: [np_dep, thread_dep, atomic_dep], -# link_args: version_link_args, -# link_with: [highs_lib, ipx_lib, basiclu_lib], -# cpp_args: [highs_flags, highs_define_macros, cython_c_args], -# install: true, -# subdir: 'scipy/optimize/_highs' -# ) - py3.install_sources([ '__init__.py', '_highs_wrapper.py', diff --git a/scipy/optimize/_linprog_highs.py b/scipy/optimize/_linprog_highs.py index 460c79fbc5a5..796ac52e18da 100644 --- a/scipy/optimize/_linprog_highs.py +++ b/scipy/optimize/_linprog_highs.py @@ -20,9 +20,9 @@ from ._highs._highs_wrapper import _highs_wrapper from scipy.sparse import csc_matrix, vstack, issparse -from highspy import HighsModelStatus as hms -from highspy import simplex_constants as simpc -import highspy as hspy +from highspy._highs import HighsModelStatus as hms +from highspy._highs import simplex_constants as simpc +import highspy._highs as hspy class SciPyRC(Enum): """Return codes for SciPy solvers""" diff --git a/subprojects/highs.wrap b/subprojects/highs.wrap new file mode 100644 index 000000000000..137b46dfeb60 --- /dev/null +++ b/subprojects/highs.wrap @@ -0,0 +1,3 @@ +[wrap-git] +url = https://github.com/HaoZeke/highs.git +revision = d0bac8a10ce0d7bed35469c68a95450bee9dce71 From 9b8416e618240e30e223fe599a795ddac10ea2f4 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sun, 1 Oct 2023 15:15:21 +0000 Subject: [PATCH 04/64] ENH: Use highspy for the interface ENH: Use highs_options TST: Fixup tests MAINT: Update to commit with no build warnings MAINT: Try building without errors MAINT: No static libraries.. MAINT: Try static again, move meson files around MAINT: Use an internally built highspy MAINT: Update to latest highspy With fixes as suggested Co-authored-by: rgommers BLD: Use variables defined by scipy BLD: Skip subproject installation Co-authored-by: rgommers MAINT: Provide link_args for highspy MAINT: Cleanup with newer API [highspy] MAINT: Cleanup to prevent building highspy ... outside of scipy at any rate MAINT: Update to streamlined highspy api TST: Revert changes to testsuite [linprog] Remain 100% backwards compatible --- pyproject.toml | 2 + scipy/_lib/meson.build | 26 --- scipy/meson.build | 2 + scipy/optimize/_highs/_highs_wrapper.py | 209 ++++++++++-------- scipy/optimize/_highs/meson.build | 42 ++++ scipy/optimize/_highs/src/HConfig.h | 0 scipy/optimize/_linprog_highs.py | 273 ++++++++++++++---------- scipy/optimize/tests/test_linprog.py | 39 ++-- scipy/optimize/tests/test_milp.py | 22 +- subprojects/highs.wrap | 2 +- 10 files changed, 350 insertions(+), 267 deletions(-) delete mode 100644 scipy/optimize/_highs/src/HConfig.h diff --git a/pyproject.toml b/pyproject.toml index 9c9de1bedae9..0b144e7969cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -127,6 +127,8 @@ tracker = "https://github.com/scipy/scipy/issues" [tool.doit] dodoFile = "dev.py" +[tool.meson-python.args] +install = ['--skip-subprojects'] [tool.cibuildwheel] skip = "cp36-* cp37-* cp38-* pp* *_ppc64le *_i686 *_s390x" diff --git a/scipy/_lib/meson.build b/scipy/_lib/meson.build index eab13731a701..7c31e19cf621 100644 --- a/scipy/_lib/meson.build +++ b/scipy/_lib/meson.build @@ -178,29 +178,3 @@ py3.install_sources( subdir('_uarray') subdir('tests') - -# Setup the highs library -highs_proj = subproject('highs', - default_options : ['default_library=static', - 'with_pybind11=True']) -highs_dep = highs_proj.get_variable('highs_dep') -highspy_py = highs_proj.get_variable('highspy_py') -highspy_cpp = highs_proj.get_variable('highspy_cpp') - -pyb11_dep = [ - py3.dependency(), - dependency('pybind11') -] - -highspyext = py3.extension_module( - '_highs', - sources : highspy_cpp, - dependencies: [pyb11_dep, highs_dep], - install: true, - subdir: 'highspy', -) - -py3.install_sources( - highspy_py, - subdir: 'highspy', -) diff --git a/scipy/meson.build b/scipy/meson.build index 768031ba6626..5656401a7176 100644 --- a/scipy/meson.build +++ b/scipy/meson.build @@ -304,6 +304,8 @@ Wno_switch = cc.get_supported_arguments('-Wno-switch') Wno_unused_label = cc.get_supported_arguments('-Wno-unused-label') Wno_unused_result = cc.get_supported_arguments('-Wno-unused-result') Wno_unused_variable = cc.get_supported_arguments('-Wno-unused-variable') +Wno_unused_but_set_variable = cc.get_supported_arguments('-Wno-unused-but-set-variable') +Wno_incompatible_pointer_types = cc.get_supported_arguments('-Wno-incompatible-pointer-types') # C++ warning flags _cpp_Wno_cpp = cpp.get_supported_arguments('-Wno-cpp') diff --git a/scipy/optimize/_highs/_highs_wrapper.py b/scipy/optimize/_highs/_highs_wrapper.py index a67e124911e2..e80a04918254 100644 --- a/scipy/optimize/_highs/_highs_wrapper.py +++ b/scipy/optimize/_highs/_highs_wrapper.py @@ -1,8 +1,8 @@ from warnings import warn import numpy as np -from scipy.optimize._highs import highs_bindings as hspy # type: ignore[attr-defined] -from scipy.optimize._highs import _highs_options as hopt # type: ignore[attr-defined] +from scipy.optimize._highs.highspy.highs import _h +from scipy.optimize._highs.highspy import _highs_options as hopt # type: ignore[attr-defined] from scipy.optimize import OptimizeWarning @@ -18,12 +18,12 @@ def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, opti } # Fill up a HighsLp object - lp = hspy.HighsLp() + lp = _h.HighsLp() lp.num_col_ = numcol lp.num_row_ = numrow lp.a_matrix_.num_col_ = numcol lp.a_matrix_.num_row_ = numrow - lp.a_matrix_.format_ = hspy.MatrixFormat.kColwise + lp.a_matrix_.format_ = _h.MatrixFormat.kColwise lp.col_cost_ = c lp.col_lower_ = lb lp.col_upper_ = ub @@ -33,18 +33,19 @@ def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, opti lp.a_matrix_.index_ = indices lp.a_matrix_.value_ = data if integrality.size > 0: - lp.integrality_ = [hspy.HighsVarType(i) for i in integrality] + lp.integrality_ = [_h.HighsVarType(i) for i in integrality] # Make a Highs object and pass it everything - highs = hspy.Highs() - highs_options = hspy.HighsOptions() + highs = _h.Highs_() + highs_options = _h.HighsOptions() + hoptmanager = hopt.HighsOptionsManager() for key, val in options.items(): # handle filtering of unsupported and default options if val is None or key in ("sense",): continue # ask for the option type - opt_type = hopt.get_option_type(key) + opt_type = hoptmanager.get_option_type(key) if -1 == opt_type: warn(f"Unrecognized options detected: {dict({key: val})}", OptimizeWarning) continue @@ -54,23 +55,21 @@ def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, opti if isinstance(val, bool): val = "on" if val else "off" else: - warn(f'Option f"{key}" is "{val}", but only True or False is ' - f'allowed. Using default.', OptimizeWarning) + warn( + f'Option f"{key}" is "{val}", but only True or False is ' + f"allowed. Using default.", + OptimizeWarning, + ) continue - opt_type = hspy.HighsOptionType(opt_type) + opt_type = _h.HighsOptionType(opt_type) status, msg = check_option(highs, key, val) - # { - # hspy.HighsOptionType.kBool: lambda _x, _y: (0, ""), - # hspy.HighsOptionType.kInt: hopt.check_int_option, - # hspy.HighsOptionType.kDouble: hopt.check_double_option, - # hspy.HighsOptionType.kString: hopt.check_string_option, - # }[opt_type](key, val) - - # have to do bool checking here because HiGHS doesn't have API - if opt_type == hspy.HighsOptionType.kBool: + if opt_type == _h.HighsOptionType.kBool: if not isinstance(val, bool): - warn(f'Option f"{key}" is "{val}", but only True or False is ' - f'allowed. Using default.', OptimizeWarning) + warn( + f'Option f"{key}" is "{val}", but only True or False is ' + f"allowed. Using default.", + OptimizeWarning, + ) continue # warn or set option @@ -80,30 +79,36 @@ def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, opti setattr(highs_options, key, val) opt_status = highs.passOptions(highs_options) - if opt_status == hspy.HighsStatus.kError: - res.update({ - "status": highs.getModelStatus(), - "message": highs.modelStatusToString(highs.getModelStatus()), - }) + if opt_status == _h.HighsStatus.kError: + res.update( + { + "status": highs.getModelStatus(), + "message": highs.modelStatusToString(highs.getModelStatus()), + } + ) return res init_status = highs.passModel(lp) - if init_status == hspy.HighsStatus.kError: + if init_status == _h.HighsStatus.kError: # if model fails to load, highs.getModelStatus() will be NOT_SET - err_model_status = hspy.HighsModelStatus.kModelError - res.update({ - "status": err_model_status, - "message": highs.modelStatusToString(err_model_status), - }) + err_model_status = _h.HighsModelStatus.kModelError + res.update( + { + "status": err_model_status, + "message": highs.modelStatusToString(err_model_status), + } + ) return res # Solve the LP run_status = highs.run() - if run_status == hspy.HighsStatus.kError: - res.update({ - "status": highs.getModelStatus(), - "message": highs.modelStatusToString(highs.getModelStatus()), - }) + if run_status == _h.HighsStatus.kError: + res.update( + { + "status": highs.getModelStatus(), + "message": highs.modelStatusToString(highs.getModelStatus()), + } + ) return res # Extract what we need from the solution @@ -122,26 +127,32 @@ def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, opti # current solution is feasible and can be returned. Else, there # is no solution. mipFailCondition = model_status not in ( - hspy.HighsModelStatus.kOptimal, - hspy.HighsModelStatus.kTimeLimit, - hspy.HighsModelStatus.kIterationLimit, - hspy.HighsModelStatus.kSolutionLimit, - ) or (model_status in { - hspy.HighsModelStatus.kTimeLimit, - hspy.HighsModelStatus.kIterationLimit, - hspy.HighsModelStatus.kSolutionLimit, - } and (info.objective_function_value == hspy.kHighsInf)) - lpFailCondition = model_status != hspy.HighsModelStatus.kOptimal + _h.HighsModelStatus.kOptimal, + _h.HighsModelStatus.kTimeLimit, + _h.HighsModelStatus.kIterationLimit, + _h.HighsModelStatus.kSolutionLimit, + ) or ( + model_status + in { + _h.HighsModelStatus.kTimeLimit, + _h.HighsModelStatus.kIterationLimit, + _h.HighsModelStatus.kSolutionLimit, + } + and (info.objective_function_value == _h.kHighsInf) + ) + lpFailCondition = model_status != _h.HighsModelStatus.kOptimal if (isMip and mipFailCondition) or (not isMip and lpFailCondition): - res.update({ - "status": model_status, - "message": f"model_status is {highs.modelStatusToString(model_status)}; " - f"primal_status is " - f"{highs.solutionStatusToString(info.primal_solution_status)}", - "simplex_nit": info.simplex_iteration_count, - "ipm_nit": info.ipm_iteration_count, - "crossover_nit": info.crossover_iteration_count, - }) + res.update( + { + "status": model_status, + "message": f"model_status is {highs.modelStatusToString(model_status)}; " + f"primal_status is " + f"{highs.solutionStatusToString(info.primal_solution_status)}", + "simplex_nit": info.simplex_iteration_count, + "ipm_nit": info.ipm_iteration_count, + "crossover_nit": info.crossover_iteration_count, + } + ) return res # Should be safe to read the solution: @@ -151,62 +162,72 @@ def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, opti # Lagrangians for bounds based on column statuses marg_bnds = np.zeros((2, numcol)) for ii in range(numcol): - if basis.col_status[ii] == hspy.HighsBasisStatus.kLower: + if basis.col_status[ii] == _h.HighsBasisStatus.kLower: marg_bnds[0, ii] = solution.col_dual[ii] - elif basis.col_status[ii] == hspy.HighsBasisStatus.kUpper: + elif basis.col_status[ii] == _h.HighsBasisStatus.kUpper: marg_bnds[1, ii] = solution.col_dual[ii] - res.update({ - "status": model_status, - "message": highs.modelStatusToString(model_status), - - # Primal solution - "x": np.array(solution.col_value), - - # Ax + s = b => Ax = b - s - # Note: this is for all constraints (A_ub and A_eq) - "slack": rhs - solution.row_value, - - # lambda are the lagrange multipliers associated with Ax=b - "lambda": np.array(solution.row_dual), - "marg_bnds": marg_bnds, - - "fun": info.objective_function_value, - "simplex_nit": info.simplex_iteration_count, - "ipm_nit": info.ipm_iteration_count, - "crossover_nit": info.crossover_iteration_count, - }) + res.update( + { + "status": model_status, + "message": highs.modelStatusToString(model_status), + # Primal solution + "x": np.array(solution.col_value), + # Ax + s = b => Ax = b - s + # Note: this is for all constraints (A_ub and A_eq) + "slack": rhs - solution.row_value, + # lambda are the lagrange multipliers associated with Ax=b + "lambda": np.array(solution.row_dual), + "marg_bnds": marg_bnds, + "fun": info.objective_function_value, + "simplex_nit": info.simplex_iteration_count, + "ipm_nit": info.ipm_iteration_count, + "crossover_nit": info.crossover_iteration_count, + } + ) if isMip: - res.update({ - "mip_node_count": info.mip_node_count, - "mip_dual_bound": info.mip_dual_bound, - "mip_gap": info.mip_gap, - }) + res.update( + { + "mip_node_count": info.mip_node_count, + "mip_dual_bound": info.mip_dual_bound, + "mip_gap": info.mip_gap, + } + ) return res + def check_option(highs_inst, option, value): status, option_type = highs_inst.getOptionType(option) + hoptmanager = hopt.HighsOptionsManager() - if status != HighsStatus.kOk: - return 1, "Invalid option name." + if status != _h.HighsStatus.kOk: + return -1, "Invalid option name." valid_types = { - HighsOptionType.kBool: bool, - HighsOptionType.kInt: int, - HighsOptionType.kDouble: float, - HighsOptionType.kString: str + _h.HighsOptionType.kBool: bool, + _h.HighsOptionType.kInt: int, + _h.HighsOptionType.kDouble: float, + _h.HighsOptionType.kString: str, } expected_type = valid_types.get(option_type, None) + + if expected_type is str: + if not hoptmanager.check_string_option(option, value): + return -1, "Invalid option value." + if expected_type is float: + if not hoptmanager.check_double_option(option, value): + return -1, "Invalid option value." + if expected_type is int: + if not hoptmanager.check_int_option(option, value): + return -1, "Invalid option value." + if expected_type is None: return 3, "Unknown option type." - if not isinstance(value, expected_type): - return 2, "Invalid option value." - status, current_value = highs_inst.getOptionValue(option) - if status != HighsStatus.kOk: + if status != _h.HighsStatus.kOk: return 4, "Failed to validate option value." return 0, "Check option succeeded." diff --git a/scipy/optimize/_highs/meson.build b/scipy/optimize/_highs/meson.build index 2e4ace80dda1..b164c2265f11 100644 --- a/scipy/optimize/_highs/meson.build +++ b/scipy/optimize/_highs/meson.build @@ -1,3 +1,45 @@ +# Setup the highs library +highs_proj = subproject('highs', + default_options : ['default_library=static']) +highs_dep = highs_proj.get_variable('highs_dep') +highspy_py = highs_proj.get_variable('highspy_py') +highspy_cpp = highs_proj.get_variable('highspy_cpp') +highsoptions_cpp = highs_proj.get_variable('highsoptions_cpp') + +scipy_highspy_dep = [ + py3_dep, + pybind11_dep, + highs_dep, +] + +py3.extension_module( + '_highs', + sources : highspy_cpp, + dependencies: scipy_highspy_dep, + c_args: [Wno_unused_variable, Wno_unused_but_set_variable], + cpp_args: [_cpp_Wno_unused_variable, _cpp_Wno_unused_but_set_variable], + link_args: version_link_args, + subdir: 'scipy/optimize/_highs/highspy', + install: true, +) + + +py3.extension_module( + '_highs_options', + sources : highsoptions_cpp, + dependencies: scipy_highspy_dep, + c_args: [Wno_unused_variable, Wno_unused_but_set_variable], + cpp_args: [_cpp_Wno_unused_variable, _cpp_Wno_unused_but_set_variable], + link_args: version_link_args, + subdir: 'scipy/optimize/_highs/highspy', + install: true, +) + +py3.install_sources( + highspy_py, + subdir: 'scipy/optimize/_highs/highspy', +) + py3.install_sources([ '__init__.py', '_highs_wrapper.py', diff --git a/scipy/optimize/_highs/src/HConfig.h b/scipy/optimize/_highs/src/HConfig.h deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/scipy/optimize/_linprog_highs.py b/scipy/optimize/_linprog_highs.py index 796ac52e18da..9dc8110ba56b 100644 --- a/scipy/optimize/_linprog_highs.py +++ b/scipy/optimize/_linprog_highs.py @@ -20,12 +20,13 @@ from ._highs._highs_wrapper import _highs_wrapper from scipy.sparse import csc_matrix, vstack, issparse -from highspy._highs import HighsModelStatus as hms -from highspy._highs import simplex_constants as simpc -import highspy._highs as hspy +from scipy.optimize._highs.highspy.highs import _h +from scipy.optimize._highs.highspy.highs import simpc + class SciPyRC(Enum): """Return codes for SciPy solvers""" + OPTIMAL = 0 ITERATION_LIMIT = 1 INFEASIBLE = 2 @@ -46,52 +47,67 @@ def to_string(self): else: return "" + class HighsStatusMapping: """Class to map HiGHS statuses to SciPy Return Codes""" + def __init__(self): # Custom mapping from HiGHS status and errors to SciPy status self.highs_to_scipy = { - hms.kNotset: (SciPyRC.NUMERICAL, "Not set"), - hms.kLoadError: (SciPyRC.NUMERICAL, "Load Error"), - hms.kModelError: (SciPyRC.INFEASIBLE, "Model Error"), - hms.kPresolveError: (SciPyRC.NUMERICAL, "Presolve Error"), - hms.kSolveError: (SciPyRC.NUMERICAL, "Solve Error"), - hms.kPostsolveError: (SciPyRC.NUMERICAL, "Postsolve Error"), - hms.kModelEmpty: (SciPyRC.NUMERICAL, "Model Empty"), - hms.kOptimal: (SciPyRC.OPTIMAL, "Optimal"), - hms.kInfeasible: (SciPyRC.INFEASIBLE, "Infeasible"), - hms.kUnboundedOrInfeasible: (SciPyRC.NUMERICAL, "Unbounded or Infeasible"), - hms.kUnbounded: (SciPyRC.UNBOUNDED, "Unbounded"), - hms.kObjectiveBound: (SciPyRC.NUMERICAL, "Objective Bound"), - hms.kObjectiveTarget: (SciPyRC.NUMERICAL, "Objective Target"), - hms.kTimeLimit: (SciPyRC.ITERATION_LIMIT, "Time Limit"), - hms.kIterationLimit: (SciPyRC.ITERATION_LIMIT, "Iteration Limit"), - hms.kUnknown: (SciPyRC.NUMERICAL, "Unknown"), - hms.kSolutionLimit: (SciPyRC.NUMERICAL, "Solution Limit"), + _h.HighsModelStatus.kNotset: (SciPyRC.NUMERICAL, "Not set"), + _h.HighsModelStatus.kLoadError: (SciPyRC.NUMERICAL, "Load Error"), + _h.HighsModelStatus.kModelError: (SciPyRC.INFEASIBLE, "Model Error"), + _h.HighsModelStatus.kPresolveError: (SciPyRC.NUMERICAL, "Presolve Error"), + _h.HighsModelStatus.kSolveError: (SciPyRC.NUMERICAL, "Solve Error"), + _h.HighsModelStatus.kPostsolveError: (SciPyRC.NUMERICAL, "Postsolve Error"), + _h.HighsModelStatus.kModelEmpty: (SciPyRC.NUMERICAL, "Model Empty"), + _h.HighsModelStatus.kOptimal: (SciPyRC.OPTIMAL, "Optimal"), + _h.HighsModelStatus.kInfeasible: (SciPyRC.INFEASIBLE, "Infeasible"), + _h.HighsModelStatus.kUnboundedOrInfeasible: ( + SciPyRC.NUMERICAL, + "Unbounded or Infeasible", + ), + _h.HighsModelStatus.kUnbounded: (SciPyRC.UNBOUNDED, "Unbounded"), + _h.HighsModelStatus.kObjectiveBound: (SciPyRC.NUMERICAL, "Objective Bound"), + _h.HighsModelStatus.kObjectiveTarget: ( + SciPyRC.NUMERICAL, + "Objective Target", + ), + _h.HighsModelStatus.kTimeLimit: (SciPyRC.ITERATION_LIMIT, "Time Limit"), + _h.HighsModelStatus.kIterationLimit: ( + SciPyRC.ITERATION_LIMIT, + "Iteration Limit", + ), + _h.HighsModelStatus.kUnknown: (SciPyRC.NUMERICAL, "Unknown"), + _h.HighsModelStatus.kSolutionLimit: (SciPyRC.NUMERICAL, "Solution Limit"), } - def get_scipy_status(self, highs_status, highs_message): """Converts HiGHS status and message to SciPy status and message""" - scipy_status, message_prefix = self.highs_to_scipy.get(hspy.HighsModelStatus(highs_status), - (SciPyRC.NUMERICAL, "Unknown HiGHS Status")) + scipy_status, message_prefix = self.highs_to_scipy.get( + _h.HighsModelStatus(highs_status), + (SciPyRC.NUMERICAL, "Unknown HiGHS Status"), + ) scip = SciPyRC(scipy_status) - scipy_message = f"{scip.to_string()} (HiGHS Status {highs_status}: {highs_message})" + scipy_message = ( + f"{scip.to_string()} (HiGHS Status {highs_status}: {highs_message})" + ) return scipy_status.value, scipy_message + def _replace_inf(x): # Replace `np.inf` with kHighsInf infs = np.isinf(x) with np.errstate(invalid="ignore"): - x[infs] = np.sign(x[infs])*hspy.kHighsInf + x[infs] = np.sign(x[infs]) * _h.kHighsInf return x class SimplexStrategy(Enum): - DANTZIG = 'dantzig' - DEVEX = 'devex' - STEEPEST_DEVEX = 'steepest-devex' # highs min, choose - STEEPEST = 'steepest' # highs max + DANTZIG = "dantzig" + DEVEX = "devex" + STEEPEST_DEVEX = "steepest-devex" # highs min, choose + STEEPEST = "steepest" # highs max def to_highs_enum(self): mapping = { @@ -102,29 +118,39 @@ def to_highs_enum(self): } return mapping.get(self) + def convert_to_highs_enum(option, option_str, choices_enum, default_value): if option is None: return choices_enum[default_value.upper()].to_highs_enum() try: enum_value = choices_enum[option.upper()] except KeyError: - warn(f"Option {option_str} is {option}, but only values in " - f"{[e.value for e in choices_enum]} are allowed. Using default: " - f"{default_value}.", - OptimizeWarning, stacklevel=3) + warn( + f"Option {option_str} is {option}, but only values in " + f"{[e.value for e in choices_enum]} are allowed. Using default: " + f"{default_value}.", + OptimizeWarning, + stacklevel=3, + ) enum_value = choices_enum[default_value.upper()] return enum_value.to_highs_enum() -def _linprog_highs(lp, solver, time_limit=None, presolve=True, - disp=False, maxiter=None, - dual_feasibility_tolerance=None, - primal_feasibility_tolerance=None, - ipm_optimality_tolerance=None, - simplex_dual_edge_weight_strategy=None, - mip_rel_gap=None, - mip_max_nodes=None, - **unknown_options): +def _linprog_highs( + lp, + solver, + time_limit=None, + presolve=True, + disp=False, + maxiter=None, + dual_feasibility_tolerance=None, + primal_feasibility_tolerance=None, + ipm_optimality_tolerance=None, + simplex_dual_edge_weight_strategy=None, + mip_rel_gap=None, + mip_max_nodes=None, + **unknown_options, +): r""" Solve the following linear programming problem using one of the HiGHS solvers: @@ -314,8 +340,10 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, simplex algorithm." Mathematical Programming 12.1 (1977): 361-371. """ if unknown_options: - message = (f"Unrecognized options detected: {unknown_options}. " - "These will be passed to HiGHS verbatim.") + message = ( + f"Unrecognized options detected: {unknown_options}. " + "These will be passed to HiGHS verbatim." + ) warn(message, OptimizeWarning, stacklevel=3) # Map options to HiGHS enum values @@ -331,7 +359,7 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, lb, ub = bounds.T.copy() # separate bounds, copy->C-cntgs # highs_wrapper solves LHS <= A*x <= RHS, not equality constraints with np.errstate(invalid="ignore"): - lhs_ub = -np.ones_like(b_ub)*np.inf # LHS of UB constraints is -inf + lhs_ub = -np.ones_like(b_ub) * np.inf # LHS of UB constraints is -inf rhs_ub = b_ub # RHS of UB constraints is b_ub lhs_eq = b_eq # Equality constraint is inequality rhs_eq = b_eq # constraint with LHS=RHS @@ -345,24 +373,23 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, A = csc_matrix(A) options = { - 'presolve': presolve, - 'sense': hspy.ObjSense.kMinimize, - 'solver': solver, - 'time_limit': time_limit, - # 'highs_debug_level': hspy.kHighs, # TODO - 'dual_feasibility_tolerance': dual_feasibility_tolerance, - 'ipm_optimality_tolerance': ipm_optimality_tolerance, - 'log_to_console': disp, - 'mip_max_nodes': mip_max_nodes, - 'output_flag': disp, - 'primal_feasibility_tolerance': primal_feasibility_tolerance, - 'simplex_dual_edge_weight_strategy': - simplex_dual_edge_weight_strategy_enum, - 'simplex_strategy': simpc.kSimplexStrategyDual.value, + "presolve": presolve, + "sense": _h.ObjSense.kMinimize, + "solver": solver, + "time_limit": time_limit, + # 'highs_debug_level': _h.kHighs, # TODO + "dual_feasibility_tolerance": dual_feasibility_tolerance, + "ipm_optimality_tolerance": ipm_optimality_tolerance, + "log_to_console": disp, + "mip_max_nodes": mip_max_nodes, + "output_flag": disp, + "primal_feasibility_tolerance": primal_feasibility_tolerance, + "simplex_dual_edge_weight_strategy": simplex_dual_edge_weight_strategy_enum, + "simplex_strategy": simpc.kSimplexStrategyDual.value, # 'simplex_crash_strategy': simpc.SimplexCrashStrategy.kSimplexCrashStrategyOff, - 'ipm_iteration_limit': maxiter, - 'simplex_iteration_limit': maxiter, - 'mip_rel_gap': mip_rel_gap, + "ipm_iteration_limit": maxiter, + "simplex_iteration_limit": maxiter, + "mip_rel_gap": mip_rel_gap, } options.update(unknown_options) @@ -377,26 +404,36 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, else: integrality = np.array(integrality) - res = _highs_wrapper(c, A.indptr, A.indices, A.data, lhs, rhs, - lb, ub, integrality.astype(np.uint8), options) + res = _highs_wrapper( + c, + A.indptr, + A.indices, + A.data, + lhs, + rhs, + lb, + ub, + integrality.astype(np.uint8), + options, + ) # HiGHS represents constraints as lhs/rhs, so # Ax + s = b => Ax = b - s # and we need to split up s by A_ub and A_eq - if 'slack' in res: - slack = res['slack'] - con = np.array(slack[len(b_ub):]) - slack = np.array(slack[:len(b_ub)]) + if "slack" in res: + slack = res["slack"] + con = np.array(slack[len(b_ub) :]) + slack = np.array(slack[: len(b_ub)]) else: slack, con = None, None # lagrange multipliers for equalities/inequalities and upper/lower bounds - if 'lambda' in res: - lamda = res['lambda'] - marg_ineqlin = np.array(lamda[:len(b_ub)]) - marg_eqlin = np.array(lamda[len(b_ub):]) - marg_upper = np.array(res['marg_bnds'][1, :]) - marg_lower = np.array(res['marg_bnds'][0, :]) + if "lambda" in res: + lamda = res["lambda"] + marg_ineqlin = np.array(lamda[: len(b_ub)]) + marg_eqlin = np.array(lamda[len(b_ub) :]) + marg_upper = np.array(res["marg_bnds"][1, :]) + marg_lower = np.array(res["marg_bnds"][0, :]) else: marg_ineqlin, marg_eqlin = None, None marg_upper, marg_lower = None, None @@ -405,44 +442,58 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, # Convert to scipy-style status and message highs_mapper = HighsStatusMapping() - highs_status = res.get('status', None) - highs_message = res.get('message', None) - status, message = highs_mapper.get_scipy_status(highs_status, - highs_message) - - x = np.array(res['x']) if 'x' in res else None - sol = {'x': x, - 'slack': slack, - 'con': con, - 'ineqlin': OptimizeResult({ - 'residual': slack, - 'marginals': marg_ineqlin, - }), - 'eqlin': OptimizeResult({ - 'residual': con, - 'marginals': marg_eqlin, - }), - 'lower': OptimizeResult({ - 'residual': None if x is None else x - lb, - 'marginals': marg_lower, - }), - 'upper': OptimizeResult({ - 'residual': None if x is None else ub - x, - 'marginals': marg_upper - }), - 'fun': res.get('fun'), - 'status': status, - 'success': res['status'] == hms.kOptimal, - 'message': message, - 'nit': res.get('simplex_nit', 0) or res.get('ipm_nit', 0), - 'crossover_nit': res.get('crossover_nit'), - } + highs_status = res.get("status", None) + highs_message = res.get("message", None) + status, message = highs_mapper.get_scipy_status(highs_status, highs_message) + + def is_valid_x(val): + if isinstance(val, np.ndarray): + if val.dtype == object and None in val: + return False + return val is not None + + x = np.array(res["x"]) if "x" in res and is_valid_x(res["x"]) else None + + sol = { + "x": x, + "slack": slack, + "con": con, + "ineqlin": OptimizeResult( + { + "residual": slack, + "marginals": marg_ineqlin, + } + ), + "eqlin": OptimizeResult( + { + "residual": con, + "marginals": marg_eqlin, + } + ), + "lower": OptimizeResult( + { + "residual": None if x is None else x - lb, + "marginals": marg_lower, + } + ), + "upper": OptimizeResult( + {"residual": None if x is None else ub - x, "marginals": marg_upper} + ), + "fun": res.get("fun"), + "status": status, + "success": res["status"] == _h.HighsModelStatus.kOptimal, + "message": message, + "nit": res.get("simplex_nit", 0) or res.get("ipm_nit", 0), + "crossover_nit": res.get("crossover_nit"), + } if np.any(x) and integrality is not None: - sol.update({ - 'mip_node_count': res.get('mip_node_count', 0), - 'mip_dual_bound': res.get('mip_dual_bound', 0.0), - 'mip_gap': res.get('mip_gap', 0.0), - }) + sol.update( + { + "mip_node_count": res.get("mip_node_count", 0), + "mip_dual_bound": res.get("mip_dual_bound", 0.0), + "mip_gap": res.get("mip_gap", 0.0), + } + ) return sol diff --git a/scipy/optimize/tests/test_linprog.py b/scipy/optimize/tests/test_linprog.py index c2026ae09656..39d09b6ad95b 100644 --- a/scipy/optimize/tests/test_linprog.py +++ b/scipy/optimize/tests/test_linprog.py @@ -303,10 +303,9 @@ def test_deprecation(): def test_highs_status_message(): res = linprog(1, method='highs') - msg = "Optimization terminated successfully." + msg = "Optimization terminated successfully. (HiGHS Status 7:" assert res.status == 0 assert res.message.startswith(msg) - assert 'HiGHS Status 7' in res.message A, b, c, numbers, M = magic_square(6) bounds = [(0, 1)] * len(c) @@ -314,46 +313,38 @@ def test_highs_status_message(): options = {"time_limit": 0.1} res = linprog(c=c, A_eq=A, b_eq=b, bounds=bounds, method='highs', options=options, integrality=integrality) - msg = "Time limit reached" + msg = "Time limit reached. (HiGHS Status 13:" assert res.status == 1 assert msg in res.message - assert 'HiGHS Status 13' in res.message options = {"maxiter": 10} res = linprog(c=c, A_eq=A, b_eq=b, bounds=bounds, method='highs-ds', options=options) - msg = "Iteration limit reached" + msg = "Iteration limit reached. (HiGHS Status 14:" assert res.status == 1 assert res.message.startswith(msg) - assert 'HiGHS Status 14' in res.message res = linprog(1, bounds=(1, -1), method='highs') - msg = "The problem is infeasible" + msg = "The problem is infeasible. (HiGHS Status 8:" assert res.status == 2 assert res.message.startswith(msg) - assert 'HiGHS Status 8' in res.message - # TODO: Fix this - # res = linprog(-1, method='highs') - # msg = "The problem is unbounded." - # assert res.status == 3 - # assert res.message.startswith(msg) - # assert 'HiGHS Status 10' in res.message + res = linprog(-1, method='highs') + msg = "The problem is unbounded. (HiGHS Status 10:" + assert res.status == 3 + assert res.message.startswith(msg) from scipy.optimize._linprog_highs import HighsStatusMapping highs_mapper = HighsStatusMapping() status, message = highs_mapper.get_scipy_status(58, "Hello!") - # msg = "The HiGHS status code was not recognized. (HiGHS Status 58:" assert status == 4 - # TODO: Fix this - # assert message.startswith(msg) - assert "HiGHS Status 58" in message - - # TODO: Fix this - # status, message = highs_mapper.get_scipy_status(None, None) - # msg = "HiGHS did not provide a status code. (HiGHS Status None: None)" - # assert status == 4 - # assert message.startswith(msg) + msg = "The HiGHS status code was not recognized. (HiGHS Status 58:" + assert message.startswith(msg) + + status, message = highs_mapper.get_scipy_status(None, None) + msg = "HiGHS did not provide a status code. (HiGHS Status None: None)" + assert status == 4 + assert message.startswith(msg) def test_bug_17380(): diff --git a/scipy/optimize/tests/test_milp.py b/scipy/optimize/tests/test_milp.py index c36ba5e19fb9..f78cb2cd0ac0 100644 --- a/scipy/optimize/tests/test_milp.py +++ b/scipy/optimize/tests/test_milp.py @@ -99,7 +99,7 @@ def test_result(): assert res.success msg = "Optimization terminated successfully." assert res.message.startswith(msg) - assert 'HiGHS Status 7' in res.message + assert 'HighsModelStatus.kOptimal' in res.message assert isinstance(res.x, np.ndarray) assert isinstance(res.fun, float) assert isinstance(res.mip_node_count, int) @@ -112,7 +112,7 @@ def test_result(): assert res.status == 1 assert not res.success msg = "Time limit reached" - assert 'HiGHS Status 13' in res.message + assert 'HighsModelStatus.kTimeLimit' in res.message assert msg in res.message assert (res.fun is res.mip_dual_bound is res.mip_gap is res.mip_node_count is res.x is None) @@ -121,19 +121,19 @@ def test_result(): assert res.status == 2 assert not res.success msg = "The problem is infeasible" - assert 'HiGHS Status 8' in res.message + assert 'HighsModelStatus.kInfeasible' in res.message assert res.message.startswith(msg) assert (res.fun is res.mip_dual_bound is res.mip_gap is res.mip_node_count is res.x is None) - # TODO: Fix this - # res = milp(-1) - # assert res.status == 3 - # assert not res.success - # msg = "The problem is unbounded. (HiGHS Status 10:" - # assert res.message.startswith(msg) - # assert (res.fun is res.mip_dual_bound is res.mip_gap - # is res.mip_node_count is res.x is None) + res = milp(-1) + assert res.status == 3 + assert not res.success + assert 'HighsModelStatus.kUnbounded' in res.message + msg = "The problem is unbounded." + assert res.message.startswith(msg) + assert (res.fun is res.mip_dual_bound is res.mip_gap + is res.mip_node_count is res.x is None) def test_milp_optional_args(): diff --git a/subprojects/highs.wrap b/subprojects/highs.wrap index 137b46dfeb60..14468dfcfa45 100644 --- a/subprojects/highs.wrap +++ b/subprojects/highs.wrap @@ -1,3 +1,3 @@ [wrap-git] url = https://github.com/HaoZeke/highs.git -revision = d0bac8a10ce0d7bed35469c68a95450bee9dce71 +revision = 325d5356835c6d258441ad45b19a357e4857a895 From cc5e00457f74161e531c43a4b9dbbe1335ae9792 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 14 Oct 2023 17:54:02 +0000 Subject: [PATCH 05/64] ENH: Rework [highspy] interface for BC TST: Revert changes to milp tests MAINT: Quiet mypy problems w.r.t highspy MAINT: Fix typo in documentation Fixes the refguide check, the primal status for an infeasible problem is None TST: Rework the one test which returns new results BLD: Rework to include gh-17777 for highs MAINT: Rework to a minimal highspy subset MAINT: Try to disable highsint64 for 32-bit fix MAINT: Update pyproject for more meson-python args BLD: Try to conditionally build with highsint64 MAINT: Add back a submodule (personal) MAINT: Update the wrap file MAINT: Simplify pyproject for now MAINT: Simplify highs_defaults upstream --- .gitignore | 2 - .gitmodules | 4 + doc/source/tutorial/optimize.rst | 2 +- mypy.ini | 11 ++ scipy/optimize/_highs/_highs_wrapper.py | 6 +- scipy/optimize/_highs/meson.build | 8 +- scipy/optimize/_linprog_highs.py | 157 ++++++++++++++++-------- scipy/optimize/_milp.py | 2 +- scipy/optimize/tests/test_milp.py | 18 ++- subprojects/highs | 1 + subprojects/highs.wrap | 2 +- 11 files changed, 136 insertions(+), 77 deletions(-) create mode 160000 subprojects/highs diff --git a/.gitignore b/.gitignore index 9e4e9b14b9b8..a4fb8751794e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ -# Wrap directories -subprojects/highs # Editor temporary/working/backup files # ######################################### .#* diff --git a/.gitmodules b/.gitmodules index 39cd9bfe50a3..dbf08905c7c9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,7 @@ [submodule "scipy/_lib/pocketfft"] path = scipy/_lib/pocketfft url = https://github.com/scipy/pocketfft +[submodule "subprojects/highs"] + path = subprojects/highs + url = https://github.com/HaoZeke/highs + branch = forSciPy diff --git a/doc/source/tutorial/optimize.rst b/doc/source/tutorial/optimize.rst index 8d6abb5f0270..23474aab5160 100644 --- a/doc/source/tutorial/optimize.rst +++ b/doc/source/tutorial/optimize.rst @@ -1596,7 +1596,7 @@ Finally, we can solve the transformed problem using :func:`linprog`. >>> bounds = [x0_bounds, x1_bounds, x2_bounds, x3_bounds] >>> result = linprog(c, A_ub=A_ub, b_ub=b_ub, A_eq=A_eq, b_eq=b_eq, bounds=bounds) >>> print(result.message) - The problem is infeasible. (HiGHS Status 8: model_status is Infeasible; primal_status is At lower/fixed bound) + The problem is infeasible. (HiGHS Status 8: model_status is Infeasible; primal_status is None) The result states that our problem is infeasible, meaning that there is no solution vector that satisfies all the constraints. That doesn't necessarily mean we did anything wrong; some problems truly are infeasible. diff --git a/mypy.ini b/mypy.ini index 3cf9d47a9aac..aa393b2c5a55 100644 --- a/mypy.ini +++ b/mypy.ini @@ -280,6 +280,17 @@ ignore_errors = True [mypy-scipy.optimize._linprog_util] ignore_errors = True +[mypy-scipy.optimize._highs.highspy._highs] +ignore_errors = True +ignore_missing_imports = True + +[mypy-scipy.optimize._highs.highspy._highs.simplex_constants] +ignore_errors = True +ignore_missing_imports = True + +[mypy-scipy.optimize._milp] +ignore_errors = True + [mypy-scipy.optimize._trustregion] ignore_errors = True diff --git a/scipy/optimize/_highs/_highs_wrapper.py b/scipy/optimize/_highs/_highs_wrapper.py index e80a04918254..909d9d651a44 100644 --- a/scipy/optimize/_highs/_highs_wrapper.py +++ b/scipy/optimize/_highs/_highs_wrapper.py @@ -1,7 +1,7 @@ from warnings import warn import numpy as np -from scipy.optimize._highs.highspy.highs import _h +import scipy.optimize._highs.highspy._highs as _h from scipy.optimize._highs.highspy import _highs_options as hopt # type: ignore[attr-defined] from scipy.optimize import OptimizeWarning @@ -36,7 +36,7 @@ def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, opti lp.integrality_ = [_h.HighsVarType(i) for i in integrality] # Make a Highs object and pass it everything - highs = _h.Highs_() + highs = _h.Highs() highs_options = _h.HighsOptions() hoptmanager = hopt.HighsOptionsManager() for key, val in options.items(): @@ -146,7 +146,7 @@ def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, opti { "status": model_status, "message": f"model_status is {highs.modelStatusToString(model_status)}; " - f"primal_status is " + "primal_status is " f"{highs.solutionStatusToString(info.primal_solution_status)}", "simplex_nit": info.simplex_iteration_count, "ipm_nit": info.ipm_iteration_count, diff --git a/scipy/optimize/_highs/meson.build b/scipy/optimize/_highs/meson.build index b164c2265f11..21e49b8b9c60 100644 --- a/scipy/optimize/_highs/meson.build +++ b/scipy/optimize/_highs/meson.build @@ -2,7 +2,6 @@ highs_proj = subproject('highs', default_options : ['default_library=static']) highs_dep = highs_proj.get_variable('highs_dep') -highspy_py = highs_proj.get_variable('highspy_py') highspy_cpp = highs_proj.get_variable('highspy_cpp') highsoptions_cpp = highs_proj.get_variable('highsoptions_cpp') @@ -10,6 +9,8 @@ scipy_highspy_dep = [ py3_dep, pybind11_dep, highs_dep, + thread_dep, + atomic_dep, ] py3.extension_module( @@ -35,11 +36,6 @@ py3.extension_module( install: true, ) -py3.install_sources( - highspy_py, - subdir: 'scipy/optimize/_highs/highspy', -) - py3.install_sources([ '__init__.py', '_highs_wrapper.py', diff --git a/scipy/optimize/_linprog_highs.py b/scipy/optimize/_linprog_highs.py index 9dc8110ba56b..dc5afa362e43 100644 --- a/scipy/optimize/_linprog_highs.py +++ b/scipy/optimize/_linprog_highs.py @@ -20,79 +20,132 @@ from ._highs._highs_wrapper import _highs_wrapper from scipy.sparse import csc_matrix, vstack, issparse -from scipy.optimize._highs.highspy.highs import _h -from scipy.optimize._highs.highspy.highs import simpc - - -class SciPyRC(Enum): - """Return codes for SciPy solvers""" - - OPTIMAL = 0 - ITERATION_LIMIT = 1 - INFEASIBLE = 2 - UNBOUNDED = 3 - NUMERICAL = 4 - - def to_string(self): - if self == SciPyRC.OPTIMAL: - return "Optimization terminated successfully. " - elif self == SciPyRC.ITERATION_LIMIT: - return "Iteration limit reached. " - elif self == SciPyRC.INFEASIBLE: - return "The problem is infeasible. " - elif self == SciPyRC.UNBOUNDED: - return "The problem is unbounded. " - elif self == SciPyRC.NUMERICAL: - return "Serious numerical difficulties encountered. " - else: - return "" +import scipy.optimize._highs.highspy._highs as _h +import scipy.optimize._highs.highspy._highs.simplex_constants as simpc class HighsStatusMapping: - """Class to map HiGHS statuses to SciPy Return Codes""" + """Class to map HiGHS statuses to SciPy-like Return Codes and Messages""" + + class SciPyRC(Enum): + """Return codes like SciPy's for solvers""" + + OPTIMAL = 0 + ITERATION_LIMIT = 1 + INFEASIBLE = 2 + UNBOUNDED = 3 + NUMERICAL = 4 def __init__(self): - # Custom mapping from HiGHS status and errors to SciPy status self.highs_to_scipy = { - _h.HighsModelStatus.kNotset: (SciPyRC.NUMERICAL, "Not set"), - _h.HighsModelStatus.kLoadError: (SciPyRC.NUMERICAL, "Load Error"), - _h.HighsModelStatus.kModelError: (SciPyRC.INFEASIBLE, "Model Error"), - _h.HighsModelStatus.kPresolveError: (SciPyRC.NUMERICAL, "Presolve Error"), - _h.HighsModelStatus.kSolveError: (SciPyRC.NUMERICAL, "Solve Error"), - _h.HighsModelStatus.kPostsolveError: (SciPyRC.NUMERICAL, "Postsolve Error"), - _h.HighsModelStatus.kModelEmpty: (SciPyRC.NUMERICAL, "Model Empty"), - _h.HighsModelStatus.kOptimal: (SciPyRC.OPTIMAL, "Optimal"), - _h.HighsModelStatus.kInfeasible: (SciPyRC.INFEASIBLE, "Infeasible"), + _h.HighsModelStatus.kNotset: ( + self.SciPyRC.NUMERICAL, + "Not set", + "Serious numerical difficulties encountered.", + ), + _h.HighsModelStatus.kLoadError: ( + self.SciPyRC.NUMERICAL, + "Load Error", + "Serious numerical difficulties encountered.", + ), + _h.HighsModelStatus.kModelError: ( + self.SciPyRC.INFEASIBLE, + "Model Error", + "The problem is infeasible.", + ), + _h.HighsModelStatus.kPresolveError: ( + self.SciPyRC.NUMERICAL, + "Presolve Error", + "Serious numerical difficulties encountered.", + ), + _h.HighsModelStatus.kSolveError: ( + self.SciPyRC.NUMERICAL, + "Solve Error", + "Serious numerical difficulties encountered.", + ), + _h.HighsModelStatus.kPostsolveError: ( + self.SciPyRC.NUMERICAL, + "Postsolve Error", + "Serious numerical difficulties encountered.", + ), + _h.HighsModelStatus.kModelEmpty: ( + self.SciPyRC.NUMERICAL, + "Model Empty", + "Serious numerical difficulties encountered.", + ), + _h.HighsModelStatus.kOptimal: ( + self.SciPyRC.OPTIMAL, + "Optimal", + "Optimization terminated successfully.", + ), + _h.HighsModelStatus.kInfeasible: ( + self.SciPyRC.INFEASIBLE, + "Infeasible", + "The problem is infeasible.", + ), _h.HighsModelStatus.kUnboundedOrInfeasible: ( - SciPyRC.NUMERICAL, + self.SciPyRC.NUMERICAL, "Unbounded or Infeasible", + "Serious numerical difficulties encountered.", + ), + _h.HighsModelStatus.kUnbounded: ( + self.SciPyRC.UNBOUNDED, + "Unbounded", + "The problem is unbounded.", + ), + _h.HighsModelStatus.kObjectiveBound: ( + self.SciPyRC.NUMERICAL, + "Objective Bound", + "Serious numerical difficulties encountered.", ), - _h.HighsModelStatus.kUnbounded: (SciPyRC.UNBOUNDED, "Unbounded"), - _h.HighsModelStatus.kObjectiveBound: (SciPyRC.NUMERICAL, "Objective Bound"), _h.HighsModelStatus.kObjectiveTarget: ( - SciPyRC.NUMERICAL, + self.SciPyRC.NUMERICAL, "Objective Target", + "Serious numerical difficulties encountered.", + ), + _h.HighsModelStatus.kTimeLimit: ( + self.SciPyRC.ITERATION_LIMIT, + "Time Limit", + "Time limit reached.", ), - _h.HighsModelStatus.kTimeLimit: (SciPyRC.ITERATION_LIMIT, "Time Limit"), _h.HighsModelStatus.kIterationLimit: ( - SciPyRC.ITERATION_LIMIT, + self.SciPyRC.ITERATION_LIMIT, "Iteration Limit", + "Iteration limit reached.", + ), + _h.HighsModelStatus.kUnknown: ( + self.SciPyRC.NUMERICAL, + "Unknown", + "Serious numerical difficulties encountered.", + ), + _h.HighsModelStatus.kSolutionLimit: ( + self.SciPyRC.NUMERICAL, + "Solution Limit", + "Serious numerical difficulties encountered.", ), - _h.HighsModelStatus.kUnknown: (SciPyRC.NUMERICAL, "Unknown"), - _h.HighsModelStatus.kSolutionLimit: (SciPyRC.NUMERICAL, "Solution Limit"), } def get_scipy_status(self, highs_status, highs_message): - """Converts HiGHS status and message to SciPy status and message""" - scipy_status, message_prefix = self.highs_to_scipy.get( + """Converts HiGHS status and message to SciPy-like status and messages""" + if highs_status is None or highs_message is None: + print(f"Highs Status: {highs_status}, Message: {highs_message}") + return ( + self.SciPyRC.NUMERICAL.value, + "HiGHS did not provide a status code. (HiGHS Status None: None)", + ) + + scipy_status_enum, message_prefix, scipy_message = self.highs_to_scipy.get( _h.HighsModelStatus(highs_status), - (SciPyRC.NUMERICAL, "Unknown HiGHS Status"), + ( + self.SciPyRC.NUMERICAL, + "Unknown HiGHS Status", + "The HiGHS status code was not recognized.", + ), ) - scip = SciPyRC(scipy_status) - scipy_message = ( - f"{scip.to_string()} (HiGHS Status {highs_status}: {highs_message})" + full_scipy_message = ( + f"{scipy_message} (HiGHS Status {int(highs_status)}: {highs_message})" ) - return scipy_status.value, scipy_message + return scipy_status_enum.value, full_scipy_message def _replace_inf(x): diff --git a/scipy/optimize/_milp.py b/scipy/optimize/_milp.py index bc4f192b4992..f70e3297af4a 100644 --- a/scipy/optimize/_milp.py +++ b/scipy/optimize/_milp.py @@ -2,7 +2,7 @@ import numpy as np from scipy.sparse import csc_array, vstack, issparse from scipy._lib._util import VisibleDeprecationWarning -from ._highs._highs_wrapper import _highs_wrapper # type: ignore[import] +from ._highs._highs_wrapper import _highs_wrapper from ._constraints import LinearConstraint, Bounds from ._optimize import OptimizeResult from ._linprog_highs import HighsStatusMapping diff --git a/scipy/optimize/tests/test_milp.py b/scipy/optimize/tests/test_milp.py index f78cb2cd0ac0..4b28ff41a371 100644 --- a/scipy/optimize/tests/test_milp.py +++ b/scipy/optimize/tests/test_milp.py @@ -97,9 +97,8 @@ def test_result(): res = milp(c=c, constraints=(A, b, b), bounds=(0, 1), integrality=1) assert res.status == 0 assert res.success - msg = "Optimization terminated successfully." + msg = "Optimization terminated successfully. (HiGHS Status 7:" assert res.message.startswith(msg) - assert 'HighsModelStatus.kOptimal' in res.message assert isinstance(res.x, np.ndarray) assert isinstance(res.fun, float) assert isinstance(res.mip_node_count, int) @@ -111,8 +110,8 @@ def test_result(): options={'time_limit': 0.05}) assert res.status == 1 assert not res.success - msg = "Time limit reached" - assert 'HighsModelStatus.kTimeLimit' in res.message + msg = "Time limit reached. (HiGHS Status 13:" + assert res.message.startswith(msg) assert msg in res.message assert (res.fun is res.mip_dual_bound is res.mip_gap is res.mip_node_count is res.x is None) @@ -120,8 +119,7 @@ def test_result(): res = milp(1, bounds=(1, -1)) assert res.status == 2 assert not res.success - msg = "The problem is infeasible" - assert 'HighsModelStatus.kInfeasible' in res.message + msg = "The problem is infeasible. (HiGHS Status 8:" assert res.message.startswith(msg) assert (res.fun is res.mip_dual_bound is res.mip_gap is res.mip_node_count is res.x is None) @@ -129,8 +127,7 @@ def test_result(): res = milp(-1) assert res.status == 3 assert not res.success - assert 'HighsModelStatus.kUnbounded' in res.message - msg = "The problem is unbounded." + msg = "The problem is unbounded. (HiGHS Status 10:" assert res.message.startswith(msg) assert (res.fun is res.mip_dual_bound is res.mip_gap is res.mip_node_count is res.x is None) @@ -294,14 +291,13 @@ def test_infeasible_prob_16609(): _msg_time = "Time limit reached. (HiGHS Status 13:" -_msg_iter = "Iteration limit reached. (HiGHS Status 14:" +_msg_sol = "Serious numerical difficulties encountered. (HiGHS Status 16:" @pytest.mark.skipif(np.intp(0).itemsize < 8, reason="Unhandled 32-bit GCC FP bug") -@pytest.mark.slow @pytest.mark.parametrize(["options", "msg"], [({"time_limit": 0.1}, _msg_time), - ({"node_limit": 1}, _msg_iter)]) + ({"node_limit": 1}, _msg_sol)]) def test_milp_timeout_16545(options, msg): # Ensure solution is not thrown away if MILP solver times out # -- see gh-16545 diff --git a/subprojects/highs b/subprojects/highs new file mode 160000 index 000000000000..704a264a4123 --- /dev/null +++ b/subprojects/highs @@ -0,0 +1 @@ +Subproject commit 704a264a4123178ab09776864ba5bcbbdb69e3f7 diff --git a/subprojects/highs.wrap b/subprojects/highs.wrap index 14468dfcfa45..045f16018e55 100644 --- a/subprojects/highs.wrap +++ b/subprojects/highs.wrap @@ -1,3 +1,3 @@ [wrap-git] url = https://github.com/HaoZeke/highs.git -revision = 325d5356835c6d258441ad45b19a357e4857a895 +revision = 704a264a4123178ab09776864ba5bcbbdb69e3f7 From 339755e2501e3ccde5a5a3b9c8590dcadb3a9b16 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 21 Oct 2023 23:17:48 +0000 Subject: [PATCH 06/64] MAINT: Pin highs to a version without gh-15888 MAINT: Switch to the scipy fork of highs TST: xfail known flaky test Co-authored-by: mckib2 MAINT: Add a note on .gitmodule location Co-authored-by: h-vetinari --- .gitmodules | 5 +++-- scipy/optimize/tests/test_milp.py | 2 ++ subprojects/highs | 2 +- subprojects/highs.wrap | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index dbf08905c7c9..f57d6adbfd78 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,7 +19,8 @@ [submodule "scipy/_lib/pocketfft"] path = scipy/_lib/pocketfft url = https://github.com/scipy/pocketfft +# All submodules are required to be under subprojects for meson: +# https://mesonbuild.com/Subprojects.html#why-must-all-subprojects-be-inside-a-single-directory [submodule "subprojects/highs"] path = subprojects/highs - url = https://github.com/HaoZeke/highs - branch = forSciPy + url = https://github.com/scipy/highs diff --git a/scipy/optimize/tests/test_milp.py b/scipy/optimize/tests/test_milp.py index 4b28ff41a371..8bf2d8e6b60e 100644 --- a/scipy/optimize/tests/test_milp.py +++ b/scipy/optimize/tests/test_milp.py @@ -294,6 +294,8 @@ def test_infeasible_prob_16609(): _msg_sol = "Serious numerical difficulties encountered. (HiGHS Status 16:" +# See https://github.com/scipy/scipy/pull/19255#issuecomment-1778438888 +@pytest.mark.xfail(reason="Often buggy, revisit with callbacks, gh-19255") @pytest.mark.skipif(np.intp(0).itemsize < 8, reason="Unhandled 32-bit GCC FP bug") @pytest.mark.parametrize(["options", "msg"], [({"time_limit": 0.1}, _msg_time), diff --git a/subprojects/highs b/subprojects/highs index 704a264a4123..a0df06fb20f2 160000 --- a/subprojects/highs +++ b/subprojects/highs @@ -1 +1 @@ -Subproject commit 704a264a4123178ab09776864ba5bcbbdb69e3f7 +Subproject commit a0df06fb20f2c67c02cf84d8a3b72862c2ab2b27 diff --git a/subprojects/highs.wrap b/subprojects/highs.wrap index 045f16018e55..6c1653068b87 100644 --- a/subprojects/highs.wrap +++ b/subprojects/highs.wrap @@ -1,3 +1,3 @@ [wrap-git] -url = https://github.com/HaoZeke/highs.git -revision = 704a264a4123178ab09776864ba5bcbbdb69e3f7 +url = https://github.com/scipy/highs.git +revision = a0df06fb20f2c67c02cf84d8a3b72862c2ab2b27 From ee0b4cacbc4397d0dcb248ccf9ce6dc8cc1eaffa Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 3 Feb 2024 16:59:17 +0000 Subject: [PATCH 07/64] DOC: Update comment for subprojects Co-authored-by: Ralf Gommers --- .gitmodules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index f57d6adbfd78..5d5e60aa5a16 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,7 +19,8 @@ [submodule "scipy/_lib/pocketfft"] path = scipy/_lib/pocketfft url = https://github.com/scipy/pocketfft -# All submodules are required to be under subprojects for meson: +# All submodules used as a Meson `subproject` are required to be under the +# subprojects/ directory - see: # https://mesonbuild.com/Subprojects.html#why-must-all-subprojects-be-inside-a-single-directory [submodule "subprojects/highs"] path = subprojects/highs From 28447be47ed07a0c635a1c56c9ee493ffefcdd34 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 3 Feb 2024 17:11:14 +0000 Subject: [PATCH 08/64] MAINT: Reverse black formatting partially --- scipy/optimize/_linprog_highs.py | 166 +++++++++++++------------------ 1 file changed, 70 insertions(+), 96 deletions(-) diff --git a/scipy/optimize/_linprog_highs.py b/scipy/optimize/_linprog_highs.py index dc5afa362e43..e152da6927bc 100644 --- a/scipy/optimize/_linprog_highs.py +++ b/scipy/optimize/_linprog_highs.py @@ -189,21 +189,15 @@ def convert_to_highs_enum(option, option_str, choices_enum, default_value): return enum_value.to_highs_enum() -def _linprog_highs( - lp, - solver, - time_limit=None, - presolve=True, - disp=False, - maxiter=None, - dual_feasibility_tolerance=None, - primal_feasibility_tolerance=None, - ipm_optimality_tolerance=None, - simplex_dual_edge_weight_strategy=None, - mip_rel_gap=None, - mip_max_nodes=None, - **unknown_options, -): +def _linprog_highs(lp, solver, time_limit=None, presolve=True, + disp=False, maxiter=None, + dual_feasibility_tolerance=None, + primal_feasibility_tolerance=None, + ipm_optimality_tolerance=None, + simplex_dual_edge_weight_strategy=None, + mip_rel_gap=None, + mip_max_nodes=None, + **unknown_options): r""" Solve the following linear programming problem using one of the HiGHS solvers: @@ -393,10 +387,8 @@ def _linprog_highs( simplex algorithm." Mathematical Programming 12.1 (1977): 361-371. """ if unknown_options: - message = ( - f"Unrecognized options detected: {unknown_options}. " - "These will be passed to HiGHS verbatim." - ) + message = (f"Unrecognized options detected: {unknown_options}. " + "These will be passed to HiGHS verbatim.") warn(message, OptimizeWarning, stacklevel=3) # Map options to HiGHS enum values @@ -412,7 +404,7 @@ def _linprog_highs( lb, ub = bounds.T.copy() # separate bounds, copy->C-cntgs # highs_wrapper solves LHS <= A*x <= RHS, not equality constraints with np.errstate(invalid="ignore"): - lhs_ub = -np.ones_like(b_ub) * np.inf # LHS of UB constraints is -inf + lhs_ub = -np.ones_like(b_ub)*np.inf # LHS of UB constraints is -inf rhs_ub = b_ub # RHS of UB constraints is b_ub lhs_eq = b_eq # Equality constraint is inequality rhs_eq = b_eq # constraint with LHS=RHS @@ -426,23 +418,23 @@ def _linprog_highs( A = csc_matrix(A) options = { - "presolve": presolve, - "sense": _h.ObjSense.kMinimize, - "solver": solver, - "time_limit": time_limit, + 'presolve': presolve, + 'sense': _h.ObjSense.kMinimize, + 'solver': solver, + 'time_limit': time_limit, # 'highs_debug_level': _h.kHighs, # TODO - "dual_feasibility_tolerance": dual_feasibility_tolerance, - "ipm_optimality_tolerance": ipm_optimality_tolerance, - "log_to_console": disp, - "mip_max_nodes": mip_max_nodes, - "output_flag": disp, - "primal_feasibility_tolerance": primal_feasibility_tolerance, - "simplex_dual_edge_weight_strategy": simplex_dual_edge_weight_strategy_enum, - "simplex_strategy": simpc.kSimplexStrategyDual.value, + 'dual_feasibility_tolerance': dual_feasibility_tolerance, + 'ipm_optimality_tolerance': ipm_optimality_tolerance, + 'log_to_console': disp, + 'mip_max_nodes': mip_max_nodes, + 'output_flag': disp, + 'primal_feasibility_tolerance': primal_feasibility_tolerance, + 'simplex_dual_edge_weight_strategy': simplex_dual_edge_weight_strategy_enum, + 'simplex_strategy': simpc.kSimplexStrategyDual.value, # 'simplex_crash_strategy': simpc.SimplexCrashStrategy.kSimplexCrashStrategyOff, - "ipm_iteration_limit": maxiter, - "simplex_iteration_limit": maxiter, - "mip_rel_gap": mip_rel_gap, + 'ipm_iteration_limit': maxiter, + 'simplex_iteration_limit': maxiter, + 'mip_rel_gap': mip_rel_gap, } options.update(unknown_options) @@ -457,36 +449,26 @@ def _linprog_highs( else: integrality = np.array(integrality) - res = _highs_wrapper( - c, - A.indptr, - A.indices, - A.data, - lhs, - rhs, - lb, - ub, - integrality.astype(np.uint8), - options, - ) + res = _highs_wrapper(c, A.indptr, A.indices, A.data, lhs, rhs, + lb, ub, integrality.astype(np.uint8), options) # HiGHS represents constraints as lhs/rhs, so # Ax + s = b => Ax = b - s # and we need to split up s by A_ub and A_eq - if "slack" in res: - slack = res["slack"] - con = np.array(slack[len(b_ub) :]) - slack = np.array(slack[: len(b_ub)]) + if 'slack' in res: + slack = res['slack'] + con = np.array(slack[len(b_ub):]) + slack = np.array(slack[:len(b_ub)]) else: slack, con = None, None # lagrange multipliers for equalities/inequalities and upper/lower bounds - if "lambda" in res: - lamda = res["lambda"] - marg_ineqlin = np.array(lamda[: len(b_ub)]) - marg_eqlin = np.array(lamda[len(b_ub) :]) - marg_upper = np.array(res["marg_bnds"][1, :]) - marg_lower = np.array(res["marg_bnds"][0, :]) + if 'lambda' in res: + lamda = res['lambda'] + marg_ineqlin = np.array(lamda[:len(b_ub)]) + marg_eqlin = np.array(lamda[len(b_ub):]) + marg_upper = np.array(res['marg_bnds'][1, :]) + marg_lower = np.array(res['marg_bnds'][0, :]) else: marg_ineqlin, marg_eqlin = None, None marg_upper, marg_lower = None, None @@ -507,46 +489,38 @@ def is_valid_x(val): x = np.array(res["x"]) if "x" in res and is_valid_x(res["x"]) else None - sol = { - "x": x, - "slack": slack, - "con": con, - "ineqlin": OptimizeResult( - { - "residual": slack, - "marginals": marg_ineqlin, - } - ), - "eqlin": OptimizeResult( - { - "residual": con, - "marginals": marg_eqlin, - } - ), - "lower": OptimizeResult( - { - "residual": None if x is None else x - lb, - "marginals": marg_lower, - } - ), - "upper": OptimizeResult( - {"residual": None if x is None else ub - x, "marginals": marg_upper} - ), - "fun": res.get("fun"), - "status": status, - "success": res["status"] == _h.HighsModelStatus.kOptimal, - "message": message, - "nit": res.get("simplex_nit", 0) or res.get("ipm_nit", 0), - "crossover_nit": res.get("crossover_nit"), - } + sol = {'x': x, + 'slack': slack, + 'con': con, + 'ineqlin': OptimizeResult({ + 'residual': slack, + 'marginals': marg_ineqlin, + }), + 'eqlin': OptimizeResult({ + 'residual': con, + 'marginals': marg_eqlin, + }), + 'lower': OptimizeResult({ + 'residual': None if x is None else x - lb, + 'marginals': marg_lower, + }), + 'upper': OptimizeResult({ + 'residual': None if x is None else ub - x, + 'marginals': marg_upper + }), + 'fun': res.get('fun'), + 'status': status, + 'success': res['status'] == _h.HighsModelStatus.kOptimal, + 'message': message, + 'nit': res.get('simplex_nit', 0) or res.get('ipm_nit', 0), + 'crossover_nit': res.get('crossover_nit'), + } if np.any(x) and integrality is not None: - sol.update( - { - "mip_node_count": res.get("mip_node_count", 0), - "mip_dual_bound": res.get("mip_dual_bound", 0.0), - "mip_gap": res.get("mip_gap", 0.0), - } - ) + sol.update({ + 'mip_node_count': res.get('mip_node_count', 0), + 'mip_dual_bound': res.get('mip_dual_bound', 0.0), + 'mip_gap': res.get('mip_gap', 0.0), + }) return sol From 9cb657cf16c125e23f3343e92bb424d2549c948a Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 3 Feb 2024 17:28:58 +0000 Subject: [PATCH 09/64] MAINT: Fix linter concerns --- scipy/optimize/_linprog_highs.py | 21 ++++++++++++++------- tools/lint.toml | 7 ++++--- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/scipy/optimize/_linprog_highs.py b/scipy/optimize/_linprog_highs.py index e152da6927bc..283476d45ecc 100644 --- a/scipy/optimize/_linprog_highs.py +++ b/scipy/optimize/_linprog_highs.py @@ -128,7 +128,10 @@ def __init__(self): def get_scipy_status(self, highs_status, highs_message): """Converts HiGHS status and message to SciPy-like status and messages""" if highs_status is None or highs_message is None: - print(f"Highs Status: {highs_status}, Message: {highs_message}") + warn( + f"Highs Status: {highs_status}, Message: {highs_message}", + stacklevel=3 + ) return ( self.SciPyRC.NUMERICAL.value, "HiGHS did not provide a status code. (HiGHS Status None: None)", @@ -142,10 +145,10 @@ def get_scipy_status(self, highs_status, highs_message): "The HiGHS status code was not recognized.", ), ) - full_scipy_message = ( + full_scipy_msg = ( f"{scipy_message} (HiGHS Status {int(highs_status)}: {highs_message})" ) - return scipy_status_enum.value, full_scipy_message + return scipy_status_enum.value, full_scipy_msg def _replace_inf(x): @@ -164,10 +167,14 @@ class SimplexStrategy(Enum): def to_highs_enum(self): mapping = { - SimplexStrategy.DANTZIG: simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategyDantzig.value, - SimplexStrategy.DEVEX: simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategyDevex.value, - SimplexStrategy.STEEPEST_DEVEX: simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategyChoose.value, - SimplexStrategy.STEEPEST: simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategySteepestEdge.value, + SimplexStrategy.DANTZIG: + simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategyDantzig.value, + SimplexStrategy.DEVEX: + simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategyDevex.value, + SimplexStrategy.STEEPEST_DEVEX: + simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategyChoose.value, + SimplexStrategy.STEEPEST: + simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategySteepestEdge.value, } return mapping.get(self) diff --git a/tools/lint.toml b/tools/lint.toml index 2f04d356026f..5d1f0f20f26c 100644 --- a/tools/lint.toml +++ b/tools/lint.toml @@ -10,11 +10,12 @@ target-version = "py39" # Also, `PGH004` which checks for blanket (non-specific) `noqa`s # and `B028` which checks that warnings include the `stacklevel` keyword. # `B028` added in gh-19623. -select = ["E", "F", "PGH004", "UP", "B028"] -ignore = ["E741"] +lint.select = ["E", "F", "PGH004", "UP", "B028"] +lint.ignore = ["E741"] +exclude = ["scipy/datasets/_registry.py"] # Allow unused variables when underscore-prefixed. -dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" +lint.dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" [lint.per-file-ignores] "**/__init__.py" = ["E402", "F401", "F403", "F405"] From 168042281c0987fcf015987599523b76ae9500c7 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 3 Feb 2024 18:27:03 +0000 Subject: [PATCH 10/64] BLD: Never use zlib for highs --- scipy/optimize/_highs/meson.build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scipy/optimize/_highs/meson.build b/scipy/optimize/_highs/meson.build index 21e49b8b9c60..9f0e30778649 100644 --- a/scipy/optimize/_highs/meson.build +++ b/scipy/optimize/_highs/meson.build @@ -1,6 +1,7 @@ # Setup the highs library highs_proj = subproject('highs', - default_options : ['default_library=static']) + default_options : ['default_library=static', + 'use_zlib=false']) highs_dep = highs_proj.get_variable('highs_dep') highspy_cpp = highs_proj.get_variable('highspy_cpp') highsoptions_cpp = highs_proj.get_variable('highsoptions_cpp') From 1466d519f9a2cffc470b6911efef36c1c295fac6 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 3 Feb 2024 18:27:26 +0000 Subject: [PATCH 11/64] BLD: Remove highs wrap, add back git subm check Co-authored-by: rgommers --- scipy/optimize/_highs/meson.build | 4 ++++ subprojects/highs.wrap | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 subprojects/highs.wrap diff --git a/scipy/optimize/_highs/meson.build b/scipy/optimize/_highs/meson.build index 9f0e30778649..2ffc324292da 100644 --- a/scipy/optimize/_highs/meson.build +++ b/scipy/optimize/_highs/meson.build @@ -1,4 +1,8 @@ # Setup the highs library +fs = import('fs') +if not fs.exists('../../../subprojects/highs/README.md') + error('Missing the `highs` submodule! Run `git submodule update --init` to fix this.') +endif highs_proj = subproject('highs', default_options : ['default_library=static', 'use_zlib=false']) diff --git a/subprojects/highs.wrap b/subprojects/highs.wrap deleted file mode 100644 index 6c1653068b87..000000000000 --- a/subprojects/highs.wrap +++ /dev/null @@ -1,3 +0,0 @@ -[wrap-git] -url = https://github.com/scipy/highs.git -revision = a0df06fb20f2c67c02cf84d8a3b72862c2ab2b27 From 2ad566ab387bf73ade3b95f760eb703187d695bf Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 3 Feb 2024 18:43:25 +0000 Subject: [PATCH 12/64] BLD: Never use zlib --- scipy/optimize/_highs/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scipy/optimize/_highs/meson.build b/scipy/optimize/_highs/meson.build index 2ffc324292da..9fdb5ec33251 100644 --- a/scipy/optimize/_highs/meson.build +++ b/scipy/optimize/_highs/meson.build @@ -5,7 +5,7 @@ if not fs.exists('../../../subprojects/highs/README.md') endif highs_proj = subproject('highs', default_options : ['default_library=static', - 'use_zlib=false']) + 'use_zlib=disabled']) highs_dep = highs_proj.get_variable('highs_dep') highspy_cpp = highs_proj.get_variable('highspy_cpp') highsoptions_cpp = highs_proj.get_variable('highsoptions_cpp') From 8d9f6e594d8cad91674a590d91d2bee9c0a45eb5 Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 3 Feb 2024 20:03:19 +0000 Subject: [PATCH 13/64] MAINT: Fix tests --- scipy/optimize/_linprog_highs.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scipy/optimize/_linprog_highs.py b/scipy/optimize/_linprog_highs.py index 283476d45ecc..e703a32c41d9 100644 --- a/scipy/optimize/_linprog_highs.py +++ b/scipy/optimize/_linprog_highs.py @@ -128,10 +128,7 @@ def __init__(self): def get_scipy_status(self, highs_status, highs_message): """Converts HiGHS status and message to SciPy-like status and messages""" if highs_status is None or highs_message is None: - warn( - f"Highs Status: {highs_status}, Message: {highs_message}", - stacklevel=3 - ) + print(f"Highs Status: {highs_status}, Message: {highs_message}") return ( self.SciPyRC.NUMERICAL.value, "HiGHS did not provide a status code. (HiGHS Status None: None)", From 181c66aa0f216555eb914d37f8a30b9005de82be Mon Sep 17 00:00:00 2001 From: Rohit Goswami Date: Sat, 17 Feb 2024 19:59:16 +0000 Subject: [PATCH 14/64] MAINT: Lint cleanup --- scipy/optimize/_highs/_highs_wrapper.py | 10 +++++++--- tools/lint.toml | 7 +++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scipy/optimize/_highs/_highs_wrapper.py b/scipy/optimize/_highs/_highs_wrapper.py index 909d9d651a44..7fe5f387c0db 100644 --- a/scipy/optimize/_highs/_highs_wrapper.py +++ b/scipy/optimize/_highs/_highs_wrapper.py @@ -47,7 +47,8 @@ def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, opti # ask for the option type opt_type = hoptmanager.get_option_type(key) if -1 == opt_type: - warn(f"Unrecognized options detected: {dict({key: val})}", OptimizeWarning) + warn(f"Unrecognized options detected: {dict({key: val})}", + OptimizeWarning, stacklevel = 2) continue else: if key in ("presolve", "parallel"): @@ -59,6 +60,7 @@ def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, opti f'Option f"{key}" is "{val}", but only True or False is ' f"allowed. Using default.", OptimizeWarning, + stacklevel = 2, ) continue opt_type = _h.HighsOptionType(opt_type) @@ -69,12 +71,13 @@ def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, opti f'Option f"{key}" is "{val}", but only True or False is ' f"allowed. Using default.", OptimizeWarning, + stacklevel = 2, ) continue # warn or set option if status != 0: - warn(msg, OptimizeWarning) + warn(msg, OptimizeWarning, stacklevel = 2) else: setattr(highs_options, key, val) @@ -145,7 +148,8 @@ def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, opti res.update( { "status": model_status, - "message": f"model_status is {highs.modelStatusToString(model_status)}; " + "message": "model_status is " + f"{highs.modelStatusToString(model_status)}; " "primal_status is " f"{highs.solutionStatusToString(info.primal_solution_status)}", "simplex_nit": info.simplex_iteration_count, diff --git a/tools/lint.toml b/tools/lint.toml index 5d1f0f20f26c..2f04d356026f 100644 --- a/tools/lint.toml +++ b/tools/lint.toml @@ -10,12 +10,11 @@ target-version = "py39" # Also, `PGH004` which checks for blanket (non-specific) `noqa`s # and `B028` which checks that warnings include the `stacklevel` keyword. # `B028` added in gh-19623. -lint.select = ["E", "F", "PGH004", "UP", "B028"] -lint.ignore = ["E741"] -exclude = ["scipy/datasets/_registry.py"] +select = ["E", "F", "PGH004", "UP", "B028"] +ignore = ["E741"] # Allow unused variables when underscore-prefixed. -lint.dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" [lint.per-file-ignores] "**/__init__.py" = ["E402", "F401", "F403", "F405"] From b7bae9c0b9c298c10a00f995a78c828cc0082f74 Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Sun, 14 Apr 2024 00:29:15 -0700 Subject: [PATCH 15/64] TST: stats: mark tests xslow --- scipy/stats/tests/test_axis_nan_policy.py | 10 ++++++++++ scipy/stats/tests/test_distributions.py | 9 ++++++--- scipy/stats/tests/test_fast_gen_inversion.py | 2 ++ scipy/stats/tests/test_fit.py | 8 ++++---- scipy/stats/tests/test_hypotests.py | 1 + scipy/stats/tests/test_morestats.py | 2 +- scipy/stats/tests/test_multivariate.py | 2 +- scipy/stats/tests/test_resampling.py | 4 +++- scipy/stats/tests/test_sampling.py | 2 ++ scipy/stats/tests/test_sensitivity_analysis.py | 1 + scipy/stats/tests/test_stats.py | 18 +++++++++--------- 11 files changed, 40 insertions(+), 19 deletions(-) diff --git a/scipy/stats/tests/test_axis_nan_policy.py b/scipy/stats/tests/test_axis_nan_policy.py index b4a0b30f8374..00539d50ed12 100644 --- a/scipy/stats/tests/test_axis_nan_policy.py +++ b/scipy/stats/tests/test_axis_nan_policy.py @@ -5,6 +5,7 @@ # functions in stats._util. from itertools import product, combinations_with_replacement, permutations +import os import re import pickle import pytest @@ -17,6 +18,9 @@ from scipy._lib._util import AxisError +SCIPY_XSLOW = int(os.environ.get('SCIPY_XSLOW', '0')) + + def unpack_ttest_result(res): low, high = res.confidence_interval() return (res.statistic, res.pvalue, res.df, res._standard_error, @@ -244,6 +248,8 @@ def nan_policy_1d(hypotest, data1d, unpacker, *args, n_outputs=2, def test_axis_nan_policy_fast(hypotest, args, kwds, n_samples, n_outputs, paired, unpacker, nan_policy, axis, data_generator): + if hypotest in {stats.cramervonmises_2samp} and not SCIPY_XSLOW: + pytest.skip("Too slow.") _axis_nan_policy_test(hypotest, args, kwds, n_samples, n_outputs, paired, unpacker, nan_policy, axis, data_generator) @@ -260,6 +266,8 @@ def test_axis_nan_policy_fast(hypotest, args, kwds, n_samples, n_outputs, def test_axis_nan_policy_full(hypotest, args, kwds, n_samples, n_outputs, paired, unpacker, nan_policy, axis, data_generator): + if hypotest in {stats.cramervonmises_2samp} and not SCIPY_XSLOW: + pytest.skip("Too slow.") _axis_nan_policy_test(hypotest, args, kwds, n_samples, n_outputs, paired, unpacker, nan_policy, axis, data_generator) @@ -680,6 +688,8 @@ def _check_arrays_broadcastable(arrays, axis): "paired", "unpacker"), axis_nan_policy_cases) def test_empty(hypotest, args, kwds, n_samples, n_outputs, paired, unpacker): # test for correct output shape when at least one input is empty + if hypotest in {stats.kruskal, stats.friedmanchisquare} and not SCIPY_XSLOW: + pytest.skip("Too slow.") if hypotest in override_propagate_funcs: reason = "Doesn't follow the usual pattern. Tested separately." diff --git a/scipy/stats/tests/test_distributions.py b/scipy/stats/tests/test_distributions.py index 61de402fbb20..c92abcbe3d2c 100644 --- a/scipy/stats/tests/test_distributions.py +++ b/scipy/stats/tests/test_distributions.py @@ -1861,6 +1861,7 @@ def test_mean_gh18511(self): rm = n / M * N assert_allclose(hm, rm) + @pytest.mark.xslow def test_sf_gh18506(self): # gh-18506 reported that `sf` was incorrect for large population; # check that this is resolved @@ -7549,13 +7550,14 @@ class TestStudentizedRange: (1, 10, np.inf, 0.000519869467083), ] + @pytest.mark.slow def test_cdf_against_tables(self): for pvk, q in self.data: p_expected, v, k = pvk res_p = stats.studentized_range.cdf(q, k, v) assert_allclose(res_p, p_expected, rtol=1e-4) - @pytest.mark.slow + @pytest.mark.xslow def test_ppf_against_tables(self): for pvk, q_expected in self.data: p, v, k = pvk @@ -7589,7 +7591,7 @@ def test_pdf_against_mp(self, case_result): atol=src_case["expected_atol"], rtol=src_case["expected_rtol"]) - @pytest.mark.slow + @pytest.mark.xslow @pytest.mark.xfail_on_32bit("intermittent RuntimeWarning: invalid value.") @pytest.mark.parametrize("case_result", pregenerated_data["moment_data"]) def test_moment_against_mp(self, case_result): @@ -7606,6 +7608,7 @@ def test_moment_against_mp(self, case_result): atol=src_case["expected_atol"], rtol=src_case["expected_rtol"]) + @pytest.mark.slow def test_pdf_integration(self): k, v = 3, 10 # Test whether PDF integration is 1 like it should be. @@ -7636,7 +7639,7 @@ def test_cdf_against_r(self, r_case_result): res = stats.studentized_range.cdf(q, k, v) assert_allclose(res, r_res) - @pytest.mark.slow + @pytest.mark.xslow @pytest.mark.xfail_on_32bit("intermittent RuntimeWarning: invalid value.") def test_moment_vectorization(self): # Test moment broadcasting. Calls `_munp` directly because diff --git a/scipy/stats/tests/test_fast_gen_inversion.py b/scipy/stats/tests/test_fast_gen_inversion.py index 3369af969150..e2d00134ccbf 100644 --- a/scipy/stats/tests/test_fast_gen_inversion.py +++ b/scipy/stats/tests/test_fast_gen_inversion.py @@ -132,6 +132,7 @@ def test_u_error(distname, args): assert u_error <= 1e-10 +@pytest.mark.xslow @pytest.mark.xfail(reason="geninvgauss CDF is not accurate") def test_geninvgauss_uerror(): dist = stats.geninvgauss(3.2, 1.5) @@ -139,6 +140,7 @@ def test_geninvgauss_uerror(): err = rng.evaluate_error(size=10_000, random_state=67982) assert err[0] < 1e-10 + # TODO: add more distributions @pytest.mark.parametrize(("distname, args"), [("beta", (0.11, 0.11))]) def test_error_extreme_params(distname, args): diff --git a/scipy/stats/tests/test_fit.py b/scipy/stats/tests/test_fit.py index bcb776f71e35..30adef754ca2 100644 --- a/scipy/stats/tests/test_fit.py +++ b/scipy/stats/tests/test_fit.py @@ -79,7 +79,7 @@ # Don't run the fit test on these: skip_fit = [ 'erlang', # Subclass of gamma, generates a warning. - 'genhyperbolic', # too slow + 'genhyperbolic', 'norminvgauss', # too slow ] @@ -292,7 +292,7 @@ def cases_test_fit_mse(): 'wald', 'weibull_max', 'weibull_min', 'wrapcauchy'} # Please keep this list in alphabetical order... - xslow_basic_fit = {'beta', 'betaprime', 'burr', 'burr12', + xslow_basic_fit = {'argus', 'beta', 'betaprime', 'burr', 'burr12', 'f', 'gengamma', 'gennorm', 'halfgennorm', 'invgamma', 'invgauss', 'kappa4', 'loguniform', @@ -840,7 +840,7 @@ def test_against_anderson_case_3(self): assert_allclose(res.statistic, 0.559) # See [1] Table 1B 1.2 assert_allclose(res.pvalue, 0.15, atol=5e-3) - @pytest.mark.slow + @pytest.mark.xslow def test_against_anderson_gumbel_r(self): rng = np.random.default_rng(7302761058217743) # c that produced critical value of statistic found w/ root_scalar @@ -901,7 +901,7 @@ def test_against_filliben_norm_table(self, case): res = stats.scoreatpercentile(res.null_distribution, percentiles*100) assert_allclose(res, ref, atol=2e-3) - @pytest.mark.slow + @pytest.mark.xslow @pytest.mark.parametrize('case', [(5, 0.95772790260469, 0.4755), (6, 0.95398832257958, 0.3848), (7, 0.9432692889277, 0.2328)]) diff --git a/scipy/stats/tests/test_hypotests.py b/scipy/stats/tests/test_hypotests.py index 064cf3dec71a..45d871298c77 100644 --- a/scipy/stats/tests/test_hypotests.py +++ b/scipy/stats/tests/test_hypotests.py @@ -1353,6 +1353,7 @@ def test_exact_pvalue(self, statistic, m, n, pval): # The values are taken from Table 2, 3, 4 and 5 assert_equal(_pval_cvm_2samp_exact(statistic, m, n), pval) + @pytest.mark.slow def test_large_sample(self): # for large samples, the statistic U gets very large # do a sanity check that p-value is not 0, 1 or nan diff --git a/scipy/stats/tests/test_morestats.py b/scipy/stats/tests/test_morestats.py index 5fa02fe42000..4b7e6a1d20c8 100644 --- a/scipy/stats/tests/test_morestats.py +++ b/scipy/stats/tests/test_morestats.py @@ -423,7 +423,7 @@ def test_example1b(self): tm[0:5], 4) assert_allclose(p, 0.0020, atol=0.00025) - @pytest.mark.slow + @pytest.mark.xslow def test_example2a(self): # Example data taken from an earlier technical report of # Scholz and Stephens diff --git a/scipy/stats/tests/test_multivariate.py b/scipy/stats/tests/test_multivariate.py index 093a991542fa..6b9ba9e16c7f 100644 --- a/scipy/stats/tests/test_multivariate.py +++ b/scipy/stats/tests/test_multivariate.py @@ -2583,7 +2583,7 @@ def test_cdf_signs(self): cdf = multivariate_normal.cdf(b, mean, cov, df, lower_limit=a) assert_allclose(cdf, cdf[0]*expected_signs) - @pytest.mark.parametrize('dim', [1, 2, 5, 10]) + @pytest.mark.parametrize('dim', [1, 2, 5]) def test_cdf_against_multivariate_normal(self, dim): # Check accuracy against MVN randomly-generated cases self.cdf_against_mvn_test(dim) diff --git a/scipy/stats/tests/test_resampling.py b/scipy/stats/tests/test_resampling.py index 4379d4b57cf6..5587d08192fa 100644 --- a/scipy/stats/tests/test_resampling.py +++ b/scipy/stats/tests/test_resampling.py @@ -788,7 +788,7 @@ def stat(x): monte_carlo_test([1, 2, 3], stats.norm.rvs, stat, alternative='ekki') - + @pytest.mark.xslow def test_batch(self): # make sure that the `batch` parameter is respected by checking the # maximum batch size provided in calls to `statistic` @@ -909,6 +909,7 @@ def statistic(x, axis): assert_allclose(res.statistic, expected.statistic) assert_allclose(res.pvalue, expected.pvalue, atol=self.atol) + @pytest.mark.xslow @pytest.mark.parametrize('a', np.linspace(-0.5, 0.5, 5)) # skewness def test_against_cramervonmises(self, a): # test that monte_carlo_test can reproduce pvalue of cramervonmises @@ -928,6 +929,7 @@ def statistic1d(x): assert_allclose(res.statistic, expected.statistic) assert_allclose(res.pvalue, expected.pvalue, atol=self.atol) + @pytest.mark.xslow @pytest.mark.parametrize('dist_name', ('norm', 'logistic')) @pytest.mark.parametrize('i', range(5)) def test_against_anderson(self, dist_name, i): diff --git a/scipy/stats/tests/test_sampling.py b/scipy/stats/tests/test_sampling.py index 10b4e5af96ec..82fe9c36ea19 100644 --- a/scipy/stats/tests/test_sampling.py +++ b/scipy/stats/tests/test_sampling.py @@ -927,6 +927,7 @@ def test_cdf(self, x): assert_allclose(res, expected, rtol=1e-11, atol=1e-11) assert res.shape == expected.shape + @pytest.mark.slow def test_u_error(self): dist = StandardNormal() rng = NumericalInversePolynomial(dist, u_resolution=1e-10) @@ -1203,6 +1204,7 @@ def test_ppf(self, u): assert_allclose(res, expected, rtol=1e-9, atol=3e-10) assert res.shape == expected.shape + @pytest.mark.slow def test_u_error(self): dist = StandardNormal() rng = NumericalInverseHermite(dist, u_resolution=1e-10) diff --git a/scipy/stats/tests/test_sensitivity_analysis.py b/scipy/stats/tests/test_sensitivity_analysis.py index ac05c8c9a3f5..27ff6cdd19de 100644 --- a/scipy/stats/tests/test_sensitivity_analysis.py +++ b/scipy/stats/tests/test_sensitivity_analysis.py @@ -74,6 +74,7 @@ def test_sample_AB(self): AB = sample_AB(A=A, B=B) assert_allclose(AB, ref) + @pytest.mark.xslow @pytest.mark.xfail_on_32bit("Can't create large array for test") @pytest.mark.parametrize( 'func', diff --git a/scipy/stats/tests/test_stats.py b/scipy/stats/tests/test_stats.py index e8c811dc7a01..545767a76409 100644 --- a/scipy/stats/tests/test_stats.py +++ b/scipy/stats/tests/test_stats.py @@ -4536,7 +4536,7 @@ def test_argument_checking(self): assert_raises(ValueError, stats.ks_2samp, [1], []) assert_raises(ValueError, stats.ks_2samp, [], []) - @pytest.mark.slow + @pytest.mark.xslow def test_gh12218(self): """Ensure gh-12218 is fixed.""" # gh-1228 triggered a TypeError calculating sqrt(n1*n2*(n1+n2)). @@ -7720,8 +7720,8 @@ def test_error_code(self): v_values = rng.random(size=(2, 2)) _ = stats.wasserstein_distance_nd(u_values, v_values) - @pytest.mark.parametrize('u_size', [1, 10, 300]) - @pytest.mark.parametrize('v_size', [1, 10, 300]) + @pytest.mark.parametrize('u_size', [1, 10, 50]) + @pytest.mark.parametrize('v_size', [1, 10, 50]) def test_optimization_vs_analytical(self, u_size, v_size): rng = np.random.default_rng(45634745675) # Test when u_weights = None, v_weights = None @@ -8199,7 +8199,7 @@ def _simulations(self, samps=100, dims=1, sim_type=""): return x, y - @pytest.mark.slow + @pytest.mark.xslow @pytest.mark.parametrize("sim_type, obs_stat, obs_pvalue", [ ("linear", 0.97, 1/1000), # test linear simulation ("nonlinear", 0.163, 1/1000), # test spiral simulation @@ -8216,7 +8216,7 @@ def test_oned(self, sim_type, obs_stat, obs_pvalue): assert_approx_equal(stat, obs_stat, significant=1) assert_approx_equal(pvalue, obs_pvalue, significant=1) - @pytest.mark.slow + @pytest.mark.xslow @pytest.mark.parametrize("sim_type, obs_stat, obs_pvalue", [ ("linear", 0.184, 1/1000), # test linear simulation ("nonlinear", 0.0190, 0.117), # test spiral simulation @@ -8253,7 +8253,7 @@ def test_twosamp(self): assert_approx_equal(stat, 1.0, significant=1) assert_approx_equal(pvalue, 0.001, significant=1) - @pytest.mark.slow + @pytest.mark.xslow def test_workers(self): np.random.seed(12345678) @@ -8265,7 +8265,7 @@ def test_workers(self): assert_approx_equal(stat, 0.97, significant=1) assert_approx_equal(pvalue, 0.001, significant=1) - @pytest.mark.slow + @pytest.mark.xslow def test_random_state(self): # generate x and y x, y = self._simulations(samps=100, dims=1, sim_type="linear") @@ -8275,7 +8275,7 @@ def test_random_state(self): assert_approx_equal(stat, 0.97, significant=1) assert_approx_equal(pvalue, 0.001, significant=1) - @pytest.mark.slow + @pytest.mark.xslow def test_dist_perm(self): np.random.seed(12345678) # generate x and y @@ -8300,7 +8300,7 @@ def test_pvalue_literature(self): _, pvalue, _ = stats.multiscale_graphcorr(x, y, random_state=1) assert_allclose(pvalue, 1/1001) - @pytest.mark.slow + @pytest.mark.xslow def test_alias(self): np.random.seed(12345678) From 86585056a63aef987bbf1baddf1d051596eaaf5e Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Sun, 14 Apr 2024 10:41:32 -0700 Subject: [PATCH 16/64] MAINT: stats: mark more tests slow --- scipy/stats/tests/test_distributions.py | 7 +++++-- scipy/stats/tests/test_kdeoth.py | 4 ++-- scipy/stats/tests/test_morestats.py | 1 + scipy/stats/tests/test_multivariate.py | 5 +++-- scipy/stats/tests/test_odds_ratio.py | 1 + scipy/stats/tests/test_resampling.py | 10 +++++++++- scipy/stats/tests/test_stats.py | 1 + 7 files changed, 22 insertions(+), 7 deletions(-) diff --git a/scipy/stats/tests/test_distributions.py b/scipy/stats/tests/test_distributions.py index c92abcbe3d2c..5ec48f690488 100644 --- a/scipy/stats/tests/test_distributions.py +++ b/scipy/stats/tests/test_distributions.py @@ -5083,6 +5083,7 @@ def nolan_loc_scale_sample_data(self): ) return data + @pytest.mark.slow @pytest.mark.parametrize( "sample_size", [ pytest.param(50), pytest.param(1500, marks=pytest.mark.slow) @@ -5111,7 +5112,7 @@ def test_rvs( ) assert p > 0.05 - @pytest.mark.slow + @pytest.mark.xslow @pytest.mark.parametrize('beta', [0.5, 1]) def test_rvs_alpha1(self, beta): """Additional test cases for rvs for alpha equal to 1.""" @@ -5209,6 +5210,7 @@ def test_fit_loc_extrap(self): assert alpha2 > 1, f"Expected alpha > 1, got {alpha2}" assert loc2 > max(x2), f"Expected loc > {max(x2)}, got {loc2}" + @pytest.mark.slow @pytest.mark.parametrize( "pct_range,alpha_range,beta_range", [ pytest.param( @@ -6026,7 +6028,7 @@ def test_uniform_fit(self): assert_raises(ValueError, stats.uniform.fit, x, floc=2.0) assert_raises(ValueError, stats.uniform.fit, x, fscale=5.0) - @pytest.mark.slow + @pytest.mark.xslow @pytest.mark.parametrize("method", ["MLE", "MM"]) def test_fshapes(self, method): # take a beta distribution, with shapes='a, b', and make sure that @@ -9546,6 +9548,7 @@ def pdf(E, M, Gamma): ref = pdf(x, rho*Gamma, Gamma) assert_allclose(res, ref, rtol=rtol) + @pytest.mark.xslow @pytest.mark.parametrize( "rho,gamma", [ pytest.param( diff --git a/scipy/stats/tests/test_kdeoth.py b/scipy/stats/tests/test_kdeoth.py index dec6fd65a19d..16741e91655e 100644 --- a/scipy/stats/tests/test_kdeoth.py +++ b/scipy/stats/tests/test_kdeoth.py @@ -70,7 +70,7 @@ def test_kde_1d_weighted(): (kdepdf*normpdf).sum()*intervall, decimal=2) -@pytest.mark.slow +@pytest.mark.xslow def test_kde_2d(): #some basic tests comparing to normal distribution np.random.seed(8765678) @@ -110,7 +110,7 @@ def test_kde_2d(): (kdepdf*normpdf).sum()*(intervall**2), decimal=2) -@pytest.mark.slow +@pytest.mark.xslow def test_kde_2d_weighted(): #some basic tests comparing to normal distribution np.random.seed(8765678) diff --git a/scipy/stats/tests/test_morestats.py b/scipy/stats/tests/test_morestats.py index 4b7e6a1d20c8..1b2970d6d632 100644 --- a/scipy/stats/tests/test_morestats.py +++ b/scipy/stats/tests/test_morestats.py @@ -2066,6 +2066,7 @@ def optimizer(fun): assert np.all(bounds[0] < maxlog) assert np.all(maxlog < bounds[1]) + @pytest.mark.slow def test_user_defined_optimizer(self): # tests an optimizer that is not based on scipy.optimize.minimize lmbda = stats.boxcox_normmax(self.x) diff --git a/scipy/stats/tests/test_multivariate.py b/scipy/stats/tests/test_multivariate.py index 6b9ba9e16c7f..3777b2d5796a 100644 --- a/scipy/stats/tests/test_multivariate.py +++ b/scipy/stats/tests/test_multivariate.py @@ -869,13 +869,13 @@ def test_fit_fix_cov_input_validation_dimension(self, fix_cov): "vectors `x`.") with pytest.raises(ValueError, match=msg): multivariate_normal.fit(np.eye(3), fix_cov=fix_cov) - + def test_fit_fix_cov_not_positive_semidefinite(self): error_msg = "`fix_cov` must be symmetric positive semidefinite." with pytest.raises(ValueError, match=error_msg): fix_cov = np.array([[1., 0.], [0., -1.]]) multivariate_normal.fit(np.eye(2), fix_cov=fix_cov) - + def test_fit_fix_mean(self): rng = np.random.default_rng(4385269356937404) loc = rng.random(3) @@ -2655,6 +2655,7 @@ def test_cdf_against_qsimvtv(self, dim, seed, singular): ref = _qsimvtv(20000, df, cov, a - mean, b - mean, rng)[0] assert_allclose(res, ref, atol=1e-4, rtol=1e-3) + @pytest.mark.slow def test_cdf_against_generic_integrators(self): # Compare result against generic numerical integrators dim = 3 diff --git a/scipy/stats/tests/test_odds_ratio.py b/scipy/stats/tests/test_odds_ratio.py index ffb38a05c8df..14b5ca88d0fc 100644 --- a/scipy/stats/tests/test_odds_ratio.py +++ b/scipy/stats/tests/test_odds_ratio.py @@ -105,6 +105,7 @@ def test_sample_odds_ratio_ci(self, case): ci = result.confidence_interval(confidence_level, alternative) assert_allclose([ci.low, ci.high], [ref_low, ref_high], rtol=1e-6) + @pytest.mark.slow @pytest.mark.parametrize('alternative', ['less', 'greater', 'two-sided']) def test_sample_odds_ratio_one_sided_ci(self, alternative): # can't find a good reference for one-sided CI, so bump up the sample diff --git a/scipy/stats/tests/test_resampling.py b/scipy/stats/tests/test_resampling.py index 5587d08192fa..95b24e3c9066 100644 --- a/scipy/stats/tests/test_resampling.py +++ b/scipy/stats/tests/test_resampling.py @@ -161,6 +161,7 @@ def my_statistic(x, y, z, axis=-1): assert_equal(res2.standard_error.shape, result_shape) +@pytest.mark.slow @pytest.mark.xfail_on_32bit("MemoryError with BCa observed in CI") @pytest.mark.parametrize("method", ['basic', 'percentile', 'BCa']) def test_bootstrap_against_theory(method): @@ -304,6 +305,7 @@ def statistic(z, y, axis=0): assert_allclose(a_hat, 0.011008228344026734) +@pytest.mark.slow @pytest.mark.parametrize("method, expected", tests_against_itself_1samp.items()) def test_bootstrap_against_itself_1samp(method, expected): @@ -347,6 +349,7 @@ def test_bootstrap_against_itself_1samp(method, expected): "percentile": 890} +@pytest.mark.slow @pytest.mark.parametrize("method, expected", tests_against_itself_2samp.items()) def test_bootstrap_against_itself_2samp(method, expected): @@ -653,6 +656,7 @@ def statistic_1d(*data): assert_allclose(res1, res2) +@pytest.mark.slow @pytest.mark.parametrize("method", ["basic", "percentile", "BCa"]) def test_vector_valued_statistic(method): # Generate 95% confidence interval around MLE of normal distribution @@ -847,6 +851,7 @@ def statistic(x, axis): assert_allclose(res.statistic, expected.statistic) assert_allclose(res.pvalue, expected.pvalue, atol=self.atol) + @pytest.mark.slow @pytest.mark.parametrize('alternative', ("less", "greater")) @pytest.mark.parametrize('a', np.linspace(-0.5, 0.5, 5)) # skewness def test_against_ks_1samp(self, alternative, a): @@ -929,7 +934,7 @@ def statistic1d(x): assert_allclose(res.statistic, expected.statistic) assert_allclose(res.pvalue, expected.pvalue, atol=self.atol) - @pytest.mark.xslow + @pytest.mark.slow @pytest.mark.parametrize('dist_name', ('norm', 'logistic')) @pytest.mark.parametrize('i', range(5)) def test_against_anderson(self, dist_name, i): @@ -1090,6 +1095,7 @@ def test_input_validation(self): with pytest.raises(ValueError, match=message): power(test, rvs, n_observations, batch=10.5) + @pytest.mark.slow def test_batch(self): # make sure that the `batch` parameter is respected by checking the # maximum batch size provided in calls to `test` @@ -1125,6 +1131,7 @@ def test(x, axis): assert_equal(res1.power, res3.power) assert_equal(res2.power, res3.power) + @pytest.mark.slow def test_vectorization(self): # Test that `power` is vectorized as expected rng = np.random.default_rng(25495254834552) @@ -1787,6 +1794,7 @@ def statistic1d(x, y): >= np.abs(S-mean))/n assert_allclose(expected_Pr_gte_S_mean, Pr_gte_S_mean) + @pytest.mark.slow @pytest.mark.parametrize('alternative, expected_pvalue', (('less', 0.9708333333333), ('greater', 0.05138888888889), diff --git a/scipy/stats/tests/test_stats.py b/scipy/stats/tests/test_stats.py index 545767a76409..ec0a9efcac57 100644 --- a/scipy/stats/tests/test_stats.py +++ b/scipy/stats/tests/test_stats.py @@ -5676,6 +5676,7 @@ def test_ttest_single_observation(): assert_allclose(res, (1.0394023007754, 0.407779907736), rtol=1e-10) +@pytest.mark.slow def test_ttest_1samp_new(): n1, n2, n3 = (10,15,20) rvn1 = stats.norm.rvs(loc=5,scale=10,size=(n1,n2,n3)) From 2188bc2cf2405884241d60aa41234ef798445f4d Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Sun, 14 Apr 2024 14:14:50 -0700 Subject: [PATCH 17/64] TST: scipy.stats: slow/xslow additional tests --- scipy/stats/tests/test_continuous_basic.py | 154 ++++++++++----------- scipy/stats/tests/test_distributions.py | 1 + scipy/stats/tests/test_fit.py | 40 +++--- scipy/stats/tests/test_stats.py | 2 +- 4 files changed, 96 insertions(+), 101 deletions(-) diff --git a/scipy/stats/tests/test_continuous_basic.py b/scipy/stats/tests/test_continuous_basic.py index 7afee32a52b7..89f6790bc18f 100644 --- a/scipy/stats/tests/test_continuous_basic.py +++ b/scipy/stats/tests/test_continuous_basic.py @@ -38,72 +38,34 @@ DECIMAL = 5 # specify the precision of the tests # increased from 0 to 5 _IS_32BIT = (sys.maxsize < 2**32) -# For skipping test_cont_basic -distslow = ['recipinvgauss', 'vonmises', 'kappa4', 'vonmises_line', - 'gausshyper', 'norminvgauss', 'geninvgauss', 'genhyperbolic', - 'truncnorm', 'truncweibull_min'] - -# distxslow are sorted by speed (very slow to slow) -distxslow = ['studentized_range', 'kstwo', 'ksone', 'wrapcauchy', 'genexpon'] - -# For skipping test_moments, which is already marked slow -distxslow_test_moments = ['studentized_range', 'vonmises', 'vonmises_line', - 'ksone', 'kstwo', 'recipinvgauss', 'genexpon'] - -# skip check_fit_args (test is slow) -skip_fit_test_mle = ['exponpow', 'exponweib', 'gausshyper', 'genexpon', - 'halfgennorm', 'gompertz', 'johnsonsb', 'johnsonsu', - 'kappa4', 'ksone', 'kstwo', 'kstwobign', 'mielke', 'ncf', - 'nct', 'powerlognorm', 'powernorm', 'recipinvgauss', - 'trapezoid', 'vonmises', 'vonmises_line', 'levy_stable', - 'rv_histogram_instance', 'studentized_range'] - -# these were really slow in `test_fit`.py. -# note that this list is used to skip both fit_test and fit_fix tests -slow_fit_test_mm = ['argus', 'exponpow', 'exponweib', 'gausshyper', 'genexpon', - 'genhalflogistic', 'halfgennorm', 'gompertz', 'johnsonsb', - 'kappa4', 'kstwobign', 'recipinvgauss', - 'trapezoid', 'truncexpon', 'vonmises', 'vonmises_line', - 'studentized_range'] -# pearson3 fails due to something weird -# the first list fails due to non-finite distribution moments encountered -# most of the rest fail due to integration warnings -# pearson3 is overridden as not implemented due to gh-11746 -fail_fit_test_mm = (['alpha', 'betaprime', 'bradford', 'burr', 'burr12', - 'cauchy', 'crystalball', 'f', 'fisk', 'foldcauchy', - 'genextreme', 'genpareto', 'halfcauchy', 'invgamma', - 'jf_skew_t', 'kappa3', 'levy', 'levy_l', 'loglaplace', - 'lomax', 'mielke', 'nakagami', 'ncf', 'skewcauchy', 't', - 'tukeylambda', 'invweibull', 'rel_breitwigner'] - + ['genhyperbolic', 'johnsonsu', 'ksone', 'kstwo', - 'nct', 'pareto', 'powernorm', 'powerlognorm'] - + ['pearson3']) - -skip_fit_test = {"MLE": skip_fit_test_mle, - "MM": slow_fit_test_mm + fail_fit_test_mm} - -# skip check_fit_args_fix (test is slow) -skip_fit_fix_test_mle = ['burr', 'exponpow', 'exponweib', 'gausshyper', - 'genexpon', 'halfgennorm', 'gompertz', 'johnsonsb', - 'johnsonsu', 'kappa4', 'ksone', 'kstwo', 'kstwobign', - 'levy_stable', 'mielke', 'ncf', 'ncx2', - 'powerlognorm', 'powernorm', 'rdist', 'recipinvgauss', - 'trapezoid', 'truncpareto', 'vonmises', 'vonmises_line', - 'studentized_range'] -# the first list fails due to non-finite distribution moments encountered -# most of the rest fail due to integration warnings -# pearson3 is overridden as not implemented due to gh-11746 -fail_fit_fix_test_mm = (['alpha', 'betaprime', 'burr', 'burr12', 'cauchy', - 'crystalball', 'f', 'fisk', 'foldcauchy', - 'genextreme', 'genpareto', 'halfcauchy', 'invgamma', - 'jf_skew_t', 'kappa3', 'levy', 'levy_l', 'loglaplace', - 'lomax', 'mielke', 'nakagami', 'ncf', 'nct', - 'skewcauchy', 't', 'truncpareto', 'invweibull'] - + ['genhyperbolic', 'johnsonsu', 'ksone', 'kstwo', - 'pareto', 'powernorm', 'powerlognorm'] - + ['pearson3']) -skip_fit_fix_test = {"MLE": skip_fit_fix_test_mle, - "MM": slow_fit_test_mm + fail_fit_fix_test_mm} +# Sets of tests to skip. +# Entries sorted by speed (very slow to slow). +# xslow took > 1s; slow took > 0.5s + +xslow_test_cont_basic = {'studentized_range', 'kstwo', 'ksone', 'vonmises', 'kappa4', + 'recipinvgauss', 'vonmises_line', 'gausshyper', 'rel_breitwigner', + 'norminvgauss'} +slow_test_cont_basic = {'crystalball', 'powerlognorm', 'pearson3'} + +# test_moments is already marked slow +xslow_test_moments = {'studentized_range', 'ksone', 'vonmises', 'vonmises_line', + 'recipinvgauss', 'kstwo', 'kappa4'} + +xslow_fit_all = {'ksone', 'kstwo', 'levy_stable', 'recipinvgauss', 'studentized_range', + 'gausshyper', 'kappa4', 'vonmises_line', 'genhyperbolic', 'ncx2', + 'powerlognorm', 'genexpon'} +xfail_fit_all = {'trapezoid', 'truncpareto'} +xslow_fit_mm = {'argus', 'beta', 'exponpow', 'exponweib', 'gengamma', + 'genhalflogistic', 'geninvgauss', 'gompertz', 'halfgennorm', + 'johnsonsb', 'johnsonsu', 'powernorm', 'vonmises', + 'truncnorm', 'kstwobign', 'rel_breitwigner', 'wrapcauchy', + 'johnsonsu', 'truncweibull_min', 'truncexpon', 'norminvgauss'} +xslow_fit_mle = {'ncf'} +xfail_fit_mm = {'alpha', 'betaprime', 'burr', 'burr12', 'cauchy', 'crystalball', 'f', + 'fisk', 'foldcauchy', 'genextreme', 'genpareto', 'halfcauchy', + 'invgamma', 'jf_skew_t', 'kappa3', 'levy', 'levy_l', 'loglaplace', + 'lomax', 'mielke', 'ncf', 'nct', 'pareto', 'skewcauchy', 't', + 'bradford', 'tukeylambda'} # These distributions fail the complex derivative test below. # Here 'fail' mean produce wrong results and/or raise exceptions, depending @@ -139,21 +101,19 @@ def cases_test_cont_basic(): for distname, arg in distcont[:] + histogram_test_instances: - if distname == 'levy_stable': + if distname == 'levy_stable': # fails; tested separately continue - elif distname in distslow: + if distname in slow_test_cont_basic: yield pytest.param(distname, arg, marks=pytest.mark.slow) - elif distname in distxslow: + elif distname in xslow_test_cont_basic: yield pytest.param(distname, arg, marks=pytest.mark.xslow) else: yield distname, arg @pytest.mark.parametrize('distname,arg', cases_test_cont_basic()) -@pytest.mark.parametrize('sn, n_fit_samples', [(500, 200)]) -def test_cont_basic(distname, arg, sn, n_fit_samples): - # this test skips slow distributions - +@pytest.mark.parametrize('sn', [500]) +def test_cont_basic(distname, arg, sn): try: distfn = getattr(stats, distname) except TypeError: @@ -237,13 +197,48 @@ def test_cont_basic(distname, arg, sn, n_fit_samples): if distname != 'truncnorm': check_ppf_private(distfn, arg, distname) - for method in ["MLE", "MM"]: - if distname not in skip_fit_test[method]: - check_fit_args(distfn, arg, rvs[:n_fit_samples], method) - if distname not in skip_fit_fix_test[method]: - check_fit_args_fix(distfn, arg, rvs[:n_fit_samples], method) +def cases_test_cont_basic_fit(): + message = "Test fails and may be slow" + fail_mark = pytest.mark.skip(reason=message) + + for distname, arg in distcont[:] + histogram_test_instances: + if distname in xfail_fit_all: + yield pytest.param(distname, arg, None, None, marks=fail_mark) + continue + if distname in xslow_fit_all: + yield pytest.param(distname, arg, None, None, marks=pytest.mark.xslow) + continue + + for method in ["MLE", "MM"]: + if method == 'MM' and distname in xfail_fit_mm: + yield pytest.param(distname, arg, method, None, marks=fail_mark) + continue + if method == 'MM' and distname in xslow_fit_mm: + yield pytest.param(distname, arg, method, None, marks=pytest.mark.xslow) + continue + if method == 'MLE' and distname in xslow_fit_mle: + yield pytest.param(distname, arg, method, None, marks=pytest.mark.xslow) + continue + + for fix_args in [True, False]: + yield distname, arg, method, fix_args + +@pytest.mark.parametrize('distname, arg, method, fix_args', + cases_test_cont_basic_fit()) +@pytest.mark.parametrize('n_fit_samples', [200]) +def test_cont_basic_fit(distname, arg, n_fit_samples, method, fix_args): + try: + distfn = getattr(stats, distname) + except TypeError: + distfn = distname + rng = np.random.RandomState(765456) + rvs = distfn.rvs(size=n_fit_samples, *arg, random_state=rng) + if fix_args: + check_fit_args_fix(distfn, arg, rvs, method) + else: + check_fit_args(distfn, arg, rvs, method) @pytest.mark.parametrize('distname,arg', cases_test_cont_basic()) def test_rvs_scalar(distname, arg): @@ -275,7 +270,7 @@ def cases_test_moments(): if distname == 'levy_stable': continue - if distname in distxslow_test_moments: + if distname in xslow_test_moments: yield pytest.param(distname, arg, True, True, True, True, marks=pytest.mark.xslow(reason="too slow")) continue @@ -920,6 +915,7 @@ def test_broadcasting_in_moments_gh12192_regression(): assert vals3.shape == expected3.shape +@pytest.mark.slow def test_kappa3_array_gh13582(): # https://github.com/scipy/scipy/pull/15140#issuecomment-994958241 shapes = [0.5, 1.5, 2.5, 3.5, 4.5] diff --git a/scipy/stats/tests/test_distributions.py b/scipy/stats/tests/test_distributions.py index 5ec48f690488..e5a12be7a634 100644 --- a/scipy/stats/tests/test_distributions.py +++ b/scipy/stats/tests/test_distributions.py @@ -220,6 +220,7 @@ def test_fit_MLE_comp_optimizer(self, rvs_loc, rvs_shape, _assert_less_or_close_loglike(stats.vonmises, data, stats.vonmises.nnlf, **kwds) + @pytest.mark.slow def test_vonmises_fit_bad_floc(self): data = [-0.92923506, -0.32498224, 0.13054989, -0.97252014, 2.79658071, -0.89110948, 1.22520295, 1.44398065, 2.49163859, 1.50315096, diff --git a/scipy/stats/tests/test_fit.py b/scipy/stats/tests/test_fit.py index 30adef754ca2..047ed2617c31 100644 --- a/scipy/stats/tests/test_fit.py +++ b/scipy/stats/tests/test_fit.py @@ -224,21 +224,20 @@ def cases_test_fit_mle(): 'arcsine'} # Please keep this list in alphabetical order... - slow_basic_fit = {'alpha', - 'betaprime', 'binom', 'bradford', 'burr12', - 'chi', 'crystalball', 'dweibull', 'exponpow', - 'f', 'fatiguelife', 'fisk', 'foldcauchy', + slow_basic_fit = {'alpha', 'betaprime', 'binom', 'bradford', 'burr12', + 'chi', 'crystalball', 'dweibull', 'erlang', 'exponnorm', + 'exponpow', 'f', 'fatiguelife', 'fisk', 'foldcauchy', 'gamma', 'genexpon', 'genextreme', 'gennorm', 'genpareto', - 'gompertz', 'halfgennorm', 'invgauss', 'invweibull', + 'gompertz', 'halfgennorm', 'invgamma', 'invgauss', 'invweibull', 'jf_skew_t', 'johnsonsb', 'johnsonsu', 'kappa3', 'kstwobign', 'loglaplace', 'lognorm', 'lomax', 'mielke', 'nakagami', 'nbinom', 'norminvgauss', 'pareto', 'pearson3', 'powerlaw', 'powernorm', - 'randint', 'rdist', 'recipinvgauss', 'rice', - 't', 'uniform', 'weibull_max', 'wrapcauchy'} + 'randint', 'rdist', 'recipinvgauss', 'rice', 'skewnorm', + 't', 'uniform', 'weibull_max', 'weibull_min', 'wrapcauchy'} # Please keep this list in alphabetical order... - xslow_basic_fit = {'beta', 'betabinom', 'burr', 'exponweib', + xslow_basic_fit = {'beta', 'betabinom', 'betanbinom', 'burr', 'exponweib', 'gausshyper', 'gengamma', 'genhalflogistic', 'genhyperbolic', 'geninvgauss', 'hypergeom', 'kappa4', 'loguniform', @@ -246,7 +245,7 @@ def cases_test_fit_mle(): 'nct', 'ncx2', 'nhypergeom', 'powerlognorm', 'reciprocal', 'rel_breitwigner', 'skellam', 'trapezoid', 'triang', 'truncnorm', - 'tukeylambda', 'zipfian'} + 'tukeylambda', 'vonmises', 'zipfian'} for dist in dict(distdiscrete + distcont): if dist in skip_basic_fit or not isinstance(dist, str): @@ -274,34 +273,33 @@ def cases_test_fit_mse(): # Please keep this list in alphabetical order... slow_basic_fit = {'alpha', 'anglit', 'arcsine', 'betabinom', 'bradford', - 'chi', 'chi2', 'crystalball', 'dgamma', 'dweibull', + 'chi', 'chi2', 'crystalball', 'dweibull', 'erlang', 'exponnorm', 'exponpow', 'exponweib', 'fatiguelife', 'fisk', 'foldcauchy', 'foldnorm', 'gamma', 'genexpon', 'genextreme', 'genhalflogistic', 'genlogistic', 'genpareto', 'gompertz', - 'hypergeom', 'invweibull', 'jf_skew_t', 'johnsonsb', + 'hypergeom', 'invweibull', 'johnsonsu', 'kappa3', 'kstwobign', 'laplace_asymmetric', 'loggamma', 'loglaplace', 'lognorm', 'lomax', - 'maxwell', 'mielke', 'nakagami', 'nhypergeom', + 'maxwell', 'nhypergeom', 'pareto', 'powernorm', 'randint', 'recipinvgauss', 'semicircular', 't', 'triang', 'truncexpon', 'truncpareto', - 'truncweibull_min', - 'uniform', 'vonmises_line', + 'uniform', 'wald', 'weibull_max', 'weibull_min', 'wrapcauchy'} # Please keep this list in alphabetical order... xslow_basic_fit = {'argus', 'beta', 'betaprime', 'burr', 'burr12', - 'f', 'gengamma', 'gennorm', - 'halfgennorm', 'invgamma', 'invgauss', - 'kappa4', 'loguniform', - 'ncf', 'nchypergeom_fisher', 'nchypergeom_wallenius', - 'nct', 'ncx2', + 'dgamma', 'f', 'gengamma', 'gennorm', + 'halfgennorm', 'invgamma', 'invgauss', 'jf_skew_t' + 'johnsonsb', 'kappa4', 'loguniform', 'mielke', + 'nakagami', 'ncf', 'nchypergeom_fisher', + 'nchypergeom_wallenius', 'nct', 'ncx2', 'pearson3', 'powerlaw', 'powerlognorm', 'rdist', 'reciprocal', 'rel_breitwigner', 'rice', - 'trapezoid', 'truncnorm', - 'zipfian'} + 'trapezoid', 'truncnorm', 'truncweibull_min', + 'vonmises_line', 'zipfian'} warns_basic_fit = {'skellam'} # can remove mark after gh-14901 is resolved diff --git a/scipy/stats/tests/test_stats.py b/scipy/stats/tests/test_stats.py index ec0a9efcac57..bb101db4402a 100644 --- a/scipy/stats/tests/test_stats.py +++ b/scipy/stats/tests/test_stats.py @@ -5171,7 +5171,7 @@ def test_ttest_ind_permutation_check_p_values(self): class Test_ttest_ind_common: # for tests that are performed on variations of the t-test such as # permutations and trimming - @pytest.mark.slow() + @pytest.mark.xslow() @pytest.mark.parametrize("kwds", [{'permutations': 200, 'random_state': 0}, {'trim': .2}, {}], ids=["permutations", "trim", "basic"]) From 3400b47f963ad123f4356f6a59ff700ed37889ab Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Sun, 14 Apr 2024 17:15:23 -0700 Subject: [PATCH 18/64] Update scipy/stats/tests/test_continuous_basic.py [lint only] --- scipy/stats/tests/test_continuous_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scipy/stats/tests/test_continuous_basic.py b/scipy/stats/tests/test_continuous_basic.py index 89f6790bc18f..01eeea8657e8 100644 --- a/scipy/stats/tests/test_continuous_basic.py +++ b/scipy/stats/tests/test_continuous_basic.py @@ -43,8 +43,8 @@ # xslow took > 1s; slow took > 0.5s xslow_test_cont_basic = {'studentized_range', 'kstwo', 'ksone', 'vonmises', 'kappa4', - 'recipinvgauss', 'vonmises_line', 'gausshyper', 'rel_breitwigner', - 'norminvgauss'} + 'recipinvgauss', 'vonmises_line', 'gausshyper', + 'rel_breitwigner', 'norminvgauss'} slow_test_cont_basic = {'crystalball', 'powerlognorm', 'pearson3'} # test_moments is already marked slow From 788b54aa38cd25262d7a1e5bd34b762379785a4c Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Sat, 20 Apr 2024 11:12:54 -0700 Subject: [PATCH 19/64] MAINT: stats.skew: remove dead code --- scipy/stats/_stats_py.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scipy/stats/_stats_py.py b/scipy/stats/_stats_py.py index 002684e54556..822e150fb944 100644 --- a/scipy/stats/_stats_py.py +++ b/scipy/stats/_stats_py.py @@ -1198,12 +1198,6 @@ def skew(a, axis=0, bias=True, nan_policy='propagate'): a, axis = _chk_asarray(a, axis) n = a.shape[axis] - contains_nan, nan_policy = _contains_nan(a, nan_policy) - - if contains_nan and nan_policy == 'omit': - a = ma.masked_invalid(a) - return mstats_basic.skew(a, axis, bias) - mean = a.mean(axis, keepdims=True) m2 = _moment(a, 2, axis, mean=mean) m3 = _moment(a, 3, axis, mean=mean) From bb3575679665d9ac6a6b359679b935f8a2eebd15 Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Sat, 20 Apr 2024 11:22:59 -0700 Subject: [PATCH 20/64] MAINT: stats.skew: simplify NumPy code --- scipy/stats/_stats_py.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scipy/stats/_stats_py.py b/scipy/stats/_stats_py.py index 822e150fb944..997c052d2b85 100644 --- a/scipy/stats/_stats_py.py +++ b/scipy/stats/_stats_py.py @@ -1199,20 +1199,22 @@ def skew(a, axis=0, bias=True, nan_policy='propagate'): n = a.shape[axis] mean = a.mean(axis, keepdims=True) + mean_reduced = mean.squeeze(axis) # needed later m2 = _moment(a, 2, axis, mean=mean) m3 = _moment(a, 3, axis, mean=mean) with np.errstate(all='ignore'): - zero = (m2 <= (np.finfo(m2.dtype).resolution * mean.squeeze(axis))**2) + eps = np.finfo(m2.dtype).resolution + zero = m2 <= (eps * mean_reduced)**2 vals = np.where(zero, np.nan, m3 / m2**1.5) if not bias: can_correct = ~zero & (n > 2) - if can_correct.any(): - m2 = np.extract(can_correct, m2) - m3 = np.extract(can_correct, m3) + if np.any(can_correct): + m2 = m2[can_correct] + m3 = m3[can_correct] nval = np.sqrt((n - 1.0) * n) / (n - 2.0) * m3 / m2**1.5 - np.place(vals, can_correct, nval) + vals[can_correct] = nval - return vals[()] + return vals[()] if vals.ndim == 0 else vals @_axis_nan_policy_factory( From 0d33968873314f9fb5dfe6bc8f41a50d5edd7608 Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Sat, 20 Apr 2024 11:30:26 -0700 Subject: [PATCH 21/64] ENH: stats.skew: convert to array API --- scipy/stats/_stats_py.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scipy/stats/_stats_py.py b/scipy/stats/_stats_py.py index 997c052d2b85..b197a0d60781 100644 --- a/scipy/stats/_stats_py.py +++ b/scipy/stats/_stats_py.py @@ -1038,14 +1038,14 @@ def moment(a, order=1, axis=0, nan_policy='propagate', *, center=None): return _moment(a, order, axis, mean=center) -def _moment(a, order, axis, *, mean=None): +def _moment(a, order, axis, *, mean=None, xp=None): """Vectorized calculation of raw moment about specified center When `mean` is None, the mean is computed and used as the center; otherwise, the provided value is used as the center. """ - xp = array_namespace(a) + xp = array_namespace(a) if xp is None else xp if xp.isdtype(a.dtype, 'integral'): a = xp.asarray(a, dtype=xp.float64) @@ -1121,6 +1121,8 @@ def _var(x, axis=0, ddof=0, mean=None): @_axis_nan_policy_factory( lambda x: x, result_to_tuple=lambda x: (x,), n_outputs=1 ) +# nan_policy handled by `_axis_nan_policy, but needs to be left +# in signature to preserve use as a positional argument def skew(a, axis=0, bias=True, nan_policy='propagate'): r"""Compute the sample skewness of a data set. @@ -1195,23 +1197,24 @@ def skew(a, axis=0, bias=True, nan_policy='propagate'): 0.2650554122698573 """ - a, axis = _chk_asarray(a, axis) + xp = array_namespace(a) + a, axis = _chk_asarray(a, axis, xp=xp) n = a.shape[axis] - mean = a.mean(axis, keepdims=True) - mean_reduced = mean.squeeze(axis) # needed later + mean = xp.mean(a, axis, keepdims=True) + mean_reduced = xp.squeeze(mean, axis) # needed later m2 = _moment(a, 2, axis, mean=mean) m3 = _moment(a, 3, axis, mean=mean) with np.errstate(all='ignore'): - eps = np.finfo(m2.dtype).resolution + eps = xp.finfo(m2.dtype).eps zero = m2 <= (eps * mean_reduced)**2 - vals = np.where(zero, np.nan, m3 / m2**1.5) + vals = xp.where(zero, np.nan, m3 / m2**1.5) if not bias: can_correct = ~zero & (n > 2) - if np.any(can_correct): + if xp.any(can_correct): m2 = m2[can_correct] m3 = m3[can_correct] - nval = np.sqrt((n - 1.0) * n) / (n - 2.0) * m3 / m2**1.5 + nval = xp.sqrt((n - 1.0) * n) / (n - 2.0) * m3 / m2**1.5 vals[can_correct] = nval return vals[()] if vals.ndim == 0 else vals From 69d0c8ab9b6acbf0e7eeafc20b99f0a539b8979d Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Sat, 20 Apr 2024 12:42:36 -0700 Subject: [PATCH 22/64] TST: stats.skew: make tests array-api compatible --- scipy/stats/_stats_py.py | 8 +-- scipy/stats/tests/test_stats.py | 98 +++++++++++++++++++++++---------- 2 files changed, 74 insertions(+), 32 deletions(-) diff --git a/scipy/stats/_stats_py.py b/scipy/stats/_stats_py.py index b197a0d60781..06faf7818318 100644 --- a/scipy/stats/_stats_py.py +++ b/scipy/stats/_stats_py.py @@ -1201,20 +1201,20 @@ def skew(a, axis=0, bias=True, nan_policy='propagate'): a, axis = _chk_asarray(a, axis, xp=xp) n = a.shape[axis] - mean = xp.mean(a, axis, keepdims=True) - mean_reduced = xp.squeeze(mean, axis) # needed later + mean = xp.mean(a, axis=axis, keepdims=True) + mean_reduced = xp.squeeze(mean, axis=axis) # needed later m2 = _moment(a, 2, axis, mean=mean) m3 = _moment(a, 3, axis, mean=mean) with np.errstate(all='ignore'): eps = xp.finfo(m2.dtype).eps zero = m2 <= (eps * mean_reduced)**2 - vals = xp.where(zero, np.nan, m3 / m2**1.5) + vals = xp.where(zero, xp.asarray(xp.nan), m3 / m2**1.5) if not bias: can_correct = ~zero & (n > 2) if xp.any(can_correct): m2 = m2[can_correct] m3 = m3[can_correct] - nval = xp.sqrt((n - 1.0) * n) / (n - 2.0) * m3 / m2**1.5 + nval = ((n - 1.0) * n)**0.5 / (n - 2.0) * m3 / m2**1.5 vals[can_correct] = nval return vals[()] if vals.ndim == 0 else vals diff --git a/scipy/stats/tests/test_stats.py b/scipy/stats/tests/test_stats.py index ca0edc365510..4697526bbb7d 100644 --- a/scipy/stats/tests/test_stats.py +++ b/scipy/stats/tests/test_stats.py @@ -3381,31 +3381,37 @@ def test_moment_array_api(self, xp, order, axis, center): xp_assert_close(res, ref) -class TestSkewKurtosis: +class SkewKurtosisTest: scalar_testcase = 4. testcase = [1., 2., 3., 4.] testmathworks = [1.165, 0.6268, 0.0751, 0.3516, -0.6965] + +class TestSkew(SkewKurtosisTest): def test_empty_1d(self): + # This is not essential behavior to maintain w/ array API message = r"Mean of empty slice\.|invalid value encountered.*" with pytest.warns(RuntimeWarning, match=message): stats.skew([]) with pytest.warns(RuntimeWarning, match=message): stats.kurtosis([]) - def test_skewness(self): + @array_api_compatible + def test_skewness(self, xp): # Scalar test case - y = stats.skew(self.scalar_testcase) - assert np.isnan(y) + y = stats.skew(xp.asarray(self.scalar_testcase)) + xp_assert_close(y, xp.asarray(xp.nan)) # sum((testmathworks-mean(testmathworks,axis=0))**3,axis=0) / # ((sqrt(var(testmathworks)*4/5))**3)/5 - y = stats.skew(self.testmathworks) - assert_approx_equal(y, -0.29322304336607, 10) - y = stats.skew(self.testmathworks, bias=0) - assert_approx_equal(y, -0.437111105023940, 10) - y = stats.skew(self.testcase) - assert_approx_equal(y, 0.0, 10) + y = stats.skew(xp.asarray(self.testmathworks, dtype=xp.float64)) + xp_assert_close(y, xp.asarray(-0.29322304336607, dtype=xp.float64), atol=1e-10) + y = stats.skew(xp.asarray(self.testmathworks, dtype=xp.float64), bias=0) + xp_assert_close(y, xp.asarray(-0.437111105023940, dtype=xp.float64), atol=1e-10) + y = stats.skew(xp.asarray(self.testcase, dtype=xp.float64)) + xp_assert_close(y, xp.asarray(0.0, dtype=xp.float64), atol=1e-10) + def test_nan_policy(self): + # initially, nan_policy is ignored with alternative backends x = np.arange(10.) x[9] = np.nan with np.errstate(invalid='ignore'): @@ -3415,42 +3421,78 @@ def test_skewness(self): assert_raises(ValueError, stats.skew, x, nan_policy='foobar') def test_skewness_scalar(self): - # `skew` must return a scalar for 1-dim input + # `skew` must return a scalar for 1-dim input (only for NumPy arrays) assert_equal(stats.skew(arange(10)), 0.0) - def test_skew_propagate_nan(self): + @array_api_compatible + def test_skew_propagate_nan(self, xp): # Check that the shape of the result is the same for inputs # with and without nans, cf gh-5817 - a = np.arange(8).reshape(2, -1).astype(float) - a[1, 0] = np.nan + a = xp.arange(8.) + a = xp.reshape(a, (2, -1)) + a[1, 0] = xp.nan with np.errstate(invalid='ignore'): s = stats.skew(a, axis=1, nan_policy="propagate") - np.testing.assert_allclose(s, [0, np.nan], atol=1e-15) + xp_assert_equal(s, xp.asarray([0, xp.nan])) - def test_skew_constant_value(self): + @array_api_compatible + def test_skew_constant_value(self, xp): # Skewness of a constant input should be zero even when the mean is not # exact (gh-13245) with pytest.warns(RuntimeWarning, match="Precision loss occurred"): - a = np.repeat(-0.27829495, 10) - assert np.isnan(stats.skew(a)) - assert np.isnan(stats.skew(a * float(2**50))) - assert np.isnan(stats.skew(a / float(2**50))) - assert np.isnan(stats.skew(a, bias=False)) - - # similarly, from gh-11086: - assert np.isnan(stats.skew([14.3]*7)) - assert np.isnan(stats.skew(1 + np.arange(-3, 4)*1e-16)) + a = xp.asarray([-0.27829495]*10) # xp.repeat not currently available + assert_equal(stats.skew(a), xp.asarray(xp.nan)) + assert_equal(stats.skew(a*2.**50), xp.asarray(xp.nan)) + assert_equal(stats.skew(a/2.**50), xp.asarray(xp.nan)) + assert_equal(stats.skew(a, bias=False), xp.asarray(xp.nan)) + + # # similarly, from gh-11086: + a = xp.asarray([14.3]*7) + assert_equal(stats.skew(a), xp.asarray(xp.nan)) + a = 1. + xp.arange(-3., 4)*1e-16 + assert_equal(stats.skew(a), xp.asarray(xp.nan)) - def test_precision_loss_gh15554(self): + @array_api_compatible + def test_precision_loss_gh15554(self, xp): # gh-15554 was one of several issues that have reported problems with # constant or near-constant input. We can't always fix these, but # make sure there's a warning. with pytest.warns(RuntimeWarning, match="Precision loss occurred"): rng = np.random.default_rng(34095309370) - a = rng.random(size=(100, 10)) + a = xp.asarray(rng.random(size=(100, 10))) a[:, 0] = 1.01 - stats.skew(a)[0] + stats.skew(a) + + @array_api_compatible + @pytest.mark.parametrize('axis', [-1, 0, 2, None]) + @pytest.mark.parametrize('bias', [False, True]) + def test_vectorization(self, xp, axis, bias): + # Behavior with array input is barely tested above. Compare + # against naive implementation. + rng = np.random.default_rng(1283413549926) + x = xp.asarray(rng.random((3, 4, 5))) + + def skewness(a, axis, bias): + # Simple implementation of skewness + if axis is None: + a = xp.reshape(a, (-1,)) + axis = 0 + xp_test = array_namespace(a) # plain torch ddof=1 by default + mean = xp_test.mean(a, axis=axis, keepdims=True) + mu3 = xp_test.mean((a - mean)**3, axis=axis) + std = xp_test.std(a, axis=axis) + res = mu3 / std ** 3 + if not bias: + n = a.shape[axis] + res *= ((n - 1.0) * n) ** 0.5 / (n - 2.0) + return res + + res = stats.skew(x, axis=axis, bias=bias) + ref = skewness(x, axis=axis, bias=bias) + xp_assert_close(res, ref) + +class TestKurtosis(SkewKurtosisTest): def test_kurtosis(self): # Scalar test case y = stats.kurtosis(self.scalar_testcase) From 0b7317de0071a2db507d828903fdec0486b5ea4c Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Sat, 20 Apr 2024 13:12:50 -0700 Subject: [PATCH 23/64] MAINT: stats: raise when keepdims/nan_policy used with non-np backend --- scipy/stats/_axis_nan_policy.py | 4 ++++ scipy/stats/tests/test_stats.py | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/scipy/stats/_axis_nan_policy.py b/scipy/stats/_axis_nan_policy.py index fefbfbf98db8..7e6d8d2d66bb 100644 --- a/scipy/stats/_axis_nan_policy.py +++ b/scipy/stats/_axis_nan_policy.py @@ -403,6 +403,10 @@ def axis_nan_policy_wrapper(*args, _no_deco=False, **kwds): temp = args[0] if not is_numpy(array_namespace(temp)): + msg = ("Use of `nan_policy` and `keepdims` " + "is incompatible with non-NumPy arrays.") + if 'nan_policy' in kwds or 'keepdims' in kwds: + raise NotImplementedError(msg) return hypotest_fun_in(*args, **kwds) # We need to be flexible about whether position or keyword diff --git a/scipy/stats/tests/test_stats.py b/scipy/stats/tests/test_stats.py index 4697526bbb7d..e7d4eaeae7eb 100644 --- a/scipy/stats/tests/test_stats.py +++ b/scipy/stats/tests/test_stats.py @@ -3342,7 +3342,7 @@ def test_moment_propagate_nan(self, xp): a = np.arange(8).reshape(2, -1).astype(float) a = xp.asarray(a) a[1, 0] = np.nan - mm = stats.moment(a, 2, axis=1, nan_policy="propagate") + mm = stats.moment(a, 2, axis=1) xp_assert_close(mm, xp.asarray([1.25, np.nan], dtype=xp.float64), atol=1e-15) @array_api_compatible @@ -3432,7 +3432,7 @@ def test_skew_propagate_nan(self, xp): a = xp.reshape(a, (2, -1)) a[1, 0] = xp.nan with np.errstate(invalid='ignore'): - s = stats.skew(a, axis=1, nan_policy="propagate") + s = stats.skew(a, axis=1) xp_assert_equal(s, xp.asarray([0, xp.nan])) @array_api_compatible @@ -8920,3 +8920,20 @@ def test_chk_asarray(xp): x_out, axis_out = _chk_asarray(x[0, 0, 0], axis=axis, xp=xp) xp_assert_equal(x_out, xp.asarray(np.atleast_1d(x0[0, 0, 0]))) assert_equal(axis_out, axis) + + +@pytest.mark.skip_xp_backends('numpy', + reasons=['These parameters *are* compatible with NumPy']) +@pytest.mark.usefixtures("skip_xp_backends") +@array_api_compatible +def test_axis_nan_policy_keepdims_nanpolicy(xp): + # this test does not need to be repeated for every function + # using the _axis_nan_policy decorator. The test is here + # rather than in `test_axis_nanpolicy.py` because there is + # no reason to run those tests on an array API CI job. + x = xp.asarray([1, 2, 3, 4]) + message = "Use of `nan_policy` and `keepdims`..." + with pytest.raises(NotImplementedError, match=message): + stats.skew(x, nan_policy='omit') + with pytest.raises(NotImplementedError, match=message): + stats.skew(x, keepdims=True) From fd014c488d1fcd63797901bc4d9d5245b81dd45b Mon Sep 17 00:00:00 2001 From: thalassemia Date: Sat, 20 Apr 2024 14:24:46 -0700 Subject: [PATCH 24/64] BLD: Accelerate builds should not define `NO_APPEND_FORTRAN` [skip circle] --- scipy/_build_utils/_wrappers_common.py | 5 +++-- scipy/_build_utils/src/npy_cblas.h | 12 +++++++----- scipy/_build_utils/src/wrap_dummy_g77_abi.c | 2 +- scipy/_build_utils/src/wrap_g77_abi.c | 2 +- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/scipy/_build_utils/_wrappers_common.py b/scipy/_build_utils/_wrappers_common.py index 8b3e3afecbbb..97f2d874844d 100644 --- a/scipy/_build_utils/_wrappers_common.py +++ b/scipy/_build_utils/_wrappers_common.py @@ -38,8 +38,8 @@ USE_OLD_ACCELERATE = ['lsame', 'dcabs1'] C_PREAMBLE = """ -#include "fortran_defs.h" #include "npy_cblas.h" +#include "fortran_defs.h" """ LAPACK_DECLS = """ @@ -128,7 +128,8 @@ def get_blas_macro_and_name(name, accelerate): elif name == 'xerbla_array': return '', name + '__' if name in WRAPPED_FUNCS: - return '', name + 'wrp_' + name = name + 'wrp' + return 'F_FUNC', f'{name},{name.upper()}' return 'BLAS_FUNC', name diff --git a/scipy/_build_utils/src/npy_cblas.h b/scipy/_build_utils/src/npy_cblas.h index 0401e999a816..de65ad903284 100644 --- a/scipy/_build_utils/src/npy_cblas.h +++ b/scipy/_build_utils/src/npy_cblas.h @@ -26,17 +26,19 @@ enum CBLAS_SIDE {CblasLeft=141, CblasRight=142}; #define CBLAS_INDEX size_t /* this may vary between platforms */ -#ifdef ACCELERATE_NEW_LAPACK -#define NO_APPEND_FORTRAN -#define BLAS_SYMBOL_SUFFIX $NEWLAPACK -#endif - #ifdef NO_APPEND_FORTRAN #define BLAS_FORTRAN_SUFFIX #else #define BLAS_FORTRAN_SUFFIX _ #endif +// New Accelerate suffix is always $NEWLAPACK (no underscore) +#ifdef ACCELERATE_NEW_LAPACK +#undef BLAS_FORTRAN_SUFFIX +#define BLAS_FORTRAN_SUFFIX +#define BLAS_SYMBOL_SUFFIX $NEWLAPACK +#endif + #ifndef BLAS_SYMBOL_PREFIX #define BLAS_SYMBOL_PREFIX #endif diff --git a/scipy/_build_utils/src/wrap_dummy_g77_abi.c b/scipy/_build_utils/src/wrap_dummy_g77_abi.c index 7dcd66b605bd..ed04abb13fa5 100644 --- a/scipy/_build_utils/src/wrap_dummy_g77_abi.c +++ b/scipy/_build_utils/src/wrap_dummy_g77_abi.c @@ -22,8 +22,8 @@ passing a pointer to a variable in which to store the computed result. Unlike return values, struct complex arguments work without segfaulting. */ -#include "fortran_defs.h" #include "npy_cblas.h" +#include "fortran_defs.h" #ifdef __cplusplus extern "C" { diff --git a/scipy/_build_utils/src/wrap_g77_abi.c b/scipy/_build_utils/src/wrap_g77_abi.c index f35c94f98435..ac11f9c53c57 100644 --- a/scipy/_build_utils/src/wrap_g77_abi.c +++ b/scipy/_build_utils/src/wrap_g77_abi.c @@ -22,8 +22,8 @@ passing a pointer to a variable in which to store the computed result. Unlike return values, struct complex arguments work without segfaulting. */ -#include "fortran_defs.h" #include "npy_cblas.h" +#include "fortran_defs.h" #ifdef __cplusplus extern "C" { From 882a92244708a988c6612aab58b2d559f31158ad Mon Sep 17 00:00:00 2001 From: Tuhin Sharma Date: Sun, 21 Apr 2024 11:41:18 +0530 Subject: [PATCH 25/64] BUG:linalg:bandwidth: Fix upper band stopping criterion (#20534) --- scipy/linalg/_cythonized_array_utils.pyx | 6 ++++-- scipy/linalg/tests/test_cythonized_array_utils.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scipy/linalg/_cythonized_array_utils.pyx b/scipy/linalg/_cythonized_array_utils.pyx index 65ae7a8135a8..ad85421f5cd1 100644 --- a/scipy/linalg/_cythonized_array_utils.pyx +++ b/scipy/linalg/_cythonized_array_utils.pyx @@ -198,7 +198,8 @@ cdef inline (int, int) band_check_internal_c(const np_numeric_t[:, ::1]A) noexce if A[r, c] != zero: upper_band = c - r break - if upper_band == c: + # If existing band falls outside matrix; we are done + if r + 1 + upper_band > m - 1: break return lower_band, upper_band @@ -229,7 +230,8 @@ cdef inline (int, int) band_check_internal_noncontig(const np_numeric_t[:, :]A) if A[r, c] != zero: upper_band = c - r break - if upper_band == c: + # If existing band falls outside matrix; we are done + if r + 1 + upper_band > m - 1: break return lower_band, upper_band diff --git a/scipy/linalg/tests/test_cythonized_array_utils.py b/scipy/linalg/tests/test_cythonized_array_utils.py index 19a0b39e2827..d52c93950b63 100644 --- a/scipy/linalg/tests/test_cythonized_array_utils.py +++ b/scipy/linalg/tests/test_cythonized_array_utils.py @@ -35,6 +35,17 @@ def test_bandwidth_square_inputs(T): R[[x for x in range(1, n)], [x for x in range(n-1)]] = 1 R[[x for x in range(k, n)], [x for x in range(n-k)]] = 1 assert bandwidth(R) == (k, k) + A = np.array([ + [1, 1, 0, 0, 0, 0, 0, 0], + [1, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 1, 1, 1], + [0, 0, 0, 0, 0, 1, 0, 0], + [0, 0, 0, 0, 0, 1, 0, 0], + ]) + assert bandwidth(A) == (2, 2) @pytest.mark.parametrize('T', [x for x in np.typecodes['All'] From 27bbb9be4adf9a59ff38e5d05869e0a9eec4419e Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Sun, 21 Apr 2024 02:56:57 -0700 Subject: [PATCH 26/64] CI: standardize job names (#20482) --- .github/workflows/{linux_meson.yml => linux.yml} | 14 +++++++------- .github/workflows/{macos_meson.yml => macos.yml} | 9 +++++---- .../workflows/{linux_musl.yml => musllinux.yml} | 1 + .github/workflows/windows.yml | 7 +++---- 4 files changed, 16 insertions(+), 15 deletions(-) rename .github/workflows/{linux_meson.yml => linux.yml} (97%) rename .github/workflows/{macos_meson.yml => macos.yml} (97%) rename .github/workflows/{linux_musl.yml => musllinux.yml} (97%) diff --git a/.github/workflows/linux_meson.yml b/.github/workflows/linux.yml similarity index 97% rename from .github/workflows/linux_meson.yml rename to .github/workflows/linux.yml index 588533acb9c3..80150126c180 100644 --- a/.github/workflows/linux_meson.yml +++ b/.github/workflows/linux.yml @@ -1,4 +1,4 @@ -name: Linux Meson tests +name: Linux tests on: push: @@ -26,7 +26,7 @@ jobs: uses: ./.github/workflows/commit_message.yml test_meson: - name: Meson build + name: mypy (py3.10) & dev deps (py3.12), fast, dev.py needs: get_commit_message # If using act to run CI locally the github object does not exist and # the usual skipping should not be enforced @@ -130,7 +130,7 @@ jobs: ################################################################################# test_venv_install: - name: Pip install into venv + name: Install into venv, cluster only, pyAny/npAny, pip+cluster.test() needs: get_commit_message if: > needs.get_commit_message.outputs.message == 1 @@ -189,7 +189,7 @@ jobs: ################################################################################# python_debug: # also uses the vcs->sdist->wheel route. - name: Python-debug & ATLAS + name: Python-debug & ATLAS & sdist+wheel, fast, py3.10/npMin, pip+pytest needs: get_commit_message if: > needs.get_commit_message.outputs.message == 1 @@ -219,7 +219,7 @@ jobs: ################################################################################# gcc9: # Purpose is to examine builds with oldest-supported gcc and test with pydata/sparse. - name: Build with gcc-9 + name: Oldest GCC & pydata/sparse, fast, py3.10/npMin, pip+pytest needs: get_commit_message if: > needs.get_commit_message.outputs.message == 1 @@ -271,7 +271,7 @@ jobs: ################################################################################# prerelease_deps_coverage_64bit_blas: # TODO: re-enable ILP64 build. - name: Prerelease deps, coverage and 64-bit BLAS + name: Prerelease deps & coverage report, full, py3.10/npMin & py3.11/npPre, dev.py needs: get_commit_message if: > needs.get_commit_message.outputs.message == 1 @@ -357,7 +357,7 @@ jobs: ################################################################################# linux_32bit: - name: Linux - 32 bit + name: 32-bit, fast, py3.10/npMin, dev.py needs: get_commit_message if: > needs.get_commit_message.outputs.message == 1 diff --git a/.github/workflows/macos_meson.yml b/.github/workflows/macos.yml similarity index 97% rename from .github/workflows/macos_meson.yml rename to .github/workflows/macos.yml index c98625d72633..f52aa73f6ca3 100644 --- a/.github/workflows/macos_meson.yml +++ b/.github/workflows/macos.yml @@ -1,4 +1,4 @@ -name: macOS tests (meson) +name: macOS tests on: push: @@ -26,7 +26,7 @@ jobs: uses: ./.github/workflows/commit_message.yml test_meson: - name: Meson build + name: Conda & umfpack/scikit-sparse, fast, py3.11/npAny, dev.py needs: get_commit_message # If using act to run CI locally the github object does not exist and # the usual skipping should not be enforced @@ -137,7 +137,7 @@ jobs: test_scipy_openblas: - name: M1 test - openblas + name: M1 & OpenBLAS, fast, py3.11/npAny, dev.py needs: get_commit_message # If using act to run CI locally the github object does not exist and # the usual skipping should not be enforced @@ -181,8 +181,9 @@ jobs: pip install pooch pytest hypothesis python dev.py -n test + test_meson_accelerate: - name: Meson build (Accelerate) + name: Accelerate, fast, py3.11/npAny, dev.py needs: get_commit_message # If using act to run CI locally the github object does not exist and # the usual skipping should not be enforced diff --git a/.github/workflows/linux_musl.yml b/.github/workflows/musllinux.yml similarity index 97% rename from .github/workflows/linux_musl.yml rename to .github/workflows/musllinux.yml index 1081834ec0ee..8137d9aff623 100644 --- a/.github/workflows/linux_musl.yml +++ b/.github/workflows/musllinux.yml @@ -24,6 +24,7 @@ jobs: uses: ./.github/workflows/commit_message.yml musllinux_x86_64: + name: musl Ubuntu-latest, fast, py3.10/npAny, dev.py needs: get_commit_message runs-on: ubuntu-latest # If using act to run CI locally the github object does not exist and diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 16d0ad14367e..2d51b817f5c4 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -22,7 +22,7 @@ jobs: uses: ./.github/workflows/commit_message.yml test: - name: cp312 (meson) fast + name: fast, py3.12/npAny, dev.py needs: get_commit_message # Ensure (a) this doesn't run on forks by default, and # (b) it does run with Act locally (`github` doesn't exist there) @@ -62,7 +62,7 @@ jobs: ############################################################################# full_dev_py_min_numpy: - name: cp310 (meson) full, dev.py, minimum numpy + name: full, py3.10/npMin, dev.py needs: get_commit_message if: > needs.get_commit_message.outputs.message == 1 @@ -103,7 +103,7 @@ jobs: ############################################################################# full_build_sdist_wheel: # TODO: enable ILP64 once possible - name: cp311 (build sdist + wheel), full, no pythran + name: no pythran & sdist+wheel, full, py3.11/npPre, pip+pytest needs: get_commit_message if: > needs.get_commit_message.outputs.message == 1 @@ -157,4 +157,3 @@ jobs: cd $RUNNER_TEMP # run full test suite pytest --pyargs scipy - From 0b949514128c3790666862229d0f956009d63ed1 Mon Sep 17 00:00:00 2001 From: h-vetinari Date: Sun, 21 Apr 2024 22:20:21 +1100 Subject: [PATCH 27/64] DEP: switch sparse methods to kwarg-only; remove tol/restrt kwargs (#20498) * DEP: switch sparse methods to kwarg-only; remove tol/restart kwargs * MAINT: reflect gmres deprecation, now that too-broad warning filter is gone * TST: teach test_warnings infrastructure to handle f-strings ... as well as more complex warning specifications, e.g. `ignore:match_msg` or `ignore::DeprecationWarning`. * don't improve detection of ignored tests just yet --- scipy/_lib/tests/test_warnings.py | 16 ++- scipy/sparse/linalg/_isolve/_gcrotmk.py | 23 +--- scipy/sparse/linalg/_isolve/iterative.py | 121 +++--------------- scipy/sparse/linalg/_isolve/lgmres.py | 22 +--- scipy/sparse/linalg/_isolve/minres.py | 19 +-- .../linalg/_isolve/tests/test_iterative.py | 43 ++++--- scipy/sparse/linalg/_isolve/tfqmr.py | 22 +--- 7 files changed, 78 insertions(+), 188 deletions(-) diff --git a/scipy/_lib/tests/test_warnings.py b/scipy/_lib/tests/test_warnings.py index 158bbd5649d3..bf2e8daad4df 100644 --- a/scipy/_lib/tests/test_warnings.py +++ b/scipy/_lib/tests/test_warnings.py @@ -40,7 +40,21 @@ def visit_Call(self, node): ast.NodeVisitor.generic_visit(self, node) if p.ls[-1] == 'simplefilter' or p.ls[-1] == 'filterwarnings': - if node.args[0].value == "ignore": + # get first argument of the `args` node of the filter call + match node.args[0]: + case ast.Constant() as c: + argtext = c.value + case ast.JoinedStr() as js: + # if we get an f-string, discard the templated pieces, which + # are likely the type or specific message; we're interested + # in the action, which is less likely to use a template + argtext = "".join( + x.value for x in js.values if isinstance(x, ast.Constant) + ) + case _: + raise ValueError("unknown ast node type") + # check if filter is set to ignore + if argtext == "ignore": self.bad_filters.append( f"{self.__filename}:{node.lineno}") diff --git a/scipy/sparse/linalg/_isolve/_gcrotmk.py b/scipy/sparse/linalg/_isolve/_gcrotmk.py index 56a398508cbd..f3a35d4110bd 100644 --- a/scipy/sparse/linalg/_isolve/_gcrotmk.py +++ b/scipy/sparse/linalg/_isolve/_gcrotmk.py @@ -6,7 +6,6 @@ from scipy.linalg import (get_blas_funcs, qr, solve, svd, qr_insert, lstsq) from .iterative import _get_atol_rtol from scipy.sparse.linalg._isolve.utils import make_system -from scipy._lib.deprecation import _NoValue, _deprecate_positional_args __all__ = ['gcrotmk'] @@ -182,10 +181,8 @@ def rpsolve(x): return Q, R, B, vs, zs, y, res -@_deprecate_positional_args(version="1.14.0") -def gcrotmk(A, b, x0=None, *, tol=_NoValue, maxiter=1000, M=None, callback=None, - m=20, k=None, CU=None, discard_C=False, truncate='oldest', - atol=None, rtol=1e-5): +def gcrotmk(A, b, x0=None, *, rtol=1e-5, atol=0., maxiter=1000, M=None, callback=None, + m=20, k=None, CU=None, discard_C=False, truncate='oldest'): """ Solve a matrix equation using flexible GCROT(m,k) algorithm. @@ -203,12 +200,7 @@ def gcrotmk(A, b, x0=None, *, tol=_NoValue, maxiter=1000, M=None, callback=None, rtol, atol : float, optional Parameters for the convergence test. For convergence, ``norm(b - A @ x) <= max(rtol*norm(b), atol)`` should be satisfied. - The default is ``rtol=1e-5``, the default for ``atol`` is ``rtol``. - - .. warning:: - - The default value for ``atol`` will be changed to ``0.0`` in - SciPy 1.14.0. + The default is ``rtol=1e-5``, the default for ``atol`` is ``0.0``. maxiter : int, optional Maximum number of iterations. Iteration will stop after maxiter steps even if the specified tolerance has not been achieved. @@ -243,11 +235,6 @@ def gcrotmk(A, b, x0=None, *, tol=_NoValue, maxiter=1000, M=None, callback=None, smallest singular values using the scheme discussed in [1,2]. See [2]_ for detailed comparison. Default: 'oldest' - tol : float, optional, deprecated - - .. deprecated:: 1.12.0 - `gcrotmk` keyword argument ``tol`` is deprecated in favor of - ``rtol`` and will be removed in SciPy 1.14.0. Returns ------- @@ -313,8 +300,8 @@ def gcrotmk(A, b, x0=None, *, tol=_NoValue, maxiter=1000, M=None, callback=None, b_norm = nrm2(b) - # we call this to get the right atol/rtol and raise warnings as necessary - atol, rtol = _get_atol_rtol('gcrotmk', b_norm, tol, atol, rtol) + # we call this to get the right atol/rtol and raise errors as necessary + atol, rtol = _get_atol_rtol('gcrotmk', b_norm, atol, rtol) if b_norm == 0: x = b diff --git a/scipy/sparse/linalg/_isolve/iterative.py b/scipy/sparse/linalg/_isolve/iterative.py index 11f71914320d..0176654cfc80 100644 --- a/scipy/sparse/linalg/_isolve/iterative.py +++ b/scipy/sparse/linalg/_isolve/iterative.py @@ -3,46 +3,25 @@ from scipy.sparse.linalg._interface import LinearOperator from .utils import make_system from scipy.linalg import get_lapack_funcs -from scipy._lib.deprecation import _NoValue, _deprecate_positional_args __all__ = ['bicg', 'bicgstab', 'cg', 'cgs', 'gmres', 'qmr'] -def _get_atol_rtol(name, b_norm, tol=_NoValue, atol=0., rtol=1e-5): +def _get_atol_rtol(name, b_norm, atol=0., rtol=1e-5): """ - A helper function to handle tolerance deprecations and normalization + A helper function to handle tolerance normalization """ - if tol is not _NoValue: - msg = (f"'scipy.sparse.linalg.{name}' keyword argument `tol` is " - "deprecated in favor of `rtol` and will be removed in SciPy " - "v1.14.0. Until then, if set, it will override `rtol`.") - warnings.warn(msg, category=DeprecationWarning, stacklevel=4) - rtol = float(tol) if tol is not None else rtol - - if atol == 'legacy': - msg = (f"'scipy.sparse.linalg.{name}' called with `atol='legacy'`. " - "This behavior is deprecated and will result in an error in " - "SciPy v1.14.0. To preserve current behaviour, set `atol=0.0`.") - warnings.warn(msg, category=DeprecationWarning, stacklevel=4) - atol = 0 - - # this branch is only hit from gcrotmk/lgmres/tfqmr - if atol is None: - msg = (f"'scipy.sparse.linalg.{name}' called without specifying " - "`atol`. This behavior is deprecated and will result in an " - "error in SciPy v1.14.0. To preserve current behaviour, set " - "`atol=rtol`, or, to adopt the future default, set `atol=0.0`.") - warnings.warn(msg, category=DeprecationWarning, stacklevel=4) - atol = rtol + if atol == 'legacy' or atol is None or atol < 0: + msg = (f"'scipy.sparse.linalg.{name}' called with invalid `atol`={atol}; " + "if set, `atol` must be a real, non-negative number.") + raise ValueError(msg) atol = max(float(atol), float(rtol) * float(b_norm)) return atol, rtol -@_deprecate_positional_args(version="1.14") -def bicg(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, callback=None, - atol=0., rtol=1e-5): +def bicg(A, b, x0=None, *, rtol=1e-5, atol=0., maxiter=None, M=None, callback=None): """Use BIConjugate Gradient iteration to solve ``Ax = b``. Parameters @@ -71,11 +50,6 @@ def bicg(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, callback=None, callback : function User-supplied function to call after each iteration. It is called as callback(xk), where xk is the current solution vector. - tol : float, optional, deprecated - - .. deprecated:: 1.12.0 - `bicg` keyword argument ``tol`` is deprecated in favor of ``rtol`` - and will be removed in SciPy 1.14.0. Returns ------- @@ -104,7 +78,7 @@ def bicg(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, callback=None, A, M, x, b, postprocess = make_system(A, M, x0, b) bnrm2 = np.linalg.norm(b) - atol, _ = _get_atol_rtol('bicg', bnrm2, tol, atol, rtol) + atol, _ = _get_atol_rtol('bicg', bnrm2, atol, rtol) if bnrm2 == 0: return postprocess(b), 0 @@ -169,9 +143,8 @@ def bicg(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, callback=None, return postprocess(x), maxiter -@_deprecate_positional_args(version="1.14") -def bicgstab(A, b, *, x0=None, tol=_NoValue, maxiter=None, M=None, - callback=None, atol=0., rtol=1e-5): +def bicgstab(A, b, x0=None, *, rtol=1e-5, atol=0., maxiter=None, M=None, + callback=None): """Use BIConjugate Gradient STABilized iteration to solve ``Ax = b``. Parameters @@ -200,11 +173,6 @@ def bicgstab(A, b, *, x0=None, tol=_NoValue, maxiter=None, M=None, callback : function User-supplied function to call after each iteration. It is called as callback(xk), where xk is the current solution vector. - tol : float, optional, deprecated - - .. deprecated:: 1.12.0 - `bicgstab` keyword argument ``tol`` is deprecated in favor of - ``rtol`` and will be removed in SciPy 1.14.0. Returns ------- @@ -237,7 +205,7 @@ def bicgstab(A, b, *, x0=None, tol=_NoValue, maxiter=None, M=None, A, M, x, b, postprocess = make_system(A, M, x0, b) bnrm2 = np.linalg.norm(b) - atol, _ = _get_atol_rtol('bicgstab', bnrm2, tol, atol, rtol) + atol, _ = _get_atol_rtol('bicgstab', bnrm2, atol, rtol) if bnrm2 == 0: return postprocess(b), 0 @@ -312,9 +280,7 @@ def bicgstab(A, b, *, x0=None, tol=_NoValue, maxiter=None, M=None, return postprocess(x), maxiter -@_deprecate_positional_args(version="1.14") -def cg(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, callback=None, - atol=0., rtol=1e-5): +def cg(A, b, x0=None, *, rtol=1e-5, atol=0., maxiter=None, M=None, callback=None): """Use Conjugate Gradient iteration to solve ``Ax = b``. Parameters @@ -344,11 +310,6 @@ def cg(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, callback=None, callback : function User-supplied function to call after each iteration. It is called as callback(xk), where xk is the current solution vector. - tol : float, optional, deprecated - - .. deprecated:: 1.12.0 - `cg` keyword argument ``tol`` is deprecated in favor of ``rtol`` and - will be removed in SciPy 1.14.0. Returns ------- @@ -380,7 +341,7 @@ def cg(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, callback=None, A, M, x, b, postprocess = make_system(A, M, x0, b) bnrm2 = np.linalg.norm(b) - atol, _ = _get_atol_rtol('cg', bnrm2, tol, atol, rtol) + atol, _ = _get_atol_rtol('cg', bnrm2, atol, rtol) if bnrm2 == 0: return postprocess(b), 0 @@ -427,9 +388,7 @@ def cg(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, callback=None, return postprocess(x), maxiter -@_deprecate_positional_args(version="1.14") -def cgs(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, callback=None, - atol=0., rtol=1e-5): +def cgs(A, b, x0=None, *, rtol=1e-5, atol=0., maxiter=None, M=None, callback=None): """Use Conjugate Gradient Squared iteration to solve ``Ax = b``. Parameters @@ -458,11 +417,6 @@ def cgs(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, callback=None, callback : function User-supplied function to call after each iteration. It is called as callback(xk), where xk is the current solution vector. - tol : float, optional, deprecated - - .. deprecated:: 1.12.0 - `cgs` keyword argument ``tol`` is deprecated in favor of ``rtol`` - and will be removed in SciPy 1.14.0. Returns ------- @@ -495,7 +449,7 @@ def cgs(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, callback=None, A, M, x, b, postprocess = make_system(A, M, x0, b) bnrm2 = np.linalg.norm(b) - atol, _ = _get_atol_rtol('cgs', bnrm2, tol, atol, rtol) + atol, _ = _get_atol_rtol('cgs', bnrm2, atol, rtol) if bnrm2 == 0: return postprocess(b), 0 @@ -580,10 +534,8 @@ def cgs(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, callback=None, return postprocess(x), maxiter -@_deprecate_positional_args(version="1.14") -def gmres(A, b, x0=None, *, tol=_NoValue, restart=None, maxiter=None, M=None, - callback=None, restrt=_NoValue, atol=0., callback_type=None, - rtol=1e-5): +def gmres(A, b, x0=None, *, rtol=1e-5, atol=0., restart=None, maxiter=None, M=None, + callback=None, callback_type=None): """ Use Generalized Minimal RESidual iteration to solve ``Ax = b``. @@ -632,16 +584,6 @@ def gmres(A, b, x0=None, *, tol=_NoValue, restart=None, maxiter=None, M=None, cycles. This keyword has no effect if `callback` is not set. - restrt : int, optional, deprecated - - .. deprecated:: 0.11.0 - `gmres` keyword argument ``restrt`` is deprecated in favor of - ``restart`` and will be removed in SciPy 1.14.0. - tol : float, optional, deprecated - - .. deprecated:: 1.12.0 - `gmres` keyword argument ``tol`` is deprecated in favor of ``rtol`` - and will be removed in SciPy 1.14.0 Returns ------- @@ -681,21 +623,6 @@ def gmres(A, b, x0=None, *, tol=_NoValue, restart=None, maxiter=None, M=None, >>> np.allclose(A.dot(x), b) True """ - - # Handle the deprecation frenzy - if restrt not in (None, _NoValue) and restart: - raise ValueError("Cannot specify both 'restart' and 'restrt'" - " keywords. Also 'rstrt' is deprecated." - " and will be removed in SciPy 1.14.0. Use " - "'restart' instead.") - if restrt is not _NoValue: - msg = ("'gmres' keyword argument 'restrt' is deprecated " - "in favor of 'restart' and will be removed in SciPy" - " 1.14.0. Until then, if set, 'rstrt' will override 'restart'." - ) - warnings.warn(msg, DeprecationWarning, stacklevel=3) - restart = restrt - if callback is not None and callback_type is None: # Warn about 'callback_type' semantic changes. # Probably should be removed only in far future, Scipy 2.0 or so. @@ -723,7 +650,7 @@ def gmres(A, b, x0=None, *, tol=_NoValue, restart=None, maxiter=None, M=None, n = len(b) bnrm2 = np.linalg.norm(b) - atol, _ = _get_atol_rtol('gmres', bnrm2, tol, atol, rtol) + atol, _ = _get_atol_rtol('gmres', bnrm2, atol, rtol) if bnrm2 == 0: return postprocess(b), 0 @@ -869,9 +796,8 @@ def gmres(A, b, x0=None, *, tol=_NoValue, restart=None, maxiter=None, M=None, return postprocess(x), info -@_deprecate_positional_args(version="1.14") -def qmr(A, b, x0=None, *, tol=_NoValue, maxiter=None, M1=None, M2=None, - callback=None, atol=0., rtol=1e-5): +def qmr(A, b, x0=None, *, rtol=1e-5, atol=0., maxiter=None, M1=None, M2=None, + callback=None): """Use Quasi-Minimal Residual iteration to solve ``Ax = b``. Parameters @@ -901,11 +827,6 @@ def qmr(A, b, x0=None, *, tol=_NoValue, maxiter=None, M1=None, M2=None, callback : function User-supplied function to call after each iteration. It is called as callback(xk), where xk is the current solution vector. - tol : float, optional, deprecated - - .. deprecated:: 1.12.0 - `qmr` keyword argument ``tol`` is deprecated in favor of ``rtol`` - and will be removed in SciPy 1.14.0. Returns ------- @@ -938,7 +859,7 @@ def qmr(A, b, x0=None, *, tol=_NoValue, maxiter=None, M1=None, M2=None, A, M, x, b, postprocess = make_system(A, None, x0, b) bnrm2 = np.linalg.norm(b) - atol, _ = _get_atol_rtol('qmr', bnrm2, tol, atol, rtol) + atol, _ = _get_atol_rtol('qmr', bnrm2, atol, rtol) if bnrm2 == 0: return postprocess(b), 0 diff --git a/scipy/sparse/linalg/_isolve/lgmres.py b/scipy/sparse/linalg/_isolve/lgmres.py index 012479576def..3e105f5283a6 100644 --- a/scipy/sparse/linalg/_isolve/lgmres.py +++ b/scipy/sparse/linalg/_isolve/lgmres.py @@ -6,17 +6,15 @@ from scipy.linalg import get_blas_funcs from .iterative import _get_atol_rtol from .utils import make_system -from scipy._lib.deprecation import _NoValue, _deprecate_positional_args from ._gcrotmk import _fgmres __all__ = ['lgmres'] -@_deprecate_positional_args(version="1.14.0") -def lgmres(A, b, x0=None, *, tol=_NoValue, maxiter=1000, M=None, callback=None, +def lgmres(A, b, x0=None, *, rtol=1e-5, atol=0., maxiter=1000, M=None, callback=None, inner_m=30, outer_k=3, outer_v=None, store_outer_Av=True, - prepend_outer_v=False, atol=None, rtol=1e-5): + prepend_outer_v=False): """ Solve a matrix equation using the LGMRES algorithm. @@ -38,12 +36,7 @@ def lgmres(A, b, x0=None, *, tol=_NoValue, maxiter=1000, M=None, callback=None, rtol, atol : float, optional Parameters for the convergence test. For convergence, ``norm(b - A @ x) <= max(rtol*norm(b), atol)`` should be satisfied. - The default is ``rtol=1e-5``, the default for ``atol`` is ``rtol``. - - .. warning:: - - The default value for ``atol`` will be changed to ``0.0`` in - SciPy 1.14.0. + The default is ``rtol=1e-5``, the default for ``atol`` is ``0.0``. maxiter : int, optional Maximum number of iterations. Iteration will stop after maxiter steps even if the specified tolerance has not been achieved. @@ -77,11 +70,6 @@ def lgmres(A, b, x0=None, *, tol=_NoValue, maxiter=1000, M=None, callback=None, prepend_outer_v : bool, optional Whether to put outer_v augmentation vectors before Krylov iterates. In standard LGMRES, prepend_outer_v=False. - tol : float, optional, deprecated - - .. deprecated:: 1.12.0 - `lgmres` keyword argument ``tol`` is deprecated in favor of ``rtol`` - and will be removed in SciPy 1.14.0. Returns ------- @@ -147,8 +135,8 @@ def lgmres(A, b, x0=None, *, tol=_NoValue, maxiter=1000, M=None, callback=None, b_norm = nrm2(b) - # we call this to get the right atol/rtol and raise warnings as necessary - atol, rtol = _get_atol_rtol('lgmres', b_norm, tol, atol, rtol) + # we call this to get the right atol/rtol and raise errors as necessary + atol, rtol = _get_atol_rtol('lgmres', b_norm, atol, rtol) if b_norm == 0: x = b diff --git a/scipy/sparse/linalg/_isolve/minres.py b/scipy/sparse/linalg/_isolve/minres.py index daea59e920ff..4efb992ba921 100644 --- a/scipy/sparse/linalg/_isolve/minres.py +++ b/scipy/sparse/linalg/_isolve/minres.py @@ -1,17 +1,14 @@ -import warnings from numpy import inner, zeros, inf, finfo from numpy.linalg import norm from math import sqrt from .utils import make_system -from scipy._lib.deprecation import _NoValue, _deprecate_positional_args __all__ = ['minres'] -@_deprecate_positional_args(version="1.14.0") -def minres(A, b, x0=None, *, shift=0.0, tol=_NoValue, maxiter=None, - M=None, callback=None, show=False, check=False, rtol=1e-5): +def minres(A, b, x0=None, *, rtol=1e-5, shift=0.0, maxiter=None, + M=None, callback=None, show=False, check=False): """ Use MINimum RESidual iteration to solve Ax=b @@ -66,11 +63,6 @@ def minres(A, b, x0=None, *, shift=0.0, tol=_NoValue, maxiter=None, check : bool If ``True``, run additional input validation to check that `A` and `M` (if specified) are symmetric. Default is ``False``. - tol : float, optional, deprecated - - .. deprecated:: 1.12.0 - `minres` keyword argument ``tol`` is deprecated in favor of ``rtol`` - and will be removed in SciPy 1.14.0. Examples -------- @@ -99,13 +91,6 @@ def minres(A, b, x0=None, *, shift=0.0, tol=_NoValue, maxiter=None, """ A, M, x, b, postprocess = make_system(A, M, x0, b) - if tol is not _NoValue: - msg = ("'scipy.sparse.linalg.minres' keyword argument `tol` is " - "deprecated in favor of `rtol` and will be removed in SciPy " - "v1.14. Until then, if set, it will override `rtol`.") - warnings.warn(msg, category=DeprecationWarning, stacklevel=4) - rtol = float(tol) if tol is not None else rtol - matvec = A.matvec psolve = M.matvec diff --git a/scipy/sparse/linalg/_isolve/tests/test_iterative.py b/scipy/sparse/linalg/_isolve/tests/test_iterative.py index 01540ff10c3e..f401f2c0271a 100644 --- a/scipy/sparse/linalg/_isolve/tests/test_iterative.py +++ b/scipy/sparse/linalg/_isolve/tests/test_iterative.py @@ -26,10 +26,7 @@ _SOLVERS = [bicg, bicgstab, cg, cgs, gcrotmk, gmres, lgmres, minres, qmr, tfqmr] -pytestmark = [ - # remove this once atol defaults to 0.0 for all methods - pytest.mark.filterwarnings("ignore:.*called without specifying.*"), -] +CB_TYPE_FILTER = ".*called without specifying `callback_type`.*" # create parametrized fixture for easy reuse in tests @@ -238,7 +235,11 @@ def test_maxiter(case): def callback(x): residuals.append(norm(b - case.A * x)) - x, info = case.solver(A, b, x0=x0, rtol=rtol, maxiter=1, callback=callback) + if case.solver == gmres: + with pytest.warns(DeprecationWarning, match=CB_TYPE_FILTER): + x, info = case.solver(A, b, x0=x0, rtol=rtol, maxiter=1, callback=callback) + else: + x, info = case.solver(A, b, x0=x0, rtol=rtol, maxiter=1, callback=callback) assert len(residuals) == 1 assert info == 1 @@ -554,7 +555,7 @@ def cb(x): assert err == "" -def test_positional_deprecation(solver): +def test_positional_error(solver): # from test_x0_working rng = np.random.default_rng(1685363802304750) n = 10 @@ -562,14 +563,25 @@ def test_positional_deprecation(solver): A = A @ A.T b = rng.random(n) x0 = rng.random(n) - with pytest.deprecated_call( - # due to the use of the _deprecate_positional_args decorator, it's not possible - # to separate the two warnings (1 for positional use, 1 for `tol` deprecation). - match="use keyword arguments.*|argument `tol` is deprecated.*" - ): + with pytest.raises(TypeError): solver(A, b, x0, 1e-5) +@pytest.mark.parametrize("atol", ["legacy", None, -1]) +def test_invalid_atol(solver, atol): + if solver == minres: + pytest.skip("minres has no `atol` argument") + # from test_x0_working + rng = np.random.default_rng(1685363802304750) + n = 10 + A = rng.random(size=[n, n]) + A = A @ A.T + b = rng.random(n) + x0 = rng.random(n) + with pytest.raises(ValueError): + solver(A, b, x0, atol=atol) + + class TestQMR: @pytest.mark.filterwarnings('ignore::scipy.sparse.SparseEfficiencyWarning') def test_leftright_precond(self): @@ -621,6 +633,7 @@ def test_basic(self): assert_allclose(x_gm[0], 0.359, rtol=1e-2) + @pytest.mark.filterwarnings(f"ignore:{CB_TYPE_FILTER}:DeprecationWarning") def test_callback(self): def store_residual(r, rvec): @@ -724,6 +737,7 @@ def test_defective_matrix_breakdown(self): # The solution should be OK outside null space of A assert_allclose(A @ (A @ x), A @ b) + @pytest.mark.filterwarnings(f"ignore:{CB_TYPE_FILTER}:DeprecationWarning") def test_callback_type(self): # The legacy callback type changes meaning of 'maxiter' np.random.seed(1) @@ -787,10 +801,3 @@ def x_cb(x): restart=10, callback_type='x') assert info == 20 assert count[0] == 20 - - def test_restrt_dep(self): - with pytest.warns( - DeprecationWarning, - match="'gmres' keyword argument 'restrt'" - ): - gmres(np.array([1]), np.array([1]), restrt=10) diff --git a/scipy/sparse/linalg/_isolve/tfqmr.py b/scipy/sparse/linalg/_isolve/tfqmr.py index c8d0231c51bb..19af0400215e 100644 --- a/scipy/sparse/linalg/_isolve/tfqmr.py +++ b/scipy/sparse/linalg/_isolve/tfqmr.py @@ -1,15 +1,13 @@ import numpy as np from .iterative import _get_atol_rtol from .utils import make_system -from scipy._lib.deprecation import _NoValue, _deprecate_positional_args __all__ = ['tfqmr'] -@_deprecate_positional_args(version="1.14.0") -def tfqmr(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, - callback=None, atol=None, rtol=1e-5, show=False): +def tfqmr(A, b, x0=None, *, rtol=1e-5, atol=0., maxiter=None, M=None, + callback=None, show=False): """ Use Transpose-Free Quasi-Minimal Residual iteration to solve ``Ax = b``. @@ -27,12 +25,7 @@ def tfqmr(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, rtol, atol : float, optional Parameters for the convergence test. For convergence, ``norm(b - A @ x) <= max(rtol*norm(b), atol)`` should be satisfied. - The default is ``rtol=1e-5``, the default for ``atol`` is ``rtol``. - - .. warning:: - - The default value for ``atol`` will be changed to ``0.0`` in - SciPy 1.14.0. + The default is ``rtol=1e-5``, the default for ``atol`` is ``0.0``. maxiter : int, optional Maximum number of iterations. Iteration will stop after maxiter steps even if the specified tolerance has not been achieved. @@ -50,11 +43,6 @@ def tfqmr(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, Specify ``show = True`` to show the convergence, ``show = False`` is to close the output of the convergence. Default is `False`. - tol : float, optional, deprecated - - .. deprecated:: 1.12.0 - `tfqmr` keyword argument ``tol`` is deprecated in favor of ``rtol`` - and will be removed in SciPy 1.14.0. Returns ------- @@ -139,8 +127,8 @@ def tfqmr(A, b, x0=None, *, tol=_NoValue, maxiter=None, M=None, if r0norm == 0: return (postprocess(x), 0) - # we call this to get the right atol and raise warnings as necessary - atol, _ = _get_atol_rtol('tfqmr', r0norm, tol, atol, rtol) + # we call this to get the right atol and raise errors as necessary + atol, _ = _get_atol_rtol('tfqmr', r0norm, atol, rtol) for iter in range(maxiter): even = iter % 2 == 0 From d1fb56e221b9c8e98f8141f4e23111daf7b753c2 Mon Sep 17 00:00:00 2001 From: lucascolley Date: Sun, 21 Apr 2024 20:51:42 +0100 Subject: [PATCH 28/64] DOC: use more correct and inclusive pronouns [skip ci] --- doc/source/dev/core-dev/decisions.rst.inc | 2 +- doc/source/dev/core-dev/licensing.rst.inc | 2 +- doc/source/dev/governance.rst | 12 ++++++------ scipy/signal/_ltisys.py | 2 +- scipy/sparse/linalg/_special_sparse_arrays.py | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/doc/source/dev/core-dev/decisions.rst.inc b/doc/source/dev/core-dev/decisions.rst.inc index 2017c120c386..78fd941cdf22 100644 --- a/doc/source/dev/core-dev/decisions.rst.inc +++ b/doc/source/dev/core-dev/decisions.rst.inc @@ -18,7 +18,7 @@ Any non-trivial change (where trivial means a typo, or a one-liner maintenance commit) has to go in through a pull request (PR). It has to be reviewed by another developer. In case review doesn't happen quickly enough and it is important that the PR is merged quickly, the submitter of the PR should send a -message to mailing list saying he/she intends to merge that PR without review +message to mailing list saying they intend to merge that PR without review at time X for reason Y unless someone reviews it before then. Changes and new additions should be tested. Untested code is broken code. diff --git a/doc/source/dev/core-dev/licensing.rst.inc b/doc/source/dev/core-dev/licensing.rst.inc index 80412e8b396c..32ce2e02a66b 100644 --- a/doc/source/dev/core-dev/licensing.rst.inc +++ b/doc/source/dev/core-dev/licensing.rst.inc @@ -18,7 +18,7 @@ code from a default license that is not compatible with SciPy's license. For instance, code published on StackOverflow is covered by a CC-BY-SA license, which is not compatible due to the share-alike clause. These contributions cannot be accepted for inclusion in SciPy unless the -original code author is willing to (re)license his/her code under the +original code author is willing to (re)license their code under the modified BSD (or compatible) license. If the original author agrees, add a comment saying so to the source files and forward the relevant communication to the scipy-dev mailing list. diff --git a/doc/source/dev/governance.rst b/doc/source/dev/governance.rst index 8b08c25bd1f2..ebd93c95050f 100644 --- a/doc/source/dev/governance.rst +++ b/doc/source/dev/governance.rst @@ -74,17 +74,17 @@ Pauli Virtanen. As Dictator, the BDFL has the authority to make all final decisions for The Project. As Benevolent, the BDFL, in practice, chooses to defer that authority to the consensus of the community discussion channels and the Steering Council (see below). It is expected, and in the past has been the -case, that the BDFL will only rarely assert his/her final authority. Because +case, that the BDFL will only rarely assert their final authority. Because rarely used, we refer to BDFL’s final authority as a “special” or “overriding” vote. When it does occur, the BDFL override typically happens in situations where there is a deadlock in the Steering Council or if the Steering Council asks the BDFL to make a decision on a specific matter. To ensure the benevolence of the BDFL, The Project encourages others to fork the project if they disagree with the overall direction the BDFL is taking. The BDFL may -delegate his/her authority on a particular decision or set of decisions to -any other Council member at his/her discretion. +delegate their authority on a particular decision or set of decisions to +any other Council member at their discretion. -The BDFL can appoint his/her successor, but it is expected that the Steering +The BDFL can appoint their successor, but it is expected that the Steering Council would be consulted on this decision. If the BDFL is unable to appoint a successor, the Steering Council will make this decision - preferably by consensus, but if needed, by a majority vote. @@ -178,7 +178,7 @@ Council Chair ~~~~~~~~~~~~~ The Chair will be appointed by the Steering Council. The Chair can stay on as -long as he/she wants, but may step down at any time and will listen to +long as they want, but may step down at any time and will listen to serious calls to do so (similar to the BDFL role). The Chair will be responsible for: @@ -224,7 +224,7 @@ All members of the Council, BDFL included, shall disclose to the rest of the Council any conflict of interest they may have. Members with a conflict of interest in a particular issue may participate in Council discussions on that issue, but must recuse themselves from voting on the issue. If the BDFL has -recused his/herself for a particular decision, the Council will appoint a +recused themself for a particular decision, the Council will appoint a substitute BDFL for that decision. Private communications of the Council diff --git a/scipy/signal/_ltisys.py b/scipy/signal/_ltisys.py index c85b61ce82df..643b23ab4866 100644 --- a/scipy/signal/_ltisys.py +++ b/scipy/signal/_ltisys.py @@ -2980,7 +2980,7 @@ def place_poles(A, B, poles, method="YT", rtol=1e-3, maxiter=30): poles, B, maxiter, rtol) if not stop and rtol > 0: # if rtol<=0 the user has probably done that on purpose, - # don't annoy him + # don't annoy them err_msg = ( "Convergence was not reached after maxiter iterations.\n" f"You asked for a tolerance of {rtol}, we got {cur_rtol}." diff --git a/scipy/sparse/linalg/_special_sparse_arrays.py b/scipy/sparse/linalg/_special_sparse_arrays.py index e80ae3c28811..d6e35f6c7e47 100644 --- a/scipy/sparse/linalg/_special_sparse_arrays.py +++ b/scipy/sparse/linalg/_special_sparse_arrays.py @@ -843,7 +843,7 @@ class MikotaPair: the system length such that vibration frequencies are subsequent integers 1, 2, ..., `n` where `n` is the number of the masses. Thus, eigenvalues of the generalized eigenvalue problem for - the matrix pair `K` and `M` where `K` is he system stiffness matrix + the matrix pair `K` and `M` where `K` is the system stiffness matrix and `M` is the system mass matrix are the squares of the integers, i.e., 1, 4, 9, ..., ``n * n``. From 71a3c3c7d7c94f05781bb95513fe11225c8f408c Mon Sep 17 00:00:00 2001 From: thalassemia <67928790+thalassemia@users.noreply.github.com> Date: Wed, 17 Apr 2024 12:59:32 -0700 Subject: [PATCH 29/64] BLD: Accelerate wheels for macOS 14+ [wheel build] --- .github/workflows/macos.yml | 15 ++-- .github/workflows/wheels.yml | 121 +++++++++++++++--------------- tools/wheels/cibw_test_command.sh | 2 +- 3 files changed, 72 insertions(+), 66 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index f52aa73f6ca3..a63e88cfdf66 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -165,14 +165,14 @@ jobs: sudo xcode-select -s /Applications/Xcode_15.2.app git submodule update --init - # for some reason gfortran is not on the path - GFORTRAN_LOC=$(brew --prefix gfortran)/bin/gfortran + GFORTRAN_LOC=$(which gfortran-13) ln -s $GFORTRAN_LOC gfortran export PATH=$PWD:$PATH - # make sure we have openblas + # make sure we have openblas and gfortran dylibs bash tools/wheels/cibw_before_build_macos.sh $PWD - export DYLD_LIBRARY_PATH=/usr/local/gfortran/lib:/opt/arm64-builds/lib + GFORTRAN_LIB=$(dirname `gfortran --print-file-name libgfortran.dylib`) + export DYLD_LIBRARY_PATH=$GFORTRAN_LIB:/opt/arm64-builds/lib export PKG_CONFIG_PATH=/opt/arm64-builds/lib/pkgconfig pip install click doit pydevtool rich_click meson cython pythran pybind11 ninja numpy @@ -211,11 +211,14 @@ jobs: sudo xcode-select -s /Applications/Xcode_15.2.app git submodule update --init - # for some reason gfortran is not on the path - GFORTRAN_LOC=$(brew --prefix gfortran)/bin/gfortran + GFORTRAN_LOC=$(which gfortran-13) ln -s $GFORTRAN_LOC gfortran export PATH=$PWD:$PATH + # Ensure we have gfortran dylib + GFORTRAN_LIB=$(dirname `gfortran --print-file-name libgfortran.dylib`) + export DYLD_LIBRARY_PATH=$GFORTRAN_LIB + pip install click doit pydevtool rich_click meson cython pythran pybind11 ninja numpy python dev.py build -C-Dblas=accelerate diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 3ed6e64e6759..331f4e08f099 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -59,7 +59,9 @@ jobs: echo github.ref ${{ github.ref }} build_wheels: - name: Build wheel for ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} ${{ matrix.buildplat[2] }} + name: Wheel, ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} + ${{ matrix.buildplat[2] }} ${{ matrix.buildplat[3] }} + ${{ matrix.buildplat[4] }} needs: get_commit_message if: >- contains(needs.get_commit_message.outputs.message, '1') || @@ -77,11 +79,13 @@ jobs: # should also be able to do multi-archs on a single entry, e.g. # [windows-2019, win*, "AMD64 x86"]. However, those two require a different compiler setup # so easier to separate out here. - - [ubuntu-22.04, manylinux, x86_64] - - [ubuntu-22.04, musllinux, x86_64] - - [macos-11, macosx, x86_64] - - [macos-14, macosx, arm64] - - [windows-2019, win, AMD64] + - [ubuntu-22.04, manylinux, x86_64, "", ""] + - [ubuntu-22.04, musllinux, x86_64, "", ""] + - [macos-12, macosx, x86_64, openblas, "10.9"] + - [macos-13, macosx, x86_64, accelerate, "14.0"] + - [macos-14, macosx, arm64, openblas, "12.0"] + - [macos-14, macosx, arm64, accelerate, "14.0"] + - [windows-2019, win, AMD64, "", ""] python: [["cp310", "3.10"], ["cp311", "3.11"], ["cp312", "3.12"]] # python[0] is used to specify the python versions made by cibuildwheel @@ -112,63 +116,53 @@ jobs: if: ${{ runner.os == 'Windows' && env.IS_32_BIT == 'false' }} - name: Setup macOS - if: matrix.buildplat[0] == 'macos-11' || matrix.buildplat[0] == 'macos-14' + if: startsWith( matrix.buildplat[0], 'macos-' ) run: | - if [[ ${{ matrix.buildplat[2] }} == 'arm64' ]]; then - # macosx_arm64 - - # use homebrew gfortran - sudo xcode-select -s /Applications/Xcode_15.2.app - # for some reason gfortran is not on the path - GFORTRAN_LOC=$(brew --prefix gfortran)/bin/gfortran - ln -s $GFORTRAN_LOC gfortran + if [[ ${{ matrix.buildplat[3] }} == 'accelerate' ]]; then + echo CIBW_CONFIG_SETTINGS=\"setup-args=-Dblas=accelerate\" >> "$GITHUB_ENV" + # Always use preinstalled gfortran for Accelerate builds + ln -s $(which gfortran-13) gfortran export PATH=$PWD:$PATH echo "PATH=$PATH" >> "$GITHUB_ENV" - - # location of the gfortran's libraries - GFORTRAN_LIB=$(dirname `gfortran --print-file-name libgfortran.dylib`) - - CIBW="MACOSX_DEPLOYMENT_TARGET=12.0\ - MACOS_DEPLOYMENT_TARGET=12.0\ - LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH\ - _PYTHON_HOST_PLATFORM=macosx-12.0-arm64\ - PIP_PRE=1\ - PIP_NO_BUILD_ISOLATION=false\ - PKG_CONFIG_PATH=/opt/arm64-builds/lib/pkgconfig\ - PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" - echo "CIBW_ENVIRONMENT_MACOS=$CIBW" >> "$GITHUB_ENV" - - CIBW="sudo xcode-select -s /Applications/Xcode_15.2.app" - echo "CIBW_BEFORE_ALL=$CIBW" >> $GITHUB_ENV - - echo "REPAIR_PATH=/opt/arm64-builds/lib" >> "$GITHUB_ENV" - - CIBW="DYLD_LIBRARY_PATH=$GFORTRAN_LIB:/opt/arm64-builds/lib delocate-listdeps {wheel} &&\ - DYLD_LIBRARY_PATH=$GFORTRAN_LIB:/opt/arm64-builds/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" - echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=$CIBW" >> "$GITHUB_ENV" - + LIB_PATH=$(dirname $(gfortran --print-file-name libgfortran.dylib)) + fi + # Add libraries installed by cibw_before_build_macos.sh to path + if [[ ${{ matrix.buildplat[2] }} == 'arm64' ]]; then + LIB_PATH=$LIB_PATH:/opt/arm64-builds/lib + else + LIB_PATH=$LIB_PATH:/usr/local/lib + fi + if [[ ${{ matrix.buildplat[4] }} == '10.9' ]]; then + # Newest version of Xcode that supports macOS 10.9 + XCODE_VER='13.4.1' else - # macosx_x86_64 with OpenBLAS - # setting SDKROOT necessary when using the gfortran compiler - # installed in cibw_before_build_macos.sh - # MACOS_DEPLOYMENT_TARGET is set because of - # https://github.com/mesonbuild/meson-python/pull/309. Once - # an update is released, then that environment variable can - # be removed. - CIBW="MACOSX_DEPLOYMENT_TARGET=10.9\ - MACOS_DEPLOYMENT_TARGET=10.9\ - SDKROOT=/Applications/Xcode_11.7.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.15.sdk\ - LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH\ - _PYTHON_HOST_PLATFORM=macosx-10.9-x86_64\ - PIP_PRE=1\ - PIP_NO_BUILD_ISOLATION=false\ - PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" - echo "CIBW_ENVIRONMENT_MACOS=$CIBW" >> "$GITHUB_ENV" - - CIBW="DYLD_LIBRARY_PATH=/usr/local/lib delocate-listdeps {wheel} &&\ - DYLD_LIBRARY_PATH=/usr/local/lib delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}" - echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=$CIBW" >> "$GITHUB_ENV" + XCODE_VER='15.2' + fi + CIBW="sudo xcode-select -s /Applications/Xcode_${XCODE_VER}.app" + echo "CIBW_BEFORE_ALL=$CIBW" >> $GITHUB_ENV + # setting SDKROOT necessary when using the gfortran compiler + # installed in cibw_before_build_macos.sh + sudo xcode-select -s /Applications/Xcode_${XCODE_VER}.app + CIBW="MACOSX_DEPLOYMENT_TARGET=${{ matrix.buildplat[4] }}\ + LD_LIBRARY_PATH=$LIB_PATH:$LD_LIBRARY_PATH\ + SDKROOT=$(xcrun --sdk macosx --show-sdk-path)\ + PIP_PRE=1\ + PIP_NO_BUILD_ISOLATION=false\ + PKG_CONFIG_PATH=$LIB_PATH/pkgconfig\ + PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" + echo "CIBW_ENVIRONMENT_MACOS=$CIBW" >> "$GITHUB_ENV" + + echo "REPAIR_PATH=$LIB_PATH" >> "$GITHUB_ENV" + GFORTRAN_LIB="\$(dirname \$(gfortran --print-file-name libgfortran.dylib))" + CIBW="DYLD_LIBRARY_PATH=$GFORTRAN_LIB:$LIB_PATH delocate-listdeps {wheel} &&\ + DYLD_LIBRARY_PATH=$GFORTRAN_LIB:$LIB_PATH delocate-wheel --require-archs \ + {delocate_archs} -w {dest_dir} {wheel}" + # Rename x86 Accelerate wheel to test on macOS 13 runner + if [[ ${{ matrix.buildplat[0] }} == 'macos-13' && ${{ matrix.buildplat[4] }} == '14.0' ]]; then + CIBW+=" && mv {dest_dir}/\$(basename {wheel}) \ + {dest_dir}/\$(echo \$(basename {wheel}) | sed 's/14_0/13_0/')" fi + echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=$CIBW" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.17.0 @@ -195,10 +189,18 @@ jobs: PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple PIP_NO_BUILD_ISOLATION=false + - name: Rename after test (macOS x86 Accelerate only) + # Rename x86 Accelerate wheel back so it targets macOS >= 14 + if: matrix.buildplat[0] == 'macos-13' && matrix.buildplat[4] == '14.0' + run: | + mv ./wheelhouse/*.whl $(find ./wheelhouse -type f -name '*.whl' | sed 's/13_0/14_0/') + - uses: actions/upload-artifact@v4 with: path: ./wheelhouse/*.whl - name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}-${{ matrix.buildplat[2] }} + name: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }} + ${{ matrix.buildplat[2] }} ${{ matrix.buildplat[3] }} + ${{ matrix.buildplat[4] }} - uses: conda-incubator/setup-miniconda@v3 with: @@ -209,6 +211,7 @@ jobs: # build and test the wheel auto-update-conda: true python-version: "3.10" + miniconda-version: "latest" - name: Upload wheels if: success() diff --git a/tools/wheels/cibw_test_command.sh b/tools/wheels/cibw_test_command.sh index 519b13cdef53..2eb214b3582b 100644 --- a/tools/wheels/cibw_test_command.sh +++ b/tools/wheels/cibw_test_command.sh @@ -3,7 +3,7 @@ set -xe PROJECT_DIR="$1" # python $PROJECT_DIR/tools/wheels/check_license.py -if [[ $(uname) == "Linux" || $(uname) == "Darwin" ]] ; then +if [[ $(uname) == "Linux" ]] ; then python $PROJECT_DIR/tools/openblas_support.py --check_version fi echo $? From 11b509c8979ddedde8f9952eb1a60d130f2889ac Mon Sep 17 00:00:00 2001 From: thalassemia <67928790+thalassemia@users.noreply.github.com> Date: Fri, 19 Apr 2024 10:00:27 -0700 Subject: [PATCH 30/64] DOC: Update wheel build toolchain info [wheel build] --- doc/source/dev/toolchain.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/doc/source/dev/toolchain.rst b/doc/source/dev/toolchain.rst index 449178232439..a7e15e53dcce 100644 --- a/doc/source/dev/toolchain.rst +++ b/doc/source/dev/toolchain.rst @@ -138,15 +138,17 @@ Official Builds Currently, SciPy wheels are being built as follows: -================ ============================== ============================== ============================= - Platform `CI`_ `Base`_ `Images`_ Compilers Comment -================ ============================== ============================== ============================= -Linux x86 ``ubuntu-22.04`` GCC 10.2.1 ``cibuildwheel`` -Linux arm ``docker-builder-arm64`` GCC 11.3.0 ``cibuildwheel`` -OSX x86_64 ``macos-11`` clang-13/gfortran 11.3 ``cibuildwheel`` -OSX arm64 ``macos-14`` clang-14/gfortran 13.0 ``cibuildwheel`` -Windows ``windows-2019`` GCC 10.3 (`rtools`_) ``cibuildwheel`` -================ ============================== ============================== ============================= +========================= ============================== ==================================== ============================= + Platform `CI`_ `Base`_ `Images`_ Compilers Comment +========================= ============================== ==================================== ============================= + Linux x86 ``ubuntu-22.04`` GCC 10.2.1 ``cibuildwheel`` + Linux arm ``docker-builder-arm64`` GCC 11.3.0 ``cibuildwheel`` + OSX x86_64 (OpenBLAS) ``macos-12`` Apple clang 13.1.6/gfortran 11.3.0 ``cibuildwheel`` + OSX x86_64 (Accelerate) ``macos-13`` Apple clang 15.0.0/gfortran 13.2.0 ``cibuildwheel`` + OSX arm64 (OpenBLAS) ``macos-14`` Apple clang 15.0.0/gfortran 12.1.0 ``cibuildwheel`` + OSX arm64 (Accelerate) ``macos-14`` Apple clang 15.0.0/gfortran 13.2.0 ``cibuildwheel`` + Windows ``windows-2019`` GCC 10.3.0 (`rtools`_) ``cibuildwheel`` +========================= ============================== ==================================== ============================= .. _CI: https://github.com/actions/runner-images .. _Base: https://cirrus-ci.org/guide/docker-builder-vm/#under-the-hood From c0edfca9ebcdb4a975ec4fbaa3d1221c785a8519 Mon Sep 17 00:00:00 2001 From: Jake Bowhay Date: Mon, 22 Apr 2024 10:56:44 +0100 Subject: [PATCH 31/64] DEP: stats: switch kendalltau to kwarg-only, remove initial_lexsort kwarg --- scipy/stats/_stats_py.py | 15 +-------------- scipy/stats/tests/test_stats.py | 8 -------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/scipy/stats/_stats_py.py b/scipy/stats/_stats_py.py index 06faf7818318..3f9691024333 100644 --- a/scipy/stats/_stats_py.py +++ b/scipy/stats/_stats_py.py @@ -66,7 +66,6 @@ from scipy._lib._bunch import _make_tuple_bunch from scipy import stats from scipy.optimize import root_scalar -from scipy._lib.deprecation import _NoValue, _deprecate_positional_args from scipy._lib._util import normalize_axis_index from scipy._lib._array_api import array_namespace, is_numpy from scipy._lib.array_api_compat import size as xp_size @@ -5662,8 +5661,7 @@ def pointbiserialr(x, y): return res -@_deprecate_positional_args(version="1.14") -def kendalltau(x, y, *, initial_lexsort=_NoValue, nan_policy='propagate', +def kendalltau(x, y, *, nan_policy='propagate', method='auto', variant='b', alternative='two-sided'): r"""Calculate Kendall's tau, a correlation measure for ordinal data. @@ -5681,12 +5679,6 @@ def kendalltau(x, y, *, initial_lexsort=_NoValue, nan_policy='propagate', x, y : array_like Arrays of rankings, of the same shape. If arrays are not 1-D, they will be flattened to 1-D. - initial_lexsort : bool, optional, deprecated - This argument is unused. - - .. deprecated:: 1.10.0 - `kendalltau` keyword argument `initial_lexsort` is deprecated as it - is unused and will be removed in SciPy 1.14.0. nan_policy : {'propagate', 'raise', 'omit'}, optional Defines how to handle when input contains nan. The following options are available (default is 'propagate'): @@ -5901,11 +5893,6 @@ def kendalltau(x, y, *, initial_lexsort=_NoValue, nan_policy='propagate', accurate results. """ - if initial_lexsort is not _NoValue: - msg = ("'kendalltau' keyword argument 'initial_lexsort' is deprecated" - " as it is unused and will be removed in SciPy 1.12.0.") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - x = np.asarray(x).ravel() y = np.asarray(y).ravel() diff --git a/scipy/stats/tests/test_stats.py b/scipy/stats/tests/test_stats.py index e7d4eaeae7eb..9f0ca52a1b6d 100644 --- a/scipy/stats/tests/test_stats.py +++ b/scipy/stats/tests/test_stats.py @@ -1559,14 +1559,6 @@ def test_kendalltau_nan_2nd_arg(): assert_allclose(r1.statistic, r2.statistic, atol=1e-15) -def test_kendalltau_deprecations(): - msg_dep = "keyword argument 'initial_lexsort'" - with pytest.deprecated_call(match=msg_dep): - stats.kendalltau([], [], initial_lexsort=True) - with pytest.deprecated_call(match=f"use keyword arguments|{msg_dep}"): - stats.kendalltau([], [], True) - - def test_kendalltau_gh18139_overflow(): # gh-18139 reported an overflow in `kendalltau` that appeared after # SciPy 0.15.1. Check that this particular overflow does not occur. From ecc5dd80c14e2d2b92b8f7b79308363cc205b14b Mon Sep 17 00:00:00 2001 From: lucascolley Date: Wed, 6 Mar 2024 13:02:09 +0000 Subject: [PATCH 32/64] DOC/MAINT: update core-dev guide [docs only] --- doc/source/dev/contributor/reviewing_prs.rst | 5 +- doc/source/dev/core-dev/deprecations.rst.inc | 5 +- doc/source/dev/core-dev/distributing.rst.inc | 53 +++----------------- doc/source/dev/core-dev/github.rst.inc | 31 +++++------- doc/source/dev/core-dev/licensing.rst.inc | 2 +- doc/source/dev/core-dev/versioning.rst.inc | 2 +- 6 files changed, 26 insertions(+), 72 deletions(-) diff --git a/doc/source/dev/contributor/reviewing_prs.rst b/doc/source/dev/contributor/reviewing_prs.rst index b52f5551d7d0..535a239d998c 100644 --- a/doc/source/dev/contributor/reviewing_prs.rst +++ b/doc/source/dev/contributor/reviewing_prs.rst @@ -45,9 +45,6 @@ Github as appropriate: This allows automatically tracking which PRs are in need of attention. -The review status is listed at: https://pav.iki.fi/scipy-needs-work/ -The page can also be generated using https://github.com/pv/github-needs-work - Some of the information is also visible on Github directly, although (as of Aug 2019) Github does not show which pull requests have been updated since the last review. @@ -98,7 +95,7 @@ and create your own branch based on one of them:: where ``BRANCH_NAME`` is the name of the branch you want to start from. This creates a copy of this branch (with the same name) in your local repository. -If make changes to this branch and push to your GitHub repository +If you make changes to this branch and push to your GitHub repository (``origin``), you can then create a pull request to merge your changes with the author's repository. diff --git a/doc/source/dev/core-dev/deprecations.rst.inc b/doc/source/dev/core-dev/deprecations.rst.inc index 8be3b89f66b7..07dbe6b503d7 100644 --- a/doc/source/dev/core-dev/deprecations.rst.inc +++ b/doc/source/dev/core-dev/deprecations.rst.inc @@ -7,8 +7,8 @@ There are various reasons for wanting to remove existing functionality: it's buggy, the API isn't understandable, it's superseded by functionality with better performance, it needs to be moved to another SciPy submodule, etc. -In general it's not a good idea to remove something without warning users about -that removal first. Therefore this is what should be done before removing +In general, it's not a good idea to remove something without warning users about +that removal first. Therefore, this is what should be done before removing something from the public API: #. Propose to deprecate the functionality on the scipy-dev mailing list and get @@ -28,4 +28,3 @@ when running the test suite so they don't pollute the output. It's possible that there is reason to want to ignore this deprecation policy for a particular deprecation; this can always be discussed on the scipy-dev mailing list. - diff --git a/doc/source/dev/core-dev/distributing.rst.inc b/doc/source/dev/core-dev/distributing.rst.inc index bddba0c9da78..ba717d3210db 100644 --- a/doc/source/dev/core-dev/distributing.rst.inc +++ b/doc/source/dev/core-dev/distributing.rst.inc @@ -13,57 +13,19 @@ Dependencies ------------ Dependencies are things that a user has to install in order to use (or build/test) a package. They usually cause trouble, especially if they're not -optional. SciPy tries to keep its dependencies to a minimum; currently they -are: +optional. SciPy tries to keep its dependencies to a minimum; the current required +and optional build time dependencies can be seen in `SciPy's configuration file`_, +``pyproject.toml``. The only non-optional runtime dependency is NumPy_. -*Unconditional run-time dependencies:* - -- Numpy_ - -*Conditional run-time dependencies:* - -- pytest (to run the test suite) -- asv_ (to run the benchmarks) -- matplotlib_ (for some functions that can produce plots) -- pooch_ (for the scipy.datasets module) -- Pillow_ (for image loading/saving) -- scikits.umfpack_ (optionally used in ``sparse.linalg``) -- mpmath_ (for more extended tests in ``special``) -- pydata/sparse (compatibility support in ``scipy.sparse``) -- threadpoolctl_ (to control BLAS/LAPACK threading in test suite) -- Hypothesis_ (to run certain unit tests) - -*Unconditional build-time dependencies:* - -- Numpy_ -- A BLAS and LAPACK implementation (reference BLAS/LAPACK, ATLAS, OpenBLAS, - MKL are all known to work) -- Cython_ -- setuptools_ -- pybind11_ - -*Conditional build-time dependencies:* - -- wheel_ (``python setup.py bdist_wheel``) -- Sphinx_ (docs) -- `PyData Sphinx theme`_ (docs) -- `Sphinx-Design`_ (docs) -- matplotlib_ (docs) -- MyST-NB_ (docs) - -Furthermore of course one needs C, C++ and Fortran compilers to build SciPy, -but those we don't consider to be dependencies and are therefore not discussed -here. For details, see https://scipy.github.io/devdocs/dev/contributor/building.html. +Furthermore, of course one needs C, C++ and Fortran compilers to build SciPy, +but we don't consider those to be dependencies, and therefore they are not discussed +here. For details, see :ref:`building-from-source`. When a package provides useful functionality and it's proposed as a new dependency, consider also if it makes sense to vendor (i.e. ship a copy of it with -scipy) the package instead. For example, decorator_ is vendored in +SciPy) the package instead. For example, decorator_ is vendored in ``scipy._lib``. -The only dependency that is reported to pip_ is Numpy_, see -``install_requires`` in SciPy's main ``setup.py``. The other dependencies -aren't needed for SciPy to function correctly - Issues with dependency handling ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ There are some issues with how Python packaging tools handle @@ -189,6 +151,7 @@ would need to be distributed via custom channels, e.g. in a Wheelhouse_, see at the wheel_ and Wheelhouse_ docs. +.. _`SciPy's configuration file`: https://github.com/scipy/scipy/blob/main/pyproject.toml .. _Numpy: https://numpy.org .. _Python: https://www.python.org .. _nose: https://nose.readthedocs.io diff --git a/doc/source/dev/core-dev/github.rst.inc b/doc/source/dev/core-dev/github.rst.inc index a47773dc419a..9e4f7a81bdf8 100644 --- a/doc/source/dev/core-dev/github.rst.inc +++ b/doc/source/dev/core-dev/github.rst.inc @@ -13,7 +13,7 @@ topic or component (``scipy.stats``, ``Documentation``, etc.), and one for the nature of the issue or pull request (``enhancement``, ``maintenance``, ``defect``, etc.). Other labels that may be added depending on the situation: -- ``easy-fix``: for issues suitable to be tackled by new contributors. +- ``good-first-issue``: for issues suitable to be tackled by new contributors. - ``needs-work``: for pull requests that have review comments that haven't been addressed. - ``needs-decision``: for issues or pull requests that need a decision. @@ -41,14 +41,12 @@ Dealing with pull requests -------------------------- - When merging contributions, a committer is responsible for ensuring that - those meet the requirements outlined in `Contributing to SciPy - `_. - Also check - that new features and backwards compatibility breaks were discussed on the - scipy-dev mailing list. + those meet the requirements outlined in :ref:`Contributing to SciPy `. + Also check that new features and backwards compatibility breaks were discussed + on the scipy-dev mailing list. - New code goes in via a pull request (PR). - Merge new code with the green button. In case of merge conflicts, ask the PR - submitter to rebase (this may require providing some git instructions). + submitter to rebase (this may require providing some ``git`` instructions). - Backports and trivial additions to finish a PR (really trivial, like a typo or PEP8 fix) can be pushed directly. - For PRs that add new features or are in some way complex, wait at least a day @@ -56,11 +54,12 @@ Dealing with pull requests the code goes in. - Squashing commits or cleaning up commit messages of a PR that you consider too messy is OK. Make sure though to retain the original author name when - doing this. + doing this. Squashing is highly recommended whenever commit messages do not + (roughly) follow the guidelines in :ref:`writing-the-commit-message`. - Make sure that the labels and milestone on a merged PR are set correctly. -- When you want to reject a PR: if it's very obvious you can just close it and - explain why, if not obvious then it's a good idea to first explain why you - think the PR is not suitable for inclusion in SciPy and then let a second +- When you want to reject a PR: if it's very obvious, you can just close it and + explain why. If it's not obvious, then it's a good idea to first explain why you + think the PR is not suitable for inclusion in SciPy, then let a second committer comment or close. @@ -92,9 +91,9 @@ release notes. What needs mentioning: new features, backwards incompatible changes, deprecations, and "other changes" (anything else noteworthy enough, see older release notes for the kinds of things worth mentioning). -Release note entries are maintained on the wiki, (e.g. -https://github.com/scipy/scipy/wiki/Release-note-entries-for-SciPy-1.2.0). The -release manager will gather content from there and integrate it into the html +Release note entries are maintained on +`the wiki `_. +The release manager will gather content from there and integrate it into the html docs. We use this mechanism to avoid merge conflicts that would happen if every PR touched the same file under ``doc/release/`` directly. @@ -104,10 +103,6 @@ and pulled (the wiki is a git repo: ``https://github.com/scipy/scipy.wiki.git``) Other ----- -*PR status page:* When new commits get added to a pull request, GitHub doesn't send out any -notifications. The ``needs-work`` label may not be justified anymore though. -`This page `_ gives an overview of PRs -that were updated, need review, need a decision, etc. *Cross-referencing:* Cross-referencing issues and pull requests on GitHub is often useful. GitHub allows doing that by using ``gh-xxxx`` or ``#xxxx`` with diff --git a/doc/source/dev/core-dev/licensing.rst.inc b/doc/source/dev/core-dev/licensing.rst.inc index 32ce2e02a66b..5d251cc74eda 100644 --- a/doc/source/dev/core-dev/licensing.rst.inc +++ b/doc/source/dev/core-dev/licensing.rst.inc @@ -29,4 +29,4 @@ cannot be included in SciPy. Simply implementing functionality with the same API as found in R/Octave/... is fine though, as long as the author doesn't look at the original incompatibly-licensed source code. -.. _modified (3-clause) BSD license: https://opensource.org/licenses/BSD-3-Clause \ No newline at end of file +.. _modified (3-clause) BSD license: https://opensource.org/licenses/BSD-3-Clause diff --git a/doc/source/dev/core-dev/versioning.rst.inc b/doc/source/dev/core-dev/versioning.rst.inc index 2d2634310019..234264ce83f5 100644 --- a/doc/source/dev/core-dev/versioning.rst.inc +++ b/doc/source/dev/core-dev/versioning.rst.inc @@ -2,7 +2,7 @@ Version numbering ================= -SciPy version numbering complies to `PEP 440`_. Released final versions, which +SciPy version numbering complies with `PEP 440`_. Released final versions, which are the only versions appearing on `PyPI`_, are numbered ``MAJOR.MINOR.MICRO`` where: From 3aba7ab3667ab6c9ba71e09ac9b3373555641f1c Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Mon, 22 Apr 2024 08:49:28 -0700 Subject: [PATCH 33/64] ENH: stats.ttest_1samp: add array-API support (#20545) --- scipy/stats/_stats_py.py | 61 +++++-- scipy/stats/tests/test_stats.py | 280 ++++++++++++++++++++------------ 2 files changed, 216 insertions(+), 125 deletions(-) diff --git a/scipy/stats/_stats_py.py b/scipy/stats/_stats_py.py index 3f9691024333..cc504a75fb1a 100644 --- a/scipy/stats/_stats_py.py +++ b/scipy/stats/_stats_py.py @@ -1108,9 +1108,10 @@ def _moment(a, order, axis, *, mean=None, xp=None): return xp.mean(s, axis=axis) -def _var(x, axis=0, ddof=0, mean=None): +def _var(x, axis=0, ddof=0, mean=None, xp=None): # Calculate variance of sample, warning if precision is lost - var = _moment(x, 2, axis, mean=mean) + xp = array_namespace(x) if xp is None else xp + var = _moment(x, 2, axis, mean=mean, xp=xp) if ddof != 0: n = x.shape[axis] if axis is not None else x.size var *= np.divide(n, n-ddof) # to avoid error on division by zero @@ -1120,7 +1121,7 @@ def _var(x, axis=0, ddof=0, mean=None): @_axis_nan_policy_factory( lambda x: x, result_to_tuple=lambda x: (x,), n_outputs=1 ) -# nan_policy handled by `_axis_nan_policy, but needs to be left +# nan_policy handled by `_axis_nan_policy`, but needs to be left # in signature to preserve use as a positional argument def skew(a, axis=0, bias=True, nan_policy='propagate'): r"""Compute the sample skewness of a data set. @@ -6773,11 +6774,16 @@ class TtestResult(TtestResultBase): """ def __init__(self, statistic, pvalue, df, # public - alternative, standard_error, estimate): # private + alternative, standard_error, estimate, # private + statistic_np=None, xp=None): # private super().__init__(statistic, pvalue, df=df) self._alternative = alternative self._standard_error = standard_error # denominator of t-statistic self._estimate = estimate # point estimate of sample mean + self._statistic_np = statistic if statistic_np is None else statistic_np + self._dtype = statistic.dtype + self._xp = array_namespace(statistic, pvalue) if xp is None else xp + def confidence_interval(self, confidence_level=0.95): """ @@ -6794,8 +6800,9 @@ def confidence_interval(self, confidence_level=0.95): fields `low` and `high`. """ - low, high = _t_confidence_interval(self.df, self.statistic, - confidence_level, self._alternative) + low, high = _t_confidence_interval(self.df, self._statistic_np, + confidence_level, self._alternative, + self._dtype, self._xp) low = low * self._standard_error + self._estimate high = high * self._standard_error + self._estimate return ConfidenceInterval(low=low, high=high) @@ -6819,6 +6826,8 @@ def unpack_TtestResult(res): @_axis_nan_policy_factory(pack_TtestResult, default_axis=0, n_samples=2, result_to_tuple=unpack_TtestResult, n_outputs=6) +# nan_policy handled by `_axis_nan_policy`, but needs to be left +# in signature to preserve use as a positional argument def ttest_1samp(a, popmean, axis=0, nan_policy='propagate', alternative="two-sided"): """Calculate the T-test for the mean of ONE group of scores. @@ -6974,35 +6983,49 @@ def ttest_1samp(a, popmean, axis=0, nan_policy='propagate', 953 """ - a, axis = _chk_asarray(a, axis) + xp = array_namespace(a) + a, axis = _chk_asarray(a, axis, xp=xp) n = a.shape[axis] df = n - 1 - mean = np.mean(a, axis) + mean = xp.mean(a, axis=axis) try: - popmean = np.squeeze(popmean, axis=axis) + popmean = xp.asarray(popmean) + popmean = xp.squeeze(popmean, axis=axis) if popmean.ndim > 0 else popmean except ValueError as e: raise ValueError("`popmean.shape[axis]` must equal 1.") from e d = mean - popmean - v = _var(a, axis, ddof=1) - denom = np.sqrt(v / n) + v = _var(a, axis=axis, ddof=1) + denom = xp.sqrt(v / n) with np.errstate(divide='ignore', invalid='ignore'): - t = np.divide(d, denom)[()] - prob = _get_pvalue(t, distributions.t(df), alternative) + t = xp.divide(d, denom) + t = t[()] if t.ndim == 0 else t + # This will only work for CPU backends for now. That's OK. In time, + # `from_dlpack` will enable the transfer from other devices, and + # `_get_pvalue` will even be reworked to support the native backend. + t_np = np.asarray(t) + prob = _get_pvalue(t_np, distributions.t(df), alternative) + prob = xp.asarray(prob, dtype=t.dtype) + prob = prob[()] if prob.ndim == 0 else prob # when nan_policy='omit', `df` can be different for different axis-slices - df = np.broadcast_to(df, t.shape)[()] + df = xp.broadcast_to(xp.asarray(df), t.shape) + df = df[()] if df.ndim == 0 else df # _axis_nan_policy decorator doesn't play well with strings alternative_num = {"less": -1, "two-sided": 0, "greater": 1}[alternative] return TtestResult(t, prob, df=df, alternative=alternative_num, - standard_error=denom, estimate=mean) + standard_error=denom, estimate=mean, + statistic_np=t_np, xp=xp) -def _t_confidence_interval(df, t, confidence_level, alternative): +def _t_confidence_interval(df, t, confidence_level, alternative, dtype=None, xp=None): # Input validation on `alternative` is already done # We just need IV on confidence_level + dtype = t.dtype if dtype is None else dtype + xp = array_namespace(t) if xp is None else xp + if confidence_level < 0 or confidence_level > 1: message = "`confidence_level` must be a number between 0 and 1." raise ValueError(message) @@ -7023,7 +7046,11 @@ def _t_confidence_interval(df, t, confidence_level, alternative): p, nans = np.broadcast_arrays(t, np.nan) low, high = nans, nans - return low[()], high[()] + low = xp.asarray(low, dtype=dtype) + low = low[()] if low.ndim == 0 else low + high = xp.asarray(high, dtype=dtype) + high = high[()] if high.ndim == 0 else high + return low, high def _ttest_ind_from_stats(mean1, mean2, denom, df, alternative): diff --git a/scipy/stats/tests/test_stats.py b/scipy/stats/tests/test_stats.py index 9f0ca52a1b6d..7671838905c4 100644 --- a/scipy/stats/tests/test_stats.py +++ b/scipy/stats/tests/test_stats.py @@ -3570,53 +3570,69 @@ def ttest_data_axis_strategy(draw): return data, axis +@pytest.mark.skip_xp_backends(cpu_only=True, + reasons=['Uses NumPy for pvalue, CI']) +@pytest.mark.usefixtures("skip_xp_backends") +@array_api_compatible class TestStudentTest: - X1 = np.array([-1, 0, 1]) - X2 = np.array([0, 1, 2]) - T1_0 = 0 - P1_0 = 1 - T1_1 = -1.7320508075 - P1_1 = 0.22540333075 - T1_2 = -3.464102 - P1_2 = 0.0741799 - T2_0 = 1.732051 - P2_0 = 0.2254033 + # Preserving original test cases. + # Recomputed statistics and p-values with R t.test, e.g. + # options(digits=16) + # t.test(c(-1., 0., 1.), mu=2) + X1 = [-1., 0., 1.] + X2 = [0., 1., 2.] + T1_0 = 0. + P1_0 = 1. + T1_1 = -1.7320508075689 + P1_1 = 0.2254033307585 + T1_2 = -3.4641016151378 + P1_2 = 0.07417990022745 + T2_0 = 1.7320508075689 + P2_0 = 0.2254033307585 P1_1_l = P1_1 / 2 P1_1_g = 1 - (P1_1 / 2) - def test_onesample(self): + def test_onesample(self, xp): with suppress_warnings() as sup, \ np.errstate(invalid="ignore", divide="ignore"): sup.filter(RuntimeWarning, "Degrees of freedom <= 0 for slice") - t, p = stats.ttest_1samp(4., 3.) - assert_(np.isnan(t)) - assert_(np.isnan(p)) + a = xp.asarray(4.) if not is_numpy(xp) else 4. + t, p = stats.ttest_1samp(a, 3.) + xp_assert_equal(t, xp.asarray(xp.nan)) + xp_assert_equal(p, xp.asarray(xp.nan)) - t, p = stats.ttest_1samp(self.X1, 0) + t, p = stats.ttest_1samp(xp.asarray(self.X1), 0.) + xp_assert_close(t, xp.asarray(self.T1_0)) + xp_assert_close(p, xp.asarray(self.P1_0)) - assert_array_almost_equal(t, self.T1_0) - assert_array_almost_equal(p, self.P1_0) - - res = stats.ttest_1samp(self.X1, 0) + res = stats.ttest_1samp(xp.asarray(self.X1), 0.) attributes = ('statistic', 'pvalue') check_named_results(res, attributes) - t, p = stats.ttest_1samp(self.X2, 0) + t, p = stats.ttest_1samp(xp.asarray(self.X2), 0.) - assert_array_almost_equal(t, self.T2_0) - assert_array_almost_equal(p, self.P2_0) + xp_assert_close(t, xp.asarray(self.T2_0)) + xp_assert_close(p, xp.asarray(self.P2_0)) - t, p = stats.ttest_1samp(self.X1, 1) + t, p = stats.ttest_1samp(xp.asarray(self.X1), 1.) - assert_array_almost_equal(t, self.T1_1) - assert_array_almost_equal(p, self.P1_1) + xp_assert_close(t, xp.asarray(self.T1_1)) + xp_assert_close(p, xp.asarray(self.P1_1)) - t, p = stats.ttest_1samp(self.X1, 2) + t, p = stats.ttest_1samp(xp.asarray(self.X1, dtype=xp.float64), 2.) - assert_array_almost_equal(t, self.T1_2) - assert_array_almost_equal(p, self.P1_2) + xp_assert_close(t, xp.asarray(self.T1_2, dtype=xp.float64)) + xp_assert_close(p, xp.asarray(self.P1_2, dtype=xp.float64)) + def test_onesample_nan_policy(self, xp): # check nan policy + if not is_numpy(xp): + x = xp.asarray([1., 2., 3., xp.nan]) + message = "Use of `nan_policy` and `keepdims`..." + with pytest.raises(NotImplementedError, match=message): + stats.ttest_1samp(x, 1., nan_policy='omit') + return + x = stats.norm.rvs(loc=5, scale=10, size=51, random_state=7654567) x[50] = np.nan with np.errstate(invalid="ignore"): @@ -3628,20 +3644,21 @@ def test_onesample(self): assert_raises(ValueError, stats.ttest_1samp, x, 5.0, nan_policy='foobar') - def test_1samp_alternative(self): - assert_raises(ValueError, stats.ttest_1samp, self.X1, 0, - alternative="error") + def test_1samp_alternative(self, xp): + message = "`alternative` must be 'less', 'greater', or 'two-sided'." + with pytest.raises(ValueError, match=message): + stats.ttest_1samp(xp.asarray(self.X1), 0., alternative="error") - t, p = stats.ttest_1samp(self.X1, 1, alternative="less") - assert_allclose(p, self.P1_1_l) - assert_allclose(t, self.T1_1) + t, p = stats.ttest_1samp(xp.asarray(self.X1), 1., alternative="less") + xp_assert_close(p, xp.asarray(self.P1_1_l)) + xp_assert_close(t, xp.asarray(self.T1_1)) - t, p = stats.ttest_1samp(self.X1, 1, alternative="greater") - assert_allclose(p, self.P1_1_g) - assert_allclose(t, self.T1_1) + t, p = stats.ttest_1samp(xp.asarray(self.X1), 1., alternative="greater") + xp_assert_close(p, xp.asarray(self.P1_1_g)) + xp_assert_close(t, xp.asarray(self.T1_1)) @pytest.mark.parametrize("alternative", ['two-sided', 'less', 'greater']) - def test_1samp_ci_1d(self, alternative): + def test_1samp_ci_1d(self, xp, alternative): # test confidence interval method against reference values rng = np.random.default_rng(8066178009154342972) n = 10 @@ -3652,18 +3669,22 @@ def test_1samp_ci_1d(self, alternative): # x = c(2.75532884, 0.93892217, 0.94835861, 1.49489446, -0.62396595, # -1.88019867, -1.55684465, 4.88777104, 5.15310979, 4.34656348) # t.test(x, conf.level=0.85, alternative='l') + dtype = xp.float32 if is_torch(xp) else xp.float64 # use default dtype + x = xp.asarray(x, dtype=dtype) + popmean = xp.asarray(popmean, dtype=dtype) ref = {'two-sided': [0.3594423211709136, 2.9333455028290860], 'greater': [0.7470806207371626, np.inf], 'less': [-np.inf, 2.545707203262837]} res = stats.ttest_1samp(x, popmean=popmean, alternative=alternative) ci = res.confidence_interval(confidence_level=0.85) - assert_allclose(ci, ref[alternative]) - assert_equal(res.df, n-1) + xp_assert_close(ci.low, xp.asarray(ref[alternative][0])) + xp_assert_close(ci.high, xp.asarray(ref[alternative][1])) + xp_assert_equal(res.df, xp.asarray(n-1)) - def test_1samp_ci_iv(self): + def test_1samp_ci_iv(self, xp): # test `confidence_interval` method input validation - res = stats.ttest_1samp(np.arange(10), 0) + res = stats.ttest_1samp(xp.arange(10.), 0.) message = '`confidence_level` must be a number between 0 and 1.' with pytest.raises(ValueError, match=message): res.confidence_interval(confidence_level=10) @@ -3672,17 +3693,22 @@ def test_1samp_ci_iv(self): @hypothesis.given(alpha=hypothesis.strategies.floats(1e-15, 1-1e-15), data_axis=ttest_data_axis_strategy()) @pytest.mark.parametrize('alternative', ['less', 'greater']) - def test_pvalue_ci(self, alpha, data_axis, alternative): + def test_pvalue_ci(self, alpha, data_axis, alternative, xp): # test relationship between one-sided p-values and confidence intervals data, axis = data_axis - res = stats.ttest_1samp(data, 0, + data = data.astype(np.float64, copy=True) # ensure byte order + data = xp.asarray(data, dtype=xp.float64) + res = stats.ttest_1samp(data, 0., alternative=alternative, axis=axis) l, u = res.confidence_interval(confidence_level=alpha) popmean = l if alternative == 'greater' else u - popmean = np.expand_dims(popmean, axis=axis) - res = stats.ttest_1samp(data, popmean, - alternative=alternative, axis=axis) - np.testing.assert_allclose(res.pvalue, 1-alpha) + xp_test = array_namespace(l) # torch needs `expand_dims` + popmean = xp_test.expand_dims(popmean, axis=axis) + res = stats.ttest_1samp(data, popmean, alternative=alternative, axis=axis) + shape = list(data.shape) + shape.pop(axis) + ref = xp.broadcast_to(xp.asarray(1-alpha, dtype=xp.float64), shape) + xp_assert_close(res.pvalue, ref) class TestPercentileOfScore: @@ -5751,99 +5777,137 @@ def test_ttest_single_observation(): assert_allclose(res, (1.0394023007754, 0.407779907736), rtol=1e-10) -def test_ttest_1samp_new(): - n1, n2, n3 = (10,15,20) - rvn1 = stats.norm.rvs(loc=5,scale=10,size=(n1,n2,n3)) +def _convert_pvalue_alternative(t, p, alt, xp): + # test alternative parameter + # Convert from two-sided p-values to one sided using T result data. + less = xp.asarray(alt == "less") + greater = xp.asarray(alt == "greater") + i = ((t < 0) & less) | ((t > 0) & greater) + return xp.where(i, p/2, 1 - p/2) + + +@pytest.mark.skip_xp_backends(cpu_only=True, + reasons=['Uses NumPy for pvalue, CI']) +@pytest.mark.usefixtures("skip_xp_backends") +@array_api_compatible +def test_ttest_1samp_new(xp): + n1, n2, n3 = (10, 15, 20) + rvn1 = stats.norm.rvs(loc=5, scale=10, size=(n1, n2, n3)) + rvn1 = xp.asarray(rvn1) # check multidimensional array and correct axis handling # deterministic rvn1 and rvn2 would be better as in test_ttest_rel - t1,p1 = stats.ttest_1samp(rvn1[:,:,:], np.ones((n2,n3)),axis=0) - t2,p2 = stats.ttest_1samp(rvn1[:,:,:], 1,axis=0) - t3,p3 = stats.ttest_1samp(rvn1[:,0,0], 1) - assert_array_almost_equal(t1,t2, decimal=14) - assert_almost_equal(t1[0,0],t3, decimal=14) - assert_equal(t1.shape, (n2,n3)) - - t1,p1 = stats.ttest_1samp(rvn1[:,:,:], np.ones((n1, 1, n3)),axis=1) - t2,p2 = stats.ttest_1samp(rvn1[:,:,:], 1,axis=1) - t3,p3 = stats.ttest_1samp(rvn1[0,:,0], 1) - assert_array_almost_equal(t1,t2, decimal=14) - assert_almost_equal(t1[0,0],t3, decimal=14) - assert_equal(t1.shape, (n1,n3)) - - t1,p1 = stats.ttest_1samp(rvn1[:,:,:], np.ones((n1,n2,1)),axis=2) - t2,p2 = stats.ttest_1samp(rvn1[:,:,:], 1,axis=2) - t3,p3 = stats.ttest_1samp(rvn1[0,0,:], 1) - assert_array_almost_equal(t1,t2, decimal=14) - assert_almost_equal(t1[0,0],t3, decimal=14) - assert_equal(t1.shape, (n1,n2)) + popmean = xp.ones((1, n2, n3)) + t1, p1 = stats.ttest_1samp(rvn1, popmean, axis=0) + t2, p2 = stats.ttest_1samp(rvn1, 1., axis=0) + t3, p3 = stats.ttest_1samp(rvn1[:, 0, 0], 1.) + xp_assert_close(t1, t2, rtol=1e-14) + xp_assert_close(t1[0, 0], t3, rtol=1e-14) + assert_equal(t1.shape, (n2, n3)) + + popmean = xp.ones((n1, 1, n3)) + t1, p1 = stats.ttest_1samp(rvn1, popmean, axis=1) + t2, p2 = stats.ttest_1samp(rvn1, 1., axis=1) + t3, p3 = stats.ttest_1samp(rvn1[0, :, 0], 1.) + xp_assert_close(t1, t2, rtol=1e-14) + xp_assert_close(t1[0, 0], t3, rtol=1e-14) + assert_equal(t1.shape, (n1, n3)) + + popmean = xp.ones((n1, n2, 1)) + t1, p1 = stats.ttest_1samp(rvn1, popmean, axis=2) + t2, p2 = stats.ttest_1samp(rvn1, 1., axis=2) + t3, p3 = stats.ttest_1samp(rvn1[0, 0, :], 1.) + xp_assert_close(t1, t2, rtol=1e-14) + xp_assert_close(t1[0, 0], t3, rtol=1e-14) + assert_equal(t1.shape, (n1, n2)) # test zero division problem - t, p = stats.ttest_1samp([0, 0, 0], 1) - assert_equal((np.abs(t), p), (np.inf, 0)) + t, p = stats.ttest_1samp(xp.asarray([0., 0., 0.]), 1.) + xp_assert_equal(xp.abs(t), xp.asarray(xp.inf)) + xp_assert_equal(p, xp.asarray(0.)) - # test alternative parameter - # Convert from two-sided p-values to one sided using T result data. - def convert(t, p, alt): - if (t < 0 and alt == "less") or (t > 0 and alt == "greater"): - return p / 2 - return 1 - (p / 2) - converter = np.vectorize(convert) - tr, pr = stats.ttest_1samp(rvn1[:, :, :], 1) + tr, pr = stats.ttest_1samp(rvn1[:, :, :], 1.) - t, p = stats.ttest_1samp(rvn1[:, :, :], 1, alternative="greater") - pc = converter(tr, pr, "greater") - assert_allclose(p, pc) - assert_allclose(t, tr) + t, p = stats.ttest_1samp(rvn1[:, :, :], 1., alternative="greater") + pc = _convert_pvalue_alternative(tr, pr, "greater", xp) + xp_assert_close(p, pc) + xp_assert_close(t, tr) - t, p = stats.ttest_1samp(rvn1[:, :, :], 1, alternative="less") - pc = converter(tr, pr, "less") - assert_allclose(p, pc) - assert_allclose(t, tr) + t, p = stats.ttest_1samp(rvn1[:, :, :], 1., alternative="less") + pc = _convert_pvalue_alternative(tr, pr, "less", xp) + xp_assert_close(p, pc) + xp_assert_close(t, tr) with np.errstate(all='ignore'): - assert_equal(stats.ttest_1samp([0, 0, 0], 0), (np.nan, np.nan)) + res = stats.ttest_1samp(xp.asarray([0., 0., 0.]), 0.) + xp_assert_equal(res.statistic, xp.asarray(xp.nan)) + xp_assert_equal(res.pvalue, xp.asarray(xp.nan)) # check that nan in input array result in nan output - anan = np.array([[1, np.nan],[-1, 1]]) - assert_equal(stats.ttest_1samp(anan, 0), ([0, np.nan], [1, np.nan])) + anan = xp.asarray([[1., np.nan], [-1., 1.]]) + res = stats.ttest_1samp(anan, 0.) + xp_assert_equal(res.statistic, xp.asarray([0., xp.nan])) + xp_assert_equal(res.pvalue, xp.asarray([1., xp.nan])) - rvn1[0:2, 1:3, 4:8] = np.nan - tr, pr = stats.ttest_1samp(rvn1[:, :, :], 1, nan_policy='omit') +@pytest.mark.skip_xp_backends(np_only=True, + reasons=["Only NumPy has nan_policy='omit' for now"]) +@pytest.mark.usefixtures("skip_xp_backends") +@array_api_compatible +def test_ttest_1samp_new_omit(xp): + n1, n2, n3 = (10, 15, 20) + rvn1 = stats.norm.rvs(loc=5, scale=10, size=(n1, n2, n3)) + rvn1 = xp.asarray(rvn1) - t, p = stats.ttest_1samp(rvn1[:, :, :], 1, nan_policy='omit', + rvn1[0:2, 1:3, 4:8] = xp.nan + + tr, pr = stats.ttest_1samp(rvn1[:, :, :], 1., nan_policy='omit') + + t, p = stats.ttest_1samp(rvn1[:, :, :], 1., nan_policy='omit', alternative="greater") - pc = converter(tr, pr, "greater") - assert_allclose(p, pc) - assert_allclose(t, tr) + pc = _convert_pvalue_alternative(tr, pr, "greater", xp) + xp_assert_close(p, pc) + xp_assert_close(t, tr) - t, p = stats.ttest_1samp(rvn1[:, :, :], 1, nan_policy='omit', + t, p = stats.ttest_1samp(rvn1[:, :, :], 1., nan_policy='omit', alternative="less") - pc = converter(tr, pr, "less") - assert_allclose(p, pc) - assert_allclose(t, tr) + pc = _convert_pvalue_alternative(tr, pr, "less", xp) + xp_assert_close(p, pc) + xp_assert_close(t, tr) -def test_ttest_1samp_popmean_array(): +@pytest.mark.skip_xp_backends(cpu_only=True, + reasons=['Uses NumPy for pvalue, CI']) +@pytest.mark.usefixtures("skip_xp_backends") +@array_api_compatible +def test_ttest_1samp_popmean_array(xp): # when popmean.shape[axis] != 1, raise an error # if the user wants to test multiple null hypotheses simultaneously, # use standard broadcasting rules rng = np.random.default_rng(2913300596553337193) x = rng.random(size=(1, 15, 20)) + x = xp.asarray(x) message = r"`popmean.shape\[axis\]` must equal 1." - popmean = rng.random(size=(5, 2, 20)) + popmean = xp.asarray(rng.random(size=(5, 2, 20))) with pytest.raises(ValueError, match=message): stats.ttest_1samp(x, popmean=popmean, axis=-2) - popmean = rng.random(size=(5, 1, 20)) + popmean = xp.asarray(rng.random(size=(5, 1, 20))) res = stats.ttest_1samp(x, popmean=popmean, axis=-2) assert res.statistic.shape == (5, 20) - ci = np.expand_dims(res.confidence_interval(), axis=-2) - res = stats.ttest_1samp(x, popmean=ci, axis=-2) - assert_allclose(res.pvalue, 0.05) + xp_test = array_namespace(x) # torch needs expand_dims + l, u = res.confidence_interval() + l = xp_test.expand_dims(l, axis=-2) + u = xp_test.expand_dims(u, axis=-2) + + res = stats.ttest_1samp(x, popmean=l, axis=-2) + ref = xp.broadcast_to(xp.asarray(0.05, dtype=xp.float64), res.pvalue.shape) + xp_assert_close(res.pvalue, ref) + + res = stats.ttest_1samp(x, popmean=u, axis=-2) + xp_assert_close(res.pvalue, ref) class TestDescribe: From 378a8c39ffe43c73a3c76903af04bf6f23480672 Mon Sep 17 00:00:00 2001 From: thalassemia Date: Sun, 21 Apr 2024 17:43:52 -0700 Subject: [PATCH 34/64] TST: Skip Cython tests for editable installs [skip circle] --- scipy/_lib/_testutils.py | 4 ++++ scipy/linalg/tests/test_extending.py | 4 +++- scipy/optimize/tests/test_extending.py | 4 +++- scipy/special/tests/test_extending.py | 4 +++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/scipy/_lib/_testutils.py b/scipy/_lib/_testutils.py index c45c9eecdc4c..e8a6d49d65cd 100644 --- a/scipy/_lib/_testutils.py +++ b/scipy/_lib/_testutils.py @@ -13,6 +13,7 @@ from importlib.util import module_from_spec, spec_from_file_location import numpy as np +import scipy try: # Need type: ignore[import-untyped] for mypy >= 1.6 @@ -43,6 +44,9 @@ IS_MUSL = True +IS_EDITABLE = 'editable' in scipy.__path__[0] + + class FPUModeChangeWarning(RuntimeWarning): """Warning about FPU mode change""" pass diff --git a/scipy/linalg/tests/test_extending.py b/scipy/linalg/tests/test_extending.py index dee7046dfdc2..b6f7b92d2a32 100644 --- a/scipy/linalg/tests/test_extending.py +++ b/scipy/linalg/tests/test_extending.py @@ -4,11 +4,13 @@ import numpy as np import pytest -from scipy._lib._testutils import _test_cython_extension, cython +from scipy._lib._testutils import IS_EDITABLE, _test_cython_extension, cython from scipy.linalg.blas import cdotu # type: ignore[attr-defined] from scipy.linalg.lapack import dgtsv # type: ignore[attr-defined] +@pytest.mark.skipif(IS_EDITABLE, + reason='Editable install cannot find .pxd headers.') @pytest.mark.skipif(platform.machine() in ["wasm32", "wasm64"], reason="Can't start subprocess") @pytest.mark.skipif(cython is None, reason="requires cython") diff --git a/scipy/optimize/tests/test_extending.py b/scipy/optimize/tests/test_extending.py index f3e554aad508..eacb3f9dccf5 100644 --- a/scipy/optimize/tests/test_extending.py +++ b/scipy/optimize/tests/test_extending.py @@ -3,9 +3,11 @@ import pytest -from scipy._lib._testutils import _test_cython_extension, cython +from scipy._lib._testutils import IS_EDITABLE, _test_cython_extension, cython +@pytest.mark.skipif(IS_EDITABLE, + reason='Editable install cannot find .pxd headers.') @pytest.mark.skipif(platform.machine() in ["wasm32", "wasm64"], reason="Can't start subprocess") @pytest.mark.skipif(cython is None, reason="requires cython") diff --git a/scipy/special/tests/test_extending.py b/scipy/special/tests/test_extending.py index fa541f6d58a5..0096b9282434 100644 --- a/scipy/special/tests/test_extending.py +++ b/scipy/special/tests/test_extending.py @@ -3,10 +3,12 @@ import pytest -from scipy._lib._testutils import _test_cython_extension, cython +from scipy._lib._testutils import IS_EDITABLE,_test_cython_extension, cython from scipy.special import beta, gamma +@pytest.mark.skipif(IS_EDITABLE, + reason='Editable install cannot find .pxd headers.') @pytest.mark.skipif(platform.machine() in ["wasm32", "wasm64"], reason="Can't start subprocess") @pytest.mark.skipif(cython is None, reason="requires cython") From c808feb22912a20e2530dbaf013a98df0b7656bb Mon Sep 17 00:00:00 2001 From: Tyler Reddy Date: Mon, 22 Apr 2024 16:02:01 -0600 Subject: [PATCH 35/64] DOC: release process updates Fixes #20461 * `setup.py` no longer exists, so don't mention it for version bounds updates. * The release schedule is now to be proposed on the appropriate Discourse forum instead of the mailing list; similar adjustments for announcements/feedback. * Clarify that docs versions switcher adjustments are to happen on the `main` branch only, for now. [docs only] --- doc/source/dev/core-dev/releasing.rst.inc | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/doc/source/dev/core-dev/releasing.rst.inc b/doc/source/dev/core-dev/releasing.rst.inc index 58ecf554b047..8ef9e0883355 100644 --- a/doc/source/dev/core-dev/releasing.rst.inc +++ b/doc/source/dev/core-dev/releasing.rst.inc @@ -6,7 +6,7 @@ Making a SciPy release At the highest level, this is what the release manager does to release a new SciPy version: -#. Propose a release schedule on the scipy-dev mailing list. +#. Propose a release schedule in the SciPy forum at https://discuss.scientific-python.org/. #. Create the maintenance branch for the release. #. Tag the release. #. Build all release artifacts (sources, installers, docs). @@ -48,7 +48,8 @@ is needed, while a simple bug-fix that's backported from main doesn't require a new RC. To propose a schedule, send a list with estimated dates for branching and -beta/rc/final releases to scipy-dev. In the same email, ask everyone to check +beta/rc/final releases to the SciPy forum at +https://discuss.scientific-python.org/. In the same message, ask everyone to check if there are important issues/PRs that need to be included and aren't tagged with the Milestone for the release or the "backport-candidate" label. @@ -63,13 +64,14 @@ Maintenance branches are named ``maintenance/..x`` (e.g. 0.19.x). To create one, simply push a branch with the correct name to the scipy repo. Immediately after, push a commit where you increment the version number on the main branch and add release notes for that new version. Send an email to -scipy-dev to let people know that you've done this. +the SciPy forum at https://discuss.scientific-python.org/ to let people know +that you've done this. Updating the version switcher ------------------------------ The version switcher dropdown needs to be updated with the new release -information. +information on the ``main`` branch only. - ``doc/source/_static/version_switcher.json``: add the new release, the new development version, and transfer ``"preferred": true`` from the old release @@ -86,7 +88,6 @@ creating a maintenance branch: - ``pyproject.toml``: all build-time dependencies, as well as supported Python and NumPy versions -- ``setup.py``: supported Python and NumPy versions - ``scipy/__init__.py``: for NumPy version check Each file has comments describing how to set the correct upper bounds. @@ -228,14 +229,11 @@ not release candidates. Wrapping up ----------- -Send an email announcing the release to the following mailing lists: +Send a message announcing the release to +https://discuss.scientific-python.org/c/announcements/. -- scipy-dev -- numpy-discussion -- python-announce (not for beta/rc releases) - -For beta and rc versions, ask people in the email to test (run the scipy tests -and test against their own code) and report issues on Github or scipy-dev. +For beta and rc versions, ask people to test (run the scipy tests +and test against their own code) and report issues on Github or Discourse. After the final release is done, port relevant changes to release notes, build scripts, author name mapping in ``tools/authors.py`` and any other changes that From d55cb95282b5cfab381e373d3c127630c7e915a0 Mon Sep 17 00:00:00 2001 From: DWesl <22566757+DWesl@users.noreply.github.com> Date: Tue, 23 Apr 2024 03:26:55 -0400 Subject: [PATCH 36/64] CI: Check whether Python.h is included first in a file (#20536) Based on script from #20149. Co-authored-by: Ralf Gommers --- .github/workflows/lint.yml | 5 + dev.py | 18 ++ .../io/_fast_matrix_market/src/_fmm_core.cpp | 5 +- scipy/optimize/_pava/pava_pybind.cpp | 2 +- scipy/spatial/ckdtree/src/count_neighbors.cxx | 6 +- scipy/spatial/ckdtree/src/query.cxx | 8 +- .../spatial/ckdtree/src/query_ball_point.cxx | 6 +- scipy/spatial/ckdtree/src/query_ball_tree.cxx | 6 +- scipy/spatial/ckdtree/src/query_pairs.cxx | 8 +- .../spatial/ckdtree/src/sparse_distances.cxx | 8 +- scipy/special/sf_error.cc | 5 - tools/check_python_h_first.py | 219 ++++++++++++++++++ 12 files changed, 266 insertions(+), 30 deletions(-) create mode 100755 tools/check_python_h_first.py diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 67fb666981b4..7dd571b7044a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -47,3 +47,8 @@ jobs: python tools/lint.py --diff-against origin/$GITHUB_BASE_REF python tools/unicode-check.py python tools/check_test_name.py + + - name: Check that Python.h is first in any file including it. + shell: bash + run: | + python tools/check_python_h_first.py diff --git a/dev.py b/dev.py index fa198a1bc27d..1b11b9f54d13 100644 --- a/dev.py +++ b/dev.py @@ -921,6 +921,23 @@ def task_lint(fix): 'doc': 'Lint only files modified since last commit (stricter rules)', } +@task_params([]) +def task_check_python_h_first(): + # Lint just the diff since branching off of main using a + # stricter configuration. + # emit_cmdstr(os.path.join('tools', 'lint.py') + ' --diff-against main') + cmd = "{!s} --diff-against=main".format( + Dirs().root / 'tools' / 'check_python_h_first.py' + ) + return { + 'basename': 'check_python_h_first', + 'actions': [cmd], + 'doc': ( + 'Check Python.h order only files modified since last commit ' + '(stricter rules)' + ), + } + def task_unicode_check(): # emit_cmdstr(os.path.join('tools', 'unicode-check.py')) @@ -955,6 +972,7 @@ def run(cls, fix): 'lint': {'fix': fix}, 'unicode-check': {}, 'check-testname': {}, + 'check_python_h_first': {}, }) diff --git a/scipy/io/_fast_matrix_market/src/_fmm_core.cpp b/scipy/io/_fast_matrix_market/src/_fmm_core.cpp index 2a6bf05bc5c8..3912910935e3 100644 --- a/scipy/io/_fast_matrix_market/src/_fmm_core.cpp +++ b/scipy/io/_fast_matrix_market/src/_fmm_core.cpp @@ -2,7 +2,8 @@ // Use of this source code is governed by the BSD 2-clause license found in the LICENSE.txt file. // SPDX-License-Identifier: BSD-2-Clause -#include +#include "_fmm_core.hpp" + #include #include namespace fast_matrix_market { @@ -17,8 +18,6 @@ namespace fast_matrix_market { } #include -#include "_fmm_core.hpp" - //////////////////////////////////////////////// //// Header methods //////////////////////////////////////////////// diff --git a/scipy/optimize/_pava/pava_pybind.cpp b/scipy/optimize/_pava/pava_pybind.cpp index a2138baba475..d2206047229a 100644 --- a/scipy/optimize/_pava/pava_pybind.cpp +++ b/scipy/optimize/_pava/pava_pybind.cpp @@ -1,7 +1,7 @@ -#include #include #include #include +#include namespace py = pybind11; diff --git a/scipy/spatial/ckdtree/src/count_neighbors.cxx b/scipy/spatial/ckdtree/src/count_neighbors.cxx index fd0ff5e261ea..872f09d78b21 100644 --- a/scipy/spatial/ckdtree/src/count_neighbors.cxx +++ b/scipy/spatial/ckdtree/src/count_neighbors.cxx @@ -1,3 +1,6 @@ +#include "ckdtree_decl.h" +#include "rectangle.h" + #include #include #include @@ -11,9 +14,6 @@ #include #include -#include "ckdtree_decl.h" -#include "rectangle.h" - struct WeightedTree { const ckdtree *tree; double *weights; diff --git a/scipy/spatial/ckdtree/src/query.cxx b/scipy/spatial/ckdtree/src/query.cxx index a8aeba4697b2..e8bad724a1e9 100644 --- a/scipy/spatial/ckdtree/src/query.cxx +++ b/scipy/spatial/ckdtree/src/query.cxx @@ -1,3 +1,7 @@ +#include "ckdtree_decl.h" +#include "ordered_pair.h" +#include "rectangle.h" + #include #include #include @@ -10,10 +14,6 @@ #include #include -#include "ckdtree_decl.h" -#include "ordered_pair.h" -#include "rectangle.h" - /* * Priority queue * ============== diff --git a/scipy/spatial/ckdtree/src/query_ball_point.cxx b/scipy/spatial/ckdtree/src/query_ball_point.cxx index 77ed1beee61c..917d2be20d53 100644 --- a/scipy/spatial/ckdtree/src/query_ball_point.cxx +++ b/scipy/spatial/ckdtree/src/query_ball_point.cxx @@ -1,3 +1,6 @@ +#include "ckdtree_decl.h" +#include "rectangle.h" + #include #include #include @@ -11,9 +14,6 @@ #include #include -#include "ckdtree_decl.h" -#include "rectangle.h" - static void traverse_no_checking(const ckdtree *self, diff --git a/scipy/spatial/ckdtree/src/query_ball_tree.cxx b/scipy/spatial/ckdtree/src/query_ball_tree.cxx index bea17eb7ed04..b0af311f289c 100644 --- a/scipy/spatial/ckdtree/src/query_ball_tree.cxx +++ b/scipy/spatial/ckdtree/src/query_ball_tree.cxx @@ -1,3 +1,6 @@ +#include "ckdtree_decl.h" +#include "rectangle.h" + #include #include #include @@ -11,9 +14,6 @@ #include #include -#include "ckdtree_decl.h" -#include "rectangle.h" - static void traverse_no_checking(const ckdtree *self, const ckdtree *other, diff --git a/scipy/spatial/ckdtree/src/query_pairs.cxx b/scipy/spatial/ckdtree/src/query_pairs.cxx index 5cc81594f57a..90d8f495dd2c 100644 --- a/scipy/spatial/ckdtree/src/query_pairs.cxx +++ b/scipy/spatial/ckdtree/src/query_pairs.cxx @@ -1,3 +1,7 @@ +#include "ckdtree_decl.h" +#include "ordered_pair.h" +#include "rectangle.h" + #include #include #include @@ -10,10 +14,6 @@ #include #include -#include "ckdtree_decl.h" -#include "ordered_pair.h" -#include "rectangle.h" - static void traverse_no_checking(const ckdtree *self, diff --git a/scipy/spatial/ckdtree/src/sparse_distances.cxx b/scipy/spatial/ckdtree/src/sparse_distances.cxx index 14ccf25a3127..1229d1f7bf56 100644 --- a/scipy/spatial/ckdtree/src/sparse_distances.cxx +++ b/scipy/spatial/ckdtree/src/sparse_distances.cxx @@ -1,3 +1,7 @@ +#include "ckdtree_decl.h" +#include "rectangle.h" +#include "coo_entries.h" + #include #include #include @@ -10,10 +14,6 @@ #include #include -#include "ckdtree_decl.h" -#include "rectangle.h" -#include "coo_entries.h" - template static void traverse(const ckdtree *self, const ckdtree *other, std::vector *results, diff --git a/scipy/special/sf_error.cc b/scipy/special/sf_error.cc index f7bae3210f87..e0ed20be5165 100644 --- a/scipy/special/sf_error.cc +++ b/scipy/special/sf_error.cc @@ -1,8 +1,3 @@ -#include - -#include -#include - #include #include diff --git a/tools/check_python_h_first.py b/tools/check_python_h_first.py new file mode 100755 index 000000000000..f16dcbe0f685 --- /dev/null +++ b/tools/check_python_h_first.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +"""Check that Python.h is included before any stdlib headers. + +May be a bit overzealous, but it should get the job done. +""" +import argparse +import fnmatch +import os.path +import re +import subprocess +import sys + +HEADER_PATTERN = re.compile( + r'^\s*#\s*include\s*[<"]((?:\w+/)*\w+(?:\.h[hp+]{0,2})?)[>"]\s*$' +) + +PYTHON_INCLUDING_HEADERS = [ + "Python.h", + # This isn't all of Python.h, but it is the visibility macros + "pyconfig.h", + "numpy/arrayobject.h", + "numpy/ndarrayobject.h", + "numpy/npy_common.h", + "numpy/npy_math.h", + "numpy/random/distributions.h", + "pybind11/pybind11.h", + # Boost::Python + "boost/python.hpp", + "boost/python/args.hpp", + "boost/python/detail/prefix.hpp", + "boost/python/detail/wrap_python.hpp", + "boost/python/ssize_t.hpp", + "boost/python/object.hpp", + "boost/mpi/python.hpp", + # Pythran + "pythonic/core.hpp", + # Python-including headers the sort doesn't pick up + "ni_support.h", +] +LEAF_HEADERS = [] + +C_CPP_EXTENSIONS = (".c", ".h", ".cpp", ".hpp", ".cc", ".hh", ".cxx", ".hxx") +# check against list in diff_files + +PARSER = argparse.ArgumentParser(description=__doc__) +PARSER.add_argument( + "--diff-against", + dest="branch", + type=str, + default=None, + help="Diff against " + "this branch and lint modified files. Use either " + "`--diff-against` or `--files`, but not both. " + "Likely to produce false positives.", +) +PARSER.add_argument( + "files", + nargs="*", + help="Lint these files or directories; " "use **/*.py to lint all files", +) + + +def check_python_h_included_first(name_to_check: str) -> int: + """Check that the passed file includes Python.h first if it does at all. + + Perhaps overzealous, but that should work around concerns with + recursion. + + Parameters + ---------- + name_to_check : str + The name of the file to check. + + Returns + ------- + int + The number of headers before Python.h + """ + included_python = False + included_non_python_header = [] + warned_python_construct = False + basename_to_check = os.path.basename(name_to_check) + in_comment = False + includes_headers = False + with open(name_to_check) as in_file: + for i, line in enumerate(in_file, 1): + # Very basic comment parsing + # Assumes /*...*/ comments are on their own lines + if "/*" in line: + if "*/" not in line: + in_comment = True + # else-branch could use regex to remove comment and continue + continue + if in_comment: + if "*/" in line: + in_comment = False + continue + match = HEADER_PATTERN.match(line) + if match: + includes_headers = True + this_header = match.group(1) + if this_header in PYTHON_INCLUDING_HEADERS: + if included_non_python_header and not included_python: + print( + f"Header before Python.h in file {name_to_check:s}\n" + f"Python.h on line {i:d}, other header(s) on line(s)" + f" {included_non_python_header}", + file=sys.stderr, + ) + included_python = True + PYTHON_INCLUDING_HEADERS.append(basename_to_check) + elif not included_python and ( + "numpy" in this_header + and this_header != "numpy/utils.h" + or "python" in this_header + ): + print( + f"Python.h not included before python-including header " + f"in file {name_to_check:s}\n" + f"{this_header:s} on line {i:d}", + file=sys.stderr, + ) + elif not included_python and this_header not in LEAF_HEADERS: + included_non_python_header.append(i) + elif ( + not included_python + and not warned_python_construct + and ".h" not in basename_to_check + ) and ("py::" in line or "PYBIND11_" in line or "npy_" in line): + print( + "Python-including header not used before python constructs " + f"in file {name_to_check:s}\nConstruct on line {i:d}", + file=sys.stderr, + ) + warned_python_construct = True + if includes_headers: + LEAF_HEADERS.append(this_header) + return included_python and len(included_non_python_header) + + +def process_files(file_list: list[str]) -> int: + n_out_of_order = 0 + for name_to_check in sorted( + file_list, key=lambda name: "h" not in os.path.splitext(name)[1].lower() + ): + try: + n_out_of_order += check_python_h_included_first(name_to_check) + except UnicodeDecodeError: + print(f"File {name_to_check:s} not utf-8", sys.stdout) + return n_out_of_order + + +def find_c_cpp_files(root: str) -> list[str]: + + result = [] + + for dirpath, dirnames, filenames in os.walk("scipy"): + # I'm assuming other people have checked boost + for name in ("build", ".git", "boost"): + try: + dirnames.remove(name) + except ValueError: + pass + for name in fnmatch.filter(dirnames, "*.p"): + dirnames.remove(name) + result.extend( + [ + os.path.join(dirpath, name) + for name in filenames + if os.path.splitext(name)[1].lower() in C_CPP_EXTENSIONS + ] + ) + return result + + +def diff_files(sha: str) -> list[str]: + """Find the diff since the given SHA. + + Adapted from lint.py + """ + res = subprocess.run( + [ + "git", + "diff", + "--name-only", + "--diff-filter=ACMR", + "-z", + sha, + "--", + # Check against C_CPP_EXTENSIONS + "*.[chCH]", + "*.[ch]pp", + "*.[ch]xx", + "*.cc", + "*.hh", + ], + stdout=subprocess.PIPE, + encoding="utf-8", + ) + res.check_returncode() + return [f for f in res.stdout.split("\0") if f] + + +if __name__ == "__main__": + from lint import find_branch_point + + args = PARSER.parse_args() + + if not ((len(args.files) == 0) ^ (args.branch is None)): + files = find_c_cpp_files("scipy") + elif args.branch: + branch_point = find_branch_point(args.branch) + files = diff_files(branch_point) + else: + files = args.files + + # See which of the headers include Python.h and add them to the list + n_out_of_order = process_files(files) + sys.exit(n_out_of_order) From ec6b0c1f78aed70d720f49f4de6ceda043b00fac Mon Sep 17 00:00:00 2001 From: lucascolley Date: Tue, 23 Apr 2024 10:28:42 +0100 Subject: [PATCH 37/64] DOC/DEV: add core-dev page on vendored code [docs only] --- doc/source/dev/core-dev/index.rst | 2 ++ doc/source/dev/core-dev/vendored-code.rst.inc | 27 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 doc/source/dev/core-dev/vendored-code.rst.inc diff --git a/doc/source/dev/core-dev/index.rst b/doc/source/dev/core-dev/index.rst index 51b8bb3fe4d1..38f91118fba3 100644 --- a/doc/source/dev/core-dev/index.rst +++ b/doc/source/dev/core-dev/index.rst @@ -16,6 +16,8 @@ SciPy Core Developer Guide .. include:: deprecations.rst.inc +.. include:: vendored-code.rst.inc + .. include:: distributing.rst.inc .. include:: releasing.rst.inc diff --git a/doc/source/dev/core-dev/vendored-code.rst.inc b/doc/source/dev/core-dev/vendored-code.rst.inc new file mode 100644 index 000000000000..ac316109b292 --- /dev/null +++ b/doc/source/dev/core-dev/vendored-code.rst.inc @@ -0,0 +1,27 @@ +.. _vendored-code: + +Vendored Code +============= +Many parts of the SciPy codebase are maintained elsewhere, and vendored in SciPy. +Some of these parts are vendored as git submodules, for example, ``boost_math``. + +Other parts are not vendored as git submodules, despite having a maintained upstream. +This is mainly for historical reasons, and it is possible that some of these parts +will see patches contributed upstream and become git submodules in the future. + +Maintainers should be careful to *not* accept contributions +(especially trivial changes) into parts of SciPy where the code is actively maintained +upstream. Instead, they should direct contributors to the upstream repo. +Currently, this includes the following parts of the codebase: + +- ARPACK_, at ``scipy/sparse/linalg/_eigen/arpack/ARPACK`` +- SuperLU_, at ``scipy/sparse/linalg/_dsolve/SuperLU`` +- QHull_, at ``scipy/spatial/qhull`` +- trlib_, at ``scipy/optimize/_trlib`` +- UNU.RAN_, at ``scipy/stats/_unuran`` + +.. _ARPACK: https://github.com/opencollab/arpack-ng +.. _SuperLU: https://github.com/xiaoyeli/superlu +.. _QHull: https://github.com/qhull/qhull +.. _trlib: https://github.com/felixlen/trlib +.. _UNU.RAN: https://statmath.wu.ac.at/unuran/ From e33e6986eb898f01b59e32acca07b1862bcec505 Mon Sep 17 00:00:00 2001 From: Jake Bowhay Date: Tue, 23 Apr 2024 14:57:33 +0100 Subject: [PATCH 38/64] DEP: linalg: remove turbo / eigvals kwargs from linalg.{eigh,eigvalsh} and switch to kwarg-only --- scipy/linalg/_decomp.py | 61 ++++--------------------------- scipy/linalg/tests/test_decomp.py | 48 ------------------------ 2 files changed, 8 insertions(+), 101 deletions(-) diff --git a/scipy/linalg/_decomp.py b/scipy/linalg/_decomp.py index 6b8a1e4e20f9..c520d6b04b6b 100644 --- a/scipy/linalg/_decomp.py +++ b/scipy/linalg/_decomp.py @@ -16,8 +16,6 @@ 'eig_banded', 'eigvals_banded', 'eigh_tridiagonal', 'eigvalsh_tridiagonal', 'hessenberg', 'cdf2rdf'] -import warnings - import numpy as np from numpy import (array, isfinite, inexact, nonzero, iscomplexobj, flatnonzero, conj, asarray, argsort, empty, @@ -26,7 +24,6 @@ from scipy._lib._util import _asarray_validated from ._misc import LinAlgError, _datacopied, norm from .lapack import get_lapack_funcs, _compute_lwork -from scipy._lib.deprecation import _NoValue, _deprecate_positional_args _I = np.array(1j, dtype='F') @@ -283,11 +280,9 @@ def eig(a, b=None, left=False, right=True, overwrite_a=False, return w, vr -@_deprecate_positional_args(version="1.14.0") def eigh(a, b=None, *, lower=True, eigvals_only=False, overwrite_a=False, - overwrite_b=False, turbo=_NoValue, eigvals=_NoValue, type=1, - check_finite=True, subset_by_index=None, subset_by_value=None, - driver=None): + overwrite_b=False, type=1, check_finite=True, subset_by_index=None, + subset_by_value=None, driver=None): """ Solve a standard or generalized eigenvalue problem for a complex Hermitian or real symmetric matrix. @@ -355,16 +350,6 @@ def eigh(a, b=None, *, lower=True, eigvals_only=False, overwrite_a=False, Whether to check that the input matrices contain only finite numbers. Disabling may give a performance gain, but may result in problems (crashes, non-termination) if the inputs do contain infinities or NaNs. - turbo : bool, optional, deprecated - .. deprecated:: 1.5.0 - `eigh` keyword argument `turbo` is deprecated in favour of - ``driver=gvd`` keyword instead and will be removed in SciPy - 1.14.0. - eigvals : tuple (lo, hi), optional, deprecated - .. deprecated:: 1.5.0 - `eigh` keyword argument `eigvals` is deprecated in favour of - `subset_by_index` keyword instead and will be removed in SciPy - 1.14.0. Returns ------- @@ -460,17 +445,6 @@ def eigh(a, b=None, *, lower=True, eigvals_only=False, overwrite_a=False, (5, 1) """ - if turbo is not _NoValue: - warnings.warn("Keyword argument 'turbo' is deprecated in favour of '" - "driver=gvd' keyword instead and will be removed in " - "SciPy 1.14.0.", - DeprecationWarning, stacklevel=2) - if eigvals is not _NoValue: - warnings.warn("Keyword argument 'eigvals' is deprecated in favour of " - "'subset_by_index' keyword instead and will be removed " - "in SciPy 1.14.0.", - DeprecationWarning, stacklevel=2) - # set lower uplo = 'L' if lower else 'U' # Set job for Fortran routines @@ -516,19 +490,12 @@ def eigh(a, b=None, *, lower=True, eigvals_only=False, overwrite_a=False, cplx = True if iscomplexobj(b1) else (cplx or False) drv_args.update({'overwrite_b': overwrite_b, 'itype': type}) - # backwards-compatibility handling - subset_by_index = subset_by_index if (eigvals in (None, _NoValue)) else eigvals - subset = (subset_by_index is not None) or (subset_by_value is not None) # Both subsets can't be given if subset_by_index and subset_by_value: raise ValueError('Either index or value subset can be requested.') - # Take turbo into account if all conditions are met otherwise ignore - if turbo not in (None, _NoValue) and b is not None: - driver = 'gvx' if subset else 'gvd' - # Check indices if given if subset_by_index: lo, hi = (int(x) for x in subset_by_index) @@ -943,11 +910,9 @@ def eigvals(a, b=None, overwrite_a=False, check_finite=True, homogeneous_eigvals=homogeneous_eigvals) -@_deprecate_positional_args(version="1.14.0") def eigvalsh(a, b=None, *, lower=True, overwrite_a=False, - overwrite_b=False, turbo=_NoValue, eigvals=_NoValue, type=1, - check_finite=True, subset_by_index=None, subset_by_value=None, - driver=None): + overwrite_b=False, type=1, check_finite=True, subset_by_index=None, + subset_by_value=None, driver=None): """ Solves a standard or generalized eigenvalue problem for a complex Hermitian or real symmetric matrix. @@ -1010,15 +975,6 @@ def eigvalsh(a, b=None, *, lower=True, overwrite_a=False, "evd", "evr", "evx" for standard problems and "gv", "gvd", "gvx" for generalized (where b is not None) problems. See the Notes section of `scipy.linalg.eigh`. - turbo : bool, optional, deprecated - .. deprecated:: 1.5.0 - 'eigvalsh' keyword argument `turbo` is deprecated in favor of - ``driver=gvd`` option and will be removed in SciPy 1.14.0. - - eigvals : tuple (lo, hi), optional - .. deprecated:: 1.5.0 - 'eigvalsh' keyword argument `eigvals` is deprecated in favor of - `subset_by_index` option and will be removed in SciPy 1.14.0. Returns ------- @@ -1066,11 +1022,10 @@ def eigvalsh(a, b=None, *, lower=True, overwrite_a=False, array([-3.74637491, -0.76263923, 6.08502336, 12.42399079]) """ - return eigh(a, b=b, lower=lower, eigvals_only=True, - overwrite_a=overwrite_a, overwrite_b=overwrite_b, - turbo=turbo, eigvals=eigvals, type=type, - check_finite=check_finite, subset_by_index=subset_by_index, - subset_by_value=subset_by_value, driver=driver) + return eigh(a, b=b, lower=lower, eigvals_only=True, overwrite_a=overwrite_a, + overwrite_b=overwrite_b, type=type, check_finite=check_finite, + subset_by_index=subset_by_index, subset_by_value=subset_by_value, + driver=driver) def eigvals_banded(a_band, lower=False, overwrite_a_band=False, diff --git a/scipy/linalg/tests/test_decomp.py b/scipy/linalg/tests/test_decomp.py index 04d8576e7007..5e171965a4bd 100644 --- a/scipy/linalg/tests/test_decomp.py +++ b/scipy/linalg/tests/test_decomp.py @@ -844,31 +844,15 @@ def test_wrong_inputs(self): # Both value and index subsets requested assert_raises(ValueError, eigh, np.ones([3, 3]), np.ones([3, 3]), subset_by_value=[1, 2], subset_by_index=[2, 4]) - with np.testing.suppress_warnings() as sup: - sup.filter(DeprecationWarning, "Keyword argument 'eigvals") - assert_raises(ValueError, eigh, np.ones([3, 3]), np.ones([3, 3]), - subset_by_value=[1, 2], eigvals=[2, 4]) # Invalid upper index spec assert_raises(ValueError, eigh, np.ones([3, 3]), np.ones([3, 3]), subset_by_index=[0, 4]) - with np.testing.suppress_warnings() as sup: - sup.filter(DeprecationWarning, "Keyword argument 'eigvals") - assert_raises(ValueError, eigh, np.ones([3, 3]), np.ones([3, 3]), - eigvals=[0, 4]) # Invalid lower index assert_raises(ValueError, eigh, np.ones([3, 3]), np.ones([3, 3]), subset_by_index=[-2, 2]) - with np.testing.suppress_warnings() as sup: - sup.filter(DeprecationWarning, "Keyword argument 'eigvals") - assert_raises(ValueError, eigh, np.ones([3, 3]), np.ones([3, 3]), - eigvals=[-2, 2]) # Invalid index spec #2 assert_raises(ValueError, eigh, np.ones([3, 3]), np.ones([3, 3]), subset_by_index=[2, 0]) - with np.testing.suppress_warnings() as sup: - sup.filter(DeprecationWarning, "Keyword argument 'eigvals") - assert_raises(ValueError, eigh, np.ones([3, 3]), np.ones([3, 3]), - subset_by_index=[2, 0]) # Invalid value spec assert_raises(ValueError, eigh, np.ones([3, 3]), np.ones([3, 3]), subset_by_value=[2, 0]) @@ -964,38 +948,6 @@ def test_eigvalsh_new_args(self): assert_equal(len(w3), 2) assert_allclose(w3, np.array([1.2, 1.3])) - @pytest.mark.parametrize("method", [eigh, eigvalsh]) - def test_deprecation_warnings(self, method): - with pytest.warns(DeprecationWarning, - match="Keyword argument 'turbo'"): - method(np.zeros((2, 2)), turbo=True) - with pytest.warns(DeprecationWarning, - match="Keyword argument 'eigvals'"): - method(np.zeros((2, 2)), eigvals=[0, 1]) - with pytest.deprecated_call(match="use keyword arguments"): - method(np.zeros((2,2)), np.eye(2, 2), True) - - def test_deprecation_results(self): - a = _random_hermitian_matrix(3) - b = _random_hermitian_matrix(3, posdef=True) - - # check turbo gives same result as driver='gvd' - with np.testing.suppress_warnings() as sup: - sup.filter(DeprecationWarning, "Keyword argument 'turbo'") - w_dep, v_dep = eigh(a, b, turbo=True) - w, v = eigh(a, b, driver='gvd') - assert_allclose(w_dep, w) - assert_allclose(v_dep, v) - - # check eigvals gives the same result as subset_by_index - with np.testing.suppress_warnings() as sup: - sup.filter(DeprecationWarning, "Keyword argument 'eigvals'") - w_dep, v_dep = eigh(a, eigvals=[0, 1]) - w, v = eigh(a, subset_by_index=[0, 1]) - assert_allclose(w_dep, w) - assert_allclose(v_dep, v) - - @pytest.mark.parametrize('dt', [int, float, np.float32, complex, np.complex64]) def test_empty(self, dt): a = np.empty((0, 0), dtype=dt) From 8431e12346ac9564e244bfce920dc8a04bcabbd9 Mon Sep 17 00:00:00 2001 From: Albert Steppi Date: Wed, 24 Apr 2024 00:25:44 -0400 Subject: [PATCH 39/64] BUG: Fix invalid default bracket selection in _bracket_minimum (#20563) * TST: Add regression test for gh-20562 * BUG: Fix gh-20562, incorrect choice of default bracket --- scipy/optimize/_bracket.py | 25 ++++++++++--------------- scipy/optimize/tests/test_bracket.py | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/scipy/optimize/_bracket.py b/scipy/optimize/_bracket.py index bb7726c234a0..2210295ea820 100644 --- a/scipy/optimize/_bracket.py +++ b/scipy/optimize/_bracket.py @@ -397,14 +397,17 @@ def _bracket_minimum_iv(func, xm0, xl0, xr0, xmin, xmax, factor, args, maxiter): xmin = -np.inf if xmin is None else xmin xmax = np.inf if xmax is None else xmax + # If xl0 (xr0) is not supplied, fill with a dummy value for the sake + # of broadcasting. We need to wait until xmin (xmax) has been validated + # to compute the default values. xl0_not_supplied = False if xl0 is None: - xl0 = xm0 - 0.5 + xl0 = np.nan xl0_not_supplied = True xr0_not_supplied = False if xr0 is None: - xr0 = xm0 + 0.5 + xr0 = np.nan xr0_not_supplied = True factor = 2.0 if factor is None else factor @@ -429,21 +432,13 @@ def _bracket_minimum_iv(func, xm0, xl0, xr0, xmin, xmax, factor, args, maxiter): if not np.all(factor > 1): raise ValueError('All elements of `factor` must be greater than 1.') - # Default choices for xl or xr might have exceeded xmin or xmax. Adjust - # to make sure this doesn't happen. We replace with copies because xl, and xr - # are read-only views produced by broadcast_arrays. + # Calculate default values of xl0 and/or xr0 if they have not been supplied + # by the user. We need to be careful to ensure xl0 and xr0 are not outside + # of (xmin, xmax). if xl0_not_supplied: - xl0 = xl0.copy() - cond = ~np.isinf(xmin) & (xl0 < xmin) - xl0[cond] = ( - xm0[cond] - xmin[cond] - ) / np.array(16, dtype=xl0.dtype) + xl0 = xm0 - np.minimum((xm0 - xmin)/16, 0.5) if xr0_not_supplied: - xr0 = xr0.copy() - cond = ~np.isinf(xmax) & (xmax < xr0) - xr0[cond] = ( - xmax[cond] - xm0[cond] - ) / np.array(16, dtype=xr0.dtype) + xr0 = xm0 + np.minimum((xmax - xm0)/16, 0.5) maxiter = np.asarray(maxiter) message = '`maxiter` must be a non-negative integer.' diff --git a/scipy/optimize/tests/test_bracket.py b/scipy/optimize/tests/test_bracket.py index 0a55a8b13d6b..af8e6c8842d4 100644 --- a/scipy/optimize/tests/test_bracket.py +++ b/scipy/optimize/tests/test_bracket.py @@ -778,3 +778,29 @@ def f(x, c): [result.fl, result.fm, result.fr], [f(xl0, *args), f(xm0, *args), f(xr0, *args)], ) + + def test_gh_20562_left(self): + # Regression test for https://github.com/scipy/scipy/issues/20562 + # minimum of f in [xmin, xmax] is at xmin. + xmin, xmax = 0.21933608, 1.39713606 + + def f(x): + log_a, log_b = np.log([xmin, xmax]) + return -((log_b - log_a)*x)**-1 + + result = _bracket_minimum(f, 0.5535723499480897, xmin=xmin, xmax=xmax) + assert not result.success + assert xmin == result.xl + + def test_gh_20562_right(self): + # Regression test for https://github.com/scipy/scipy/issues/20562 + # minimum of f in [xmin, xmax] is at xmax. + xmin, xmax = -1.39713606, -0.21933608, + + def f(x): + log_a, log_b = np.log([-xmax, -xmin]) + return ((log_b - log_a)*x)**-1 + + result = _bracket_minimum(f, -0.5535723499480897, xmin=xmin, xmax=xmax) + assert not result.success + assert xmax == result.xr From dcd9e935bda596ce9c2176bf24e5ce0301fdab64 Mon Sep 17 00:00:00 2001 From: Jake Bowhay Date: Wed, 24 Apr 2024 07:36:40 +0100 Subject: [PATCH 40/64] DEP: linalg: remove cond / rcond kwargs from linalg.pinv and switch to kwarg-only --- scipy/linalg/_basic.py | 30 +----------------------------- scipy/linalg/tests/test_basic.py | 16 ---------------- 2 files changed, 1 insertion(+), 45 deletions(-) diff --git a/scipy/linalg/_basic.py b/scipy/linalg/_basic.py index f64c29ef013e..84e4cafdc4a6 100644 --- a/scipy/linalg/_basic.py +++ b/scipy/linalg/_basic.py @@ -14,7 +14,6 @@ from . import _decomp, _decomp_svd from ._solve_toeplitz import levinson from ._cythonized_array_utils import find_det_from_lu -from scipy._lib.deprecation import _NoValue, _deprecate_positional_args __all__ = ['solve', 'solve_triangular', 'solveh_banded', 'solve_banded', 'solve_toeplitz', 'solve_circulant', 'inv', 'det', 'lstsq', @@ -1345,9 +1344,7 @@ def lstsq(a, b, cond=None, overwrite_a=False, overwrite_b=False, lstsq.default_lapack_driver = 'gelsd' -@_deprecate_positional_args(version="1.14") -def pinv(a, *, atol=None, rtol=None, return_rank=False, check_finite=True, - cond=_NoValue, rcond=_NoValue): +def pinv(a, *, atol=None, rtol=None, return_rank=False, check_finite=True): """ Compute the (Moore-Penrose) pseudo-inverse of a matrix. @@ -1381,20 +1378,6 @@ def pinv(a, *, atol=None, rtol=None, return_rank=False, check_finite=True, Whether to check that the input matrix contains only finite numbers. Disabling may give a performance gain, but may result in problems (crashes, non-termination) if the inputs do contain infinities or NaNs. - cond, rcond : float, optional - In older versions, these values were meant to be used as ``atol`` with - ``rtol=0``. If both were given ``rcond`` overwrote ``cond`` and hence - the code was not correct. Thus using these are strongly discouraged and - the tolerances above are recommended instead. In fact, if provided, - atol, rtol takes precedence over these keywords. - - .. deprecated:: 1.7.0 - Deprecated in favor of ``rtol`` and ``atol`` parameters above and - will be removed in SciPy 1.14.0. - - .. versionchanged:: 1.3.0 - Previously the default cutoff value was just ``eps*f`` where ``f`` - was ``1e3`` for single precision and ``1e6`` for double precision. Returns ------- @@ -1465,17 +1448,6 @@ def pinv(a, *, atol=None, rtol=None, return_rank=False, check_finite=True, t = u.dtype.char.lower() maxS = np.max(s, initial=0.) - if rcond is not _NoValue or cond is not _NoValue: - warn('Use of the "cond" and "rcond" keywords are deprecated and ' - 'will be removed in SciPy 1.14.0. Use "atol" and ' - '"rtol" keywords instead', DeprecationWarning, stacklevel=2) - - # backwards compatible only atol and rtol are both missing - if ((rcond not in (_NoValue, None) or cond not in (_NoValue, None)) - and (atol is None) and (rtol is None)): - atol = rcond if rcond not in (_NoValue, None) else cond - rtol = 0. - atol = 0. if atol is None else atol rtol = max(a.shape) * np.finfo(t).eps if (rtol is None) else rtol diff --git a/scipy/linalg/tests/test_basic.py b/scipy/linalg/tests/test_basic.py index a99b127f4780..4449ad83f772 100644 --- a/scipy/linalg/tests/test_basic.py +++ b/scipy/linalg/tests/test_basic.py @@ -20,7 +20,6 @@ from scipy.linalg._testutils import assert_no_overwrite from scipy._lib._testutils import check_free_memory, IS_MUSL from scipy.linalg.blas import HAS_ILP64 -from scipy._lib.deprecation import _NoValue REAL_DTYPES = (np.float32, np.float64, np.longdouble) COMPLEX_DTYPES = (np.complex64, np.complex128, np.clongdouble) @@ -1535,21 +1534,6 @@ def test_atol_rtol(self): assert_allclose(np.linalg.norm(adiff1), 4.233, rtol=0.01) assert_allclose(np.linalg.norm(adiff2), 4.233, rtol=0.01) - @pytest.mark.parametrize("cond", [1, None, _NoValue]) - @pytest.mark.parametrize("rcond", [1, None, _NoValue]) - def test_cond_rcond_deprecation(self, cond, rcond): - if cond is _NoValue and rcond is _NoValue: - # the defaults if cond/rcond aren't set -> no warning - pinv(np.ones((2,2)), cond=cond, rcond=rcond) - else: - # at least one of cond/rcond has a user-supplied value -> warn - with pytest.deprecated_call(match='"cond" and "rcond"'): - pinv(np.ones((2,2)), cond=cond, rcond=rcond) - - def test_positional_deprecation(self): - with pytest.deprecated_call(match="use keyword arguments"): - pinv(np.ones((2,2)), 0., 1e-10) - @pytest.mark.parametrize('dt', [float, np.float32, complex, np.complex64]) def test_empty(self, dt): a = np.empty((0, 0), dtype=dt) From 3b1c1f06d728a421748b43c5c65c9ca72f444f77 Mon Sep 17 00:00:00 2001 From: Matti Picus Date: Wed, 24 Apr 2024 22:31:38 +1000 Subject: [PATCH 41/64] update openblas to 0.3.27 --- tools/openblas_support.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/openblas_support.py b/tools/openblas_support.py index bdb42f3da230..37b60dd69e55 100644 --- a/tools/openblas_support.py +++ b/tools/openblas_support.py @@ -13,8 +13,8 @@ from urllib.request import urlopen, Request from urllib.error import HTTPError -OPENBLAS_V = '0.3.26.dev' -OPENBLAS_LONG = 'v0.3.26-382-gb1e8ba50' +OPENBLAS_V = '0.3.27' +OPENBLAS_LONG = 'v0.3.27' BASE_LOC = 'https://anaconda.org/multibuild-wheels-staging/openblas-libs' NIGHTLY_BASE_LOC = ( 'https://anaconda.org/scientific-python-nightly-wheels/openblas-libs' From 8acedb1ba7d54ef63d742ed71c7c7c357d2601f8 Mon Sep 17 00:00:00 2001 From: Andrew Nelson Date: Wed, 24 Apr 2024 23:12:29 +1000 Subject: [PATCH 42/64] DOC: change approx_fprime doctest (#20568) --- scipy/optimize/_optimize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scipy/optimize/_optimize.py b/scipy/optimize/_optimize.py index e0b634981ea9..cd189e16a109 100644 --- a/scipy/optimize/_optimize.py +++ b/scipy/optimize/_optimize.py @@ -998,7 +998,7 @@ def approx_fprime(xk, f, epsilon=_epsilon, *args): >>> c0, c1 = (1, 200) >>> eps = np.sqrt(np.finfo(float).eps) >>> optimize.approx_fprime(x, func, [eps, np.sqrt(200) * eps], c0, c1) - array([ 2. , 400.00004198]) + array([ 2. , 400.00004208]) """ xk = np.asarray(xk, float) From deb3c184592ffcea99ce44a760220f7f29b718b1 Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Wed, 24 Apr 2024 11:43:15 -0700 Subject: [PATCH 43/64] TST: stats.skew: assert_equal -> xp_assert_equal as appropriate --- scipy/stats/tests/test_stats.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scipy/stats/tests/test_stats.py b/scipy/stats/tests/test_stats.py index 2bba0dd380f3..ce37e481e318 100644 --- a/scipy/stats/tests/test_stats.py +++ b/scipy/stats/tests/test_stats.py @@ -3433,16 +3433,16 @@ def test_skew_constant_value(self, xp): # exact (gh-13245) with pytest.warns(RuntimeWarning, match="Precision loss occurred"): a = xp.asarray([-0.27829495]*10) # xp.repeat not currently available - assert_equal(stats.skew(a), xp.asarray(xp.nan)) - assert_equal(stats.skew(a*2.**50), xp.asarray(xp.nan)) - assert_equal(stats.skew(a/2.**50), xp.asarray(xp.nan)) - assert_equal(stats.skew(a, bias=False), xp.asarray(xp.nan)) + xp_assert_equal(stats.skew(a), xp.asarray(xp.nan)) + xp_assert_equal(stats.skew(a*2.**50), xp.asarray(xp.nan)) + xp_assert_equal(stats.skew(a/2.**50), xp.asarray(xp.nan)) + xp_assert_equal(stats.skew(a, bias=False), xp.asarray(xp.nan)) # # similarly, from gh-11086: a = xp.asarray([14.3]*7) - assert_equal(stats.skew(a), xp.asarray(xp.nan)) + xp_assert_equal(stats.skew(a), xp.asarray(xp.nan)) a = 1. + xp.arange(-3., 4)*1e-16 - assert_equal(stats.skew(a), xp.asarray(xp.nan)) + xp_assert_equal(stats.skew(a), xp.asarray(xp.nan)) @array_api_compatible def test_precision_loss_gh15554(self, xp): From 91ac9d79249c58ff9ce3ead47092a003c0e429e6 Mon Sep 17 00:00:00 2001 From: lucascolley Date: Wed, 24 Apr 2024 22:01:27 +0100 Subject: [PATCH 44/64] DEV: add unicode check to pre-commit-hook [lint only] --- tools/pre-commit-hook.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/pre-commit-hook.py b/tools/pre-commit-hook.py index 5d78b77fa304..4b25e2074e9f 100755 --- a/tools/pre-commit-hook.py +++ b/tools/pre-commit-hook.py @@ -20,6 +20,14 @@ linter = [f for f in linters if os.path.exists(f)][0] +unicode_checks = [ + '../../tools/unicode-check.py', + 'tools/unicode-check.py', + 'unicode-check.py' # in case pre-commit hook is run from tools dir +] + +unicode_check = [f for f in unicode_checks if os.path.exists(f)][0] + # names of files that were staged # add '*.pxd', '*.pxi' once cython-lint supports it @@ -86,3 +94,8 @@ print(' ./tools/pre-commit-hook.py --fix') sys.exit(p.returncode) + +p = subprocess.run(unicode_check, cwd=work_dir) + +if p.returncode != 0: + sys.exit(p.returncode) From d4c46ded1f6269b0451b072f04d9a546cf736136 Mon Sep 17 00:00:00 2001 From: Jake Bowhay Date: Thu, 25 Apr 2024 10:34:52 +0100 Subject: [PATCH 45/64] DEP: signal: remove nyq / Hz kwargs in firwin* and switch to kwarg-only --- scipy/signal/_fir_filter_design.py | 78 +++----------------- scipy/signal/tests/test_fir_filter_design.py | 55 -------------- 2 files changed, 11 insertions(+), 122 deletions(-) diff --git a/scipy/signal/_fir_filter_design.py b/scipy/signal/_fir_filter_design.py index 3982ff749cc0..3d1c9c13dc26 100644 --- a/scipy/signal/_fir_filter_design.py +++ b/scipy/signal/_fir_filter_design.py @@ -10,7 +10,6 @@ from scipy.special import sinc from scipy.linalg import (toeplitz, hankel, solve, LinAlgError, LinAlgWarning, lstsq) -from scipy._lib.deprecation import _NoValue, _deprecate_positional_args from scipy.signal._arraytools import _validate_fs from . import _sigtools @@ -19,25 +18,6 @@ 'firwin', 'firwin2', 'remez', 'firls', 'minimum_phase'] -def _get_fs(fs, nyq): - """ - Utility for replacing the argument 'nyq' (with default 1) with 'fs'. - """ - if nyq is _NoValue and fs is None: - fs = 2 - elif nyq is not _NoValue: - if fs is not None: - raise ValueError("Values cannot be given for both 'nyq' and 'fs'.") - msg = ("Keyword argument 'nyq' is deprecated in favour of 'fs' and " - "will be removed in SciPy 1.14.0.") - warnings.warn(msg, DeprecationWarning, stacklevel=3) - if nyq is None: - fs = 2 - else: - fs = 2*nyq - return fs - - # Some notes on function parameters: # # `cutoff` and `width` are given as numbers between 0 and 1. These are @@ -268,9 +248,8 @@ def kaiserord(ripple, width): return int(ceil(numtaps)), beta -@_deprecate_positional_args(version="1.14") def firwin(numtaps, cutoff, *, width=None, window='hamming', pass_zero=True, - scale=True, nyq=_NoValue, fs=None): + scale=True, fs=None): """ FIR filter design using the window method. @@ -320,13 +299,6 @@ def firwin(numtaps, cutoff, *, width=None, window='hamming', pass_zero=True, `fs/2` (i.e the filter is a single band highpass filter); center of first passband otherwise - nyq : float, optional, deprecated - This is the Nyquist frequency. Each frequency in `cutoff` must be - between 0 and `nyq`. Default is 1. - - .. deprecated:: 1.0.0 - `firwin` keyword argument `nyq` is deprecated in favour of `fs` and - will be removed in SciPy 1.14.0. fs : float, optional The sampling frequency of the signal. Each frequency in `cutoff` must be between 0 and ``fs/2``. Default is 2. @@ -397,8 +369,9 @@ def firwin(numtaps, cutoff, *, width=None, window='hamming', pass_zero=True, # The major enhancements to this function added in November 2010 were # developed by Tom Krauss (see ticket #902). fs = _validate_fs(fs, allow_none=True) + fs = 2 if fs is None else fs - nyq = 0.5 * _get_fs(fs, nyq) + nyq = 0.5 * fs cutoff = np.atleast_1d(cutoff) / float(nyq) @@ -493,8 +466,7 @@ def firwin(numtaps, cutoff, *, width=None, window='hamming', pass_zero=True, # Original version of firwin2 from scipy ticket #457, submitted by "tash". # # Rewritten by Warren Weckesser, 2010. -@_deprecate_positional_args(version="1.14") -def firwin2(numtaps, freq, gain, *, nfreqs=None, window='hamming', nyq=_NoValue, +def firwin2(numtaps, freq, gain, *, nfreqs=None, window='hamming', antisymmetric=False, fs=None): """ FIR filter design using the window method. @@ -529,13 +501,6 @@ def firwin2(numtaps, freq, gain, *, nfreqs=None, window='hamming', nyq=_NoValue, Window function to use. Default is "hamming". See `scipy.signal.get_window` for the complete list of possible values. If None, no window function is applied. - nyq : float, optional, deprecated - This is the Nyquist frequency. Each frequency in `freq` must be - between 0 and `nyq`. Default is 1. - - .. deprecated:: 1.0.0 - `firwin2` keyword argument `nyq` is deprecated in favour of `fs` and - will be removed in SciPy 1.14.0. antisymmetric : bool, optional Whether resulting impulse response is symmetric/antisymmetric. See Notes for more details. @@ -603,7 +568,8 @@ def firwin2(numtaps, freq, gain, *, nfreqs=None, window='hamming', nyq=_NoValue, """ fs = _validate_fs(fs, allow_none=True) - nyq = 0.5 * _get_fs(fs, nyq) + fs = 2 if fs is None else fs + nyq = 0.5 * fs if len(freq) != len(gain): raise ValueError('freq and gain must be of same length.') @@ -697,8 +663,7 @@ def firwin2(numtaps, freq, gain, *, nfreqs=None, window='hamming', nyq=_NoValue, return out -@_deprecate_positional_args(version="1.14") -def remez(numtaps, bands, desired, *, weight=None, Hz=_NoValue, type='bandpass', +def remez(numtaps, bands, desired, *, weight=None, type='bandpass', maxiter=25, grid_density=16, fs=None): """ Calculate the minimax optimal filter using the Remez exchange algorithm. @@ -723,12 +688,6 @@ def remez(numtaps, bands, desired, *, weight=None, Hz=_NoValue, type='bandpass', weight : array_like, optional A relative weighting to give to each band region. The length of `weight` has to be half the length of `bands`. - Hz : scalar, optional, deprecated - The sampling frequency in Hz. Default is 1. - - .. deprecated:: 1.0.0 - `remez` keyword argument `Hz` is deprecated in favour of `fs` and - will be removed in SciPy 1.14.0. type : {'bandpass', 'differentiator', 'hilbert'}, optional The type of filter: @@ -857,15 +816,7 @@ def remez(numtaps, bands, desired, *, weight=None, Hz=_NoValue, type='bandpass', """ fs = _validate_fs(fs, allow_none=True) - if Hz is _NoValue and fs is None: - fs = 1.0 - elif Hz is not _NoValue: - if fs is not None: - raise ValueError("Values cannot be given for both 'Hz' and 'fs'.") - msg = ("'remez' keyword argument 'Hz' is deprecated in favour of 'fs'" - " and will be removed in SciPy 1.14.0.") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - fs = Hz + fs = 1.0 if fs is None else fs # Convert type try: @@ -883,8 +834,7 @@ def remez(numtaps, bands, desired, *, weight=None, Hz=_NoValue, type='bandpass', maxiter, grid_density) -@_deprecate_positional_args(version="1.14") -def firls(numtaps, bands, desired, *, weight=None, nyq=_NoValue, fs=None): +def firls(numtaps, bands, desired, *, weight=None, fs=None): """ FIR filter design using least-squares error minimization. @@ -914,13 +864,6 @@ def firls(numtaps, bands, desired, *, weight=None, nyq=_NoValue, fs=None): A relative weighting to give to each band region when solving the least squares problem. `weight` has to be half the size of `bands`. - nyq : float, optional, deprecated - This is the Nyquist frequency. Each frequency in `bands` must be - between 0 and `nyq` (inclusive). Default is 1. - - .. deprecated:: 1.0.0 - `firls` keyword argument `nyq` is deprecated in favour of `fs` and - will be removed in SciPy 1.14.0. fs : float, optional The sampling frequency of the signal. Each frequency in `bands` must be between 0 and ``fs/2`` (inclusive). Default is 2. @@ -1002,7 +945,8 @@ def firls(numtaps, bands, desired, *, weight=None, nyq=_NoValue, fs=None): """ fs = _validate_fs(fs, allow_none=True) - nyq = 0.5 * _get_fs(fs, nyq) + fs = 2 if fs is None else fs + nyq = 0.5 * fs numtaps = int(numtaps) if numtaps % 2 == 0 or numtaps < 1: diff --git a/scipy/signal/tests/test_fir_filter_design.py b/scipy/signal/tests/test_fir_filter_design.py index ea50d402d7c6..d9fc22982965 100644 --- a/scipy/signal/tests/test_fir_filter_design.py +++ b/scipy/signal/tests/test_fir_filter_design.py @@ -234,11 +234,6 @@ def test_fs_nyq(self): freqs, response = freqz(taps, worN=np.pi*freq_samples/nyquist) assert_array_almost_equal(np.abs(response), [0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0], decimal=5) - with np.testing.suppress_warnings() as sup: - sup.filter(DeprecationWarning, "Keyword argument 'nyq'") - taps2 = firwin(ntaps, cutoff=[300, 700], window=('kaiser', beta), - pass_zero=False, scale=False, nyq=nyquist) - assert_allclose(taps2, taps) def test_bad_cutoff(self): """Test that invalid cutoff argument raises ValueError.""" @@ -256,10 +251,6 @@ def test_bad_cutoff(self): # 2D array not allowed. assert_raises(ValueError, firwin, 99, [[0.1, 0.2],[0.3, 0.4]]) # cutoff values must be less than nyq. - with np.testing.suppress_warnings() as sup: - sup.filter(DeprecationWarning, "Keyword argument 'nyq'") - assert_raises(ValueError, firwin, 99, 50.0, nyq=40) - assert_raises(ValueError, firwin, 99, [10, 20, 30], nyq=25) assert_raises(ValueError, firwin, 99, 50.0, fs=80) assert_raises(ValueError, firwin, 99, [10, 20, 30], fs=50) @@ -282,12 +273,6 @@ def test_bad_pass_zero(self): with assert_raises(ValueError, match='must have at least two'): firwin(41, [0.5], pass_zero=pass_zero) - def test_firwin_deprecations(self): - with pytest.deprecated_call(match="argument 'nyq' is deprecated"): - firwin(1, 1, nyq=10) - with pytest.deprecated_call(match="use keyword arguments"): - firwin(58, 0.1, 0.03) - def test_fs_validation(self): with pytest.raises(ValueError, match="Sampling.*single scalar"): firwin2(51, .5, 1, fs=np.array([10, 20])) @@ -427,10 +412,6 @@ def test_fs_nyq(self): taps1 = firwin2(80, [0.0, 0.5, 1.0], [1.0, 1.0, 0.0]) taps2 = firwin2(80, [0.0, 30.0, 60.0], [1.0, 1.0, 0.0], fs=120.0) assert_array_almost_equal(taps1, taps2) - with np.testing.suppress_warnings() as sup: - sup.filter(DeprecationWarning, "Keyword argument 'nyq'") - taps2 = firwin2(80, [0.0, 30.0, 60.0], [1.0, 1.0, 0.0], nyq=60.0) - assert_array_almost_equal(taps1, taps2) def test_tuple(self): taps1 = firwin2(150, (0.0, 0.5, 0.5, 1.0), (1.0, 1.0, 0.0, 0.0)) @@ -443,13 +424,6 @@ def test_input_modyfication(self): firwin2(80, freq1, [1.0, 1.0, 0.0, 0.0]) assert_equal(freq1, freq2) - def test_firwin2_deprecations(self): - with pytest.deprecated_call(match="argument 'nyq' is deprecated"): - firwin2(1, [0, 10], [1, 1], nyq=10) - with pytest.deprecated_call(match="use keyword arguments"): - # from test04 - firwin2(5, [0.0, 0.5, 0.5, 1.0], [1.0, 1.0, 0.0, 0.0], 8193, None) - class TestRemez: @@ -491,10 +465,6 @@ def test_compare(self): -0.003530911231040, 0.193140296954975, 0.373400753484939, 0.373400753484939, 0.193140296954975, -0.003530911231040, -0.075943803756711, -0.041314581814658, 0.024590270518440] - with np.testing.suppress_warnings() as sup: - sup.filter(DeprecationWarning, "'remez'") - h = remez(12, [0, 0.3, 0.5, 1], [1, 0], Hz=2.) - assert_allclose(h, k) h = remez(12, [0, 0.3, 0.5, 1], [1, 0], fs=2.) assert_allclose(h, k) @@ -505,18 +475,8 @@ def test_compare(self): 0.129770906801075, -0.103908158578635, 0.073641298245579, -0.043276706138248, 0.016849978528150, 0.002879152556419, -0.014644062687875, 0.018704846485491, -0.038976016082299] - with np.testing.suppress_warnings() as sup: - sup.filter(DeprecationWarning, "'remez'") - assert_allclose(remez(21, [0, 0.8, 0.9, 1], [0, 1], Hz=2.), h) assert_allclose(remez(21, [0, 0.8, 0.9, 1], [0, 1], fs=2.), h) - def test_remez_deprecations(self): - with pytest.deprecated_call(match="'remez' keyword argument 'Hz'"): - remez(12, [0, 0.3, 0.5, 1], [1, 0], Hz=2.) - with pytest.deprecated_call(match="use keyword arguments"): - # from test_hilbert - remez(11, [0.1, 0.4], [1], None) - def test_fs_validation(self): with pytest.raises(ValueError, match="Sampling.*single scalar"): remez(11, .1, 1, fs=np.array([10, 20])) @@ -607,14 +567,6 @@ def test_compare(self): 1.156090832768218] assert_allclose(taps, known_taps) - with np.testing.suppress_warnings() as sup: - sup.filter(DeprecationWarning, "Keyword argument 'nyq'") - taps = firls(7, (0, 1, 2, 3, 4, 5), [1, 0, 0, 1, 1, 0], nyq=10) - assert_allclose(taps, known_taps) - - with pytest.raises(ValueError, match='between 0 and 1'): - firls(7, [0, 1], [0, 1], nyq=0.5) - def test_rank_deficient(self): # solve() runs but warns (only sometimes, so here we don't use match) x = firls(21, [0, 0.1, 0.9, 1], [1, 1, 0, 0]) @@ -633,13 +585,6 @@ def test_rank_deficient(self): assert mask.sum() > 3 assert_allclose(np.abs(h[mask]), 0., atol=1e-4) - def test_firls_deprecations(self): - with pytest.deprecated_call(match="argument 'nyq' is deprecated"): - firls(1, (0, 1), (0, 0), nyq=10) - with pytest.deprecated_call(match="use keyword arguments"): - # from test_firls - firls(11, [0, 0.1, 0.4, 0.5], [1, 1, 0, 0], None) - def test_fs_validation(self): with pytest.raises(ValueError, match="Sampling.*single scalar"): firls(11, .1, 1, fs=np.array([10, 20])) From 73f719a37c57e1e64406f976de92e4574fdcf0de Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Tue, 23 Apr 2024 14:57:30 +0200 Subject: [PATCH 46/64] MAINT: vendor Tempita in `scipy/_build_utils` This is an unmodified copy from `Cython/Tempita/` as of commit `023d4af35` in Cython's master branch (21 April 2024). Tempita hasn't been maintained independently on PyPI for 10+ years; we've used the Cython version which is semi-public (undocumented) for a while, with the note that we should vendor it if that ever fails. It now failed once (see scipy#20535), and conceptually that makes sense - this is the only place where we don't invoke the `cython` executable but actually do `import Cython`. That can fail in a distro setup where Cython is installed for a different Python interpreter than the active one. The license file was taken over from the `maintenance/1.26.x` branch in the `numpy` repo - which came from the original upstream repo for Tempita (now disappeared) as discussed in numpy#8096. See scipy#20572 for more clarification about the MIT license that this code is under. --- LICENSES_bundled.txt | 5 + scipy/_build_utils/tempita.py | 4 +- scipy/_build_utils/tempita/LICENSE.txt | 20 + scipy/_build_utils/tempita/__init__.py | 4 + scipy/_build_utils/tempita/_looper.py | 156 ++++ scipy/_build_utils/tempita/_tempita.py | 1092 ++++++++++++++++++++++++ tools/lint.toml | 5 +- 7 files changed, 1282 insertions(+), 4 deletions(-) create mode 100644 scipy/_build_utils/tempita/LICENSE.txt create mode 100644 scipy/_build_utils/tempita/__init__.py create mode 100644 scipy/_build_utils/tempita/_looper.py create mode 100644 scipy/_build_utils/tempita/_tempita.py diff --git a/LICENSES_bundled.txt b/LICENSES_bundled.txt index 9a3943898822..4828b64e30aa 100644 --- a/LICENSES_bundled.txt +++ b/LICENSES_bundled.txt @@ -281,3 +281,8 @@ Name: array-api-compat Files: scipy/_lib/array-api-compat/* License: MIT For details, see scipy/_lib/array-api-compat/LICENCE + +Name: Tempita +Files: scipy/_build_utils/tempita/* +License: MIT + For details, see scipy/_build_utils/tempita/LICENCE.txt diff --git a/scipy/_build_utils/tempita.py b/scipy/_build_utils/tempita.py index b9e74032a026..a08374dd7c65 100644 --- a/scipy/_build_utils/tempita.py +++ b/scipy/_build_utils/tempita.py @@ -3,9 +3,7 @@ import os import argparse -from Cython import Tempita as tempita -# XXX: If this import ever fails (does it really?), vendor either -# cython.tempita or numpy/npy_tempita. +import tempita def process_tempita(fromfile, outfile=None): diff --git a/scipy/_build_utils/tempita/LICENSE.txt b/scipy/_build_utils/tempita/LICENSE.txt new file mode 100644 index 000000000000..0ba6f23c440f --- /dev/null +++ b/scipy/_build_utils/tempita/LICENSE.txt @@ -0,0 +1,20 @@ +Copyright (c) 2008 Ian Bicking and Contributors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/scipy/_build_utils/tempita/__init__.py b/scipy/_build_utils/tempita/__init__.py new file mode 100644 index 000000000000..41a0ce3d0efa --- /dev/null +++ b/scipy/_build_utils/tempita/__init__.py @@ -0,0 +1,4 @@ +# The original Tempita implements all of its templating code here. +# Moved it to _tempita.py to make the compilation portable. + +from ._tempita import * diff --git a/scipy/_build_utils/tempita/_looper.py b/scipy/_build_utils/tempita/_looper.py new file mode 100644 index 000000000000..4864f2949605 --- /dev/null +++ b/scipy/_build_utils/tempita/_looper.py @@ -0,0 +1,156 @@ +""" +Helper for looping over sequences, particular in templates. + +Often in a loop in a template it's handy to know what's next up, +previously up, if this is the first or last item in the sequence, etc. +These can be awkward to manage in a normal Python loop, but using the +looper you can get a better sense of the context. Use like:: + + >>> for loop, item in looper(['a', 'b', 'c']): + ... print loop.number, item + ... if not loop.last: + ... print '---' + 1 a + --- + 2 b + --- + 3 c + +""" + +basestring_ = (bytes, str) + +__all__ = ['looper'] + + +class looper: + """ + Helper for looping (particularly in templates) + + Use this like:: + + for loop, item in looper(seq): + if loop.first: + ... + """ + + def __init__(self, seq): + self.seq = seq + + def __iter__(self): + return looper_iter(self.seq) + + def __repr__(self): + return '<%s for %r>' % ( + self.__class__.__name__, self.seq) + + +class looper_iter: + + def __init__(self, seq): + self.seq = list(seq) + self.pos = 0 + + def __iter__(self): + return self + + def __next__(self): + if self.pos >= len(self.seq): + raise StopIteration + result = loop_pos(self.seq, self.pos), self.seq[self.pos] + self.pos += 1 + return result + + +class loop_pos: + + def __init__(self, seq, pos): + self.seq = seq + self.pos = pos + + def __repr__(self): + return '' % ( + self.seq[self.pos], self.pos) + + def index(self): + return self.pos + index = property(index) + + def number(self): + return self.pos + 1 + number = property(number) + + def item(self): + return self.seq[self.pos] + item = property(item) + + def __next__(self): + try: + return self.seq[self.pos + 1] + except IndexError: + return None + __next__ = property(__next__) + + def previous(self): + if self.pos == 0: + return None + return self.seq[self.pos - 1] + previous = property(previous) + + def odd(self): + return not self.pos % 2 + odd = property(odd) + + def even(self): + return self.pos % 2 + even = property(even) + + def first(self): + return self.pos == 0 + first = property(first) + + def last(self): + return self.pos == len(self.seq) - 1 + last = property(last) + + def length(self): + return len(self.seq) + length = property(length) + + def first_group(self, getter=None): + """ + Returns true if this item is the start of a new group, + where groups mean that some attribute has changed. The getter + can be None (the item itself changes), an attribute name like + ``'.attr'``, a function, or a dict key or list index. + """ + if self.first: + return True + return self._compare_group(self.item, self.previous, getter) + + def last_group(self, getter=None): + """ + Returns true if this item is the end of a new group, + where groups mean that some attribute has changed. The getter + can be None (the item itself changes), an attribute name like + ``'.attr'``, a function, or a dict key or list index. + """ + if self.last: + return True + return self._compare_group(self.item, self.__next__, getter) + + def _compare_group(self, item, other, getter): + if getter is None: + return item != other + elif (isinstance(getter, basestring_) + and getter.startswith('.')): + getter = getter[1:] + if getter.endswith('()'): + getter = getter[:-2] + return getattr(item, getter)() != getattr(other, getter)() + else: + return getattr(item, getter) != getattr(other, getter) + elif hasattr(getter, '__call__'): + return getter(item) != getter(other) + else: + return item[getter] != other[getter] diff --git a/scipy/_build_utils/tempita/_tempita.py b/scipy/_build_utils/tempita/_tempita.py new file mode 100644 index 000000000000..c5269f25ff39 --- /dev/null +++ b/scipy/_build_utils/tempita/_tempita.py @@ -0,0 +1,1092 @@ +""" +A small templating language + +This implements a small templating language. This language implements +if/elif/else, for/continue/break, expressions, and blocks of Python +code. The syntax is:: + + {{any expression (function calls etc)}} + {{any expression | filter}} + {{for x in y}}...{{endfor}} + {{if x}}x{{elif y}}y{{else}}z{{endif}} + {{py:x=1}} + {{py: + def foo(bar): + return 'baz' + }} + {{default var = default_value}} + {{# comment}} + +You use this with the ``Template`` class or the ``sub`` shortcut. +The ``Template`` class takes the template string and the name of +the template (for errors) and a default namespace. Then (like +``string.Template``) you can call the ``tmpl.substitute(**kw)`` +method to make a substitution (or ``tmpl.substitute(a_dict)``). + +``sub(content, **kw)`` substitutes the template immediately. You +can use ``__name='tmpl.html'`` to set the name of the template. + +If there are syntax errors ``TemplateError`` will be raised. +""" + + +import re +import sys +import os +import tokenize +from io import StringIO + +from ._looper import looper + +__all__ = ['TemplateError', 'Template', 'sub', 'bunch'] + +in_re = re.compile(r'\s+in\s+') +var_re = re.compile(r'^[a-z_][a-z0-9_]*$', re.I) +basestring_ = (bytes, str) + +def coerce_text(v): + if not isinstance(v, basestring_): + if hasattr(v, '__str__'): + return str(v) + else: + return bytes(v) + return v + +class TemplateError(Exception): + """Exception raised while parsing a template + """ + + def __init__(self, message, position, name=None): + Exception.__init__(self, message) + self.position = position + self.name = name + + def __str__(self): + msg = ' '.join(self.args) + if self.position: + msg = '%s at line %s column %s' % ( + msg, self.position[0], self.position[1]) + if self.name: + msg += ' in %s' % self.name + return msg + + +class _TemplateContinue(Exception): + pass + + +class _TemplateBreak(Exception): + pass + + +def get_file_template(name, from_template): + path = os.path.join(os.path.dirname(from_template.name), name) + return from_template.__class__.from_filename( + path, namespace=from_template.namespace, + get_template=from_template.get_template) + + +class Template: + + default_namespace = { + 'start_braces': '{{', + 'end_braces': '}}', + 'looper': looper, + } + + default_encoding = 'utf8' + default_inherit = None + + def __init__(self, content, name=None, namespace=None, stacklevel=None, + get_template=None, default_inherit=None, line_offset=0, + delimiters=None, delimeters=None): + self.content = content + + # set delimiters + if delimeters: + import warnings + warnings.warn( + "'delimeters' kwarg is being deprecated in favor of correctly" + " spelled 'delimiters'. Please adjust your code.", + DeprecationWarning + ) + if delimiters is None: + delimiters = delimeters + if delimiters is None: + delimiters = (self.default_namespace['start_braces'], + self.default_namespace['end_braces']) + else: + #assert len(delimiters) == 2 and all([isinstance(delimiter, basestring) + # for delimiter in delimiters]) + self.default_namespace = self.__class__.default_namespace.copy() + self.default_namespace['start_braces'] = delimiters[0] + self.default_namespace['end_braces'] = delimiters[1] + self.delimiters = self.delimeters = delimiters # Keep a legacy read-only copy, but don't use it. + + self._unicode = isinstance(content, str) + if name is None and stacklevel is not None: + try: + caller = sys._getframe(stacklevel) + except ValueError: + pass + else: + globals = caller.f_globals + lineno = caller.f_lineno + if '__file__' in globals: + name = globals['__file__'] + if name.endswith('.pyc') or name.endswith('.pyo'): + name = name[:-1] + elif '__name__' in globals: + name = globals['__name__'] + else: + name = '' + if lineno: + name += ':%s' % lineno + self.name = name + self._parsed = parse(content, name=name, line_offset=line_offset, delimiters=self.delimiters) + if namespace is None: + namespace = {} + self.namespace = namespace + self.get_template = get_template + if default_inherit is not None: + self.default_inherit = default_inherit + + def from_filename(cls, filename, namespace=None, encoding=None, + default_inherit=None, get_template=get_file_template): + with open(filename, 'rb') as f: + c = f.read() + if encoding: + c = c.decode(encoding) + return cls(content=c, name=filename, namespace=namespace, + default_inherit=default_inherit, get_template=get_template) + + from_filename = classmethod(from_filename) + + def __repr__(self): + return '<%s %s name=%r>' % ( + self.__class__.__name__, + hex(id(self))[2:], self.name) + + def substitute(self, *args, **kw): + if args: + if kw: + raise TypeError( + "You can only give positional *or* keyword arguments") + if len(args) > 1: + raise TypeError( + "You can only give one positional argument") + if not hasattr(args[0], 'items'): + raise TypeError( + "If you pass in a single argument, you must pass in a dictionary-like object (with a .items() method); you gave %r" + % (args[0],)) + kw = args[0] + ns = kw + ns['__template_name__'] = self.name + if self.namespace: + ns.update(self.namespace) + result, defs, inherit = self._interpret(ns) + if not inherit: + inherit = self.default_inherit + if inherit: + result = self._interpret_inherit(result, defs, inherit, ns) + return result + + def _interpret(self, ns): + __traceback_hide__ = True + parts = [] + defs = {} + self._interpret_codes(self._parsed, ns, out=parts, defs=defs) + if '__inherit__' in defs: + inherit = defs.pop('__inherit__') + else: + inherit = None + return ''.join(parts), defs, inherit + + def _interpret_inherit(self, body, defs, inherit_template, ns): + __traceback_hide__ = True + if not self.get_template: + raise TemplateError( + 'You cannot use inheritance without passing in get_template', + position=None, name=self.name) + templ = self.get_template(inherit_template, self) + self_ = TemplateObject(self.name) + for name, value in defs.items(): + setattr(self_, name, value) + self_.body = body + ns = ns.copy() + ns['self'] = self_ + return templ.substitute(ns) + + def _interpret_codes(self, codes, ns, out, defs): + __traceback_hide__ = True + for item in codes: + if isinstance(item, basestring_): + out.append(item) + else: + self._interpret_code(item, ns, out, defs) + + def _interpret_code(self, code, ns, out, defs): + __traceback_hide__ = True + name, pos = code[0], code[1] + if name == 'py': + self._exec(code[2], ns, pos) + elif name == 'continue': + raise _TemplateContinue() + elif name == 'break': + raise _TemplateBreak() + elif name == 'for': + vars, expr, content = code[2], code[3], code[4] + expr = self._eval(expr, ns, pos) + self._interpret_for(vars, expr, content, ns, out, defs) + elif name == 'cond': + parts = code[2:] + self._interpret_if(parts, ns, out, defs) + elif name == 'expr': + parts = code[2].split('|') + base = self._eval(parts[0], ns, pos) + for part in parts[1:]: + func = self._eval(part, ns, pos) + base = func(base) + out.append(self._repr(base, pos)) + elif name == 'default': + var, expr = code[2], code[3] + if var not in ns: + result = self._eval(expr, ns, pos) + ns[var] = result + elif name == 'inherit': + expr = code[2] + value = self._eval(expr, ns, pos) + defs['__inherit__'] = value + elif name == 'def': + name = code[2] + signature = code[3] + parts = code[4] + ns[name] = defs[name] = TemplateDef(self, name, signature, body=parts, ns=ns, + pos=pos) + elif name == 'comment': + return + else: + assert 0, "Unknown code: %r" % name + + def _interpret_for(self, vars, expr, content, ns, out, defs): + __traceback_hide__ = True + for item in expr: + if len(vars) == 1: + ns[vars[0]] = item + else: + if len(vars) != len(item): + raise ValueError( + 'Need %i items to unpack (got %i items)' + % (len(vars), len(item))) + for name, value in zip(vars, item): + ns[name] = value + try: + self._interpret_codes(content, ns, out, defs) + except _TemplateContinue: + continue + except _TemplateBreak: + break + + def _interpret_if(self, parts, ns, out, defs): + __traceback_hide__ = True + # @@: if/else/else gets through + for part in parts: + assert not isinstance(part, basestring_) + name, pos = part[0], part[1] + if name == 'else': + result = True + else: + result = self._eval(part[2], ns, pos) + if result: + self._interpret_codes(part[3], ns, out, defs) + break + + def _eval(self, code, ns, pos): + __traceback_hide__ = True + try: + try: + value = eval(code, self.default_namespace, ns) + except SyntaxError as e: + raise SyntaxError( + 'invalid syntax in expression: %s' % code) + return value + except Exception as e: + if getattr(e, 'args', None): + arg0 = e.args[0] + else: + arg0 = coerce_text(e) + e.args = (self._add_line_info(arg0, pos),) + raise + + def _exec(self, code, ns, pos): + __traceback_hide__ = True + try: + exec(code, self.default_namespace, ns) + except Exception as e: + if e.args: + e.args = (self._add_line_info(e.args[0], pos),) + else: + e.args = (self._add_line_info(None, pos),) + raise + + def _repr(self, value, pos): + __traceback_hide__ = True + try: + if value is None: + return '' + if self._unicode: + try: + value = str(value) + except UnicodeDecodeError: + value = bytes(value) + else: + if not isinstance(value, basestring_): + value = coerce_text(value) + if (isinstance(value, str) + and self.default_encoding): + value = value.encode(self.default_encoding) + except Exception as e: + e.args = (self._add_line_info(e.args[0], pos),) + raise + else: + if self._unicode and isinstance(value, bytes): + if not self.default_encoding: + raise UnicodeDecodeError( + 'Cannot decode bytes value %r into unicode ' + '(no default_encoding provided)' % value) + try: + value = value.decode(self.default_encoding) + except UnicodeDecodeError as e: + raise UnicodeDecodeError( + e.encoding, + e.object, + e.start, + e.end, + e.reason + ' in string %r' % value) + elif not self._unicode and isinstance(value, str): + if not self.default_encoding: + raise UnicodeEncodeError( + 'Cannot encode unicode value %r into bytes ' + '(no default_encoding provided)' % value) + value = value.encode(self.default_encoding) + return value + + def _add_line_info(self, msg, pos): + msg = "%s at line %s column %s" % ( + msg, pos[0], pos[1]) + if self.name: + msg += " in file %s" % self.name + return msg + + +def sub(content, delimiters=None, **kw): + name = kw.get('__name') + delimeters = kw.pop('delimeters') if 'delimeters' in kw else None # for legacy code + tmpl = Template(content, name=name, delimiters=delimiters, delimeters=delimeters) + return tmpl.substitute(kw) + + +def paste_script_template_renderer(content, vars, filename=None): + tmpl = Template(content, name=filename) + return tmpl.substitute(vars) + + +class bunch(dict): + + def __init__(self, **kw): + for name, value in kw.items(): + setattr(self, name, value) + + def __setattr__(self, name, value): + self[name] = value + + def __getattr__(self, name): + try: + return self[name] + except KeyError: + raise AttributeError(name) + + def __getitem__(self, key): + if 'default' in self: + try: + return dict.__getitem__(self, key) + except KeyError: + return dict.__getitem__(self, 'default') + else: + return dict.__getitem__(self, key) + + def __repr__(self): + return '<%s %s>' % ( + self.__class__.__name__, + ' '.join(['%s=%r' % (k, v) for k, v in sorted(self.items())])) + + +class TemplateDef: + def __init__(self, template, func_name, func_signature, + body, ns, pos, bound_self=None): + self._template = template + self._func_name = func_name + self._func_signature = func_signature + self._body = body + self._ns = ns + self._pos = pos + self._bound_self = bound_self + + def __repr__(self): + return '' % ( + self._func_name, self._func_signature, + self._template.name, self._pos) + + def __str__(self): + return self() + + def __call__(self, *args, **kw): + values = self._parse_signature(args, kw) + ns = self._ns.copy() + ns.update(values) + if self._bound_self is not None: + ns['self'] = self._bound_self + out = [] + subdefs = {} + self._template._interpret_codes(self._body, ns, out, subdefs) + return ''.join(out) + + def __get__(self, obj, type=None): + if obj is None: + return self + return self.__class__( + self._template, self._func_name, self._func_signature, + self._body, self._ns, self._pos, bound_self=obj) + + def _parse_signature(self, args, kw): + values = {} + sig_args, var_args, var_kw, defaults = self._func_signature + extra_kw = {} + for name, value in kw.items(): + if not var_kw and name not in sig_args: + raise TypeError( + 'Unexpected argument %s' % name) + if name in sig_args: + values[sig_args] = value + else: + extra_kw[name] = value + args = list(args) + sig_args = list(sig_args) + while args: + while sig_args and sig_args[0] in values: + sig_args.pop(0) + if sig_args: + name = sig_args.pop(0) + values[name] = args.pop(0) + elif var_args: + values[var_args] = tuple(args) + break + else: + raise TypeError( + 'Extra position arguments: %s' + % ', '.join([repr(v) for v in args])) + for name, value_expr in defaults.items(): + if name not in values: + values[name] = self._template._eval( + value_expr, self._ns, self._pos) + for name in sig_args: + if name not in values: + raise TypeError( + 'Missing argument: %s' % name) + if var_kw: + values[var_kw] = extra_kw + return values + + +class TemplateObject: + + def __init__(self, name): + self.__name = name + self.get = TemplateObjectGetter(self) + + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self.__name) + + +class TemplateObjectGetter: + + def __init__(self, template_obj): + self.__template_obj = template_obj + + def __getattr__(self, attr): + return getattr(self.__template_obj, attr, Empty) + + def __repr__(self): + return '<%s around %r>' % (self.__class__.__name__, self.__template_obj) + + +class _Empty: + def __call__(self, *args, **kw): + return self + + def __str__(self): + return '' + + def __repr__(self): + return 'Empty' + + def __unicode__(self): + return '' + + def __iter__(self): + return iter(()) + + def __bool__(self): + return False + +Empty = _Empty() +del _Empty + +############################################################ +## Lexing and Parsing +############################################################ + + +def lex(s, name=None, trim_whitespace=True, line_offset=0, delimiters=None): + """ + Lex a string into chunks: + + >>> lex('hey') + ['hey'] + >>> lex('hey {{you}}') + ['hey ', ('you', (1, 7))] + >>> lex('hey {{') + Traceback (most recent call last): + ... + TemplateError: No }} to finish last expression at line 1 column 7 + >>> lex('hey }}') + Traceback (most recent call last): + ... + TemplateError: }} outside expression at line 1 column 7 + >>> lex('hey {{ {{') + Traceback (most recent call last): + ... + TemplateError: {{ inside expression at line 1 column 10 + + """ + if delimiters is None: + delimiters = ( Template.default_namespace['start_braces'], + Template.default_namespace['end_braces'] ) + in_expr = False + chunks = [] + last = 0 + last_pos = (line_offset + 1, 1) + + token_re = re.compile(r'%s|%s' % (re.escape(delimiters[0]), + re.escape(delimiters[1]))) + for match in token_re.finditer(s): + expr = match.group(0) + pos = find_position(s, match.end(), last, last_pos) + if expr == delimiters[0] and in_expr: + raise TemplateError('%s inside expression' % delimiters[0], + position=pos, + name=name) + elif expr == delimiters[1] and not in_expr: + raise TemplateError('%s outside expression' % delimiters[1], + position=pos, + name=name) + if expr == delimiters[0]: + part = s[last:match.start()] + if part: + chunks.append(part) + in_expr = True + else: + chunks.append((s[last:match.start()], last_pos)) + in_expr = False + last = match.end() + last_pos = pos + if in_expr: + raise TemplateError('No %s to finish last expression' % delimiters[1], + name=name, position=last_pos) + part = s[last:] + if part: + chunks.append(part) + if trim_whitespace: + chunks = trim_lex(chunks) + return chunks + +statement_re = re.compile(r'^(?:if |elif |for |def |inherit |default |py:)') +single_statements = ['else', 'endif', 'endfor', 'enddef', 'continue', 'break'] +trail_whitespace_re = re.compile(r'\n\r?[\t ]*$') +lead_whitespace_re = re.compile(r'^[\t ]*\n') + + +def trim_lex(tokens): + r""" + Takes a lexed set of tokens, and removes whitespace when there is + a directive on a line by itself: + + >>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False) + >>> tokens + [('if x', (1, 3)), '\nx\n', ('endif', (3, 3)), '\ny'] + >>> trim_lex(tokens) + [('if x', (1, 3)), 'x\n', ('endif', (3, 3)), 'y'] + """ + last_trim = None + for i, current in enumerate(tokens): + if isinstance(current, basestring_): + # we don't trim this + continue + item = current[0] + if not statement_re.search(item) and item not in single_statements: + continue + if not i: + prev = '' + else: + prev = tokens[i - 1] + if i + 1 >= len(tokens): + next_chunk = '' + else: + next_chunk = tokens[i + 1] + if (not isinstance(next_chunk, basestring_) + or not isinstance(prev, basestring_)): + continue + prev_ok = not prev or trail_whitespace_re.search(prev) + if i == 1 and not prev.strip(): + prev_ok = True + if last_trim is not None and last_trim + 2 == i and not prev.strip(): + prev_ok = 'last' + if (prev_ok + and (not next_chunk or lead_whitespace_re.search(next_chunk) + or (i == len(tokens) - 2 and not next_chunk.strip()))): + if prev: + if ((i == 1 and not prev.strip()) + or prev_ok == 'last'): + tokens[i - 1] = '' + else: + m = trail_whitespace_re.search(prev) + # +1 to leave the leading \n on: + prev = prev[:m.start() + 1] + tokens[i - 1] = prev + if next_chunk: + last_trim = i + if i == len(tokens) - 2 and not next_chunk.strip(): + tokens[i + 1] = '' + else: + m = lead_whitespace_re.search(next_chunk) + next_chunk = next_chunk[m.end():] + tokens[i + 1] = next_chunk + return tokens + + +def find_position(string, index, last_index, last_pos): + """Given a string and index, return (line, column)""" + lines = string.count('\n', last_index, index) + if lines > 0: + column = index - string.rfind('\n', last_index, index) + else: + column = last_pos[1] + (index - last_index) + return (last_pos[0] + lines, column) + + +def parse(s, name=None, line_offset=0, delimiters=None): + r""" + Parses a string into a kind of AST + + >>> parse('{{x}}') + [('expr', (1, 3), 'x')] + >>> parse('foo') + ['foo'] + >>> parse('{{if x}}test{{endif}}') + [('cond', (1, 3), ('if', (1, 3), 'x', ['test']))] + >>> parse('series->{{for x in y}}x={{x}}{{endfor}}') + ['series->', ('for', (1, 11), ('x',), 'y', ['x=', ('expr', (1, 27), 'x')])] + >>> parse('{{for x, y in z:}}{{continue}}{{endfor}}') + [('for', (1, 3), ('x', 'y'), 'z', [('continue', (1, 21))])] + >>> parse('{{py:x=1}}') + [('py', (1, 3), 'x=1')] + >>> parse('{{if x}}a{{elif y}}b{{else}}c{{endif}}') + [('cond', (1, 3), ('if', (1, 3), 'x', ['a']), ('elif', (1, 12), 'y', ['b']), ('else', (1, 23), None, ['c']))] + + Some exceptions:: + + >>> parse('{{continue}}') + Traceback (most recent call last): + ... + TemplateError: continue outside of for loop at line 1 column 3 + >>> parse('{{if x}}foo') + Traceback (most recent call last): + ... + TemplateError: No {{endif}} at line 1 column 3 + >>> parse('{{else}}') + Traceback (most recent call last): + ... + TemplateError: else outside of an if block at line 1 column 3 + >>> parse('{{if x}}{{for x in y}}{{endif}}{{endfor}}') + Traceback (most recent call last): + ... + TemplateError: Unexpected endif at line 1 column 25 + >>> parse('{{if}}{{endif}}') + Traceback (most recent call last): + ... + TemplateError: if with no expression at line 1 column 3 + >>> parse('{{for x y}}{{endfor}}') + Traceback (most recent call last): + ... + TemplateError: Bad for (no "in") in 'x y' at line 1 column 3 + >>> parse('{{py:x=1\ny=2}}') + Traceback (most recent call last): + ... + TemplateError: Multi-line py blocks must start with a newline at line 1 column 3 + """ + if delimiters is None: + delimiters = ( Template.default_namespace['start_braces'], + Template.default_namespace['end_braces'] ) + tokens = lex(s, name=name, line_offset=line_offset, delimiters=delimiters) + result = [] + while tokens: + next_chunk, tokens = parse_expr(tokens, name) + result.append(next_chunk) + return result + + +def parse_expr(tokens, name, context=()): + if isinstance(tokens[0], basestring_): + return tokens[0], tokens[1:] + expr, pos = tokens[0] + expr = expr.strip() + if expr.startswith('py:'): + expr = expr[3:].lstrip(' \t') + if expr.startswith('\n') or expr.startswith('\r'): + expr = expr.lstrip('\r\n') + if '\r' in expr: + expr = expr.replace('\r\n', '\n') + expr = expr.replace('\r', '') + expr += '\n' + else: + if '\n' in expr: + raise TemplateError( + 'Multi-line py blocks must start with a newline', + position=pos, name=name) + return ('py', pos, expr), tokens[1:] + elif expr in ('continue', 'break'): + if 'for' not in context: + raise TemplateError( + 'continue outside of for loop', + position=pos, name=name) + return (expr, pos), tokens[1:] + elif expr.startswith('if '): + return parse_cond(tokens, name, context) + elif (expr.startswith('elif ') + or expr == 'else'): + raise TemplateError( + '%s outside of an if block' % expr.split()[0], + position=pos, name=name) + elif expr in ('if', 'elif', 'for'): + raise TemplateError( + '%s with no expression' % expr, + position=pos, name=name) + elif expr in ('endif', 'endfor', 'enddef'): + raise TemplateError( + 'Unexpected %s' % expr, + position=pos, name=name) + elif expr.startswith('for '): + return parse_for(tokens, name, context) + elif expr.startswith('default '): + return parse_default(tokens, name, context) + elif expr.startswith('inherit '): + return parse_inherit(tokens, name, context) + elif expr.startswith('def '): + return parse_def(tokens, name, context) + elif expr.startswith('#'): + return ('comment', pos, tokens[0][0]), tokens[1:] + return ('expr', pos, tokens[0][0]), tokens[1:] + + +def parse_cond(tokens, name, context): + start = tokens[0][1] + pieces = [] + context = context + ('if',) + while 1: + if not tokens: + raise TemplateError( + 'Missing {{endif}}', + position=start, name=name) + if (isinstance(tokens[0], tuple) + and tokens[0][0] == 'endif'): + return ('cond', start) + tuple(pieces), tokens[1:] + next_chunk, tokens = parse_one_cond(tokens, name, context) + pieces.append(next_chunk) + + +def parse_one_cond(tokens, name, context): + (first, pos), tokens = tokens[0], tokens[1:] + content = [] + if first.endswith(':'): + first = first[:-1] + if first.startswith('if '): + part = ('if', pos, first[3:].lstrip(), content) + elif first.startswith('elif '): + part = ('elif', pos, first[5:].lstrip(), content) + elif first == 'else': + part = ('else', pos, None, content) + else: + assert 0, "Unexpected token %r at %s" % (first, pos) + while 1: + if not tokens: + raise TemplateError( + 'No {{endif}}', + position=pos, name=name) + if (isinstance(tokens[0], tuple) + and (tokens[0][0] == 'endif' + or tokens[0][0].startswith('elif ') + or tokens[0][0] == 'else')): + return part, tokens + next_chunk, tokens = parse_expr(tokens, name, context) + content.append(next_chunk) + + +def parse_for(tokens, name, context): + first, pos = tokens[0] + tokens = tokens[1:] + context = ('for',) + context + content = [] + assert first.startswith('for '), first + if first.endswith(':'): + first = first[:-1] + first = first[3:].strip() + match = in_re.search(first) + if not match: + raise TemplateError( + 'Bad for (no "in") in %r' % first, + position=pos, name=name) + vars = first[:match.start()] + if '(' in vars: + raise TemplateError( + 'You cannot have () in the variable section of a for loop (%r)' + % vars, position=pos, name=name) + vars = tuple([ + v.strip() for v in first[:match.start()].split(',') + if v.strip()]) + expr = first[match.end():] + while 1: + if not tokens: + raise TemplateError( + 'No {{endfor}}', + position=pos, name=name) + if (isinstance(tokens[0], tuple) + and tokens[0][0] == 'endfor'): + return ('for', pos, vars, expr, content), tokens[1:] + next_chunk, tokens = parse_expr(tokens, name, context) + content.append(next_chunk) + + +def parse_default(tokens, name, context): + first, pos = tokens[0] + assert first.startswith('default ') + first = first.split(None, 1)[1] + parts = first.split('=', 1) + if len(parts) == 1: + raise TemplateError( + "Expression must be {{default var=value}}; no = found in %r" % first, + position=pos, name=name) + var = parts[0].strip() + if ',' in var: + raise TemplateError( + "{{default x, y = ...}} is not supported", + position=pos, name=name) + if not var_re.search(var): + raise TemplateError( + "Not a valid variable name for {{default}}: %r" + % var, position=pos, name=name) + expr = parts[1].strip() + return ('default', pos, var, expr), tokens[1:] + + +def parse_inherit(tokens, name, context): + first, pos = tokens[0] + assert first.startswith('inherit ') + expr = first.split(None, 1)[1] + return ('inherit', pos, expr), tokens[1:] + + +def parse_def(tokens, name, context): + first, start = tokens[0] + tokens = tokens[1:] + assert first.startswith('def ') + first = first.split(None, 1)[1] + if first.endswith(':'): + first = first[:-1] + if '(' not in first: + func_name = first + sig = ((), None, None, {}) + elif not first.endswith(')'): + raise TemplateError("Function definition doesn't end with ): %s" % first, + position=start, name=name) + else: + first = first[:-1] + func_name, sig_text = first.split('(', 1) + sig = parse_signature(sig_text, name, start) + context = context + ('def',) + content = [] + while 1: + if not tokens: + raise TemplateError( + 'Missing {{enddef}}', + position=start, name=name) + if (isinstance(tokens[0], tuple) + and tokens[0][0] == 'enddef'): + return ('def', start, func_name, sig, content), tokens[1:] + next_chunk, tokens = parse_expr(tokens, name, context) + content.append(next_chunk) + + +def parse_signature(sig_text, name, pos): + tokens = tokenize.generate_tokens(StringIO(sig_text).readline) + sig_args = [] + var_arg = None + var_kw = None + defaults = {} + + def get_token(pos=False): + try: + tok_type, tok_string, (srow, scol), (erow, ecol), line = next(tokens) + except StopIteration: + return tokenize.ENDMARKER, '' + if pos: + return tok_type, tok_string, (srow, scol), (erow, ecol) + else: + return tok_type, tok_string + while 1: + var_arg_type = None + tok_type, tok_string = get_token() + if tok_type == tokenize.ENDMARKER: + break + if tok_type == tokenize.OP and (tok_string == '*' or tok_string == '**'): + var_arg_type = tok_string + tok_type, tok_string = get_token() + if tok_type != tokenize.NAME: + raise TemplateError('Invalid signature: (%s)' % sig_text, + position=pos, name=name) + var_name = tok_string + tok_type, tok_string = get_token() + if tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','): + if var_arg_type == '*': + var_arg = var_name + elif var_arg_type == '**': + var_kw = var_name + else: + sig_args.append(var_name) + if tok_type == tokenize.ENDMARKER: + break + continue + if var_arg_type is not None: + raise TemplateError('Invalid signature: (%s)' % sig_text, + position=pos, name=name) + if tok_type == tokenize.OP and tok_string == '=': + nest_type = None + unnest_type = None + nest_count = 0 + start_pos = end_pos = None + parts = [] + while 1: + tok_type, tok_string, s, e = get_token(True) + if start_pos is None: + start_pos = s + end_pos = e + if tok_type == tokenize.ENDMARKER and nest_count: + raise TemplateError('Invalid signature: (%s)' % sig_text, + position=pos, name=name) + if (not nest_count and + (tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','))): + default_expr = isolate_expression(sig_text, start_pos, end_pos) + defaults[var_name] = default_expr + sig_args.append(var_name) + break + parts.append((tok_type, tok_string)) + if nest_count and tok_type == tokenize.OP and tok_string == nest_type: + nest_count += 1 + elif nest_count and tok_type == tokenize.OP and tok_string == unnest_type: + nest_count -= 1 + if not nest_count: + nest_type = unnest_type = None + elif not nest_count and tok_type == tokenize.OP and tok_string in ('(', '[', '{'): + nest_type = tok_string + nest_count = 1 + unnest_type = {'(': ')', '[': ']', '{': '}'}[nest_type] + return sig_args, var_arg, var_kw, defaults + + +def isolate_expression(string, start_pos, end_pos): + srow, scol = start_pos + srow -= 1 + erow, ecol = end_pos + erow -= 1 + lines = string.splitlines(True) + if srow == erow: + return lines[srow][scol:ecol] + parts = [lines[srow][scol:]] + parts.extend(lines[srow+1:erow]) + if erow < len(lines): + # It'll sometimes give (end_row_past_finish, 0) + parts.append(lines[erow][:ecol]) + return ''.join(parts) + +_fill_command_usage = """\ +%prog [OPTIONS] TEMPLATE arg=value + +Use py:arg=value to set a Python value; otherwise all values are +strings. +""" + + +def fill_command(args=None): + import sys + import optparse + import pkg_resources + import os + if args is None: + args = sys.argv[1:] + dist = pkg_resources.get_distribution('Paste') + parser = optparse.OptionParser( + version=coerce_text(dist), + usage=_fill_command_usage) + parser.add_option( + '-o', '--output', + dest='output', + metavar="FILENAME", + help="File to write output to (default stdout)") + parser.add_option( + '--env', + dest='use_env', + action='store_true', + help="Put the environment in as top-level variables") + options, args = parser.parse_args(args) + if len(args) < 1: + print('You must give a template filename') + sys.exit(2) + template_name = args[0] + args = args[1:] + vars = {} + if options.use_env: + vars.update(os.environ) + for value in args: + if '=' not in value: + print('Bad argument: %r' % value) + sys.exit(2) + name, value = value.split('=', 1) + if name.startswith('py:'): + name = name[:3] + value = eval(value) + vars[name] = value + if template_name == '-': + template_content = sys.stdin.read() + template_name = '' + else: + with open(template_name, 'rb') as f: + template_content = f.read() + template = Template(template_content, name=template_name) + result = template.substitute(vars) + if options.output: + with open(options.output, 'wb') as f: + f.write(result) + else: + sys.stdout.write(result) + +if __name__ == '__main__': + fill_command() diff --git a/tools/lint.toml b/tools/lint.toml index 69f7e9e2e812..c392dc59dd16 100644 --- a/tools/lint.toml +++ b/tools/lint.toml @@ -1,4 +1,7 @@ -exclude = ["scipy/datasets/_registry.py"] +exclude = [ + "scipy/_build_utils/tempita/*", + "scipy/datasets/_registry.py", +] line-length = 88 From 42da1b44f4f853ef6a2cf9f9ff1901aaa9877e4c Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Thu, 25 Apr 2024 14:22:05 +0200 Subject: [PATCH 47/64] DEV: ensure `python dev.py lint` honors the excluded files in lint.toml Co-authored-by: Lucas Colley --- tools/lint.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/lint.toml b/tools/lint.toml index c392dc59dd16..4470891b85b1 100644 --- a/tools/lint.toml +++ b/tools/lint.toml @@ -3,6 +3,8 @@ exclude = [ "scipy/datasets/_registry.py", ] +force-exclude = true + line-length = 88 # Assume Python 3.9 From 13e0cc50fadf1e7753990d21124351f97792e822 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Thu, 25 Apr 2024 15:31:35 +0200 Subject: [PATCH 48/64] BLD: move the `optimize` build steps earlier into the build sequence This makes the build more balanced, since HiGHS takes a long time to build and finishes last otherwise. --- scipy/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scipy/meson.build b/scipy/meson.build index 8ba9c6e3ceea..56dba5de4001 100644 --- a/scipy/meson.build +++ b/scipy/meson.build @@ -538,6 +538,7 @@ subdir('sparse') subdir('stats') subdir('fft') subdir('io') +subdir('optimize') subdir('spatial') subdir('cluster') subdir('constants') @@ -547,6 +548,5 @@ subdir('signal') subdir('interpolate') subdir('ndimage') subdir('odr') -subdir('optimize') subdir('datasets') subdir('misc') From ebd1bc9125cf3d3b9d8685f1e8f961c6b78c5744 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Thu, 25 Apr 2024 17:45:57 +0200 Subject: [PATCH 49/64] BENCH: disable very slow benchmark in `optimize_milp.py` This was no slower after the update to HiGHS (verified locally) in PR 19255. However it takes very long to run and was already near the limit for CircleCI. The tracking issue for this already existed as well: scipy#19389 [skip actions] [skip cirrus] --- benchmarks/benchmarks/optimize_milp.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmarks/benchmarks/optimize_milp.py b/benchmarks/benchmarks/optimize_milp.py index f59ebeb081e4..ff7cce24e52f 100644 --- a/benchmarks/benchmarks/optimize_milp.py +++ b/benchmarks/benchmarks/optimize_milp.py @@ -48,7 +48,7 @@ def setup(self, prob): self.integrality = integrality def time_milp(self, prob): - # TODO: fix this benchmark (timing out in Aug. 2023) + # TODO: fix this benchmark (timing out in Aug. 2023); see gh-19389 # res = milp(c=self.c, constraints=self.constraints, bounds=self.bounds, # integrality=self.integrality) # assert res.success @@ -57,8 +57,9 @@ def time_milp(self, prob): class MilpMagicSquare(Benchmark): - # TODO: re-add 6, timing out in Aug. 2023 - params = [[3, 4, 5]] + # TODO: look at 5,6 - timing out and disabled in Apr'24 (5) and Aug'23 (6) + # see gh-19389 for details + params = [[3, 4]] param_names = ['size'] def setup(self, n): From fb1db9d108fe9427565fa4a1ae6b1cef5d7e5d7c Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Thu, 25 Apr 2024 15:25:55 -0700 Subject: [PATCH 50/64] TST: stats.rv_continuous.fit: adjust fit XSLOW/XFAIL/skip sets --- scipy/stats/tests/test_continuous_basic.py | 79 +++++++++++++--------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/scipy/stats/tests/test_continuous_basic.py b/scipy/stats/tests/test_continuous_basic.py index 01eeea8657e8..cd114facbb51 100644 --- a/scipy/stats/tests/test_continuous_basic.py +++ b/scipy/stats/tests/test_continuous_basic.py @@ -51,21 +51,22 @@ xslow_test_moments = {'studentized_range', 'ksone', 'vonmises', 'vonmises_line', 'recipinvgauss', 'kstwo', 'kappa4'} -xslow_fit_all = {'ksone', 'kstwo', 'levy_stable', 'recipinvgauss', 'studentized_range', - 'gausshyper', 'kappa4', 'vonmises_line', 'genhyperbolic', 'ncx2', - 'powerlognorm', 'genexpon'} -xfail_fit_all = {'trapezoid', 'truncpareto'} -xslow_fit_mm = {'argus', 'beta', 'exponpow', 'exponweib', 'gengamma', +xslow_fit_mle = {'gausshyper', 'ncf', 'ncx2', 'vonmises_line'} +xfail_fit_mle = {'ksone', 'kstwo', 'trapezoid', 'truncpareto'} +skip_fit_mle = {'levy_stable', 'studentized_range'} # far too slow (>10min) +xslow_fit_mm = {'argus', 'beta', 'exponpow', 'gausshyper', 'gengamma', 'genhalflogistic', 'geninvgauss', 'gompertz', 'halfgennorm', - 'johnsonsb', 'johnsonsu', 'powernorm', 'vonmises', - 'truncnorm', 'kstwobign', 'rel_breitwigner', 'wrapcauchy', - 'johnsonsu', 'truncweibull_min', 'truncexpon', 'norminvgauss'} -xslow_fit_mle = {'ncf'} -xfail_fit_mm = {'alpha', 'betaprime', 'burr', 'burr12', 'cauchy', 'crystalball', 'f', - 'fisk', 'foldcauchy', 'genextreme', 'genpareto', 'halfcauchy', - 'invgamma', 'jf_skew_t', 'kappa3', 'levy', 'levy_l', 'loglaplace', - 'lomax', 'mielke', 'ncf', 'nct', 'pareto', 'skewcauchy', 't', - 'bradford', 'tukeylambda'} + 'johnsonsb', 'kstwobign', 'ncx2', 'norminvgauss', 'truncnorm', + 'truncweibull_min', 'wrapcauchy'} +xfail_fit_mm = {'alpha', 'betaprime', 'bradford', 'burr', 'burr12', 'cauchy', + 'crystalball', 'exponweib', 'f', 'fisk', 'foldcauchy', 'genextreme', + 'genpareto', 'halfcauchy', 'invgamma', 'jf_skew_t', 'johnsonsu', + 'kappa3', 'kappa4', 'levy', 'levy_l', 'loglaplace', 'lomax', 'mielke', + 'ncf', 'nct', 'pareto', 'powerlognorm', 'powernorm', 'rel_breitwigner', + 'skewcauchy', 't', 'trapezoid', 'truncexpon', 'truncpareto', + 'tukeylambda', 'vonmises', 'vonmises_line'} +skip_fit_mm = {'genexpon', 'genhyperbolic', 'ksone', 'kstwo', 'levy_stable', + 'recipinvgauss', 'studentized_range'} # far too slow (>10min) # These distributions fail the complex derivative test below. # Here 'fail' mean produce wrong results and/or raise exceptions, depending @@ -199,31 +200,43 @@ def test_cont_basic(distname, arg, sn): def cases_test_cont_basic_fit(): - message = "Test fails and may be slow" - fail_mark = pytest.mark.skip(reason=message) + xslow = pytest.mark.xslow + fail = pytest.mark.skip(reason="Test fails and may be slow.") + skip = pytest.mark.skip(reason="Test too slow to run to completion (>10m).") for distname, arg in distcont[:] + histogram_test_instances: - if distname in xfail_fit_all: - yield pytest.param(distname, arg, None, None, marks=fail_mark) - continue - if distname in xslow_fit_all: - yield pytest.param(distname, arg, None, None, marks=pytest.mark.xslow) - continue - for method in ["MLE", "MM"]: - if method == 'MM' and distname in xfail_fit_mm: - yield pytest.param(distname, arg, method, None, marks=fail_mark) - continue - if method == 'MM' and distname in xslow_fit_mm: - yield pytest.param(distname, arg, method, None, marks=pytest.mark.xslow) - continue - if method == 'MLE' and distname in xslow_fit_mle: - yield pytest.param(distname, arg, method, None, marks=pytest.mark.xslow) - continue - for fix_args in [True, False]: + if method == 'MLE' and distname in xslow_fit_mle: + yield pytest.param(distname, arg, method, fix_args, marks=xslow) + continue + if method == 'MLE' and distname in xfail_fit_mle: + yield pytest.param(distname, arg, method, fix_args, marks=fail) + continue + if method == 'MLE' and distname in skip_fit_mle: + yield pytest.param(distname, arg, method, fix_args, marks=skip) + continue + if method == 'MM' and distname in xslow_fit_mm: + yield pytest.param(distname, arg, method, fix_args, marks=xslow) + continue + if method == 'MM' and distname in xfail_fit_mm: + yield pytest.param(distname, arg, method, fix_args, marks=fail) + continue + if method == 'MM' and distname in skip_fit_mm: + yield pytest.param(distname, arg, method, fix_args, marks=skip) + continue + yield distname, arg, method, fix_args + +def test_cont_basic_fit_cases(): + # Distribution names should not be in multiple MLE or MM sets + assert (len(xslow_fit_mle.union(xfail_fit_mle).union(skip_fit_mle)) == + len(xslow_fit_mle) + len(xfail_fit_mle) + len(skip_fit_mle)) + assert (len(xslow_fit_mm.union(xfail_fit_mm).union(skip_fit_mm)) == + len(xslow_fit_mm) + len(xfail_fit_mm) + len(skip_fit_mm)) + + @pytest.mark.parametrize('distname, arg, method, fix_args', cases_test_cont_basic_fit()) @pytest.mark.parametrize('n_fit_samples', [200]) From ed1e4ebbdaa00e293d886c60393bfa50595415d2 Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Fri, 26 Apr 2024 02:30:43 -0700 Subject: [PATCH 51/64] MAINT: optimize.linprog: fix bug when integrality is a list of all zeros (#20586) --- scipy/optimize/_linprog.py | 2 ++ scipy/optimize/tests/test_linprog.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/scipy/optimize/_linprog.py b/scipy/optimize/_linprog.py index 5deb51bd4558..181218217196 100644 --- a/scipy/optimize/_linprog.py +++ b/scipy/optimize/_linprog.py @@ -625,6 +625,8 @@ def linprog(c, A_ub=None, b_ub=None, A_eq=None, b_eq=None, warn(warning_message, OptimizeWarning, stacklevel=2) elif np.any(integrality): integrality = np.broadcast_to(integrality, np.shape(c)) + else: + integrality = None lp = _LPProblem(c, A_ub, b_ub, A_eq, b_eq, bounds, x0, integrality) lp, solver_options = _parse_linprog(lp, options, meth) diff --git a/scipy/optimize/tests/test_linprog.py b/scipy/optimize/tests/test_linprog.py index f219fcb07f88..63ceaa84049b 100644 --- a/scipy/optimize/tests/test_linprog.py +++ b/scipy/optimize/tests/test_linprog.py @@ -1703,6 +1703,21 @@ def test_bug_10466(self): method=self.method, options=o) assert_allclose(res.fun, -8589934560) + def test_bug_20584(self): + """ + Test that when integrality is a list of all zeros, linprog gives the + same result as when it is an array of all zeros / integrality=None + """ + c = [1, 1] + A_ub = [[-1, 0]] + b_ub = [-2.5] + res1 = linprog(c, A_ub=A_ub, b_ub=b_ub, integrality=[0, 0]) + res2 = linprog(c, A_ub=A_ub, b_ub=b_ub, integrality=np.asarray([0, 0])) + res3 = linprog(c, A_ub=A_ub, b_ub=b_ub, integrality=None) + assert_equal(res1.x, res2.x) + assert_equal(res1.x, res3.x) + + ######################### # Method-specific Tests # ######################### From 76f2d3c04852a5aeb155f7a233ee1538760e033e Mon Sep 17 00:00:00 2001 From: Jake Bowhay <60778417+j-bowhay@users.noreply.github.com> Date: Fri, 26 Apr 2024 16:10:00 +0100 Subject: [PATCH 52/64] DEP: integrate: switch simpson to kwarg-only, remove even kwarg (#20554) --- scipy/integrate/_quadrature.py | 93 +----------------------- scipy/integrate/tests/test_quadrature.py | 35 ++------- 2 files changed, 12 insertions(+), 116 deletions(-) diff --git a/scipy/integrate/_quadrature.py b/scipy/integrate/_quadrature.py index d23d13ee918a..045704204480 100644 --- a/scipy/integrate/_quadrature.py +++ b/scipy/integrate/_quadrature.py @@ -9,8 +9,7 @@ from scipy.special import roots_legendre from scipy.special import gammaln, logsumexp from scipy._lib._util import _rng_spawn -from scipy._lib.deprecation import (_NoValue, _deprecate_positional_args, - _deprecated) +from scipy._lib.deprecation import _deprecated __all__ = ['fixed_quad', 'quadrature', 'romberg', 'romb', @@ -544,8 +543,7 @@ def _basic_simpson(y, start, stop, x, dx, axis): return result -@_deprecate_positional_args(version="1.14") -def simpson(y, *, x=None, dx=1.0, axis=-1, even=_NoValue): +def simpson(y, *, x=None, dx=1.0, axis=-1): """ Integrate y(x) using samples along the given axis and the composite Simpson's rule. If x is None, spacing of dx is assumed. @@ -565,37 +563,6 @@ def simpson(y, *, x=None, dx=1.0, axis=-1, even=_NoValue): `x` is None. Default is 1. axis : int, optional Axis along which to integrate. Default is the last axis. - even : {None, 'simpson', 'avg', 'first', 'last'}, optional - 'avg' : Average two results: - 1) use the first N-2 intervals with - a trapezoidal rule on the last interval and - 2) use the last - N-2 intervals with a trapezoidal rule on the first interval. - - 'first' : Use Simpson's rule for the first N-2 intervals with - a trapezoidal rule on the last interval. - - 'last' : Use Simpson's rule for the last N-2 intervals with a - trapezoidal rule on the first interval. - - None : equivalent to 'simpson' (default) - - 'simpson' : Use Simpson's rule for the first N-2 intervals with the - addition of a 3-point parabolic segment for the last - interval using equations outlined by Cartwright [1]_. - If the axis to be integrated over only has two points then - the integration falls back to a trapezoidal integration. - - .. versionadded:: 1.11.0 - - .. versionchanged:: 1.11.0 - The newly added 'simpson' option is now the default as it is more - accurate in most situations. - - .. deprecated:: 1.11.0 - Parameter `even` is deprecated and will be removed in SciPy - 1.14.0. After this time the behaviour for an even number of - points will follow that of `even='simpson'`. Returns ------- @@ -605,16 +572,12 @@ def simpson(y, *, x=None, dx=1.0, axis=-1, even=_NoValue): See Also -------- quad : adaptive quadrature using QUADPACK - romberg : adaptive Romberg quadrature - quadrature : adaptive Gaussian quadrature fixed_quad : fixed-order Gaussian quadrature dblquad : double integrals tplquad : triple integrals romb : integrators for sampled data cumulative_trapezoid : cumulative integration for sampled data cumulative_simpson : cumulative integration using Simpson's 1/3 rule - ode : ODE integrators - odeint : ODE integrators Notes ----- @@ -645,15 +608,11 @@ def simpson(y, *, x=None, dx=1.0, axis=-1, even=_NoValue): >>> integrate.quad(lambda x: x**3, 0, 9)[0] 1640.25 - >>> integrate.simpson(y, x=x, even='first') - 1644.5 - """ y = np.asarray(y) nd = len(y.shape) N = y.shape[axis] last_dx = dx - first_dx = dx returnshape = 0 if x is not None: x = np.asarray(x) @@ -670,28 +629,11 @@ def simpson(y, *, x=None, dx=1.0, axis=-1, even=_NoValue): raise ValueError("If given, length of x along axis must be the " "same as y.") - # even keyword parameter is deprecated - if even is not _NoValue: - warnings.warn( - "The 'even' keyword is deprecated as of SciPy 1.11.0 and will be " - "removed in SciPy 1.14.0", - DeprecationWarning, stacklevel=2 - ) - if N % 2 == 0: val = 0.0 result = 0.0 slice_all = (slice(None),) * nd - # default is 'simpson' - even = even if even not in (_NoValue, None) else "simpson" - - if even not in ['avg', 'last', 'first', 'simpson']: - raise ValueError( - "Parameter 'even' must be 'simpson', " - "'avg', 'last', or 'first'." - ) - if N == 2: # need at least 3 points in integration axis to form parabolic # segment. If there are two points then any of 'avg', 'first', @@ -701,12 +643,7 @@ def simpson(y, *, x=None, dx=1.0, axis=-1, even=_NoValue): if x is not None: last_dx = x[slice1] - x[slice2] val += 0.5 * last_dx * (y[slice1] + y[slice2]) - - # calculation is finished. Set `even` to None to skip other - # scenarios - even = None - - if even == 'simpson': + else: # use Simpson's rule on first intervals result = _basic_simpson(y, 0, N-3, x, dx, axis) @@ -763,29 +700,7 @@ def simpson(y, *, x=None, dx=1.0, axis=-1, even=_NoValue): result += alpha*y[slice1] + beta*y[slice2] - eta*y[slice3] - # The following code (down to result=result+val) can be removed - # once the 'even' keyword is removed. - - # Compute using Simpson's rule on first intervals - if even in ['avg', 'first']: - slice1 = tupleset(slice_all, axis, -1) - slice2 = tupleset(slice_all, axis, -2) - if x is not None: - last_dx = x[slice1] - x[slice2] - val += 0.5*last_dx*(y[slice1]+y[slice2]) - result = _basic_simpson(y, 0, N-3, x, dx, axis) - # Compute using Simpson's rule on last set of intervals - if even in ['avg', 'last']: - slice1 = tupleset(slice_all, axis, 0) - slice2 = tupleset(slice_all, axis, 1) - if x is not None: - first_dx = x[tuple(slice2)] - x[tuple(slice1)] - val += 0.5*first_dx*(y[slice2]+y[slice1]) - result += _basic_simpson(y, 1, N-2, x, dx, axis) - if even == 'avg': - val /= 2.0 - result /= 2.0 - result = result + val + result += val else: result = _basic_simpson(y, 0, N-2, x, dx, axis) if returnshape: diff --git a/scipy/integrate/tests/test_quadrature.py b/scipy/integrate/tests/test_quadrature.py index 00fc2f6d45a1..87f8d248243a 100644 --- a/scipy/integrate/tests/test_quadrature.py +++ b/scipy/integrate/tests/test_quadrature.py @@ -148,40 +148,29 @@ def test_newton_cotes2(self): numeric_integral = np.dot(wts, y) assert_almost_equal(numeric_integral, exact_integral) - # ignore the DeprecationWarning emitted by the even kwd - @pytest.mark.filterwarnings('ignore::DeprecationWarning') def test_simpson(self): y = np.arange(17) assert_equal(simpson(y), 128) assert_equal(simpson(y, dx=0.5), 64) assert_equal(simpson(y, x=np.linspace(0, 4, 17)), 32) - y = np.arange(4) - x = 2**y - assert_equal(simpson(y, x=x, even='avg'), 13.875) - assert_equal(simpson(y, x=x, even='first'), 13.75) - assert_equal(simpson(y, x=x, even='last'), 14) - - # `even='simpson'` # integral should be exactly 21 x = np.linspace(1, 4, 4) def f(x): return x**2 - assert_allclose(simpson(f(x), x=x, even='simpson'), 21.0) - assert_allclose(simpson(f(x), x=x, even='avg'), 21 + 1/6) + assert_allclose(simpson(f(x), x=x), 21.0) # integral should be exactly 114 x = np.linspace(1, 7, 4) - assert_allclose(simpson(f(x), dx=2.0, even='simpson'), 114) - assert_allclose(simpson(f(x), dx=2.0, even='avg'), 115 + 1/3) + assert_allclose(simpson(f(x), dx=2.0), 114) - # `even='simpson'`, test multi-axis behaviour + # test multi-axis behaviour a = np.arange(16).reshape(4, 4) x = np.arange(64.).reshape(4, 4, 4) y = f(x) for i in range(3): - r = simpson(y, x=x, even='simpson', axis=i) + r = simpson(y, x=x, axis=i) it = np.nditer(a, flags=['multi_index']) for _ in it: idx = list(it.multi_index) @@ -192,11 +181,10 @@ def f(x): # test when integration axis only has two points x = np.arange(16).reshape(8, 2) y = f(x) - for even in ['simpson', 'avg', 'first', 'last']: - r = simpson(y, x=x, even=even, axis=-1) + r = simpson(y, x=x, axis=-1) - integral = 0.5 * (y[:, 1] + y[:, 0]) * (x[:, 1] - x[:, 0]) - assert_allclose(r, integral) + integral = 0.5 * (y[:, 1] + y[:, 0]) * (x[:, 1] - x[:, 0]) + assert_allclose(r, integral) # odd points, test multi-axis behaviour a = np.arange(25).reshape(5, 5) @@ -227,7 +215,7 @@ def f(x): zero_axis = [0.0, 0.0, 0.0, 0.0] default_axis = [170 + 1/3] * 3 # 8**3 / 3 - 1/3 assert_allclose(simpson(y, x=x, axis=0), zero_axis) - # the following should be exact for even='simpson' + # the following should be exact assert_allclose(simpson(y, x=x, axis=-1), default_axis) x = np.array([[1, 2, 4, 8], [1, 2, 4, 8], [1, 8, 16, 32]]) @@ -237,13 +225,6 @@ def f(x): assert_allclose(simpson(y, x=x, axis=0), zero_axis) assert_allclose(simpson(y, x=x, axis=-1), default_axis) - def test_simpson_deprecations(self): - x = np.linspace(0, 3, 4) - y = x**2 - with pytest.deprecated_call(match="The 'even' keyword is deprecated"): - simpson(y, x=x, even='first') - with pytest.deprecated_call(match="use keyword arguments"): - simpson(y, x) @pytest.mark.parametrize('droplast', [False, True]) def test_simpson_2d_integer_no_x(self, droplast): From 38fcc9b2d34d56e8a2b5b5645cf5fdc43b79f82b Mon Sep 17 00:00:00 2001 From: Jake Bowhay Date: Fri, 26 Apr 2024 10:53:07 +0100 Subject: [PATCH 53/64] DEP: special: remove legacy kwarg from special.comb and switch to kwarg-only --- scipy/special/_basic.py | 32 ++-------------- scipy/special/_special_ufuncs_docs.cpp | 12 +++--- scipy/special/tests/test_basic.py | 53 +++----------------------- 3 files changed, 15 insertions(+), 82 deletions(-) diff --git a/scipy/special/_basic.py b/scipy/special/_basic.py index 86072c5cf46f..7b3fd8106c52 100644 --- a/scipy/special/_basic.py +++ b/scipy/special/_basic.py @@ -18,7 +18,6 @@ _sph_harm_all as _sph_harm_all_gufunc) from . import _specfun from ._comb import _comb_int -from scipy._lib.deprecation import _NoValue, _deprecate_positional_args __all__ = [ @@ -2679,8 +2678,7 @@ def obl_cv_seq(m, n, c): return _specfun.segv(m, n, c, -1)[1][:maxL] -@_deprecate_positional_args(version="1.14") -def comb(N, k, *, exact=False, repetition=False, legacy=_NoValue): +def comb(N, k, *, exact=False, repetition=False): """The number of combinations of N things taken k at a time. This is often expressed as "N choose k". @@ -2693,21 +2691,10 @@ def comb(N, k, *, exact=False, repetition=False, legacy=_NoValue): Number of elements taken. exact : bool, optional For integers, if `exact` is False, then floating point precision is - used, otherwise the result is computed exactly. For non-integers, if - `exact` is True, is disregarded. + used, otherwise the result is computed exactly. repetition : bool, optional If `repetition` is True, then the number of combinations with repetition is computed. - legacy : bool, optional - If `legacy` is True and `exact` is True, then non-integral arguments - are cast to ints; if `legacy` is False, the result for non-integral - arguments is unaffected by the value of `exact`. - - .. deprecated:: 1.9.0 - Using `legacy` is deprecated and will removed by - Scipy 1.14.0. If you want to keep the legacy behaviour, cast - your inputs directly, e.g. - ``comb(int(your_N), int(your_k), exact=True)``. Returns ------- @@ -2739,25 +2726,12 @@ def comb(N, k, *, exact=False, repetition=False, legacy=_NoValue): 220 """ - if legacy is not _NoValue: - warnings.warn( - "Using 'legacy' keyword is deprecated and will be removed by " - "Scipy 1.14.0. If you want to keep the legacy behaviour, cast " - "your inputs directly, e.g. " - "'comb(int(your_N), int(your_k), exact=True)'.", - DeprecationWarning, - stacklevel=2 - ) if repetition: - return comb(N + k - 1, k, exact=exact, legacy=legacy) + return comb(N + k - 1, k, exact=exact) if exact: if int(N) == N and int(k) == k: # _comb_int casts inputs to integers, which is safe & intended here return _comb_int(N, k) - elif legacy: - # here at least one number is not an integer; legacy behavior uses - # lossy casts to int - return _comb_int(N, k) # otherwise, we disregard `exact=True`; it makes no sense for # non-integral arguments return comb(N, k) diff --git a/scipy/special/_special_ufuncs_docs.cpp b/scipy/special/_special_ufuncs_docs.cpp index eb388016228a..0965b35fb6ae 100644 --- a/scipy/special/_special_ufuncs_docs.cpp +++ b/scipy/special/_special_ufuncs_docs.cpp @@ -398,16 +398,16 @@ const char *binom_doc = R"( ``y`` is negative or ``x`` is less than ``y``. >>> x, y = -3, 2 - >>> (binom(x, y), comb(x, y), comb(x, y, exact=True)) - (nan, 0.0, 0) + >>> (binom(x, y), comb(x, y)) + (nan, 0.0) >>> x, y = -3.1, 2.2 - >>> (binom(x, y), comb(x, y), comb(x, y, exact=True)) - (18.714147876804432, 0.0, 0) + >>> (binom(x, y), comb(x, y)) + (18.714147876804432, 0.0) >>> x, y = 2.2, 3.1 - >>> (binom(x, y), comb(x, y), comb(x, y, exact=True)) - (0.037399983365134115, 0.0, 0) + >>> (binom(x, y), comb(x, y)) + (0.037399983365134115, 0.0) )"; const char *exp1_doc = R"( diff --git a/scipy/special/tests/test_basic.py b/scipy/special/tests/test_basic.py index 99f4bae6dcc7..6d0be6be2743 100644 --- a/scipy/special/tests/test_basic.py +++ b/scipy/special/tests/test_basic.py @@ -40,7 +40,6 @@ from scipy.special import ellipe, ellipk, ellipkm1 from scipy.special import elliprc, elliprd, elliprf, elliprg, elliprj from scipy.special import mathieu_odd_coef, mathieu_even_coef, stirling2 -from scipy._lib.deprecation import _NoValue from scipy._lib._util import np_long, np_ulong from scipy.special._basic import _FACTORIALK_LIMITS_64BITS, \ @@ -1429,8 +1428,8 @@ def test_betainc_domain_errors(self, func, args): class TestCombinatorics: def test_comb(self): - assert_array_almost_equal(special.comb([10, 10], [3, 4]), [120., 210.]) - assert_almost_equal(special.comb(10, 3), 120.) + assert_allclose(special.comb([10, 10], [3, 4]), [120., 210.]) + assert_allclose(special.comb(10, 3), 120.) assert_equal(special.comb(10, 3, exact=True), 120) assert_equal(special.comb(10, 3, exact=True, repetition=True), 220) @@ -1443,39 +1442,6 @@ def test_comb(self): expected = 100891344545564193334812497256 assert special.comb(100, 50, exact=True) == expected - @pytest.mark.parametrize("repetition", [True, False]) - @pytest.mark.parametrize("legacy", [True, False, _NoValue]) - @pytest.mark.parametrize("k", [3.5, 3]) - @pytest.mark.parametrize("N", [4.5, 4]) - def test_comb_legacy(self, N, k, legacy, repetition): - # test is only relevant for exact=True - if legacy is not _NoValue: - with pytest.warns( - DeprecationWarning, - match=r"Using 'legacy' keyword is deprecated" - ): - result = special.comb(N, k, exact=True, legacy=legacy, - repetition=repetition) - else: - result = special.comb(N, k, exact=True, legacy=legacy, - repetition=repetition) - if legacy: - # for exact=True and legacy=True, cast input arguments, else don't - if repetition: - # the casting in legacy mode happens AFTER transforming N & k, - # so rounding can change (e.g. both floats, but sum to int); - # hence we need to emulate the repetition-transformation here - N, k = int(N + k - 1), int(k) - repetition = False - else: - N, k = int(N), int(k) - # expected result is the same as with exact=False - with suppress_warnings() as sup: - if legacy is not _NoValue: - sup.filter(DeprecationWarning) - expected = special.comb(N, k, legacy=legacy, repetition=repetition) - assert_equal(result, expected) - def test_comb_with_np_int64(self): n = 70 k = 30 @@ -1490,11 +1456,10 @@ def test_comb_zeros(self): assert_equal(special.comb(-1, 3, exact=True), 0) assert_equal(special.comb(2, -1, exact=True), 0) assert_equal(special.comb(2, -1, exact=False), 0) - assert_array_almost_equal(special.comb([2, -1, 2, 10], [3, 3, -1, 3]), - [0., 0., 0., 120.]) + assert_allclose(special.comb([2, -1, 2, 10], [3, 3, -1, 3]), [0., 0., 0., 120.]) def test_perm(self): - assert_array_almost_equal(special.perm([10, 10], [3, 4]), [720., 5040.]) + assert_allclose(special.perm([10, 10], [3, 4]), [720., 5040.]) assert_almost_equal(special.perm(10, 3), 720.) assert_equal(special.perm(10, 3, exact=True), 720) @@ -1503,14 +1468,8 @@ def test_perm_zeros(self): assert_equal(special.perm(-1, 3, exact=True), 0) assert_equal(special.perm(2, -1, exact=True), 0) assert_equal(special.perm(2, -1, exact=False), 0) - assert_array_almost_equal(special.perm([2, -1, 2, 10], [3, 3, -1, 3]), - [0., 0., 0., 720.]) - - def test_positional_deprecation(self): - with pytest.deprecated_call(match="use keyword arguments"): - # from test_comb - special.comb([10, 10], [3, 4], False, False) - + assert_allclose(special.perm([2, -1, 2, 10], [3, 3, -1, 3]), [0., 0., 0., 720.]) + class TestTrigonometric: def test_cbrt(self): From c46ef137b690456d0859d07855b7236b33d90d68 Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Fri, 26 Apr 2024 23:01:38 +0200 Subject: [PATCH 54/64] Revert "ENH: Use `highspy` in `linprog`" --- .gitignore | 2 + .gitmodules | 10 +- LICENSES_bundled.txt | 4 +- benchmarks/benchmarks/optimize_milp.py | 7 +- doc/source/tutorial/optimize.rst | 2 +- mypy.ini | 8 +- pyproject.toml | 2 - scipy/_lib/highs | 1 + scipy/_lib/meson.build | 3 + scipy/meson.build | 4 +- scipy/optimize/_highs/_highs_wrapper.py | 237 ------ scipy/optimize/_highs/cython/__init__.py | 0 scipy/optimize/_highs/cython/src/HConfig.h | 0 scipy/optimize/_highs/cython/src/HConst.pxd | 106 +++ scipy/optimize/_highs/cython/src/Highs.pxd | 56 ++ scipy/optimize/_highs/cython/src/HighsIO.pxd | 20 + .../optimize/_highs/cython/src/HighsInfo.pxd | 22 + scipy/optimize/_highs/cython/src/HighsLp.pxd | 46 ++ .../_highs/cython/src/HighsLpUtils.pxd | 9 + .../_highs/cython/src/HighsModelUtils.pxd | 10 + .../_highs/cython/src/HighsOptions.pxd | 110 +++ .../_highs/cython/src/HighsRuntimeOptions.pxd | 9 + .../_highs/cython/src/HighsSparseMatrix.pxd | 15 + .../_highs/cython/src/HighsStatus.pxd | 12 + .../_highs/cython/src/SimplexConst.pxd | 95 +++ scipy/optimize/_highs/cython/src/__init__.py | 0 .../_highs/cython/src/_highs_constants.pyx | 117 +++ .../_highs/cython/src/_highs_wrapper.pyx | 736 ++++++++++++++++++ .../_highs/cython/src/highs_c_api.pxd | 7 + scipy/optimize/_highs/meson.build | 307 +++++++- scipy/optimize/_highs/src/HConfig.h | 0 scipy/optimize/_highs/src/libhighs_export.h | 50 ++ scipy/optimize/_linprog_highs.py | 280 +++---- scipy/optimize/_milp.py | 9 +- scipy/optimize/tests/test_linprog.py | 11 +- scipy/optimize/tests/test_milp.py | 8 +- subprojects/highs | 1 - 37 files changed, 1818 insertions(+), 498 deletions(-) create mode 160000 scipy/_lib/highs delete mode 100644 scipy/optimize/_highs/_highs_wrapper.py create mode 100644 scipy/optimize/_highs/cython/__init__.py create mode 100644 scipy/optimize/_highs/cython/src/HConfig.h create mode 100644 scipy/optimize/_highs/cython/src/HConst.pxd create mode 100644 scipy/optimize/_highs/cython/src/Highs.pxd create mode 100644 scipy/optimize/_highs/cython/src/HighsIO.pxd create mode 100644 scipy/optimize/_highs/cython/src/HighsInfo.pxd create mode 100644 scipy/optimize/_highs/cython/src/HighsLp.pxd create mode 100644 scipy/optimize/_highs/cython/src/HighsLpUtils.pxd create mode 100644 scipy/optimize/_highs/cython/src/HighsModelUtils.pxd create mode 100644 scipy/optimize/_highs/cython/src/HighsOptions.pxd create mode 100644 scipy/optimize/_highs/cython/src/HighsRuntimeOptions.pxd create mode 100644 scipy/optimize/_highs/cython/src/HighsSparseMatrix.pxd create mode 100644 scipy/optimize/_highs/cython/src/HighsStatus.pxd create mode 100644 scipy/optimize/_highs/cython/src/SimplexConst.pxd create mode 100644 scipy/optimize/_highs/cython/src/__init__.py create mode 100644 scipy/optimize/_highs/cython/src/_highs_constants.pyx create mode 100644 scipy/optimize/_highs/cython/src/_highs_wrapper.pyx create mode 100644 scipy/optimize/_highs/cython/src/highs_c_api.pxd create mode 100644 scipy/optimize/_highs/src/HConfig.h create mode 100644 scipy/optimize/_highs/src/libhighs_export.h delete mode 160000 subprojects/highs diff --git a/.gitignore b/.gitignore index cd7b3a89475d..547eff51a193 100644 --- a/.gitignore +++ b/.gitignore @@ -322,3 +322,5 @@ scipy/optimize/_group_columns.c scipy/optimize/cython_optimize/_zeros.c scipy/optimize/cython_optimize/_zeros.pyx scipy/optimize/lbfgsb/_lbfgsbmodule.c +scipy/optimize/_highs/cython/src/_highs_wrapper.cxx +scipy/optimize/_highs/cython/src/_highs_constants.cxx diff --git a/.gitmodules b/.gitmodules index 5d5e60aa5a16..b6920b8714b2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,6 +9,10 @@ path = scipy/_lib/unuran url = https://github.com/scipy/unuran.git shallow = true +[submodule "HiGHS"] + path = scipy/_lib/highs + url = https://github.com/scipy/highs + shallow = true [submodule "scipy/_lib/boost_math"] path = scipy/_lib/boost_math url = https://github.com/boostorg/math.git @@ -19,9 +23,3 @@ [submodule "scipy/_lib/pocketfft"] path = scipy/_lib/pocketfft url = https://github.com/scipy/pocketfft -# All submodules used as a Meson `subproject` are required to be under the -# subprojects/ directory - see: -# https://mesonbuild.com/Subprojects.html#why-must-all-subprojects-be-inside-a-single-directory -[submodule "subprojects/highs"] - path = subprojects/highs - url = https://github.com/scipy/highs diff --git a/LICENSES_bundled.txt b/LICENSES_bundled.txt index 0c867c19c7b2..9a3943898822 100644 --- a/LICENSES_bundled.txt +++ b/LICENSES_bundled.txt @@ -252,9 +252,9 @@ License: OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Name: HiGHS -Files: subprojects/highs/* +Files: scipy/optimize/_highs/* License: MIT - For details, see subprojects/highs/LICENCE + For details, see scipy/optimize/_highs/LICENCE Name: Boost Files: scipy/_lib/boost_math/* diff --git a/benchmarks/benchmarks/optimize_milp.py b/benchmarks/benchmarks/optimize_milp.py index ff7cce24e52f..f59ebeb081e4 100644 --- a/benchmarks/benchmarks/optimize_milp.py +++ b/benchmarks/benchmarks/optimize_milp.py @@ -48,7 +48,7 @@ def setup(self, prob): self.integrality = integrality def time_milp(self, prob): - # TODO: fix this benchmark (timing out in Aug. 2023); see gh-19389 + # TODO: fix this benchmark (timing out in Aug. 2023) # res = milp(c=self.c, constraints=self.constraints, bounds=self.bounds, # integrality=self.integrality) # assert res.success @@ -57,9 +57,8 @@ def time_milp(self, prob): class MilpMagicSquare(Benchmark): - # TODO: look at 5,6 - timing out and disabled in Apr'24 (5) and Aug'23 (6) - # see gh-19389 for details - params = [[3, 4]] + # TODO: re-add 6, timing out in Aug. 2023 + params = [[3, 4, 5]] param_names = ['size'] def setup(self, n): diff --git a/doc/source/tutorial/optimize.rst b/doc/source/tutorial/optimize.rst index 23474aab5160..8d6abb5f0270 100644 --- a/doc/source/tutorial/optimize.rst +++ b/doc/source/tutorial/optimize.rst @@ -1596,7 +1596,7 @@ Finally, we can solve the transformed problem using :func:`linprog`. >>> bounds = [x0_bounds, x1_bounds, x2_bounds, x3_bounds] >>> result = linprog(c, A_ub=A_ub, b_ub=b_ub, A_eq=A_eq, b_eq=b_eq, bounds=bounds) >>> print(result.message) - The problem is infeasible. (HiGHS Status 8: model_status is Infeasible; primal_status is None) + The problem is infeasible. (HiGHS Status 8: model_status is Infeasible; primal_status is At lower/fixed bound) The result states that our problem is infeasible, meaning that there is no solution vector that satisfies all the constraints. That doesn't necessarily mean we did anything wrong; some problems truly are infeasible. diff --git a/mypy.ini b/mypy.ini index 918b9a76a033..11fe54c67475 100644 --- a/mypy.ini +++ b/mypy.ini @@ -283,15 +283,13 @@ ignore_errors = True [mypy-scipy.optimize._linprog_util] ignore_errors = True -[mypy-scipy.optimize._highs.highspy._highs] +[mypy-scipy.optimize._linprog_highs] ignore_errors = True -ignore_missing_imports = True -[mypy-scipy.optimize._highs.highspy._highs.simplex_constants] +[mypy-scipy.optimize._highs.highs_wrapper] ignore_errors = True -ignore_missing_imports = True -[mypy-scipy.optimize._milp] +[mypy-scipy.optimize._highs.constants] ignore_errors = True [mypy-scipy.optimize._trustregion] diff --git a/pyproject.toml b/pyproject.toml index eb0b1d560a7a..0a5efd36115e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,8 +119,6 @@ tracker = "https://github.com/scipy/scipy/issues" [tool.doit] dodoFile = "dev.py" -[tool.meson-python.args] -install = ['--skip-subprojects'] [tool.cibuildwheel] skip = "cp36-* cp37-* cp38-* pp* *_ppc64le *_i686 *_s390x" diff --git a/scipy/_lib/highs b/scipy/_lib/highs new file mode 160000 index 000000000000..4a122958a82e --- /dev/null +++ b/scipy/_lib/highs @@ -0,0 +1 @@ +Subproject commit 4a122958a82e67e725d08153e099efe4dad099a2 diff --git a/scipy/_lib/meson.build b/scipy/_lib/meson.build index 1edbd94aafcc..7cdab24a5eeb 100644 --- a/scipy/_lib/meson.build +++ b/scipy/_lib/meson.build @@ -2,6 +2,9 @@ fs = import('fs') if not fs.exists('boost_math/README.md') error('Missing the `boost` submodule! Run `git submodule update --init` to fix this.') endif +if not fs.exists('highs/README.md') + error('Missing the `highs` submodule! Run `git submodule update --init` to fix this.') +endif if not fs.exists('unuran/README.md') error('Missing the `unuran` submodule! Run `git submodule update --init` to fix this.') endif diff --git a/scipy/meson.build b/scipy/meson.build index 1d25a3940a90..5f7279fa8cb9 100644 --- a/scipy/meson.build +++ b/scipy/meson.build @@ -342,8 +342,6 @@ Wno_switch = cc.get_supported_arguments('-Wno-switch') Wno_unused_label = cc.get_supported_arguments('-Wno-unused-label') Wno_unused_result = cc.get_supported_arguments('-Wno-unused-result') Wno_unused_variable = cc.get_supported_arguments('-Wno-unused-variable') -Wno_unused_but_set_variable = cc.get_supported_arguments('-Wno-unused-but-set-variable') -Wno_incompatible_pointer_types = cc.get_supported_arguments('-Wno-incompatible-pointer-types') # C++ warning flags _cpp_Wno_cpp = cpp.get_supported_arguments('-Wno-cpp') @@ -547,7 +545,6 @@ subdir('sparse') subdir('stats') subdir('fft') subdir('io') -subdir('optimize') subdir('spatial') subdir('cluster') subdir('constants') @@ -557,5 +554,6 @@ subdir('signal') subdir('interpolate') subdir('ndimage') subdir('odr') +subdir('optimize') subdir('datasets') subdir('misc') diff --git a/scipy/optimize/_highs/_highs_wrapper.py b/scipy/optimize/_highs/_highs_wrapper.py deleted file mode 100644 index 7fe5f387c0db..000000000000 --- a/scipy/optimize/_highs/_highs_wrapper.py +++ /dev/null @@ -1,237 +0,0 @@ -from warnings import warn - -import numpy as np -import scipy.optimize._highs.highspy._highs as _h -from scipy.optimize._highs.highspy import _highs_options as hopt # type: ignore[attr-defined] -from scipy.optimize import OptimizeWarning - - -def _highs_wrapper(c, indptr, indices, data, lhs, rhs, lb, ub, integrality, options): - numcol = c.size - numrow = rhs.size - isMip = integrality is not None and np.sum(integrality) > 0 - - # default "null" return values - res = { - "x": None, - "fun": None, - } - - # Fill up a HighsLp object - lp = _h.HighsLp() - lp.num_col_ = numcol - lp.num_row_ = numrow - lp.a_matrix_.num_col_ = numcol - lp.a_matrix_.num_row_ = numrow - lp.a_matrix_.format_ = _h.MatrixFormat.kColwise - lp.col_cost_ = c - lp.col_lower_ = lb - lp.col_upper_ = ub - lp.row_lower_ = lhs - lp.row_upper_ = rhs - lp.a_matrix_.start_ = indptr - lp.a_matrix_.index_ = indices - lp.a_matrix_.value_ = data - if integrality.size > 0: - lp.integrality_ = [_h.HighsVarType(i) for i in integrality] - - # Make a Highs object and pass it everything - highs = _h.Highs() - highs_options = _h.HighsOptions() - hoptmanager = hopt.HighsOptionsManager() - for key, val in options.items(): - # handle filtering of unsupported and default options - if val is None or key in ("sense",): - continue - - # ask for the option type - opt_type = hoptmanager.get_option_type(key) - if -1 == opt_type: - warn(f"Unrecognized options detected: {dict({key: val})}", - OptimizeWarning, stacklevel = 2) - continue - else: - if key in ("presolve", "parallel"): - # handle fake bools (require bool -> str conversions) - if isinstance(val, bool): - val = "on" if val else "off" - else: - warn( - f'Option f"{key}" is "{val}", but only True or False is ' - f"allowed. Using default.", - OptimizeWarning, - stacklevel = 2, - ) - continue - opt_type = _h.HighsOptionType(opt_type) - status, msg = check_option(highs, key, val) - if opt_type == _h.HighsOptionType.kBool: - if not isinstance(val, bool): - warn( - f'Option f"{key}" is "{val}", but only True or False is ' - f"allowed. Using default.", - OptimizeWarning, - stacklevel = 2, - ) - continue - - # warn or set option - if status != 0: - warn(msg, OptimizeWarning, stacklevel = 2) - else: - setattr(highs_options, key, val) - - opt_status = highs.passOptions(highs_options) - if opt_status == _h.HighsStatus.kError: - res.update( - { - "status": highs.getModelStatus(), - "message": highs.modelStatusToString(highs.getModelStatus()), - } - ) - return res - - init_status = highs.passModel(lp) - if init_status == _h.HighsStatus.kError: - # if model fails to load, highs.getModelStatus() will be NOT_SET - err_model_status = _h.HighsModelStatus.kModelError - res.update( - { - "status": err_model_status, - "message": highs.modelStatusToString(err_model_status), - } - ) - return res - - # Solve the LP - run_status = highs.run() - if run_status == _h.HighsStatus.kError: - res.update( - { - "status": highs.getModelStatus(), - "message": highs.modelStatusToString(highs.getModelStatus()), - } - ) - return res - - # Extract what we need from the solution - model_status = highs.getModelStatus() - - # it should always be safe to get the info object - info = highs.getInfo() - - # Failure modes: - # LP: if we have anything other than an Optimal status, it - # is unsafe (and unhelpful) to read any results - # MIP: has a non-Optimal status or has timed out/reached max iterations - # 1) If not Optimal/TimedOut/MaxIter status, there is no solution - # 2) If TimedOut/MaxIter status, there may be a feasible solution. - # if the objective function value is not Infinity, then the - # current solution is feasible and can be returned. Else, there - # is no solution. - mipFailCondition = model_status not in ( - _h.HighsModelStatus.kOptimal, - _h.HighsModelStatus.kTimeLimit, - _h.HighsModelStatus.kIterationLimit, - _h.HighsModelStatus.kSolutionLimit, - ) or ( - model_status - in { - _h.HighsModelStatus.kTimeLimit, - _h.HighsModelStatus.kIterationLimit, - _h.HighsModelStatus.kSolutionLimit, - } - and (info.objective_function_value == _h.kHighsInf) - ) - lpFailCondition = model_status != _h.HighsModelStatus.kOptimal - if (isMip and mipFailCondition) or (not isMip and lpFailCondition): - res.update( - { - "status": model_status, - "message": "model_status is " - f"{highs.modelStatusToString(model_status)}; " - "primal_status is " - f"{highs.solutionStatusToString(info.primal_solution_status)}", - "simplex_nit": info.simplex_iteration_count, - "ipm_nit": info.ipm_iteration_count, - "crossover_nit": info.crossover_iteration_count, - } - ) - return res - - # Should be safe to read the solution: - solution = highs.getSolution() - basis = highs.getBasis() - - # Lagrangians for bounds based on column statuses - marg_bnds = np.zeros((2, numcol)) - for ii in range(numcol): - if basis.col_status[ii] == _h.HighsBasisStatus.kLower: - marg_bnds[0, ii] = solution.col_dual[ii] - elif basis.col_status[ii] == _h.HighsBasisStatus.kUpper: - marg_bnds[1, ii] = solution.col_dual[ii] - - res.update( - { - "status": model_status, - "message": highs.modelStatusToString(model_status), - # Primal solution - "x": np.array(solution.col_value), - # Ax + s = b => Ax = b - s - # Note: this is for all constraints (A_ub and A_eq) - "slack": rhs - solution.row_value, - # lambda are the lagrange multipliers associated with Ax=b - "lambda": np.array(solution.row_dual), - "marg_bnds": marg_bnds, - "fun": info.objective_function_value, - "simplex_nit": info.simplex_iteration_count, - "ipm_nit": info.ipm_iteration_count, - "crossover_nit": info.crossover_iteration_count, - } - ) - - if isMip: - res.update( - { - "mip_node_count": info.mip_node_count, - "mip_dual_bound": info.mip_dual_bound, - "mip_gap": info.mip_gap, - } - ) - - return res - - -def check_option(highs_inst, option, value): - status, option_type = highs_inst.getOptionType(option) - hoptmanager = hopt.HighsOptionsManager() - - if status != _h.HighsStatus.kOk: - return -1, "Invalid option name." - - valid_types = { - _h.HighsOptionType.kBool: bool, - _h.HighsOptionType.kInt: int, - _h.HighsOptionType.kDouble: float, - _h.HighsOptionType.kString: str, - } - - expected_type = valid_types.get(option_type, None) - - if expected_type is str: - if not hoptmanager.check_string_option(option, value): - return -1, "Invalid option value." - if expected_type is float: - if not hoptmanager.check_double_option(option, value): - return -1, "Invalid option value." - if expected_type is int: - if not hoptmanager.check_int_option(option, value): - return -1, "Invalid option value." - - if expected_type is None: - return 3, "Unknown option type." - - status, current_value = highs_inst.getOptionValue(option) - if status != _h.HighsStatus.kOk: - return 4, "Failed to validate option value." - return 0, "Check option succeeded." diff --git a/scipy/optimize/_highs/cython/__init__.py b/scipy/optimize/_highs/cython/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scipy/optimize/_highs/cython/src/HConfig.h b/scipy/optimize/_highs/cython/src/HConfig.h new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scipy/optimize/_highs/cython/src/HConst.pxd b/scipy/optimize/_highs/cython/src/HConst.pxd new file mode 100644 index 000000000000..503d9e74a263 --- /dev/null +++ b/scipy/optimize/_highs/cython/src/HConst.pxd @@ -0,0 +1,106 @@ +# cython: language_level=3 + +from libcpp cimport bool +from libcpp.string cimport string + +cdef extern from "HConst.h" nogil: + + const int HIGHS_CONST_I_INF "kHighsIInf" + const double HIGHS_CONST_INF "kHighsInf" + const double kHighsTiny + const double kHighsZero + const int kHighsThreadLimit + + cdef enum HighsDebugLevel: + HighsDebugLevel_kHighsDebugLevelNone "kHighsDebugLevelNone" = 0 + HighsDebugLevel_kHighsDebugLevelCheap "kHighsDebugLevelCheap" + HighsDebugLevel_kHighsDebugLevelCostly "kHighsDebugLevelCostly" + HighsDebugLevel_kHighsDebugLevelExpensive "kHighsDebugLevelExpensive" + HighsDebugLevel_kHighsDebugLevelMin "kHighsDebugLevelMin" = HighsDebugLevel_kHighsDebugLevelNone + HighsDebugLevel_kHighsDebugLevelMax "kHighsDebugLevelMax" = HighsDebugLevel_kHighsDebugLevelExpensive + + ctypedef enum HighsModelStatus: + HighsModelStatusNOTSET "HighsModelStatus::kNotset" = 0 + HighsModelStatusLOAD_ERROR "HighsModelStatus::kLoadError" + HighsModelStatusMODEL_ERROR "HighsModelStatus::kModelError" + HighsModelStatusPRESOLVE_ERROR "HighsModelStatus::kPresolveError" + HighsModelStatusSOLVE_ERROR "HighsModelStatus::kSolveError" + HighsModelStatusPOSTSOLVE_ERROR "HighsModelStatus::kPostsolveError" + HighsModelStatusMODEL_EMPTY "HighsModelStatus::kModelEmpty" + HighsModelStatusOPTIMAL "HighsModelStatus::kOptimal" + HighsModelStatusINFEASIBLE "HighsModelStatus::kInfeasible" + HighsModelStatus_UNBOUNDED_OR_INFEASIBLE "HighsModelStatus::kUnboundedOrInfeasible" + HighsModelStatusUNBOUNDED "HighsModelStatus::kUnbounded" + HighsModelStatusREACHED_DUAL_OBJECTIVE_VALUE_UPPER_BOUND "HighsModelStatus::kObjectiveBound" + HighsModelStatusREACHED_OBJECTIVE_TARGET "HighsModelStatus::kObjectiveTarget" + HighsModelStatusREACHED_TIME_LIMIT "HighsModelStatus::kTimeLimit" + HighsModelStatusREACHED_ITERATION_LIMIT "HighsModelStatus::kIterationLimit" + HighsModelStatusUNKNOWN "HighsModelStatus::kUnknown" + HighsModelStatusHIGHS_MODEL_STATUS_MIN "HighsModelStatus::kMin" = HighsModelStatusNOTSET + HighsModelStatusHIGHS_MODEL_STATUS_MAX "HighsModelStatus::kMax" = HighsModelStatusUNKNOWN + + cdef enum HighsBasisStatus: + HighsBasisStatusLOWER "HighsBasisStatus::kLower" = 0, # (slack) variable is at its lower bound [including fixed variables] + HighsBasisStatusBASIC "HighsBasisStatus::kBasic" # (slack) variable is basic + HighsBasisStatusUPPER "HighsBasisStatus::kUpper" # (slack) variable is at its upper bound + HighsBasisStatusZERO "HighsBasisStatus::kZero" # free variable is non-basic and set to zero + HighsBasisStatusNONBASIC "HighsBasisStatus::kNonbasic" # nonbasic with no specific bound information - useful for users and postsolve + + cdef enum SolverOption: + SOLVER_OPTION_SIMPLEX "SolverOption::SOLVER_OPTION_SIMPLEX" = -1 + SOLVER_OPTION_CHOOSE "SolverOption::SOLVER_OPTION_CHOOSE" + SOLVER_OPTION_IPM "SolverOption::SOLVER_OPTION_IPM" + + cdef enum PrimalDualStatus: + PrimalDualStatusSTATUS_NOT_SET "PrimalDualStatus::STATUS_NOT_SET" = -1 + PrimalDualStatusSTATUS_MIN "PrimalDualStatus::STATUS_MIN" = PrimalDualStatusSTATUS_NOT_SET + PrimalDualStatusSTATUS_NO_SOLUTION "PrimalDualStatus::STATUS_NO_SOLUTION" + PrimalDualStatusSTATUS_UNKNOWN "PrimalDualStatus::STATUS_UNKNOWN" + PrimalDualStatusSTATUS_INFEASIBLE_POINT "PrimalDualStatus::STATUS_INFEASIBLE_POINT" + PrimalDualStatusSTATUS_FEASIBLE_POINT "PrimalDualStatus::STATUS_FEASIBLE_POINT" + PrimalDualStatusSTATUS_MAX "PrimalDualStatus::STATUS_MAX" = PrimalDualStatusSTATUS_FEASIBLE_POINT + + cdef enum HighsOptionType: + HighsOptionTypeBOOL "HighsOptionType::kBool" = 0 + HighsOptionTypeINT "HighsOptionType::kInt" + HighsOptionTypeDOUBLE "HighsOptionType::kDouble" + HighsOptionTypeSTRING "HighsOptionType::kString" + + # workaround for lack of enum class support in Cython < 3.x + # cdef enum class ObjSense(int): + # ObjSenseMINIMIZE "ObjSense::kMinimize" = 1 + # ObjSenseMAXIMIZE "ObjSense::kMaximize" = -1 + + cdef cppclass ObjSense: + pass + + cdef ObjSense ObjSenseMINIMIZE "ObjSense::kMinimize" + cdef ObjSense ObjSenseMAXIMIZE "ObjSense::kMaximize" + + # cdef enum class MatrixFormat(int): + # MatrixFormatkColwise "MatrixFormat::kColwise" = 1 + # MatrixFormatkRowwise "MatrixFormat::kRowwise" + # MatrixFormatkRowwisePartitioned "MatrixFormat::kRowwisePartitioned" + + cdef cppclass MatrixFormat: + pass + + cdef MatrixFormat MatrixFormatkColwise "MatrixFormat::kColwise" + cdef MatrixFormat MatrixFormatkRowwise "MatrixFormat::kRowwise" + cdef MatrixFormat MatrixFormatkRowwisePartitioned "MatrixFormat::kRowwisePartitioned" + + # cdef enum class HighsVarType(int): + # kContinuous "HighsVarType::kContinuous" + # kInteger "HighsVarType::kInteger" + # kSemiContinuous "HighsVarType::kSemiContinuous" + # kSemiInteger "HighsVarType::kSemiInteger" + # kImplicitInteger "HighsVarType::kImplicitInteger" + + cdef cppclass HighsVarType: + pass + + cdef HighsVarType kContinuous "HighsVarType::kContinuous" + cdef HighsVarType kInteger "HighsVarType::kInteger" + cdef HighsVarType kSemiContinuous "HighsVarType::kSemiContinuous" + cdef HighsVarType kSemiInteger "HighsVarType::kSemiInteger" + cdef HighsVarType kImplicitInteger "HighsVarType::kImplicitInteger" diff --git a/scipy/optimize/_highs/cython/src/Highs.pxd b/scipy/optimize/_highs/cython/src/Highs.pxd new file mode 100644 index 000000000000..7139908d0341 --- /dev/null +++ b/scipy/optimize/_highs/cython/src/Highs.pxd @@ -0,0 +1,56 @@ +# cython: language_level=3 + +from libc.stdio cimport FILE + +from libcpp cimport bool +from libcpp.string cimport string + +from .HighsStatus cimport HighsStatus +from .HighsOptions cimport HighsOptions +from .HighsInfo cimport HighsInfo +from .HighsLp cimport ( + HighsLp, + HighsSolution, + HighsBasis, + ObjSense, +) +from .HConst cimport HighsModelStatus + +cdef extern from "Highs.h": + # From HiGHS/src/Highs.h + cdef cppclass Highs: + HighsStatus passHighsOptions(const HighsOptions& options) + HighsStatus passModel(const HighsLp& lp) + HighsStatus run() + HighsStatus setHighsLogfile(FILE* logfile) + HighsStatus setHighsOutput(FILE* output) + HighsStatus writeHighsOptions(const string filename, const bool report_only_non_default_values = true) + + # split up for cython below + #const HighsModelStatus& getModelStatus(const bool scaled_model = False) const + const HighsModelStatus & getModelStatus() const + + const HighsInfo& getHighsInfo "getInfo" () const + string modelStatusToString(const HighsModelStatus model_status) const + #HighsStatus getHighsInfoValue(const string& info, int& value) + HighsStatus getHighsInfoValue(const string& info, double& value) const + const HighsOptions& getHighsOptions() const + + const HighsLp& getLp() const + + HighsStatus writeSolution(const string filename, const bool pretty) const + + HighsStatus setBasis() + const HighsSolution& getSolution() const + const HighsBasis& getBasis() const + + bool changeObjectiveSense(const ObjSense sense) + + HighsStatus setHighsOptionValueBool "setOptionValue" (const string & option, const bool value) + HighsStatus setHighsOptionValueInt "setOptionValue" (const string & option, const int value) + HighsStatus setHighsOptionValueStr "setOptionValue" (const string & option, const string & value) + HighsStatus setHighsOptionValueDbl "setOptionValue" (const string & option, const double value) + + string primalDualStatusToString(const int primal_dual_status) + + void resetGlobalScheduler(bool blocking) diff --git a/scipy/optimize/_highs/cython/src/HighsIO.pxd b/scipy/optimize/_highs/cython/src/HighsIO.pxd new file mode 100644 index 000000000000..82b80ae643f1 --- /dev/null +++ b/scipy/optimize/_highs/cython/src/HighsIO.pxd @@ -0,0 +1,20 @@ +# cython: language_level=3 + + +cdef extern from "HighsIO.h" nogil: + # workaround for lack of enum class support in Cython < 3.x + # cdef enum class HighsLogType(int): + # kInfo "HighsLogType::kInfo" = 1 + # kDetailed "HighsLogType::kDetailed" + # kVerbose "HighsLogType::kVerbose" + # kWarning "HighsLogType::kWarning" + # kError "HighsLogType::kError" + + cdef cppclass HighsLogType: + pass + + cdef HighsLogType kInfo "HighsLogType::kInfo" + cdef HighsLogType kDetailed "HighsLogType::kDetailed" + cdef HighsLogType kVerbose "HighsLogType::kVerbose" + cdef HighsLogType kWarning "HighsLogType::kWarning" + cdef HighsLogType kError "HighsLogType::kError" diff --git a/scipy/optimize/_highs/cython/src/HighsInfo.pxd b/scipy/optimize/_highs/cython/src/HighsInfo.pxd new file mode 100644 index 000000000000..789b51089896 --- /dev/null +++ b/scipy/optimize/_highs/cython/src/HighsInfo.pxd @@ -0,0 +1,22 @@ +# cython: language_level=3 + +cdef extern from "HighsInfo.h" nogil: + # From HiGHS/src/lp_data/HighsInfo.h + cdef cppclass HighsInfo: + # Inherited from HighsInfoStruct: + int mip_node_count + int simplex_iteration_count + int ipm_iteration_count + int crossover_iteration_count + int primal_solution_status + int dual_solution_status + int basis_validity + double objective_function_value + double mip_dual_bound + double mip_gap + int num_primal_infeasibilities + double max_primal_infeasibility + double sum_primal_infeasibilities + int num_dual_infeasibilities + double max_dual_infeasibility + double sum_dual_infeasibilities diff --git a/scipy/optimize/_highs/cython/src/HighsLp.pxd b/scipy/optimize/_highs/cython/src/HighsLp.pxd new file mode 100644 index 000000000000..0944f083743f --- /dev/null +++ b/scipy/optimize/_highs/cython/src/HighsLp.pxd @@ -0,0 +1,46 @@ +# cython: language_level=3 + +from libcpp cimport bool +from libcpp.string cimport string +from libcpp.vector cimport vector + +from .HConst cimport HighsBasisStatus, ObjSense, HighsVarType +from .HighsSparseMatrix cimport HighsSparseMatrix + + +cdef extern from "HighsLp.h" nogil: + # From HiGHS/src/lp_data/HighsLp.h + cdef cppclass HighsLp: + int num_col_ + int num_row_ + + vector[double] col_cost_ + vector[double] col_lower_ + vector[double] col_upper_ + vector[double] row_lower_ + vector[double] row_upper_ + + HighsSparseMatrix a_matrix_ + + ObjSense sense_ + double offset_ + + string model_name_ + + vector[string] row_names_ + vector[string] col_names_ + + vector[HighsVarType] integrality_ + + bool isMip() const + + cdef cppclass HighsSolution: + vector[double] col_value + vector[double] col_dual + vector[double] row_value + vector[double] row_dual + + cdef cppclass HighsBasis: + bool valid_ + vector[HighsBasisStatus] col_status + vector[HighsBasisStatus] row_status diff --git a/scipy/optimize/_highs/cython/src/HighsLpUtils.pxd b/scipy/optimize/_highs/cython/src/HighsLpUtils.pxd new file mode 100644 index 000000000000..18ede36c146a --- /dev/null +++ b/scipy/optimize/_highs/cython/src/HighsLpUtils.pxd @@ -0,0 +1,9 @@ +# cython: language_level=3 + +from .HighsStatus cimport HighsStatus +from .HighsLp cimport HighsLp +from .HighsOptions cimport HighsOptions + +cdef extern from "HighsLpUtils.h" nogil: + # From HiGHS/src/lp_data/HighsLpUtils.h + HighsStatus assessLp(HighsLp& lp, const HighsOptions& options) diff --git a/scipy/optimize/_highs/cython/src/HighsModelUtils.pxd b/scipy/optimize/_highs/cython/src/HighsModelUtils.pxd new file mode 100644 index 000000000000..4fccc2e80046 --- /dev/null +++ b/scipy/optimize/_highs/cython/src/HighsModelUtils.pxd @@ -0,0 +1,10 @@ +# cython: language_level=3 + +from libcpp.string cimport string + +from .HConst cimport HighsModelStatus + +cdef extern from "HighsModelUtils.h" nogil: + # From HiGHS/src/lp_data/HighsModelUtils.h + string utilHighsModelStatusToString(const HighsModelStatus model_status) + string utilBasisStatusToString(const int primal_dual_status) diff --git a/scipy/optimize/_highs/cython/src/HighsOptions.pxd b/scipy/optimize/_highs/cython/src/HighsOptions.pxd new file mode 100644 index 000000000000..920c10c19e30 --- /dev/null +++ b/scipy/optimize/_highs/cython/src/HighsOptions.pxd @@ -0,0 +1,110 @@ +# cython: language_level=3 + +from libc.stdio cimport FILE + +from libcpp cimport bool +from libcpp.string cimport string +from libcpp.vector cimport vector + +from .HConst cimport HighsOptionType + +cdef extern from "HighsOptions.h" nogil: + + cdef cppclass OptionRecord: + HighsOptionType type + string name + string description + bool advanced + + cdef cppclass OptionRecordBool(OptionRecord): + bool* value + bool default_value + + cdef cppclass OptionRecordInt(OptionRecord): + int* value + int lower_bound + int default_value + int upper_bound + + cdef cppclass OptionRecordDouble(OptionRecord): + double* value + double lower_bound + double default_value + double upper_bound + + cdef cppclass OptionRecordString(OptionRecord): + string* value + string default_value + + cdef cppclass HighsOptions: + # From HighsOptionsStruct: + + # Options read from the command line + string model_file + string presolve + string solver + string parallel + double time_limit + string options_file + + # Options read from the file + double infinite_cost + double infinite_bound + double small_matrix_value + double large_matrix_value + double primal_feasibility_tolerance + double dual_feasibility_tolerance + double ipm_optimality_tolerance + double dual_objective_value_upper_bound + int highs_debug_level + int simplex_strategy + int simplex_scale_strategy + int simplex_crash_strategy + int simplex_dual_edge_weight_strategy + int simplex_primal_edge_weight_strategy + int simplex_iteration_limit + int simplex_update_limit + int ipm_iteration_limit + int highs_min_threads + int highs_max_threads + int message_level + string solution_file + bool write_solution_to_file + bool write_solution_pretty + + # Advanced options + bool run_crossover + bool mps_parser_type_free + int keep_n_rows + int allowed_simplex_matrix_scale_factor + int allowed_simplex_cost_scale_factor + int simplex_dualise_strategy + int simplex_permute_strategy + int dual_simplex_cleanup_strategy + int simplex_price_strategy + int dual_chuzc_sort_strategy + bool simplex_initial_condition_check + double simplex_initial_condition_tolerance + double dual_steepest_edge_weight_log_error_threshhold + double dual_simplex_cost_perturbation_multiplier + double start_crossover_tolerance + bool less_infeasible_DSE_check + bool less_infeasible_DSE_choose_row + bool use_original_HFactor_logic + + # Options for MIP solver + int mip_max_nodes + int mip_report_level + + # Switch for MIP solver + bool mip + + # Options for HighsPrintMessage and HighsLogMessage + FILE* logfile + FILE* output + int message_level + string solution_file + bool write_solution_to_file + bool write_solution_pretty + + vector[OptionRecord*] records diff --git a/scipy/optimize/_highs/cython/src/HighsRuntimeOptions.pxd b/scipy/optimize/_highs/cython/src/HighsRuntimeOptions.pxd new file mode 100644 index 000000000000..3e227b7a44f7 --- /dev/null +++ b/scipy/optimize/_highs/cython/src/HighsRuntimeOptions.pxd @@ -0,0 +1,9 @@ +# cython: language_level=3 + +from libcpp cimport bool + +from .HighsOptions cimport HighsOptions + +cdef extern from "HighsRuntimeOptions.h" nogil: + # From HiGHS/src/lp_data/HighsRuntimeOptions.h + bool loadOptions(int argc, char** argv, HighsOptions& options) diff --git a/scipy/optimize/_highs/cython/src/HighsSparseMatrix.pxd b/scipy/optimize/_highs/cython/src/HighsSparseMatrix.pxd new file mode 100644 index 000000000000..7eaa9ef79eee --- /dev/null +++ b/scipy/optimize/_highs/cython/src/HighsSparseMatrix.pxd @@ -0,0 +1,15 @@ +# cython: language_level=3 + +from libcpp.vector cimport vector + +from .HConst cimport MatrixFormat + + +cdef extern from "HighsSparseMatrix.h" nogil: + cdef cppclass HighsSparseMatrix: + MatrixFormat format_ + int num_col_ + int num_row_ + vector[int] start_ + vector[int] index_ + vector[double] value_ diff --git a/scipy/optimize/_highs/cython/src/HighsStatus.pxd b/scipy/optimize/_highs/cython/src/HighsStatus.pxd new file mode 100644 index 000000000000..b47813b5d391 --- /dev/null +++ b/scipy/optimize/_highs/cython/src/HighsStatus.pxd @@ -0,0 +1,12 @@ +# cython: language_level=3 + +from libcpp.string cimport string + +cdef extern from "HighsStatus.h" nogil: + ctypedef enum HighsStatus: + HighsStatusError "HighsStatus::kError" = -1 + HighsStatusOK "HighsStatus::kOk" = 0 + HighsStatusWarning "HighsStatus::kWarning" = 1 + + + string highsStatusToString(HighsStatus status) diff --git a/scipy/optimize/_highs/cython/src/SimplexConst.pxd b/scipy/optimize/_highs/cython/src/SimplexConst.pxd new file mode 100644 index 000000000000..77e7b96320d6 --- /dev/null +++ b/scipy/optimize/_highs/cython/src/SimplexConst.pxd @@ -0,0 +1,95 @@ +# cython: language_level=3 + +from libcpp cimport bool + +cdef extern from "SimplexConst.h" nogil: + + cdef enum SimplexAlgorithm: + PRIMAL "SimplexAlgorithm::kPrimal" = 0 + DUAL "SimplexAlgorithm::kDual" + + cdef enum SimplexStrategy: + SIMPLEX_STRATEGY_MIN "SimplexStrategy::kSimplexStrategyMin" = 0 + SIMPLEX_STRATEGY_CHOOSE "SimplexStrategy::kSimplexStrategyChoose" = SIMPLEX_STRATEGY_MIN + SIMPLEX_STRATEGY_DUAL "SimplexStrategy::kSimplexStrategyDual" + SIMPLEX_STRATEGY_DUAL_PLAIN "SimplexStrategy::kSimplexStrategyDualPlain" = SIMPLEX_STRATEGY_DUAL + SIMPLEX_STRATEGY_DUAL_TASKS "SimplexStrategy::kSimplexStrategyDualTasks" + SIMPLEX_STRATEGY_DUAL_MULTI "SimplexStrategy::kSimplexStrategyDualMulti" + SIMPLEX_STRATEGY_PRIMAL "SimplexStrategy::kSimplexStrategyPrimal" + SIMPLEX_STRATEGY_MAX "SimplexStrategy::kSimplexStrategyMax" = SIMPLEX_STRATEGY_PRIMAL + SIMPLEX_STRATEGY_NUM "SimplexStrategy::kSimplexStrategyNum" + + cdef enum SimplexCrashStrategy: + SIMPLEX_CRASH_STRATEGY_MIN "SimplexCrashStrategy::kSimplexCrashStrategyMin" = 0 + SIMPLEX_CRASH_STRATEGY_OFF "SimplexCrashStrategy::kSimplexCrashStrategyOff" = SIMPLEX_CRASH_STRATEGY_MIN + SIMPLEX_CRASH_STRATEGY_LTSSF_K "SimplexCrashStrategy::kSimplexCrashStrategyLtssfK" + SIMPLEX_CRASH_STRATEGY_LTSSF "SimplexCrashStrategy::kSimplexCrashStrategyLtssf" = SIMPLEX_CRASH_STRATEGY_LTSSF_K + SIMPLEX_CRASH_STRATEGY_BIXBY "SimplexCrashStrategy::kSimplexCrashStrategyBixby" + SIMPLEX_CRASH_STRATEGY_LTSSF_PRI "SimplexCrashStrategy::kSimplexCrashStrategyLtssfPri" + SIMPLEX_CRASH_STRATEGY_LTSF_K "SimplexCrashStrategy::kSimplexCrashStrategyLtsfK" + SIMPLEX_CRASH_STRATEGY_LTSF_PRI "SimplexCrashStrategy::kSimplexCrashStrategyLtsfPri" + SIMPLEX_CRASH_STRATEGY_LTSF "SimplexCrashStrategy::kSimplexCrashStrategyLtsf" + SIMPLEX_CRASH_STRATEGY_BIXBY_NO_NONZERO_COL_COSTS "SimplexCrashStrategy::kSimplexCrashStrategyBixbyNoNonzeroColCosts" + SIMPLEX_CRASH_STRATEGY_BASIC "SimplexCrashStrategy::kSimplexCrashStrategyBasic" + SIMPLEX_CRASH_STRATEGY_TEST_SING "SimplexCrashStrategy::kSimplexCrashStrategyTestSing" + SIMPLEX_CRASH_STRATEGY_MAX "SimplexCrashStrategy::kSimplexCrashStrategyMax" = SIMPLEX_CRASH_STRATEGY_TEST_SING + + cdef enum SimplexEdgeWeightStrategy: + SIMPLEX_EDGE_WEIGHT_STRATEGY_MIN "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategyMin" = -1 + SIMPLEX_EDGE_WEIGHT_STRATEGY_CHOOSE "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategyChoose" = SIMPLEX_EDGE_WEIGHT_STRATEGY_MIN + SIMPLEX_EDGE_WEIGHT_STRATEGY_DANTZIG "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategyDantzig" + SIMPLEX_EDGE_WEIGHT_STRATEGY_DEVEX "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategyDevex" + SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategySteepestEdge" + SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE_UNIT_INITIAL "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategySteepestEdgeUnitInitial" + SIMPLEX_EDGE_WEIGHT_STRATEGY_MAX "SimplexEdgeWeightStrategy::kSimplexEdgeWeightStrategyMax" = SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE_UNIT_INITIAL + + cdef enum SimplexPriceStrategy: + SIMPLEX_PRICE_STRATEGY_MIN = 0 + SIMPLEX_PRICE_STRATEGY_COL = SIMPLEX_PRICE_STRATEGY_MIN + SIMPLEX_PRICE_STRATEGY_ROW + SIMPLEX_PRICE_STRATEGY_ROW_SWITCH + SIMPLEX_PRICE_STRATEGY_ROW_SWITCH_COL_SWITCH + SIMPLEX_PRICE_STRATEGY_MAX = SIMPLEX_PRICE_STRATEGY_ROW_SWITCH_COL_SWITCH + + cdef enum SimplexDualChuzcStrategy: + SIMPLEX_DUAL_CHUZC_STRATEGY_MIN = 0 + SIMPLEX_DUAL_CHUZC_STRATEGY_CHOOSE = SIMPLEX_DUAL_CHUZC_STRATEGY_MIN + SIMPLEX_DUAL_CHUZC_STRATEGY_QUAD + SIMPLEX_DUAL_CHUZC_STRATEGY_HEAP + SIMPLEX_DUAL_CHUZC_STRATEGY_BOTH + SIMPLEX_DUAL_CHUZC_STRATEGY_MAX = SIMPLEX_DUAL_CHUZC_STRATEGY_BOTH + + cdef enum InvertHint: + INVERT_HINT_NO = 0 + INVERT_HINT_UPDATE_LIMIT_REACHED + INVERT_HINT_SYNTHETIC_CLOCK_SAYS_INVERT + INVERT_HINT_POSSIBLY_OPTIMAL + INVERT_HINT_POSSIBLY_PRIMAL_UNBOUNDED + INVERT_HINT_POSSIBLY_DUAL_UNBOUNDED + INVERT_HINT_POSSIBLY_SINGULAR_BASIS + INVERT_HINT_PRIMAL_INFEASIBLE_IN_PRIMAL_SIMPLEX + INVERT_HINT_CHOOSE_COLUMN_FAIL + INVERT_HINT_Count + + cdef enum DualEdgeWeightMode: + DANTZIG "DualEdgeWeightMode::DANTZIG" = 0 + DEVEX "DualEdgeWeightMode::DEVEX" + STEEPEST_EDGE "DualEdgeWeightMode::STEEPEST_EDGE" + Count "DualEdgeWeightMode::Count" + + cdef enum PriceMode: + ROW "PriceMode::ROW" = 0 + COL "PriceMode::COL" + + const int PARALLEL_THREADS_DEFAULT + const int DUAL_TASKS_MIN_THREADS + const int DUAL_MULTI_MIN_THREADS + + const bool invert_if_row_out_negative + + const int NONBASIC_FLAG_TRUE + const int NONBASIC_FLAG_FALSE + + const int NONBASIC_MOVE_UP + const int NONBASIC_MOVE_DN + const int NONBASIC_MOVE_ZE diff --git a/scipy/optimize/_highs/cython/src/__init__.py b/scipy/optimize/_highs/cython/src/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scipy/optimize/_highs/cython/src/_highs_constants.pyx b/scipy/optimize/_highs/cython/src/_highs_constants.pyx new file mode 100644 index 000000000000..815e7d79e9e1 --- /dev/null +++ b/scipy/optimize/_highs/cython/src/_highs_constants.pyx @@ -0,0 +1,117 @@ +# cython: language_level=3 + +'''Export enum values and constants from HiGHS.''' + +from .HConst cimport ( + HIGHS_CONST_I_INF, + HIGHS_CONST_INF, + + HighsDebugLevel_kHighsDebugLevelNone, + HighsDebugLevel_kHighsDebugLevelCheap, + + HighsModelStatusNOTSET, + HighsModelStatusLOAD_ERROR, + HighsModelStatusMODEL_ERROR, + HighsModelStatusMODEL_EMPTY, + HighsModelStatusPRESOLVE_ERROR, + HighsModelStatusSOLVE_ERROR, + HighsModelStatusPOSTSOLVE_ERROR, + HighsModelStatusINFEASIBLE, + HighsModelStatus_UNBOUNDED_OR_INFEASIBLE, + HighsModelStatusUNBOUNDED, + HighsModelStatusOPTIMAL, + HighsModelStatusREACHED_DUAL_OBJECTIVE_VALUE_UPPER_BOUND, + HighsModelStatusREACHED_OBJECTIVE_TARGET, + HighsModelStatusREACHED_TIME_LIMIT, + HighsModelStatusREACHED_ITERATION_LIMIT, + + ObjSenseMINIMIZE, + kContinuous, + kInteger, + kSemiContinuous, + kSemiInteger, + kImplicitInteger, +) +from .HighsIO cimport ( + kInfo, + kDetailed, + kVerbose, + kWarning, + kError, +) +from .SimplexConst cimport ( + # Simplex strategy + SIMPLEX_STRATEGY_CHOOSE, + SIMPLEX_STRATEGY_DUAL, + SIMPLEX_STRATEGY_PRIMAL, + + # Crash strategy + SIMPLEX_CRASH_STRATEGY_OFF, + SIMPLEX_CRASH_STRATEGY_BIXBY, + SIMPLEX_CRASH_STRATEGY_LTSF, + + # Edge weight strategy + SIMPLEX_EDGE_WEIGHT_STRATEGY_CHOOSE, + SIMPLEX_EDGE_WEIGHT_STRATEGY_DANTZIG, + SIMPLEX_EDGE_WEIGHT_STRATEGY_DEVEX, + SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE, +) + +# HConst +CONST_I_INF = HIGHS_CONST_I_INF +CONST_INF = HIGHS_CONST_INF + +# Debug level +MESSAGE_LEVEL_NONE = HighsDebugLevel_kHighsDebugLevelNone +MESSAGE_LEVEL_MINIMAL = HighsDebugLevel_kHighsDebugLevelCheap + +# HighsIO +LOG_TYPE_INFO = kInfo +LOG_TYPE_DETAILED = kDetailed +LOG_TYPE_VERBOSE = kVerbose +LOG_TYPE_WARNING = kWarning +LOG_TYPE_ERROR = kError + +# HighsLp +MODEL_STATUS_NOTSET = HighsModelStatusNOTSET +MODEL_STATUS_LOAD_ERROR = HighsModelStatusLOAD_ERROR +MODEL_STATUS_MODEL_ERROR = HighsModelStatusMODEL_ERROR +MODEL_STATUS_PRESOLVE_ERROR = HighsModelStatusPRESOLVE_ERROR +MODEL_STATUS_SOLVE_ERROR = HighsModelStatusSOLVE_ERROR +MODEL_STATUS_POSTSOLVE_ERROR = HighsModelStatusPOSTSOLVE_ERROR +MODEL_STATUS_MODEL_EMPTY = HighsModelStatusMODEL_EMPTY +MODEL_STATUS_INFEASIBLE = HighsModelStatusINFEASIBLE +MODEL_STATUS_UNBOUNDED_OR_INFEASIBLE = HighsModelStatus_UNBOUNDED_OR_INFEASIBLE +MODEL_STATUS_UNBOUNDED = HighsModelStatusUNBOUNDED +MODEL_STATUS_OPTIMAL = HighsModelStatusOPTIMAL +MODEL_STATUS_REACHED_DUAL_OBJECTIVE_VALUE_UPPER_BOUND = HighsModelStatusREACHED_DUAL_OBJECTIVE_VALUE_UPPER_BOUND +MODEL_STATUS_REACHED_OBJECTIVE_TARGET = HighsModelStatusREACHED_OBJECTIVE_TARGET +MODEL_STATUS_REACHED_TIME_LIMIT = HighsModelStatusREACHED_TIME_LIMIT +MODEL_STATUS_REACHED_ITERATION_LIMIT = HighsModelStatusREACHED_ITERATION_LIMIT + +# Simplex strategy +HIGHS_SIMPLEX_STRATEGY_CHOOSE = SIMPLEX_STRATEGY_CHOOSE +HIGHS_SIMPLEX_STRATEGY_DUAL = SIMPLEX_STRATEGY_DUAL +HIGHS_SIMPLEX_STRATEGY_PRIMAL = SIMPLEX_STRATEGY_PRIMAL + +# Crash strategy +HIGHS_SIMPLEX_CRASH_STRATEGY_OFF = SIMPLEX_CRASH_STRATEGY_OFF +HIGHS_SIMPLEX_CRASH_STRATEGY_BIXBY = SIMPLEX_CRASH_STRATEGY_BIXBY +HIGHS_SIMPLEX_CRASH_STRATEGY_LTSF = SIMPLEX_CRASH_STRATEGY_LTSF + +# Edge weight strategy +HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_CHOOSE = SIMPLEX_EDGE_WEIGHT_STRATEGY_CHOOSE +HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_DANTZIG = SIMPLEX_EDGE_WEIGHT_STRATEGY_DANTZIG +HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_DEVEX = SIMPLEX_EDGE_WEIGHT_STRATEGY_DEVEX +HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE = SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE +# HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE_UNIT_INITIAL = SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE_UNIT_INITIAL + +# Objective sense +HIGHS_OBJECTIVE_SENSE_MINIMIZE = ObjSenseMINIMIZE + +# Variable types +HIGHS_VAR_TYPE_CONTINUOUS = kContinuous +HIGHS_VAR_TYPE_INTEGER = kInteger +HIGHS_VAR_TYPE_SEMI_CONTINUOUS = kSemiContinuous +HIGHS_VAR_TYPE_SEMI_INTEGER = kSemiInteger +HIGHS_VAR_TYPE_IMPLICIT_INTEGER = kImplicitInteger diff --git a/scipy/optimize/_highs/cython/src/_highs_wrapper.pyx b/scipy/optimize/_highs/cython/src/_highs_wrapper.pyx new file mode 100644 index 000000000000..d5da4e4fea25 --- /dev/null +++ b/scipy/optimize/_highs/cython/src/_highs_wrapper.pyx @@ -0,0 +1,736 @@ +# cython: language_level=3 + +import numpy as np +cimport numpy as np +from scipy.optimize import OptimizeWarning +from warnings import warn +import numbers + +from libcpp.string cimport string +from libcpp.map cimport map as cppmap +from libcpp.cast cimport reinterpret_cast + +from .HConst cimport ( + HIGHS_CONST_INF, + + HighsModelStatus, + HighsModelStatusNOTSET, + HighsModelStatusMODEL_ERROR, + HighsModelStatusOPTIMAL, + HighsModelStatusREACHED_TIME_LIMIT, + HighsModelStatusREACHED_ITERATION_LIMIT, + + HighsOptionTypeBOOL, + HighsOptionTypeINT, + HighsOptionTypeDOUBLE, + HighsOptionTypeSTRING, + + HighsBasisStatus, + HighsBasisStatusLOWER, + HighsBasisStatusUPPER, + + MatrixFormatkColwise, + HighsVarType, +) +from .Highs cimport Highs +from .HighsStatus cimport ( + HighsStatus, + highsStatusToString, + HighsStatusError, + HighsStatusWarning, + HighsStatusOK, +) +from .HighsLp cimport ( + HighsLp, + HighsSolution, + HighsBasis, +) +from .HighsInfo cimport HighsInfo +from .HighsOptions cimport ( + HighsOptions, + OptionRecord, + OptionRecordBool, + OptionRecordInt, + OptionRecordDouble, + OptionRecordString, +) +from .HighsModelUtils cimport utilBasisStatusToString + +np.import_array() + +# options to reference for default values and bounds; +# make a map to quickly lookup +cdef HighsOptions _ref_opts +cdef cppmap[string, OptionRecord*] _ref_opt_lookup +cdef OptionRecord * _r = NULL +for _r in _ref_opts.records: + _ref_opt_lookup[_r.name] = _r + + +cdef str _opt_warning(string name, val, valid_set=None) noexcept: + cdef OptionRecord * r = _ref_opt_lookup[name] + + # BOOL + if r.type == HighsOptionTypeBOOL: + default_value = ( r).default_value + return ('Option "%s" is "%s", but only True or False is allowed. ' + 'Using default: %s.' % (name.decode(), str(val), default_value)) + + # INT + if r.type == HighsOptionTypeINT: + lower_bound = int(( r).lower_bound) + upper_bound = int(( r).upper_bound) + default_value = int(( r).default_value) + if upper_bound - lower_bound < 10: + int_range = str(set(range(lower_bound, upper_bound + 1))) + else: + int_range = '[%d, %d]' % (lower_bound, upper_bound) + return ('Option "%s" is "%s", but only values in %s are allowed. ' + 'Using default: %d.' % (name.decode(), str(val), int_range, default_value)) + + # DOUBLE + if r.type == HighsOptionTypeDOUBLE: + lower_bound = ( r).lower_bound + upper_bound = ( r).upper_bound + default_value = ( r).default_value + return ('Option "%s" is "%s", but only values in (%g, %g) are allowed. ' + 'Using default: %g.' % (name.decode(), str(val), lower_bound, upper_bound, default_value)) + + # STRING + if r.type == HighsOptionTypeSTRING: + if valid_set is not None: + descr = 'but only values in %s are allowed. ' % str(set(valid_set)) + else: + descr = 'but this is an invalid value. %s. ' % r.description.decode() + default_value = ( r).default_value.decode() + return ('Option "%s" is "%s", ' + '%s' + 'Using default: %s.' % (name.decode(), str(val), descr, default_value)) + + # We don't know what type (should be unreachable)? + return('Option "%s" is "%s", but this is not a valid value. ' + 'See documentation for valid options. ' + 'Using default.' % (name.decode(), str(val))) + +cdef apply_options(dict options, Highs & highs) noexcept: + '''Take options from dictionary and apply to HiGHS object.''' + + # Initialize for error checking + cdef HighsStatus opt_status = HighsStatusOK + + # Do all the ints + for opt in set([ + 'allowed_simplex_cost_scale_factor', + 'allowed_simplex_matrix_scale_factor', + 'dual_simplex_cleanup_strategy', + 'ipm_iteration_limit', + 'keep_n_rows', + 'threads', + 'mip_max_nodes', + 'highs_debug_level', + 'simplex_crash_strategy', + 'simplex_dual_edge_weight_strategy', + 'simplex_dualise_strategy', + 'simplex_iteration_limit', + 'simplex_permute_strategy', + 'simplex_price_strategy', + 'simplex_primal_edge_weight_strategy', + 'simplex_scale_strategy', + 'simplex_strategy', + 'simplex_update_limit', + 'small_matrix_value', + ]): + val = options.get(opt, None) + if val is not None: + if not isinstance(val, int): + warn(_opt_warning(opt.encode(), val), OptimizeWarning) + else: + opt_status = highs.setHighsOptionValueInt(opt.encode(), val) + if opt_status != HighsStatusOK: + warn(_opt_warning(opt.encode(), val), OptimizeWarning) + else: + if opt == "threads": + highs.resetGlobalScheduler(blocking=True) + + # Do all the doubles + for opt in set([ + 'dual_feasibility_tolerance', + 'dual_objective_value_upper_bound', + 'dual_simplex_cost_perturbation_multiplier', + 'dual_steepest_edge_weight_log_error_threshhold', + 'infinite_bound', + 'infinite_cost', + 'ipm_optimality_tolerance', + 'large_matrix_value', + 'primal_feasibility_tolerance', + 'simplex_initial_condition_tolerance', + 'small_matrix_value', + 'start_crossover_tolerance', + 'time_limit', + 'mip_rel_gap' + ]): + val = options.get(opt, None) + if val is not None: + if not isinstance(val, numbers.Number): + warn(_opt_warning(opt.encode(), val), OptimizeWarning) + else: + opt_status = highs.setHighsOptionValueDbl(opt.encode(), val) + if opt_status != HighsStatusOK: + warn(_opt_warning(opt.encode(), val), OptimizeWarning) + + + # Do all the strings + for opt in set(['solver']): + val = options.get(opt, None) + if val is not None: + if not isinstance(val, str): + warn(_opt_warning(opt.encode(), val), OptimizeWarning) + else: + opt_status = highs.setHighsOptionValueStr(opt.encode(), val.encode()) + if opt_status != HighsStatusOK: + warn(_opt_warning(opt.encode(), val), OptimizeWarning) + + + # Do all the bool to strings + for opt in set([ + 'parallel', + 'presolve', + ]): + val = options.get(opt, None) + if val is not None: + if isinstance(val, bool): + if val: + val0 = b'on' + else: + val0 = b'off' + opt_status = highs.setHighsOptionValueStr(opt.encode(), val0) + if opt_status != HighsStatusOK: + warn(_opt_warning(opt.encode(), val, valid_set=[True, False]), OptimizeWarning) + else: + warn(_opt_warning(opt.encode(), val, valid_set=[True, False]), OptimizeWarning) + + + # Do the actual bools + for opt in set([ + 'less_infeasible_DSE_check', + 'less_infeasible_DSE_choose_row', + 'log_to_console', + 'mps_parser_type_free', + 'output_flag', + 'run_as_hsol', + 'run_crossover', + 'simplex_initial_condition_check', + 'use_original_HFactor_logic', + ]): + val = options.get(opt, None) + if val is not None: + if val in [True, False]: + opt_status = highs.setHighsOptionValueBool(opt.encode(), val) + if opt_status != HighsStatusOK: + warn(_opt_warning(opt.encode(), val), OptimizeWarning) + else: + warn(_opt_warning(opt.encode(), val), OptimizeWarning) + + +ctypedef HighsVarType* HighsVarType_ptr + + +def _highs_wrapper( + double[::1] c, + int[::1] astart, + int[::1] aindex, + double[::1] avalue, + double[::1] lhs, + double[::1] rhs, + double[::1] lb, + double[::1] ub, + np.uint8_t[::1] integrality, + dict options): + '''Solve linear programs using HiGHS [1]_. + + Assume problems of the form: + + MIN c.T @ x + s.t. lhs <= A @ x <= rhs + lb <= x <= ub + + Parameters + ---------- + c : 1-D array, (n,) + Array of objective value coefficients. + astart : 1-D array + CSC format index array. + aindex : 1-D array + CSC format index array. + avalue : 1-D array + Data array of the matrix. + lhs : 1-D array (or None), (m,) + Array of left hand side values of the inequality constraints. + If ``lhs=None``, then an array of ``-inf`` is assumed. + rhs : 1-D array, (m,) + Array of right hand side values of the inequality constraints. + lb : 1-D array (or None), (n,) + Lower bounds on solution variables x. If ``lb=None``, then an + array of all `0` is assumed. + ub : 1-D array (or None), (n,) + Upper bounds on solution variables x. If ``ub=None``, then an + array of ``inf`` is assumed. + options : dict + A dictionary of solver options with the following fields: + + - allowed_simplex_cost_scale_factor : int + Undocumented advanced option. + + - allowed_simplex_matrix_scale_factor : int + Undocumented advanced option. + + - dual_feasibility_tolerance : double + Dual feasibility tolerance for simplex. + ``min(dual_feasibility_tolerance, + primal_feasibility_tolerance)`` will be used for + ipm feasibility tolerance. + + - dual_objective_value_upper_bound : double + Upper bound on objective value for dual simplex: + algorithm terminates if reached + + - dual_simplex_cleanup_strategy : int + Undocumented advanced option. + + - dual_simplex_cost_perturbation_multiplier : double + Undocumented advanced option. + + - dual_steepest_edge_weight_log_error_threshhold : double + Undocumented advanced option. + + - infinite_bound : double + Limit on abs(constraint bound): values larger than + this will be treated as infinite + + - infinite_cost : double + Limit on cost coefficient: values larger than this + will be treated as infinite. + + - ipm_iteration_limit : int + Iteration limit for interior-point solver. + + - ipm_optimality_tolerance : double + Optimality tolerance for IPM. + + - keep_n_rows : int {-1, 0, 1} + Undocumented advanced option. + + - ``-1``: ``KEEP_N_ROWS_DELETE_ROWS`` + - ``0``: ``KEEP_N_ROWS_DELETE_ENTRIES`` + - ``1``: ``KEEP_N_ROWS_KEEP_ROWS`` + + - large_matrix_value : double + Upper limit on abs(matrix entries): values larger than + this will be treated as infinite + + - less_infeasible_DSE_check : bool + Undocumented advanced option. + + - less_infeasible_DSE_choose_row : bool + Undocumented advanced option. + + - threads : int + Maximum number of threads in parallel execution. + + - message_level : int {0, 1, 2, 4, 7} + Verbosity level, corresponds to: + + - ``0``: ``ML_NONE`` + All messaging to stdout is suppressed. + + - ``1``: ``ML_VERBOSE`` + Includes a once-per-iteration report on simplex/ipm + progress and information about each nonzero row and + column. + + - ``2``: ``ML_DETAILED`` + Includes technical information about progress and + events in applying the simplex method. + + - ``4``: ``ML_MINIMAL`` + Once-per-solve information about progress as well as a + once-per-basis-matrix-reinversion report on progress in + simplex or a once-per-iteration report on progress in IPX. + + ``message_level`` behaves like a bitmask, i.e., any + combination of levels is possible using the bit-or + operator. + + - mps_parser_type_free : bool + Use free format MPS parsing. + + - parallel : bool + Run the solver in serial (False) or parallel (True). + + - presolve : bool + Run the presolve or not (or if ``None``, then choose). + + - primal_feasibility_tolerance : double + Primal feasibility tolerance. + ``min(dual_feasibility_tolerance, + primal_feasibility_tolerance)`` will be used for + ipm feasibility tolerance. + + - run_as_hsol : bool + Undocumented advanced option. + + - run_crossover : bool + Advanced option. Toggles running the crossover routine + for IPX. + + - sense : int {1, -1} + ``sense=1`` corresponds to the MIN problem, ``sense=-1`` + corresponds to the MAX problem. TODO: NOT IMPLEMENTED + + - simplex_crash_strategy : int {0, 1, 2, 3, 4, 5, 6, 7, 8, 9} + Strategy for simplex crash: off / LTSSF / Bixby (0/1/2). + Default is ``0``. Corresponds to the following: + + - ``0``: ``SIMPLEX_CRASH_STRATEGY_OFF`` + - ``1``: ``SIMPLEX_CRASH_STRATEGY_LTSSF_K`` + - ``2``: ``SIMPLEX_CRASH_STRATEGY_BIXBY`` + - ``3``: ``SIMPLEX_CRASH_STRATEGY_LTSSF_PRI`` + - ``4``: ``SIMPLEX_CRASH_STRATEGY_LTSF_K`` + - ``5``: ``SIMPLEX_CRASH_STRATEGY_LTSF_PRI`` + - ``6``: ``SIMPLEX_CRASH_STRATEGY_LTSF`` + - ``7``: ``SIMPLEX_CRASH_STRATEGY_BIXBY_NO_NONZERO_COL_COSTS`` + - ``8``: ``SIMPLEX_CRASH_STRATEGY_BASIC`` + - ``9``: ``SIMPLE_CRASH_STRATEGY_TEST_SING`` + + - simplex_dualise_strategy : int + Undocumented advanced option. + + - simplex_dual_edge_weight_strategy : int {0, 1, 2, 3, 4} + Strategy for simplex dual edge weights: + Dantzig / Devex / Steepest Edge. Corresponds + to the following: + + - ``0``: ``SIMPLEX_DUAL_EDGE_WEIGHT_STRATEGY_DANTZIG`` + - ``1``: ``SIMPLEX_DUAL_EDGE_WEIGHT_STRATEGY_DEVEX`` + - ``2``: ``SIMPLEX_DUAL_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE_TO_DEVEX_SWITCH`` + - ``3``: ``SIMPLEX_DUAL_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE`` + - ``4``: ``SIMPLEX_DUAL_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE_UNIT_INITIAL`` + + - simplex_initial_condition_check : bool + Undocumented advanced option. + + - simplex_initial_condition_tolerance : double + Undocumented advanced option. + + - simplex_iteration_limit : int + Iteration limit for simplex solver. + + - simplex_permute_strategy : int + Undocumented advanced option. + + - simplex_price_strategy : int + Undocumented advanced option. + + - simplex_primal_edge_weight_strategy : int {0, 1} + Strategy for simplex primal edge weights: + Dantzig / Devex. Corresponds to the following: + + - ``0``: ``SIMPLEX_PRIMAL_EDGE_WEIGHT_STRATEGY_DANTZIG`` + - ``1``: ``SIMPLEX_PRIMAL_EDGE_WEIGHT_STRATEGY_DEVEX`` + + - simplex_scale_strategy : int {0, 1, 2, 3, 4, 5} + Strategy for scaling before simplex solver: + off / on (0/1) + + - ``0``: ``SIMPLEX_SCALE_STRATEGY_OFF`` + - ``1``: ``SIMPLEX_SCALE_STRATEGY_HIGHS`` + - ``2``: ``SIMPLEX_SCALE_STRATEGY_HIGHS_FORCED`` + - ``3``: ``SIMPLEX_SCALE_STRATEGY_HIGHS_015`` + - ``4``: ``SIMPLEX_SCALE_STRATEGY_HIGHS_0157`` + - ``5``: ``SIMPLEX_SCALE_STRATEGY_HSOL`` + + - simplex_strategy : int {0, 1, 2, 3, 4} + Strategy for simplex solver. Default: 1. Corresponds + to the following: + + - ``0``: ``SIMPLEX_STRATEGY_MIN`` + - ``1``: ``SIMPLEX_STRATEGY_DUAL`` + - ``2``: ``SIMPLEX_STRATEGY_DUAL_TASKS`` + - ``3``: ``SIMPLEX_STRATEGY_DUAL_MULTI`` + - ``4``: ``SIMPLEX_STRATEGY_PRIMAL`` + + - simplex_update_limit : int + Limit on the number of simplex UPDATE operations. + + - small_matrix_value : double + Lower limit on abs(matrix entries): values smaller + than this will be treated as zero. + + - solution_file : str + Solution file + + - solver : str {'simplex', 'ipm'} + Choose which solver to use. If ``solver='simplex'`` + and ``parallel=True`` then PAMI will be used. + + - start_crossover_tolerance : double + Tolerance to be satisfied before IPM crossover will + start. + + - time_limit : double + Max number of seconds to run the solver for. + + - use_original_HFactor_logic : bool + Undocumented advanced option. + + - write_solution_to_file : bool + Write the primal and dual solution to a file + + - write_solution_pretty : bool + Write the primal and dual solution in a pretty + (human-readable) format + + See [2]_ for a list of all non-advanced options. + + Returns + ------- + res : dict + + If model_status is one of OPTIMAL, + REACHED_DUAL_OBJECTIVE_VALUE_UPPER_BOUND, REACHED_TIME_LIMIT, + REACHED_ITERATION_LIMIT: + + - ``status`` : int + Model status code. + + - ``message`` : str + Message corresponding to model status code. + + - ``x`` : list + Solution variables. + + - ``slack`` : list + Slack variables. + + - ``lambda`` : list + Lagrange multipliers associated with the constraints + Ax = b. + + - ``s`` : list + Lagrange multipliers associated with the constraints + x >= 0. + + - ``fun`` + Final objective value. + + - ``simplex_nit`` : int + Number of iterations accomplished by the simplex + solver. + + - ``ipm_nit`` : int + Number of iterations accomplished by the interior- + point solver. + + If model_status is not one of the above: + + - ``status`` : int + Model status code. + + - ``message`` : str + Message corresponding to model status code. + + Notes + ----- + If ``options['write_solution_to_file']`` is ``True`` but + ``options['solution_file']`` is unset or ``''``, then the solution + will be printed to ``stdout``. + + If any iteration limit is reached, no solution will be + available. + + ``OptimizeWarning`` will be raised if any option value set by + the user is found to be incorrect. + + References + ---------- + .. [1] https://highs.dev/ + .. [2] https://www.maths.ed.ac.uk/hall/HiGHS/HighsOptions.html + ''' + + cdef int numcol = c.size + cdef int numrow = rhs.size + cdef int numnz = avalue.size + cdef int numintegrality = integrality.size + + # Fill up a HighsLp object + cdef HighsLp lp + lp.num_col_ = numcol + lp.num_row_ = numrow + lp.a_matrix_.num_col_ = numcol + lp.a_matrix_.num_row_ = numrow + lp.a_matrix_.format_ = MatrixFormatkColwise + + lp.col_cost_.resize(numcol) + lp.col_lower_.resize(numcol) + lp.col_upper_.resize(numcol) + + lp.row_lower_.resize(numrow) + lp.row_upper_.resize(numrow) + lp.a_matrix_.start_.resize(numcol + 1) + lp.a_matrix_.index_.resize(numnz) + lp.a_matrix_.value_.resize(numnz) + + # only need to set integrality if it's not's empty + cdef HighsVarType * integrality_ptr = NULL + if numintegrality > 0: + lp.integrality_.resize(numintegrality) + integrality_ptr = reinterpret_cast[HighsVarType_ptr](&integrality[0]) + lp.integrality_.assign(integrality_ptr, integrality_ptr + numcol) + + # Explicitly create pointers to pass to HiGHS C++ API; + # do checking to make sure null memory-views are not + # accessed (e.g., &lhs[0] raises exception when lhs is + # empty!) + cdef: + double * colcost_ptr = NULL + double * collower_ptr = NULL + double * colupper_ptr = NULL + double * rowlower_ptr = NULL + double * rowupper_ptr = NULL + int * astart_ptr = NULL + int * aindex_ptr = NULL + double * avalue_ptr = NULL + if numrow > 0: + rowlower_ptr = &lhs[0] + rowupper_ptr = &rhs[0] + lp.row_lower_.assign(rowlower_ptr, rowlower_ptr + numrow) + lp.row_upper_.assign(rowupper_ptr, rowupper_ptr + numrow) + else: + lp.row_lower_.empty() + lp.row_upper_.empty() + if numcol > 0: + colcost_ptr = &c[0] + collower_ptr = &lb[0] + colupper_ptr = &ub[0] + lp.col_cost_.assign(colcost_ptr, colcost_ptr + numcol) + lp.col_lower_.assign(collower_ptr, collower_ptr + numcol) + lp.col_upper_.assign(colupper_ptr, colupper_ptr + numcol) + else: + lp.col_cost_.empty() + lp.col_lower_.empty() + lp.col_upper_.empty() + lp.integrality_.empty() + if numnz > 0: + astart_ptr = &astart[0] + aindex_ptr = &aindex[0] + avalue_ptr = &avalue[0] + lp.a_matrix_.start_.assign(astart_ptr, astart_ptr + numcol + 1) + lp.a_matrix_.index_.assign(aindex_ptr, aindex_ptr + numnz) + lp.a_matrix_.value_.assign(avalue_ptr, avalue_ptr + numnz) + else: + lp.a_matrix_.start_.empty() + lp.a_matrix_.index_.empty() + lp.a_matrix_.value_.empty() + + # Create the options + cdef Highs highs + apply_options(options, highs) + + # Make a Highs object and pass it everything + cdef HighsModelStatus err_model_status = HighsModelStatusNOTSET + cdef HighsStatus init_status = highs.passModel(lp) + if init_status != HighsStatusOK: + if init_status != HighsStatusWarning: + err_model_status = HighsModelStatusMODEL_ERROR + return { + 'status': err_model_status, + 'message': highs.modelStatusToString(err_model_status).decode(), + } + + # Solve the LP + cdef HighsStatus run_status = highs.run() + if run_status == HighsStatusError: + return { + 'status': highs.getModelStatus(), + 'message': highsStatusToString(run_status).decode(), + } + + # Extract what we need from the solution + cdef HighsModelStatus model_status = highs.getModelStatus() + + # We might need an info object if we can look up the solution and a place to put solution + cdef HighsInfo info = highs.getHighsInfo() # it should always be safe to get the info object + cdef HighsSolution solution + cdef HighsBasis basis + cdef double[:, ::1] marg_bnds = np.zeros((2, numcol)) # marg_bnds[0, :]: lower + + # Failure modes: + # LP: if we have anything other than an Optimal status, it + # is unsafe (and unhelpful) to read any results + # MIP: has a non-Optimal status or has timed out/reached max iterations + # 1) If not Optimal/TimedOut/MaxIter status, there is no solution + # 2) If TimedOut/MaxIter status, there may be a feasible solution. + # if the objective function value is not Infinity, then the + # current solution is feasible and can be returned. Else, there + # is no solution. + mipFailCondition = model_status not in { + HighsModelStatusOPTIMAL, + HighsModelStatusREACHED_TIME_LIMIT, + HighsModelStatusREACHED_ITERATION_LIMIT, + } or (model_status in { + HighsModelStatusREACHED_TIME_LIMIT, + HighsModelStatusREACHED_ITERATION_LIMIT, + } and (info.objective_function_value == HIGHS_CONST_INF)) + lpFailCondition = model_status != HighsModelStatusOPTIMAL + if (highs.getLp().isMip() and mipFailCondition) or (not highs.getLp().isMip() and lpFailCondition): + return { + 'status': model_status, + 'message': f'model_status is {highs.modelStatusToString(model_status).decode()}; ' + f'primal_status is {utilBasisStatusToString( info.primal_solution_status).decode()}', + 'simplex_nit': info.simplex_iteration_count, + 'ipm_nit': info.ipm_iteration_count, + 'fun': None, + 'crossover_nit': info.crossover_iteration_count, + } + # If the model status is such that the solution can be read + else: + # Should be safe to read the solution: + solution = highs.getSolution() + basis = highs.getBasis() + + # lagrangians for bounds based on column statuses + for ii in range(numcol): + if HighsBasisStatusLOWER == basis.col_status[ii]: + marg_bnds[0, ii] = solution.col_dual[ii] + elif HighsBasisStatusUPPER == basis.col_status[ii]: + marg_bnds[1, ii] = solution.col_dual[ii] + + res = { + 'status': model_status, + 'message': highs.modelStatusToString(model_status).decode(), + + # Primal solution + 'x': [solution.col_value[ii] for ii in range(numcol)], + + # Ax + s = b => Ax = b - s + # Note: this is for all constraints (A_ub and A_eq) + 'slack': [rhs[ii] - solution.row_value[ii] for ii in range(numrow)], + + # lambda are the lagrange multipliers associated with Ax=b + 'lambda': [solution.row_dual[ii] for ii in range(numrow)], + 'marg_bnds': marg_bnds, + + 'fun': info.objective_function_value, + 'simplex_nit': info.simplex_iteration_count, + 'ipm_nit': info.ipm_iteration_count, + 'crossover_nit': info.crossover_iteration_count, + } + + if highs.getLp().isMip(): + res.update({ + 'mip_node_count': info.mip_node_count, + 'mip_dual_bound': info.mip_dual_bound, + 'mip_gap': info.mip_gap, + }) + + return res diff --git a/scipy/optimize/_highs/cython/src/highs_c_api.pxd b/scipy/optimize/_highs/cython/src/highs_c_api.pxd new file mode 100644 index 000000000000..b7097caf30bc --- /dev/null +++ b/scipy/optimize/_highs/cython/src/highs_c_api.pxd @@ -0,0 +1,7 @@ +# cython: language_level=3 + +cdef extern from "highs_c_api.h" nogil: + int Highs_passLp(void* highs, int numcol, int numrow, int numnz, + double* colcost, double* collower, double* colupper, + double* rowlower, double* rowupper, + int* astart, int* aindex, double* avalue) diff --git a/scipy/optimize/_highs/meson.build b/scipy/optimize/_highs/meson.build index 9fdb5ec33251..8d701e5e3f67 100644 --- a/scipy/optimize/_highs/meson.build +++ b/scipy/optimize/_highs/meson.build @@ -1,49 +1,284 @@ -# Setup the highs library -fs = import('fs') -if not fs.exists('../../../subprojects/highs/README.md') - error('Missing the `highs` submodule! Run `git submodule update --init` to fix this.') -endif -highs_proj = subproject('highs', - default_options : ['default_library=static', - 'use_zlib=disabled']) -highs_dep = highs_proj.get_variable('highs_dep') -highspy_cpp = highs_proj.get_variable('highspy_cpp') -highsoptions_cpp = highs_proj.get_variable('highsoptions_cpp') - -scipy_highspy_dep = [ - py3_dep, - pybind11_dep, - highs_dep, - thread_dep, - atomic_dep, +highs_define_macros = [ + '-DCMAKE_BUILD_TYPE="RELEASE"', + '-DFAST_BUILD=ON', + '-DHIGHS_GITHASH="n/a"', + '-DHIGHS_COMPILATION_DATE="2021-07-09"', # cannot generate dynamically + '-DHIGHS_VERSION_MAJOR=1', # don't care about this, look at CMakelists.txt + '-DHIGHS_VERSION_MINOR=2', + '-DHIGHS_VERSION_PATCH=0', + '-DHIGHS_DIR=' + meson.current_source_dir() / '..' / '..' / '_lib' / 'highs', + '-UOPENMP', + '-UEXT_PRESOLVE', + '-USCIP_DEV', + '-UHiGHSDEV', + '-UOSI_FOUND', + '-DNDEBUG' ] -py3.extension_module( - '_highs', - sources : highspy_cpp, - dependencies: scipy_highspy_dep, - c_args: [Wno_unused_variable, Wno_unused_but_set_variable], - cpp_args: [_cpp_Wno_unused_variable, _cpp_Wno_unused_but_set_variable], +basiclu_lib = static_library('basiclu', + [ + '../../_lib/highs/src/ipm/basiclu/src/basiclu_factorize.c', + '../../_lib/highs/src/ipm/basiclu/src/basiclu_get_factors.c', + '../../_lib/highs/src/ipm/basiclu/src/basiclu_initialize.c', + '../../_lib/highs/src/ipm/basiclu/src/basiclu_object.c', + '../../_lib/highs/src/ipm/basiclu/src/basiclu_solve_dense.c', + '../../_lib/highs/src/ipm/basiclu/src/basiclu_solve_for_update.c', + '../../_lib/highs/src/ipm/basiclu/src/basiclu_solve_sparse.c', + '../../_lib/highs/src/ipm/basiclu/src/basiclu_update.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_build_factors.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_condest.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_dfs.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_factorize_bump.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_file.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_garbage_perm.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_initialize.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_internal.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_markowitz.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_matrix_norm.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_pivot.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_residual_test.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_setup_bump.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_singletons.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_solve_dense.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_solve_for_update.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_solve_sparse.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_solve_symbolic.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_solve_triangular.c', + '../../_lib/highs/src/ipm/basiclu/src/lu_update.c' + ], + include_directories: [ + 'src', + '../../_lib/highs/src', + '../../_lib/highs/src/ipm/basiclu/include' + ], + c_args: [Wno_unused_variable, highs_define_macros] +) + +highs_flags = [ + _cpp_Wno_class_memaccess, + _cpp_Wno_format_truncation, + _cpp_Wno_non_virtual_dtor, + _cpp_Wno_sign_compare, + _cpp_Wno_switch, + _cpp_Wno_unused_but_set_variable, + _cpp_Wno_unused_variable, +] + +ipx_lib = static_library('ipx', + [ + '../../_lib/highs/src/ipm/ipx/src/basiclu_kernel.cc', + '../../_lib/highs/src/ipm/ipx/src/basiclu_wrapper.cc', + '../../_lib/highs/src/ipm/ipx/src/basis.cc', + '../../_lib/highs/src/ipm/ipx/src/conjugate_residuals.cc', + '../../_lib/highs/src/ipm/ipx/src/control.cc', + '../../_lib/highs/src/ipm/ipx/src/crossover.cc', + '../../_lib/highs/src/ipm/ipx/src/diagonal_precond.cc', + '../../_lib/highs/src/ipm/ipx/src/forrest_tomlin.cc', + '../../_lib/highs/src/ipm/ipx/src/guess_basis.cc', + '../../_lib/highs/src/ipm/ipx/src/indexed_vector.cc', + '../../_lib/highs/src/ipm/ipx/src/info.cc', + '../../_lib/highs/src/ipm/ipx/src/ipm.cc', + '../../_lib/highs/src/ipm/ipx/src/ipx_c.cc', + '../../_lib/highs/src/ipm/ipx/src/iterate.cc', + '../../_lib/highs/src/ipm/ipx/src/kkt_solver.cc', + '../../_lib/highs/src/ipm/ipx/src/kkt_solver_basis.cc', + '../../_lib/highs/src/ipm/ipx/src/kkt_solver_diag.cc', + '../../_lib/highs/src/ipm/ipx/src/linear_operator.cc', + '../../_lib/highs/src/ipm/ipx/src/lp_solver.cc', + '../../_lib/highs/src/ipm/ipx/src/lu_factorization.cc', + '../../_lib/highs/src/ipm/ipx/src/lu_update.cc', + '../../_lib/highs/src/ipm/ipx/src/maxvolume.cc', + '../../_lib/highs/src/ipm/ipx/src/model.cc', + '../../_lib/highs/src/ipm/ipx/src/normal_matrix.cc', + '../../_lib/highs/src/ipm/ipx/src/sparse_matrix.cc', + '../../_lib/highs/src/ipm/ipx/src/sparse_utils.cc', + '../../_lib/highs/src/ipm/ipx/src/splitted_normal_matrix.cc', + '../../_lib/highs/src/ipm/ipx/src/starting_basis.cc', + '../../_lib/highs/src/ipm/ipx/src/symbolic_invert.cc', + '../../_lib/highs/src/ipm/ipx/src/timer.cc', + '../../_lib/highs/src/ipm/ipx/src/utils.cc' + ], + include_directories: [ + '../../_lib/highs/src/ipm/ipx/include/', + '../../_lib/highs/src/ipm/basiclu/include/', + '../../_lib/highs/src/', + '../../_lib/highs/extern/', + 'cython/src/' + ], + dependencies: thread_dep, + cpp_args: [highs_flags, highs_define_macros] +) + +highs_lib = static_library('highs', + [ + '../../_lib/highs/extern/filereaderlp/reader.cpp', + '../../_lib/highs/src/io/Filereader.cpp', + '../../_lib/highs/src/io/FilereaderLp.cpp', + '../../_lib/highs/src/io/FilereaderEms.cpp', + '../../_lib/highs/src/io/FilereaderMps.cpp', + '../../_lib/highs/src/io/HighsIO.cpp', + '../../_lib/highs/src/io/HMPSIO.cpp', + '../../_lib/highs/src/io/HMpsFF.cpp', + '../../_lib/highs/src/io/LoadOptions.cpp', + '../../_lib/highs/src/ipm/IpxWrapper.cpp', + '../../_lib/highs/src/lp_data/Highs.cpp', + '../../_lib/highs/src/lp_data/HighsDebug.cpp', + '../../_lib/highs/src/lp_data/HighsInfo.cpp', + '../../_lib/highs/src/lp_data/HighsInfoDebug.cpp', + '../../_lib/highs/src/lp_data/HighsDeprecated.cpp', + '../../_lib/highs/src/lp_data/HighsInterface.cpp', + '../../_lib/highs/src/lp_data/HighsLp.cpp', + '../../_lib/highs/src/lp_data/HighsLpUtils.cpp', + '../../_lib/highs/src/lp_data/HighsModelUtils.cpp', + '../../_lib/highs/src/lp_data/HighsRanging.cpp', + '../../_lib/highs/src/lp_data/HighsSolution.cpp', + '../../_lib/highs/src/lp_data/HighsSolutionDebug.cpp', + '../../_lib/highs/src/lp_data/HighsSolve.cpp', + '../../_lib/highs/src/lp_data/HighsStatus.cpp', + '../../_lib/highs/src/lp_data/HighsOptions.cpp', + '../../_lib/highs/src/mip/HighsMipSolver.cpp', + '../../_lib/highs/src/mip/HighsMipSolverData.cpp', + '../../_lib/highs/src/mip/HighsDomain.cpp', + '../../_lib/highs/src/mip/HighsDynamicRowMatrix.cpp', + '../../_lib/highs/src/mip/HighsLpRelaxation.cpp', + '../../_lib/highs/src/mip/HighsSeparation.cpp', + '../../_lib/highs/src/mip/HighsSeparator.cpp', + '../../_lib/highs/src/mip/HighsTableauSeparator.cpp', + '../../_lib/highs/src/mip/HighsModkSeparator.cpp', + '../../_lib/highs/src/mip/HighsPathSeparator.cpp', + '../../_lib/highs/src/mip/HighsCutGeneration.cpp', + '../../_lib/highs/src/mip/HighsSearch.cpp', + '../../_lib/highs/src/mip/HighsConflictPool.cpp', + '../../_lib/highs/src/mip/HighsCutPool.cpp', + '../../_lib/highs/src/mip/HighsCliqueTable.cpp', + '../../_lib/highs/src/mip/HighsGFkSolve.cpp', + '../../_lib/highs/src/mip/HighsTransformedLp.cpp', + '../../_lib/highs/src/mip/HighsLpAggregator.cpp', + '../../_lib/highs/src/mip/HighsDebugSol.cpp', + '../../_lib/highs/src/mip/HighsImplications.cpp', + '../../_lib/highs/src/mip/HighsPrimalHeuristics.cpp', + '../../_lib/highs/src/mip/HighsPseudocost.cpp', + '../../_lib/highs/src/mip/HighsRedcostFixing.cpp', + '../../_lib/highs/src/mip/HighsNodeQueue.cpp', + '../../_lib/highs/src/mip/HighsObjectiveFunction.cpp', + '../../_lib/highs/src/model/HighsHessian.cpp', + '../../_lib/highs/src/model/HighsHessianUtils.cpp', + '../../_lib/highs/src/model/HighsModel.cpp', + '../../_lib/highs/src/parallel/HighsTaskExecutor.cpp', + '../../_lib/highs/src/presolve/ICrash.cpp', + '../../_lib/highs/src/presolve/ICrashUtil.cpp', + '../../_lib/highs/src/presolve/ICrashX.cpp', + '../../_lib/highs/src/presolve/HighsPostsolveStack.cpp', + '../../_lib/highs/src/presolve/HighsSymmetry.cpp', + '../../_lib/highs/src/presolve/HPresolve.cpp', + '../../_lib/highs/src/presolve/PresolveComponent.cpp', + '../../_lib/highs/src/qpsolver/basis.cpp', + '../../_lib/highs/src/qpsolver/quass.cpp', + '../../_lib/highs/src/qpsolver/ratiotest.cpp', + '../../_lib/highs/src/qpsolver/scaling.cpp', + '../../_lib/highs/src/qpsolver/perturbation.cpp', + '../../_lib/highs/src/simplex/HEkk.cpp', + '../../_lib/highs/src/simplex/HEkkControl.cpp', + '../../_lib/highs/src/simplex/HEkkDebug.cpp', + '../../_lib/highs/src/simplex/HEkkPrimal.cpp', + '../../_lib/highs/src/simplex/HEkkDual.cpp', + '../../_lib/highs/src/simplex/HEkkDualRHS.cpp', + '../../_lib/highs/src/simplex/HEkkDualRow.cpp', + '../../_lib/highs/src/simplex/HEkkDualMulti.cpp', + '../../_lib/highs/src/simplex/HEkkInterface.cpp', + '../../_lib/highs/src/simplex/HighsSimplexAnalysis.cpp', + '../../_lib/highs/src/simplex/HSimplex.cpp', + '../../_lib/highs/src/simplex/HSimplexDebug.cpp', + '../../_lib/highs/src/simplex/HSimplexNla.cpp', + '../../_lib/highs/src/simplex/HSimplexNlaDebug.cpp', + '../../_lib/highs/src/simplex/HSimplexNlaFreeze.cpp', + '../../_lib/highs/src/simplex/HSimplexNlaProductForm.cpp', + '../../_lib/highs/src/simplex/HSimplexReport.cpp', + '../../_lib/highs/src/test/DevKkt.cpp', + '../../_lib/highs/src/test/KktCh2.cpp', + '../../_lib/highs/src/util/HFactor.cpp', + '../../_lib/highs/src/util/HFactorDebug.cpp', + '../../_lib/highs/src/util/HFactorExtend.cpp', + '../../_lib/highs/src/util/HFactorRefactor.cpp', + '../../_lib/highs/src/util/HFactorUtils.cpp', + '../../_lib/highs/src/util/HighsHash.cpp', + '../../_lib/highs/src/util/HighsLinearSumBounds.cpp', + '../../_lib/highs/src/util/HighsMatrixPic.cpp', + '../../_lib/highs/src/util/HighsMatrixUtils.cpp', + '../../_lib/highs/src/util/HighsSort.cpp', + '../../_lib/highs/src/util/HighsSparseMatrix.cpp', + '../../_lib/highs/src/util/HighsUtils.cpp', + '../../_lib/highs/src/util/HSet.cpp', + '../../_lib/highs/src/util/HVectorBase.cpp', + '../../_lib/highs/src/util/stringutil.cpp', + '../../_lib/highs/src/interfaces/highs_c_api.cpp' + ], + include_directories: [ + 'src/', + '../../_lib/highs/extern/', + '../../_lib/highs/src/', + '../../_lib/highs/src/io/', + '../../_lib/highs/src/ipm/ipx/include/', + '../../_lib/highs/src/lp_data/', + '../../_lib/highs/src/util/', + ], + dependencies: thread_dep, + cpp_args: [highs_flags, highs_define_macros] +) + +_highs_wrapper = py3.extension_module('_highs_wrapper', + cython_gen_cpp.process('cython/src/_highs_wrapper.pyx'), + include_directories: [ + 'cython/src/', + 'src/', + '../../_lib/highs/src/', + '../../_lib/highs/src/io/', + '../../_lib/highs/src/lp_data/', + '../../_lib/highs/src/util/' + ], + dependencies: [np_dep, thread_dep, atomic_dep], link_args: version_link_args, - subdir: 'scipy/optimize/_highs/highspy', + link_with: [highs_lib, ipx_lib, basiclu_lib], + cpp_args: [highs_flags, highs_define_macros, cython_c_args], install: true, + subdir: 'scipy/optimize/_highs' ) - -py3.extension_module( - '_highs_options', - sources : highsoptions_cpp, - dependencies: scipy_highspy_dep, - c_args: [Wno_unused_variable, Wno_unused_but_set_variable], - cpp_args: [_cpp_Wno_unused_variable, _cpp_Wno_unused_but_set_variable], +_highs_constants = py3.extension_module('_highs_constants', + cython_gen_cpp.process('cython/src/_highs_constants.pyx'), + c_args: cython_c_args, + include_directories: [ + 'cython/src/', + 'src', + '../../_lib/highs/src/', + '../../_lib/highs/src/io/', + '../../_lib/highs/src/lp_data/', + '../../_lib/highs/src/simplex/' + ], + dependencies: thread_dep, link_args: version_link_args, - subdir: 'scipy/optimize/_highs/highspy', install: true, + subdir: 'scipy/optimize/_highs' ) py3.install_sources([ - '__init__.py', - '_highs_wrapper.py', -], + 'cython/src/HConst.pxd', + 'cython/src/Highs.pxd', + 'cython/src/HighsIO.pxd', + 'cython/src/HighsInfo.pxd', + 'cython/src/HighsLp.pxd', + 'cython/src/HighsLpUtils.pxd', + 'cython/src/HighsModelUtils.pxd', + 'cython/src/HighsOptions.pxd', + 'cython/src/HighsRuntimeOptions.pxd', + 'cython/src/HighsStatus.pxd', + 'cython/src/SimplexConst.pxd', + 'cython/src/highs_c_api.pxd' + ], + subdir: 'scipy/optimize/_highs/src/cython' +) + +py3.install_sources( + ['__init__.py'], subdir: 'scipy/optimize/_highs' ) diff --git a/scipy/optimize/_highs/src/HConfig.h b/scipy/optimize/_highs/src/HConfig.h new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scipy/optimize/_highs/src/libhighs_export.h b/scipy/optimize/_highs/src/libhighs_export.h new file mode 100644 index 000000000000..095eab54c196 --- /dev/null +++ b/scipy/optimize/_highs/src/libhighs_export.h @@ -0,0 +1,50 @@ + +#ifndef LIBHIGHS_EXPORT_H +#define LIBHIGHS_EXPORT_H + +#ifdef LIBHIGHS_STATIC_DEFINE +# define LIBHIGHS_EXPORT +# define LIBHIGHS_NO_EXPORT +#else +# ifndef LIBHIGHS_EXPORT +# ifdef libhighs_EXPORTS + /* We are building this library */ +# if defined(_MSC_VER) +# define LIBHIGHS_EXPORT __declspec(dllexport) +# else +# define LIBHIGHS_EXPORT __attribute__((visibility("default"))) +# endif +# else + /* We are using this library */ +# if defined(_MSC_VER) +# define LIBHIGHS_EXPORT __declspec(dllexport) +# else +# define LIBHIGHS_EXPORT __attribute__((visibility("default"))) +# endif +# endif +# endif + +# ifndef LIBHIGHS_NO_EXPORT +# define LIBHIGHS_NO_EXPORT __attribute__((visibility("hidden"))) +# endif +#endif + +#ifndef LIBHIGHS_DEPRECATED +# define LIBHIGHS_DEPRECATED __attribute__ ((__deprecated__)) +#endif + +#ifndef LIBHIGHS_DEPRECATED_EXPORT +# define LIBHIGHS_DEPRECATED_EXPORT LIBHIGHS_EXPORT LIBHIGHS_DEPRECATED +#endif + +#ifndef LIBHIGHS_DEPRECATED_NO_EXPORT +# define LIBHIGHS_DEPRECATED_NO_EXPORT LIBHIGHS_NO_EXPORT LIBHIGHS_DEPRECATED +#endif + +#if 0 /* DEFINE_NO_DEPRECATED */ +# ifndef LIBHIGHS_NO_DEPRECATED +# define LIBHIGHS_NO_DEPRECATED +# endif +#endif + +#endif /* LIBHIGHS_EXPORT_H */ diff --git a/scipy/optimize/_linprog_highs.py b/scipy/optimize/_linprog_highs.py index e703a32c41d9..eb07443bb255 100644 --- a/scipy/optimize/_linprog_highs.py +++ b/scipy/optimize/_linprog_highs.py @@ -13,184 +13,97 @@ """ -from enum import Enum +import inspect import numpy as np from ._optimize import OptimizeWarning, OptimizeResult from warnings import warn from ._highs._highs_wrapper import _highs_wrapper +from ._highs._highs_constants import ( + CONST_INF, + MESSAGE_LEVEL_NONE, + HIGHS_OBJECTIVE_SENSE_MINIMIZE, + + MODEL_STATUS_NOTSET, + MODEL_STATUS_LOAD_ERROR, + MODEL_STATUS_MODEL_ERROR, + MODEL_STATUS_PRESOLVE_ERROR, + MODEL_STATUS_SOLVE_ERROR, + MODEL_STATUS_POSTSOLVE_ERROR, + MODEL_STATUS_MODEL_EMPTY, + MODEL_STATUS_OPTIMAL, + MODEL_STATUS_INFEASIBLE, + MODEL_STATUS_UNBOUNDED_OR_INFEASIBLE, + MODEL_STATUS_UNBOUNDED, + MODEL_STATUS_REACHED_DUAL_OBJECTIVE_VALUE_UPPER_BOUND + as MODEL_STATUS_RDOVUB, + MODEL_STATUS_REACHED_OBJECTIVE_TARGET, + MODEL_STATUS_REACHED_TIME_LIMIT, + MODEL_STATUS_REACHED_ITERATION_LIMIT, + + HIGHS_SIMPLEX_STRATEGY_DUAL, + + HIGHS_SIMPLEX_CRASH_STRATEGY_OFF, + + HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_CHOOSE, + HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_DANTZIG, + HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_DEVEX, + HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE, +) from scipy.sparse import csc_matrix, vstack, issparse -import scipy.optimize._highs.highspy._highs as _h -import scipy.optimize._highs.highspy._highs.simplex_constants as simpc - - -class HighsStatusMapping: - """Class to map HiGHS statuses to SciPy-like Return Codes and Messages""" - - class SciPyRC(Enum): - """Return codes like SciPy's for solvers""" - - OPTIMAL = 0 - ITERATION_LIMIT = 1 - INFEASIBLE = 2 - UNBOUNDED = 3 - NUMERICAL = 4 - - def __init__(self): - self.highs_to_scipy = { - _h.HighsModelStatus.kNotset: ( - self.SciPyRC.NUMERICAL, - "Not set", - "Serious numerical difficulties encountered.", - ), - _h.HighsModelStatus.kLoadError: ( - self.SciPyRC.NUMERICAL, - "Load Error", - "Serious numerical difficulties encountered.", - ), - _h.HighsModelStatus.kModelError: ( - self.SciPyRC.INFEASIBLE, - "Model Error", - "The problem is infeasible.", - ), - _h.HighsModelStatus.kPresolveError: ( - self.SciPyRC.NUMERICAL, - "Presolve Error", - "Serious numerical difficulties encountered.", - ), - _h.HighsModelStatus.kSolveError: ( - self.SciPyRC.NUMERICAL, - "Solve Error", - "Serious numerical difficulties encountered.", - ), - _h.HighsModelStatus.kPostsolveError: ( - self.SciPyRC.NUMERICAL, - "Postsolve Error", - "Serious numerical difficulties encountered.", - ), - _h.HighsModelStatus.kModelEmpty: ( - self.SciPyRC.NUMERICAL, - "Model Empty", - "Serious numerical difficulties encountered.", - ), - _h.HighsModelStatus.kOptimal: ( - self.SciPyRC.OPTIMAL, - "Optimal", - "Optimization terminated successfully.", - ), - _h.HighsModelStatus.kInfeasible: ( - self.SciPyRC.INFEASIBLE, - "Infeasible", - "The problem is infeasible.", - ), - _h.HighsModelStatus.kUnboundedOrInfeasible: ( - self.SciPyRC.NUMERICAL, - "Unbounded or Infeasible", - "Serious numerical difficulties encountered.", - ), - _h.HighsModelStatus.kUnbounded: ( - self.SciPyRC.UNBOUNDED, - "Unbounded", - "The problem is unbounded.", - ), - _h.HighsModelStatus.kObjectiveBound: ( - self.SciPyRC.NUMERICAL, - "Objective Bound", - "Serious numerical difficulties encountered.", - ), - _h.HighsModelStatus.kObjectiveTarget: ( - self.SciPyRC.NUMERICAL, - "Objective Target", - "Serious numerical difficulties encountered.", - ), - _h.HighsModelStatus.kTimeLimit: ( - self.SciPyRC.ITERATION_LIMIT, - "Time Limit", - "Time limit reached.", - ), - _h.HighsModelStatus.kIterationLimit: ( - self.SciPyRC.ITERATION_LIMIT, - "Iteration Limit", - "Iteration limit reached.", - ), - _h.HighsModelStatus.kUnknown: ( - self.SciPyRC.NUMERICAL, - "Unknown", - "Serious numerical difficulties encountered.", - ), - _h.HighsModelStatus.kSolutionLimit: ( - self.SciPyRC.NUMERICAL, - "Solution Limit", - "Serious numerical difficulties encountered.", - ), - } - - def get_scipy_status(self, highs_status, highs_message): - """Converts HiGHS status and message to SciPy-like status and messages""" - if highs_status is None or highs_message is None: - print(f"Highs Status: {highs_status}, Message: {highs_message}") - return ( - self.SciPyRC.NUMERICAL.value, - "HiGHS did not provide a status code. (HiGHS Status None: None)", - ) - - scipy_status_enum, message_prefix, scipy_message = self.highs_to_scipy.get( - _h.HighsModelStatus(highs_status), - ( - self.SciPyRC.NUMERICAL, - "Unknown HiGHS Status", - "The HiGHS status code was not recognized.", - ), - ) - full_scipy_msg = ( - f"{scipy_message} (HiGHS Status {int(highs_status)}: {highs_message})" - ) - return scipy_status_enum.value, full_scipy_msg + +def _highs_to_scipy_status_message(highs_status, highs_message): + """Converts HiGHS status number/message to SciPy status number/message""" + + scipy_statuses_messages = { + None: (4, "HiGHS did not provide a status code. "), + MODEL_STATUS_NOTSET: (4, ""), + MODEL_STATUS_LOAD_ERROR: (4, ""), + MODEL_STATUS_MODEL_ERROR: (2, ""), + MODEL_STATUS_PRESOLVE_ERROR: (4, ""), + MODEL_STATUS_SOLVE_ERROR: (4, ""), + MODEL_STATUS_POSTSOLVE_ERROR: (4, ""), + MODEL_STATUS_MODEL_EMPTY: (4, ""), + MODEL_STATUS_RDOVUB: (4, ""), + MODEL_STATUS_REACHED_OBJECTIVE_TARGET: (4, ""), + MODEL_STATUS_OPTIMAL: (0, "Optimization terminated successfully. "), + MODEL_STATUS_REACHED_TIME_LIMIT: (1, "Time limit reached. "), + MODEL_STATUS_REACHED_ITERATION_LIMIT: (1, "Iteration limit reached. "), + MODEL_STATUS_INFEASIBLE: (2, "The problem is infeasible. "), + MODEL_STATUS_UNBOUNDED: (3, "The problem is unbounded. "), + MODEL_STATUS_UNBOUNDED_OR_INFEASIBLE: (4, "The problem is unbounded " + "or infeasible. ")} + unrecognized = (4, "The HiGHS status code was not recognized. ") + scipy_status, scipy_message = ( + scipy_statuses_messages.get(highs_status, unrecognized)) + scipy_message = (f"{scipy_message}" + f"(HiGHS Status {highs_status}: {highs_message})") + return scipy_status, scipy_message def _replace_inf(x): - # Replace `np.inf` with kHighsInf + # Replace `np.inf` with CONST_INF infs = np.isinf(x) with np.errstate(invalid="ignore"): - x[infs] = np.sign(x[infs]) * _h.kHighsInf + x[infs] = np.sign(x[infs])*CONST_INF return x -class SimplexStrategy(Enum): - DANTZIG = "dantzig" - DEVEX = "devex" - STEEPEST_DEVEX = "steepest-devex" # highs min, choose - STEEPEST = "steepest" # highs max - - def to_highs_enum(self): - mapping = { - SimplexStrategy.DANTZIG: - simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategyDantzig.value, - SimplexStrategy.DEVEX: - simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategyDevex.value, - SimplexStrategy.STEEPEST_DEVEX: - simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategyChoose.value, - SimplexStrategy.STEEPEST: - simpc.SimplexEdgeWeightStrategy.kSimplexEdgeWeightStrategySteepestEdge.value, - } - return mapping.get(self) - - -def convert_to_highs_enum(option, option_str, choices_enum, default_value): - if option is None: - return choices_enum[default_value.upper()].to_highs_enum() +def _convert_to_highs_enum(option, option_str, choices): + # If option is in the choices we can look it up, if not use + # the default value taken from function signature and warn: try: - enum_value = choices_enum[option.upper()] + return choices[option.lower()] + except AttributeError: + return choices[option] except KeyError: - warn( - f"Option {option_str} is {option}, but only values in " - f"{[e.value for e in choices_enum]} are allowed. Using default: " - f"{default_value}.", - OptimizeWarning, - stacklevel=3, - ) - enum_value = choices_enum[default_value.upper()] - return enum_value.to_highs_enum() + sig = inspect.signature(_linprog_highs) + default_str = sig.parameters[option_str].default + warn(f"Option {option_str} is {option}, but only values in " + f"{set(choices.keys())} are allowed. Using default: " + f"{default_str}.", + OptimizeWarning, stacklevel=3) + return choices[default_str] def _linprog_highs(lp, solver, time_limit=None, presolve=True, @@ -396,12 +309,15 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, warn(message, OptimizeWarning, stacklevel=3) # Map options to HiGHS enum values - simplex_dual_edge_weight_strategy_enum = convert_to_highs_enum( + simplex_dual_edge_weight_strategy_enum = _convert_to_highs_enum( simplex_dual_edge_weight_strategy, - "simplex_dual_edge_weight_strategy", - choices_enum=SimplexStrategy, - default_value="dantzig", - ) + 'simplex_dual_edge_weight_strategy', + choices={'dantzig': HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_DANTZIG, + 'devex': HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_DEVEX, + 'steepest-devex': HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_CHOOSE, + 'steepest': + HIGHS_SIMPLEX_EDGE_WEIGHT_STRATEGY_STEEPEST_EDGE, + None: None}) c, A_ub, b_ub, A_eq, b_eq, bounds, x0, integrality = lp @@ -423,19 +339,20 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, options = { 'presolve': presolve, - 'sense': _h.ObjSense.kMinimize, + 'sense': HIGHS_OBJECTIVE_SENSE_MINIMIZE, 'solver': solver, 'time_limit': time_limit, - # 'highs_debug_level': _h.kHighs, # TODO + 'highs_debug_level': MESSAGE_LEVEL_NONE, 'dual_feasibility_tolerance': dual_feasibility_tolerance, 'ipm_optimality_tolerance': ipm_optimality_tolerance, 'log_to_console': disp, 'mip_max_nodes': mip_max_nodes, 'output_flag': disp, 'primal_feasibility_tolerance': primal_feasibility_tolerance, - 'simplex_dual_edge_weight_strategy': simplex_dual_edge_weight_strategy_enum, - 'simplex_strategy': simpc.kSimplexStrategyDual.value, - # 'simplex_crash_strategy': simpc.SimplexCrashStrategy.kSimplexCrashStrategyOff, + 'simplex_dual_edge_weight_strategy': + simplex_dual_edge_weight_strategy_enum, + 'simplex_strategy': HIGHS_SIMPLEX_STRATEGY_DUAL, + 'simplex_crash_strategy': HIGHS_SIMPLEX_CRASH_STRATEGY_OFF, 'ipm_iteration_limit': maxiter, 'simplex_iteration_limit': maxiter, 'mip_rel_gap': mip_rel_gap, @@ -480,19 +397,12 @@ def _linprog_highs(lp, solver, time_limit=None, presolve=True, # this needs to be updated if we start choosing the solver intelligently # Convert to scipy-style status and message - highs_mapper = HighsStatusMapping() - highs_status = res.get("status", None) - highs_message = res.get("message", None) - status, message = highs_mapper.get_scipy_status(highs_status, highs_message) - - def is_valid_x(val): - if isinstance(val, np.ndarray): - if val.dtype == object and None in val: - return False - return val is not None - - x = np.array(res["x"]) if "x" in res and is_valid_x(res["x"]) else None + highs_status = res.get('status', None) + highs_message = res.get('message', None) + status, message = _highs_to_scipy_status_message(highs_status, + highs_message) + x = np.array(res['x']) if 'x' in res else None sol = {'x': x, 'slack': slack, 'con': con, @@ -514,7 +424,7 @@ def is_valid_x(val): }), 'fun': res.get('fun'), 'status': status, - 'success': res['status'] == _h.HighsModelStatus.kOptimal, + 'success': res['status'] == MODEL_STATUS_OPTIMAL, 'message': message, 'nit': res.get('simplex_nit', 0) or res.get('ipm_nit', 0), 'crossover_nit': res.get('crossover_nit'), diff --git a/scipy/optimize/_milp.py b/scipy/optimize/_milp.py index f70e3297af4a..fd9ecf52083f 100644 --- a/scipy/optimize/_milp.py +++ b/scipy/optimize/_milp.py @@ -2,10 +2,10 @@ import numpy as np from scipy.sparse import csc_array, vstack, issparse from scipy._lib._util import VisibleDeprecationWarning -from ._highs._highs_wrapper import _highs_wrapper +from ._highs._highs_wrapper import _highs_wrapper # type: ignore[import] from ._constraints import LinearConstraint, Bounds from ._optimize import OptimizeResult -from ._linprog_highs import HighsStatusMapping +from ._linprog_highs import _highs_to_scipy_status_message def _constraints_to_components(constraints): @@ -377,9 +377,8 @@ def milp(c, *, integrality=None, bounds=None, constraints=None, options=None): # Convert to scipy-style status and message highs_status = highs_res.get('status', None) highs_message = highs_res.get('message', None) - hstat = HighsStatusMapping() - status, message = hstat.get_scipy_status(highs_status, - highs_message) + status, message = _highs_to_scipy_status_message(highs_status, + highs_message) res['status'] = status res['message'] = message res['success'] = (status == 0) diff --git a/scipy/optimize/tests/test_linprog.py b/scipy/optimize/tests/test_linprog.py index 63ceaa84049b..3a3f88ce41ea 100644 --- a/scipy/optimize/tests/test_linprog.py +++ b/scipy/optimize/tests/test_linprog.py @@ -315,7 +315,7 @@ def test_highs_status_message(): options=options, integrality=integrality) msg = "Time limit reached. (HiGHS Status 13:" assert res.status == 1 - assert msg in res.message + assert res.message.startswith(msg) options = {"maxiter": 10} res = linprog(c=c, A_eq=A, b_eq=b, bounds=bounds, method='highs-ds', @@ -334,14 +334,13 @@ def test_highs_status_message(): assert res.status == 3 assert res.message.startswith(msg) - from scipy.optimize._linprog_highs import HighsStatusMapping - highs_mapper = HighsStatusMapping() - status, message = highs_mapper.get_scipy_status(58, "Hello!") - assert status == 4 + from scipy.optimize._linprog_highs import _highs_to_scipy_status_message + status, message = _highs_to_scipy_status_message(58, "Hello!") msg = "The HiGHS status code was not recognized. (HiGHS Status 58:" + assert status == 4 assert message.startswith(msg) - status, message = highs_mapper.get_scipy_status(None, None) + status, message = _highs_to_scipy_status_message(None, None) msg = "HiGHS did not provide a status code. (HiGHS Status None: None)" assert status == 4 assert message.startswith(msg) diff --git a/scipy/optimize/tests/test_milp.py b/scipy/optimize/tests/test_milp.py index 8bf2d8e6b60e..0970a15a8bcc 100644 --- a/scipy/optimize/tests/test_milp.py +++ b/scipy/optimize/tests/test_milp.py @@ -112,7 +112,6 @@ def test_result(): assert not res.success msg = "Time limit reached. (HiGHS Status 13:" assert res.message.startswith(msg) - assert msg in res.message assert (res.fun is res.mip_dual_bound is res.mip_gap is res.mip_node_count is res.x is None) @@ -291,15 +290,14 @@ def test_infeasible_prob_16609(): _msg_time = "Time limit reached. (HiGHS Status 13:" -_msg_sol = "Serious numerical difficulties encountered. (HiGHS Status 16:" +_msg_iter = "Iteration limit reached. (HiGHS Status 14:" -# See https://github.com/scipy/scipy/pull/19255#issuecomment-1778438888 -@pytest.mark.xfail(reason="Often buggy, revisit with callbacks, gh-19255") @pytest.mark.skipif(np.intp(0).itemsize < 8, reason="Unhandled 32-bit GCC FP bug") +@pytest.mark.slow @pytest.mark.parametrize(["options", "msg"], [({"time_limit": 0.1}, _msg_time), - ({"node_limit": 1}, _msg_sol)]) + ({"node_limit": 1}, _msg_iter)]) def test_milp_timeout_16545(options, msg): # Ensure solution is not thrown away if MILP solver times out # -- see gh-16545 diff --git a/subprojects/highs b/subprojects/highs deleted file mode 160000 index a0df06fb20f2..000000000000 --- a/subprojects/highs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a0df06fb20f2c67c02cf84d8a3b72862c2ab2b27 From efbcba15d3624201aeb6387270726812996e7836 Mon Sep 17 00:00:00 2001 From: Jonas Eschle Date: Sat, 27 Apr 2024 05:06:38 -0400 Subject: [PATCH 55/64] ENH: Add more initialization methods for HessianUpdateStrategy (#13534) * Remove init_scale restriction to scalar * Remove init_scale restriction to scalar * ENH: add arbitrary shaped initial scale for HessianUpdateStrategy Remove the float conversion that implies a scalar. This adds a lot of flexibility to e.g. intialize with an array or even a matrix. If a matrix is given, it will be used as the initial matrix. * ENH: add arbitrary shaped initial scale for HessianUpdateStrategy Remove the float conversion that implies a scalar. This adds a lot of flexibility to e.g. intialize with an array or even a matrix. If a matrix is given, it will be used as the initial matrix. * ENH: add arbitrary shaped initial scale for HessianUpdateStrategy Remove the float conversion that implies a scalar. This adds a lot of flexibility to e.g. intialize with an array or even a matrix. If a matrix is given, it will be used as the initial matrix. * ENH: add arbitrary shaped initial scale for HessianUpdateStrategy Remove the float conversion that implies a scalar. This adds a lot of flexibility to e.g. intialize with an array or even a matrix. If a matrix is given, it will be used as the initial matrix. * ENH: add arbitrary shaped initial scale for HessianUpdateStrategy Remove the float conversion that implies a scalar. This adds a lot of flexibility to e.g. intialize with an array or even a matrix. If a matrix is given, it will be used as the initial matrix. * ENH: add arbitrary shaped initial scale for HessianUpdateStrategy Remove the float conversion that implies a scalar. This adds a lot of flexibility to e.g. intialize with an array or even a matrix. If a matrix is given, it will be used as the initial matrix. * ENH: add arbitrary shaped initial scale for HessianUpdateStrategy Remove the float conversion that implies a scalar. This adds a lot of flexibility to e.g. intialize with an array or even a matrix. If a matrix is given, it will be used as the initial matrix. * docs: improve HessianUpdateStrategy parameter init_scale * ENH: make init_scale in HessianUpdateStrategy a real array This will fail loudly if it can't be cast to the exact dtype, e.g. if it is complex. Added an extra check to ensure that this will do. Also extended the tests to check for both approximation types, hess and inv_hess * ENH: add legacy cast in HessianUpdateStrategys init_scale Added the float cast again to ensure full legacy behavior * FIX: Check if init_scale is str in HessianUpdateStrategy Gives FutureWarning otherwise as a string comparison with an array * TEST: Check if init_scale is str in HessianUpdateStrategy test Gives FutureWarning otherwise as a string comparison with an array * REFACTOR: Make elif for if that would have been elif anyway * chore: remove whitespaces * chore: remove whitespaces * chore: remove unused import * enh: add input checks * tests: add match check in message * tests: use pytests raises * tests: typo * fix: init multiply * fix: init multiply * style: fix * tests: escape regex * tests: escape regex 2 * tests: escape regex 2 * tests: escape regex 3 * tests: escape regex 4 * tests: escape regex 4 * tests: fix test with correct error * enh: add more descriptive errors * enh: add more descriptive errors * enh: be explicit about copy * chore: reformat * tests: fix exceptions * chore: fix linting * chore: fix linting 2 --- scipy/optimize/_hessian_update_strategy.py | 79 +++++++++++---- .../tests/test_hessian_update_strategy.py | 98 +++++++++++++++++-- 2 files changed, 153 insertions(+), 24 deletions(-) diff --git a/scipy/optimize/_hessian_update_strategy.py b/scipy/optimize/_hessian_update_strategy.py index b8529e51e83b..c72d1159314e 100644 --- a/scipy/optimize/_hessian_update_strategy.py +++ b/scipy/optimize/_hessian_update_strategy.py @@ -1,7 +1,7 @@ """Hessian update strategies for quasi-Newton optimization methods.""" import numpy as np from numpy.linalg import norm -from scipy.linalg import get_blas_funcs +from scipy.linalg import get_blas_funcs, issymmetric from warnings import warn @@ -188,15 +188,56 @@ def update(self, delta_x, delta_grad): return if self.first_iteration: # Get user specific scale - if self.init_scale == "auto": + if isinstance(self.init_scale, str) and self.init_scale == "auto": scale = self._auto_scale(delta_x, delta_grad) else: - scale = float(self.init_scale) - # Scale initial matrix with ``scale * np.eye(n)`` + scale = self.init_scale + + # Check for complex: numpy will silently cast a complex array to + # a real one but not so for scalar as it raises a TypeError. + # Checking here brings a consistent behavior. + replace = False + if np.size(scale) == 1: + # to account for the legacy behavior having the exact same cast + scale = float(scale) + elif np.iscomplexobj(scale): + raise TypeError("init_scale contains complex elements, " + "must be real.") + else: # test explicitly for allowed shapes and values + replace = True + if self.approx_type == 'hess': + shape = np.shape(self.B) + dtype = self.B.dtype + else: + shape = np.shape(self.H) + dtype = self.H.dtype + # copy, will replace the original + scale = np.array(scale, dtype=dtype, copy=True) + + # it has to match the shape of the matrix for the multiplication, + # no implicit broadcasting is allowed + if shape != (init_shape := np.shape(scale)): + raise ValueError("If init_scale is an array, it must have the " + f"dimensions of the hess/inv_hess: {shape}." + f" Got {init_shape}.") + if not issymmetric(scale): + raise ValueError("If init_scale is an array, it must be" + " symmetric (passing scipy.linalg.issymmetric)" + " to be an approximation of a hess/inv_hess.") + + # Scale initial matrix with ``scale * np.eye(n)`` or replace + # This is not ideal, we could assign the scale directly in + # initialize, but we would need to if self.approx_type == 'hess': - self.B *= scale + if replace: + self.B = scale + else: + self.B *= scale else: - self.H *= scale + if replace: + self.H = scale + else: + self.H *= scale self.first_iteration = False self._update_implementation(delta_x, delta_grad) @@ -254,13 +295,15 @@ class BFGS(FullHessianUpdateStrategy): unaffected by the exception strategy. By default is equal to 1e-8 when ``exception_strategy = 'skip_update'`` and equal to 0.2 when ``exception_strategy = 'damp_update'``. - init_scale : {float, 'auto'} - Matrix scale at first iteration. At the first - iteration the Hessian matrix or its inverse will be initialized - with ``init_scale*np.eye(n)``, where ``n`` is the problem dimension. + init_scale : {float, np.array, 'auto'} + This parameter can be used to initialize the Hessian or its + inverse. When a float is given, the relevant array is initialized + to ``np.eye(n) * init_scale``, where ``n`` is the problem dimension. + Alternatively, if a precisely ``(n, n)`` shaped, symmetric array is given, + this array will be used. Otherwise an error is generated. Set it to 'auto' in order to use an automatic heuristic for choosing the initial scale. The heuristic is described in [1]_, p.143. - By default uses 'auto'. + The default is 'auto'. Notes ----- @@ -309,7 +352,7 @@ def _update_inverse_hessian(self, ys, Hy, yHy, s): Second Edition (2006). """ self.H = self._syr2(-1.0 / ys, s, Hy, a=self.H) - self.H = self._syr((ys+yHy)/ys**2, s, a=self.H) + self.H = self._syr((ys + yHy) / ys ** 2, s, a=self.H) def _update_hessian(self, ys, Bs, sBs, y): """Update the Hessian matrix. @@ -385,13 +428,15 @@ class SR1(FullHessianUpdateStrategy): defines the minimum denominator magnitude allowed in the update. When the condition is violated we skip the update. By default uses ``1e-8``. - init_scale : {float, 'auto'}, optional - Matrix scale at first iteration. At the first - iteration the Hessian matrix or its inverse will be initialized - with ``init_scale*np.eye(n)``, where ``n`` is the problem dimension. + init_scale : {float, np.array, 'auto'}, optional + This parameter can be used to initialize the Hessian or its + inverse. When a float is given, the relevant array is initialized + to ``np.eye(n) * init_scale``, where ``n`` is the problem dimension. + Alternatively, if a precisely ``(n, n)`` shaped, symmetric array is given, + this array will be used. Otherwise an error is generated. Set it to 'auto' in order to use an automatic heuristic for choosing the initial scale. The heuristic is described in [1]_, p.143. - By default uses 'auto'. + The default is 'auto'. Notes ----- diff --git a/scipy/optimize/tests/test_hessian_update_strategy.py b/scipy/optimize/tests/test_hessian_update_strategy.py index e6872997881c..fe9d7a059b47 100644 --- a/scipy/optimize/tests/test_hessian_update_strategy.py +++ b/scipy/optimize/tests/test_hessian_update_strategy.py @@ -1,5 +1,8 @@ -import numpy as np +import re from copy import deepcopy + +import numpy as np +import pytest from numpy.linalg import norm from numpy.testing import (TestCase, assert_array_almost_equal, assert_array_equal, assert_array_less) @@ -49,19 +52,100 @@ def hess(self, x): class TestHessianUpdateStrategy(TestCase): + def test_hessian_initialization(self): - quasi_newton = (BFGS(), SR1()) - for qn in quasi_newton: - qn.initialize(5, 'hess') - B = qn.get_matrix() + ndims = 5 + symmetric_matrix = np.array([[43, 24, 33, 34, 49], + [24, 36, 44, 15, 44], + [33, 44, 37, 1, 30], + [34, 15, 1, 5, 46], + [49, 44, 30, 46, 22]]) + init_scales = ( + ('auto', np.eye(ndims)), + (2, np.eye(ndims) * 2), + (np.arange(1, ndims + 1) * np.eye(ndims), + np.arange(1, ndims + 1) * np.eye(ndims)), + (symmetric_matrix, symmetric_matrix),) + for approx_type in ['hess', 'inv_hess']: + for init_scale, true_matrix in init_scales: + # large min_{denominator,curvatur} makes them skip an update, + # so we can have our initial matrix + quasi_newton = (BFGS(init_scale=init_scale, + min_curvature=1e50, + exception_strategy='skip_update'), + SR1(init_scale=init_scale, + min_denominator=1e50)) - assert_array_equal(B, np.eye(5)) + for qn in quasi_newton: + qn.initialize(ndims, approx_type) + B = qn.get_matrix() + + assert_array_equal(B, np.eye(ndims)) + # don't test the auto init scale + if isinstance(init_scale, str) and init_scale == 'auto': + continue + + qn.update(np.ones(ndims) * 1e-5, np.arange(ndims) + 0.2) + B = qn.get_matrix() + assert_array_equal(B, true_matrix) # For this list of points, it is known # that no exception occur during the # Hessian update. Hence no update is - # skipped or damped. + # skiped or damped. + + + def test_initialize_catch_illegal(self): + ndims = 3 + # no complex allowed + inits_msg_errtype = ((complex(3.14), + re.escape("float() argument must be a " + "string or a real number, " + "not 'complex'"), + TypeError), + + (np.array([3.2, 2.3, 1.2]).astype(np.complex128), + "init_scale contains complex elements, " + "must be real.", + TypeError), + + (np.array([[43, 24, 33], + [24, 36, 44, ], + [33, 44, 37, ]]).astype(np.complex128), + "init_scale contains complex elements, " + "must be real.", + TypeError), + + # not square + (np.array([[43, 55, 66]]), + re.escape( + "If init_scale is an array, it must have the " + "dimensions of the hess/inv_hess: (3, 3)." + " Got (1, 3)."), + ValueError), + + # not symmetric + (np.array([[43, 24, 33], + [24.1, 36, 44, ], + [33, 44, 37, ]]), + re.escape("If init_scale is an array, it must be" + " symmetric (passing scipy.linalg.issymmetric)" + " to be an approximation of a hess/inv_hess."), + ValueError), + ) + for approx_type in ['hess', 'inv_hess']: + for init_scale, message, errortype in inits_msg_errtype: + # large min_{denominator,curvatur} makes it skip an update, + # so we can retrieve our initial matrix + quasi_newton = (BFGS(init_scale=init_scale), + SR1(init_scale=init_scale)) + + for qn in quasi_newton: + qn.initialize(ndims, approx_type) + with pytest.raises(errortype, match=message): + qn.update(np.ones(ndims), np.arange(ndims)) + def test_rosenbrock_with_no_exception(self): # Define auxiliary problem prob = Rosenbrock(n=5) From 701d8da42825f7c5124dcdaba470a841598091b3 Mon Sep 17 00:00:00 2001 From: Scott Shambaugh <14363975+scottshambaugh@users.noreply.github.com> Date: Sun, 28 Apr 2024 08:00:45 -0600 Subject: [PATCH 56/64] BUG: Fix error with 180 degree rotation in Rotation.align_vectors() with an infinite weight (#20573) * Fix exact rotations at 180 deg, improve near 180 deg Comments * Tests for exact near 180 deg rotations * Fix tests * Code review updates --------- Co-authored-by: Scott Shambaugh --- scipy/spatial/transform/_rotation.pyx | 25 +++++++++++++++--- .../spatial/transform/tests/test_rotation.py | 26 +++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/scipy/spatial/transform/_rotation.pyx b/scipy/spatial/transform/_rotation.pyx index f18a24270277..2c4a3b70df41 100644 --- a/scipy/spatial/transform/_rotation.pyx +++ b/scipy/spatial/transform/_rotation.pyx @@ -3483,13 +3483,32 @@ cdef class Rotation: # We first find the minimum angle rotation between the primary # vectors. cross = np.cross(b_pri[0], a_pri[0]) - theta = atan2(_norm3(cross), np.dot(a_pri[0], b_pri[0])) - if theta < 1e-3: # small angle Taylor series approximation + cross_norm = _norm3(cross) + theta = atan2(cross_norm, _dot3(a_pri[0], b_pri[0])) + tolerance = 1e-3 # tolerance for small angle approximation (rad) + R_flip = cls.identity() + if (np.pi - theta) < tolerance: + # Near pi radians, the Taylor series appoximation of x/sin(x) + # diverges, so for numerical stability we flip pi and then + # rotate back by the small angle pi - theta + if cross_norm == 0: + # For antiparallel vectors, cross = [0, 0, 0] so we need to + # manually set an arbitrary orthogonal axis of rotation + i = np.argmin(np.abs(a_pri[0])) + r = np.zeros(3) + r[i - 1], r[i - 2] = a_pri[0][i - 2], -a_pri[0][i - 1] + else: + r = cross # Shortest angle orthogonal axis of rotation + R_flip = Rotation.from_rotvec(r / np.linalg.norm(r) * np.pi) + theta = np.pi - theta + cross = -cross + if abs(theta) < tolerance: + # Small angle Taylor series approximation for numerical stability theta2 = theta * theta r = cross * (1 + theta2 / 6 + theta2 * theta2 * 7 / 360) else: r = cross * theta / np.sin(theta) - R_pri = cls.from_rotvec(r) + R_pri = cls.from_rotvec(r) * R_flip if N == 1: # No secondary vectors, so we are done diff --git a/scipy/spatial/transform/tests/test_rotation.py b/scipy/spatial/transform/tests/test_rotation.py index bf208f34437b..381ce7ca06f2 100644 --- a/scipy/spatial/transform/tests/test_rotation.py +++ b/scipy/spatial/transform/tests/test_rotation.py @@ -1467,6 +1467,32 @@ def test_align_vectors_parallel(): assert_allclose(R.apply(b[0]), a[0], atol=atol) +def test_align_vectors_antiparallel(): + # Test exact 180 deg rotation + atol = 1e-12 + as_to_test = np.array([[[1, 0, 0], [0, 1, 0]], + [[0, 1, 0], [1, 0, 0]], + [[0, 0, 1], [0, 1, 0]]]) + bs_to_test = [[-a[0], a[1]] for a in as_to_test] + for a, b in zip(as_to_test, bs_to_test): + R, _ = Rotation.align_vectors(a, b, weights=[np.inf, 1]) + assert_allclose(R.magnitude(), np.pi, atol=atol) + assert_allclose(R.apply(b[0]), a[0], atol=atol) + + # Test exact rotations near 180 deg + Rs = Rotation.random(100, random_state=0) + dRs = Rotation.from_rotvec(Rs.as_rotvec()*1e-4) # scale down to small angle + a = [[ 1, 0, 0], [0, 1, 0]] + b = [[-1, 0, 0], [0, 1, 0]] + as_to_test = [] + for dR in dRs: + as_to_test.append([dR.apply(a[0]), a[1]]) + for a in as_to_test: + R, _ = Rotation.align_vectors(a, b, weights=[np.inf, 1]) + R2, _ = Rotation.align_vectors(a, b, weights=[1e10, 1]) + assert_allclose(R.as_matrix(), R2.as_matrix(), atol=atol) + + def test_align_vectors_primary_only(): atol = 1e-12 mats_a = Rotation.random(100, random_state=0).as_matrix() From d575726949d7ac5bc2b75d0f17d0dc13006db22f Mon Sep 17 00:00:00 2001 From: Ralf Gommers Date: Sun, 28 Apr 2024 20:11:38 +0200 Subject: [PATCH 57/64] TYP: update supported Mypy version from 1.0.0 to 1.10.0 (#20600) --- .github/workflows/linux.yml | 2 +- pyproject.toml | 2 +- requirements/dev.txt | 2 +- scipy/_lib/_testutils.py | 4 ++-- scipy/conftest.py | 4 ++-- scipy/optimize/_milp.py | 2 +- scipy/spatial/_ckdtree.pyi | 16 ++++++++-------- scipy/special/_orthogonal.pyi | 2 +- scipy/special/_support_alternative_backends.py | 2 +- scipy/stats/_qmc.py | 6 +++--- scipy/stats/_unuran/unuran_wrapper.pyi | 12 ++++++------ 11 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 80150126c180..1096e7f06e4f 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -118,7 +118,7 @@ jobs: if: matrix.python-version == '3.10' run: | # Packages that are only needed for their annotations - python -m pip install mypy==1.0.0 types-psutil typing_extensions + python -m pip install mypy==1.10.0 types-psutil typing_extensions python -m pip install pybind11 sphinx python -u dev.py mypy diff --git a/pyproject.toml b/pyproject.toml index 0a5efd36115e..826a4976c65f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -98,7 +98,7 @@ doc = [ "jupyterlite-pyodide-kernel", ] dev = [ - "mypy", + "mypy==1.10.0", "typing_extensions", "types-psutil", "pycodestyle", diff --git a/requirements/dev.txt b/requirements/dev.txt index 46086ccdc0fb..47aea16dfb5e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,6 +1,6 @@ # Generated via tools/generate_requirements.py. # Do not edit this file; modify `pyproject.toml` instead and run `python tools/generate_requirements.py`. -mypy +mypy==1.10.0 typing_extensions types-psutil pycodestyle diff --git a/scipy/_lib/_testutils.py b/scipy/_lib/_testutils.py index e8a6d49d65cd..e71c665e51c7 100644 --- a/scipy/_lib/_testutils.py +++ b/scipy/_lib/_testutils.py @@ -17,8 +17,8 @@ try: # Need type: ignore[import-untyped] for mypy >= 1.6 - import cython # type: ignore[import] - from Cython.Compiler.Version import ( # type: ignore[import] + import cython # type: ignore[import-untyped] + from Cython.Compiler.Version import ( # type: ignore[import-untyped] version as cython_version, ) except ImportError: diff --git a/scipy/conftest.py b/scipy/conftest.py index 62fd6d745dcf..90713b01376e 100644 --- a/scipy/conftest.py +++ b/scipy/conftest.py @@ -116,7 +116,7 @@ def check_fpu_mode(request): pass try: - import torch # type: ignore[import] + import torch # type: ignore[import-not-found] xp_available_backends.update({'pytorch': torch}) # can use `mps` or `cpu` torch.set_default_device(SCIPY_DEVICE) @@ -124,7 +124,7 @@ def check_fpu_mode(request): pass try: - import cupy # type: ignore[import] + import cupy # type: ignore[import-not-found] xp_available_backends.update({'cupy': cupy}) except ImportError: pass diff --git a/scipy/optimize/_milp.py b/scipy/optimize/_milp.py index fd9ecf52083f..c1d74499d5b5 100644 --- a/scipy/optimize/_milp.py +++ b/scipy/optimize/_milp.py @@ -2,7 +2,7 @@ import numpy as np from scipy.sparse import csc_array, vstack, issparse from scipy._lib._util import VisibleDeprecationWarning -from ._highs._highs_wrapper import _highs_wrapper # type: ignore[import] +from ._highs._highs_wrapper import _highs_wrapper # type: ignore[import-not-found] from ._constraints import LinearConstraint, Bounds from ._optimize import OptimizeResult from ._linprog_highs import _highs_to_scipy_status_message diff --git a/scipy/spatial/_ckdtree.pyi b/scipy/spatial/_ckdtree.pyi index 7b16a0f73614..c6624b79a6a6 100644 --- a/scipy/spatial/_ckdtree.pyi +++ b/scipy/spatial/_ckdtree.pyi @@ -73,7 +73,7 @@ class cKDTree(Generic[_BoxType]): # The latter gives us more flexibility in setting the generic parameter # though. @overload - def __new__( # type: ignore[misc] + def __new__( # type: ignore[overload-overlap] cls, data: npt.ArrayLike, leafsize: int = ..., @@ -127,7 +127,7 @@ class cKDTree(Generic[_BoxType]): ) -> list[list[int]]: ... @overload - def query_pairs( # type: ignore[misc] + def query_pairs( # type: ignore[overload-overlap] self, r: float, p: float = ..., @@ -144,7 +144,7 @@ class cKDTree(Generic[_BoxType]): ) -> npt.NDArray[np.intp]: ... @overload - def count_neighbors( # type: ignore[misc] + def count_neighbors( # type: ignore[overload-overlap] self, other: cKDTree, r: _ArrayLike0D, @@ -153,7 +153,7 @@ class cKDTree(Generic[_BoxType]): cumulative: bool = ..., ) -> int: ... @overload - def count_neighbors( # type: ignore[misc] + def count_neighbors( # type: ignore[overload-overlap] self, other: cKDTree, r: _ArrayLike0D, @@ -162,7 +162,7 @@ class cKDTree(Generic[_BoxType]): cumulative: bool = ..., ) -> np.float64: ... @overload - def count_neighbors( # type: ignore[misc] + def count_neighbors( # type: ignore[overload-overlap] self, other: cKDTree, r: npt.ArrayLike, @@ -181,7 +181,7 @@ class cKDTree(Generic[_BoxType]): ) -> npt.NDArray[np.float64]: ... @overload - def sparse_distance_matrix( # type: ignore[misc] + def sparse_distance_matrix( # type: ignore[overload-overlap] self, other: cKDTree, max_distance: float, @@ -189,7 +189,7 @@ class cKDTree(Generic[_BoxType]): output_type: Literal["dok_matrix"] = ..., ) -> dok_matrix: ... @overload - def sparse_distance_matrix( # type: ignore[misc] + def sparse_distance_matrix( # type: ignore[overload-overlap] self, other: cKDTree, max_distance: float, @@ -197,7 +197,7 @@ class cKDTree(Generic[_BoxType]): output_type: Literal["coo_matrix"] = ..., ) -> coo_matrix: ... @overload - def sparse_distance_matrix( # type: ignore[misc] + def sparse_distance_matrix( # type: ignore[overload-overlap] self, other: cKDTree, max_distance: float, diff --git a/scipy/special/_orthogonal.pyi b/scipy/special/_orthogonal.pyi index 84da71ee3473..9388513e8735 100644 --- a/scipy/special/_orthogonal.pyi +++ b/scipy/special/_orthogonal.pyi @@ -288,7 +288,7 @@ class orthopoly1d(np.poly1d): @overload def __call__(self, x: _ArrayLike0D) -> Any: ... @overload - def __call__(self, x: np.poly1d) -> np.poly1d: ... # type: ignore[misc] + def __call__(self, x: np.poly1d) -> np.poly1d: ... # type: ignore[overload-overlap] @overload def __call__(self, x: np.typing.ArrayLike) -> np.ndarray: ... diff --git a/scipy/special/_support_alternative_backends.py b/scipy/special/_support_alternative_backends.py index f1b8ba28c2fe..1aa54969063e 100644 --- a/scipy/special/_support_alternative_backends.py +++ b/scipy/special/_support_alternative_backends.py @@ -21,7 +21,7 @@ def get_array_special_func(f_name, xp, n_array_args): elif is_torch(xp): f = getattr(xp.special, f_name, None) elif is_cupy(xp): - import cupyx # type: ignore[import] + import cupyx # type: ignore[import-not-found] f = getattr(cupyx.scipy.special, f_name, None) elif xp.__name__ == f"{array_api_compat_prefix}.jax": f = getattr(xp.scipy.special, f_name, None) diff --git a/scipy/stats/_qmc.py b/scipy/stats/_qmc.py index 484cc1367c03..1aea6a694c8f 100644 --- a/scipy/stats/_qmc.py +++ b/scipy/stats/_qmc.py @@ -668,7 +668,7 @@ def n_primes(n: IntNumber) -> list[int]: 677, 683, 691, 701, 709, 719, 727, 733, 739, 743, 751, 757, 761, 769, 773, 787, 797, 809, 811, 821, 823, 827, 829, 839, 853, 857, 859, 863, 877, 881, 883, 887, 907, 911, 919, 929, 937, 941, 947, - 953, 967, 971, 977, 983, 991, 997][:n] # type: ignore[misc] + 953, 967, 971, 977, 983, 991, 997][:n] if len(primes) < n: big_number = 2000 @@ -1508,7 +1508,7 @@ def _random_oa_lhs(self, n: IntNumber = 4) -> np.ndarray: oa_lhs_sample /= p - return oa_lhs_sample[:, :self.d] # type: ignore + return oa_lhs_sample[:, :self.d] class Sobol(QMCEngine): @@ -1767,7 +1767,7 @@ def _random( ) sample = np.concatenate( [self._first_point, sample] - )[:n] # type: ignore[misc] + )[:n] else: _draw( n=n, num_gen=self.num_generated - 1, dim=self.d, diff --git a/scipy/stats/_unuran/unuran_wrapper.pyi b/scipy/stats/_unuran/unuran_wrapper.pyi index 96b6d4e65edf..46387c5891c6 100644 --- a/scipy/stats/_unuran/unuran_wrapper.pyi +++ b/scipy/stats/_unuran/unuran_wrapper.pyi @@ -18,7 +18,7 @@ class UNURANError(RuntimeError): class Method: @overload - def rvs(self, size: None = ...) -> float | int: ... # type: ignore[misc] + def rvs(self, size: None = ...) -> float | int: ... # type: ignore[overload-overlap] @overload def rvs(self, size: int | tuple[int, ...] = ...) -> np.ndarray: ... def set_random_state(self, random_state: SeedType) -> None: ... @@ -50,7 +50,7 @@ class TransformedDensityRejection(Method): @property def squeeze_area(self) -> float: ... @overload - def ppf_hat(self, u: ArrayLike0D) -> float: ... # type: ignore[misc] + def ppf_hat(self, u: ArrayLike0D) -> float: ... # type: ignore[overload-overlap] @overload def ppf_hat(self, u: npt.ArrayLike) -> np.ndarray: ... @@ -99,11 +99,11 @@ class NumericalInversePolynomial(Method): @property def intervals(self) -> int: ... @overload - def ppf(self, u: ArrayLike0D) -> float: ... # type: ignore[misc] + def ppf(self, u: ArrayLike0D) -> float: ... # type: ignore[overload-overlap] @overload def ppf(self, u: npt.ArrayLike) -> np.ndarray: ... @overload - def cdf(self, x: ArrayLike0D) -> float: ... # type: ignore[misc] + def cdf(self, x: ArrayLike0D) -> float: ... # type: ignore[overload-overlap] @overload def cdf(self, x: npt.ArrayLike) -> np.ndarray: ... def u_error(self, sample_size: int = ...) -> UError: ... @@ -135,7 +135,7 @@ class NumericalInverseHermite(Method): @property def intervals(self) -> int: ... @overload - def ppf(self, u: ArrayLike0D) -> float: ... # type: ignore[misc] + def ppf(self, u: ArrayLike0D) -> float: ... # type: ignore[overload-overlap] @overload def ppf(self, u: npt.ArrayLike) -> np.ndarray: ... def qrvs(self, @@ -174,6 +174,6 @@ class DiscreteGuideTable(Method): guide_factor: float = ..., random_state: SeedType = ...) -> None: ... @overload - def ppf(self, u: ArrayLike0D) -> float: ... # type: ignore[misc] + def ppf(self, u: ArrayLike0D) -> float: ... # type: ignore[overload-overlap] @overload def ppf(self, u: npt.ArrayLike) -> np.ndarray: ... From b9668a0f5571425620f92d7f0f3556af6f8cdfce Mon Sep 17 00:00:00 2001 From: lucascolley Date: Sun, 28 Apr 2024 23:32:53 +0100 Subject: [PATCH 58/64] MAINT: lint: temporarily disable UP031 [lint only] --- tools/lint.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/lint.toml b/tools/lint.toml index 69f7e9e2e812..4e7e66a09967 100644 --- a/tools/lint.toml +++ b/tools/lint.toml @@ -12,7 +12,8 @@ target-version = "py39" # `B028` added in gh-19623. # `ICN001` added in gh-20382 to enforce common conventions for imports select = ["E", "F", "PGH004", "UP", "B028", "ICN001"] -ignore = ["E741"] +# UP031 should be enabled once someone fixes the errors. +ignore = ["E741", "UP031"] # Allow unused variables when underscore-prefixed. dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" From 4ee0de31b758f81be85435a0ad72d315a840441d Mon Sep 17 00:00:00 2001 From: Jake Bowhay <60778417+j-bowhay@users.noreply.github.com> Date: Mon, 29 Apr 2024 10:08:45 +0100 Subject: [PATCH 59/64] ENH: constants: add array api support (#20593) Co-authored-by: Ralf Gommers --- .github/workflows/array_api.yml | 1 + doc/source/dev/api-dev/array_api.rst | 1 + scipy/_lib/_array_api.py | 13 ++- scipy/constants/_constants.py | 32 ++++--- scipy/constants/tests/test_constants.py | 116 +++++++++++++++++------- 5 files changed, 116 insertions(+), 47 deletions(-) diff --git a/.github/workflows/array_api.yml b/.github/workflows/array_api.yml index 25550618857a..d046777ac3ca 100644 --- a/.github/workflows/array_api.yml +++ b/.github/workflows/array_api.yml @@ -91,6 +91,7 @@ jobs: export OMP_NUM_THREADS=2 # expand as new modules are added python dev.py --no-build test -b all -s cluster -- --durations 3 --timeout=60 + python dev.py --no-build test -b all -s constants -- --durations 3 --timeout=60 python dev.py --no-build test -b all -s fft -- --durations 3 --timeout=60 python dev.py --no-build test -b all -t scipy.special.tests.test_support_alternative_backends -- --durations 3 --timeout=60 python dev.py --no-build test -b all -t scipy._lib.tests.test_array_api diff --git a/doc/source/dev/api-dev/array_api.rst b/doc/source/dev/api-dev/array_api.rst index 660871bcd5cb..0a437c5f1a10 100644 --- a/doc/source/dev/api-dev/array_api.rst +++ b/doc/source/dev/api-dev/array_api.rst @@ -95,6 +95,7 @@ variable is set: - `scipy.cluster.hierarchy` - `scipy.cluster.vq` +- `scipy.constants` - `scipy.fft` Support is provided in `scipy.special` for the following functions: diff --git a/scipy/_lib/_array_api.py b/scipy/_lib/_array_api.py index e32d81c70cf3..98d7f50bb6ee 100644 --- a/scipy/_lib/_array_api.py +++ b/scipy/_lib/_array_api.py @@ -120,9 +120,11 @@ def array_namespace(*arrays): def _asarray( - array, dtype=None, order=None, copy=None, *, xp=None, check_finite=False + array, dtype=None, order=None, copy=None, *, xp=None, check_finite=False, + subok=False ): - """SciPy-specific replacement for `np.asarray` with `order` and `check_finite`. + """SciPy-specific replacement for `np.asarray` with `order`, `check_finite`, and + `subok`. Memory layout parameter `order` is not exposed in the Array API standard. `order` is only enforced if the input array implementation @@ -131,13 +133,18 @@ def _asarray( `check_finite` is also not a keyword in the array API standard; included here for convenience rather than that having to be a separate function call inside SciPy functions. + + `subok` is included to allow this function to preserve the behaviour of + `np.asanyarray` for NumPy based inputs. """ if xp is None: xp = array_namespace(array) if xp.__name__ in {"numpy", "scipy._lib.array_api_compat.numpy"}: # Use NumPy API to support order if copy is True: - array = np.array(array, order=order, dtype=dtype) + array = np.array(array, order=order, dtype=dtype, subok=subok) + elif subok: + array = np.asanyarray(array, order=order, dtype=dtype) else: array = np.asarray(array, order=order, dtype=dtype) diff --git a/scipy/constants/_constants.py b/scipy/constants/_constants.py index 666690d47a83..7163518c0c15 100644 --- a/scipy/constants/_constants.py +++ b/scipy/constants/_constants.py @@ -13,11 +13,13 @@ from typing import TYPE_CHECKING, Any from ._codata import value as _cd -import numpy as np if TYPE_CHECKING: import numpy.typing as npt +from scipy._lib._array_api import array_namespace, _asarray + + """ BasSw 2006 physical constants: imported from CODATA @@ -269,19 +271,21 @@ def convert_temperature( array([ 233.15, 313.15]) """ + xp = array_namespace(val) + _val = _asarray(val, xp=xp, subok=True) # Convert from `old_scale` to Kelvin if old_scale.lower() in ['celsius', 'c']: - tempo = np.asanyarray(val) + zero_Celsius + tempo = _val + zero_Celsius elif old_scale.lower() in ['kelvin', 'k']: - tempo = np.asanyarray(val) + tempo = _val elif old_scale.lower() in ['fahrenheit', 'f']: - tempo = (np.asanyarray(val) - 32) * 5 / 9 + zero_Celsius + tempo = (_val - 32) * 5 / 9 + zero_Celsius elif old_scale.lower() in ['rankine', 'r']: - tempo = np.asanyarray(val) * 5 / 9 + tempo = _val * 5 / 9 else: - raise NotImplementedError("%s scale is unsupported: supported scales " - "are Celsius, Kelvin, Fahrenheit, and " - "Rankine" % old_scale) + raise NotImplementedError(f"{old_scale=} is unsupported: supported scales " + "are Celsius, Kelvin, Fahrenheit, and " + "Rankine") # and from Kelvin to `new_scale`. if new_scale.lower() in ['celsius', 'c']: res = tempo - zero_Celsius @@ -292,9 +296,9 @@ def convert_temperature( elif new_scale.lower() in ['rankine', 'r']: res = tempo * 9 / 5 else: - raise NotImplementedError("'%s' scale is unsupported: supported " - "scales are 'Celsius', 'Kelvin', " - "'Fahrenheit', and 'Rankine'" % new_scale) + raise NotImplementedError(f"{new_scale=} is unsupported: supported " + "scales are 'Celsius', 'Kelvin', " + "'Fahrenheit', and 'Rankine'") return res @@ -329,7 +333,8 @@ def lambda2nu(lambda_: npt.ArrayLike) -> Any: array([ 2.99792458e+08, 1.00000000e+00]) """ - return c / np.asanyarray(lambda_) + xp = array_namespace(lambda_) + return c / _asarray(lambda_, xp=xp, subok=True) def nu2lambda(nu: npt.ArrayLike) -> Any: @@ -359,4 +364,5 @@ def nu2lambda(nu: npt.ArrayLike) -> Any: array([ 2.99792458e+08, 1.00000000e+00]) """ - return c / np.asanyarray(nu) + xp = array_namespace(nu) + return c / _asarray(nu, xp=xp, subok=True) diff --git a/scipy/constants/tests/test_constants.py b/scipy/constants/tests/test_constants.py index 8d7461d978fa..9f5c241b13f7 100644 --- a/scipy/constants/tests/test_constants.py +++ b/scipy/constants/tests/test_constants.py @@ -1,35 +1,89 @@ -from numpy.testing import assert_equal, assert_allclose +import pytest + import scipy.constants as sc +from scipy.conftest import array_api_compatible +from scipy._lib._array_api import xp_assert_equal, xp_assert_close + + +pytestmark = [array_api_compatible, pytest.mark.usefixtures("skip_xp_backends")] +skip_xp_backends = pytest.mark.skip_xp_backends + + +class TestConvertTemperature: + def test_convert_temperature(self, xp): + xp_assert_equal(sc.convert_temperature(xp.asarray(32.), 'f', 'Celsius'), + xp.asarray(0.0)) + xp_assert_equal(sc.convert_temperature(xp.asarray([0., 0.]), + 'celsius', 'Kelvin'), + xp.asarray([273.15, 273.15])) + xp_assert_equal(sc.convert_temperature(xp.asarray([0., 0.]), 'kelvin', 'c'), + xp.asarray([-273.15, -273.15])) + xp_assert_equal(sc.convert_temperature(xp.asarray([32., 32.]), 'f', 'k'), + xp.asarray([273.15, 273.15])) + xp_assert_equal(sc.convert_temperature(xp.asarray([273.15, 273.15]), + 'kelvin', 'F'), + xp.asarray([32., 32.])) + xp_assert_equal(sc.convert_temperature(xp.asarray([0., 0.]), 'C', 'fahrenheit'), + xp.asarray([32., 32.])) + xp_assert_close(sc.convert_temperature(xp.asarray([0., 0.], dtype=xp.float64), + 'c', 'r'), + xp.asarray([491.67, 491.67], dtype=xp.float64), + rtol=0., atol=1e-13) + xp_assert_close(sc.convert_temperature(xp.asarray([491.67, 491.67], + dtype=xp.float64), + 'Rankine', 'C'), + xp.asarray([0., 0.], dtype=xp.float64), rtol=0., atol=1e-13) + xp_assert_close(sc.convert_temperature(xp.asarray([491.67, 491.67], + dtype=xp.float64), + 'r', 'F'), + xp.asarray([32., 32.], dtype=xp.float64), rtol=0., atol=1e-13) + xp_assert_close(sc.convert_temperature(xp.asarray([32., 32.], dtype=xp.float64), + 'fahrenheit', 'R'), + xp.asarray([491.67, 491.67], dtype=xp.float64), + rtol=0., atol=1e-13) + xp_assert_close(sc.convert_temperature(xp.asarray([273.15, 273.15], + dtype=xp.float64), + 'K', 'R'), + xp.asarray([491.67, 491.67], dtype=xp.float64), + rtol=0., atol=1e-13) + xp_assert_close(sc.convert_temperature(xp.asarray([491.67, 0.], + dtype=xp.float64), + 'rankine', 'kelvin'), + xp.asarray([273.15, 0.], dtype=xp.float64), rtol=0., atol=1e-13) + + @skip_xp_backends(np_only=True, reasons=['Python list input uses NumPy backend']) + def test_convert_temperature_array_like(self): + xp_assert_close(sc.convert_temperature([491.67, 0.], 'rankine', 'kelvin'), + [273.15, 0.], rtol=0., atol=1e-13) + + + @skip_xp_backends(np_only=True, reasons=['Python int input uses NumPy backend']) + def test_convert_temperature_errors(self, xp): + with pytest.raises(NotImplementedError, match="old_scale="): + sc.convert_temperature(1, old_scale="cheddar", new_scale="kelvin") + with pytest.raises(NotImplementedError, match="new_scale="): + sc.convert_temperature(1, old_scale="kelvin", new_scale="brie") + + +class TestLambdaToNu: + def test_lambda_to_nu(self, xp): + xp_assert_equal(sc.lambda2nu(xp.asarray([sc.speed_of_light, 1])), + xp.asarray([1, sc.speed_of_light])) + + + @skip_xp_backends(np_only=True, reasons=['Python list input uses NumPy backend']) + def test_lambda_to_nu_array_like(self, xp): + xp_assert_equal(sc.lambda2nu([sc.speed_of_light, 1]), + [1, sc.speed_of_light]) + +class TestNuToLambda: + def test_nu_to_lambda(self, xp): + xp_assert_equal(sc.nu2lambda(xp.asarray([sc.speed_of_light, 1])), + xp.asarray([1, sc.speed_of_light])) -def test_convert_temperature(): - assert_equal(sc.convert_temperature(32, 'f', 'Celsius'), 0) - assert_equal(sc.convert_temperature([0, 0], 'celsius', 'Kelvin'), - [273.15, 273.15]) - assert_equal(sc.convert_temperature([0, 0], 'kelvin', 'c'), - [-273.15, -273.15]) - assert_equal(sc.convert_temperature([32, 32], 'f', 'k'), [273.15, 273.15]) - assert_equal(sc.convert_temperature([273.15, 273.15], 'kelvin', 'F'), - [32, 32]) - assert_equal(sc.convert_temperature([0, 0], 'C', 'fahrenheit'), [32, 32]) - assert_allclose(sc.convert_temperature([0, 0], 'c', 'r'), [491.67, 491.67], - rtol=0., atol=1e-13) - assert_allclose(sc.convert_temperature([491.67, 491.67], 'Rankine', 'C'), - [0., 0.], rtol=0., atol=1e-13) - assert_allclose(sc.convert_temperature([491.67, 491.67], 'r', 'F'), - [32., 32.], rtol=0., atol=1e-13) - assert_allclose(sc.convert_temperature([32, 32], 'fahrenheit', 'R'), - [491.67, 491.67], rtol=0., atol=1e-13) - assert_allclose(sc.convert_temperature([273.15, 273.15], 'K', 'R'), - [491.67, 491.67], rtol=0., atol=1e-13) - assert_allclose(sc.convert_temperature([491.67, 0.], 'rankine', 'kelvin'), - [273.15, 0.], rtol=0., atol=1e-13) - - -def test_lambda_to_nu(): - assert_equal(sc.lambda2nu([sc.speed_of_light, 1]), [1, sc.speed_of_light]) - - -def test_nu_to_lambda(): - assert_equal(sc.nu2lambda([sc.speed_of_light, 1]), [1, sc.speed_of_light]) + @skip_xp_backends(np_only=True, reasons=['Python list input uses NumPy backend']) + def test_nu_to_lambda_array_like(self, xp): + xp_assert_equal(sc.nu2lambda([sc.speed_of_light, 1]), + [1, sc.speed_of_light]) From fa9852cf57667c4fe8f51c8861af1bd41a4489f5 Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Mon, 29 Apr 2024 17:29:02 -0700 Subject: [PATCH 60/64] MAINT: optimize._chandrupatla: reduce xatol (#20501) * MAINT: optimize._chandrupatla: reduce default xatol * MAINT: optimize._chandrupatla: increase maximum number of iterations * MAINT: optimize._chandrupatla: scale maxiter with dtype --- scipy/optimize/_chandrupatla.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/scipy/optimize/_chandrupatla.py b/scipy/optimize/_chandrupatla.py index 26c65c7bcd87..6ce98230af68 100644 --- a/scipy/optimize/_chandrupatla.py +++ b/scipy/optimize/_chandrupatla.py @@ -1,10 +1,10 @@ import numpy as np -from ._zeros_py import _xtol, _rtol, _iter +from ._zeros_py import _rtol import scipy._lib._elementwise_iterative_method as eim from scipy._lib._util import _RichResult -def _chandrupatla(func, a, b, *, args=(), xatol=_xtol, xrtol=_rtol, - fatol=None, frtol=0, maxiter=_iter, callback=None): +def _chandrupatla(func, a, b, *, args=(), xatol=None, xrtol=_rtol, + fatol=None, frtol=0, maxiter=None, callback=None): """Find the root of an elementwise function using Chandrupatla's algorithm. For each element of the output of `func`, `chandrupatla` seeks the scalar @@ -33,6 +33,8 @@ def _chandrupatla(func, a, b, *, args=(), xatol=_xtol, xrtol=_rtol, See Notes for details. maxiter : int, optional The maximum number of iterations of the algorithm to perform. + The default is the maximum possible number of bisections within + the (normal) floating point numbers of the relevant dtype. callback : callable, optional An optional user-supplied function to be called before the first iteration and after each iteration. @@ -84,9 +86,9 @@ def _chandrupatla(func, a, b, *, args=(), xatol=_xtol, xrtol=_rtol, ``fun(xmin) <= fatol + abs(fmin0) * frtol``. This is equivalent to the termination condition described in [1]_ with ``xrtol = 4e-10``, ``xatol = 1e-5``, and ``fatol = frtol = 0``. The default values are - ``xatol = 2e-12``, ``xrtol = 4 * np.finfo(float).eps``, ``frtol = 0``, - and ``fatol`` is the smallest normal number of the ``dtype`` returned - by ``func``. + ``xatol = 4*tiny``, ``xrtol = 4*eps``, ``frtol = 0``, and ``fatol = tiny``, + where ``eps`` and ``tiny`` are the precision and smallest normal number + of the result ``dtype`` of function inputs and outputs. References ---------- @@ -128,10 +130,11 @@ def _chandrupatla(func, a, b, *, args=(), xatol=_xtol, xrtol=_rtol, f1, f2 = fs status = np.full_like(x1, eim._EINPROGRESS, dtype=int) # in progress nit, nfev = 0, 2 # two function evaluations performed above - xatol = _xtol if xatol is None else xatol + xatol = 4*np.finfo(dtype).tiny if xatol is None else xatol xrtol = _rtol if xrtol is None else xrtol fatol = np.finfo(dtype).tiny if fatol is None else fatol frtol = frtol * np.minimum(np.abs(f1), np.abs(f2)) + maxiter = 2**np.finfo(dtype).nexp if maxiter is None else maxiter work = _RichResult(x1=x1, f1=f1, x2=x2, f2=f2, x3=None, f3=None, t=0.5, xatol=xatol, xrtol=xrtol, fatol=fatol, frtol=frtol, nit=nit, nfev=nfev, status=status) @@ -245,9 +248,10 @@ def _chandrupatla_iv(func, args, xatol, xrtol, or np.any(np.isnan(tols)) or tols.shape != (4,)): raise ValueError('Tolerances must be non-negative scalars.') - maxiter_int = int(maxiter) - if maxiter != maxiter_int or maxiter < 0: - raise ValueError('`maxiter` must be a non-negative integer.') + if maxiter is not None: + maxiter_int = int(maxiter) + if maxiter != maxiter_int or maxiter < 0: + raise ValueError('`maxiter` must be a non-negative integer.') if callback is not None and not callable(callback): raise ValueError('`callback` must be callable.') @@ -344,7 +348,7 @@ def _chandrupatla_minimize(func, x1, x2, x3, *, args=(), xatol=None, or ``(f1 - 2*f2 + f3)/2 <= abs(f2)*frtol + fatol``. Note that first of these differs from the termination conditions described in [1]_. The default values of `xrtol` is the square root of the precision of the - appropriate dtype, and ``xatol=fatol = frtol`` is the smallest normal + appropriate dtype, and ``xatol = fatol = frtol`` is the smallest normal number of the appropriate dtype. References From 4222ece47be5b15654b6f9889947514f9723e198 Mon Sep 17 00:00:00 2001 From: Warren Weckesser Date: Tue, 30 Apr 2024 00:28:53 -0400 Subject: [PATCH 61/64] ENH: stats: Implement _isf for burr12 (#20615) * STY: Add a few missing blank lines. * ENH: stats: Implement _isf for burr12 to improve accuracy when p is small. --- scipy/stats/_continuous_distns.py | 3 +++ scipy/stats/tests/test_distributions.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/scipy/stats/_continuous_distns.py b/scipy/stats/_continuous_distns.py index d51602133b6d..5e6832de507d 100644 --- a/scipy/stats/_continuous_distns.py +++ b/scipy/stats/_continuous_distns.py @@ -1321,6 +1321,9 @@ def _ppf(self, q, c, d): # that does a better job handling small values of q. return sc.expm1(-1/d * sc.log1p(-q))**(1/c) + def _isf(self, p, c, d): + return sc.expm1(-1/d * np.log(p))**(1/c) + def _munp(self, n, c, d): def moment_if_exists(n, c, d): nc = 1. * n / c diff --git a/scipy/stats/tests/test_distributions.py b/scipy/stats/tests/test_distributions.py index e5a12be7a634..ad9fc1d37516 100644 --- a/scipy/stats/tests/test_distributions.py +++ b/scipy/stats/tests/test_distributions.py @@ -3158,6 +3158,7 @@ def test_fit_analytic_mle(self, c, loc, scale, fix_c, fix_scale): _assert_less_or_close_loglike(stats.loglaplace, data, **kwds) + class TestPowerlaw: # In the following data, `sf` was computed with mpmath. @@ -3810,6 +3811,7 @@ def test_compare_with_gamlss_r(self, gamlss_pdf_data, a, b): x, pdf = data["x"], data["pdf"] assert_allclose(pdf, stats.jf_skew_t(a, b).pdf(x), rtol=1e-12) + # Test data for TestSkewNorm.test_noncentral_moments() # The expected noncentral moments were computed by Wolfram Alpha. # In Wolfram Alpha, enter @@ -4000,6 +4002,7 @@ def test_fit_gh19332(self): # Compare overridden fit against stats.fit rng = np.random.default_rng(9842356982345693637) bounds = {'a': (-5, 5), 'loc': (-10, 10), 'scale': (1e-16, 10)} + def optimizer(fun, bounds): return differential_evolution(fun, bounds, seed=rng) @@ -4793,6 +4796,7 @@ def test_fit_mm(self, a, loc, scale, fix_a, fix_loc, fix_scale): if nfree >= 3: assert_allclose(dist.moment(3), np.mean(data**3)) + def test_pdf_overflow_gh19616(): # Confirm that gh19616 (intermediate over/underflows in PDF) is resolved # Reference value from R GeneralizedHyperbolic library @@ -7509,6 +7513,19 @@ def test_moments_edge(self): res = stats.burr12(c, d).stats('mvsk') assert_allclose(res, ref, rtol=1e-14) + # Reference values were computed with mpmath using mp.dps = 80 + # and then cast to float. + @pytest.mark.parametrize( + 'p, c, d, ref', + [(1e-12, 20, 0.5, 15.848931924611135), + (1e-19, 20, 0.5, 79.43282347242815), + (1e-12, 0.25, 35, 2.0888618213462466), + (1e-80, 0.25, 35, 1360930951.7972188)] + ) + def test_isf_near_zero(self, p, c, d, ref): + x = stats.burr12.isf(p, c, d) + assert_allclose(x, ref, rtol=1e-14) + class TestStudentizedRange: # For alpha = .05, .01, and .001, and for each value of From 6ae5a2e707fa702200a66da3c089f1533fd72180 Mon Sep 17 00:00:00 2001 From: h-vetinari Date: Tue, 30 Apr 2024 17:39:07 +1100 Subject: [PATCH 62/64] BUG: special: handle uint arrays in factorial{,2,k} (#20607) --- scipy/special/_basic.py | 3 ++- scipy/special/tests/test_basic.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scipy/special/_basic.py b/scipy/special/_basic.py index 7b3fd8106c52..2ef20eb0c1d7 100644 --- a/scipy/special/_basic.py +++ b/scipy/special/_basic.py @@ -2943,7 +2943,8 @@ def corr(k, r): return np.power(k, -r / k) / gamma(r / k + 1) * r for r in np.unique(n_mod_k): if r == 0: continue - result[n_mod_k == r] *= corr(k, r) + # cast to int because uint types break on `-r` + result[n_mod_k == r] *= corr(k, int(r)) return result diff --git a/scipy/special/tests/test_basic.py b/scipy/special/tests/test_basic.py index 6d0be6be2743..9284110e937e 100644 --- a/scipy/special/tests/test_basic.py +++ b/scipy/special/tests/test_basic.py @@ -2130,9 +2130,13 @@ def _check(res, nucleus): _check(special.factorialk(n, 3, exact=exact), exp_nucleus[3]) @pytest.mark.parametrize("exact", [True, False]) + @pytest.mark.parametrize("dtype", [ + None, int, np.int8, np.int16, np.int32, np.int64, + np.uint8, np.uint16, np.uint32, np.uint64 + ]) @pytest.mark.parametrize("dim", range(0, 5)) - def test_factorialx_array_dimension(self, dim, exact): - n = np.array(5, ndmin=dim) + def test_factorialx_array_dimension(self, dim, dtype, exact): + n = np.array(5, dtype=dtype, ndmin=dim) exp = {1: 120, 2: 15, 3: 10} assert_allclose(special.factorial(n, exact=exact), np.array(exp[1], ndmin=dim)) From 34c8578484485559a3282f2bd53e546c047cf910 Mon Sep 17 00:00:00 2001 From: Jake Bowhay Date: Tue, 30 Apr 2024 09:09:31 +0100 Subject: [PATCH 63/64] DOC: integrate: remove references to deprecated and legacy functions [docs only] --- doc/source/tutorial/integrate.rst | 25 ++++++------------------- scipy/integrate/_quadpack_py.py | 10 ---------- scipy/integrate/_quadrature.py | 17 ----------------- 3 files changed, 6 insertions(+), 46 deletions(-) diff --git a/doc/source/tutorial/integrate.rst b/doc/source/tutorial/integrate.rst index f6d5965a59e7..970b6353ddce 100644 --- a/doc/source/tutorial/integrate.rst +++ b/doc/source/tutorial/integrate.rst @@ -265,23 +265,12 @@ which is the same result as before. Gaussian quadrature ------------------- -A few functions are also provided in order to perform simple Gaussian -quadrature over a fixed interval. The first is :obj:`fixed_quad`, which -performs fixed-order Gaussian quadrature. The second function is -:obj:`quadrature`, which performs Gaussian quadrature of multiple -orders until the difference in the integral estimate is beneath some -tolerance supplied by the user. These functions both use the module -``scipy.special.orthogonal``, which can calculate the roots and quadrature -weights of a large variety of orthogonal polynomials (the polynomials -themselves are available as special functions returning instances of -the polynomial class --- e.g., :obj:`special.legendre `). - - -Romberg Integration -------------------- - -Romberg's method [WPR]_ is another method for numerically evaluating an -integral. See the help function for :func:`romberg` for further details. +:obj:`fixed_quad` performs fixed-order Gaussian quadrature over a fixed interval. This +function uses the collection of orthogonal polynomials provided by ``scipy.special``, +which can calculate the roots and quadrature weights of a large variety of orthogonal +polynomials (the polynomials themselves are available as special functions returning +instances of the polynomial class --- e.g., +:obj:`special.legendre `). Integrating using Samples @@ -787,6 +776,4 @@ Let's ensure that they have computed the same result:: References ~~~~~~~~~~ -.. [WPR] https://en.wikipedia.org/wiki/Romberg's_method - .. [MOL] https://en.wikipedia.org/wiki/Method_of_lines diff --git a/scipy/integrate/_quadpack_py.py b/scipy/integrate/_quadpack_py.py index a9d200e6ddca..599a30dffe5c 100644 --- a/scipy/integrate/_quadpack_py.py +++ b/scipy/integrate/_quadpack_py.py @@ -122,9 +122,6 @@ def quad(func, a, b, args=(), full_output=0, epsabs=1.49e-8, epsrel=1.49e-8, tplquad : triple integral nquad : n-dimensional integrals (uses `quad` recursively) fixed_quad : fixed-order Gaussian quadrature - quadrature : adaptive Gaussian quadrature - odeint : ODE integrator - ode : ODE integrator simpson : integrator for sampled data romb : integrator for sampled data scipy.special : for coefficients and roots of orthogonal polynomials @@ -727,9 +724,6 @@ def dblquad(func, a, b, gfun, hfun, args=(), epsabs=1.49e-8, epsrel=1.49e-8): tplquad : triple integral nquad : N-dimensional integrals fixed_quad : fixed-order Gaussian quadrature - quadrature : adaptive Gaussian quadrature - odeint : ODE integrator - ode : ODE integrator simpson : integrator for sampled data romb : integrator for sampled data scipy.special : for coefficients and roots of orthogonal polynomials @@ -860,14 +854,11 @@ def tplquad(func, a, b, gfun, hfun, qfun, rfun, args=(), epsabs=1.49e-8, See Also -------- quad : Adaptive quadrature using QUADPACK - quadrature : Adaptive Gaussian quadrature fixed_quad : Fixed-order Gaussian quadrature dblquad : Double integrals nquad : N-dimensional integrals romb : Integrators for sampled data simpson : Integrators for sampled data - ode : ODE integrators - odeint : ODE integrators scipy.special : For coefficients and roots of orthogonal polynomials Notes @@ -1045,7 +1036,6 @@ def nquad(func, ranges, args=None, opts=None, full_output=False): quad : 1-D numerical integration dblquad, tplquad : double and triple integrals fixed_quad : fixed-order Gaussian quadrature - quadrature : adaptive Gaussian quadrature Notes ----- diff --git a/scipy/integrate/_quadrature.py b/scipy/integrate/_quadrature.py index 045704204480..7fe4ef9424eb 100644 --- a/scipy/integrate/_quadrature.py +++ b/scipy/integrate/_quadrature.py @@ -217,13 +217,9 @@ def fixed_quad(func, a, b, args=(), n=5): quad : adaptive quadrature using QUADPACK dblquad : double integrals tplquad : triple integrals - romberg : adaptive Romberg quadrature - quadrature : adaptive Gaussian quadrature romb : integrators for sampled data simpson : integrators for sampled data cumulative_trapezoid : cumulative integration for sampled data - ode : ODE integrator - odeint : ODE integrator Examples -------- @@ -345,7 +341,6 @@ def quadrature(func, a, b, args=(), tol=1.49e-8, rtol=1.49e-8, maxiter=50, See Also -------- - romberg : adaptive Romberg quadrature fixed_quad : fixed-order Gaussian quadrature quad : adaptive quadrature using QUADPACK dblquad : double integrals @@ -353,8 +348,6 @@ def quadrature(func, a, b, args=(), tol=1.49e-8, rtol=1.49e-8, maxiter=50, romb : integrator for sampled data simpson : integrator for sampled data cumulative_trapezoid : cumulative integration for sampled data - ode : ODE integrator - odeint : ODE integrator Examples -------- @@ -437,14 +430,10 @@ def cumulative_trapezoid(y, x=None, dx=1.0, axis=-1, initial=None): numpy.cumsum, numpy.cumprod cumulative_simpson : cumulative integration using Simpson's 1/3 rule quad : adaptive quadrature using QUADPACK - romberg : adaptive Romberg quadrature - quadrature : adaptive Gaussian quadrature fixed_quad : fixed-order Gaussian quadrature dblquad : double integrals tplquad : triple integrals romb : integrators for sampled data - ode : ODE integrators - odeint : ODE integrators Examples -------- @@ -987,15 +976,11 @@ def romb(y, dx=1.0, axis=-1, show=False): See Also -------- quad : adaptive quadrature using QUADPACK - romberg : adaptive Romberg quadrature - quadrature : adaptive Gaussian quadrature fixed_quad : fixed-order Gaussian quadrature dblquad : double integrals tplquad : triple integrals simpson : integrators for sampled data cumulative_trapezoid : cumulative integration for sampled data - ode : ODE integrators - odeint : ODE integrators Examples -------- @@ -1202,8 +1187,6 @@ def romberg(function, a, b, args=(), tol=1.48e-8, rtol=1.48e-8, show=False, romb : Integrators for sampled data. simpson : Integrators for sampled data. cumulative_trapezoid : Cumulative integration for sampled data. - ode : ODE integrator. - odeint : ODE integrator. References ---------- From 713bce97dd8a6c755a9e8a59d36e6e92ea39a80e Mon Sep 17 00:00:00 2001 From: Matt Haberland Date: Tue, 30 Apr 2024 06:05:43 -0700 Subject: [PATCH 64/64] TST: adjust other very slow tests (#20487) --- scipy/_lib/tests/test_import_cycles.py | 2 + scipy/integrate/tests/test_quadpack.py | 1 + scipy/interpolate/tests/test_gil.py | 2 +- scipy/interpolate/tests/test_rgi.py | 2 +- .../tests/test__differential_evolution.py | 2 +- .../tests/test_minimize_constrained.py | 129 +++++++++--------- .../linalg/_eigen/lobpcg/tests/test_lobpcg.py | 16 +-- scipy/sparse/linalg/tests/test_propack.py | 1 + 8 files changed, 79 insertions(+), 76 deletions(-) diff --git a/scipy/_lib/tests/test_import_cycles.py b/scipy/_lib/tests/test_import_cycles.py index e61c57093f62..feaf2ff4cf64 100644 --- a/scipy/_lib/tests/test_import_cycles.py +++ b/scipy/_lib/tests/test_import_cycles.py @@ -1,3 +1,4 @@ +import pytest import sys import subprocess @@ -7,6 +8,7 @@ # Check that all modules are importable in a new Python process. # This is not necessarily true if there are import cycles present. +@pytest.mark.slow def test_public_modules_importable(): pids = [subprocess.Popen([sys.executable, '-c', f'import {module}']) for module in PUBLIC_MODULES] diff --git a/scipy/integrate/tests/test_quadpack.py b/scipy/integrate/tests/test_quadpack.py index 90bf6006cf1f..a7f6c7d195b0 100644 --- a/scipy/integrate/tests/test_quadpack.py +++ b/scipy/integrate/tests/test_quadpack.py @@ -342,6 +342,7 @@ def simpfunc(z, y, x, t): # Note order of arguments. (2.,)), 2*8/3.0 * (b**4.0 - a**4.0)) + @pytest.mark.xslow @pytest.mark.parametrize( "x_lower, x_upper, y_lower, y_upper, z_lower, z_upper, expected", [ diff --git a/scipy/interpolate/tests/test_gil.py b/scipy/interpolate/tests/test_gil.py index 0902308fb6af..818cd7275bf3 100644 --- a/scipy/interpolate/tests/test_gil.py +++ b/scipy/interpolate/tests/test_gil.py @@ -28,7 +28,7 @@ def run(self): return WorkerThread() - @pytest.mark.slow + @pytest.mark.xslow @pytest.mark.xfail(reason='race conditions, may depend on system load') def test_rectbivariatespline(self): def generate_params(n_points): diff --git a/scipy/interpolate/tests/test_rgi.py b/scipy/interpolate/tests/test_rgi.py index 95b5e2253031..e4855d205293 100644 --- a/scipy/interpolate/tests/test_rgi.py +++ b/scipy/interpolate/tests/test_rgi.py @@ -476,7 +476,7 @@ def f(x, y): ]) def test_descending_points_nd(self, method, ndims, func): - if ndims == 5 and method in {"cubic", "quintic"}: + if ndims >= 4 and method in {"cubic", "quintic"}: pytest.skip("too slow; OOM (quintic); or nearly so (cubic)") rng = np.random.default_rng(42) diff --git a/scipy/optimize/tests/test__differential_evolution.py b/scipy/optimize/tests/test__differential_evolution.py index d5638a120aaa..8f83c1601648 100644 --- a/scipy/optimize/tests/test__differential_evolution.py +++ b/scipy/optimize/tests/test__differential_evolution.py @@ -1339,7 +1339,7 @@ def c1(x): assert_(np.all(res.x >= np.array(bounds)[:, 0])) assert_(np.all(res.x <= np.array(bounds)[:, 1])) - @pytest.mark.slow + @pytest.mark.xslow @pytest.mark.xfail(platform.machine() == 'ppc64le', reason="fails on ppc64le") def test_L8(self): diff --git a/scipy/optimize/tests/test_minimize_constrained.py b/scipy/optimize/tests/test_minimize_constrained.py index 6dad4bad5a39..1f68bfafae66 100644 --- a/scipy/optimize/tests/test_minimize_constrained.py +++ b/scipy/optimize/tests/test_minimize_constrained.py @@ -2,7 +2,7 @@ import pytest from scipy.linalg import block_diag from scipy.sparse import csc_matrix -from numpy.testing import (TestCase, assert_array_almost_equal, +from numpy.testing import (assert_array_almost_equal, assert_array_less, assert_, assert_allclose, suppress_warnings) from scipy.optimize import (NonlinearConstraint, @@ -443,67 +443,70 @@ def hess(x, v): return NonlinearConstraint(fun, -np.inf, 0, jac, hess) -class TestTrustRegionConstr(TestCase): - - @pytest.mark.slow - def test_list_of_problems(self): - list_of_problems = [Maratos(), - Maratos(constr_hess='2-point'), - Maratos(constr_hess=SR1()), - Maratos(constr_jac='2-point', constr_hess=SR1()), - MaratosGradInFunc(), - HyperbolicIneq(), - HyperbolicIneq(constr_hess='3-point'), - HyperbolicIneq(constr_hess=BFGS()), - HyperbolicIneq(constr_jac='3-point', - constr_hess=BFGS()), - Rosenbrock(), - IneqRosenbrock(), - EqIneqRosenbrock(), - BoundedRosenbrock(), - Elec(n_electrons=2), - Elec(n_electrons=2, constr_hess='2-point'), - Elec(n_electrons=2, constr_hess=SR1()), - Elec(n_electrons=2, constr_jac='3-point', - constr_hess=SR1())] - - for prob in list_of_problems: - for grad in (prob.grad, '3-point', False): - for hess in (prob.hess, - '3-point', - SR1(), - BFGS(exception_strategy='damp_update'), - BFGS(exception_strategy='skip_update')): - - # Remove exceptions - if grad in ('2-point', '3-point', 'cs', False) and \ - hess in ('2-point', '3-point', 'cs'): - continue - if prob.grad is True and grad in ('3-point', False): - continue - with suppress_warnings() as sup: - sup.filter(UserWarning, "delta_grad == 0.0") - result = minimize(prob.fun, prob.x0, - method='trust-constr', - jac=grad, hess=hess, - bounds=prob.bounds, - constraints=prob.constr) - - if prob.x_opt is not None: - assert_array_almost_equal(result.x, prob.x_opt, - decimal=5) - # gtol - if result.status == 1: - assert_array_less(result.optimality, 1e-8) - # xtol - if result.status == 2: - assert_array_less(result.tr_radius, 1e-8) - - if result.method == "tr_interior_point": - assert_array_less(result.barrier_parameter, 1e-8) - # max iter - if result.status in (0, 3): - raise RuntimeError("Invalid termination condition.") +class TestTrustRegionConstr: + list_of_problems = [Maratos(), + Maratos(constr_hess='2-point'), + Maratos(constr_hess=SR1()), + Maratos(constr_jac='2-point', constr_hess=SR1()), + MaratosGradInFunc(), + HyperbolicIneq(), + HyperbolicIneq(constr_hess='3-point'), + HyperbolicIneq(constr_hess=BFGS()), + HyperbolicIneq(constr_jac='3-point', + constr_hess=BFGS()), + Rosenbrock(), + IneqRosenbrock(), + EqIneqRosenbrock(), + BoundedRosenbrock(), + Elec(n_electrons=2), + Elec(n_electrons=2, constr_hess='2-point'), + Elec(n_electrons=2, constr_hess=SR1()), + Elec(n_electrons=2, constr_jac='3-point', + constr_hess=SR1())] + + @pytest.mark.parametrize('prob', list_of_problems) + @pytest.mark.parametrize('grad', ('prob.grad', '3-point', False)) + @pytest.mark.parametrize('hess', ("prob.hess", '3-point', SR1(), + BFGS(exception_strategy='damp_update'), + BFGS(exception_strategy='skip_update'))) + def test_list_of_problems(self, prob, grad, hess): + grad = prob.grad if grad == "prob.grad" else grad + hess = prob.hess if hess == "prob.hess" else hess + # Remove exceptions + if (grad in {'2-point', '3-point', 'cs', False} and + hess in {'2-point', '3-point', 'cs'}): + pytest.skip("Numerical Hessian needs analytical gradient") + if prob.grad is True and grad in {'3-point', False}: + pytest.skip("prob.grad incompatible with grad in {'3-point', False}") + sensitive = (isinstance(prob, BoundedRosenbrock) and grad == '3-point' + and isinstance(hess, BFGS)) + if sensitive: + pytest.xfail("Seems sensitive to initial conditions w/ Accelerate") + with suppress_warnings() as sup: + sup.filter(UserWarning, "delta_grad == 0.0") + result = minimize(prob.fun, prob.x0, + method='trust-constr', + jac=grad, hess=hess, + bounds=prob.bounds, + constraints=prob.constr) + + if prob.x_opt is not None: + assert_array_almost_equal(result.x, prob.x_opt, + decimal=5) + # gtol + if result.status == 1: + assert_array_less(result.optimality, 1e-8) + # xtol + if result.status == 2: + assert_array_less(result.tr_radius, 1e-8) + + if result.method == "tr_interior_point": + assert_array_less(result.barrier_parameter, 1e-8) + + # check for max iter + message = f"Invalid termination condition: {result.status}." + assert result.status not in {0, 3}, message + def test_default_jac_and_hess(self): def fun(x): @@ -641,7 +644,7 @@ def obj(x): assert result['success'] -class TestEmptyConstraint(TestCase): +class TestEmptyConstraint: """ Here we minimize x^2+y^2 subject to x^2-y^2>1. The actual minimum is at (0, 0) which fails the constraint. diff --git a/scipy/sparse/linalg/_eigen/lobpcg/tests/test_lobpcg.py b/scipy/sparse/linalg/_eigen/lobpcg/tests/test_lobpcg.py index 4b060d77d1f7..3fe07ba31296 100644 --- a/scipy/sparse/linalg/_eigen/lobpcg/tests/test_lobpcg.py +++ b/scipy/sparse/linalg/_eigen/lobpcg/tests/test_lobpcg.py @@ -548,9 +548,7 @@ def test_diagonal_data_types(n, m): # and where we choose A and B to be diagonal. vals = np.arange(1, n + 1) - # list_sparse_format = ['bsr', 'coo', 'csc', 'csr', 'dia', 'dok', 'lil'] - list_sparse_format = ['coo'] - sparse_formats = len(list_sparse_format) + list_sparse_format = ['bsr', 'coo', 'csc', 'csr', 'dia', 'dok', 'lil'] for s_f_i, s_f in enumerate(list_sparse_format): As64 = diags([vals * vals], [0], (n, n), format=s_f) @@ -629,15 +627,13 @@ def Mf32precond(x): listY = [Yf64, Yf32] tests = list(itertools.product(listA, listB, listM, listX, listY)) - # This is one of the slower tests because there are >1,000 configs - # to test here, instead of checking product of all input, output types - # test each configuration for the first sparse format, and then - # for one additional sparse format. this takes 2/7=30% as long as - # testing all configurations for all sparse formats. - if s_f_i > 0: - tests = tests[s_f_i - 1::sparse_formats-1] for A, B, M, X, Y in tests: + # This is one of the slower tests because there are >1,000 configs + # to test here. Flip a biased coin to decide whether to run each + # test to get decent coverage in less time. + if rnd.random() < 0.95: + continue # too many tests eigvals, _ = lobpcg(A, X, B=B, M=M, Y=Y, tol=1e-4, maxiter=100, largest=False) assert_allclose(eigvals, diff --git a/scipy/sparse/linalg/tests/test_propack.py b/scipy/sparse/linalg/tests/test_propack.py index 64eb888fd994..2dac7133997a 100644 --- a/scipy/sparse/linalg/tests/test_propack.py +++ b/scipy/sparse/linalg/tests/test_propack.py @@ -89,6 +89,7 @@ def test_svdp(ctor, dtype, irl, which): check_svdp(n, m, ctor, dtype, k, irl, which) +@pytest.mark.xslow @pytest.mark.parametrize('dtype', _dtypes) @pytest.mark.parametrize('irl', (False, True)) @pytest.mark.timeout(120) # True, complex64 > 60 s: prerel deps cov 64bit blas