From d3a7732dd54c27ae523bef73efbb0c580ce2fbc0 Mon Sep 17 00:00:00 2001 From: Sadra Barikbin Date: Tue, 4 Apr 2023 01:21:43 +0330 Subject: [PATCH 01/76] gh-103215: Remove redundant if stmt from `enum.EnumType._find_data_type_` (GH-103222) --- Lib/enum.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/enum.py b/Lib/enum.py index 4e231e7e8ea779..b8ea7af9e74775 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -987,8 +987,6 @@ def _find_data_type_(mcls, class_name, bases): data_types.add(base._member_type_) break elif '__new__' in base.__dict__ or '__init__' in base.__dict__: - if isinstance(base, EnumType): - continue data_types.add(candidate or base) break else: From 5ffc1e5a21de9a30566095386236db44695d184a Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Mon, 3 Apr 2023 14:57:42 -0700 Subject: [PATCH 02/76] gh-98298, gh-74730: [Enum] update docs (GH-103163) fix FlagBoundary statements add warning about reloading modules and enum identity --- Doc/howto/enum.rst | 5 +++++ Doc/library/enum.rst | 51 ++++++++++++++++++++++---------------------- Lib/enum.py | 8 +++---- 3 files changed, 34 insertions(+), 30 deletions(-) diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 4525acb04503b3..fc8ea55069a9ec 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -372,6 +372,11 @@ below):: >>> Color.BLUE == 2 False +.. warning:: + + It is possible to reload modules -- if a reloaded module contains + enums, they will be recreated, and the new members may not + compare identical/equal to the original members. Allowed members and attributes of enumerations ---------------------------------------------- diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 24b6dbfe37cd38..c690c837309ea5 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -141,9 +141,8 @@ Module Contents :func:`global_enum` Modify the :class:`str() ` and :func:`repr` of an enum - to show its members as belonging to the module instead of its class. - Should only be used if the enum members will be exported to the - module global namespace. + to show its members as belonging to the module instead of its class, + and export the enum members to the global namespace. :func:`show_flag_values` @@ -170,6 +169,27 @@ Data Types final *enum*, as well as creating the enum members, properly handling duplicates, providing iteration over the enum class, etc. + .. method:: EnumType.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) + + This method is called in two different ways: + + * to look up an existing member: + + :cls: The enum class being called. + :value: The value to lookup. + + * to use the ``cls`` enum to create a new enum (only if the existing enum + does not have any members): + + :cls: The enum class being called. + :value: The name of the new Enum to create. + :names: The names/values of the members for the new Enum. + :module: The name of the module the new Enum is created in. + :qualname: The actual location in the module where this Enum can be found. + :type: A mix-in type for the new Enum. + :start: The first integer value for the Enum (used by :class:`auto`). + :boundary: How to handle out-of-range values from bit operations (:class:`Flag` only). + .. method:: EnumType.__contains__(cls, member) Returns ``True`` if member belongs to the ``cls``:: @@ -255,26 +275,6 @@ Data Types names will also be removed from the completed enumeration. See :ref:`TimePeriod ` for an example. - .. method:: Enum.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) - - This method is called in two different ways: - - * to look up an existing member: - - :cls: The enum class being called. - :value: The value to lookup. - - * to use the ``cls`` enum to create a new enum: - - :cls: The enum class being called. - :value: The name of the new Enum to create. - :names: The names/values of the members for the new Enum. - :module: The name of the module the new Enum is created in. - :qualname: The actual location in the module where this Enum can be found. - :type: A mix-in type for the new Enum. - :start: The first integer value for the Enum (used by :class:`auto`). - :boundary: How to handle out-of-range values from bit operations (:class:`Flag` only). - .. method:: Enum.__dir__(self) Returns ``['__class__', '__doc__', '__module__', 'name', 'value']`` and @@ -728,7 +728,6 @@ Data Types .. attribute:: EJECT Out-of-range values lose their *Flag* membership and revert to :class:`int`. - This is the default for :class:`IntFlag`:: >>> from enum import Flag, EJECT, auto >>> class EjectFlag(Flag, boundary=EJECT): @@ -741,8 +740,8 @@ Data Types .. attribute:: KEEP - Out-of-range values are kept, and the *Flag* membership is kept. This is - used for some stdlib flags:: + Out-of-range values are kept, and the *Flag* membership is kept. + This is the default for :class:`IntFlag`:: >>> from enum import Flag, KEEP, auto >>> class KeepFlag(Flag, boundary=KEEP): diff --git a/Lib/enum.py b/Lib/enum.py index b8ea7af9e74775..9d836c716d8423 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1301,10 +1301,10 @@ def _reduce_ex_by_global_name(self, proto): class FlagBoundary(StrEnum): """ control how out of range values are handled - "strict" -> error is raised [default for Flag] - "conform" -> extra bits are discarded - "eject" -> lose flag status [default for IntFlag] - "keep" -> keep flag status and all bits + "strict" -> error is raised + "conform" -> extra bits are discarded [default for Flag] + "eject" -> lose flag status + "keep" -> keep flag status and all bits [default for IntFlag] """ STRICT = auto() CONFORM = auto() From 02f9920900551fd0281c8989d65521d4fce4ead1 Mon Sep 17 00:00:00 2001 From: David Benjamin Date: Tue, 4 Apr 2023 07:44:00 +0900 Subject: [PATCH 03/76] gh-100062: Remove error code tables from _ssl and err_names_to_codes (GH-100063) Prior to https://github.com/python/cpython/pull/25300, the make_ssl_data.py script used various tables, exposed in _ssl, to update the error list. After that PR, this is no longer used. Moreover, the err_names_to_codes map isn't used at all. Clean those up. This gets them out of the way if, in the future, OpenSSL provides an API to do what the code here is doing directly. (https://github.com/openssl/openssl/issues/19848) --- Modules/_ssl.c | 14 -------------- Modules/_ssl.h | 1 - 2 files changed, 15 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index e0351a89e6a1e9..5f17cd502d4598 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -5991,9 +5991,6 @@ sslmodule_init_errorcodes(PyObject *module) state->err_codes_to_names = PyDict_New(); if (state->err_codes_to_names == NULL) return -1; - state->err_names_to_codes = PyDict_New(); - if (state->err_names_to_codes == NULL) - return -1; state->lib_codes_to_names = PyDict_New(); if (state->lib_codes_to_names == NULL) return -1; @@ -6007,8 +6004,6 @@ sslmodule_init_errorcodes(PyObject *module) return -1; if (PyDict_SetItem(state->err_codes_to_names, key, mnemo)) return -1; - if (PyDict_SetItem(state->err_names_to_codes, mnemo, key)) - return -1; Py_DECREF(key); Py_DECREF(mnemo); errcode++; @@ -6028,13 +6023,6 @@ sslmodule_init_errorcodes(PyObject *module) libcode++; } - if (PyModule_AddObjectRef(module, "err_codes_to_names", state->err_codes_to_names)) - return -1; - if (PyModule_AddObjectRef(module, "err_names_to_codes", state->err_names_to_codes)) - return -1; - if (PyModule_AddObjectRef(module, "lib_codes_to_names", state->lib_codes_to_names)) - return -1; - return 0; } @@ -6189,7 +6177,6 @@ sslmodule_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(state->PySSLSyscallErrorObject); Py_VISIT(state->PySSLEOFErrorObject); Py_VISIT(state->err_codes_to_names); - Py_VISIT(state->err_names_to_codes); Py_VISIT(state->lib_codes_to_names); Py_VISIT(state->Sock_Type); @@ -6214,7 +6201,6 @@ sslmodule_clear(PyObject *m) Py_CLEAR(state->PySSLSyscallErrorObject); Py_CLEAR(state->PySSLEOFErrorObject); Py_CLEAR(state->err_codes_to_names); - Py_CLEAR(state->err_names_to_codes); Py_CLEAR(state->lib_codes_to_names); Py_CLEAR(state->Sock_Type); Py_CLEAR(state->str_library); diff --git a/Modules/_ssl.h b/Modules/_ssl.h index d68ccdec5e88c4..c1da8b46b536d3 100644 --- a/Modules/_ssl.h +++ b/Modules/_ssl.h @@ -25,7 +25,6 @@ typedef struct { PyObject *PySSLEOFErrorObject; /* Error mappings */ PyObject *err_codes_to_names; - PyObject *err_names_to_codes; PyObject *lib_codes_to_names; /* socket type from module CAPI */ PyTypeObject *Sock_Type; From c71756fa651e51041ac8b55776320e3297ba081d Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 3 Apr 2023 16:47:48 -0700 Subject: [PATCH 04/76] Specify more settings for the C extension of VS Code for dev containers (GH-103229) Should help with auto-complete. --- .devcontainer/devcontainer.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e3fb4c6c88f497..9fbaf7fddd8514 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -37,10 +37,16 @@ // "ms-python.python" ], "settings": { + "C_Cpp.default.compilerPath": "/usr/bin/clang", "C_Cpp.default.cStandard": "c11", "C_Cpp.default.defines": [ + "CONFIG_64", "Py_BUILD_CORE" ], + "C_Cpp.default.includePath": [ + "${workspaceFolder}/*", + "${workspaceFolder}/Include/**" + ], // https://github.com/microsoft/vscode-cpptools/issues/10732 "C_Cpp.errorSquiggles": "disabled", "editor.insertSpaces": true, From 810d365b5eb2cf3043957ca2971f6e7a7cd87d0d Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Mon, 3 Apr 2023 17:47:40 -0700 Subject: [PATCH 05/76] gh-103056: [Enum] use staticmethod decorator for _gnv_ (GH-103231) _gnv_ --> _generate_next_value_ --- Doc/howto/enum.rst | 1 + Lib/enum.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index fc8ea55069a9ec..38951ed7348f56 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -284,6 +284,7 @@ The values are chosen by :func:`_generate_next_value_`, which can be overridden:: >>> class AutoName(Enum): + ... @staticmethod ... def _generate_next_value_(name, start, count, last_values): ... return name ... diff --git a/Lib/enum.py b/Lib/enum.py index 9d836c716d8423..ec698d5fa3c300 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1146,6 +1146,7 @@ def __new__(cls, value): def __init__(self, *args, **kwds): pass + @staticmethod def _generate_next_value_(name, start, count, last_values): """ Generate the next value when not given. @@ -1288,6 +1289,7 @@ def __new__(cls, *values): member._value_ = value return member + @staticmethod def _generate_next_value_(name, start, count, last_values): """ Return the lower-cased version of the member name. @@ -1337,6 +1339,7 @@ def __reduce_ex__(self, proto): _numeric_repr_ = repr + @staticmethod def _generate_next_value_(name, start, count, last_values): """ Generate the next value when not given. From c00dcf0e381a090f7e905f887a98bbf63c88af5a Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 4 Apr 2023 12:08:54 +0200 Subject: [PATCH 06/76] gh-103092: Isolate `_pickle` module (#102982) Co-authored-by: Mohamed Koubaa Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> --- Lib/test/test_pickle.py | 28 + ...2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst | 1 + Modules/_pickle.c | 1262 ++++++++--------- Modules/clinic/_pickle.c.h | 76 +- Tools/c-analyzer/cpython/globals-to-fix.tsv | 5 - 5 files changed, 703 insertions(+), 669 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 80e7a4d23a4ba8..1a55da39bdc58d 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -293,6 +293,34 @@ class CustomCPicklerClass(_pickle.Pickler, AbstractCustomPicklerClass): pass pickler_class = CustomCPicklerClass + @support.cpython_only + class HeapTypesTests(unittest.TestCase): + def setUp(self): + pickler = _pickle.Pickler(io.BytesIO()) + unpickler = _pickle.Unpickler(io.BytesIO()) + + self._types = ( + _pickle.Pickler, + _pickle.Unpickler, + type(pickler.memo), + type(unpickler.memo), + + # We cannot test the _pickle.Pdata; + # there's no way to get to it. + ) + + def test_have_gc(self): + import gc + for tp in self._types: + with self.subTest(tp=tp): + self.assertTrue(gc.is_tracked(tp)) + + def test_immutable(self): + for tp in self._types: + with self.subTest(tp=tp): + with self.assertRaisesRegex(TypeError, "immutable"): + tp.foo = "bar" + @support.cpython_only class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst b/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst new file mode 100644 index 00000000000000..9f9c8121bea5ae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst @@ -0,0 +1 @@ +Adapt :mod:`!_pickle` to :pep:`687`. Patch by Mohamed Koubaa and Erlend Aasland. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index a26732af8ba2a1..360c7910f67187 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -22,12 +22,12 @@ PyDoc_STRVAR(pickle_module_doc, /*[clinic input] module _pickle -class _pickle.Pickler "PicklerObject *" "&Pickler_Type" -class _pickle.PicklerMemoProxy "PicklerMemoProxyObject *" "&PicklerMemoProxyType" -class _pickle.Unpickler "UnpicklerObject *" "&Unpickler_Type" -class _pickle.UnpicklerMemoProxy "UnpicklerMemoProxyObject *" "&UnpicklerMemoProxyType" +class _pickle.Pickler "PicklerObject *" "" +class _pickle.PicklerMemoProxy "PicklerMemoProxyObject *" "" +class _pickle.Unpickler "UnpicklerObject *" "" +class _pickle.UnpicklerMemoProxy "UnpicklerMemoProxyObject *" "" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4b3e113468a58e6c]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=b6d7191ab6466cda]*/ /* Bump HIGHEST_PROTOCOL when new opcodes are added to the pickle protocol. Bump DEFAULT_PROTOCOL only when the oldest still supported version of Python @@ -192,24 +192,41 @@ typedef struct { /* functools.partial, used for implementing __newobj_ex__ with protocols 2 and 3 */ PyObject *partial; + + /* Types */ + PyTypeObject *Pickler_Type; + PyTypeObject *Unpickler_Type; + PyTypeObject *Pdata_Type; + PyTypeObject *PicklerMemoProxyType; + PyTypeObject *UnpicklerMemoProxyType; } PickleState; /* Forward declaration of the _pickle module definition. */ static struct PyModuleDef _picklemodule; /* Given a module object, get its per-module state. */ -static PickleState * +static inline PickleState * _Pickle_GetState(PyObject *module) { - return (PickleState *)_PyModule_GetState(module); + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (PickleState *)state; +} + +static inline PickleState * +_Pickle_GetStateByClass(PyTypeObject *cls) +{ + void *state = _PyType_GetModuleState(cls); + assert(state != NULL); + return (PickleState *)state; } -/* Find the module instance imported in the currently running sub-interpreter - and get its state. */ -static PickleState * -_Pickle_GetGlobalState(void) +static inline PickleState * +_Pickle_FindStateByType(PyTypeObject *tp) { - return _Pickle_GetState(PyState_FindModule(&_picklemodule)); + PyObject *module = PyType_GetModuleByDef(tp, &_picklemodule); + assert(module != NULL); + return _Pickle_GetState(module); } /* Clear the given pickle module state. */ @@ -230,6 +247,11 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->codecs_encode); Py_CLEAR(st->getattr); Py_CLEAR(st->partial); + Py_CLEAR(st->Pickler_Type); + Py_CLEAR(st->Unpickler_Type); + Py_CLEAR(st->Pdata_Type); + Py_CLEAR(st->PicklerMemoProxyType); + Py_CLEAR(st->UnpicklerMemoProxyType); } /* Initialize the given pickle module state. */ @@ -439,39 +461,58 @@ typedef struct { Py_ssize_t allocated; /* number of slots in data allocated */ } Pdata; +static int +Pdata_traverse(Pdata *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + static void Pdata_dealloc(Pdata *self) { + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); Py_ssize_t i = Py_SIZE(self); while (--i >= 0) { Py_DECREF(self->data[i]); } PyMem_Free(self->data); - PyObject_Free(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } -static PyTypeObject Pdata_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pickle.Pdata", /*tp_name*/ - sizeof(Pdata), /*tp_basicsize*/ - sizeof(PyObject *), /*tp_itemsize*/ - (destructor)Pdata_dealloc, /*tp_dealloc*/ +static PyType_Slot pdata_slots[] = { + {Py_tp_dealloc, Pdata_dealloc}, + {Py_tp_traverse, Pdata_traverse}, + {0, NULL}, +}; + +static PyType_Spec pdata_spec = { + .name = "_pickle.Pdata", + .basicsize = sizeof(Pdata), + .itemsize = sizeof(PyObject *), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pdata_slots, }; static PyObject * -Pdata_New(void) +Pdata_New(PickleState *state) { Pdata *self; - if (!(self = PyObject_New(Pdata, &Pdata_Type))) + if (!(self = PyObject_GC_New(Pdata, state->Pdata_Type))) return NULL; Py_SET_SIZE(self, 0); self->mark_set = 0; self->fence = 0; self->allocated = 8; self->data = PyMem_Malloc(self->allocated * sizeof(PyObject *)); - if (self->data) + if (self->data) { + PyObject_GC_Track(self); return (PyObject *)self; + } Py_DECREF(self); return PyErr_NoMemory(); } @@ -522,9 +563,8 @@ Pdata_grow(Pdata *self) } static int -Pdata_stack_underflow(Pdata *self) +Pdata_stack_underflow(PickleState *st, Pdata *self) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, self->mark_set ? "unexpected MARK found" : @@ -537,16 +577,16 @@ Pdata_stack_underflow(Pdata *self) * is raised and V is set to NULL. */ static PyObject * -Pdata_pop(Pdata *self) +Pdata_pop(PickleState *state, Pdata *self) { if (Py_SIZE(self) <= self->fence) { - Pdata_stack_underflow(self); + Pdata_stack_underflow(state, self); return NULL; } Py_SET_SIZE(self, Py_SIZE(self) - 1); return self->data[Py_SIZE(self)]; } -#define PDATA_POP(D, V) do { (V) = Pdata_pop((D)); } while (0) +#define PDATA_POP(S, D, V) do { (V) = Pdata_pop(S, (D)); } while (0) static int Pdata_push(Pdata *self, PyObject *obj) @@ -569,13 +609,13 @@ Pdata_push(Pdata *self, PyObject *obj) if (Pdata_push((D), (O)) < 0) return (ER); } while(0) static PyObject * -Pdata_poptuple(Pdata *self, Py_ssize_t start) +Pdata_poptuple(PickleState *state, Pdata *self, Py_ssize_t start) { PyObject *tuple; Py_ssize_t len, i, j; if (start < self->fence) { - Pdata_stack_underflow(self); + Pdata_stack_underflow(state, self); return NULL; } len = Py_SIZE(self) - start; @@ -710,10 +750,8 @@ typedef struct { } UnpicklerMemoProxyObject; /* Forward declarations */ -static int save(PicklerObject *, PyObject *, int); -static int save_reduce(PicklerObject *, PyObject *, PyObject *); -static PyTypeObject Pickler_Type; -static PyTypeObject Unpickler_Type; +static int save(PickleState *state, PicklerObject *, PyObject *, int); +static int save_reduce(PickleState *, PicklerObject *, PyObject *, PyObject *); #include "clinic/_pickle.c.h" @@ -1105,11 +1143,11 @@ _Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t data_len) } static PicklerObject * -_Pickler_New(void) +_Pickler_New(PickleState *st) { PicklerObject *self; - self = PyObject_GC_New(PicklerObject, &Pickler_Type); + self = PyObject_GC_New(PicklerObject, st->Pickler_Type); if (self == NULL) return NULL; @@ -1220,9 +1258,8 @@ _Unpickler_SetStringInput(UnpicklerObject *self, PyObject *input) } static int -bad_readline(void) +bad_readline(PickleState *st) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "pickle data was truncated"); return -1; } @@ -1317,13 +1354,12 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) /* Don't call it directly: use _Unpickler_Read() */ static Py_ssize_t -_Unpickler_ReadImpl(UnpicklerObject *self, char **s, Py_ssize_t n) +_Unpickler_ReadImpl(UnpicklerObject *self, PickleState *st, char **s, Py_ssize_t n) { Py_ssize_t num_read; *s = NULL; if (self->next_read_idx > PY_SSIZE_T_MAX - n) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "read would overflow (invalid bytecode)"); return -1; @@ -1333,14 +1369,14 @@ _Unpickler_ReadImpl(UnpicklerObject *self, char **s, Py_ssize_t n) assert(self->next_read_idx + n > self->input_len); if (!self->read) - return bad_readline(); + return bad_readline(st); /* Extend the buffer to satisfy desired size */ num_read = _Unpickler_ReadFromFile(self, n); if (num_read < 0) return -1; if (num_read < n) - return bad_readline(); + return bad_readline(st); *s = self->input_buffer; self->next_read_idx = n; return n; @@ -1355,7 +1391,8 @@ _Unpickler_ReadImpl(UnpicklerObject *self, char **s, Py_ssize_t n) * _Unpickler_Read() is recommended in most cases. */ static Py_ssize_t -_Unpickler_ReadInto(UnpicklerObject *self, char *buf, Py_ssize_t n) +_Unpickler_ReadInto(PickleState *state, UnpicklerObject *self, char *buf, + Py_ssize_t n) { assert(n != READ_WHOLE_LINE); @@ -1376,7 +1413,7 @@ _Unpickler_ReadInto(UnpicklerObject *self, char *buf, Py_ssize_t n) /* Read from file */ if (!self->read) { /* We're unpickling memory, this means the input is truncated */ - return bad_readline(); + return bad_readline(state); } if (_Unpickler_SkipConsumed(self) < 0) { return -1; @@ -1403,7 +1440,7 @@ _Unpickler_ReadInto(UnpicklerObject *self, char *buf, Py_ssize_t n) Py_ssize_t read_size = PyBytes_GET_SIZE(data); if (read_size < n) { Py_DECREF(data); - return bad_readline(); + return bad_readline(state); } memcpy(buf, PyBytes_AS_STRING(data), n); Py_DECREF(data); @@ -1430,7 +1467,7 @@ _Unpickler_ReadInto(UnpicklerObject *self, char *buf, Py_ssize_t n) return -1; } if (read_size < n) { - return bad_readline(); + return bad_readline(state); } return n; } @@ -1448,12 +1485,12 @@ _Unpickler_ReadInto(UnpicklerObject *self, char *buf, Py_ssize_t n) Returns -1 (with an exception set) on failure. On success, return the number of chars read. */ -#define _Unpickler_Read(self, s, n) \ +#define _Unpickler_Read(self, state, s, n) \ (((n) <= (self)->input_len - (self)->next_read_idx) \ ? (*(s) = (self)->input_buffer + (self)->next_read_idx, \ (self)->next_read_idx += (n), \ (n)) \ - : _Unpickler_ReadImpl(self, (s), (n))) + : _Unpickler_ReadImpl(self, state, (s), (n))) static Py_ssize_t _Unpickler_CopyLine(UnpicklerObject *self, char *line, Py_ssize_t len, @@ -1477,7 +1514,7 @@ _Unpickler_CopyLine(UnpicklerObject *self, char *line, Py_ssize_t len, Returns the number of chars read, or -1 on failure. */ static Py_ssize_t -_Unpickler_Readline(UnpicklerObject *self, char **result) +_Unpickler_Readline(PickleState *state, UnpicklerObject *self, char **result) { Py_ssize_t i, num_read; @@ -1490,13 +1527,13 @@ _Unpickler_Readline(UnpicklerObject *self, char **result) } } if (!self->read) - return bad_readline(); + return bad_readline(state); num_read = _Unpickler_ReadFromFile(self, READ_WHOLE_LINE); if (num_read < 0) return -1; if (num_read == 0 || self->input_buffer[num_read - 1] != '\n') - return bad_readline(); + return bad_readline(state); self->next_read_idx = num_read; return _Unpickler_CopyLine(self, self->input_buffer, num_read, result); } @@ -1586,11 +1623,12 @@ _Unpickler_MemoCleanup(UnpicklerObject *self) } static UnpicklerObject * -_Unpickler_New(void) +_Unpickler_New(PyObject *module) { UnpicklerObject *self; + PickleState *st = _Pickle_GetState(module); - self = PyObject_GC_New(UnpicklerObject, &Unpickler_Type); + self = PyObject_GC_New(UnpicklerObject, st->Unpickler_Type); if (self == NULL) return NULL; @@ -1616,7 +1654,7 @@ _Unpickler_New(void) self->memo_size = 32; self->memo_len = 0; self->memo = _Unpickler_NewMemo(self->memo_size); - self->stack = (Pdata *)Pdata_New(); + self->stack = (Pdata *)Pdata_New(st); if (self->memo == NULL || self->stack == NULL) { Py_DECREF(self); @@ -1695,7 +1733,7 @@ _Unpickler_SetBuffers(UnpicklerObject *self, PyObject *buffers) /* Generate a GET opcode for an object stored in the memo. */ static int -memo_get(PicklerObject *self, PyObject *key) +memo_get(PickleState *st, PicklerObject *self, PyObject *key) { Py_ssize_t *value; char pdata[30]; @@ -1728,7 +1766,6 @@ memo_get(PicklerObject *self, PyObject *key) len = 5; } else { /* unlikely */ - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->PicklingError, "memo id too large for LONG_BINGET"); return -1; @@ -1744,7 +1781,7 @@ memo_get(PicklerObject *self, PyObject *key) /* Store an object in the memo, assign it a new unique ID based on the number of objects currently stored in the memo and generate a PUT opcode. */ static int -memo_put(PicklerObject *self, PyObject *obj) +memo_put(PickleState *st, PicklerObject *self, PyObject *obj) { char pdata[30]; Py_ssize_t len; @@ -1785,7 +1822,6 @@ memo_put(PicklerObject *self, PyObject *obj) len = 5; } else { /* unlikely */ - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->PicklingError, "memo id too large for LONG_BINPUT"); return -1; @@ -2338,8 +2374,8 @@ _Pickler_write_bytes(PicklerObject *self, } static int -_save_bytes_data(PicklerObject *self, PyObject *obj, const char *data, - Py_ssize_t size) +_save_bytes_data(PickleState *st, PicklerObject *self, PyObject *obj, + const char *data, Py_ssize_t size) { assert(self->proto >= 3); @@ -2378,7 +2414,7 @@ _save_bytes_data(PicklerObject *self, PyObject *obj, const char *data, return -1; } - if (memo_put(self, obj) < 0) { + if (memo_put(st, self, obj) < 0) { return -1; } @@ -2386,7 +2422,7 @@ _save_bytes_data(PicklerObject *self, PyObject *obj, const char *data, } static int -save_bytes(PicklerObject *self, PyObject *obj) +save_bytes(PickleState *st, PicklerObject *self, PyObject *obj) { if (self->proto < 3) { /* Older pickle protocols do not have an opcode for pickling bytes @@ -2407,7 +2443,6 @@ save_bytes(PicklerObject *self, PyObject *obj) reduce_value = Py_BuildValue("(O())", (PyObject*)&PyBytes_Type); } else { - PickleState *st = _Pickle_GetGlobalState(); PyObject *unicode_str = PyUnicode_DecodeLatin1(PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj), @@ -2425,19 +2460,19 @@ save_bytes(PicklerObject *self, PyObject *obj) return -1; /* save_reduce() will memoize the object automatically. */ - status = save_reduce(self, reduce_value, obj); + status = save_reduce(st, self, reduce_value, obj); Py_DECREF(reduce_value); return status; } else { - return _save_bytes_data(self, obj, PyBytes_AS_STRING(obj), + return _save_bytes_data(st, self, obj, PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); } } static int -_save_bytearray_data(PicklerObject *self, PyObject *obj, const char *data, - Py_ssize_t size) +_save_bytearray_data(PickleState *state, PicklerObject *self, PyObject *obj, + const char *data, Py_ssize_t size) { assert(self->proto >= 5); @@ -2455,7 +2490,7 @@ _save_bytearray_data(PicklerObject *self, PyObject *obj, const char *data, return -1; } - if (memo_put(self, obj) < 0) { + if (memo_put(state, self, obj) < 0) { return -1; } @@ -2463,7 +2498,7 @@ _save_bytearray_data(PicklerObject *self, PyObject *obj, const char *data, } static int -save_bytearray(PicklerObject *self, PyObject *obj) +save_bytearray(PickleState *state, PicklerObject *self, PyObject *obj) { if (self->proto < 5) { /* Older pickle protocols do not have an opcode for pickling @@ -2488,21 +2523,21 @@ save_bytearray(PicklerObject *self, PyObject *obj) return -1; /* save_reduce() will memoize the object automatically. */ - status = save_reduce(self, reduce_value, obj); + status = save_reduce(state, self, reduce_value, obj); Py_DECREF(reduce_value); return status; } else { - return _save_bytearray_data(self, obj, PyByteArray_AS_STRING(obj), + return _save_bytearray_data(state, self, obj, + PyByteArray_AS_STRING(obj), PyByteArray_GET_SIZE(obj)); } } static int -save_picklebuffer(PicklerObject *self, PyObject *obj) +save_picklebuffer(PickleState *st, PicklerObject *self, PyObject *obj) { if (self->proto < 5) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->PicklingError, "PickleBuffer can only pickled with protocol >= 5"); return -1; @@ -2512,7 +2547,6 @@ save_picklebuffer(PicklerObject *self, PyObject *obj) return -1; } if (view->suboffsets != NULL || !PyBuffer_IsContiguous(view, 'A')) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->PicklingError, "PickleBuffer can not be pickled when " "pointing to a non-contiguous buffer"); @@ -2533,11 +2567,11 @@ save_picklebuffer(PicklerObject *self, PyObject *obj) if (in_band) { /* Write data in-band */ if (view->readonly) { - return _save_bytes_data(self, obj, (const char*) view->buf, + return _save_bytes_data(st, self, obj, (const char *)view->buf, view->len); } else { - return _save_bytearray_data(self, obj, (const char*) view->buf, + return _save_bytearray_data(st, self, obj, (const char *)view->buf, view->len); } } @@ -2692,7 +2726,7 @@ write_unicode_binary(PicklerObject *self, PyObject *obj) } static int -save_unicode(PicklerObject *self, PyObject *obj) +save_unicode(PickleState *state, PicklerObject *self, PyObject *obj) { if (self->bin) { if (write_unicode_binary(self, obj) < 0) @@ -2722,7 +2756,7 @@ save_unicode(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, "\n", 1) < 0) return -1; } - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) return -1; return 0; @@ -2730,7 +2764,8 @@ save_unicode(PicklerObject *self, PyObject *obj) /* A helper for save_tuple. Push the len elements in tuple t on the stack. */ static int -store_tuple_elements(PicklerObject *self, PyObject *t, Py_ssize_t len) +store_tuple_elements(PickleState *state, PicklerObject *self, PyObject *t, + Py_ssize_t len) { Py_ssize_t i; @@ -2741,7 +2776,7 @@ store_tuple_elements(PicklerObject *self, PyObject *t, Py_ssize_t len) if (element == NULL) return -1; - if (save(self, element, 0) < 0) + if (save(state, self, element, 0) < 0) return -1; } @@ -2755,7 +2790,7 @@ store_tuple_elements(PicklerObject *self, PyObject *t, Py_ssize_t len) * magic so that it works in all cases. IOW, this is a long routine. */ static int -save_tuple(PicklerObject *self, PyObject *obj) +save_tuple(PickleState *state, PicklerObject *self, PyObject *obj) { Py_ssize_t len, i; @@ -2792,7 +2827,7 @@ save_tuple(PicklerObject *self, PyObject *obj) */ if (len <= 3 && self->proto >= 2) { /* Use TUPLE{1,2,3} opcodes. */ - if (store_tuple_elements(self, obj, len) < 0) + if (store_tuple_elements(state, self, obj, len) < 0) return -1; if (PyMemoTable_Get(self->memo, obj)) { @@ -2801,7 +2836,7 @@ save_tuple(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, &pop_op, 1) < 0) return -1; /* fetch from memo */ - if (memo_get(self, obj) < 0) + if (memo_get(state, self, obj) < 0) return -1; return 0; @@ -2819,7 +2854,7 @@ save_tuple(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, &mark_op, 1) < 0) return -1; - if (store_tuple_elements(self, obj, len) < 0) + if (store_tuple_elements(state, self, obj, len) < 0) return -1; if (PyMemoTable_Get(self->memo, obj)) { @@ -2837,7 +2872,7 @@ save_tuple(PicklerObject *self, PyObject *obj) return -1; } /* fetch from memo */ - if (memo_get(self, obj) < 0) + if (memo_get(state, self, obj) < 0) return -1; return 0; @@ -2848,7 +2883,7 @@ save_tuple(PicklerObject *self, PyObject *obj) } memoize: - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) return -1; return 0; @@ -2861,7 +2896,7 @@ save_tuple(PicklerObject *self, PyObject *obj) * Returns 0 on success, <0 on error. */ static int -batch_list(PicklerObject *self, PyObject *iter) +batch_list(PickleState *state, PicklerObject *self, PyObject *iter) { PyObject *obj = NULL; PyObject *firstitem = NULL; @@ -2887,7 +2922,7 @@ batch_list(PicklerObject *self, PyObject *iter) return -1; break; } - i = save(self, obj, 0); + i = save(state, self, obj, 0); Py_DECREF(obj); if (i < 0) return -1; @@ -2916,7 +2951,7 @@ batch_list(PicklerObject *self, PyObject *iter) goto error; /* Only one item to write */ - if (save(self, firstitem, 0) < 0) + if (save(state, self, firstitem, 0) < 0) goto error; if (_Pickler_Write(self, &append_op, 1) < 0) goto error; @@ -2930,14 +2965,14 @@ batch_list(PicklerObject *self, PyObject *iter) if (_Pickler_Write(self, &mark_op, 1) < 0) goto error; - if (save(self, firstitem, 0) < 0) + if (save(state, self, firstitem, 0) < 0) goto error; Py_CLEAR(firstitem); n = 1; /* Fetch and save up to BATCHSIZE items */ while (obj) { - if (save(self, obj, 0) < 0) + if (save(state, self, obj, 0) < 0) goto error; Py_CLEAR(obj); n += 1; @@ -2977,7 +3012,7 @@ batch_list(PicklerObject *self, PyObject *iter) * Note that this only works for protocols > 0. */ static int -batch_list_exact(PicklerObject *self, PyObject *obj) +batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *item = NULL; Py_ssize_t this_batch, total; @@ -2993,7 +3028,7 @@ batch_list_exact(PicklerObject *self, PyObject *obj) if (PyList_GET_SIZE(obj) == 1) { item = PyList_GET_ITEM(obj, 0); Py_INCREF(item); - int err = save(self, item, 0); + int err = save(state, self, item, 0); Py_DECREF(item); if (err < 0) return -1; @@ -3011,7 +3046,7 @@ batch_list_exact(PicklerObject *self, PyObject *obj) while (total < PyList_GET_SIZE(obj)) { item = PyList_GET_ITEM(obj, total); Py_INCREF(item); - int err = save(self, item, 0); + int err = save(state, self, item, 0); Py_DECREF(item); if (err < 0) return -1; @@ -3028,7 +3063,7 @@ batch_list_exact(PicklerObject *self, PyObject *obj) } static int -save_list(PicklerObject *self, PyObject *obj) +save_list(PickleState *state, PicklerObject *self, PyObject *obj) { char header[3]; Py_ssize_t len; @@ -3055,7 +3090,7 @@ save_list(PicklerObject *self, PyObject *obj) if ((len = PyList_Size(obj)) < 0) goto error; - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) goto error; if (len != 0) { @@ -3063,7 +3098,7 @@ save_list(PicklerObject *self, PyObject *obj) if (PyList_CheckExact(obj) && self->proto > 0) { if (_Py_EnterRecursiveCall(" while pickling an object")) goto error; - status = batch_list_exact(self, obj); + status = batch_list_exact(state, self, obj); _Py_LeaveRecursiveCall(); } else { PyObject *iter = PyObject_GetIter(obj); @@ -3074,7 +3109,7 @@ save_list(PicklerObject *self, PyObject *obj) Py_DECREF(iter); goto error; } - status = batch_list(self, iter); + status = batch_list(state, self, iter); _Py_LeaveRecursiveCall(); Py_DECREF(iter); } @@ -3102,7 +3137,7 @@ save_list(PicklerObject *self, PyObject *obj) * ugly to bear. */ static int -batch_dict(PicklerObject *self, PyObject *iter) +batch_dict(PickleState *state, PicklerObject *self, PyObject *iter) { PyObject *obj = NULL; PyObject *firstitem = NULL; @@ -3128,9 +3163,9 @@ batch_dict(PicklerObject *self, PyObject *iter) "iterator must return 2-tuples"); return -1; } - i = save(self, PyTuple_GET_ITEM(obj, 0), 0); + i = save(state, self, PyTuple_GET_ITEM(obj, 0), 0); if (i >= 0) - i = save(self, PyTuple_GET_ITEM(obj, 1), 0); + i = save(state, self, PyTuple_GET_ITEM(obj, 1), 0); Py_DECREF(obj); if (i < 0) return -1; @@ -3164,9 +3199,9 @@ batch_dict(PicklerObject *self, PyObject *iter) goto error; /* Only one item to write */ - if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0) goto error; - if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) goto error; if (_Pickler_Write(self, &setitem_op, 1) < 0) goto error; @@ -3180,9 +3215,9 @@ batch_dict(PicklerObject *self, PyObject *iter) if (_Pickler_Write(self, &mark_op, 1) < 0) goto error; - if (save(self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(firstitem, 0), 0) < 0) goto error; - if (save(self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(firstitem, 1), 0) < 0) goto error; Py_CLEAR(firstitem); n = 1; @@ -3194,8 +3229,8 @@ batch_dict(PicklerObject *self, PyObject *iter) "iterator must return 2-tuples"); goto error; } - if (save(self, PyTuple_GET_ITEM(obj, 0), 0) < 0 || - save(self, PyTuple_GET_ITEM(obj, 1), 0) < 0) + if (save(state, self, PyTuple_GET_ITEM(obj, 0), 0) < 0 || + save(state, self, PyTuple_GET_ITEM(obj, 1), 0) < 0) goto error; Py_CLEAR(obj); n += 1; @@ -3233,7 +3268,7 @@ batch_dict(PicklerObject *self, PyObject *iter) * Note that this currently doesn't work for protocol 0. */ static int -batch_dict_exact(PicklerObject *self, PyObject *obj) +batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *key = NULL, *value = NULL; int i; @@ -3253,10 +3288,10 @@ batch_dict_exact(PicklerObject *self, PyObject *obj) PyDict_Next(obj, &ppos, &key, &value); Py_INCREF(key); Py_INCREF(value); - if (save(self, key, 0) < 0) { + if (save(state, self, key, 0) < 0) { goto error; } - if (save(self, value, 0) < 0) { + if (save(state, self, value, 0) < 0) { goto error; } Py_CLEAR(key); @@ -3274,10 +3309,10 @@ batch_dict_exact(PicklerObject *self, PyObject *obj) while (PyDict_Next(obj, &ppos, &key, &value)) { Py_INCREF(key); Py_INCREF(value); - if (save(self, key, 0) < 0) { + if (save(state, self, key, 0) < 0) { goto error; } - if (save(self, value, 0) < 0) { + if (save(state, self, value, 0) < 0) { goto error; } Py_CLEAR(key); @@ -3303,7 +3338,7 @@ batch_dict_exact(PicklerObject *self, PyObject *obj) } static int -save_dict(PicklerObject *self, PyObject *obj) +save_dict(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *items, *iter; char header[3]; @@ -3328,7 +3363,7 @@ save_dict(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, header, len) < 0) goto error; - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) goto error; if (PyDict_GET_SIZE(obj)) { @@ -3338,7 +3373,7 @@ save_dict(PicklerObject *self, PyObject *obj) not a dict subclass. */ if (_Py_EnterRecursiveCall(" while pickling an object")) goto error; - status = batch_dict_exact(self, obj); + status = batch_dict_exact(state, self, obj); _Py_LeaveRecursiveCall(); } else { items = PyObject_CallMethodNoArgs(obj, &_Py_ID(items)); @@ -3352,7 +3387,7 @@ save_dict(PicklerObject *self, PyObject *obj) Py_DECREF(iter); goto error; } - status = batch_dict(self, iter); + status = batch_dict(state, self, iter); _Py_LeaveRecursiveCall(); Py_DECREF(iter); } @@ -3370,7 +3405,7 @@ save_dict(PicklerObject *self, PyObject *obj) } static int -save_set(PicklerObject *self, PyObject *obj) +save_set(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *item; int i; @@ -3396,7 +3431,7 @@ save_set(PicklerObject *self, PyObject *obj) return -1; } /* save_reduce() will memoize the object automatically. */ - status = save_reduce(self, reduce_value, obj); + status = save_reduce(state, self, reduce_value, obj); Py_DECREF(reduce_value); return status; } @@ -3404,7 +3439,7 @@ save_set(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, &empty_set_op, 1) < 0) return -1; - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) return -1; set_size = PySet_GET_SIZE(obj); @@ -3418,7 +3453,7 @@ save_set(PicklerObject *self, PyObject *obj) return -1; while (_PySet_NextEntry(obj, &ppos, &item, &hash)) { Py_INCREF(item); - int err = save(self, item, 0); + int err = save(state, self, item, 0); Py_CLEAR(item); if (err < 0) return -1; @@ -3439,7 +3474,7 @@ save_set(PicklerObject *self, PyObject *obj) } static int -save_frozenset(PicklerObject *self, PyObject *obj) +save_frozenset(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *iter; @@ -3465,7 +3500,7 @@ save_frozenset(PicklerObject *self, PyObject *obj) return -1; } /* save_reduce() will memoize the object automatically. */ - status = save_reduce(self, reduce_value, obj); + status = save_reduce(state, self, reduce_value, obj); Py_DECREF(reduce_value); return status; } @@ -3488,7 +3523,7 @@ save_frozenset(PicklerObject *self, PyObject *obj) } break; } - if (save(self, item, 0) < 0) { + if (save(state, self, item, 0) < 0) { Py_DECREF(item); Py_DECREF(iter); return -1; @@ -3505,25 +3540,24 @@ save_frozenset(PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, &pop_mark_op, 1) < 0) return -1; - if (memo_get(self, obj) < 0) + if (memo_get(state, self, obj) < 0) return -1; return 0; } if (_Pickler_Write(self, &frozenset_op, 1) < 0) return -1; - if (memo_put(self, obj) < 0) + if (memo_put(state, self, obj) < 0) return -1; return 0; } static int -fix_imports(PyObject **module_name, PyObject **global_name) +fix_imports(PickleState *st, PyObject **module_name, PyObject **global_name) { PyObject *key; PyObject *item; - PickleState *st = _Pickle_GetGlobalState(); key = PyTuple_Pack(2, *module_name, *global_name); if (key == NULL) @@ -3582,7 +3616,8 @@ fix_imports(PyObject **module_name, PyObject **global_name) } static int -save_global(PicklerObject *self, PyObject *obj, PyObject *name) +save_global(PickleState *st, PicklerObject *self, PyObject *obj, + PyObject *name) { PyObject *global_name = NULL; PyObject *module_name = NULL; @@ -3591,7 +3626,6 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) PyObject *dotted_path = NULL; PyObject *lastname = NULL; PyObject *cls; - PickleState *st = _Pickle_GetGlobalState(); int status = 0; const char global_op = GLOBAL; @@ -3727,21 +3761,20 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) if (self->proto >= 4) { const char stack_global_op = STACK_GLOBAL; - if (save(self, module_name, 0) < 0) + if (save(st, self, module_name, 0) < 0) goto error; - if (save(self, global_name, 0) < 0) + if (save(st, self, global_name, 0) < 0) goto error; if (_Pickler_Write(self, &stack_global_op, 1) < 0) goto error; } else if (parent != module) { - PickleState *st = _Pickle_GetGlobalState(); PyObject *reduce_value = Py_BuildValue("(O(OO))", st->getattr, parent, lastname); if (reduce_value == NULL) goto error; - status = save_reduce(self, reduce_value, NULL); + status = save_reduce(st, self, reduce_value, NULL); Py_DECREF(reduce_value); if (status < 0) goto error; @@ -3759,7 +3792,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) /* For protocol < 3 and if the user didn't request against doing so, we convert module names to the old 2.x module names. */ if (self->proto < 3 && self->fix_imports) { - if (fix_imports(&module_name, &global_name) < 0) { + if (fix_imports(st, &module_name, &global_name) < 0) { goto error; } } @@ -3813,7 +3846,7 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) goto error; } /* Memoize the object. */ - if (memo_put(self, obj) < 0) + if (memo_put(st, self, obj) < 0) goto error; } @@ -3832,7 +3865,8 @@ save_global(PicklerObject *self, PyObject *obj, PyObject *name) } static int -save_singleton_type(PicklerObject *self, PyObject *obj, PyObject *singleton) +save_singleton_type(PickleState *state, PicklerObject *self, PyObject *obj, + PyObject *singleton) { PyObject *reduce_value; int status; @@ -3841,28 +3875,28 @@ save_singleton_type(PicklerObject *self, PyObject *obj, PyObject *singleton) if (reduce_value == NULL) { return -1; } - status = save_reduce(self, reduce_value, obj); + status = save_reduce(state, self, reduce_value, obj); Py_DECREF(reduce_value); return status; } static int -save_type(PicklerObject *self, PyObject *obj) +save_type(PickleState *state, PicklerObject *self, PyObject *obj) { if (obj == (PyObject *)&_PyNone_Type) { - return save_singleton_type(self, obj, Py_None); + return save_singleton_type(state, self, obj, Py_None); } else if (obj == (PyObject *)&PyEllipsis_Type) { - return save_singleton_type(self, obj, Py_Ellipsis); + return save_singleton_type(state, self, obj, Py_Ellipsis); } else if (obj == (PyObject *)&_PyNotImplemented_Type) { - return save_singleton_type(self, obj, Py_NotImplemented); + return save_singleton_type(state, self, obj, Py_NotImplemented); } - return save_global(self, obj, NULL); + return save_global(state, self, obj, NULL); } static int -save_pers(PicklerObject *self, PyObject *obj) +save_pers(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *pid = NULL; int status = 0; @@ -3876,7 +3910,7 @@ save_pers(PicklerObject *self, PyObject *obj) if (pid != Py_None) { if (self->bin) { - if (save(self, pid, 1) < 0 || + if (save(state, self, pid, 1) < 0 || _Pickler_Write(self, &binpersid_op, 1) < 0) goto error; } @@ -3890,7 +3924,7 @@ save_pers(PicklerObject *self, PyObject *obj) /* XXX: Should it check whether the pid contains embedded newlines? */ if (!PyUnicode_IS_ASCII(pid_str)) { - PyErr_SetString(_Pickle_GetGlobalState()->PicklingError, + PyErr_SetString(state->PicklingError, "persistent IDs in protocol 0 must be " "ASCII strings"); Py_DECREF(pid_str); @@ -3933,7 +3967,8 @@ get_class(PyObject *obj) * appropriate __reduce__ method for obj. */ static int -save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) +save_reduce(PickleState *st, PicklerObject *self, PyObject *args, + PyObject *obj) { PyObject *callable; PyObject *argtup; @@ -3941,7 +3976,6 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) PyObject *listitems = Py_None; PyObject *dictitems = Py_None; PyObject *state_setter = Py_None; - PickleState *st = _Pickle_GetGlobalState(); Py_ssize_t size; int use_newobj = 0, use_newobj_ex = 0; @@ -4053,9 +4087,9 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) } if (self->proto >= 4) { - if (save(self, cls, 0) < 0 || - save(self, args, 0) < 0 || - save(self, kwargs, 0) < 0 || + if (save(st, self, cls, 0) < 0 || + save(st, self, args, 0) < 0 || + save(st, self, kwargs, 0) < 0 || _Pickler_Write(self, &newobj_ex_op, 1) < 0) { return -1; } @@ -4092,8 +4126,8 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) return -1; } - if (save(self, callable, 0) < 0 || - save(self, newargs, 0) < 0 || + if (save(st, self, callable, 0) < 0 || + save(st, self, newargs, 0) < 0 || _Pickler_Write(self, &reduce_op, 1) < 0) { Py_DECREF(newargs); Py_DECREF(callable); @@ -4163,14 +4197,15 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) function. */ /* Save the class and its __new__ arguments. */ - if (save(self, cls, 0) < 0) + if (save(st, self, cls, 0) < 0) { return -1; + } newargtup = PyTuple_GetSlice(argtup, 1, PyTuple_GET_SIZE(argtup)); if (newargtup == NULL) return -1; - p = save(self, newargtup, 0); + p = save(st, self, newargtup, 0); Py_DECREF(newargtup); if (p < 0) return -1; @@ -4180,8 +4215,8 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) return -1; } else { /* Not using NEWOBJ. */ - if (save(self, callable, 0) < 0 || - save(self, argtup, 0) < 0 || + if (save(st, self, callable, 0) < 0 || + save(st, self, argtup, 0) < 0 || _Pickler_Write(self, &reduce_op, 1) < 0) return -1; } @@ -4199,24 +4234,24 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) if (_Pickler_Write(self, &pop_op, 1) < 0) return -1; - if (memo_get(self, obj) < 0) + if (memo_get(st, self, obj) < 0) return -1; return 0; } - else if (memo_put(self, obj) < 0) + else if (memo_put(st, self, obj) < 0) return -1; } - if (listitems && batch_list(self, listitems) < 0) + if (listitems && batch_list(st, self, listitems) < 0) return -1; - if (dictitems && batch_dict(self, dictitems) < 0) + if (dictitems && batch_dict(st, self, dictitems) < 0) return -1; if (state) { if (state_setter == NULL) { - if (save(self, state, 0) < 0 || + if (save(st, self, state, 0) < 0 || _Pickler_Write(self, &build_op, 1) < 0) return -1; } @@ -4233,8 +4268,8 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) const char tupletwo_op = TUPLE2; const char pop_op = POP; - if (save(self, state_setter, 0) < 0 || - save(self, obj, 0) < 0 || save(self, state, 0) < 0 || + if (save(st, self, state_setter, 0) < 0 || + save(st, self, obj, 0) < 0 || save(st, self, state, 0) < 0 || _Pickler_Write(self, &tupletwo_op, 1) < 0 || _Pickler_Write(self, &reduce_op, 1) < 0 || _Pickler_Write(self, &pop_op, 1) < 0) @@ -4245,7 +4280,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) } static int -save(PicklerObject *self, PyObject *obj, int pers_save) +save(PickleState *st, PicklerObject *self, PyObject *obj, int pers_save) { PyTypeObject *type; PyObject *reduce_func = NULL; @@ -4263,7 +4298,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) 0 if it did nothing successfully; 1 if a persistent id was saved. */ - if ((status = save_pers(self, obj)) != 0) + if ((status = save_pers(st, self, obj)) != 0) return status; } @@ -4293,14 +4328,14 @@ save(PicklerObject *self, PyObject *obj, int pers_save) a GET (or BINGET) opcode, instead of pickling the object once again. */ if (PyMemoTable_Get(self->memo, obj)) { - return memo_get(self, obj); + return memo_get(st, self, obj); } if (type == &PyBytes_Type) { - return save_bytes(self, obj); + return save_bytes(st, self, obj); } else if (type == &PyUnicode_Type) { - return save_unicode(self, obj); + return save_unicode(st, self, obj); } /* We're only calling _Py_EnterRecursiveCall here so that atomic @@ -4310,31 +4345,31 @@ save(PicklerObject *self, PyObject *obj, int pers_save) } if (type == &PyDict_Type) { - status = save_dict(self, obj); + status = save_dict(st, self, obj); goto done; } else if (type == &PySet_Type) { - status = save_set(self, obj); + status = save_set(st, self, obj); goto done; } else if (type == &PyFrozenSet_Type) { - status = save_frozenset(self, obj); + status = save_frozenset(st, self, obj); goto done; } else if (type == &PyList_Type) { - status = save_list(self, obj); + status = save_list(st, self, obj); goto done; } else if (type == &PyTuple_Type) { - status = save_tuple(self, obj); + status = save_tuple(st, self, obj); goto done; } else if (type == &PyByteArray_Type) { - status = save_bytearray(self, obj); + status = save_bytearray(st, self, obj); goto done; } else if (type == &PyPickleBuffer_Type) { - status = save_picklebuffer(self, obj); + status = save_picklebuffer(st, self, obj); goto done; } @@ -4354,11 +4389,11 @@ save(PicklerObject *self, PyObject *obj, int pers_save) } if (type == &PyType_Type) { - status = save_type(self, obj); + status = save_type(st, self, obj); goto done; } else if (type == &PyFunction_Type) { - status = save_global(self, obj, NULL); + status = save_global(st, self, obj, NULL); goto done; } @@ -4369,7 +4404,6 @@ save(PicklerObject *self, PyObject *obj, int pers_save) * __reduce_ex__ method, or the object's __reduce__ method. */ if (self->dispatch_table == NULL) { - PickleState *st = _Pickle_GetGlobalState(); reduce_func = PyDict_GetItemWithError(st->dispatch_table, (PyObject *)type); if (reduce_func == NULL) { @@ -4396,7 +4430,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) reduce_value = _Pickle_FastCall(reduce_func, Py_NewRef(obj)); } else if (PyType_IsSubtype(type, &PyType_Type)) { - status = save_global(self, obj, NULL); + status = save_global(st, self, obj, NULL); goto done; } else { @@ -4428,7 +4462,6 @@ save(PicklerObject *self, PyObject *obj, int pers_save) reduce_value = PyObject_CallNoArgs(reduce_func); } else { - PickleState *st = _Pickle_GetGlobalState(); PyErr_Format(st->PicklingError, "can't pickle '%.200s' object: %R", type->tp_name, obj); @@ -4442,18 +4475,17 @@ save(PicklerObject *self, PyObject *obj, int pers_save) reduce: if (PyUnicode_Check(reduce_value)) { - status = save_global(self, obj, reduce_value); + status = save_global(st, self, obj, reduce_value); goto done; } if (!PyTuple_Check(reduce_value)) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->PicklingError, "__reduce__ must return a string or tuple"); goto error; } - status = save_reduce(self, reduce_value, obj); + status = save_reduce(st, self, reduce_value, obj); if (0) { error: @@ -4469,7 +4501,7 @@ save(PicklerObject *self, PyObject *obj, int pers_save) } static int -dump(PicklerObject *self, PyObject *obj) +dump(PickleState *state, PicklerObject *self, PyObject *obj) { const char stop_op = STOP; int status = -1; @@ -4499,7 +4531,7 @@ dump(PicklerObject *self, PyObject *obj) self->framing = 1; } - if (save(self, obj, 0) < 0 || + if (save(state, self, obj, 0) < 0 || _Pickler_Write(self, &stop_op, 1) < 0 || _Pickler_CommitFrame(self) < 0) goto error; @@ -4546,6 +4578,7 @@ _pickle_Pickler_clear_memo_impl(PicklerObject *self) _pickle.Pickler.dump + cls: defining_class obj: object / @@ -4553,14 +4586,15 @@ Write a pickled representation of the given object to the open file. [clinic start generated code]*/ static PyObject * -_pickle_Pickler_dump(PicklerObject *self, PyObject *obj) -/*[clinic end generated code: output=87ecad1261e02ac7 input=552eb1c0f52260d9]*/ +_pickle_Pickler_dump_impl(PicklerObject *self, PyTypeObject *cls, + PyObject *obj) +/*[clinic end generated code: output=952cf7f68b1445bb input=f949d84151983594]*/ { + PickleState *st = _Pickle_GetStateByClass(cls); /* Check whether the Pickler was initialized correctly (issue3664). Developers often forget to call __init__() in their subclasses, which would trigger a segfault without this check. */ if (self->write == NULL) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_Format(st->PicklingError, "Pickler.__init__() was not called by %s.__init__()", Py_TYPE(self)->tp_name); @@ -4570,7 +4604,7 @@ _pickle_Pickler_dump(PicklerObject *self, PyObject *obj) if (_Pickler_ClearBuffer(self) < 0) return NULL; - if (dump(self, obj) < 0) + if (dump(st, self, obj) < 0) return NULL; if (_Pickler_FlushToFile(self) < 0) @@ -4612,36 +4646,6 @@ static struct PyMethodDef Pickler_methods[] = { {NULL, NULL} /* sentinel */ }; -static void -Pickler_dealloc(PicklerObject *self) -{ - PyObject_GC_UnTrack(self); - - Py_XDECREF(self->output_buffer); - Py_XDECREF(self->write); - Py_XDECREF(self->pers_func); - Py_XDECREF(self->dispatch_table); - Py_XDECREF(self->fast_memo); - Py_XDECREF(self->reducer_override); - Py_XDECREF(self->buffer_callback); - - PyMemoTable_Del(self->memo); - - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static int -Pickler_traverse(PicklerObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->write); - Py_VISIT(self->pers_func); - Py_VISIT(self->dispatch_table); - Py_VISIT(self->fast_memo); - Py_VISIT(self->reducer_override); - Py_VISIT(self->buffer_callback); - return 0; -} - static int Pickler_clear(PicklerObject *self) { @@ -4661,6 +4665,29 @@ Pickler_clear(PicklerObject *self) return 0; } +static void +Pickler_dealloc(PicklerObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + (void)Pickler_clear(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); +} + +static int +Pickler_traverse(PicklerObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->write); + Py_VISIT(self->pers_func); + Py_VISIT(self->dispatch_table); + Py_VISIT(self->fast_memo); + Py_VISIT(self->reducer_override); + Py_VISIT(self->buffer_callback); + return 0; +} + /*[clinic input] @@ -4870,15 +4897,18 @@ static PyMethodDef picklerproxy_methods[] = { static void PicklerMemoProxy_dealloc(PicklerMemoProxyObject *self) { + PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - Py_XDECREF(self->pickler); - PyObject_GC_Del((PyObject *)self); + Py_CLEAR(self->pickler); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } static int PicklerMemoProxy_traverse(PicklerMemoProxyObject *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->pickler); return 0; } @@ -4890,43 +4920,29 @@ PicklerMemoProxy_clear(PicklerMemoProxyObject *self) return 0; } -static PyTypeObject PicklerMemoProxyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pickle.PicklerMemoProxy", /*tp_name*/ - sizeof(PicklerMemoProxyObject), /*tp_basicsize*/ - 0, - (destructor)PicklerMemoProxy_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - PyObject_HashNotImplemented, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - PyObject_GenericSetAttr, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - 0, /* tp_doc */ - (traverseproc)PicklerMemoProxy_traverse, /* tp_traverse */ - (inquiry)PicklerMemoProxy_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - picklerproxy_methods, /* tp_methods */ +static PyType_Slot memoproxy_slots[] = { + {Py_tp_dealloc, PicklerMemoProxy_dealloc}, + {Py_tp_traverse, PicklerMemoProxy_traverse}, + {Py_tp_clear, PicklerMemoProxy_clear}, + {Py_tp_methods, picklerproxy_methods}, + {Py_tp_hash, PyObject_HashNotImplemented}, + {0, NULL}, +}; + +static PyType_Spec memoproxy_spec = { + .name = "_pickle.PicklerMemoProxy", + .basicsize = sizeof(PicklerMemoProxyObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = memoproxy_slots, }; static PyObject * PicklerMemoProxy_New(PicklerObject *pickler) { PicklerMemoProxyObject *self; - - self = PyObject_GC_New(PicklerMemoProxyObject, &PicklerMemoProxyType); + PickleState *st = _Pickle_FindStateByType(Py_TYPE(pickler)); + self = PyObject_GC_New(PicklerMemoProxyObject, st->PicklerMemoProxyType); if (self == NULL) return NULL; self->pickler = (PicklerObject*)Py_NewRef(pickler); @@ -4953,7 +4969,8 @@ Pickler_set_memo(PicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored)) return -1; } - if (Py_IS_TYPE(obj, &PicklerMemoProxyType)) { + PickleState *st = _Pickle_FindStateByType(Py_TYPE(self)); + if (Py_IS_TYPE(obj, st->PicklerMemoProxyType)) { PicklerObject *pickler = ((PicklerMemoProxyObject *)obj)->pickler; @@ -5049,47 +5066,27 @@ static PyGetSetDef Pickler_getsets[] = { {NULL} }; -static PyTypeObject Pickler_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pickle.Pickler" , /*tp_name*/ - sizeof(PicklerObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)Pickler_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - _pickle_Pickler___init____doc__, /*tp_doc*/ - (traverseproc)Pickler_traverse, /*tp_traverse*/ - (inquiry)Pickler_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - Pickler_methods, /*tp_methods*/ - Pickler_members, /*tp_members*/ - Pickler_getsets, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - _pickle_Pickler___init__, /*tp_init*/ - PyType_GenericAlloc, /*tp_alloc*/ - PyType_GenericNew, /*tp_new*/ - PyObject_GC_Del, /*tp_free*/ - 0, /*tp_is_gc*/ +static PyType_Slot pickler_type_slots[] = { + {Py_tp_dealloc, Pickler_dealloc}, + {Py_tp_methods, Pickler_methods}, + {Py_tp_members, Pickler_members}, + {Py_tp_getset, Pickler_getsets}, + {Py_tp_clear, Pickler_clear}, + {Py_tp_doc, (char*)_pickle_Pickler___init____doc__}, + {Py_tp_traverse, Pickler_traverse}, + {Py_tp_init, _pickle_Pickler___init__}, + {Py_tp_new, PyType_GenericNew}, + {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec pickler_type_spec = { + .name = "_pickle.Pickler", + .basicsize = sizeof(PicklerObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = pickler_type_slots, }; /* Temporary helper for calling self.find_class(). @@ -5107,17 +5104,14 @@ find_class(UnpicklerObject *self, PyObject *module_name, PyObject *global_name) } static Py_ssize_t -marker(UnpicklerObject *self) +marker(PickleState *st, UnpicklerObject *self) { - Py_ssize_t mark; - if (self->num_marks < 1) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "could not find MARK"); return -1; } - mark = self->marks[--self->num_marks]; + Py_ssize_t mark = self->marks[--self->num_marks]; self->stack->mark_set = self->num_marks != 0; self->stack->fence = self->num_marks ? self->marks[self->num_marks - 1] : 0; @@ -5125,24 +5119,24 @@ marker(UnpicklerObject *self) } static int -load_none(UnpicklerObject *self) +load_none(PickleState *state, UnpicklerObject *self) { PDATA_APPEND(self->stack, Py_None, -1); return 0; } static int -load_int(UnpicklerObject *self) +load_int(PickleState *state, UnpicklerObject *self) { PyObject *value; char *endptr, *s; Py_ssize_t len; long x; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(state); errno = 0; /* XXX: Should the base argument of strtol() be explicitly set to 10? @@ -5177,7 +5171,7 @@ load_int(UnpicklerObject *self) } static int -load_bool(UnpicklerObject *self, PyObject *boolean) +load_bool(PickleState *state, UnpicklerObject *self, PyObject *boolean) { assert(boolean == Py_True || boolean == Py_False); PDATA_APPEND(self->stack, boolean, -1); @@ -5257,49 +5251,46 @@ load_binintx(UnpicklerObject *self, char *s, int size) } static int -load_binint(UnpicklerObject *self) +load_binint(PickleState *state, UnpicklerObject *self) { char *s; - - if (_Unpickler_Read(self, &s, 4) < 0) + if (_Unpickler_Read(self, state, &s, 4) < 0) return -1; return load_binintx(self, s, 4); } static int -load_binint1(UnpicklerObject *self) +load_binint1(PickleState *state, UnpicklerObject *self) { char *s; - - if (_Unpickler_Read(self, &s, 1) < 0) + if (_Unpickler_Read(self, state, &s, 1) < 0) return -1; return load_binintx(self, s, 1); } static int -load_binint2(UnpicklerObject *self) +load_binint2(PickleState *state, UnpicklerObject *self) { char *s; - - if (_Unpickler_Read(self, &s, 2) < 0) + if (_Unpickler_Read(self, state, &s, 2) < 0) return -1; return load_binintx(self, s, 2); } static int -load_long(UnpicklerObject *self) +load_long(PickleState *state, UnpicklerObject *self) { PyObject *value; char *s = NULL; Py_ssize_t len; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(state); /* s[len-2] will usually be 'L' (and s[len-1] is '\n'); we need to remove the 'L' before calling PyLong_FromString. In order to maintain @@ -5320,19 +5311,18 @@ load_long(UnpicklerObject *self) * data following. */ static int -load_counted_long(UnpicklerObject *self, int size) +load_counted_long(PickleState *st, UnpicklerObject *self, int size) { PyObject *value; char *nbytes; char *pdata; assert(size == 1 || size == 4); - if (_Unpickler_Read(self, &nbytes, size) < 0) + if (_Unpickler_Read(self, st, &nbytes, size) < 0) return -1; size = calc_binint(nbytes, size); if (size < 0) { - PickleState *st = _Pickle_GetGlobalState(); /* Corrupt or hostile pickle -- we never write one like this */ PyErr_SetString(st->UnpicklingError, "LONG pickle has negative byte count"); @@ -5343,7 +5333,7 @@ load_counted_long(UnpicklerObject *self, int size) value = PyLong_FromLong(0L); else { /* Read the raw little-endian bytes and convert. */ - if (_Unpickler_Read(self, &pdata, size) < 0) + if (_Unpickler_Read(self, st, &pdata, size) < 0) return -1; value = _PyLong_FromByteArray((unsigned char *)pdata, (size_t)size, 1 /* little endian */ , 1 /* signed */ ); @@ -5355,17 +5345,17 @@ load_counted_long(UnpicklerObject *self, int size) } static int -load_float(UnpicklerObject *self) +load_float(PickleState *state, UnpicklerObject *self) { PyObject *value; char *endptr, *s; Py_ssize_t len; double d; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(state); errno = 0; d = PyOS_string_to_double(s, &endptr, PyExc_OverflowError); @@ -5384,13 +5374,13 @@ load_float(UnpicklerObject *self) } static int -load_binfloat(UnpicklerObject *self) +load_binfloat(PickleState *state, UnpicklerObject *self) { PyObject *value; double x; char *s; - if (_Unpickler_Read(self, &s, 8) < 0) + if (_Unpickler_Read(self, state, &s, 8) < 0) return -1; x = PyFloat_Unpack8(s, 0); @@ -5405,14 +5395,14 @@ load_binfloat(UnpicklerObject *self) } static int -load_string(UnpicklerObject *self) +load_string(PickleState *st, UnpicklerObject *self) { PyObject *bytes; PyObject *obj; Py_ssize_t len; char *s, *p; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(st, self, &s)) < 0) return -1; /* Strip the newline */ len--; @@ -5422,7 +5412,6 @@ load_string(UnpicklerObject *self) len -= 2; } else { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "the STRING opcode argument must be quoted"); return -1; @@ -5453,25 +5442,24 @@ load_string(UnpicklerObject *self) } static int -load_counted_binstring(UnpicklerObject *self, int nbytes) +load_counted_binstring(PickleState *st, UnpicklerObject *self, int nbytes) { PyObject *obj; Py_ssize_t size; char *s; - if (_Unpickler_Read(self, &s, nbytes) < 0) + if (_Unpickler_Read(self, st, &s, nbytes) < 0) return -1; size = calc_binsize(s, nbytes); if (size < 0) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_Format(st->UnpicklingError, "BINSTRING exceeds system's maximum size of %zd bytes", PY_SSIZE_T_MAX); return -1; } - if (_Unpickler_Read(self, &s, size) < 0) + if (_Unpickler_Read(self, st, &s, size) < 0) return -1; /* Convert Python 2.x strings to bytes if the *encoding* given to the @@ -5491,13 +5479,13 @@ load_counted_binstring(UnpicklerObject *self, int nbytes) } static int -load_counted_binbytes(UnpicklerObject *self, int nbytes) +load_counted_binbytes(PickleState *state, UnpicklerObject *self, int nbytes) { PyObject *bytes; Py_ssize_t size; char *s; - if (_Unpickler_Read(self, &s, nbytes) < 0) + if (_Unpickler_Read(self, state, &s, nbytes) < 0) return -1; size = calc_binsize(s, nbytes); @@ -5511,7 +5499,7 @@ load_counted_binbytes(UnpicklerObject *self, int nbytes) bytes = PyBytes_FromStringAndSize(NULL, size); if (bytes == NULL) return -1; - if (_Unpickler_ReadInto(self, PyBytes_AS_STRING(bytes), size) < 0) { + if (_Unpickler_ReadInto(state, self, PyBytes_AS_STRING(bytes), size) < 0) { Py_DECREF(bytes); return -1; } @@ -5521,13 +5509,13 @@ load_counted_binbytes(UnpicklerObject *self, int nbytes) } static int -load_counted_bytearray(UnpicklerObject *self) +load_counted_bytearray(PickleState *state, UnpicklerObject *self) { PyObject *bytearray; Py_ssize_t size; char *s; - if (_Unpickler_Read(self, &s, 8) < 0) { + if (_Unpickler_Read(self, state, &s, 8) < 0) { return -1; } @@ -5543,7 +5531,8 @@ load_counted_bytearray(UnpicklerObject *self) if (bytearray == NULL) { return -1; } - if (_Unpickler_ReadInto(self, PyByteArray_AS_STRING(bytearray), size) < 0) { + char *str = PyByteArray_AS_STRING(bytearray); + if (_Unpickler_ReadInto(state, self, str, size) < 0) { Py_DECREF(bytearray); return -1; } @@ -5553,10 +5542,9 @@ load_counted_bytearray(UnpicklerObject *self) } static int -load_next_buffer(UnpicklerObject *self) +load_next_buffer(PickleState *st, UnpicklerObject *self) { if (self->buffers == NULL) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "pickle stream refers to out-of-band data " "but no *buffers* argument was given"); @@ -5565,7 +5553,6 @@ load_next_buffer(UnpicklerObject *self) PyObject *buf = PyIter_Next(self->buffers); if (buf == NULL) { if (!PyErr_Occurred()) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "not enough out-of-band buffers"); } @@ -5577,11 +5564,11 @@ load_next_buffer(UnpicklerObject *self) } static int -load_readonly_buffer(UnpicklerObject *self) +load_readonly_buffer(PickleState *state, UnpicklerObject *self) { Py_ssize_t len = Py_SIZE(self->stack); if (len <= self->stack->fence) { - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); } PyObject *obj = self->stack->data[len - 1]; @@ -5603,16 +5590,16 @@ load_readonly_buffer(UnpicklerObject *self) } static int -load_unicode(UnpicklerObject *self) +load_unicode(PickleState *state, UnpicklerObject *self) { PyObject *str; Py_ssize_t len; char *s = NULL; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 1) - return bad_readline(); + return bad_readline(state); str = PyUnicode_DecodeRawUnicodeEscape(s, len - 1, NULL); if (str == NULL) @@ -5623,13 +5610,13 @@ load_unicode(UnpicklerObject *self) } static int -load_counted_binunicode(UnpicklerObject *self, int nbytes) +load_counted_binunicode(PickleState *state, UnpicklerObject *self, int nbytes) { PyObject *str; Py_ssize_t size; char *s; - if (_Unpickler_Read(self, &s, nbytes) < 0) + if (_Unpickler_Read(self, state, &s, nbytes) < 0) return -1; size = calc_binsize(s, nbytes); @@ -5640,7 +5627,7 @@ load_counted_binunicode(UnpicklerObject *self, int nbytes) return -1; } - if (_Unpickler_Read(self, &s, size) < 0) + if (_Unpickler_Read(self, state, &s, size) < 0) return -1; str = PyUnicode_DecodeUTF8(s, size, "surrogatepass"); @@ -5652,14 +5639,14 @@ load_counted_binunicode(UnpicklerObject *self, int nbytes) } static int -load_counted_tuple(UnpicklerObject *self, Py_ssize_t len) +load_counted_tuple(PickleState *state, UnpicklerObject *self, Py_ssize_t len) { PyObject *tuple; if (Py_SIZE(self->stack) < len) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); - tuple = Pdata_poptuple(self->stack, Py_SIZE(self->stack) - len); + tuple = Pdata_poptuple(state, self->stack, Py_SIZE(self->stack) - len); if (tuple == NULL) return -1; PDATA_PUSH(self->stack, tuple, -1); @@ -5667,18 +5654,18 @@ load_counted_tuple(UnpicklerObject *self, Py_ssize_t len) } static int -load_tuple(UnpicklerObject *self) +load_tuple(PickleState *state, UnpicklerObject *self) { Py_ssize_t i; - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) return -1; - return load_counted_tuple(self, Py_SIZE(self->stack) - i); + return load_counted_tuple(state, self, Py_SIZE(self->stack) - i); } static int -load_empty_list(UnpicklerObject *self) +load_empty_list(PickleState *state, UnpicklerObject *self) { PyObject *list; @@ -5689,7 +5676,7 @@ load_empty_list(UnpicklerObject *self) } static int -load_empty_dict(UnpicklerObject *self) +load_empty_dict(PickleState *state, UnpicklerObject *self) { PyObject *dict; @@ -5700,7 +5687,7 @@ load_empty_dict(UnpicklerObject *self) } static int -load_empty_set(UnpicklerObject *self) +load_empty_set(PickleState *state, UnpicklerObject *self) { PyObject *set; @@ -5711,12 +5698,12 @@ load_empty_set(UnpicklerObject *self) } static int -load_list(UnpicklerObject *self) +load_list(PickleState *state, UnpicklerObject *self) { PyObject *list; Py_ssize_t i; - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) return -1; list = Pdata_poplist(self->stack, i); @@ -5727,12 +5714,12 @@ load_list(UnpicklerObject *self) } static int -load_dict(UnpicklerObject *self) +load_dict(PickleState *st, UnpicklerObject *self) { PyObject *dict, *key, *value; Py_ssize_t i, j, k; - if ((i = marker(self)) < 0) + if ((i = marker(st, self)) < 0) return -1; j = Py_SIZE(self->stack); @@ -5740,7 +5727,6 @@ load_dict(UnpicklerObject *self) return -1; if ((j - i) % 2 != 0) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "odd number of items for DICT"); Py_DECREF(dict); return -1; @@ -5760,16 +5746,16 @@ load_dict(UnpicklerObject *self) } static int -load_frozenset(UnpicklerObject *self) +load_frozenset(PickleState *state, UnpicklerObject *self) { PyObject *items; PyObject *frozenset; Py_ssize_t i; - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) return -1; - items = Pdata_poptuple(self->stack, i); + items = Pdata_poptuple(state, self->stack, i); if (items == NULL) return -1; @@ -5803,22 +5789,22 @@ instantiate(PyObject *cls, PyObject *args) } static int -load_obj(UnpicklerObject *self) +load_obj(PickleState *state, UnpicklerObject *self) { PyObject *cls, *args, *obj = NULL; Py_ssize_t i; - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) return -1; if (Py_SIZE(self->stack) - i < 1) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); - args = Pdata_poptuple(self->stack, i + 1); + args = Pdata_poptuple(state, self->stack, i + 1); if (args == NULL) return -1; - PDATA_POP(self->stack, cls); + PDATA_POP(state, self->stack, cls); if (cls) { obj = instantiate(cls, args); Py_DECREF(cls); @@ -5832,7 +5818,7 @@ load_obj(UnpicklerObject *self) } static int -load_inst(UnpicklerObject *self) +load_inst(PickleState *state, UnpicklerObject *self) { PyObject *cls = NULL; PyObject *args = NULL; @@ -5843,12 +5829,12 @@ load_inst(UnpicklerObject *self) Py_ssize_t i; char *s; - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) return -1; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(state); /* Here it is safe to use PyUnicode_DecodeASCII(), even though non-ASCII identifiers are permitted in Python 3.0, since the INST opcode is only @@ -5857,10 +5843,10 @@ load_inst(UnpicklerObject *self) if (module_name == NULL) return -1; - if ((len = _Unpickler_Readline(self, &s)) >= 0) { + if ((len = _Unpickler_Readline(state, self, &s)) >= 0) { if (len < 2) { Py_DECREF(module_name); - return bad_readline(); + return bad_readline(state); } class_name = PyUnicode_DecodeASCII(s, len - 1, "strict"); if (class_name != NULL) { @@ -5873,7 +5859,7 @@ load_inst(UnpicklerObject *self) if (cls == NULL) return -1; - if ((args = Pdata_poptuple(self->stack, i)) != NULL) { + if ((args = Pdata_poptuple(state, self->stack, i)) != NULL) { obj = instantiate(cls, args); Py_DECREF(args); } @@ -5887,16 +5873,16 @@ load_inst(UnpicklerObject *self) } static void -newobj_unpickling_error(const char * msg, int use_kwargs, PyObject *arg) +newobj_unpickling_error(PickleState *st, const char *msg, int use_kwargs, + PyObject *arg) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_Format(st->UnpicklingError, msg, use_kwargs ? "NEWOBJ_EX" : "NEWOBJ", Py_TYPE(arg)->tp_name); } static int -load_newobj(UnpicklerObject *self, int use_kwargs) +load_newobj(PickleState *state, UnpicklerObject *self, int use_kwargs) { PyObject *cls, *args, *kwargs = NULL; PyObject *obj; @@ -5905,17 +5891,17 @@ load_newobj(UnpicklerObject *self, int use_kwargs) * cls.__new__(cls, *args, **kwargs). */ if (use_kwargs) { - PDATA_POP(self->stack, kwargs); + PDATA_POP(state, self->stack, kwargs); if (kwargs == NULL) { return -1; } } - PDATA_POP(self->stack, args); + PDATA_POP(state, self->stack, args); if (args == NULL) { Py_XDECREF(kwargs); return -1; } - PDATA_POP(self->stack, cls); + PDATA_POP(state, self->stack, cls); if (cls == NULL) { Py_XDECREF(kwargs); Py_DECREF(args); @@ -5923,22 +5909,26 @@ load_newobj(UnpicklerObject *self, int use_kwargs) } if (!PyType_Check(cls)) { - newobj_unpickling_error("%s class argument must be a type, not %.200s", + newobj_unpickling_error(state, + "%s class argument must be a type, not %.200s", use_kwargs, cls); goto error; } if (((PyTypeObject *)cls)->tp_new == NULL) { - newobj_unpickling_error("%s class argument '%.200s' doesn't have __new__", + newobj_unpickling_error(state, + "%s class argument '%.200s' doesn't have __new__", use_kwargs, cls); goto error; } if (!PyTuple_Check(args)) { - newobj_unpickling_error("%s args argument must be a tuple, not %.200s", + newobj_unpickling_error(state, + "%s args argument must be a tuple, not %.200s", use_kwargs, args); goto error; } if (use_kwargs && !PyDict_Check(kwargs)) { - newobj_unpickling_error("%s kwargs argument must be a dict, not %.200s", + newobj_unpickling_error(state, + "%s kwargs argument must be a dict, not %.200s", use_kwargs, kwargs); goto error; } @@ -5961,7 +5951,7 @@ load_newobj(UnpicklerObject *self, int use_kwargs) } static int -load_global(UnpicklerObject *self) +load_global(PickleState *state, UnpicklerObject *self) { PyObject *global = NULL; PyObject *module_name; @@ -5969,18 +5959,18 @@ load_global(UnpicklerObject *self) Py_ssize_t len; char *s; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(state); module_name = PyUnicode_DecodeUTF8(s, len - 1, "strict"); if (!module_name) return -1; - if ((len = _Unpickler_Readline(self, &s)) >= 0) { + if ((len = _Unpickler_Readline(state, self, &s)) >= 0) { if (len < 2) { Py_DECREF(module_name); - return bad_readline(); + return bad_readline(state); } global_name = PyUnicode_DecodeUTF8(s, len - 1, "strict"); if (global_name) { @@ -5997,17 +5987,16 @@ load_global(UnpicklerObject *self) } static int -load_stack_global(UnpicklerObject *self) +load_stack_global(PickleState *st, UnpicklerObject *self) { PyObject *global; PyObject *module_name; PyObject *global_name; - PDATA_POP(self->stack, global_name); - PDATA_POP(self->stack, module_name); + PDATA_POP(st, self->stack, global_name); + PDATA_POP(st, self->stack, module_name); if (module_name == NULL || !PyUnicode_CheckExact(module_name) || global_name == NULL || !PyUnicode_CheckExact(global_name)) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "STACK_GLOBAL requires str"); Py_XDECREF(global_name); Py_XDECREF(module_name); @@ -6023,22 +6012,22 @@ load_stack_global(UnpicklerObject *self) } static int -load_persid(UnpicklerObject *self) +load_persid(PickleState *st, UnpicklerObject *self) { PyObject *pid, *obj; Py_ssize_t len; char *s; if (self->pers_func) { - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(st, self, &s)) < 0) return -1; if (len < 1) - return bad_readline(); + return bad_readline(st); pid = PyUnicode_DecodeASCII(s, len - 1, "strict"); if (pid == NULL) { if (PyErr_ExceptionMatches(PyExc_UnicodeDecodeError)) { - PyErr_SetString(_Pickle_GetGlobalState()->UnpicklingError, + PyErr_SetString(st->UnpicklingError, "persistent IDs in protocol 0 must be " "ASCII strings"); } @@ -6054,7 +6043,6 @@ load_persid(UnpicklerObject *self) return 0; } else { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "A load persistent id instruction was encountered, " "but no persistent_load function was specified."); @@ -6063,12 +6051,12 @@ load_persid(UnpicklerObject *self) } static int -load_binpersid(UnpicklerObject *self) +load_binpersid(PickleState *st, UnpicklerObject *self) { PyObject *pid, *obj; if (self->pers_func) { - PDATA_POP(self->stack, pid); + PDATA_POP(st, self->stack, pid); if (pid == NULL) return -1; @@ -6081,7 +6069,6 @@ load_binpersid(UnpicklerObject *self) return 0; } else { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "A load persistent id instruction was encountered, " "but no persistent_load function was specified."); @@ -6090,7 +6077,7 @@ load_binpersid(UnpicklerObject *self) } static int -load_pop(UnpicklerObject *self) +load_pop(PickleState *state, UnpicklerObject *self) { Py_ssize_t len = Py_SIZE(self->stack); @@ -6107,7 +6094,7 @@ load_pop(UnpicklerObject *self) self->stack->fence = self->num_marks ? self->marks[self->num_marks - 1] : 0; } else if (len <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); else { len--; Py_DECREF(self->stack->data[len]); @@ -6117,11 +6104,10 @@ load_pop(UnpicklerObject *self) } static int -load_pop_mark(UnpicklerObject *self) +load_pop_mark(PickleState *state, UnpicklerObject *self) { Py_ssize_t i; - - if ((i = marker(self)) < 0) + if ((i = marker(state, self)) < 0) return -1; Pdata_clear(self->stack, i); @@ -6130,30 +6116,30 @@ load_pop_mark(UnpicklerObject *self) } static int -load_dup(UnpicklerObject *self) +load_dup(PickleState *state, UnpicklerObject *self) { PyObject *last; Py_ssize_t len = Py_SIZE(self->stack); if (len <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); last = self->stack->data[len - 1]; PDATA_APPEND(self->stack, last, -1); return 0; } static int -load_get(UnpicklerObject *self) +load_get(PickleState *st, UnpicklerObject *self) { PyObject *key, *value; Py_ssize_t idx; Py_ssize_t len; char *s; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(st, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(st); key = PyLong_FromString(s, NULL, 10); if (key == NULL) @@ -6167,7 +6153,6 @@ load_get(UnpicklerObject *self) value = _Unpickler_MemoGet(self, idx); if (value == NULL) { if (!PyErr_Occurred()) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); } Py_DECREF(key); @@ -6180,13 +6165,13 @@ load_get(UnpicklerObject *self) } static int -load_binget(UnpicklerObject *self) +load_binget(PickleState *st, UnpicklerObject *self) { PyObject *value; Py_ssize_t idx; char *s; - if (_Unpickler_Read(self, &s, 1) < 0) + if (_Unpickler_Read(self, st, &s, 1) < 0) return -1; idx = Py_CHARMASK(s[0]); @@ -6195,7 +6180,6 @@ load_binget(UnpicklerObject *self) if (value == NULL) { PyObject *key = PyLong_FromSsize_t(idx); if (key != NULL) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); Py_DECREF(key); } @@ -6207,13 +6191,13 @@ load_binget(UnpicklerObject *self) } static int -load_long_binget(UnpicklerObject *self) +load_long_binget(PickleState *st, UnpicklerObject *self) { PyObject *value; Py_ssize_t idx; char *s; - if (_Unpickler_Read(self, &s, 4) < 0) + if (_Unpickler_Read(self, st, &s, 4) < 0) return -1; idx = calc_binsize(s, 4); @@ -6222,7 +6206,6 @@ load_long_binget(UnpicklerObject *self) if (value == NULL) { PyObject *key = PyLong_FromSsize_t(idx); if (key != NULL) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_Format(st->UnpicklingError, "Memo value not found at index %ld", idx); Py_DECREF(key); } @@ -6237,7 +6220,7 @@ load_long_binget(UnpicklerObject *self) * the number of bytes following the opcode, holding the index (code) value. */ static int -load_extension(UnpicklerObject *self, int nbytes) +load_extension(PickleState *st, UnpicklerObject *self, int nbytes) { char *codebytes; /* the nbytes bytes after the opcode */ long code; /* calc_binint returns long */ @@ -6245,10 +6228,9 @@ load_extension(UnpicklerObject *self, int nbytes) PyObject *obj; /* the object to push */ PyObject *pair; /* (module_name, class_name) */ PyObject *module_name, *class_name; - PickleState *st = _Pickle_GetGlobalState(); assert(nbytes == 1 || nbytes == 2 || nbytes == 4); - if (_Unpickler_Read(self, &codebytes, nbytes) < 0) + if (_Unpickler_Read(self, st, &codebytes, nbytes) < 0) return -1; code = calc_binint(codebytes, nbytes); if (code <= 0) { /* note that 0 is forbidden */ @@ -6324,19 +6306,19 @@ load_extension(UnpicklerObject *self, int nbytes) } static int -load_put(UnpicklerObject *self) +load_put(PickleState *state, UnpicklerObject *self) { PyObject *key, *value; Py_ssize_t idx; Py_ssize_t len; char *s = NULL; - if ((len = _Unpickler_Readline(self, &s)) < 0) + if ((len = _Unpickler_Readline(state, self, &s)) < 0) return -1; if (len < 2) - return bad_readline(); + return bad_readline(state); if (Py_SIZE(self->stack) <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; key = PyLong_FromString(s, NULL, 10); @@ -6355,17 +6337,17 @@ load_put(UnpicklerObject *self) } static int -load_binput(UnpicklerObject *self) +load_binput(PickleState *state, UnpicklerObject *self) { PyObject *value; Py_ssize_t idx; char *s; - if (_Unpickler_Read(self, &s, 1) < 0) + if (_Unpickler_Read(self, state, &s, 1) < 0) return -1; if (Py_SIZE(self->stack) <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; idx = Py_CHARMASK(s[0]); @@ -6374,17 +6356,17 @@ load_binput(UnpicklerObject *self) } static int -load_long_binput(UnpicklerObject *self) +load_long_binput(PickleState *state, UnpicklerObject *self) { PyObject *value; Py_ssize_t idx; char *s; - if (_Unpickler_Read(self, &s, 4) < 0) + if (_Unpickler_Read(self, state, &s, 4) < 0) return -1; if (Py_SIZE(self->stack) <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; idx = calc_binsize(s, 4); @@ -6398,19 +6380,19 @@ load_long_binput(UnpicklerObject *self) } static int -load_memoize(UnpicklerObject *self) +load_memoize(PickleState *state, UnpicklerObject *self) { PyObject *value; if (Py_SIZE(self->stack) <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); value = self->stack->data[Py_SIZE(self->stack) - 1]; return _Unpickler_MemoPut(self, self->memo_len, value); } static int -do_append(UnpicklerObject *self, Py_ssize_t x) +do_append(PickleState *state, UnpicklerObject *self, Py_ssize_t x) { PyObject *value; PyObject *slice; @@ -6420,7 +6402,7 @@ do_append(UnpicklerObject *self, Py_ssize_t x) len = Py_SIZE(self->stack); if (x > len || x <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); if (len == x) /* nothing to do */ return 0; @@ -6485,24 +6467,24 @@ do_append(UnpicklerObject *self, Py_ssize_t x) } static int -load_append(UnpicklerObject *self) +load_append(PickleState *state, UnpicklerObject *self) { if (Py_SIZE(self->stack) - 1 <= self->stack->fence) - return Pdata_stack_underflow(self->stack); - return do_append(self, Py_SIZE(self->stack) - 1); + return Pdata_stack_underflow(state, self->stack); + return do_append(state, self, Py_SIZE(self->stack) - 1); } static int -load_appends(UnpicklerObject *self) +load_appends(PickleState *state, UnpicklerObject *self) { - Py_ssize_t i = marker(self); + Py_ssize_t i = marker(state, self); if (i < 0) return -1; - return do_append(self, i); + return do_append(state, self, i); } static int -do_setitems(UnpicklerObject *self, Py_ssize_t x) +do_setitems(PickleState *st, UnpicklerObject *self, Py_ssize_t x) { PyObject *value, *key; PyObject *dict; @@ -6511,11 +6493,10 @@ do_setitems(UnpicklerObject *self, Py_ssize_t x) len = Py_SIZE(self->stack); if (x > len || x <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(st, self->stack); if (len == x) /* nothing to do */ return 0; if ((len - x) % 2 != 0) { - PickleState *st = _Pickle_GetGlobalState(); /* Corrupt or hostile pickle -- we never write one like this. */ PyErr_SetString(st->UnpicklingError, "odd number of items for SETITEMS"); @@ -6540,32 +6521,32 @@ do_setitems(UnpicklerObject *self, Py_ssize_t x) } static int -load_setitem(UnpicklerObject *self) +load_setitem(PickleState *state, UnpicklerObject *self) { - return do_setitems(self, Py_SIZE(self->stack) - 2); + return do_setitems(state, self, Py_SIZE(self->stack) - 2); } static int -load_setitems(UnpicklerObject *self) +load_setitems(PickleState *state, UnpicklerObject *self) { - Py_ssize_t i = marker(self); + Py_ssize_t i = marker(state, self); if (i < 0) return -1; - return do_setitems(self, i); + return do_setitems(state, self, i); } static int -load_additems(UnpicklerObject *self) +load_additems(PickleState *state, UnpicklerObject *self) { PyObject *set; Py_ssize_t mark, len, i; - mark = marker(self); + mark = marker(state, self); if (mark < 0) return -1; len = Py_SIZE(self->stack); if (mark > len || mark <= self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(state, self->stack); if (len == mark) /* nothing to do */ return 0; @@ -6575,7 +6556,7 @@ load_additems(UnpicklerObject *self) PyObject *items; int status; - items = Pdata_poptuple(self->stack, mark); + items = Pdata_poptuple(state, self->stack, mark); if (items == NULL) return -1; @@ -6609,9 +6590,9 @@ load_additems(UnpicklerObject *self) } static int -load_build(UnpicklerObject *self) +load_build(PickleState *st, UnpicklerObject *self) { - PyObject *state, *inst, *slotstate; + PyObject *inst, *slotstate; PyObject *setstate; int status = 0; @@ -6619,9 +6600,10 @@ load_build(UnpicklerObject *self) * the stack top, possibly mutated via instance.__setstate__(state). */ if (Py_SIZE(self->stack) - 2 < self->stack->fence) - return Pdata_stack_underflow(self->stack); + return Pdata_stack_underflow(st, self->stack); - PDATA_POP(self->stack, state); + PyObject *state; + PDATA_POP(st, self->stack, state); if (state == NULL) return -1; @@ -6665,7 +6647,6 @@ load_build(UnpicklerObject *self) Py_ssize_t i; if (!PyDict_Check(state)) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "state is not a dictionary"); goto error; } @@ -6695,7 +6676,6 @@ load_build(UnpicklerObject *self) Py_ssize_t i; if (!PyDict_Check(slotstate)) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_SetString(st->UnpicklingError, "slot state is not a dictionary"); goto error; @@ -6718,7 +6698,7 @@ load_build(UnpicklerObject *self) } static int -load_mark(UnpicklerObject *self) +load_mark(PickleState *state, UnpicklerObject *self) { /* Note that we split the (pickle.py) stack into two stacks, an @@ -6745,16 +6725,16 @@ load_mark(UnpicklerObject *self) } static int -load_reduce(UnpicklerObject *self) +load_reduce(PickleState *state, UnpicklerObject *self) { PyObject *callable = NULL; PyObject *argtup = NULL; PyObject *obj = NULL; - PDATA_POP(self->stack, argtup); + PDATA_POP(state, self->stack, argtup); if (argtup == NULL) return -1; - PDATA_POP(self->stack, callable); + PDATA_POP(state, self->stack, callable); if (callable) { obj = PyObject_CallObject(callable, argtup); Py_DECREF(callable); @@ -6772,12 +6752,12 @@ load_reduce(UnpicklerObject *self) * is the first opcode for protocols >= 2. */ static int -load_proto(UnpicklerObject *self) +load_proto(PickleState *state, UnpicklerObject *self) { char *s; int i; - if (_Unpickler_Read(self, &s, 1) < 0) + if (_Unpickler_Read(self, state, &s, 1) < 0) return -1; i = (unsigned char)s[0]; @@ -6791,12 +6771,12 @@ load_proto(UnpicklerObject *self) } static int -load_frame(UnpicklerObject *self) +load_frame(PickleState *state, UnpicklerObject *self) { char *s; Py_ssize_t frame_len; - if (_Unpickler_Read(self, &s, 8) < 0) + if (_Unpickler_Read(self, state, &s, 8) < 0) return -1; frame_len = calc_binsize(s, 8); @@ -6807,7 +6787,7 @@ load_frame(UnpicklerObject *self) return -1; } - if (_Unpickler_Read(self, &s, frame_len) < 0) + if (_Unpickler_Read(self, state, &s, frame_len) < 0) return -1; /* Rewind to start of frame */ @@ -6816,7 +6796,7 @@ load_frame(UnpicklerObject *self) } static PyObject * -load(UnpicklerObject *self) +load(PickleState *st, UnpicklerObject *self) { PyObject *value = NULL; char *s = NULL; @@ -6830,14 +6810,13 @@ load(UnpicklerObject *self) /* Convenient macros for the dispatch while-switch loop just below. */ #define OP(opcode, load_func) \ - case opcode: if (load_func(self) < 0) break; continue; + case opcode: if (load_func(st, self) < 0) break; continue; #define OP_ARG(opcode, load_func, arg) \ - case opcode: if (load_func(self, (arg)) < 0) break; continue; + case opcode: if (load_func(st, self, (arg)) < 0) break; continue; while (1) { - if (_Unpickler_Read(self, &s, 1) < 0) { - PickleState *st = _Pickle_GetGlobalState(); + if (_Unpickler_Read(self, st, &s, 1) < 0) { if (PyErr_ExceptionMatches(st->UnpicklingError)) { PyErr_Format(PyExc_EOFError, "Ran out of input"); } @@ -6918,7 +6897,6 @@ load(UnpicklerObject *self) default: { - PickleState *st = _Pickle_GetGlobalState(); unsigned char c = (unsigned char) *s; if (0x20 <= c && c <= 0x7e && c != '\'' && c != '\\') { PyErr_Format(st->UnpicklingError, @@ -6942,7 +6920,7 @@ load(UnpicklerObject *self) if (_Unpickler_SkipConsumed(self) < 0) return NULL; - PDATA_POP(self->stack, value); + PDATA_POP(st, self->stack, value); return value; } @@ -6950,6 +6928,8 @@ load(UnpicklerObject *self) _pickle.Unpickler.load + cls: defining_class + Load a pickle. Read a pickled object representation from the open file object given @@ -6958,24 +6938,25 @@ specified therein. [clinic start generated code]*/ static PyObject * -_pickle_Unpickler_load_impl(UnpicklerObject *self) -/*[clinic end generated code: output=fdcc488aad675b14 input=acbb91a42fa9b7b9]*/ +_pickle_Unpickler_load_impl(UnpicklerObject *self, PyTypeObject *cls) +/*[clinic end generated code: output=cc88168f608e3007 input=f5d2f87e61d5f07f]*/ { UnpicklerObject *unpickler = (UnpicklerObject*)self; + PickleState *st = _Pickle_GetStateByClass(cls); + /* Check whether the Unpickler was initialized correctly. This prevents segfaulting if a subclass overridden __init__ with a function that does not call Unpickler.__init__(). Here, we simply ensure that self->read is not NULL. */ if (unpickler->read == NULL) { - PickleState *st = _Pickle_GetGlobalState(); PyErr_Format(st->UnpicklingError, "Unpickler.__init__() was not called by %s.__init__()", Py_TYPE(unpickler)->tp_name); return NULL; } - return load(unpickler); + return load(st, unpickler); } /* The name of find_class() is misleading. In newer pickle protocols, this @@ -6986,6 +6967,7 @@ _pickle_Unpickler_load_impl(UnpicklerObject *self) _pickle.Unpickler.find_class + cls: defining_class module_name: object global_name: object / @@ -7001,10 +6983,10 @@ needed. Both arguments passed are str objects. [clinic start generated code]*/ static PyObject * -_pickle_Unpickler_find_class_impl(UnpicklerObject *self, +_pickle_Unpickler_find_class_impl(UnpicklerObject *self, PyTypeObject *cls, PyObject *module_name, PyObject *global_name) -/*[clinic end generated code: output=becc08d7f9ed41e3 input=e2e6a865de093ef4]*/ +/*[clinic end generated code: output=99577948abb0be81 input=9577745719219fc7]*/ { PyObject *global; PyObject *module; @@ -7020,7 +7002,7 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, if (self->proto < 3 && self->fix_imports) { PyObject *key; PyObject *item; - PickleState *st = _Pickle_GetGlobalState(); + PickleState *st = _Pickle_GetStateByClass(cls); /* Check if the global (i.e., a function or a class) was renamed or moved to another module. */ @@ -7114,44 +7096,6 @@ static struct PyMethodDef Unpickler_methods[] = { {NULL, NULL} /* sentinel */ }; -static void -Unpickler_dealloc(UnpicklerObject *self) -{ - PyObject_GC_UnTrack((PyObject *)self); - Py_XDECREF(self->readline); - Py_XDECREF(self->readinto); - Py_XDECREF(self->read); - Py_XDECREF(self->peek); - Py_XDECREF(self->stack); - Py_XDECREF(self->pers_func); - Py_XDECREF(self->buffers); - if (self->buffer.buf != NULL) { - PyBuffer_Release(&self->buffer); - self->buffer.buf = NULL; - } - - _Unpickler_MemoCleanup(self); - PyMem_Free(self->marks); - PyMem_Free(self->input_line); - PyMem_Free(self->encoding); - PyMem_Free(self->errors); - - Py_TYPE(self)->tp_free((PyObject *)self); -} - -static int -Unpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->readline); - Py_VISIT(self->readinto); - Py_VISIT(self->read); - Py_VISIT(self->peek); - Py_VISIT(self->stack); - Py_VISIT(self->pers_func); - Py_VISIT(self->buffers); - return 0; -} - static int Unpickler_clear(UnpicklerObject *self) { @@ -7180,6 +7124,30 @@ Unpickler_clear(UnpicklerObject *self) return 0; } +static void +Unpickler_dealloc(UnpicklerObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack((PyObject *)self); + (void)Unpickler_clear(self); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); +} + +static int +Unpickler_traverse(UnpicklerObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + Py_VISIT(self->readline); + Py_VISIT(self->readinto); + Py_VISIT(self->read); + Py_VISIT(self->peek); + Py_VISIT(self->stack); + Py_VISIT(self->pers_func); + Py_VISIT(self->buffers); + return 0; +} + /*[clinic input] _pickle.Unpickler.__init__ @@ -7240,7 +7208,9 @@ _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file, return -1; } - self->stack = (Pdata *)Pdata_New(); + PyTypeObject *tp = Py_TYPE(self); + PickleState *state = _Pickle_FindStateByType(tp); + self->stack = (Pdata *)Pdata_New(state); if (self->stack == NULL) return -1; @@ -7366,15 +7336,18 @@ static PyMethodDef unpicklerproxy_methods[] = { static void UnpicklerMemoProxy_dealloc(UnpicklerMemoProxyObject *self) { + PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - Py_XDECREF(self->unpickler); - PyObject_GC_Del((PyObject *)self); + Py_CLEAR(self->unpickler); + tp->tp_free((PyObject *)self); + Py_DECREF(tp); } static int UnpicklerMemoProxy_traverse(UnpicklerMemoProxyObject *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); Py_VISIT(self->unpickler); return 0; } @@ -7386,44 +7359,30 @@ UnpicklerMemoProxy_clear(UnpicklerMemoProxyObject *self) return 0; } -static PyTypeObject UnpicklerMemoProxyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pickle.UnpicklerMemoProxy", /*tp_name*/ - sizeof(UnpicklerMemoProxyObject), /*tp_basicsize*/ - 0, - (destructor)UnpicklerMemoProxy_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - PyObject_HashNotImplemented, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - PyObject_GenericSetAttr, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - 0, /* tp_doc */ - (traverseproc)UnpicklerMemoProxy_traverse, /* tp_traverse */ - (inquiry)UnpicklerMemoProxy_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - unpicklerproxy_methods, /* tp_methods */ +static PyType_Slot unpickler_memoproxy_slots[] = { + {Py_tp_dealloc, UnpicklerMemoProxy_dealloc}, + {Py_tp_traverse, UnpicklerMemoProxy_traverse}, + {Py_tp_clear, UnpicklerMemoProxy_clear}, + {Py_tp_methods, unpicklerproxy_methods}, + {Py_tp_hash, PyObject_HashNotImplemented}, + {0, NULL}, +}; + +static PyType_Spec unpickler_memoproxy_spec = { + .name = "_pickle.UnpicklerMemoProxy", + .basicsize = sizeof(UnpicklerMemoProxyObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = unpickler_memoproxy_slots, }; static PyObject * UnpicklerMemoProxy_New(UnpicklerObject *unpickler) { + PickleState *state = _Pickle_FindStateByType(Py_TYPE(unpickler)); UnpicklerMemoProxyObject *self; - self = PyObject_GC_New(UnpicklerMemoProxyObject, - &UnpicklerMemoProxyType); + state->UnpicklerMemoProxyType); if (self == NULL) return NULL; self->unpickler = (UnpicklerObject*)Py_NewRef(unpickler); @@ -7452,7 +7411,8 @@ Unpickler_set_memo(UnpicklerObject *self, PyObject *obj, void *Py_UNUSED(ignored return -1; } - if (Py_IS_TYPE(obj, &UnpicklerMemoProxyType)) { + PickleState *state = _Pickle_FindStateByType(Py_TYPE(self)); + if (Py_IS_TYPE(obj, state->UnpicklerMemoProxyType)) { UnpicklerObject *unpickler = ((UnpicklerMemoProxyObject *)obj)->unpickler; @@ -7554,47 +7514,26 @@ static PyGetSetDef Unpickler_getsets[] = { {NULL} }; -static PyTypeObject Unpickler_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pickle.Unpickler", /*tp_name*/ - sizeof(UnpicklerObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)Unpickler_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - _pickle_Unpickler___init____doc__, /*tp_doc*/ - (traverseproc)Unpickler_traverse, /*tp_traverse*/ - (inquiry)Unpickler_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - Unpickler_methods, /*tp_methods*/ - 0, /*tp_members*/ - Unpickler_getsets, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - _pickle_Unpickler___init__, /*tp_init*/ - PyType_GenericAlloc, /*tp_alloc*/ - PyType_GenericNew, /*tp_new*/ - PyObject_GC_Del, /*tp_free*/ - 0, /*tp_is_gc*/ +static PyType_Slot unpickler_type_slots[] = { + {Py_tp_dealloc, Unpickler_dealloc}, + {Py_tp_doc, (char *)_pickle_Unpickler___init____doc__}, + {Py_tp_traverse, Unpickler_traverse}, + {Py_tp_clear, Unpickler_clear}, + {Py_tp_methods, Unpickler_methods}, + {Py_tp_getset, Unpickler_getsets}, + {Py_tp_init, _pickle_Unpickler___init__}, + {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_new, PyType_GenericNew}, + {Py_tp_free, PyObject_GC_Del}, + {0, NULL}, +}; + +static PyType_Spec unpickler_type_spec = { + .name = "_pickle.Unpickler", + .basicsize = sizeof(UnpicklerObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = unpickler_type_slots, }; /*[clinic input] @@ -7643,7 +7582,8 @@ _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file, PyObject *buffer_callback) /*[clinic end generated code: output=706186dba996490c input=5ed6653da99cd97c]*/ { - PicklerObject *pickler = _Pickler_New(); + PickleState *state = _Pickle_GetState(module); + PicklerObject *pickler = _Pickler_New(state); if (pickler == NULL) return NULL; @@ -7657,7 +7597,7 @@ _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file, if (_Pickler_SetBufferCallback(pickler, buffer_callback) < 0) goto error; - if (dump(pickler, obj) < 0) + if (dump(state, pickler, obj) < 0) goto error; if (_Pickler_FlushToFile(pickler) < 0) @@ -7708,7 +7648,8 @@ _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol, /*[clinic end generated code: output=fbab0093a5580fdf input=e543272436c6f987]*/ { PyObject *result; - PicklerObject *pickler = _Pickler_New(); + PickleState *state = _Pickle_GetState(module); + PicklerObject *pickler = _Pickler_New(state); if (pickler == NULL) return NULL; @@ -7719,7 +7660,7 @@ _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol, if (_Pickler_SetBufferCallback(pickler, buffer_callback) < 0) goto error; - if (dump(pickler, obj) < 0) + if (dump(state, pickler, obj) < 0) goto error; result = _Pickler_GetString(pickler); @@ -7774,7 +7715,7 @@ _pickle_load_impl(PyObject *module, PyObject *file, int fix_imports, /*[clinic end generated code: output=250452d141c23e76 input=46c7c31c92f4f371]*/ { PyObject *result; - UnpicklerObject *unpickler = _Unpickler_New(); + UnpicklerObject *unpickler = _Unpickler_New(module); if (unpickler == NULL) return NULL; @@ -7790,7 +7731,8 @@ _pickle_load_impl(PyObject *module, PyObject *file, int fix_imports, unpickler->fix_imports = fix_imports; - result = load(unpickler); + PickleState *state = _Pickle_GetState(module); + result = load(state, unpickler); Py_DECREF(unpickler); return result; @@ -7834,7 +7776,7 @@ _pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports, /*[clinic end generated code: output=82ac1e6b588e6d02 input=b3615540d0535087]*/ { PyObject *result; - UnpicklerObject *unpickler = _Unpickler_New(); + UnpicklerObject *unpickler = _Unpickler_New(module); if (unpickler == NULL) return NULL; @@ -7850,7 +7792,8 @@ _pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports, unpickler->fix_imports = fix_imports; - result = load(unpickler); + PickleState *state = _Pickle_GetState(module); + result = load(state, unpickler); Py_DECREF(unpickler); return result; @@ -7898,81 +7841,94 @@ pickle_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->codecs_encode); Py_VISIT(st->getattr); Py_VISIT(st->partial); + Py_VISIT(st->Pickler_Type); + Py_VISIT(st->Unpickler_Type); + Py_VISIT(st->Pdata_Type); + Py_VISIT(st->PicklerMemoProxyType); + Py_VISIT(st->UnpicklerMemoProxyType); return 0; } -static struct PyModuleDef _picklemodule = { - PyModuleDef_HEAD_INIT, - "_pickle", /* m_name */ - pickle_module_doc, /* m_doc */ - sizeof(PickleState), /* m_size */ - pickle_methods, /* m_methods */ - NULL, /* m_reload */ - pickle_traverse, /* m_traverse */ - pickle_clear, /* m_clear */ - (freefunc)pickle_free /* m_free */ -}; - -PyMODINIT_FUNC -PyInit__pickle(void) +static int +_pickle_exec(PyObject *m) { - PyObject *m; - PickleState *st; + PickleState *st = _Pickle_GetState(m); - m = PyState_FindModule(&_picklemodule); - if (m) { - return Py_NewRef(m); - } +#define CREATE_TYPE(mod, type, spec) \ + do { \ + type = (PyTypeObject *)PyType_FromMetaclass(NULL, mod, spec, NULL); \ + if (type == NULL) { \ + return -1; \ + } \ + } while (0) - if (PyType_Ready(&Pdata_Type) < 0) - return NULL; - if (PyType_Ready(&PicklerMemoProxyType) < 0) - return NULL; - if (PyType_Ready(&UnpicklerMemoProxyType) < 0) - return NULL; + CREATE_TYPE(m, st->Pdata_Type, &pdata_spec); + CREATE_TYPE(m, st->PicklerMemoProxyType, &memoproxy_spec); + CREATE_TYPE(m, st->UnpicklerMemoProxyType, &unpickler_memoproxy_spec); + CREATE_TYPE(m, st->Pickler_Type, &pickler_type_spec); + CREATE_TYPE(m, st->Unpickler_Type, &unpickler_type_spec); - /* Create the module and add the functions. */ - m = PyModule_Create(&_picklemodule); - if (m == NULL) - return NULL; +#undef CREATE_TYPE /* Add types */ - if (PyModule_AddType(m, &Pickler_Type) < 0) { - return NULL; + if (PyModule_AddType(m, &PyPickleBuffer_Type) < 0) { + return -1; } - if (PyModule_AddType(m, &Unpickler_Type) < 0) { - return NULL; + if (PyModule_AddType(m, st->Pickler_Type) < 0) { + return -1; } - if (PyModule_AddType(m, &PyPickleBuffer_Type) < 0) { - return NULL; + if (PyModule_AddType(m, st->Unpickler_Type) < 0) { + return -1; } - st = _Pickle_GetState(m); - /* Initialize the exceptions. */ st->PickleError = PyErr_NewException("_pickle.PickleError", NULL, NULL); if (st->PickleError == NULL) - return NULL; + return -1; st->PicklingError = \ PyErr_NewException("_pickle.PicklingError", st->PickleError, NULL); if (st->PicklingError == NULL) - return NULL; + return -1; st->UnpicklingError = \ PyErr_NewException("_pickle.UnpicklingError", st->PickleError, NULL); if (st->UnpicklingError == NULL) - return NULL; + return -1; if (PyModule_AddObjectRef(m, "PickleError", st->PickleError) < 0) { - return NULL; + return -1; } if (PyModule_AddObjectRef(m, "PicklingError", st->PicklingError) < 0) { - return NULL; + return -1; } if (PyModule_AddObjectRef(m, "UnpicklingError", st->UnpicklingError) < 0) { - return NULL; + return -1; } + if (_Pickle_InitState(st) < 0) - return NULL; + return -1; - return m; + return 0; +} + +static PyModuleDef_Slot pickle_slots[] = { + {Py_mod_exec, _pickle_exec}, + {0, NULL}, +}; + +static struct PyModuleDef _picklemodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_pickle", + .m_doc = pickle_module_doc, + .m_size = sizeof(PickleState), + .m_methods = pickle_methods, + .m_slots = pickle_slots, + .m_traverse = pickle_traverse, + .m_clear = pickle_clear, + .m_free = (freefunc)pickle_free, +}; + +PyMODINIT_FUNC +PyInit__pickle(void) +{ + return PyModuleDef_Init(&_picklemodule); } diff --git a/Modules/clinic/_pickle.c.h b/Modules/clinic/_pickle.c.h index adb3abc5eb2372..539acc34a05cc1 100644 --- a/Modules/clinic/_pickle.c.h +++ b/Modules/clinic/_pickle.c.h @@ -38,7 +38,42 @@ PyDoc_STRVAR(_pickle_Pickler_dump__doc__, "Write a pickled representation of the given object to the open file."); #define _PICKLE_PICKLER_DUMP_METHODDEF \ - {"dump", (PyCFunction)_pickle_Pickler_dump, METH_O, _pickle_Pickler_dump__doc__}, + {"dump", _PyCFunction_CAST(_pickle_Pickler_dump), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _pickle_Pickler_dump__doc__}, + +static PyObject * +_pickle_Pickler_dump_impl(PicklerObject *self, PyTypeObject *cls, + PyObject *obj); + +static PyObject * +_pickle_Pickler_dump(PicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "dump", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *obj; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + obj = args[0]; + return_value = _pickle_Pickler_dump_impl(self, cls, obj); + +exit: + return return_value; +} PyDoc_STRVAR(_pickle_Pickler___sizeof____doc__, "__sizeof__($self, /)\n" @@ -242,15 +277,19 @@ PyDoc_STRVAR(_pickle_Unpickler_load__doc__, "specified therein."); #define _PICKLE_UNPICKLER_LOAD_METHODDEF \ - {"load", (PyCFunction)_pickle_Unpickler_load, METH_NOARGS, _pickle_Unpickler_load__doc__}, + {"load", _PyCFunction_CAST(_pickle_Unpickler_load), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _pickle_Unpickler_load__doc__}, static PyObject * -_pickle_Unpickler_load_impl(UnpicklerObject *self); +_pickle_Unpickler_load_impl(UnpicklerObject *self, PyTypeObject *cls); static PyObject * -_pickle_Unpickler_load(UnpicklerObject *self, PyObject *Py_UNUSED(ignored)) +_pickle_Unpickler_load(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _pickle_Unpickler_load_impl(self); + if (nargs) { + PyErr_SetString(PyExc_TypeError, "load() takes no arguments"); + return NULL; + } + return _pickle_Unpickler_load_impl(self, cls); } PyDoc_STRVAR(_pickle_Unpickler_find_class__doc__, @@ -267,26 +306,41 @@ PyDoc_STRVAR(_pickle_Unpickler_find_class__doc__, "needed. Both arguments passed are str objects."); #define _PICKLE_UNPICKLER_FIND_CLASS_METHODDEF \ - {"find_class", _PyCFunction_CAST(_pickle_Unpickler_find_class), METH_FASTCALL, _pickle_Unpickler_find_class__doc__}, + {"find_class", _PyCFunction_CAST(_pickle_Unpickler_find_class), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _pickle_Unpickler_find_class__doc__}, static PyObject * -_pickle_Unpickler_find_class_impl(UnpicklerObject *self, +_pickle_Unpickler_find_class_impl(UnpicklerObject *self, PyTypeObject *cls, PyObject *module_name, PyObject *global_name); static PyObject * -_pickle_Unpickler_find_class(UnpicklerObject *self, PyObject *const *args, Py_ssize_t nargs) +_pickle_Unpickler_find_class(UnpicklerObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", "", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "find_class", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; PyObject *module_name; PyObject *global_name; - if (!_PyArg_CheckPositional("find_class", nargs, 2, 2)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { goto exit; } module_name = args[0]; global_name = args[1]; - return_value = _pickle_Unpickler_find_class_impl(self, module_name, global_name); + return_value = _pickle_Unpickler_find_class_impl(self, cls, module_name, global_name); exit: return return_value; @@ -980,4 +1034,4 @@ _pickle_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=730dc26938561313 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a0e04b85e7bae626 input=a9049054013a1b77]*/ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index e0e45265209fff..0620c7e13925b5 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -390,11 +390,6 @@ Modules/_decimal/_decimal.c - PyDecContextManager_Type - Modules/_decimal/_decimal.c - PyDecContext_Type - Modules/_decimal/_decimal.c - PyDecSignalDictMixin_Type - Modules/_decimal/_decimal.c - PyDec_Type - -Modules/_pickle.c - Pdata_Type - -Modules/_pickle.c - PicklerMemoProxyType - -Modules/_pickle.c - Pickler_Type - -Modules/_pickle.c - UnpicklerMemoProxyType - -Modules/_pickle.c - Unpickler_Type - Modules/ossaudiodev.c - OSSAudioType - Modules/ossaudiodev.c - OSSMixerType - Modules/socketmodule.c - sock_type - From 89e6a3446184925ee7f17cd0d948c7784a88b8d7 Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Tue, 4 Apr 2023 16:51:30 +0200 Subject: [PATCH 07/76] Fix a compiler warning in _xxsubinterpretermodule.c (#103245) Fix a (correct) warning about potential uses of uninitialized memory in _xxsubinterpreter. Unlike newly allocated PyObject structs or global structs, stack-allocated structs are not initialised, and a few places in the code expect the _sharedexception struct data to be either NULL or initialised. --- Modules/_xxsubinterpretersmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 9648f080cd756c..11164676c4d107 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -481,7 +481,7 @@ _run_script_in_interpreter(PyObject *mod, PyInterpreterState *interp, } // Run the script. - _sharedexception exc; + _sharedexception exc = {NULL, NULL}; int result = _run_script(interp, codestr, shared, &exc); // Switch back. From b861ba4a8247af8159df1e01d33157cf57705067 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Tue, 4 Apr 2023 17:51:44 +0200 Subject: [PATCH 08/76] Python 3.12.0a7 --- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 146 ++-- Misc/NEWS.d/3.12.0a7.rst | 745 ++++++++++++++++++ ...-03-15-02-03-39.gh-issue-102711.zTkjts.rst | 1 - ...-03-23-20-58-56.gh-issue-102973.EaJUrw.rst | 2 - ...-02-18-00-55-14.gh-issue-102013.83mrtI.rst | 1 - ...2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst | 1 - ...-02-21-17-22-06.gh-issue-101865.fwrTOA.rst | 2 - ...-02-21-23-42-39.gh-issue-102027.fQARG0.rst | 2 - ...-02-26-11-43-56.gh-issue-102255.cRnI5x.rst | 1 - ...-02-26-13-12-55.gh-issue-102213.fTH8X7.rst | 1 - ...-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst | 1 - ...-03-02-13-49-21.gh-issue-102281.QCuu2N.rst | 1 - ...-03-03-23-21-16.gh-issue-102406.XLqYO3.rst | 1 - ...-03-04-06-48-34.gh-issue-102397.ACJaOf.rst | 2 - ...-03-06-10-02-22.gh-issue-101291.0FT2QS.rst | 7 - ...-03-08-08-37-36.gh-issue-102491.SFvvsC.rst | 2 - ...3-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst | 2 - ...-03-14-00-11-46.gh-issue-102594.BjU-m2.rst | 1 - ...-03-16-14-44-29.gh-issue-102755.j1GxlV.rst | 3 - ...-03-16-17-24-44.gh-issue-102701.iNGVaS.rst | 1 - ...-03-17-12-09-45.gh-issue-100982.Pf_BI6.rst | 1 - ...-03-17-13-43-34.gh-issue-102778.ANDv8I.rst | 3 - ...-03-18-02-36-39.gh-issue-101975.HwMR1d.rst | 1 - ...-03-21-00-46-36.gh-issue-102859.PRkGca.rst | 2 - ...3-03-24-02-50-33.gh-issue-89987.oraTzh.rst | 2 - ...-03-31-12-22-25.gh-issue-102192.gYxJP_.rst | 2 - ...-03-29-14-51-39.gh-issue-103112.XgGSEO.rst | 1 - .../2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst | 2 - ...2-04-11-18-34-33.gh-issue-72346.pC7gnM.rst | 1 - ...2-06-30-21-28-41.gh-issue-94440.LtgX0d.rst | 2 - ...2-07-09-13-07-30.gh-issue-94684.nV5yno.rst | 1 - ...2-07-30-23-01-43.gh-issue-95495.RA-q1d.rst | 7 - ...2-09-19-08-12-58.gh-issue-96931.x0WQhh.rst | 1 - ...2-10-10-19-14-51.gh-issue-98169.DBWIxL.rst | 2 - ...2-11-24-13-23-07.gh-issue-48330.6uAX9F.rst | 8 - ...-12-09-11-21-38.gh-issue-100131.v863yR.rst | 1 - ...2-12-16-10-27-58.gh-issue-89727.y64ZLM.rst | 1 - ...-12-20-10-55-14.gh-issue-100372.utfP65.rst | 2 - ...-01-27-14-51-07.gh-issue-101313.10AEXh.rst | 1 - ...-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst | 1 - ...3-02-18-23-03-50.gh-issue-98886.LkKGWv.rst | 1 - ...-02-19-01-49-46.gh-issue-102038.n3if3D.rst | 1 - ...-02-20-16-47-56.gh-issue-102069.FS7f1j.rst | 1 - ...-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst | 2 - ...3-02-26-17-29-57.gh-issue-79940.SAfmAy.rst | 2 - ...-03-03-19-53-08.gh-issue-102378.kRdOZc.rst | 1 - ...3-03-04-20-58-29.gh-issue-74468.Ac5Ew_.rst | 3 - ...-03-08-23-08-38.gh-issue-102519.wlcsFI.rst | 2 - ...-03-10-13-21-16.gh-issue-102578.-gujoI.rst | 4 - ...-03-10-13-51-21.gh-issue-100112.VHh4mw.rst | 1 - ...-03-13-12-05-55.gh-issue-102615.NcA_ZL.rst | 3 - ...-03-13-18-27-00.gh-issue-102670.GyoThv.rst | 2 - ...-03-16-08-17-29.gh-issue-102748.WNACpI.rst | 3 - ...3-03-16-16-43-04.gh-issue-78530.Lr8eq_.rst | 1 - ...3-03-17-19-14-26.gh-issue-76846.KEamjK.rst | 3 - ...3-03-18-14-59-21.gh-issue-88965.kA70Km.rst | 7 - ...-03-19-15-30-59.gh-issue-102828.NKClXg.rst | 3 - ...-03-20-12-21-19.gh-issue-102839.RjRi12.rst | 1 - ...-03-21-15-17-07.gh-issue-102871.U9mchn.rst | 3 - ...-03-22-16-15-18.gh-issue-102780.NEcljy.rst | 3 - ...-03-23-13-34-33.gh-issue-102947.cTwcpU.rst | 2 - ...-03-25-02-08-05.gh-issue-103023.Qfn7Hl.rst | 2 - ...-03-25-16-57-18.gh-issue-102433.L-7x2Q.rst | 10 - ...-03-26-20-54-57.gh-issue-103046.xBlA2l.rst | 1 - ...-03-27-15-01-16.gh-issue-103056.-Efh5Q.rst | 1 - ...-03-27-19-21-51.gh-issue-102549.NQ6Nlv.rst | 1 - ...-03-28-05-14-59.gh-issue-103068.YQTmrA.rst | 2 - ...-03-28-15-12-53.gh-issue-103085.DqNehf.rst | 1 - ...-01-27-18-10-40.gh-issue-101377.IJGpqh.rst | 1 - ...-03-08-13-54-20.gh-issue-102537.Vfplpb.rst | 2 - ...-03-23-23-25-18.gh-issue-102980.Zps4QF.rst | 1 - ...-03-21-01-27-07.gh-issue-102809.2F1Byz.rst | 1 - ...3-02-22-17-26-10.gh-issue-99726.76t957.rst | 2 - ...-03-14-10-52-43.gh-issue-102690.sbXtqk.rst | 1 - README.rst | 2 +- 76 files changed, 822 insertions(+), 224 deletions(-) create mode 100644 Misc/NEWS.d/3.12.0a7.rst delete mode 100644 Misc/NEWS.d/next/Build/2023-03-15-02-03-39.gh-issue-102711.zTkjts.rst delete mode 100644 Misc/NEWS.d/next/Build/2023-03-23-20-58-56.gh-issue-102973.EaJUrw.rst delete mode 100644 Misc/NEWS.d/next/C API/2023-02-18-00-55-14.gh-issue-102013.83mrtI.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-21-17-22-06.gh-issue-101865.fwrTOA.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-21-23-42-39.gh-issue-102027.fQARG0.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-02-13-49-21.gh-issue-102281.QCuu2N.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-03-23-21-16.gh-issue-102406.XLqYO3.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-06-10-02-22.gh-issue-101291.0FT2QS.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-08-08-37-36.gh-issue-102491.SFvvsC.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-16-14-44-29.gh-issue-102755.j1GxlV.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-16-17-24-44.gh-issue-102701.iNGVaS.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-17-12-09-45.gh-issue-100982.Pf_BI6.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-17-13-43-34.gh-issue-102778.ANDv8I.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-18-02-36-39.gh-issue-101975.HwMR1d.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-21-00-46-36.gh-issue-102859.PRkGca.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-24-02-50-33.gh-issue-89987.oraTzh.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-31-12-22-25.gh-issue-102192.gYxJP_.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2023-03-29-14-51-39.gh-issue-103112.XgGSEO.rst delete mode 100644 Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-04-11-18-34-33.gh-issue-72346.pC7gnM.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-07-09-13-07-30.gh-issue-94684.nV5yno.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-07-30-23-01-43.gh-issue-95495.RA-q1d.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-09-19-08-12-58.gh-issue-96931.x0WQhh.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-10-10-19-14-51.gh-issue-98169.DBWIxL.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-11-24-13-23-07.gh-issue-48330.6uAX9F.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-12-09-11-21-38.gh-issue-100131.v863yR.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-12-16-10-27-58.gh-issue-89727.y64ZLM.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-01-27-14-51-07.gh-issue-101313.10AEXh.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-18-23-03-50.gh-issue-98886.LkKGWv.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-19-01-49-46.gh-issue-102038.n3if3D.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-04-20-58-29.gh-issue-74468.Ac5Ew_.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-13-12-05-55.gh-issue-102615.NcA_ZL.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-13-18-27-00.gh-issue-102670.GyoThv.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-16-08-17-29.gh-issue-102748.WNACpI.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-16-16-43-04.gh-issue-78530.Lr8eq_.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-17-19-14-26.gh-issue-76846.KEamjK.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-18-14-59-21.gh-issue-88965.kA70Km.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-19-15-30-59.gh-issue-102828.NKClXg.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-20-12-21-19.gh-issue-102839.RjRi12.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-21-15-17-07.gh-issue-102871.U9mchn.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-22-16-15-18.gh-issue-102780.NEcljy.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-23-13-34-33.gh-issue-102947.cTwcpU.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-25-02-08-05.gh-issue-103023.Qfn7Hl.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-25-16-57-18.gh-issue-102433.L-7x2Q.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-26-20-54-57.gh-issue-103046.xBlA2l.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-27-15-01-16.gh-issue-103056.-Efh5Q.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-27-19-21-51.gh-issue-102549.NQ6Nlv.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-28-05-14-59.gh-issue-103068.YQTmrA.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-03-28-15-12-53.gh-issue-103085.DqNehf.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-01-27-18-10-40.gh-issue-101377.IJGpqh.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst delete mode 100644 Misc/NEWS.d/next/Tests/2023-03-23-23-25-18.gh-issue-102980.Zps4QF.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2023-03-21-01-27-07.gh-issue-102809.2F1Byz.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-02-22-17-26-10.gh-issue-99726.76t957.rst delete mode 100644 Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 049cdfa30897ca..1a8a6d9c8f7a3d 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 12 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 6 +#define PY_RELEASE_SERIAL 7 /* Version as a string */ -#define PY_VERSION "3.12.0a6+" +#define PY_VERSION "3.12.0a7" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index ad1b6aca6b95bc..1babb5ce9476c9 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Tue Mar 7 22:42:28 2023 +# Autogenerated by Sphinx on Tue Apr 4 17:52:21 2023 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -1134,10 +1134,11 @@ 'future, a\n' ' check may be added to prevent this.\n' '\n' - '* Nonempty *__slots__* does not work for classes derived ' - 'from\n' - ' “variable-length” built-in types such as "int", ' - '"bytes" and "tuple".\n' + '* "TypeError" will be raised if nonempty *__slots__* are ' + 'defined for a\n' + ' class derived from a ""variable-length" built-in type" ' + 'such as\n' + ' "int", "bytes", and "tuple".\n' '\n' '* Any non-string *iterable* may be assigned to ' '*__slots__*.\n' @@ -3072,7 +3073,7 @@ 'AS\n' 'pattern binds the subject to the name on the right of the as ' 'keyword\n' - 'and succeeds. "capture_pattern" cannot be a a "_".\n' + 'and succeeds. "capture_pattern" cannot be a "_".\n' '\n' 'In simple terms "P as NAME" will match with "P", and on success ' 'it\n' @@ -4675,7 +4676,7 @@ 'scripts. For\n' 'example:\n' '\n' - ' python3 -m pdb myscript.py\n' + ' python -m pdb myscript.py\n' '\n' 'When invoked as a script, pdb will automatically enter ' 'post-mortem\n' @@ -4695,7 +4696,7 @@ '\n' 'New in version 3.7: "pdb.py" now accepts a "-m" option that ' 'execute\n' - 'modules similar to the way "python3 -m" does. As with a script, ' + 'modules similar to the way "python -m" does. As with a script, ' 'the\n' 'debugger will pause execution just before the first line of the\n' 'module.\n' @@ -4759,8 +4760,8 @@ 'object)\n' ' under debugger control. When "runeval()" returns, it returns ' 'the\n' - ' value of the expression. Otherwise this function is similar ' - 'to\n' + ' value of the *expression*. Otherwise this function is ' + 'similar to\n' ' "run()".\n' '\n' 'pdb.runcall(function, *args, **kwds)\n' @@ -4799,7 +4800,7 @@ 'pdb.pm()\n' '\n' ' Enter post-mortem debugging of the traceback found in\n' - ' "sys.last_exc".\n' + ' "sys.last_traceback".\n' '\n' 'The "run*" functions and "set_trace()" are aliases for ' 'instantiating\n' @@ -5022,14 +5023,15 @@ 'ignore bpnumber [count]\n' '\n' ' Set the ignore count for the given breakpoint number. If ' - 'count is\n' - ' omitted, the ignore count is set to 0. A breakpoint becomes ' - 'active\n' - ' when the ignore count is zero. When non-zero, the count is\n' - ' decremented each time the breakpoint is reached and the ' - 'breakpoint\n' - ' is not disabled and any associated condition evaluates to ' - 'true.\n' + '*count*\n' + ' is omitted, the ignore count is set to 0. A breakpoint ' + 'becomes\n' + ' active when the ignore count is zero. When non-zero, the ' + '*count*\n' + ' is decremented each time the breakpoint is reached and the\n' + ' breakpoint is not disabled and any associated condition ' + 'evaluates\n' + ' to true.\n' '\n' 'condition bpnumber [condition]\n' '\n' @@ -5079,7 +5081,7 @@ ' breakpoint—which could have its own command list, leading to\n' ' ambiguities about which list to execute.\n' '\n' - ' If you use the ‘silent’ command in the command list, the ' + ' If you use the "silent" command in the command list, the ' 'usual\n' ' message about stopping at a breakpoint is not printed. This ' 'may be\n' @@ -5114,11 +5116,10 @@ 'number\n' ' greater than the current one is reached.\n' '\n' - ' With a line number, continue execution until a line with a ' - 'number\n' - ' greater or equal to that is reached. In both cases, also ' - 'stop when\n' - ' the current frame returns.\n' + ' With *lineno*, continue execution until a line with a number\n' + ' greater or equal to *lineno* is reached. In both cases, also ' + 'stop\n' + ' when the current frame returns.\n' '\n' ' Changed in version 3.2: Allow giving an explicit line ' 'number.\n' @@ -5182,9 +5183,8 @@ '\n' 'p expression\n' '\n' - ' Evaluate the *expression* in the current context and print ' - 'its\n' - ' value.\n' + ' Evaluate *expression* in the current context and print its ' + 'value.\n' '\n' ' Note:\n' '\n' @@ -5194,26 +5194,26 @@ '\n' 'pp expression\n' '\n' - ' Like the "p" command, except the value of the expression is ' + ' Like the "p" command, except the value of *expression* is ' 'pretty-\n' ' printed using the "pprint" module.\n' '\n' 'whatis expression\n' '\n' - ' Print the type of the *expression*.\n' + ' Print the type of *expression*.\n' '\n' 'source expression\n' '\n' - ' Try to get source code for the given object and display it.\n' + ' Try to get source code of *expression* and display it.\n' '\n' ' New in version 3.2.\n' '\n' 'display [expression]\n' '\n' - ' Display the value of the expression if it changed, each time\n' + ' Display the value of *expression* if it changed, each time\n' ' execution stops in the current frame.\n' '\n' - ' Without expression, list all display expressions for the ' + ' Without *expression*, list all display expressions for the ' 'current\n' ' frame.\n' '\n' @@ -5221,10 +5221,10 @@ '\n' 'undisplay [expression]\n' '\n' - ' Do not display the expression any more in the current frame.\n' - ' Without expression, clear all display expressions for the ' - 'current\n' - ' frame.\n' + ' Do not display *expression* anymore in the current frame. ' + 'Without\n' + ' *expression*, clear all display expressions for the current ' + 'frame.\n' '\n' ' New in version 3.2.\n' '\n' @@ -5240,16 +5240,16 @@ '\n' 'alias [name [command]]\n' '\n' - ' Create an alias called *name* that executes *command*. The ' - 'command\n' - ' must *not* be enclosed in quotes. Replaceable parameters can ' - 'be\n' - ' indicated by "%1", "%2", and so on, while "%*" is replaced by ' - 'all\n' - ' the parameters. If no command is given, the current alias ' - 'for\n' - ' *name* is shown. If no arguments are given, all aliases are ' - 'listed.\n' + ' Create an alias called *name* that executes *command*. The\n' + ' *command* must *not* be enclosed in quotes. Replaceable ' + 'parameters\n' + ' can be indicated by "%1", "%2", and so on, while "%*" is ' + 'replaced\n' + ' by all the parameters. If *command* is omitted, the current ' + 'alias\n' + ' for *name* is shown. If no arguments are given, all aliases ' + 'are\n' + ' listed.\n' '\n' ' Aliases may be nested and can contain anything that can be ' 'legally\n' @@ -5268,14 +5268,14 @@ ' in the ".pdbrc" file):\n' '\n' ' # Print instance variables (usage "pi classInst")\n' - ' alias pi for k in %1.__dict__.keys(): ' - 'print("%1.",k,"=",%1.__dict__[k])\n' + ' alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = ' + '{%1.__dict__[k]}")\n' ' # Print instance variables in self\n' ' alias ps pi self\n' '\n' 'unalias name\n' '\n' - ' Delete the specified alias.\n' + ' Delete the specified alias *name*.\n' '\n' '! statement\n' '\n' @@ -5295,12 +5295,13 @@ 'run [args ...]\n' 'restart [args ...]\n' '\n' - ' Restart the debugged Python program. If an argument is ' - 'supplied,\n' - ' it is split with "shlex" and the result is used as the new\n' - ' "sys.argv". History, breakpoints, actions and debugger ' - 'options are\n' - ' preserved. "restart" is an alias for "run".\n' + ' Restart the debugged Python program. If *args* is supplied, ' + 'it is\n' + ' split with "shlex" and the result is used as the new ' + '"sys.argv".\n' + ' History, breakpoints, actions and debugger options are ' + 'preserved.\n' + ' "restart" is an alias for "run".\n' '\n' 'q(uit)\n' '\n' @@ -5309,11 +5310,11 @@ '\n' 'debug code\n' '\n' - ' Enter a recursive debugger that steps through the code ' - 'argument\n' - ' (which is an arbitrary expression or statement to be executed ' - 'in\n' - ' the current environment).\n' + ' Enter a recursive debugger that steps through *code* (which ' + 'is an\n' + ' arbitrary expression or statement to be executed in the ' + 'current\n' + ' environment).\n' '\n' 'retval\n' '\n' @@ -6170,7 +6171,8 @@ 'The general form of a *standard format specifier* is:\n' '\n' ' format_spec ::= ' - '[[fill]align][sign][z][#][0][width][grouping_option][.precision][type]\n' + '[[fill]align][sign]["z"]["#"]["0"][width][grouping_option]["." ' + 'precision][type]\n' ' fill ::= \n' ' align ::= "<" | ">" | "=" | "^"\n' ' sign ::= "+" | "-" | " "\n' @@ -9981,10 +9983,11 @@ 'future, a\n' ' check may be added to prevent this.\n' '\n' - '* Nonempty *__slots__* does not work for classes derived ' - 'from\n' - ' “variable-length” built-in types such as "int", "bytes" ' - 'and "tuple".\n' + '* "TypeError" will be raised if nonempty *__slots__* are ' + 'defined for a\n' + ' class derived from a ""variable-length" built-in type" ' + 'such as\n' + ' "int", "bytes", and "tuple".\n' '\n' '* Any non-string *iterable* may be assigned to *__slots__*.\n' '\n' @@ -13691,11 +13694,10 @@ ' compiled; "co_firstlineno" is the first line number of the\n' ' function; "co_lnotab" is a string encoding the mapping from\n' ' bytecode offsets to line numbers (for details see the source\n' - ' code of the interpreter); "co_stacksize" is the required ' - 'stack\n' - ' size; "co_flags" is an integer encoding a number of flags ' - 'for\n' - ' the interpreter.\n' + ' code of the interpreter, is deprecated since 3.12 and may be\n' + ' removed in 3.14); "co_stacksize" is the required stack size;\n' + ' "co_flags" is an integer encoding a number of flags for the\n' + ' interpreter.\n' '\n' ' The following flag bits are defined for "co_flags": bit ' '"0x04"\n' @@ -13858,7 +13860,7 @@ 'if\n' ' the interpreter is interactive, it is also made available to ' 'the\n' - ' user as "sys.last_exc".\n' + ' user as "sys.last_traceback".\n' '\n' ' For explicitly created tracebacks, it is up to the creator ' 'of\n' diff --git a/Misc/NEWS.d/3.12.0a7.rst b/Misc/NEWS.d/3.12.0a7.rst new file mode 100644 index 00000000000000..8f078e50823a00 --- /dev/null +++ b/Misc/NEWS.d/3.12.0a7.rst @@ -0,0 +1,745 @@ +.. date: 2023-03-31-12-22-25 +.. gh-issue: 102192 +.. nonce: gYxJP_ +.. release date: 2023-04-04 +.. section: Core and Builtins + +Deprecated ``_PyErr_ChainExceptions`` in favour of +``_PyErr_ChainExceptions1``. + +.. + +.. date: 2023-03-24-02-50-33 +.. gh-issue: 89987 +.. nonce: oraTzh +.. section: Core and Builtins + +Reduce the number of inline :opcode:`CACHE` entries for +:opcode:`BINARY_SUBSCR`. + +.. + +.. date: 2023-03-21-00-46-36 +.. gh-issue: 102859 +.. nonce: PRkGca +.. section: Core and Builtins + +Removed :opcode:`JUMP_IF_FALSE_OR_POP` and :opcode:`JUMP_IF_TRUE_OR_POP` +instructions. + +.. + +.. date: 2023-03-18-02-36-39 +.. gh-issue: 101975 +.. nonce: HwMR1d +.. section: Core and Builtins + +Fixed ``stacktop`` value on tracing entries to avoid corruption on garbage +collection. + +.. + +.. date: 2023-03-17-13-43-34 +.. gh-issue: 102778 +.. nonce: ANDv8I +.. section: Core and Builtins + +Add :data:`sys.last_exc` and deprecate :data:`sys.last_type`, +:data:`sys.last_value` and :data:`sys.last_traceback`, which hold the same +information in its legacy form. + +.. + +.. date: 2023-03-17-12-09-45 +.. gh-issue: 100982 +.. nonce: Pf_BI6 +.. section: Core and Builtins + +Replace all occurrences of ``COMPARE_AND_BRANCH`` with :opcode:`COMPARE_OP`. + +.. + +.. date: 2023-03-16-17-24-44 +.. gh-issue: 102701 +.. nonce: iNGVaS +.. section: Core and Builtins + +Fix overflow when creating very large dict. + +.. + +.. date: 2023-03-16-14-44-29 +.. gh-issue: 102755 +.. nonce: j1GxlV +.. section: Core and Builtins + +Add :c:func:`PyErr_DisplayException` which takes just an exception instance, +to replace the legacy :c:func:`PyErr_Display` which takes the ``(typ, exc, +tb)`` triplet. + +.. + +.. date: 2023-03-14-00-11-46 +.. gh-issue: 102594 +.. nonce: BjU-m2 +.. section: Core and Builtins + +Add note to exception raised in ``PyErr_SetObject`` when normalization +fails. + +.. + +.. date: 2023-03-09-13-57-35 +.. gh-issue: 90997 +.. nonce: J-Yhn2 +.. section: Core and Builtins + +Shrink the number of inline :opcode:`CACHE` entries used by +:opcode:`LOAD_GLOBAL`. + +.. + +.. date: 2023-03-08-08-37-36 +.. gh-issue: 102491 +.. nonce: SFvvsC +.. section: Core and Builtins + +Improve import time of ``platform`` by removing IronPython version parsing. +The IronPython version parsing was not functional (see +https://github.com/IronLanguages/ironpython3/issues/1667). + +.. + +.. date: 2023-03-06-10-02-22 +.. gh-issue: 101291 +.. nonce: 0FT2QS +.. section: Core and Builtins + +Rearrage bits in first field (after header) of PyLongObject. * Bits 0 and 1: +1 - sign. I.e. 0 for positive numbers, 1 for zero and 2 for negative +numbers. * Bit 2 reserved (probably for the immortal bit) * Bits 3+ the +unsigned size. + +This makes a few operations slightly more efficient, and will enable a more +compact and faster 2s-complement representation of most ints in future. + +.. + +.. date: 2023-03-04-06-48-34 +.. gh-issue: 102397 +.. nonce: ACJaOf +.. section: Core and Builtins + +Fix segfault from race condition in signal handling during garbage +collection. Patch by Kumar Aditya. + +.. + +.. date: 2023-03-03-23-21-16 +.. gh-issue: 102406 +.. nonce: XLqYO3 +.. section: Core and Builtins + +:mod:`codecs` encoding/decoding errors now get the context information +(which operation and which codecs) attached as :pep:`678` notes instead of +through chaining a new instance of the exception. + +.. + +.. date: 2023-03-02-13-49-21 +.. gh-issue: 102281 +.. nonce: QCuu2N +.. section: Core and Builtins + +Fix potential nullptr dereference and use of uninitialized memory in +fileutils. Patch by Max Bachmann. + +.. + +.. date: 2023-02-27-15-48-31 +.. gh-issue: 102300 +.. nonce: 8o-_Mt +.. section: Core and Builtins + +Reuse operands with refcount of 1 in float specializations of BINARY_OP. + +.. + +.. date: 2023-02-26-13-12-55 +.. gh-issue: 102213 +.. nonce: fTH8X7 +.. section: Core and Builtins + +Fix performance loss when accessing an object's attributes with +``__getattr__`` defined. + +.. + +.. date: 2023-02-26-11-43-56 +.. gh-issue: 102255 +.. nonce: cRnI5x +.. section: Core and Builtins + +Improve build support for the Xbox. Patch by Max Bachmann. + +.. + +.. date: 2023-02-21-23-42-39 +.. gh-issue: 102027 +.. nonce: fQARG0 +.. section: Core and Builtins + +Fix SSE2 and SSE3 detection in ``_blake2`` internal module. Patch by Max +Bachmann. + +.. + +.. date: 2023-02-21-17-22-06 +.. gh-issue: 101865 +.. nonce: fwrTOA +.. section: Core and Builtins + +Deprecate ``co_lnotab`` in code objects, schedule it for removal in Python +3.14 + +.. + +.. bpo: 1635741 +.. date: 2020-07-04-09-04-41 +.. nonce: ZsP31Y +.. section: Core and Builtins + +Adapt :mod:`!_pickle` to :pep:`687`. Patch by Mohamed Koubaa and Erlend +Aasland. + +.. + +.. date: 2023-03-28-15-12-53 +.. gh-issue: 103085 +.. nonce: DqNehf +.. section: Library + +Pure python :func:`locale.getencoding()` will not warn deprecation. + +.. + +.. date: 2023-03-28-05-14-59 +.. gh-issue: 103068 +.. nonce: YQTmrA +.. section: Library + +It's no longer possible to register conditional breakpoints in +:class:`~pdb.Pdb` that raise :exc:`SyntaxError`. Patch by Tian Gao. + +.. + +.. date: 2023-03-27-19-21-51 +.. gh-issue: 102549 +.. nonce: NQ6Nlv +.. section: Library + +Don't ignore exceptions in member type creation. + +.. + +.. date: 2023-03-27-15-01-16 +.. gh-issue: 103056 +.. nonce: -Efh5Q +.. section: Library + +Ensure final ``_generate_next_value_`` is a ``staticmethod``. + +.. + +.. date: 2023-03-26-20-54-57 +.. gh-issue: 103046 +.. nonce: xBlA2l +.. section: Library + +Display current line label correctly in :mod:`dis` when ``show_caches`` is +False and ``lasti`` points to a CACHE entry. + +.. + +.. date: 2023-03-25-16-57-18 +.. gh-issue: 102433 +.. nonce: L-7x2Q +.. section: Library + +:func:`isinstance` checks against :func:`runtime-checkable protocols +` now use :func:`inspect.getattr_static` rather +than :func:`hasattr` to lookup whether attributes exist. This means that +descriptors and :meth:`~object.__getattr__` methods are no longer +unexpectedly evaluated during ``isinstance()`` checks against +runtime-checkable protocols. However, it may also mean that some objects +which used to be considered instances of a runtime-checkable protocol may no +longer be considered instances of that protocol on Python 3.12+, and vice +versa. Most users are unlikely to be affected by this change. Patch by Alex +Waygood. + +.. + +.. date: 2023-03-25-02-08-05 +.. gh-issue: 103023 +.. nonce: Qfn7Hl +.. section: Library + +It's no longer possible to register expressions to display in +:class:`~pdb.Pdb` that raise :exc:`SyntaxError`. Patch by Tian Gao. + +.. + +.. date: 2023-03-23-13-34-33 +.. gh-issue: 102947 +.. nonce: cTwcpU +.. section: Library + +Improve traceback when :func:`dataclasses.fields` is called on a +non-dataclass. Patch by Alex Waygood + +.. + +.. date: 2023-03-22-16-15-18 +.. gh-issue: 102780 +.. nonce: NEcljy +.. section: Library + +The :class:`asyncio.Timeout` context manager now works reliably even when +performing cleanup due to task cancellation. Previously it could raise a +:exc:`~asyncio.CancelledError` instead of an :exc:`~asyncio.TimeoutError` in +such cases. + +.. + +.. date: 2023-03-21-15-17-07 +.. gh-issue: 102871 +.. nonce: U9mchn +.. section: Library + +Remove support for obsolete browsers from :mod:`webbrowser`. Removed +browsers include Grail, Mosaic, Netscape, Galeon, Skipstone, Iceape, +Firebird, and Firefox versions 35 and below. + +.. + +.. date: 2023-03-20-12-21-19 +.. gh-issue: 102839 +.. nonce: RjRi12 +.. section: Library + +Improve performance of :func:`math.log` arguments handling by removing the +argument clinic. + +.. + +.. date: 2023-03-19-15-30-59 +.. gh-issue: 102828 +.. nonce: NKClXg +.. section: Library + +Add the ``onexc`` arg to :func:`shutil.rmtree`, which is like ``onerror`` +but expects an exception instance rather than an exc_info tuple. Deprecate +``onerror``. + +.. + +.. date: 2023-03-18-14-59-21 +.. gh-issue: 88965 +.. nonce: kA70Km +.. section: Library + +typing: Fix a bug relating to substitution in custom classes generic over a +:class:`~typing.ParamSpec`. Previously, if the ``ParamSpec`` was substituted +with a parameters list that itself contained a :class:`~typing.TypeVar`, the +``TypeVar`` in the parameters list could not be subsequently substituted. +This is now fixed. + +Patch by Nikita Sobolev. + +.. + +.. date: 2023-03-17-19-14-26 +.. gh-issue: 76846 +.. nonce: KEamjK +.. section: Library + +Fix issue where ``__new__()`` and ``__init__()`` methods of +:class:`pathlib.PurePath` and :class:`~pathlib.Path` subclasses were not +called in some circumstances. + +.. + +.. date: 2023-03-16-16-43-04 +.. gh-issue: 78530 +.. nonce: Lr8eq_ +.. section: Library + +:func:`asyncio.wait` now accepts generators yielding tasks. Patch by Kumar +Aditya. + +.. + +.. date: 2023-03-16-08-17-29 +.. gh-issue: 102748 +.. nonce: WNACpI +.. section: Library + +:func:`asyncio.iscoroutine` now returns ``False`` for generators as +:mod:`asyncio` does not support legacy generator-based coroutines. Patch by +Kumar Aditya. + +.. + +.. date: 2023-03-13-18-27-00 +.. gh-issue: 102670 +.. nonce: GyoThv +.. section: Library + +Optimized fmean(), correlation(), covariance(), and linear_regression() +using the new math.sumprod() function. + +.. + +.. date: 2023-03-13-12-05-55 +.. gh-issue: 102615 +.. nonce: NcA_ZL +.. section: Library + +Typing: Improve the ``repr`` of generic aliases for classes generic over a +:class:`~typing.ParamSpec`. (Use square brackets to represent a parameter +list.) + +.. + +.. date: 2023-03-10-13-51-21 +.. gh-issue: 100112 +.. nonce: VHh4mw +.. section: Library + +:meth:`asyncio.Task.get_coro` now always returns a coroutine when wrapping +an awaitable object. Patch by Kumar Aditya. + +.. + +.. date: 2023-03-10-13-21-16 +.. gh-issue: 102578 +.. nonce: -gujoI +.. section: Library + +Speed up setting or deleting mutable attributes on non-dataclass subclasses +of frozen dataclasses. Due to the implementation of ``__setattr__`` and +``__delattr__`` for frozen dataclasses, this previously had a time +complexity of ``O(n)``. It now has a time complexity of ``O(1)``. + +.. + +.. date: 2023-03-08-23-08-38 +.. gh-issue: 102519 +.. nonce: wlcsFI +.. section: Library + +Add :func:`os.listdrives`, :func:`os.listvolumes` and :func:`os.listmounts` +functions on Windows for enumerating drives, volumes and mount points + +.. + +.. date: 2023-03-04-20-58-29 +.. gh-issue: 74468 +.. nonce: Ac5Ew_ +.. section: Library + +Attribute name of the extracted :mod:`tarfile` file object now holds +filename of itself rather than of the archive it is contained in. Patch by +Oleg Iarygin. + +.. + +.. date: 2023-03-03-19-53-08 +.. gh-issue: 102378 +.. nonce: kRdOZc +.. section: Library + +Private helper method ``inspect._signature_strip_non_python_syntax`` will no +longer strip ``/`` from the input string. + +.. + +.. date: 2023-02-26-17-29-57 +.. gh-issue: 79940 +.. nonce: SAfmAy +.. section: Library + +Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals`. +Patch by Thomas Krennwallner. + +.. + +.. date: 2023-02-21-11-56-16 +.. gh-issue: 102103 +.. nonce: Dj0WEj +.. section: Library + +Add ``module`` argument to :func:`dataclasses.make_dataclass` and make +classes produced by it pickleable. + +.. + +.. date: 2023-02-20-16-47-56 +.. gh-issue: 102069 +.. nonce: FS7f1j +.. section: Library + +Fix ``__weakref__`` descriptor generation for custom dataclasses. + +.. + +.. date: 2023-02-19-01-49-46 +.. gh-issue: 102038 +.. nonce: n3if3D +.. section: Library + +Skip a ``stat`` in :mod:`site` if we have already found a ``pyvenv.cfg`` + +.. + +.. date: 2023-02-18-23-03-50 +.. gh-issue: 98886 +.. nonce: LkKGWv +.. section: Library + +Fix issues when defining dataclasses that have fields with specific +underscore names that aren't clearly reserved by :mod:`dataclasses`. + +.. + +.. date: 2023-02-09-19-40-41 +.. gh-issue: 101673 +.. nonce: mX-Ppq +.. section: Library + +Fix a :mod:`pdb` bug where ``ll`` clears the changes to local variables. + +.. + +.. date: 2023-01-27-14-51-07 +.. gh-issue: 101313 +.. nonce: 10AEXh +.. section: Library + +Added -h and --help arguments to the webbrowser CLI + +.. + +.. date: 2022-12-20-10-55-14 +.. gh-issue: 100372 +.. nonce: utfP65 +.. section: Library + +:meth:`ssl.SSLContext.load_verify_locations` no longer incorrectly accepts +some cases of trailing data when parsing DER. + +.. + +.. date: 2022-12-16-10-27-58 +.. gh-issue: 89727 +.. nonce: y64ZLM +.. section: Library + +Fix pathlib.Path.walk RecursionError on deep directory trees by rewriting it +using iteration instead of recursion. + +.. + +.. date: 2022-12-09-11-21-38 +.. gh-issue: 100131 +.. nonce: v863yR +.. section: Library + +Added an optional ``delete`` keyword argument to +:class:`tempfile.TemporaryDirectory`. + +.. + +.. date: 2022-11-24-13-23-07 +.. gh-issue: 48330 +.. nonce: 6uAX9F +.. section: Library + +Added ``--durations`` command line option, showing the N slowest test cases. +:class:`unittest.TextTestRunner` and :class:`unittest.TextTestResult` +constructors accept a new *durations* keyword argument. Subclasses should +take this into account or accept ``**kwargs``. Added +:meth:`unittest.TestResult.addDuration` method and +:attr:`unittest.TestResult.collectedDurations` attribute. + +(Contributed by Giampaolo Rodola) + +.. + +.. date: 2022-10-10-19-14-51 +.. gh-issue: 98169 +.. nonce: DBWIxL +.. section: Library + +Fix :func:`dataclasses.astuple` crash when :class:`collections.defaultdict` +is present in the attributes. + +.. + +.. date: 2022-09-19-08-12-58 +.. gh-issue: 96931 +.. nonce: x0WQhh +.. section: Library + +Fix incorrect results from :meth:`ssl.SSLSocket.shared_ciphers` + +.. + +.. date: 2022-07-30-23-01-43 +.. gh-issue: 95495 +.. nonce: RA-q1d +.. section: Library + +When built against OpenSSL 3.0, the :mod:`ssl` module had a bug where it +reported unauthenticated EOFs (i.e. without close_notify) as a clean +TLS-level EOF. It now raises :exc:`~ssl.SSLEOFError`, matching the behavior +in previous versions of OpenSSL. The :attr:`~ssl.SSLContext.options` +attribute on :class:`~ssl.SSLContext` also no longer includes +:data:`~ssl.OP_IGNORE_UNEXPECTED_EOF` by default. This option may be set to +specify the previous OpenSSL 3.0 behavior. + +.. + +.. date: 2022-07-09-13-07-30 +.. gh-issue: 94684 +.. nonce: nV5yno +.. section: Library + +Now :func:`uuid.uuid3` and :func:`uuid.uuid5` functions support +:class:`bytes` objects as their *name* argument. + +.. + +.. date: 2022-06-30-21-28-41 +.. gh-issue: 94440 +.. nonce: LtgX0d +.. section: Library + +Fix a :mod:`concurrent.futures.process` bug where ``ProcessPoolExecutor`` +shutdown could hang after a future has been quickly submitted and canceled. + +.. + +.. date: 2022-04-11-18-34-33 +.. gh-issue: 72346 +.. nonce: pC7gnM +.. section: Library + +Added deprecation warning to *isdst* parameter of +:func:`email.utils.localtime`. + +.. + +.. bpo: 36305 +.. date: 2019-03-15-22-50-27 +.. nonce: Pbkv6u +.. section: Library + +Fix handling of Windows filenames that resemble drives, such as ``./a:b``, +in :mod:`pathlib`. + +.. + +.. date: 2023-03-29-14-51-39 +.. gh-issue: 103112 +.. nonce: XgGSEO +.. section: Documentation + +Add docstring to :meth:`http.client.HTTPResponse.read` to fix ``pydoc`` +output. + +.. + +.. date: 2023-03-23-23-25-18 +.. gh-issue: 102980 +.. nonce: Zps4QF +.. section: Tests + +Improve test coverage on :mod:`pdb`. + +.. + +.. date: 2023-03-08-13-54-20 +.. gh-issue: 102537 +.. nonce: Vfplpb +.. section: Tests + +Adjust the error handling strategy in +``test_zoneinfo.TzPathTest.python_tzpath_context``. Patch by Paul Ganssle. + +.. + +.. date: 2023-01-27-18-10-40 +.. gh-issue: 101377 +.. nonce: IJGpqh +.. section: Tests + +Improved test_locale_calendar_formatweekday of calendar. + +.. + +.. date: 2023-03-23-20-58-56 +.. gh-issue: 102973 +.. nonce: EaJUrw +.. section: Build + +Add a dev container (along with accompanying Dockerfile) for development +purposes. + +.. + +.. date: 2023-03-15-02-03-39 +.. gh-issue: 102711 +.. nonce: zTkjts +.. section: Build + +Fix ``-Wstrict-prototypes`` compiler warnings. + +.. + +.. date: 2023-03-14-10-52-43 +.. gh-issue: 102690 +.. nonce: sbXtqk +.. section: Windows + +Update :mod:`webbrowser` to fall back to Microsoft Edge instead of Internet +Explorer. + +.. + +.. date: 2023-02-22-17-26-10 +.. gh-issue: 99726 +.. nonce: 76t957 +.. section: Windows + +Improves correctness of stat results for Windows, and uses faster API when +available + +.. + +.. date: 2023-03-21-01-27-07 +.. gh-issue: 102809 +.. nonce: 2F1Byz +.. section: Tools/Demos + +``Misc/gdbinit`` was removed. + +.. + +.. date: 2023-02-18-00-55-14 +.. gh-issue: 102013 +.. nonce: 83mrtI +.. section: C API + +Add a new (unstable) C-API function for iterating over GC'able objects using +a callback: ``PyUnstable_VisitObjects``. diff --git a/Misc/NEWS.d/next/Build/2023-03-15-02-03-39.gh-issue-102711.zTkjts.rst b/Misc/NEWS.d/next/Build/2023-03-15-02-03-39.gh-issue-102711.zTkjts.rst deleted file mode 100644 index 51184396877721..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-03-15-02-03-39.gh-issue-102711.zTkjts.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``-Wstrict-prototypes`` compiler warnings. diff --git a/Misc/NEWS.d/next/Build/2023-03-23-20-58-56.gh-issue-102973.EaJUrw.rst b/Misc/NEWS.d/next/Build/2023-03-23-20-58-56.gh-issue-102973.EaJUrw.rst deleted file mode 100644 index 38b02391266ffb..00000000000000 --- a/Misc/NEWS.d/next/Build/2023-03-23-20-58-56.gh-issue-102973.EaJUrw.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add a dev container (along with accompanying Dockerfile) for development -purposes. diff --git a/Misc/NEWS.d/next/C API/2023-02-18-00-55-14.gh-issue-102013.83mrtI.rst b/Misc/NEWS.d/next/C API/2023-02-18-00-55-14.gh-issue-102013.83mrtI.rst deleted file mode 100644 index 0350237ebc7390..00000000000000 --- a/Misc/NEWS.d/next/C API/2023-02-18-00-55-14.gh-issue-102013.83mrtI.rst +++ /dev/null @@ -1 +0,0 @@ -Add a new (unstable) C-API function for iterating over GC'able objects using a callback: ``PyUnstable_VisitObjects``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst b/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst deleted file mode 100644 index 9f9c8121bea5ae..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst +++ /dev/null @@ -1 +0,0 @@ -Adapt :mod:`!_pickle` to :pep:`687`. Patch by Mohamed Koubaa and Erlend Aasland. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-21-17-22-06.gh-issue-101865.fwrTOA.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-21-17-22-06.gh-issue-101865.fwrTOA.rst deleted file mode 100644 index 876cc223a0e727..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-21-17-22-06.gh-issue-101865.fwrTOA.rst +++ /dev/null @@ -1,2 +0,0 @@ -Deprecate ``co_lnotab`` in code objects, schedule it for removal in Python -3.14 diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-21-23-42-39.gh-issue-102027.fQARG0.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-21-23-42-39.gh-issue-102027.fQARG0.rst deleted file mode 100644 index 42d96b54677e41..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-21-23-42-39.gh-issue-102027.fQARG0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix SSE2 and SSE3 detection in ``_blake2`` internal module. Patch by Max -Bachmann. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst deleted file mode 100644 index daabc3c15f6ee2..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-11-43-56.gh-issue-102255.cRnI5x.rst +++ /dev/null @@ -1 +0,0 @@ -Improve build support for the Xbox. Patch by Max Bachmann. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst deleted file mode 100644 index 997bef226e713f..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-26-13-12-55.gh-issue-102213.fTH8X7.rst +++ /dev/null @@ -1 +0,0 @@ -Fix performance loss when accessing an object's attributes with ``__getattr__`` defined. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst b/Misc/NEWS.d/next/Core and Builtins/2023-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst deleted file mode 100644 index 4227014582b7e6..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-02-27-15-48-31.gh-issue-102300.8o-_Mt.rst +++ /dev/null @@ -1 +0,0 @@ -Reuse operands with refcount of 1 in float specializations of BINARY_OP. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-02-13-49-21.gh-issue-102281.QCuu2N.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-02-13-49-21.gh-issue-102281.QCuu2N.rst deleted file mode 100644 index b0269dd3d92bd5..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-02-13-49-21.gh-issue-102281.QCuu2N.rst +++ /dev/null @@ -1 +0,0 @@ -Fix potential nullptr dereference and use of uninitialized memory in fileutils. Patch by Max Bachmann. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-03-23-21-16.gh-issue-102406.XLqYO3.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-03-23-21-16.gh-issue-102406.XLqYO3.rst deleted file mode 100644 index e0d061c37299f2..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-03-23-21-16.gh-issue-102406.XLqYO3.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`codecs` encoding/decoding errors now get the context information (which operation and which codecs) attached as :pep:`678` notes instead of through chaining a new instance of the exception. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst deleted file mode 100644 index db0b3f32c2ec0b..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-04-06-48-34.gh-issue-102397.ACJaOf.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix segfault from race condition in signal handling during garbage collection. -Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-06-10-02-22.gh-issue-101291.0FT2QS.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-06-10-02-22.gh-issue-101291.0FT2QS.rst deleted file mode 100644 index 46f0f325f91630..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-06-10-02-22.gh-issue-101291.0FT2QS.rst +++ /dev/null @@ -1,7 +0,0 @@ -Rearrage bits in first field (after header) of PyLongObject. -* Bits 0 and 1: 1 - sign. I.e. 0 for positive numbers, 1 for zero and 2 for negative numbers. -* Bit 2 reserved (probably for the immortal bit) -* Bits 3+ the unsigned size. - -This makes a few operations slightly more efficient, and will enable a more -compact and faster 2s-complement representation of most ints in future. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-08-08-37-36.gh-issue-102491.SFvvsC.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-08-08-37-36.gh-issue-102491.SFvvsC.rst deleted file mode 100644 index 5bdc9ed2f37adc..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-08-08-37-36.gh-issue-102491.SFvvsC.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve import time of ``platform`` by removing IronPython version parsing. The IronPython version parsing -was not functional (see https://github.com/IronLanguages/ironpython3/issues/1667). diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst deleted file mode 100644 index 723a4b9fa777d6..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-09-13-57-35.gh-issue-90997.J-Yhn2.rst +++ /dev/null @@ -1,2 +0,0 @@ -Shrink the number of inline :opcode:`CACHE` entries used by -:opcode:`LOAD_GLOBAL`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst deleted file mode 100644 index 0b95b5ec98e811..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-14-00-11-46.gh-issue-102594.BjU-m2.rst +++ /dev/null @@ -1 +0,0 @@ -Add note to exception raised in ``PyErr_SetObject`` when normalization fails. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-16-14-44-29.gh-issue-102755.j1GxlV.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-16-14-44-29.gh-issue-102755.j1GxlV.rst deleted file mode 100644 index d09af8d060d405..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-16-14-44-29.gh-issue-102755.j1GxlV.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :c:func:`PyErr_DisplayException` which takes just an exception instance, -to replace the legacy :c:func:`PyErr_Display` which takes the ``(typ, exc, -tb)`` triplet. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-16-17-24-44.gh-issue-102701.iNGVaS.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-16-17-24-44.gh-issue-102701.iNGVaS.rst deleted file mode 100644 index 4e1f31893377ba..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-16-17-24-44.gh-issue-102701.iNGVaS.rst +++ /dev/null @@ -1 +0,0 @@ -Fix overflow when creating very large dict. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-17-12-09-45.gh-issue-100982.Pf_BI6.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-17-12-09-45.gh-issue-100982.Pf_BI6.rst deleted file mode 100644 index 31a8660836c759..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-17-12-09-45.gh-issue-100982.Pf_BI6.rst +++ /dev/null @@ -1 +0,0 @@ -Replace all occurrences of ``COMPARE_AND_BRANCH`` with :opcode:`COMPARE_OP`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-17-13-43-34.gh-issue-102778.ANDv8I.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-17-13-43-34.gh-issue-102778.ANDv8I.rst deleted file mode 100644 index b5da227afa5a69..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-17-13-43-34.gh-issue-102778.ANDv8I.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add :data:`sys.last_exc` and deprecate :data:`sys.last_type`, :data:`sys.last_value` -and :data:`sys.last_traceback`, -which hold the same information in its legacy form. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-18-02-36-39.gh-issue-101975.HwMR1d.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-18-02-36-39.gh-issue-101975.HwMR1d.rst deleted file mode 100644 index 28c9a8465180db..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-18-02-36-39.gh-issue-101975.HwMR1d.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed ``stacktop`` value on tracing entries to avoid corruption on garbage collection. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-21-00-46-36.gh-issue-102859.PRkGca.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-21-00-46-36.gh-issue-102859.PRkGca.rst deleted file mode 100644 index d2e2232c33cc4d..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-21-00-46-36.gh-issue-102859.PRkGca.rst +++ /dev/null @@ -1,2 +0,0 @@ -Removed :opcode:`JUMP_IF_FALSE_OR_POP` and :opcode:`JUMP_IF_TRUE_OR_POP` -instructions. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-24-02-50-33.gh-issue-89987.oraTzh.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-24-02-50-33.gh-issue-89987.oraTzh.rst deleted file mode 100644 index 507f68b0c5affe..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-24-02-50-33.gh-issue-89987.oraTzh.rst +++ /dev/null @@ -1,2 +0,0 @@ -Reduce the number of inline :opcode:`CACHE` entries for -:opcode:`BINARY_SUBSCR`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-12-22-25.gh-issue-102192.gYxJP_.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-31-12-22-25.gh-issue-102192.gYxJP_.rst deleted file mode 100644 index 10dd72b1abc422..00000000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2023-03-31-12-22-25.gh-issue-102192.gYxJP_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Deprecated ``_PyErr_ChainExceptions`` in favour of -``_PyErr_ChainExceptions1``. diff --git a/Misc/NEWS.d/next/Documentation/2023-03-29-14-51-39.gh-issue-103112.XgGSEO.rst b/Misc/NEWS.d/next/Documentation/2023-03-29-14-51-39.gh-issue-103112.XgGSEO.rst deleted file mode 100644 index babc8150966117..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2023-03-29-14-51-39.gh-issue-103112.XgGSEO.rst +++ /dev/null @@ -1 +0,0 @@ -Add docstring to :meth:`http.client.HTTPResponse.read` to fix ``pydoc`` output. diff --git a/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst b/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst deleted file mode 100644 index d9360496ac24cb..00000000000000 --- a/Misc/NEWS.d/next/Library/2019-03-15-22-50-27.bpo-36305.Pbkv6u.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix handling of Windows filenames that resemble drives, such as ``./a:b``, -in :mod:`pathlib`. diff --git a/Misc/NEWS.d/next/Library/2022-04-11-18-34-33.gh-issue-72346.pC7gnM.rst b/Misc/NEWS.d/next/Library/2022-04-11-18-34-33.gh-issue-72346.pC7gnM.rst deleted file mode 100644 index 149ddd706c358f..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-04-11-18-34-33.gh-issue-72346.pC7gnM.rst +++ /dev/null @@ -1 +0,0 @@ -Added deprecation warning to *isdst* parameter of :func:`email.utils.localtime`. diff --git a/Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst b/Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst deleted file mode 100644 index 3eee82e59dfafb..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-06-30-21-28-41.gh-issue-94440.LtgX0d.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a :mod:`concurrent.futures.process` bug where ``ProcessPoolExecutor`` shutdown -could hang after a future has been quickly submitted and canceled. diff --git a/Misc/NEWS.d/next/Library/2022-07-09-13-07-30.gh-issue-94684.nV5yno.rst b/Misc/NEWS.d/next/Library/2022-07-09-13-07-30.gh-issue-94684.nV5yno.rst deleted file mode 100644 index 1fa38c0044d36f..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-09-13-07-30.gh-issue-94684.nV5yno.rst +++ /dev/null @@ -1 +0,0 @@ -Now :func:`uuid.uuid3` and :func:`uuid.uuid5` functions support :class:`bytes` objects as their *name* argument. diff --git a/Misc/NEWS.d/next/Library/2022-07-30-23-01-43.gh-issue-95495.RA-q1d.rst b/Misc/NEWS.d/next/Library/2022-07-30-23-01-43.gh-issue-95495.RA-q1d.rst deleted file mode 100644 index d0f4ccbdd3e39f..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-07-30-23-01-43.gh-issue-95495.RA-q1d.rst +++ /dev/null @@ -1,7 +0,0 @@ -When built against OpenSSL 3.0, the :mod:`ssl` module had a bug where it -reported unauthenticated EOFs (i.e. without close_notify) as a clean TLS-level -EOF. It now raises :exc:`~ssl.SSLEOFError`, matching the behavior in previous -versions of OpenSSL. The :attr:`~ssl.SSLContext.options` attribute on -:class:`~ssl.SSLContext` also no longer includes -:data:`~ssl.OP_IGNORE_UNEXPECTED_EOF` by default. This option may be set to -specify the previous OpenSSL 3.0 behavior. diff --git a/Misc/NEWS.d/next/Library/2022-09-19-08-12-58.gh-issue-96931.x0WQhh.rst b/Misc/NEWS.d/next/Library/2022-09-19-08-12-58.gh-issue-96931.x0WQhh.rst deleted file mode 100644 index 766b1d4d477b72..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-09-19-08-12-58.gh-issue-96931.x0WQhh.rst +++ /dev/null @@ -1 +0,0 @@ -Fix incorrect results from :meth:`ssl.SSLSocket.shared_ciphers` diff --git a/Misc/NEWS.d/next/Library/2022-10-10-19-14-51.gh-issue-98169.DBWIxL.rst b/Misc/NEWS.d/next/Library/2022-10-10-19-14-51.gh-issue-98169.DBWIxL.rst deleted file mode 100644 index 24c3aeecc83f18..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-10-10-19-14-51.gh-issue-98169.DBWIxL.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`dataclasses.astuple` crash when :class:`collections.defaultdict` -is present in the attributes. diff --git a/Misc/NEWS.d/next/Library/2022-11-24-13-23-07.gh-issue-48330.6uAX9F.rst b/Misc/NEWS.d/next/Library/2022-11-24-13-23-07.gh-issue-48330.6uAX9F.rst deleted file mode 100644 index 5b63a0a6b96dfa..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-11-24-13-23-07.gh-issue-48330.6uAX9F.rst +++ /dev/null @@ -1,8 +0,0 @@ -Added ``--durations`` command line option, showing the N slowest test cases. -:class:`unittest.TextTestRunner` and :class:`unittest.TextTestResult` -constructors accept a new *durations* keyword argument. Subclasses should take -this into account or accept ``**kwargs``. Added -:meth:`unittest.TestResult.addDuration` method and -:attr:`unittest.TestResult.collectedDurations` attribute. - -(Contributed by Giampaolo Rodola) diff --git a/Misc/NEWS.d/next/Library/2022-12-09-11-21-38.gh-issue-100131.v863yR.rst b/Misc/NEWS.d/next/Library/2022-12-09-11-21-38.gh-issue-100131.v863yR.rst deleted file mode 100644 index 07891f2c1e9eb6..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-09-11-21-38.gh-issue-100131.v863yR.rst +++ /dev/null @@ -1 +0,0 @@ -Added an optional ``delete`` keyword argument to :class:`tempfile.TemporaryDirectory`. diff --git a/Misc/NEWS.d/next/Library/2022-12-16-10-27-58.gh-issue-89727.y64ZLM.rst b/Misc/NEWS.d/next/Library/2022-12-16-10-27-58.gh-issue-89727.y64ZLM.rst deleted file mode 100644 index f9ac1475dceb00..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-16-10-27-58.gh-issue-89727.y64ZLM.rst +++ /dev/null @@ -1 +0,0 @@ -Fix pathlib.Path.walk RecursionError on deep directory trees by rewriting it using iteration instead of recursion. diff --git a/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst b/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst deleted file mode 100644 index ec37aff5092c3a..00000000000000 --- a/Misc/NEWS.d/next/Library/2022-12-20-10-55-14.gh-issue-100372.utfP65.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`ssl.SSLContext.load_verify_locations` no longer incorrectly accepts -some cases of trailing data when parsing DER. diff --git a/Misc/NEWS.d/next/Library/2023-01-27-14-51-07.gh-issue-101313.10AEXh.rst b/Misc/NEWS.d/next/Library/2023-01-27-14-51-07.gh-issue-101313.10AEXh.rst deleted file mode 100644 index 63d0a7286920a4..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-01-27-14-51-07.gh-issue-101313.10AEXh.rst +++ /dev/null @@ -1 +0,0 @@ -Added -h and --help arguments to the webbrowser CLI diff --git a/Misc/NEWS.d/next/Library/2023-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst b/Misc/NEWS.d/next/Library/2023-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst deleted file mode 100644 index 4e673ba9811568..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-09-19-40-41.gh-issue-101673.mX-Ppq.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a :mod:`pdb` bug where ``ll`` clears the changes to local variables. diff --git a/Misc/NEWS.d/next/Library/2023-02-18-23-03-50.gh-issue-98886.LkKGWv.rst b/Misc/NEWS.d/next/Library/2023-02-18-23-03-50.gh-issue-98886.LkKGWv.rst deleted file mode 100644 index 64e4d6eed2f62d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-18-23-03-50.gh-issue-98886.LkKGWv.rst +++ /dev/null @@ -1 +0,0 @@ -Fix issues when defining dataclasses that have fields with specific underscore names that aren't clearly reserved by :mod:`dataclasses`. diff --git a/Misc/NEWS.d/next/Library/2023-02-19-01-49-46.gh-issue-102038.n3if3D.rst b/Misc/NEWS.d/next/Library/2023-02-19-01-49-46.gh-issue-102038.n3if3D.rst deleted file mode 100644 index 40df20cb596049..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-19-01-49-46.gh-issue-102038.n3if3D.rst +++ /dev/null @@ -1 +0,0 @@ -Skip a ``stat`` in :mod:`site` if we have already found a ``pyvenv.cfg`` diff --git a/Misc/NEWS.d/next/Library/2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst b/Misc/NEWS.d/next/Library/2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst deleted file mode 100644 index 04c87e515cca93..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-20-16-47-56.gh-issue-102069.FS7f1j.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``__weakref__`` descriptor generation for custom dataclasses. diff --git a/Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst b/Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst deleted file mode 100644 index feba433f5bee89..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-21-11-56-16.gh-issue-102103.Dj0WEj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add ``module`` argument to :func:`dataclasses.make_dataclass` and make -classes produced by it pickleable. diff --git a/Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst b/Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst deleted file mode 100644 index 31b8ead8433279..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-02-26-17-29-57.gh-issue-79940.SAfmAy.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :func:`inspect.getasyncgenstate` and :func:`inspect.getasyncgenlocals`. -Patch by Thomas Krennwallner. diff --git a/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst b/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst deleted file mode 100644 index d30f65f30d109a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-03-19-53-08.gh-issue-102378.kRdOZc.rst +++ /dev/null @@ -1 +0,0 @@ -Private helper method ``inspect._signature_strip_non_python_syntax`` will no longer strip ``/`` from the input string. diff --git a/Misc/NEWS.d/next/Library/2023-03-04-20-58-29.gh-issue-74468.Ac5Ew_.rst b/Misc/NEWS.d/next/Library/2023-03-04-20-58-29.gh-issue-74468.Ac5Ew_.rst deleted file mode 100644 index 8fad551f3a4ece..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-04-20-58-29.gh-issue-74468.Ac5Ew_.rst +++ /dev/null @@ -1,3 +0,0 @@ -Attribute name of the extracted :mod:`tarfile` file object now holds -filename of itself rather than of the archive it is contained in. -Patch by Oleg Iarygin. diff --git a/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst b/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst deleted file mode 100644 index f47e4f70b1301d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-08-23-08-38.gh-issue-102519.wlcsFI.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add :func:`os.listdrives`, :func:`os.listvolumes` and :func:`os.listmounts` -functions on Windows for enumerating drives, volumes and mount points diff --git a/Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst b/Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst deleted file mode 100644 index 7307148d9a81ef..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-10-13-21-16.gh-issue-102578.-gujoI.rst +++ /dev/null @@ -1,4 +0,0 @@ -Speed up setting or deleting mutable attributes on non-dataclass subclasses of -frozen dataclasses. Due to the implementation of ``__setattr__`` and -``__delattr__`` for frozen dataclasses, this previously had a time complexity -of ``O(n)``. It now has a time complexity of ``O(1)``. diff --git a/Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst b/Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst deleted file mode 100644 index eff77c40e30c48..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-10-13-51-21.gh-issue-100112.VHh4mw.rst +++ /dev/null @@ -1 +0,0 @@ -:meth:`asyncio.Task.get_coro` now always returns a coroutine when wrapping an awaitable object. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2023-03-13-12-05-55.gh-issue-102615.NcA_ZL.rst b/Misc/NEWS.d/next/Library/2023-03-13-12-05-55.gh-issue-102615.NcA_ZL.rst deleted file mode 100644 index 333068369bc4f7..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-13-12-05-55.gh-issue-102615.NcA_ZL.rst +++ /dev/null @@ -1,3 +0,0 @@ -Typing: Improve the ``repr`` of generic aliases for classes generic over a -:class:`~typing.ParamSpec`. (Use square brackets to represent a parameter -list.) diff --git a/Misc/NEWS.d/next/Library/2023-03-13-18-27-00.gh-issue-102670.GyoThv.rst b/Misc/NEWS.d/next/Library/2023-03-13-18-27-00.gh-issue-102670.GyoThv.rst deleted file mode 100644 index 3de09f86754f3e..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-13-18-27-00.gh-issue-102670.GyoThv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Optimized fmean(), correlation(), covariance(), and linear_regression() -using the new math.sumprod() function. diff --git a/Misc/NEWS.d/next/Library/2023-03-16-08-17-29.gh-issue-102748.WNACpI.rst b/Misc/NEWS.d/next/Library/2023-03-16-08-17-29.gh-issue-102748.WNACpI.rst deleted file mode 100644 index b1dc67f38fe85d..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-16-08-17-29.gh-issue-102748.WNACpI.rst +++ /dev/null @@ -1,3 +0,0 @@ -:func:`asyncio.iscoroutine` now returns ``False`` for generators as -:mod:`asyncio` does not support legacy generator-based coroutines. -Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2023-03-16-16-43-04.gh-issue-78530.Lr8eq_.rst b/Misc/NEWS.d/next/Library/2023-03-16-16-43-04.gh-issue-78530.Lr8eq_.rst deleted file mode 100644 index bdb46d08c5c4af..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-16-16-43-04.gh-issue-78530.Lr8eq_.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`asyncio.wait` now accepts generators yielding tasks. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2023-03-17-19-14-26.gh-issue-76846.KEamjK.rst b/Misc/NEWS.d/next/Library/2023-03-17-19-14-26.gh-issue-76846.KEamjK.rst deleted file mode 100644 index 9fba11f074ee1c..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-17-19-14-26.gh-issue-76846.KEamjK.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix issue where ``__new__()`` and ``__init__()`` methods of -:class:`pathlib.PurePath` and :class:`~pathlib.Path` subclasses were not -called in some circumstances. diff --git a/Misc/NEWS.d/next/Library/2023-03-18-14-59-21.gh-issue-88965.kA70Km.rst b/Misc/NEWS.d/next/Library/2023-03-18-14-59-21.gh-issue-88965.kA70Km.rst deleted file mode 100644 index 6e9642100cd8cb..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-18-14-59-21.gh-issue-88965.kA70Km.rst +++ /dev/null @@ -1,7 +0,0 @@ -typing: Fix a bug relating to substitution in custom classes generic over a -:class:`~typing.ParamSpec`. Previously, if the ``ParamSpec`` was substituted -with a parameters list that itself contained a :class:`~typing.TypeVar`, the -``TypeVar`` in the parameters list could not be subsequently substituted. This -is now fixed. - -Patch by Nikita Sobolev. diff --git a/Misc/NEWS.d/next/Library/2023-03-19-15-30-59.gh-issue-102828.NKClXg.rst b/Misc/NEWS.d/next/Library/2023-03-19-15-30-59.gh-issue-102828.NKClXg.rst deleted file mode 100644 index be9b2bab24a381..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-19-15-30-59.gh-issue-102828.NKClXg.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add the ``onexc`` arg to :func:`shutil.rmtree`, which is like ``onerror`` -but expects an exception instance rather than an exc_info tuple. Deprecate -``onerror``. diff --git a/Misc/NEWS.d/next/Library/2023-03-20-12-21-19.gh-issue-102839.RjRi12.rst b/Misc/NEWS.d/next/Library/2023-03-20-12-21-19.gh-issue-102839.RjRi12.rst deleted file mode 100644 index 673b38974e4d1b..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-20-12-21-19.gh-issue-102839.RjRi12.rst +++ /dev/null @@ -1 +0,0 @@ -Improve performance of :func:`math.log` arguments handling by removing the argument clinic. diff --git a/Misc/NEWS.d/next/Library/2023-03-21-15-17-07.gh-issue-102871.U9mchn.rst b/Misc/NEWS.d/next/Library/2023-03-21-15-17-07.gh-issue-102871.U9mchn.rst deleted file mode 100644 index 3ef0e74d21ca00..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-21-15-17-07.gh-issue-102871.U9mchn.rst +++ /dev/null @@ -1,3 +0,0 @@ -Remove support for obsolete browsers from :mod:`webbrowser`. -Removed browsers include Grail, Mosaic, Netscape, Galeon, Skipstone, -Iceape, Firebird, and Firefox versions 35 and below. diff --git a/Misc/NEWS.d/next/Library/2023-03-22-16-15-18.gh-issue-102780.NEcljy.rst b/Misc/NEWS.d/next/Library/2023-03-22-16-15-18.gh-issue-102780.NEcljy.rst deleted file mode 100644 index 2aaffe34b86414..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-22-16-15-18.gh-issue-102780.NEcljy.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :class:`asyncio.Timeout` context manager now works reliably even when performing cleanup due -to task cancellation. Previously it could raise a -:exc:`~asyncio.CancelledError` instead of an :exc:`~asyncio.TimeoutError` in such cases. diff --git a/Misc/NEWS.d/next/Library/2023-03-23-13-34-33.gh-issue-102947.cTwcpU.rst b/Misc/NEWS.d/next/Library/2023-03-23-13-34-33.gh-issue-102947.cTwcpU.rst deleted file mode 100644 index b59c9820356697..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-23-13-34-33.gh-issue-102947.cTwcpU.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve traceback when :func:`dataclasses.fields` is called on a -non-dataclass. Patch by Alex Waygood diff --git a/Misc/NEWS.d/next/Library/2023-03-25-02-08-05.gh-issue-103023.Qfn7Hl.rst b/Misc/NEWS.d/next/Library/2023-03-25-02-08-05.gh-issue-103023.Qfn7Hl.rst deleted file mode 100644 index e7958f6f002055..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-25-02-08-05.gh-issue-103023.Qfn7Hl.rst +++ /dev/null @@ -1,2 +0,0 @@ -It's no longer possible to register expressions to display in -:class:`~pdb.Pdb` that raise :exc:`SyntaxError`. Patch by Tian Gao. diff --git a/Misc/NEWS.d/next/Library/2023-03-25-16-57-18.gh-issue-102433.L-7x2Q.rst b/Misc/NEWS.d/next/Library/2023-03-25-16-57-18.gh-issue-102433.L-7x2Q.rst deleted file mode 100644 index a4de4b4448f391..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-25-16-57-18.gh-issue-102433.L-7x2Q.rst +++ /dev/null @@ -1,10 +0,0 @@ -:func:`isinstance` checks against :func:`runtime-checkable protocols -` now use :func:`inspect.getattr_static` rather -than :func:`hasattr` to lookup whether attributes exist. This means that -descriptors and :meth:`~object.__getattr__` methods are no longer -unexpectedly evaluated during ``isinstance()`` checks against -runtime-checkable protocols. However, it may also mean that some objects -which used to be considered instances of a runtime-checkable protocol may no -longer be considered instances of that protocol on Python 3.12+, and vice -versa. Most users are unlikely to be affected by this change. Patch by Alex -Waygood. diff --git a/Misc/NEWS.d/next/Library/2023-03-26-20-54-57.gh-issue-103046.xBlA2l.rst b/Misc/NEWS.d/next/Library/2023-03-26-20-54-57.gh-issue-103046.xBlA2l.rst deleted file mode 100644 index f9bd0a10056ef1..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-26-20-54-57.gh-issue-103046.xBlA2l.rst +++ /dev/null @@ -1 +0,0 @@ -Display current line label correctly in :mod:`dis` when ``show_caches`` is False and ``lasti`` points to a CACHE entry. diff --git a/Misc/NEWS.d/next/Library/2023-03-27-15-01-16.gh-issue-103056.-Efh5Q.rst b/Misc/NEWS.d/next/Library/2023-03-27-15-01-16.gh-issue-103056.-Efh5Q.rst deleted file mode 100644 index c892d8376503f8..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-27-15-01-16.gh-issue-103056.-Efh5Q.rst +++ /dev/null @@ -1 +0,0 @@ -Ensure final ``_generate_next_value_`` is a ``staticmethod``. diff --git a/Misc/NEWS.d/next/Library/2023-03-27-19-21-51.gh-issue-102549.NQ6Nlv.rst b/Misc/NEWS.d/next/Library/2023-03-27-19-21-51.gh-issue-102549.NQ6Nlv.rst deleted file mode 100644 index e4def038175b87..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-27-19-21-51.gh-issue-102549.NQ6Nlv.rst +++ /dev/null @@ -1 +0,0 @@ -Don't ignore exceptions in member type creation. diff --git a/Misc/NEWS.d/next/Library/2023-03-28-05-14-59.gh-issue-103068.YQTmrA.rst b/Misc/NEWS.d/next/Library/2023-03-28-05-14-59.gh-issue-103068.YQTmrA.rst deleted file mode 100644 index 71c142c30f4eff..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-28-05-14-59.gh-issue-103068.YQTmrA.rst +++ /dev/null @@ -1,2 +0,0 @@ -It's no longer possible to register conditional breakpoints in -:class:`~pdb.Pdb` that raise :exc:`SyntaxError`. Patch by Tian Gao. diff --git a/Misc/NEWS.d/next/Library/2023-03-28-15-12-53.gh-issue-103085.DqNehf.rst b/Misc/NEWS.d/next/Library/2023-03-28-15-12-53.gh-issue-103085.DqNehf.rst deleted file mode 100644 index fa07fa5226c06a..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-28-15-12-53.gh-issue-103085.DqNehf.rst +++ /dev/null @@ -1 +0,0 @@ -Pure python :func:`locale.getencoding()` will not warn deprecation. diff --git a/Misc/NEWS.d/next/Tests/2023-01-27-18-10-40.gh-issue-101377.IJGpqh.rst b/Misc/NEWS.d/next/Tests/2023-01-27-18-10-40.gh-issue-101377.IJGpqh.rst deleted file mode 100644 index a9c19ce060e3ab..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-01-27-18-10-40.gh-issue-101377.IJGpqh.rst +++ /dev/null @@ -1 +0,0 @@ -Improved test_locale_calendar_formatweekday of calendar. diff --git a/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst b/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst deleted file mode 100644 index 94d160dd4127a6..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-03-08-13-54-20.gh-issue-102537.Vfplpb.rst +++ /dev/null @@ -1,2 +0,0 @@ -Adjust the error handling strategy in -``test_zoneinfo.TzPathTest.python_tzpath_context``. Patch by Paul Ganssle. diff --git a/Misc/NEWS.d/next/Tests/2023-03-23-23-25-18.gh-issue-102980.Zps4QF.rst b/Misc/NEWS.d/next/Tests/2023-03-23-23-25-18.gh-issue-102980.Zps4QF.rst deleted file mode 100644 index 48277367fc8755..00000000000000 --- a/Misc/NEWS.d/next/Tests/2023-03-23-23-25-18.gh-issue-102980.Zps4QF.rst +++ /dev/null @@ -1 +0,0 @@ -Improve test coverage on :mod:`pdb`. diff --git a/Misc/NEWS.d/next/Tools-Demos/2023-03-21-01-27-07.gh-issue-102809.2F1Byz.rst b/Misc/NEWS.d/next/Tools-Demos/2023-03-21-01-27-07.gh-issue-102809.2F1Byz.rst deleted file mode 100644 index 5c282769878563..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2023-03-21-01-27-07.gh-issue-102809.2F1Byz.rst +++ /dev/null @@ -1 +0,0 @@ -``Misc/gdbinit`` was removed. diff --git a/Misc/NEWS.d/next/Windows/2023-02-22-17-26-10.gh-issue-99726.76t957.rst b/Misc/NEWS.d/next/Windows/2023-02-22-17-26-10.gh-issue-99726.76t957.rst deleted file mode 100644 index e2578620017894..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-02-22-17-26-10.gh-issue-99726.76t957.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improves correctness of stat results for Windows, and uses faster API when -available diff --git a/Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst b/Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst deleted file mode 100644 index 5669ebbb442c24..00000000000000 --- a/Misc/NEWS.d/next/Windows/2023-03-14-10-52-43.gh-issue-102690.sbXtqk.rst +++ /dev/null @@ -1 +0,0 @@ -Update :mod:`webbrowser` to fall back to Microsoft Edge instead of Internet Explorer. diff --git a/README.rst b/README.rst index 6923b692f6c971..06c4f0a1fa3917 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.12.0 alpha 6 +This is Python version 3.12.0 alpha 7 ===================================== .. image:: https://github.com/python/cpython/workflows/Tests/badge.svg From fd1947ecfba589feebd48c1ebb60d1c01ee0a89d Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 4 Apr 2023 17:00:03 +0100 Subject: [PATCH 09/76] bpo-44844: Enable detection of Microsoft Edge browser in webbrowser module (GH-29908) --- Lib/test/test_webbrowser.py | 25 +++++++++++++++++++ Lib/webbrowser.py | 16 ++++++++++++ .../2021-12-03-23-00-56.bpo-44844.tvg2VY.rst | 1 + 3 files changed, 42 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2021-12-03-23-00-56.bpo-44844.tvg2VY.rst diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 147a113c7fd730..2d695bc883131f 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -95,6 +95,31 @@ def test_open_new_tab(self): arguments=[URL]) +class EdgeCommandTest(CommandTestMixin, unittest.TestCase): + + browser_class = webbrowser.Edge + + def test_open(self): + self._test('open', + options=[], + arguments=[URL]) + + def test_open_with_autoraise_false(self): + self._test('open', kw=dict(autoraise=False), + options=[], + arguments=[URL]) + + def test_open_new(self): + self._test('open_new', + options=['--new-window'], + arguments=[URL]) + + def test_open_new_tab(self): + self._test('open_new_tab', + options=[], + arguments=[URL]) + + class MozillaCommandTest(CommandTestMixin, unittest.TestCase): browser_class = webbrowser.Mozilla diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index d98c5997d2f44f..b86d131f030d80 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -400,6 +400,16 @@ def open(self, url, new=0, autoraise=True): return ok +class Edge(UnixBrowser): + "Launcher class for Microsoft Edge browser." + + remote_args = ['%action', '%s'] + remote_action = "" + remote_action_newwin = "--new-window" + remote_action_newtab = "" + background = True + + # # Platform support for Unix # @@ -456,6 +466,10 @@ def register_X_browsers(): register("opera", None, Opera("opera")) + if shutil.which("microsoft-edge"): + register("microsoft-edge", None, Edge("microsoft-edge")) + + def register_standard_browsers(): global _tryorder _tryorder = [] @@ -487,6 +501,8 @@ def register_standard_browsers(): "opera", edge64, edge32): if shutil.which(browser): register(browser, None, BackgroundBrowser(browser)) + if shutil.which("MicrosoftEdge.exe"): + register("microsoft-edge", None, Edge("MicrosoftEdge.exe")) else: # Prefer X browsers if present if os.environ.get("DISPLAY") or os.environ.get("WAYLAND_DISPLAY"): diff --git a/Misc/NEWS.d/next/Library/2021-12-03-23-00-56.bpo-44844.tvg2VY.rst b/Misc/NEWS.d/next/Library/2021-12-03-23-00-56.bpo-44844.tvg2VY.rst new file mode 100644 index 00000000000000..f0c91236dfdf21 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-12-03-23-00-56.bpo-44844.tvg2VY.rst @@ -0,0 +1 @@ +Enables :mod:`webbrowser` to detect and launch Microsoft Edge browser. From a90f5ca87b9ad713280af50b30558a2aca566be0 Mon Sep 17 00:00:00 2001 From: Thomas Wouters Date: Tue, 4 Apr 2023 22:00:41 +0200 Subject: [PATCH 10/76] Post 3.12.0a7 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 1a8a6d9c8f7a3d..aaedd563a905e9 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 7 /* Version as a string */ -#define PY_VERSION "3.12.0a7" +#define PY_VERSION "3.12.0a7+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. From f184abbdc9ac3a5656de5f606faf505aa42ff391 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 4 Apr 2023 17:06:00 -0400 Subject: [PATCH 11/76] =?UTF-8?q?gh-103207:=20Add=20instructions=20to=20th?= =?UTF-8?q?e=20macOS=20installer=20welcome=20display=20on=20how=20to=20wor?= =?UTF-8?q?karound=20=20the=20macOS=2013=20Ventura=20=E2=80=9CThe=20instal?= =?UTF-8?q?ler=20encountered=20an=20error=E2=80=9D=20failure.=20(GH-103251?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Mac/BuildScript/resources/Welcome.rtf | 39 ++++++++++++++++--- ...-04-04-13-37-28.gh-issue-103207.x0vvQp.rst | 2 + 2 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2023-04-04-13-37-28.gh-issue-103207.x0vvQp.rst diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 7819241b618d87..efdcb12f849b66 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,9 +1,9 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2639 +{\rtf1\ansi\ansicpg1252\cocoartf2708 \cocoascreenfonts1\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fmodern\fcharset0 CourierNewPSMT; -} -{\colortbl;\red255\green255\blue255;} -{\*\expandedcolortbl;;} -\paperw11900\paperh16840\margl1440\margr1440\vieww12200\viewh10880\viewkind0 +\f3\fnil\fcharset0 HelveticaNeue;\f4\fnil\fcharset0 HelveticaNeue-Bold;} +{\colortbl;\red255\green255\blue255;\red24\green26\blue30;\red255\green255\blue255;} +{\*\expandedcolortbl;;\cssrgb\c12157\c13725\c15686;\cssrgb\c100000\c100000\c100000;} +\margl1440\margr1440\vieww12820\viewh10620\viewkind0 \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 \f0\fs24 \cf0 This package will install @@ -25,6 +25,35 @@ At the end of this install, click on \f0 to install a set of current SSL root certificates.\ \ +\f1\b macOS 13 Ventura users +\f0\b0 : due to an issue with macOS +\f1\b Installer +\f0\b0 app, installation of some +\f3 \cf2 \cb3 \expnd0\expndtw0\kerning0 +third-party +\f0 \cf0 \cb1 \kerning1\expnd0\expndtw0 packages including this Python package may fail with a vague +\f4\b \cf2 \cb3 \expnd0\expndtw0\kerning0 +\'93The installer encountered an error\'94 +\f3\b0 message if the +\f4\b Installer +\f3\b0 app does not have permission to access the folder containing the downloaded installer file, typically in the +\f4\b Downloads +\f3\b0 folder. Go to +\f4\b System Settings +\f3\b0 -> +\f4\b Privacy & Security +\f3\b0 -> +\f4\b Files and Folders +\f3\b0 , then click the mark in front of +\f4\b Installer +\f3\b0 to expand, and enable +\f4\b Downloads Folder +\f0\b0 \cf0 \cb1 \kerning1\expnd0\expndtw0 by moving the toggle to the right +\f3 \cf2 \cb3 \expnd0\expndtw0\kerning0 +. See {\field{\*\fldinst{HYPERLINK "https://github.com/python/cpython/issues/103207"}}{\fldrslt https://github.com/python/cpython/issues/103207}} for up-to-date information on this issue. +\f0 \cf0 \cb1 \kerning1\expnd0\expndtw0 \ +\ + \f1\b NOTE: \f0\b0 This is an alpha test preview of Python 3.12.0, the next feature release of Python 3. It is not intended for production use.\ } \ No newline at end of file diff --git a/Misc/NEWS.d/next/macOS/2023-04-04-13-37-28.gh-issue-103207.x0vvQp.rst b/Misc/NEWS.d/next/macOS/2023-04-04-13-37-28.gh-issue-103207.x0vvQp.rst new file mode 100644 index 00000000000000..3c176e3a6b5310 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2023-04-04-13-37-28.gh-issue-103207.x0vvQp.rst @@ -0,0 +1,2 @@ +Add instructions to the macOS installer welcome display on how to workaround +the macOS 13 Ventura “The installer encountered an error” failure. From 935aa452359ac3f79febefcdb4387b962cf528af Mon Sep 17 00:00:00 2001 From: Charles Machalow Date: Tue, 4 Apr 2023 15:24:13 -0700 Subject: [PATCH 12/76] GH-75586: Make shutil.which() on Windows more consistent with the OS (GH-103179) --- Doc/library/shutil.rst | 34 +++++-- Doc/whatsnew/3.12.rst | 14 +++ Lib/shutil.py | 89 ++++++++++--------- Lib/test/test_shutil.py | 86 ++++++++++++++++-- ...3-04-02-22-04-26.gh-issue-75586.526iJm.rst | 1 + Modules/_winapi.c | 21 +++++ Modules/clinic/_winapi.c.h | 42 ++++++++- 7 files changed, 233 insertions(+), 54 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-02-22-04-26.gh-issue-75586.526iJm.rst diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index acba66258fe8f0..373cc7d6072031 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -433,23 +433,43 @@ Directory and files operations When no *path* is specified, the results of :func:`os.environ` are used, returning either the "PATH" value or a fallback of :attr:`os.defpath`. - On Windows, the current directory is always prepended to the *path* whether - or not you use the default or provide your own, which is the behavior the - command shell uses when finding executables. Additionally, when finding the - *cmd* in the *path*, the ``PATHEXT`` environment variable is checked. For - example, if you call ``shutil.which("python")``, :func:`which` will search - ``PATHEXT`` to know that it should look for ``python.exe`` within the *path* - directories. For example, on Windows:: + On Windows, the current directory is prepended to the *path* if *mode* does + not include ``os.X_OK``. When the *mode* does include ``os.X_OK``, the + Windows API ``NeedCurrentDirectoryForExePathW`` will be consulted to + determine if the current directory should be prepended to *path*. To avoid + consulting the current working directory for executables: set the environment + variable ``NoDefaultCurrentDirectoryInExePath``. + + Also on Windows, the ``PATHEXT`` variable is used to resolve commands + that may not already include an extension. For example, if you call + ``shutil.which("python")``, :func:`which` will search ``PATHEXT`` + to know that it should look for ``python.exe`` within the *path* + directories. For example, on Windows:: >>> shutil.which("python") 'C:\\Python33\\python.EXE' + This is also applied when *cmd* is a path that contains a directory + component:: + + >> shutil.which("C:\\Python33\\python") + 'C:\\Python33\\python.EXE' + .. versionadded:: 3.3 .. versionchanged:: 3.8 The :class:`bytes` type is now accepted. If *cmd* type is :class:`bytes`, the result type is also :class:`bytes`. + .. versionchanged:: 3.12 + On Windows, the current directory is no longer prepended to the search + path if *mode* includes ``os.X_OK`` and WinAPI + ``NeedCurrentDirectoryForExePathW(cmd)`` is false, else the current + directory is prepended even if it is already in the search path; + ``PATHEXT`` is used now even when *cmd* includes a directory component + or ends with an extension that is in ``PATHEXT``; and filenames that + have no extension can now be found. + .. exception:: Error This exception collects exceptions that are raised during a multi-file diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 88b99828f0beeb..cc2c0560a57c85 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -343,6 +343,20 @@ shutil will be removed in Python 3.14. (Contributed by Irit Katriel in :gh:`102828`.) +* :func:`shutil.which` now consults the *PATHEXT* environment variable to + find matches within *PATH* on Windows even when the given *cmd* includes + a directory component. + (Contributed by Charles Machalow in :gh:`103179`.) + + :func:`shutil.which` will call ``NeedCurrentDirectoryForExePathW`` when + querying for executables on Windows to determine if the current working + directory should be prepended to the search path. + (Contributed by Charles Machalow in :gh:`103179`.) + + :func:`shutil.which` will return a path matching the *cmd* with a component + from ``PATHEXT`` prior to a direct match elsewhere in the search path on + Windows. + (Contributed by Charles Machalow in :gh:`103179`.) sqlite3 ------- diff --git a/Lib/shutil.py b/Lib/shutil.py index b0576407e02ffb..8b378645a5a375 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -40,6 +40,9 @@ elif _WINDOWS: import nt +if sys.platform == 'win32': + import _winapi + COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 @@ -1449,6 +1452,16 @@ def _access_check(fn, mode): and not os.path.isdir(fn)) +def _win_path_needs_curdir(cmd, mode): + """ + On Windows, we can use NeedCurrentDirectoryForExePath to figure out + if we should add the cwd to PATH when searching for executables if + the mode is executable. + """ + return (not (mode & os.X_OK)) or _winapi.NeedCurrentDirectoryForExePath( + os.fsdecode(cmd)) + + def which(cmd, mode=os.F_OK | os.X_OK, path=None): """Given a command, mode, and a PATH string, return the path which conforms to the given mode on the PATH, or None if there is no such @@ -1459,60 +1472,54 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): path. """ - # If we're given a path with a directory part, look it up directly rather - # than referring to PATH directories. This includes checking relative to the - # current directory, e.g. ./script - if os.path.dirname(cmd): - if _access_check(cmd, mode): - return cmd - return None - use_bytes = isinstance(cmd, bytes) - if path is None: - path = os.environ.get("PATH", None) - if path is None: - try: - path = os.confstr("CS_PATH") - except (AttributeError, ValueError): - # os.confstr() or CS_PATH is not available - path = os.defpath - # bpo-35755: Don't use os.defpath if the PATH environment variable is - # set to an empty string - - # PATH='' doesn't match, whereas PATH=':' looks in the current directory - if not path: - return None - - if use_bytes: - path = os.fsencode(path) - path = path.split(os.fsencode(os.pathsep)) + # If we're given a path with a directory part, look it up directly rather + # than referring to PATH directories. This includes checking relative to + # the current directory, e.g. ./script + dirname, cmd = os.path.split(cmd) + if dirname: + path = [dirname] else: - path = os.fsdecode(path) - path = path.split(os.pathsep) + if path is None: + path = os.environ.get("PATH", None) + if path is None: + try: + path = os.confstr("CS_PATH") + except (AttributeError, ValueError): + # os.confstr() or CS_PATH is not available + path = os.defpath + # bpo-35755: Don't use os.defpath if the PATH environment variable + # is set to an empty string + + # PATH='' doesn't match, whereas PATH=':' looks in the current + # directory + if not path: + return None - if sys.platform == "win32": - # The current directory takes precedence on Windows. - curdir = os.curdir if use_bytes: - curdir = os.fsencode(curdir) - if curdir not in path: + path = os.fsencode(path) + path = path.split(os.fsencode(os.pathsep)) + else: + path = os.fsdecode(path) + path = path.split(os.pathsep) + + if sys.platform == "win32" and _win_path_needs_curdir(cmd, mode): + curdir = os.curdir + if use_bytes: + curdir = os.fsencode(curdir) path.insert(0, curdir) + if sys.platform == "win32": # PATHEXT is necessary to check on Windows. pathext_source = os.getenv("PATHEXT") or _WIN_DEFAULT_PATHEXT pathext = [ext for ext in pathext_source.split(os.pathsep) if ext] if use_bytes: pathext = [os.fsencode(ext) for ext in pathext] - # See if the given file matches any of the expected path extensions. - # This will allow us to short circuit when given "python.exe". - # If it does match, only test that one, otherwise we have to try - # others. - if any(cmd.lower().endswith(ext.lower()) for ext in pathext): - files = [cmd] - else: - files = [cmd + ext for ext in pathext] + + # Always try checking the originally given cmd, if it doesn't match, try pathext + files = [cmd] + [cmd + ext for ext in pathext] else: # On other platforms you don't have things like PATHEXT to tell you # what file suffixes are executable, so just pass on cmd as-is. diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 1c0589ced9ea89..9eaf167a9fa3c9 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2034,18 +2034,68 @@ def test_relative_cmd(self): rv = shutil.which(relpath, path=base_dir) self.assertIsNone(rv) - def test_cwd(self): + @unittest.skipUnless(sys.platform != "win32", + "test is for non win32") + def test_cwd_non_win32(self): # Issue #16957 base_dir = os.path.dirname(self.dir) with os_helper.change_cwd(path=self.dir): rv = shutil.which(self.file, path=base_dir) - if sys.platform == "win32": - # Windows: current directory implicitly on PATH + # non-win32: shouldn't match in the current directory. + self.assertIsNone(rv) + + @unittest.skipUnless(sys.platform == "win32", + "test is for win32") + def test_cwd_win32(self): + base_dir = os.path.dirname(self.dir) + with os_helper.change_cwd(path=self.dir): + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): + rv = shutil.which(self.file, path=base_dir) + # Current directory implicitly on PATH self.assertEqual(rv, os.path.join(self.curdir, self.file)) - else: - # Other platforms: shouldn't match in the current directory. + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=False): + rv = shutil.which(self.file, path=base_dir) + # Current directory not on PATH self.assertIsNone(rv) + @unittest.skipUnless(sys.platform == "win32", + "test is for win32") + def test_cwd_win32_added_before_all_other_path(self): + base_dir = pathlib.Path(os.fsdecode(self.dir)) + + elsewhere_in_path_dir = base_dir / 'dir1' + elsewhere_in_path_dir.mkdir() + match_elsewhere_in_path = elsewhere_in_path_dir / 'hello.exe' + match_elsewhere_in_path.touch() + + exe_in_cwd = base_dir / 'hello.exe' + exe_in_cwd.touch() + + with os_helper.change_cwd(path=base_dir): + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): + rv = shutil.which('hello.exe', path=elsewhere_in_path_dir) + + self.assertEqual(os.path.abspath(rv), os.path.abspath(exe_in_cwd)) + + @unittest.skipUnless(sys.platform == "win32", + "test is for win32") + def test_pathext_match_before_path_full_match(self): + base_dir = pathlib.Path(os.fsdecode(self.dir)) + dir1 = base_dir / 'dir1' + dir2 = base_dir / 'dir2' + dir1.mkdir() + dir2.mkdir() + + pathext_match = dir1 / 'hello.com.exe' + path_match = dir2 / 'hello.com' + pathext_match.touch() + path_match.touch() + + test_path = os.pathsep.join([str(dir1), str(dir2)]) + assert os.path.basename(shutil.which( + 'hello.com', path=test_path, mode = os.F_OK + )).lower() == 'hello.com.exe' + @os_helper.skip_if_dac_override def test_non_matching_mode(self): # Set the file read-only and ask for writeable files. @@ -2179,6 +2229,32 @@ def test_pathext_with_empty_str(self): rv = shutil.which(program, path=self.temp_dir) self.assertEqual(rv, temp_filexyz.name) + # See GH-75586 + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_applied_on_files_in_path(self): + with os_helper.EnvironmentVarGuard() as env: + env["PATH"] = self.temp_dir + env["PATHEXT"] = ".test" + + test_path = pathlib.Path(self.temp_dir) / "test_program.test" + test_path.touch(mode=0o755) + + self.assertEqual(shutil.which("test_program"), str(test_path)) + + # See GH-75586 + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_win_path_needs_curdir(self): + with unittest.mock.patch('_winapi.NeedCurrentDirectoryForExePath', return_value=True) as need_curdir_mock: + self.assertTrue(shutil._win_path_needs_curdir('dontcare', os.X_OK)) + need_curdir_mock.assert_called_once_with('dontcare') + need_curdir_mock.reset_mock() + self.assertTrue(shutil._win_path_needs_curdir('dontcare', 0)) + need_curdir_mock.assert_not_called() + + with unittest.mock.patch('_winapi.NeedCurrentDirectoryForExePath', return_value=False) as need_curdir_mock: + self.assertFalse(shutil._win_path_needs_curdir('dontcare', os.X_OK)) + need_curdir_mock.assert_called_once_with('dontcare') + class TestWhichBytes(TestWhich): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2023-04-02-22-04-26.gh-issue-75586.526iJm.rst b/Misc/NEWS.d/next/Library/2023-04-02-22-04-26.gh-issue-75586.526iJm.rst new file mode 100644 index 00000000000000..8ec568ec4e4775 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-02-22-04-26.gh-issue-75586.526iJm.rst @@ -0,0 +1 @@ +Fix various Windows-specific issues with ``shutil.which``. diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 83cde7501176b6..fa380b8b798405 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -2054,6 +2054,26 @@ _winapi__mimetypes_read_windows_registry_impl(PyObject *module, #undef CB_TYPE } +/*[clinic input] +_winapi.NeedCurrentDirectoryForExePath -> bool + + exe_name: LPCWSTR + / +[clinic start generated code]*/ + +static int +_winapi_NeedCurrentDirectoryForExePath_impl(PyObject *module, + LPCWSTR exe_name) +/*[clinic end generated code: output=a65ec879502b58fc input=972aac88a1ec2f00]*/ +{ + BOOL result; + + Py_BEGIN_ALLOW_THREADS + result = NeedCurrentDirectoryForExePathW(exe_name); + Py_END_ALLOW_THREADS + + return result; +} static PyMethodDef winapi_functions[] = { _WINAPI_CLOSEHANDLE_METHODDEF @@ -2089,6 +2109,7 @@ static PyMethodDef winapi_functions[] = { _WINAPI_GETACP_METHODDEF _WINAPI_GETFILETYPE_METHODDEF _WINAPI__MIMETYPES_READ_WINDOWS_REGISTRY_METHODDEF + _WINAPI_NEEDCURRENTDIRECTORYFOREXEPATH_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index 891b3f851d1243..7bc63e612be348 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -1371,4 +1371,44 @@ _winapi__mimetypes_read_windows_registry(PyObject *module, PyObject *const *args exit: return return_value; } -/*[clinic end generated code: output=edb1a9d1bbfd6394 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_winapi_NeedCurrentDirectoryForExePath__doc__, +"NeedCurrentDirectoryForExePath($module, exe_name, /)\n" +"--\n" +"\n"); + +#define _WINAPI_NEEDCURRENTDIRECTORYFOREXEPATH_METHODDEF \ + {"NeedCurrentDirectoryForExePath", (PyCFunction)_winapi_NeedCurrentDirectoryForExePath, METH_O, _winapi_NeedCurrentDirectoryForExePath__doc__}, + +static int +_winapi_NeedCurrentDirectoryForExePath_impl(PyObject *module, + LPCWSTR exe_name); + +static PyObject * +_winapi_NeedCurrentDirectoryForExePath(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + LPCWSTR exe_name = NULL; + int _return_value; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("NeedCurrentDirectoryForExePath", "argument", "str", arg); + goto exit; + } + exe_name = PyUnicode_AsWideCharString(arg, NULL); + if (exe_name == NULL) { + goto exit; + } + _return_value = _winapi_NeedCurrentDirectoryForExePath_impl(module, exe_name); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + /* Cleanup for exe_name */ + PyMem_Free((void *)exe_name); + + return return_value; +} +/*[clinic end generated code: output=96ea65ece7912d0a input=a9049054013a1b77]*/ From bceb9e00ad2998e5193ad5b477e92a114dd31024 Mon Sep 17 00:00:00 2001 From: Tim Burke Date: Tue, 4 Apr 2023 15:46:46 -0700 Subject: [PATCH 13/76] Improve some grammar in the socket docs (#103254) --- Doc/library/socket.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index aec79da57f0576..c8ca555700a3c9 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1775,7 +1775,7 @@ to sockets. much data, if any, was successfully sent. .. versionchanged:: 3.5 - The socket timeout is no more reset each time data is sent successfully. + The socket timeout is no longer reset each time data is sent successfully. The socket timeout is now the maximum total duration to send all data. .. versionchanged:: 3.5 @@ -1998,8 +1998,8 @@ can be changed by calling :func:`setdefaulttimeout`. * In *non-blocking mode*, operations fail (with an error that is unfortunately system-dependent) if they cannot be completed immediately: functions from the - :mod:`select` can be used to know when and whether a socket is available for - reading or writing. + :mod:`select` module can be used to know when and whether a socket is available + for reading or writing. * In *timeout mode*, operations fail if they cannot be completed within the timeout specified for the socket (they raise a :exc:`timeout` exception) @@ -2188,7 +2188,7 @@ manager protocol instead, open a socket with:: socket.socket(socket.AF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) After binding (:const:`CAN_RAW`) or connecting (:const:`CAN_BCM`) the socket, you -can use the :meth:`socket.send`, and the :meth:`socket.recv` operations (and +can use the :meth:`socket.send` and :meth:`socket.recv` operations (and their counterparts) on the socket object as usual. This last example might require special privileges:: From f513d5c80672c76acbdaf7d5b601f4bbe9fae56a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 4 Apr 2023 17:03:40 -0600 Subject: [PATCH 14/76] gh-102660: Fix is_core_module() (gh-103257) In gh-102744 we added is_core_module() (in Python/import.c), which relies on get_core_module_dict() (also added in that PR). The problem is that_PyImport_FixupBuiltin(), which ultimately calls is_core_module(), is called on the builtins module before interp->builtins_copyis set. Consequently, the builtins module isn't considered a "core" module while it is getting "fixed up" and its module def m_copy erroneously gets set. Under isolated interpreters this causes problems since sys and builtins are allowed even though they are still single-phase init modules. (This was discovered while working on gh-101660.) The solution is to stop relying on get_core_module_dict() in is_core_module(). --- Python/import.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/Python/import.c b/Python/import.c index 24249ae4a6ade1..1db5b9333bbba1 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1110,7 +1110,17 @@ get_core_module_dict(PyInterpreterState *interp, static inline int is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *filename) { - return get_core_module_dict(interp, name, filename) != NULL; + /* This might be called before the core dict copies are in place, + so we can't rely on get_core_module_dict() here. */ + if (filename == name) { + if (PyUnicode_CompareWithASCIIString(name, "sys") == 0) { + return 1; + } + if (PyUnicode_CompareWithASCIIString(name, "builtins") == 0) { + return 1; + } + } + return 0; } static int @@ -1136,6 +1146,8 @@ fix_up_extension(PyObject *mod, PyObject *name, PyObject *filename) // when the extension module doesn't support sub-interpreters. if (def->m_size == -1) { if (!is_core_module(tstate->interp, name, filename)) { + assert(PyUnicode_CompareWithASCIIString(name, "sys") != 0); + assert(PyUnicode_CompareWithASCIIString(name, "builtins") != 0); if (def->m_base.m_copy) { /* Somebody already imported the module, likely under a different name. From a62ff970750267441a7175b1bd6d6b2ec59d9a2c Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Wed, 5 Apr 2023 09:10:45 +0900 Subject: [PATCH 15/76] gh-101525: Disable peephole optimization process of BOLT (gh-103187) Co-authored-by: Dong-hee Na --- Doc/whatsnew/3.12.rst | 2 +- Makefile.pre.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index cc2c0560a57c85..23524ec5d7d452 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -465,7 +465,7 @@ Optimizations * Added experimental support for using the BOLT binary optimizer in the build process, which improves performance by 1-5%. - (Contributed by Kevin Modzelewski in :gh:`90536`.) + (Contributed by Kevin Modzelewski in :gh:`90536` and tuned by Dong-hee Na in :gh:`101525`) * Speed up the regular expression substitution (functions :func:`re.sub` and :func:`re.subn` and corresponding :class:`!re.Pattern` methods) for diff --git a/Makefile.pre.in b/Makefile.pre.in index b97daaf6f445cc..9fdbd8db19bb33 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -670,7 +670,7 @@ bolt-opt: @PREBOLT_RULE@ @LLVM_BOLT@ ./$(BUILDPYTHON) -instrument -instrumentation-file-append-pid -instrumentation-file=$(abspath $(BUILDPYTHON).bolt) -o $(BUILDPYTHON).bolt_inst ./$(BUILDPYTHON).bolt_inst $(PROFILE_TASK) || true @MERGE_FDATA@ $(BUILDPYTHON).*.fdata > $(BUILDPYTHON).fdata - @LLVM_BOLT@ ./$(BUILDPYTHON) -o $(BUILDPYTHON).bolt -data=$(BUILDPYTHON).fdata -update-debug-sections -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=all -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot + @LLVM_BOLT@ ./$(BUILDPYTHON) -o $(BUILDPYTHON).bolt -data=$(BUILDPYTHON).fdata -update-debug-sections -reorder-blocks=ext-tsp -reorder-functions=hfsort+ -split-functions -icf=1 -inline-all -split-eh -reorder-functions-use-hot-size -peepholes=none -jump-tables=aggressive -inline-ap -indirect-call-promotion=all -dyno-stats -use-gnu-stack -frame-opt=hot rm -f *.fdata rm -f $(BUILDPYTHON).bolt_inst mv $(BUILDPYTHON).bolt $(BUILDPYTHON) From 1a8f862e329c3872a11d4ef8eb85cf353ca2f4d5 Mon Sep 17 00:00:00 2001 From: Michael Handler Date: Tue, 4 Apr 2023 21:55:24 -0700 Subject: [PATCH 16/76] gh-66897: Upgrade HTTP CONNECT to protocol HTTP/1.1 (#8305) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * bpo-22708: Upgrade HTTP CONNECT to protocol HTTP/1.1 (GH-NNNN) Use protocol HTTP/1.1 when sending HTTP CONNECT tunnelling requests; generate Host: headers if one is not already provided (required by HTTP/1.1), convert IDN domains to punycode in HTTP CONNECT requests. * Refactor tests to pass under -bb (fix ByteWarnings); missed some lines >80. * Use consistent 'tunnelling' spelling in Lib/http/client.py * Lib/test/test_httplib: Remove remnant of obsoleted test. * Use dict.copy() not copy.copy() * fix version changed * Update Lib/http/client.py Co-authored-by: bgehman * Switch to for/else: syntax, as suggested * Don't use for: else: * Sure, fine, w/e * Oops * 1nm to the left --------- Co-authored-by: Éric Co-authored-by: bgehman Co-authored-by: Oleg Iarygin --- Doc/library/http.client.rst | 12 ++ Lib/http/client.py | 25 ++- Lib/test/test_httplib.py | 147 ++++++++++++++++-- Misc/ACKS | 1 + .../2018-07-16-14-10-29.bpo-22708.592iRR.rst | 3 + 5 files changed, 167 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2018-07-16-14-10-29.bpo-22708.592iRR.rst diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index ad3416135e307b..38821b32c91cf1 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -353,6 +353,13 @@ HTTPConnection Objects The *headers* argument should be a mapping of extra HTTP headers to send with the CONNECT request. + As HTTP/1.1 is used for HTTP CONNECT tunnelling request, `as per the RFC + `_, a HTTP ``Host:`` + header must be provided, matching the authority-form of the request target + provided as the destination for the CONNECT request. If a HTTP ``Host:`` + header is not provided via the headers argument, one is generated and + transmitted automatically. + For example, to tunnel through a HTTPS proxy server running locally on port 8080, we would pass the address of the proxy to the :class:`HTTPSConnection` constructor, and the address of the host that we eventually want to reach to @@ -365,6 +372,11 @@ HTTPConnection Objects .. versionadded:: 3.2 + .. versionchanged:: 3.12 + HTTP CONNECT tunnelling requests use protocol HTTP/1.1, upgraded from + protocol HTTP/1.0. ``Host:`` HTTP headers are mandatory for HTTP/1.1, so + one will be automatically generated and transmitted if not provided in + the headers argument. .. method:: HTTPConnection.connect() diff --git a/Lib/http/client.py b/Lib/http/client.py index bd55e7d239af0c..0f5cd35247ae82 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -870,9 +870,9 @@ def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, def set_tunnel(self, host, port=None, headers=None): """Set up host and port for HTTP CONNECT tunnelling. - In a connection that uses HTTP CONNECT tunneling, the host passed to the - constructor is used as a proxy server that relays all communication to - the endpoint passed to `set_tunnel`. This done by sending an HTTP + In a connection that uses HTTP CONNECT tunnelling, the host passed to + the constructor is used as a proxy server that relays all communication + to the endpoint passed to `set_tunnel`. This done by sending an HTTP CONNECT request to the proxy server when the connection is established. This method must be called before the HTTP connection has been @@ -880,6 +880,13 @@ def set_tunnel(self, host, port=None, headers=None): The headers argument should be a mapping of extra HTTP headers to send with the CONNECT request. + + As HTTP/1.1 is used for HTTP CONNECT tunnelling request, as per the RFC + (https://tools.ietf.org/html/rfc7231#section-4.3.6), a HTTP Host: + header must be provided, matching the authority-form of the request + target provided as the destination for the CONNECT request. If a + HTTP Host: header is not provided via the headers argument, one + is generated and transmitted automatically. """ if self.sock: @@ -887,10 +894,15 @@ def set_tunnel(self, host, port=None, headers=None): self._tunnel_host, self._tunnel_port = self._get_hostport(host, port) if headers: - self._tunnel_headers = headers + self._tunnel_headers = headers.copy() else: self._tunnel_headers.clear() + if not any(header.lower() == "host" for header in self._tunnel_headers): + encoded_host = self._tunnel_host.encode("idna").decode("ascii") + self._tunnel_headers["Host"] = "%s:%d" % ( + encoded_host, self._tunnel_port) + def _get_hostport(self, host, port): if port is None: i = host.rfind(':') @@ -915,8 +927,9 @@ def set_debuglevel(self, level): self.debuglevel = level def _tunnel(self): - connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( - self._tunnel_host.encode("ascii"), self._tunnel_port) + connect = b"CONNECT %s:%d %s\r\n" % ( + self._tunnel_host.encode("idna"), self._tunnel_port, + self._http_vsn_str.encode("ascii")) headers = [connect] for header, value in self._tunnel_headers.items(): headers.append(f"{header}: {value}\r\n".encode("latin-1")) diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 9ff6afcbadec54..b4f4e2b14351a6 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -2187,11 +2187,12 @@ def test_getting_header_defaultint(self): class TunnelTests(TestCase): def setUp(self): response_text = ( - 'HTTP/1.0 200 OK\r\n\r\n' # Reply to CONNECT + 'HTTP/1.1 200 OK\r\n\r\n' # Reply to CONNECT 'HTTP/1.1 200 OK\r\n' # Reply to HEAD 'Content-Length: 42\r\n\r\n' ) self.host = 'proxy.com' + self.port = client.HTTP_PORT self.conn = client.HTTPConnection(self.host) self.conn._create_connection = self._create_connection(response_text) @@ -2203,15 +2204,45 @@ def create_connection(address, timeout=None, source_address=None): return FakeSocket(response_text, host=address[0], port=address[1]) return create_connection - def test_set_tunnel_host_port_headers(self): + def test_set_tunnel_host_port_headers_add_host_missing(self): tunnel_host = 'destination.com' tunnel_port = 8888 tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)'} + tunnel_headers_after = tunnel_headers.copy() + tunnel_headers_after['Host'] = '%s:%d' % (tunnel_host, tunnel_port) self.conn.set_tunnel(tunnel_host, port=tunnel_port, headers=tunnel_headers) self.conn.request('HEAD', '/', '') self.assertEqual(self.conn.sock.host, self.host) - self.assertEqual(self.conn.sock.port, client.HTTP_PORT) + self.assertEqual(self.conn.sock.port, self.port) + self.assertEqual(self.conn._tunnel_host, tunnel_host) + self.assertEqual(self.conn._tunnel_port, tunnel_port) + self.assertEqual(self.conn._tunnel_headers, tunnel_headers_after) + + def test_set_tunnel_host_port_headers_set_host_identical(self): + tunnel_host = 'destination.com' + tunnel_port = 8888 + tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)', + 'Host': '%s:%d' % (tunnel_host, tunnel_port)} + self.conn.set_tunnel(tunnel_host, port=tunnel_port, + headers=tunnel_headers) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertEqual(self.conn._tunnel_host, tunnel_host) + self.assertEqual(self.conn._tunnel_port, tunnel_port) + self.assertEqual(self.conn._tunnel_headers, tunnel_headers) + + def test_set_tunnel_host_port_headers_set_host_different(self): + tunnel_host = 'destination.com' + tunnel_port = 8888 + tunnel_headers = {'User-Agent': 'Mozilla/5.0 (compatible, MSIE 11)', + 'Host': '%s:%d' % ('example.com', 4200)} + self.conn.set_tunnel(tunnel_host, port=tunnel_port, + headers=tunnel_headers) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) self.assertEqual(self.conn._tunnel_host, tunnel_host) self.assertEqual(self.conn._tunnel_port, tunnel_port) self.assertEqual(self.conn._tunnel_headers, tunnel_headers) @@ -2223,17 +2254,96 @@ def test_disallow_set_tunnel_after_connect(self): 'destination.com') def test_connect_with_tunnel(self): - self.conn.set_tunnel('destination.com') + d = { + b'host': b'destination.com', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel(d[b'host'].decode('ascii')) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_with_tunnel_with_default_port(self): + d = { + b'host': b'destination.com', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel(d[b'host'].decode('ascii'), port=d[b'port']) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_with_tunnel_with_nonstandard_port(self): + d = { + b'host': b'destination.com', + b'port': 8888, + } + self.conn.set_tunnel(d[b'host'].decode('ascii'), port=d[b'port']) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s:%(port)d\r\n' % d, + self.conn.sock.data) + + # This request is not RFC-valid, but it's been possible with the library + # for years, so don't break it unexpectedly... This also tests + # case-insensitivity when injecting Host: headers if they're missing. + def test_connect_with_tunnel_with_different_host_header(self): + d = { + b'host': b'destination.com', + b'tunnel_host_header': b'example.com:9876', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel( + d[b'host'].decode('ascii'), + headers={'HOST': d[b'tunnel_host_header'].decode('ascii')}) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'HOST: %(tunnel_host_header)s\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_with_tunnel_different_host(self): + d = { + b'host': b'destination.com', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel(d[b'host'].decode('ascii')) + self.conn.request('HEAD', '/', '') + self.assertEqual(self.conn.sock.host, self.host) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'HEAD / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) + + def test_connect_with_tunnel_idna(self): + dest = '\u03b4\u03c0\u03b8.gr' + dest_port = b'%s:%d' % (dest.encode('idna'), client.HTTP_PORT) + expected = b'CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n' % ( + dest_port, dest_port) + self.conn.set_tunnel(dest) self.conn.request('HEAD', '/', '') self.assertEqual(self.conn.sock.host, self.host) self.assertEqual(self.conn.sock.port, client.HTTP_PORT) - self.assertIn(b'CONNECT destination.com', self.conn.sock.data) - # issue22095 - self.assertNotIn(b'Host: destination.com:None', self.conn.sock.data) - self.assertIn(b'Host: destination.com', self.conn.sock.data) - - # This test should be removed when CONNECT gets the HTTP/1.1 blessing - self.assertNotIn(b'Host: proxy.com', self.conn.sock.data) + self.assertIn(expected, self.conn.sock.data) def test_tunnel_connect_single_send_connection_setup(self): """Regresstion test for https://bugs.python.org/issue43332.""" @@ -2253,12 +2363,19 @@ def test_tunnel_connect_single_send_connection_setup(self): msg=f'unexpected proxy data sent {proxy_setup_data_sent!r}') def test_connect_put_request(self): - self.conn.set_tunnel('destination.com') + d = { + b'host': b'destination.com', + b'port': client.HTTP_PORT, + } + self.conn.set_tunnel(d[b'host'].decode('ascii')) self.conn.request('PUT', '/', '') self.assertEqual(self.conn.sock.host, self.host) - self.assertEqual(self.conn.sock.port, client.HTTP_PORT) - self.assertIn(b'CONNECT destination.com', self.conn.sock.data) - self.assertIn(b'Host: destination.com', self.conn.sock.data) + self.assertEqual(self.conn.sock.port, self.port) + self.assertIn(b'CONNECT %(host)s:%(port)d HTTP/1.1\r\n' + b'Host: %(host)s:%(port)d\r\n\r\n' % d, + self.conn.sock.data) + self.assertIn(b'PUT / HTTP/1.1\r\nHost: %(host)s\r\n' % d, + self.conn.sock.data) def test_tunnel_debuglog(self): expected_header = 'X-Dummy: 1' diff --git a/Misc/ACKS b/Misc/ACKS index 8cf5166a2bb1f4..929e06a87cb794 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -693,6 +693,7 @@ Anders Hammarquist Mark Hammond Harald Hanche-Olsen Manus Hand +Michael Handler Andreas Hangauer Milton L. Hankins Carl Bordum Hansen diff --git a/Misc/NEWS.d/next/Library/2018-07-16-14-10-29.bpo-22708.592iRR.rst b/Misc/NEWS.d/next/Library/2018-07-16-14-10-29.bpo-22708.592iRR.rst new file mode 100644 index 00000000000000..00bcf38bbcdf51 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2018-07-16-14-10-29.bpo-22708.592iRR.rst @@ -0,0 +1,3 @@ +http.client CONNECT method tunnel improvements: Use HTTP 1.1 protocol; send +a matching Host: header with CONNECT, if one is not provided; convert IDN +domain names to Punycode. Patch by Michael Handler. From 119f67de08f1fddc2a3f7b7caac7454cb57ef800 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Wed, 5 Apr 2023 10:22:33 +0300 Subject: [PATCH 17/76] gh-103167: Fix `-Wstrict-prototypes` warnings by using `(void)` for functions with no args (GH-103168) --- Modules/_tkinter.c | 2 +- Modules/posixmodule.c | 4 ++-- PC/launcher.c | 12 ++++++------ PC/launcher2.c | 6 +++--- Python/ceval_gil.c | 2 +- Python/initconfig.c | 4 ++-- Python/sysmodule.c | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 606e578a1f3116..20e01c79668549 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -121,7 +121,7 @@ Copyright (C) 1994 Steen Lumholt. #define WAIT_FOR_STDIN static PyObject * -_get_tcl_lib_path() +_get_tcl_lib_path(void) { static PyObject *tcl_library_path = NULL; static int already_checked = 0; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e38caf7cc0abee..dd150107e4a9de 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8546,7 +8546,7 @@ os_setpgrp_impl(PyObject *module) #include static PyObject* -win32_getppid() +win32_getppid(void) { DWORD error; PyObject* result = NULL; @@ -13330,7 +13330,7 @@ static int has_ShellExecute = -1; static HINSTANCE (CALLBACK *Py_ShellExecuteW)(HWND, LPCWSTR, LPCWSTR, LPCWSTR, LPCWSTR, INT); static int -check_ShellExecute() +check_ShellExecute(void) { HINSTANCE hShell32; diff --git a/PC/launcher.c b/PC/launcher.c index 0776e57249c427..dc265533740b67 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -449,7 +449,7 @@ locate_pythons_for_key(HKEY root, REGSAM flags) } static void -locate_store_pythons() +locate_store_pythons(void) { #if defined(_M_X64) /* 64bit process, so look in native registry */ @@ -466,7 +466,7 @@ locate_store_pythons() } static void -locate_venv_python() +locate_venv_python(void) { static wchar_t venv_python[MAX_PATH]; INSTALLED_PYTHON * ip; @@ -495,7 +495,7 @@ locate_venv_python() } static void -locate_all_pythons() +locate_all_pythons(void) { /* venv Python is highest priority */ locate_venv_python(); @@ -694,7 +694,7 @@ static wchar_t wrapped_script_path[MAX_PATH]; * valid wrapped script file. */ static void -locate_wrapped_script() +locate_wrapped_script(void) { wchar_t * p; size_t plen; @@ -1034,7 +1034,7 @@ read_config_file(wchar_t * config_path) } } -static void read_commands() +static void read_commands(void) { if (launcher_ini_path[0]) read_config_file(launcher_ini_path); @@ -1684,7 +1684,7 @@ wcsdup_pad(const wchar_t *s, int padding, int *newlen) } static wchar_t * -get_process_name() +get_process_name(void) { DWORD bufferLen = MAX_PATH; DWORD len = bufferLen; diff --git a/PC/launcher2.c b/PC/launcher2.c index 932665387f1966..bb500d4b6bfb07 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -132,7 +132,7 @@ typedef BOOL (*PIsWow64Process2)(HANDLE, USHORT*, USHORT*); USHORT -_getNativeMachine() +_getNativeMachine(void) { static USHORT _nativeMachine = IMAGE_FILE_MACHINE_UNKNOWN; if (_nativeMachine == IMAGE_FILE_MACHINE_UNKNOWN) { @@ -163,14 +163,14 @@ _getNativeMachine() bool -isAMD64Host() +isAMD64Host(void) { return _getNativeMachine() == IMAGE_FILE_MACHINE_AMD64; } bool -isARM64Host() +isARM64Host(void) { return _getNativeMachine() == IMAGE_FILE_MACHINE_ARM64; } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 749d8144bf7a23..29796be4b80e93 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -467,7 +467,7 @@ void _PyEval_SetSwitchInterval(unsigned long microseconds) gil->interval = microseconds; } -unsigned long _PyEval_GetSwitchInterval() +unsigned long _PyEval_GetSwitchInterval(void) { struct _gil_runtime_state *gil = &_PyRuntime.ceval.gil; return gil->interval; diff --git a/Python/initconfig.c b/Python/initconfig.c index db7f11e17d6662..0d42b7ea082d61 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2355,13 +2355,13 @@ config_usage(int error, const wchar_t* program) } static void -config_envvars_usage() +config_envvars_usage(void) { printf(usage_envvars, (wint_t)DELIM, (wint_t)DELIM, PYTHONHOMEHELP); } static void -config_xoptions_usage() +config_xoptions_usage(void) { puts(usage_xoptions); } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 4afb0f1d0b5ed2..f1a294de598420 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1488,7 +1488,7 @@ static PyStructSequence_Desc windows_version_desc = { }; static PyObject * -_sys_getwindowsversion_from_kernel32() +_sys_getwindowsversion_from_kernel32(void) { #ifndef MS_WINDOWS_DESKTOP return NULL; From 264c00a1c512a9bd58f47c80e72e436c639763c8 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 5 Apr 2023 08:27:01 +0100 Subject: [PATCH 18/76] gh-103193: Micro-optimise helper functions for `inspect.getattr_static` (#103195) --- Lib/inspect.py | 7 +++---- .../Library/2023-04-02-17-51-08.gh-issue-103193.xrZbM1.rst | 2 ++ 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-02-17-51-08.gh-issue-103193.xrZbM1.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 0eceaaf9a24f5d..8739c9c2572643 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1772,9 +1772,9 @@ def trace(context=1): # ------------------------------------------------ static version of getattr _sentinel = object() +_static_getmro = type.__dict__['__mro__'].__get__ +_get_dunder_dict_of_class = type.__dict__["__dict__"].__get__ -def _static_getmro(klass): - return type.__dict__['__mro__'].__get__(klass) def _check_instance(obj, attr): instance_dict = {} @@ -1802,10 +1802,9 @@ def _is_type(obj): return True def _shadowed_dict(klass): - dict_attr = type.__dict__["__dict__"] for entry in _static_getmro(klass): try: - class_dict = dict_attr.__get__(entry)["__dict__"] + class_dict = _get_dunder_dict_of_class(entry)["__dict__"] except KeyError: pass else: diff --git a/Misc/NEWS.d/next/Library/2023-04-02-17-51-08.gh-issue-103193.xrZbM1.rst b/Misc/NEWS.d/next/Library/2023-04-02-17-51-08.gh-issue-103193.xrZbM1.rst new file mode 100644 index 00000000000000..f0b76a605a5610 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-02-17-51-08.gh-issue-103193.xrZbM1.rst @@ -0,0 +1,2 @@ +Improve performance of :func:`inspect.getattr_static`. Patch by Alex +Waygood. From 47753ecde21b79b5c5f11d883946fda2a340e427 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 5 Apr 2023 10:07:30 +0100 Subject: [PATCH 19/76] gh-74690: typing: Simplify and optimise `_ProtocolMeta.__instancecheck__` (#103159) --- Lib/typing.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index 442d684d14c928..f50e9b0a93b061 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2024,20 +2024,12 @@ def __instancecheck__(cls, instance): raise TypeError("Instance and class checks can only be used with" " @runtime_checkable protocols") - if not is_protocol_cls and issubclass(instance.__class__, cls): - return True - - protocol_attrs = _get_protocol_attrs(cls) - - if ( - _is_callable_members_only(cls, protocol_attrs) - and issubclass(instance.__class__, cls) - ): + if super().__instancecheck__(instance): return True if is_protocol_cls: getattr_static = _lazy_load_getattr_static() - for attr in protocol_attrs: + for attr in _get_protocol_attrs(cls): try: val = getattr_static(instance, attr) except AttributeError: @@ -2047,7 +2039,7 @@ def __instancecheck__(cls, instance): else: return True - return super().__instancecheck__(instance) + return False class Protocol(Generic, metaclass=_ProtocolMeta): From c396b6ddf3da784349bac9ebf7f28c55bde016ea Mon Sep 17 00:00:00 2001 From: "C.A.M. Gerlach" Date: Wed, 5 Apr 2023 06:16:36 -0500 Subject: [PATCH 20/76] gh-81762: Clarify and simplify description of print's flush param (#103264) --- Doc/library/functions.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index f0f374771b0cf1..8797485cd05d83 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1444,8 +1444,9 @@ are always available. They are listed here in alphabetical order. arguments are converted to text strings, :func:`print` cannot be used with binary mode file objects. For these, use ``file.write(...)`` instead. - Whether the output is buffered is usually determined by *file*, but if the - *flush* keyword argument is true, the stream is forcibly flushed. + Output buffering is usually determined by *file*. + However, if *flush* is true, the stream is forcibly flushed. + .. versionchanged:: 3.3 Added the *flush* keyword argument. From 3246688918a428738b61c4adb5fbc6525eae96f9 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 5 Apr 2023 12:19:03 +0100 Subject: [PATCH 21/76] gh-74690: typing: Call `_get_protocol_attrs` and `_callable_members_only` at protocol class creation time, not during `isinstance()` checks (#103160) --- Lib/typing.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index f50e9b0a93b061..b8420f619a1d05 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1905,7 +1905,8 @@ class _TypingEllipsis: _TYPING_INTERNALS = frozenset({ '__parameters__', '__orig_bases__', '__orig_class__', - '_is_protocol', '_is_runtime_protocol' + '_is_protocol', '_is_runtime_protocol', '__protocol_attrs__', + '__callable_proto_members_only__', }) _SPECIAL_NAMES = frozenset({ @@ -1935,11 +1936,6 @@ def _get_protocol_attrs(cls): return attrs -def _is_callable_members_only(cls, protocol_attrs): - # PEP 544 prohibits using issubclass() with protocols that have non-method members. - return all(callable(getattr(cls, attr, None)) for attr in protocol_attrs) - - def _no_init_or_replace_init(self, *args, **kwargs): cls = type(self) @@ -2012,6 +2008,15 @@ def _lazy_load_getattr_static(): class _ProtocolMeta(ABCMeta): # This metaclass is really unfortunate and exists only because of # the lack of __instancehook__. + def __init__(cls, *args, **kwargs): + super().__init__(*args, **kwargs) + cls.__protocol_attrs__ = _get_protocol_attrs(cls) + # PEP 544 prohibits using issubclass() + # with protocols that have non-method members. + cls.__callable_proto_members_only__ = all( + callable(getattr(cls, attr, None)) for attr in cls.__protocol_attrs__ + ) + def __instancecheck__(cls, instance): # We need this method for situations where attributes are # assigned in __init__. @@ -2029,7 +2034,7 @@ def __instancecheck__(cls, instance): if is_protocol_cls: getattr_static = _lazy_load_getattr_static() - for attr in _get_protocol_attrs(cls): + for attr in cls.__protocol_attrs__: try: val = getattr_static(instance, attr) except AttributeError: @@ -2095,9 +2100,7 @@ def _proto_hook(other): raise TypeError("Instance and class checks can only be used with" " @runtime_checkable protocols") - protocol_attrs = _get_protocol_attrs(cls) - - if not _is_callable_members_only(cls, protocol_attrs): + if not cls.__callable_proto_members_only__ : if _allow_reckless_class_checks(): return NotImplemented raise TypeError("Protocols with non-method members" @@ -2107,7 +2110,7 @@ def _proto_hook(other): raise TypeError('issubclass() arg 1 must be a class') # Second, perform the actual structural compatibility check. - for attr in protocol_attrs: + for attr in cls.__protocol_attrs__: for base in other.__mro__: # Check if the members appears in the class dictionary... if attr in base.__dict__: From a28d4edb23b7150942f1eceb9e97c6f53aa4de42 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Wed, 5 Apr 2023 14:43:26 +0300 Subject: [PATCH 22/76] gh-100408: Fix a traceback in multiprocessing example (#100409) --- Doc/library/multiprocessing.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 0ec47bb956a99e..8454296b815b41 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -460,16 +460,16 @@ process which created it. ... return x*x ... >>> with p: - ... p.map(f, [1,2,3]) + ... p.map(f, [1,2,3]) Process PoolWorker-1: Process PoolWorker-2: Process PoolWorker-3: Traceback (most recent call last): Traceback (most recent call last): Traceback (most recent call last): - AttributeError: 'module' object has no attribute 'f' - AttributeError: 'module' object has no attribute 'f' - AttributeError: 'module' object has no attribute 'f' + AttributeError: Can't get attribute 'f' on )> + AttributeError: Can't get attribute 'f' on )> + AttributeError: Can't get attribute 'f' on )> (If you try this it will actually output three full tracebacks interleaved in a semi-random fashion, and then you may have to From 8f70b16e3397ad32757ddbabd5180cbef0036a4b Mon Sep 17 00:00:00 2001 From: Yeojin Kim Date: Wed, 5 Apr 2023 20:54:48 +0900 Subject: [PATCH 23/76] gh-86094: Add support for Unicode Path Extra Field in ZipFile (gh-102566) --- Lib/test/test_zipfile/test_core.py | 27 ++++++++++ Lib/zipfile/__init__.py | 50 ++++++++++++++----- Misc/ACKS | 1 + ...3-03-10-04-59-35.gh-issue-86094.zOYdy8.rst | 2 + 4 files changed, 67 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2023-03-10-04-59-35.gh-issue-86094.zOYdy8.rst diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index e23f5c2a8556f2..73c6b0185a1a0e 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -1616,6 +1616,33 @@ def test_write_unicode_filenames(self): self.assertEqual(zf.filelist[0].filename, "foo.txt") self.assertEqual(zf.filelist[1].filename, "\xf6.txt") + @requires_zlib() + def test_read_zipfile_containing_unicode_path_extra_field(self): + with zipfile.ZipFile(TESTFN, mode='w') as zf: + # create a file with a non-ASCII name + filename = '이름.txt' + filename_encoded = filename.encode('utf-8') + + # create a ZipInfo object with Unicode path extra field + zip_info = zipfile.ZipInfo(filename) + + tag_for_unicode_path = b'\x75\x70' + version_of_unicode_path = b'\x01' + + import zlib + filename_crc = struct.pack('= 0: + filename = filename[0:null_byte] + # This is used to ensure paths in generated ZIP files always use + # forward slashes as the directory separator, as required by the + # ZIP format specification. + if os.sep != "/" and os.sep in filename: + filename = filename.replace(os.sep, "/") + return filename + class ZipInfo (object): """Class with attributes describing each file in the ZIP archive.""" @@ -368,16 +384,9 @@ class ZipInfo (object): def __init__(self, filename="NoName", date_time=(1980,1,1,0,0,0)): self.orig_filename = filename # Original file name in archive - # Terminate the file name at the first null byte. Null bytes in file - # names are used as tricks by viruses in archives. - null_byte = filename.find(chr(0)) - if null_byte >= 0: - filename = filename[0:null_byte] - # This is used to ensure paths in generated ZIP files always use - # forward slashes as the directory separator, as required by the - # ZIP format specification. - if os.sep != "/" and os.sep in filename: - filename = filename.replace(os.sep, "/") + # Terminate the file name at the first null byte and + # ensure paths always use forward slashes as the directory separator. + filename = _sanitize_filename(filename) self.filename = filename # Normalized file name self.date_time = date_time # year, month, day, hour, min, sec @@ -482,7 +491,7 @@ def _encodeFilenameFlags(self): except UnicodeEncodeError: return self.filename.encode('utf-8'), self.flag_bits | _MASK_UTF_FILENAME - def _decodeExtra(self): + def _decodeExtra(self, filename_crc): # Try to decode the extra field. extra = self.extra unpack = struct.unpack @@ -508,6 +517,21 @@ def _decodeExtra(self): except struct.error: raise BadZipFile(f"Corrupt zip64 extra field. " f"{field} not found.") from None + elif tp == 0x7075: + data = extra[4:ln+4] + # Unicode Path Extra Field + try: + up_version, up_name_crc = unpack(' 2: print(centdir) filename = fp.read(centdir[_CD_FILENAME_LENGTH]) + orig_filename_crc = crc32(filename) flags = centdir[_CD_FLAG_BITS] if flags & _MASK_UTF_FILENAME: # UTF-8 file names extension @@ -1432,8 +1457,7 @@ def _RealGetContents(self): x._raw_time = t x.date_time = ( (d>>9)+1980, (d>>5)&0xF, d&0x1F, t>>11, (t>>5)&0x3F, (t&0x1F) * 2 ) - - x._decodeExtra() + x._decodeExtra(orig_filename_crc) x.header_offset = x.header_offset + concat self.filelist.append(x) self.NameToInfo[x.filename] = x diff --git a/Misc/ACKS b/Misc/ACKS index 929e06a87cb794..49f3692dfd6b8f 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -627,6 +627,7 @@ Julian Gindi Yannick Gingras Neil Girdhar Matt Giuca +Andrea Giudiceandrea Franz Glasner Wim Glenn Michael Goderbauer diff --git a/Misc/NEWS.d/next/Documentation/2023-03-10-04-59-35.gh-issue-86094.zOYdy8.rst b/Misc/NEWS.d/next/Documentation/2023-03-10-04-59-35.gh-issue-86094.zOYdy8.rst new file mode 100644 index 00000000000000..39461f3f84c9ac --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-03-10-04-59-35.gh-issue-86094.zOYdy8.rst @@ -0,0 +1,2 @@ +Add support for Unicode Path Extra Field in ZipFile. Patch by Yeojin Kim +and Andrea Giudiceandrea From 5e7c468fc4ceb801c30844b794e04a2ac6a35743 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 5 Apr 2023 15:52:38 +0100 Subject: [PATCH 24/76] gh-89058: remove skip from test_no_hang_on_context_chain_cycle2 (#102903) --- Lib/test/test_exceptions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 284f26be717794..74a5884264d577 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1135,7 +1135,6 @@ def cycle(): self.assertIsInstance(exc.__context__, ValueError) self.assertIs(exc.__context__.__context__, exc.__context__) - @unittest.skip("See issue 44895") def test_no_hang_on_context_chain_cycle2(self): # See issue 25782. Cycle at head of context chain. From fdd0fff277a55c010a4da0a7af0e986e38560545 Mon Sep 17 00:00:00 2001 From: Olivier Gayot Date: Wed, 5 Apr 2023 16:54:43 +0200 Subject: [PATCH 25/76] gh-102899: Fix doc link for getting filesystem error handler (#102901) --- Doc/library/sys.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index b3b9b5e74ac068..00721efd1cf65e 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -697,7 +697,7 @@ always available. the encoding used with the :term:`filesystem error handler ` to convert between Unicode filenames and bytes filenames. The filesystem error handler is returned from - :func:`getfilesystemencoding`. + :func:`getfilesystemencodeerrors`. For best compatibility, str should be used for filenames in all cases, although representing filenames as bytes is also supported. Functions From 96e1901a59ed3bb6188743d60395666969a3ba42 Mon Sep 17 00:00:00 2001 From: Joshua Root Date: Thu, 6 Apr 2023 01:09:19 +1000 Subject: [PATCH 26/76] gh-99069: Consolidate checks for static_assert (#94766) Several platforms don't define the static_assert macro despite having compiler support for the _Static_assert keyword. The macro needs to be defined since it is used unconditionally in the Python code. So it should always be safe to define it if undefined and not in C++11 (or later) mode. Hence, remove the checks for particular platforms or libc versions, and just define static_assert anytime it needs to be defined but isn't. That way, all platforms that need the fix will get it, regardless of whether someone specifically thought of them. Also document that certain macOS versions are among the platforms that need this. The C2x draft (currently expected to become C23) makes static_assert a keyword to match C++. So only define the macro for up to C17. Co-authored-by: Victor Stinner --- Include/pymacro.h | 29 ++++++++++--------- ...3-02-11-05-31-05.gh-issue-99069.X4LDvY.rst | 1 + 2 files changed, 17 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-02-11-05-31-05.gh-issue-99069.X4LDvY.rst diff --git a/Include/pymacro.h b/Include/pymacro.h index e37cda44c5ebf1..342d2a7b844adf 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -3,20 +3,23 @@ // gh-91782: On FreeBSD 12, if the _POSIX_C_SOURCE and _XOPEN_SOURCE macros are // defined, disables C11 support and does not define -// the static_assert() macro. Define the static_assert() macro in Python until -// suports C11: +// the static_assert() macro. // https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=255290 -#if defined(__FreeBSD__) && !defined(static_assert) -# define static_assert _Static_assert -#endif - -// static_assert is defined in glibc from version 2.16. Before it requires -// compiler support (gcc >= 4.6) and is called _Static_assert. -// In C++ 11 static_assert is a keyword, redefining is undefined behaviour. -#if (defined(__GLIBC__) \ - && (__GLIBC__ < 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ <= 16)) \ - && !(defined(__cplusplus) && __cplusplus >= 201103L) \ - && !defined(static_assert)) +// +// macOS <= 10.10 doesn't define static_assert in assert.h at all despite +// having C11 compiler support. +// +// static_assert is defined in glibc from version 2.16. Compiler support for +// the C11 _Static_assert keyword is in gcc >= 4.6. +// +// MSVC makes static_assert a keyword in C11-17, contrary to the standards. +// +// In C++11 and C2x, static_assert is a keyword, redefining is undefined +// behaviour. So only define if building as C (if __STDC_VERSION__ is defined), +// not C++, and only for C11-17. +#if !defined(static_assert) && (defined(__GNUC__) || defined(__clang__)) \ + && defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \ + && __STDC_VERSION__ <= 201710L # define static_assert _Static_assert #endif diff --git a/Misc/NEWS.d/next/Build/2023-02-11-05-31-05.gh-issue-99069.X4LDvY.rst b/Misc/NEWS.d/next/Build/2023-02-11-05-31-05.gh-issue-99069.X4LDvY.rst new file mode 100644 index 00000000000000..ae9b4d59ca8cec --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-02-11-05-31-05.gh-issue-99069.X4LDvY.rst @@ -0,0 +1 @@ +Extended workaround defining ``static_assert`` when missing from the libc headers to all clang and gcc builds. In particular, this fixes building on macOS <= 10.10. From de182676853e8de22363e8a0641c42392c0fdaa9 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Wed, 5 Apr 2023 17:37:36 +0100 Subject: [PATCH 27/76] gh-74690: Further optimise `typing._ProtocolMeta.__instancecheck__` (#103280) --- Lib/typing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/typing.py b/Lib/typing.py index b8420f619a1d05..1f1c4ffa2566ab 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2039,7 +2039,7 @@ def __instancecheck__(cls, instance): val = getattr_static(instance, attr) except AttributeError: break - if callable(getattr(cls, attr, None)) and val is None: + if val is None and callable(getattr(cls, attr, None)): break else: return True From aa5a9b5eb767fd849ea09d0842e33691e690de7c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 5 Apr 2023 15:13:12 -0600 Subject: [PATCH 28/76] gh-101659: Use the Raw Allocator in the _xxinterpchannels Module (gh-103287) Using the raw allocator for any of the global state makes sense, especially as we move to a per-interpreter obmalloc state (gh-101660). --- Modules/_xxinterpchannelsmodule.c | 115 ++++++++++++++++++++++++------ 1 file changed, 93 insertions(+), 22 deletions(-) diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index fead12c963da26..ef1cdcab952271 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -10,9 +10,77 @@ #include "pycore_interpreteridobject.h" +/* +This module has the following process-global state: + +_globals (static struct globals): + module_count (int) + channels (struct _channels): + numopen (int64_t) + next_id; (int64_t) + mutex (PyThread_type_lock) + head (linked list of struct _channelref *): + id (int64_t) + objcount (Py_ssize_t) + next (struct _channelref *): + ... + chan (struct _channel *): + open (int) + mutex (PyThread_type_lock) + closing (struct _channel_closing *): + ref (struct _channelref *): + ... + ends (struct _channelends *): + numsendopen (int64_t) + numrecvopen (int64_t) + send (struct _channelend *): + interp (int64_t) + open (int) + next (struct _channelend *) + recv (struct _channelend *): + ... + queue (struct _channelqueue *): + count (int64_t) + first (struct _channelitem *): + next (struct _channelitem *): + ... + data (_PyCrossInterpreterData *): + data (void *) + obj (PyObject *) + interp (int64_t) + new_object (xid_newobjectfunc) + free (xid_freefunc) + last (struct _channelitem *): + ... + +The above state includes the following allocations by the module: + +* 1 top-level mutex (to protect the rest of the state) +* for each channel: + * 1 struct _channelref + * 1 struct _channel + * 0-1 struct _channel_closing + * 1 struct _channelends + * 2 struct _channelend + * 1 struct _channelqueue +* for each item in each channel: + * 1 struct _channelitem + * 1 _PyCrossInterpreterData + +The only objects in that global state are the references held by each +channel's queue, which are safely managed via the _PyCrossInterpreterData_*() +API.. The module does not create any objects that are shared globally. +*/ + #define MODULE_NAME "_xxinterpchannels" +#define GLOBAL_MALLOC(TYPE) \ + PyMem_RawMalloc(sizeof(TYPE)) +#define GLOBAL_FREE(VAR) \ + PyMem_RawFree(VAR) + + static PyInterpreterState * _get_current_interp(void) { @@ -301,7 +369,7 @@ typedef struct _channelitem { static _channelitem * _channelitem_new(void) { - _channelitem *item = PyMem_NEW(_channelitem, 1); + _channelitem *item = GLOBAL_MALLOC(_channelitem); if (item == NULL) { PyErr_NoMemory(); return NULL; @@ -316,7 +384,8 @@ _channelitem_clear(_channelitem *item) { if (item->data != NULL) { (void)_release_xid_data(item->data, 1); - PyMem_Free(item->data); + // It was allocated in _channel_send(). + GLOBAL_FREE(item->data); item->data = NULL; } item->next = NULL; @@ -326,7 +395,7 @@ static void _channelitem_free(_channelitem *item) { _channelitem_clear(item); - PyMem_Free(item); + GLOBAL_FREE(item); } static void @@ -357,7 +426,7 @@ typedef struct _channelqueue { static _channelqueue * _channelqueue_new(void) { - _channelqueue *queue = PyMem_NEW(_channelqueue, 1); + _channelqueue *queue = GLOBAL_MALLOC(_channelqueue); if (queue == NULL) { PyErr_NoMemory(); return NULL; @@ -381,7 +450,7 @@ static void _channelqueue_free(_channelqueue *queue) { _channelqueue_clear(queue); - PyMem_Free(queue); + GLOBAL_FREE(queue); } static int @@ -433,7 +502,7 @@ typedef struct _channelend { static _channelend * _channelend_new(int64_t interp) { - _channelend *end = PyMem_NEW(_channelend, 1); + _channelend *end = GLOBAL_MALLOC(_channelend); if (end == NULL) { PyErr_NoMemory(); return NULL; @@ -447,7 +516,7 @@ _channelend_new(int64_t interp) static void _channelend_free(_channelend *end) { - PyMem_Free(end); + GLOBAL_FREE(end); } static void @@ -492,7 +561,7 @@ typedef struct _channelassociations { static _channelends * _channelends_new(void) { - _channelends *ends = PyMem_NEW(_channelends, 1); + _channelends *ends = GLOBAL_MALLOC(_channelends); if (ends== NULL) { return NULL; } @@ -519,7 +588,7 @@ static void _channelends_free(_channelends *ends) { _channelends_clear(ends); - PyMem_Free(ends); + GLOBAL_FREE(ends); } static _channelend * @@ -660,20 +729,20 @@ typedef struct _channel { static _PyChannelState * _channel_new(PyThread_type_lock mutex) { - _PyChannelState *chan = PyMem_NEW(_PyChannelState, 1); + _PyChannelState *chan = GLOBAL_MALLOC(_PyChannelState); if (chan == NULL) { return NULL; } chan->mutex = mutex; chan->queue = _channelqueue_new(); if (chan->queue == NULL) { - PyMem_Free(chan); + GLOBAL_FREE(chan); return NULL; } chan->ends = _channelends_new(); if (chan->ends == NULL) { _channelqueue_free(chan->queue); - PyMem_Free(chan); + GLOBAL_FREE(chan); return NULL; } chan->open = 1; @@ -691,7 +760,7 @@ _channel_free(_PyChannelState *chan) PyThread_release_lock(chan->mutex); PyThread_free_lock(chan->mutex); - PyMem_Free(chan); + GLOBAL_FREE(chan); } static int @@ -814,7 +883,7 @@ typedef struct _channelref { static _channelref * _channelref_new(int64_t id, _PyChannelState *chan) { - _channelref *ref = PyMem_NEW(_channelref, 1); + _channelref *ref = GLOBAL_MALLOC(_channelref); if (ref == NULL) { return NULL; } @@ -841,7 +910,7 @@ _channelref_free(_channelref *ref) _channel_clear_closing(ref->chan); } //_channelref_clear(ref); - PyMem_Free(ref); + GLOBAL_FREE(ref); } static _channelref * @@ -1163,7 +1232,7 @@ _channel_set_closing(struct _channelref *ref, PyThread_type_lock mutex) { res = ERR_CHANNEL_CLOSED; goto done; } - chan->closing = PyMem_NEW(struct _channel_closing, 1); + chan->closing = GLOBAL_MALLOC(struct _channel_closing); if (chan->closing == NULL) { goto done; } @@ -1179,7 +1248,7 @@ static void _channel_clear_closing(struct _channel *chan) { PyThread_acquire_lock(chan->mutex, WAIT_LOCK); if (chan->closing != NULL) { - PyMem_Free(chan->closing); + GLOBAL_FREE(chan->closing); chan->closing = NULL; } PyThread_release_lock(chan->mutex); @@ -1257,14 +1326,14 @@ _channel_send(_channels *channels, int64_t id, PyObject *obj) } // Convert the object to cross-interpreter data. - _PyCrossInterpreterData *data = PyMem_NEW(_PyCrossInterpreterData, 1); + _PyCrossInterpreterData *data = GLOBAL_MALLOC(_PyCrossInterpreterData); if (data == NULL) { PyThread_release_lock(mutex); return -1; } if (_PyObject_GetCrossInterpreterData(obj, data) != 0) { PyThread_release_lock(mutex); - PyMem_Free(data); + GLOBAL_FREE(data); return -1; } @@ -1274,7 +1343,7 @@ _channel_send(_channels *channels, int64_t id, PyObject *obj) if (res != 0) { // We may chain an exception here: (void)_release_xid_data(data, 0); - PyMem_Free(data); + GLOBAL_FREE(data); return res; } @@ -1323,11 +1392,13 @@ _channel_recv(_channels *channels, int64_t id, PyObject **res) if (obj == NULL) { assert(PyErr_Occurred()); (void)_release_xid_data(data, 1); - PyMem_Free(data); + // It was allocated in _channel_send(). + GLOBAL_FREE(data); return -1; } int release_res = _release_xid_data(data, 0); - PyMem_Free(data); + // It was allocated in _channel_send(). + GLOBAL_FREE(data); if (release_res < 0) { // The source interpreter has been destroyed already. assert(PyErr_Occurred()); From b4978ff872be5102117b4e25d93dbbb4e04c8292 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 5 Apr 2023 14:15:49 -0700 Subject: [PATCH 29/76] GH-88691: Shrink the CALL caches (GH-103230) --- Doc/library/dis.rst | 2 +- Include/internal/pycore_code.h | 1 - Include/internal/pycore_opcode.h | 2 +- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 1 - Lib/test/test_dis.py | 258 +++++++++--------- ...3-03-25-23-24-38.gh-issue-88691.2SWBd1.rst | 1 + Programs/test_frozenmain.h | 66 ++--- Python/bytecodes.c | 43 +-- Python/generated_cases.c.h | 146 +++++----- Python/opcode_metadata.h | 38 +-- Python/specialize.c | 5 - 12 files changed, 285 insertions(+), 281 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-25-23-24-38.gh-issue-88691.2SWBd1.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 8703cddb3448cc..19c08b225ef2bc 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -59,7 +59,7 @@ the following command can be used to display the disassembly of 3 2 LOAD_GLOBAL 1 (NULL + len) 12 LOAD_FAST 0 (alist) 14 CALL 1 - 24 RETURN_VALUE + 22 RETURN_VALUE (The "2" is a line number). diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 7f3148a38ff13e..faf1be585e17ef 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -73,7 +73,6 @@ typedef struct { typedef struct { uint16_t counter; uint16_t func_version[2]; - uint16_t min_args; } _PyCallCache; #define INLINE_CACHE_ENTRIES_CALL CACHE_ENTRIES(_PyCallCache) diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 58f7da51aebd5e..81ca91465950b7 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -55,7 +55,7 @@ const uint8_t _PyOpcode_Caches[256] = { [LOAD_GLOBAL] = 4, [BINARY_OP] = 1, [SEND] = 1, - [CALL] = 4, + [CALL] = 3, }; const uint8_t _PyOpcode_Deopt[256] = { diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 9c4c5f907744cd..de6c434450fc82 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -438,6 +438,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a7 3522 (Removed JUMP_IF_FALSE_OR_POP/JUMP_IF_TRUE_OR_POP) # Python 3.12a7 3523 (Convert COMPARE_AND_BRANCH back to COMPARE_OP) # Python 3.12a7 3524 (Shrink the BINARY_SUBSCR caches) +# Python 3.12b1 3525 (Shrink the CALL caches) # Python 3.13 will start with 3550 @@ -454,7 +455,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3524).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3525).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 16a61dff47b5b3..b62dfa1bcb42c5 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -410,7 +410,6 @@ def pseudo_op(name, op, real_ops): "CALL": { "counter": 1, "func_version": 2, - "min_args": 1, }, "STORE_SUBSCR": { "counter": 1, diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 7e501045662f4c..4a2144743f6567 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -138,10 +138,10 @@ def bug708901(): %3d CALL 2 GET_ITER - >> FOR_ITER 2 (to 36) + >> FOR_ITER 2 (to 34) STORE_FAST 0 (res) -%3d JUMP_BACKWARD 4 (to 28) +%3d JUMP_BACKWARD 4 (to 26) %3d >> END_FOR RETURN_CONST 0 (None) @@ -437,7 +437,7 @@ def _with(c): %3d >> PUSH_EXC_INFO WITH_EXCEPT_START - POP_JUMP_IF_TRUE 1 (to 44) + POP_JUMP_IF_TRUE 1 (to 42) RERAISE 2 >> POP_TOP POP_EXCEPT @@ -492,10 +492,10 @@ async def _asyncwith(c): CALL 2 GET_AWAITABLE 2 LOAD_CONST 0 (None) - >> SEND 3 (to 64) + >> SEND 3 (to 62) YIELD_VALUE 2 RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 54) + JUMP_BACKWARD_NO_INTERRUPT 5 (to 52) >> POP_TOP POP_TOP @@ -504,21 +504,21 @@ async def _asyncwith(c): RETURN_CONST 0 (None) %3d >> CLEANUP_THROW - JUMP_BACKWARD 27 (to 24) + JUMP_BACKWARD 26 (to 24) >> CLEANUP_THROW - JUMP_BACKWARD 9 (to 64) + JUMP_BACKWARD 9 (to 62) >> PUSH_EXC_INFO WITH_EXCEPT_START GET_AWAITABLE 2 LOAD_CONST 0 (None) - >> SEND 4 (to 102) + >> SEND 4 (to 100) YIELD_VALUE 3 RESUME 3 - JUMP_BACKWARD_NO_INTERRUPT 5 (to 90) + JUMP_BACKWARD_NO_INTERRUPT 5 (to 88) >> CLEANUP_THROW >> SWAP 2 POP_TOP - POP_JUMP_IF_TRUE 1 (to 110) + POP_JUMP_IF_TRUE 1 (to 108) RERAISE 2 >> POP_TOP POP_EXCEPT @@ -730,14 +730,14 @@ def loop_test(): LOAD_CONST 2 (3) BINARY_OP 5 (*) GET_ITER - >> FOR_ITER_LIST 14 (to 48) + >> FOR_ITER_LIST 13 (to 46) STORE_FAST 0 (i) %3d LOAD_GLOBAL_MODULE 1 (NULL + load_test) LOAD_FAST 0 (i) CALL_PY_WITH_DEFAULTS 1 POP_TOP - JUMP_BACKWARD 16 (to 16) + JUMP_BACKWARD 15 (to 16) %3d >> END_FOR RETURN_CONST 0 (None) @@ -1193,7 +1193,7 @@ def test_show_caches(self): caches = list(self.get_cached_values(quickened, adaptive)) for cache in caches: self.assertRegex(cache, pattern) - total_caches = 22 + total_caches = 20 empty_caches = 7 self.assertEqual(caches.count(""), empty_caches) self.assertEqual(len(caches), total_caches) @@ -1524,9 +1524,9 @@ def _prepare_test_cases(): Instruction(opname='BUILD_MAP', opcode=105, arg=0, argval=0, argrepr='', offset=40, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='Hello world!', argrepr="'Hello world!'", offset=42, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=7, argval=7, argrepr='', offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=56, starts_line=8, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=52, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='f', argrepr='f', offset=54, starts_line=8, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=56, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_f = [ @@ -1549,9 +1549,9 @@ def _prepare_test_cases(): Instruction(opname='LOAD_DEREF', opcode=137, arg=0, argval='c', argrepr='c', offset=40, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_DEREF', opcode=137, arg=1, argval='d', argrepr='d', offset=42, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=4, argval=4, argrepr='', offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=56, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=58, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=52, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=2, argval='inner', argrepr='inner', offset=54, starts_line=6, is_jump_target=False, positions=None), + Instruction(opname='RETURN_VALUE', opcode=83, arg=None, argval=None, argrepr='', offset=56, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_inner = [ @@ -1565,8 +1565,8 @@ def _prepare_test_cases(): Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='e', argrepr='e', offset=22, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='LOAD_FAST', opcode=124, arg=1, argval='f', argrepr='f', offset=24, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=6, argval=6, argrepr='', offset=26, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=36, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=38, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=34, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=36, starts_line=None, is_jump_target=False, positions=None), ] expected_opinfo_jumpy = [ @@ -1574,115 +1574,115 @@ def _prepare_test_cases(): Instruction(opname='LOAD_GLOBAL', opcode=116, arg=1, argval='range', argrepr='NULL + range', offset=2, starts_line=3, is_jump_target=False, positions=None), Instruction(opname='LOAD_CONST', opcode=100, arg=1, argval=10, argrepr='10', offset=12, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=14, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=24, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='FOR_ITER', opcode=93, arg=27, argval=84, argrepr='to 84', offset=26, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=30, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=32, starts_line=4, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=42, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=44, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=54, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=56, starts_line=5, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=58, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=2, argval='<', argrepr='<', offset=60, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=68, argrepr='to 68', offset=64, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=21, argval=26, argrepr='to 26', offset=66, starts_line=6, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=68, starts_line=7, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=70, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=68, argval='>', argrepr='>', offset=72, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=80, argrepr='to 80', offset=76, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=27, argval=26, argrepr='to 26', offset=78, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=80, starts_line=8, is_jump_target=True, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=13, argval=110, argrepr='to 110', offset=82, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='END_FOR', opcode=4, arg=None, argval=None, argrepr='', offset=84, starts_line=3, is_jump_target=True, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=86, starts_line=10, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=96, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=98, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=108, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=110, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=32, argval=178, argrepr='to 178', offset=112, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=114, starts_line=12, is_jump_target=True, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=124, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=126, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=136, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=138, starts_line=13, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=140, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=122, arg=23, argval=23, argrepr='-=', offset=142, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=146, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=148, starts_line=14, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=150, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=68, argval='>', argrepr='>', offset=152, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=160, argrepr='to 160', offset=156, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=25, argval=110, argrepr='to 110', offset=158, starts_line=15, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=160, starts_line=16, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=162, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COMPARE_OP', opcode=107, arg=2, argval='<', argrepr='<', offset=164, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=172, argrepr='to 172', offset=168, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_FORWARD', opcode=110, arg=15, argval=202, argrepr='to 202', offset=170, starts_line=17, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=172, starts_line=11, is_jump_target=True, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=178, argrepr='to 178', offset=174, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=32, argval=114, argrepr='to 114', offset=176, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=178, starts_line=19, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=188, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=190, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=200, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='NOP', opcode=9, arg=None, argval=None, argrepr='', offset=202, starts_line=20, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=204, starts_line=21, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=206, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='BINARY_OP', opcode=122, arg=11, argval=11, argrepr='/', offset=208, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=212, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=214, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='BEFORE_WITH', opcode=53, arg=None, argval=None, argrepr='', offset=216, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=218, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=220, starts_line=26, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=230, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=232, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=242, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=244, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=246, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=248, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=2, argval=2, argrepr='', offset=250, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=260, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=262, starts_line=28, is_jump_target=True, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=272, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=274, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='GET_ITER', opcode=68, arg=None, argval=None, argrepr='', offset=22, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='FOR_ITER', opcode=93, arg=26, argval=80, argrepr='to 80', offset=24, starts_line=None, is_jump_target=True, positions=None), + Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=28, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=30, starts_line=4, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=40, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=42, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=50, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=52, starts_line=5, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=54, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=107, arg=2, argval='<', argrepr='<', offset=56, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=64, argrepr='to 64', offset=60, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=20, argval=24, argrepr='to 24', offset=62, starts_line=6, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=64, starts_line=7, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=66, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=107, arg=68, argval='>', argrepr='>', offset=68, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=76, argrepr='to 76', offset=72, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=26, argval=24, argrepr='to 24', offset=74, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=76, starts_line=8, is_jump_target=True, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=12, argval=104, argrepr='to 104', offset=78, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='END_FOR', opcode=4, arg=None, argval=None, argrepr='', offset=80, starts_line=3, is_jump_target=True, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=82, starts_line=10, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=4, argval='I can haz else clause?', argrepr="'I can haz else clause?'", offset=92, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=94, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=102, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST_CHECK', opcode=127, arg=0, argval='i', argrepr='i', offset=104, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=31, argval=170, argrepr='to 170', offset=106, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=108, starts_line=12, is_jump_target=True, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=118, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=120, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=128, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=130, starts_line=13, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=132, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=122, arg=23, argval=23, argrepr='-=', offset=134, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=125, arg=0, argval='i', argrepr='i', offset=138, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=140, starts_line=14, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=3, argval=6, argrepr='6', offset=142, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=107, arg=68, argval='>', argrepr='>', offset=144, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=152, argrepr='to 152', offset=148, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=24, argval=104, argrepr='to 104', offset=150, starts_line=15, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=152, starts_line=16, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=2, argval=4, argrepr='4', offset=154, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COMPARE_OP', opcode=107, arg=2, argval='<', argrepr='<', offset=156, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=164, argrepr='to 164', offset=160, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_FORWARD', opcode=110, arg=14, argval=192, argrepr='to 192', offset=162, starts_line=17, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=164, starts_line=11, is_jump_target=True, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=1, argval=170, argrepr='to 170', offset=166, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=31, argval=108, argrepr='to 108', offset=168, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=170, starts_line=19, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=6, argval='Who let lolcatz into this test suite?', argrepr="'Who let lolcatz into this test suite?'", offset=180, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=182, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=190, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='NOP', opcode=9, arg=None, argval=None, argrepr='', offset=192, starts_line=20, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=5, argval=1, argrepr='1', offset=194, starts_line=21, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=7, argval=0, argrepr='0', offset=196, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='BINARY_OP', opcode=122, arg=11, argval=11, argrepr='/', offset=198, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=202, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_FAST', opcode=124, arg=0, argval='i', argrepr='i', offset=204, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='BEFORE_WITH', opcode=53, arg=None, argval=None, argrepr='', offset=206, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='STORE_FAST', opcode=125, arg=1, argval='dodgy', argrepr='dodgy', offset=208, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=210, starts_line=26, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=8, argval='Never reach this', argrepr="'Never reach this'", offset=220, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=222, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=230, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=232, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=234, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=0, argval=None, argrepr='None', offset=236, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=2, argval=2, argrepr='', offset=238, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=246, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=248, starts_line=28, is_jump_target=True, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=258, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=260, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=268, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=270, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=272, starts_line=25, is_jump_target=False, positions=None), + Instruction(opname='WITH_EXCEPT_START', opcode=49, arg=None, argval=None, argrepr='', offset=274, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=280, argrepr='to 280', offset=276, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=2, argval=2, argrepr='', offset=278, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=280, starts_line=None, is_jump_target=True, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=282, starts_line=None, is_jump_target=False, positions=None), Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=284, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RETURN_CONST', opcode=121, arg=0, argval=None, argrepr='None', offset=286, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=288, starts_line=25, is_jump_target=False, positions=None), - Instruction(opname='WITH_EXCEPT_START', opcode=49, arg=None, argval=None, argrepr='', offset=290, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_TRUE', opcode=115, arg=1, argval=296, argrepr='to 296', offset=292, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=2, argval=2, argrepr='', offset=294, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=296, starts_line=None, is_jump_target=True, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=298, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=300, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=302, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=22, argval=262, argrepr='to 262', offset=304, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=306, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=308, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=310, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=312, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=314, starts_line=22, is_jump_target=False, positions=None), - Instruction(opname='CHECK_EXC_MATCH', opcode=36, arg=None, argval=None, argrepr='', offset=324, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=15, argval=358, argrepr='to 358', offset=326, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=328, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=330, starts_line=23, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=340, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=342, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=352, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=354, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='JUMP_BACKWARD', opcode=140, arg=48, argval=262, argrepr='to 262', offset=356, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=358, starts_line=22, is_jump_target=True, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=360, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=362, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=364, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=366, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=368, starts_line=28, is_jump_target=False, positions=None), - Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=378, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=380, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=390, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=392, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=394, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=396, starts_line=None, is_jump_target=False, positions=None), - Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=398, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=286, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=21, argval=248, argrepr='to 248', offset=288, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=290, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=292, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=294, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=296, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=4, argval='ZeroDivisionError', argrepr='ZeroDivisionError', offset=298, starts_line=22, is_jump_target=False, positions=None), + Instruction(opname='CHECK_EXC_MATCH', opcode=36, arg=None, argval=None, argrepr='', offset=308, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_JUMP_IF_FALSE', opcode=114, arg=14, argval=340, argrepr='to 340', offset=310, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=312, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=314, starts_line=23, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=9, argval='Here we go, here we go, here we go...', argrepr="'Here we go, here we go, here we go...'", offset=324, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=326, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=334, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=336, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='JUMP_BACKWARD', opcode=140, arg=46, argval=248, argrepr='to 248', offset=338, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=340, starts_line=22, is_jump_target=True, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=342, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=344, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=346, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='PUSH_EXC_INFO', opcode=35, arg=None, argval=None, argrepr='', offset=348, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='LOAD_GLOBAL', opcode=116, arg=3, argval='print', argrepr='NULL + print', offset=350, starts_line=28, is_jump_target=False, positions=None), + Instruction(opname='LOAD_CONST', opcode=100, arg=10, argval="OK, now we're done", argrepr='"OK, now we\'re done"', offset=360, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='CALL', opcode=171, arg=1, argval=1, argrepr='', offset=362, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_TOP', opcode=1, arg=None, argval=None, argrepr='', offset=370, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=0, argval=0, argrepr='', offset=372, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='COPY', opcode=120, arg=3, argval=3, argrepr='', offset=374, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='POP_EXCEPT', opcode=89, arg=None, argval=None, argrepr='', offset=376, starts_line=None, is_jump_target=False, positions=None), + Instruction(opname='RERAISE', opcode=119, arg=1, argval=1, argrepr='', offset=378, starts_line=None, is_jump_target=False, positions=None), ] # One last piece of inspect fodder to check the default line number handling diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-25-23-24-38.gh-issue-88691.2SWBd1.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-25-23-24-38.gh-issue-88691.2SWBd1.rst new file mode 100644 index 00000000000000..761d45b0a3a84f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-25-23-24-38.gh-issue-88691.2SWBd1.rst @@ -0,0 +1 @@ +Reduce the number of inline :opcode:`CACHE` entries for :opcode:`CALL`. diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index c33bb18f4da8b3..4ac472a88261e1 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -1,38 +1,38 @@ // Auto-generated by Programs/freeze_test_frozenmain.py unsigned char M_test_frozenmain[] = { 227,0,0,0,0,0,0,0,0,0,0,0,0,8,0,0, - 0,0,0,0,0,243,170,0,0,0,151,0,100,0,100,1, + 0,0,0,0,0,243,162,0,0,0,151,0,100,0,100,1, 108,0,90,0,100,0,100,1,108,1,90,1,2,0,101,2, - 100,2,171,1,0,0,0,0,0,0,0,0,1,0,2,0, - 101,2,100,3,101,0,106,6,0,0,0,0,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,171,2,0,0,0,0, - 0,0,0,0,1,0,2,0,101,1,106,8,0,0,0,0, - 0,0,0,0,0,0,0,0,0,0,0,0,0,0,171,0, - 0,0,0,0,0,0,0,0,100,4,25,0,0,0,90,5, - 100,5,68,0,93,20,0,0,90,6,2,0,101,2,100,6, - 101,6,155,0,100,7,101,5,101,6,25,0,0,0,155,0, - 157,4,171,1,0,0,0,0,0,0,0,0,1,0,140,22, - 4,0,121,1,41,8,233,0,0,0,0,78,122,18,70,114, - 111,122,101,110,32,72,101,108,108,111,32,87,111,114,108,100, - 122,8,115,121,115,46,97,114,103,118,218,6,99,111,110,102, - 105,103,41,5,218,12,112,114,111,103,114,97,109,95,110,97, - 109,101,218,10,101,120,101,99,117,116,97,98,108,101,218,15, - 117,115,101,95,101,110,118,105,114,111,110,109,101,110,116,218, - 17,99,111,110,102,105,103,117,114,101,95,99,95,115,116,100, - 105,111,218,14,98,117,102,102,101,114,101,100,95,115,116,100, - 105,111,122,7,99,111,110,102,105,103,32,122,2,58,32,41, - 7,218,3,115,121,115,218,17,95,116,101,115,116,105,110,116, - 101,114,110,97,108,99,97,112,105,218,5,112,114,105,110,116, - 218,4,97,114,103,118,218,11,103,101,116,95,99,111,110,102, - 105,103,115,114,3,0,0,0,218,3,107,101,121,169,0,243, - 0,0,0,0,250,18,116,101,115,116,95,102,114,111,122,101, - 110,109,97,105,110,46,112,121,250,8,60,109,111,100,117,108, - 101,62,114,18,0,0,0,1,0,0,0,115,100,0,0,0, - 240,3,1,1,1,243,8,0,1,11,219,0,24,225,0,5, - 208,6,26,213,0,27,217,0,5,128,106,144,35,151,40,145, - 40,213,0,27,216,9,38,208,9,26,215,9,38,209,9,38, - 212,9,40,168,24,209,9,50,128,6,240,2,6,12,2,242, - 0,7,1,42,128,67,241,14,0,5,10,208,10,40,144,67, - 209,10,40,152,54,160,35,153,59,209,10,40,214,4,41,241, - 15,7,1,42,114,16,0,0,0, + 100,2,171,1,0,0,0,0,0,0,1,0,2,0,101,2, + 100,3,101,0,106,6,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,171,2,0,0,0,0,0,0, + 1,0,2,0,101,1,106,8,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,171,0,0,0,0,0, + 0,0,100,4,25,0,0,0,90,5,100,5,68,0,93,19, + 0,0,90,6,2,0,101,2,100,6,101,6,155,0,100,7, + 101,5,101,6,25,0,0,0,155,0,157,4,171,1,0,0, + 0,0,0,0,1,0,140,21,4,0,121,1,41,8,233,0, + 0,0,0,78,122,18,70,114,111,122,101,110,32,72,101,108, + 108,111,32,87,111,114,108,100,122,8,115,121,115,46,97,114, + 103,118,218,6,99,111,110,102,105,103,41,5,218,12,112,114, + 111,103,114,97,109,95,110,97,109,101,218,10,101,120,101,99, + 117,116,97,98,108,101,218,15,117,115,101,95,101,110,118,105, + 114,111,110,109,101,110,116,218,17,99,111,110,102,105,103,117, + 114,101,95,99,95,115,116,100,105,111,218,14,98,117,102,102, + 101,114,101,100,95,115,116,100,105,111,122,7,99,111,110,102, + 105,103,32,122,2,58,32,41,7,218,3,115,121,115,218,17, + 95,116,101,115,116,105,110,116,101,114,110,97,108,99,97,112, + 105,218,5,112,114,105,110,116,218,4,97,114,103,118,218,11, + 103,101,116,95,99,111,110,102,105,103,115,114,3,0,0,0, + 218,3,107,101,121,169,0,243,0,0,0,0,250,18,116,101, + 115,116,95,102,114,111,122,101,110,109,97,105,110,46,112,121, + 250,8,60,109,111,100,117,108,101,62,114,18,0,0,0,1, + 0,0,0,115,100,0,0,0,240,3,1,1,1,243,8,0, + 1,11,219,0,24,225,0,5,208,6,26,212,0,27,217,0, + 5,128,106,144,35,151,40,145,40,212,0,27,216,9,38,208, + 9,26,215,9,38,209,9,38,211,9,40,168,24,209,9,50, + 128,6,240,2,6,12,2,242,0,7,1,42,128,67,241,14, + 0,5,10,208,10,40,144,67,209,10,40,152,54,160,35,153, + 59,209,10,40,213,4,41,241,15,7,1,42,114,16,0,0, + 0, }; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 617b6f311e2f77..72f85cc92b0c92 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2318,7 +2318,7 @@ dummy_func( kwnames = GETITEM(frame->f_code->co_consts, oparg); } - // Cache layout: counter/1, func_version/2, min_args/1 + // Cache layout: counter/1, func_version/2 // Neither CALL_INTRINSIC_1/2 nor CALL_FUNCTION_EX are members! family(call, INLINE_CACHE_ENTRIES_CALL) = { CALL, @@ -2348,7 +2348,7 @@ dummy_func( // (Some args may be keywords, see KW_NAMES, which sets 'kwnames'.) // On exit, the stack is [result]. // When calling Python, inline the call using DISPATCH_INLINED(). - inst(CALL, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { + inst(CALL, (unused/1, unused/2, method, callable, args[oparg] -- res)) { int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -2426,7 +2426,7 @@ dummy_func( // Start out with [NULL, bound_method, arg1, arg2, ...] // Transform to [callable, self, arg1, arg2, ...] // Then fall through to CALL_PY_EXACT_ARGS - inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, unused/1, method, callable, unused[oparg] -- unused)) { + inst(CALL_BOUND_METHOD_EXACT_ARGS, (unused/1, unused/2, method, callable, unused[oparg] -- unused)) { DEOPT_IF(method != NULL, CALL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); STAT_INC(CALL, hit); @@ -2438,7 +2438,7 @@ dummy_func( GO_TO_INSTRUCTION(CALL_PY_EXACT_ARGS); } - inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, unused/1, method, callable, args[oparg] -- unused)) { + inst(CALL_PY_EXACT_ARGS, (unused/1, func_version/2, method, callable, args[oparg] -- unused)) { assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -2465,7 +2465,7 @@ dummy_func( DISPATCH_INLINED(new_frame); } - inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, min_args/1, method, callable, args[oparg] -- unused)) { + inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, method, callable, args[oparg] -- unused)) { assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); int is_meth = method != NULL; @@ -2479,6 +2479,11 @@ dummy_func( PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; + assert(func->func_defaults); + assert(PyTuple_CheckExact(func->func_defaults)); + int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); + assert(defcount <= code->co_argcount); + int min_args = code->co_argcount - defcount; DEOPT_IF(argcount > code->co_argcount, CALL); DEOPT_IF(argcount < min_args, CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); @@ -2497,7 +2502,7 @@ dummy_func( DISPATCH_INLINED(new_frame); } - inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_TYPE_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); @@ -2510,7 +2515,7 @@ dummy_func( Py_DECREF(&PyType_Type); // I.e., callable } - inst(CALL_NO_KW_STR_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_STR_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); @@ -2525,7 +2530,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_TUPLE_1, (unused/1, unused/2, unused/1, null, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_TUPLE_1, (unused/1, unused/2, null, callable, args[oparg] -- res)) { assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -2539,7 +2544,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { + inst(CALL_BUILTIN_CLASS, (unused/1, unused/2, method, callable, args[oparg] -- res)) { int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -2564,7 +2569,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_BUILTIN_O, (unused/1, unused/2, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); @@ -2596,7 +2601,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_BUILTIN_FAST, (unused/1, unused/2, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); @@ -2632,7 +2637,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { + inst(CALL_BUILTIN_FAST_WITH_KEYWORDS, (unused/1, unused/2, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; @@ -2668,7 +2673,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_LEN, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_LEN, (unused/1, unused/2, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ @@ -2696,7 +2701,7 @@ dummy_func( ERROR_IF(res == NULL, error); } - inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, unused/1, method, callable, args[oparg] -- res)) { + inst(CALL_NO_KW_ISINSTANCE, (unused/1, unused/2, method, callable, args[oparg] -- res)) { assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ @@ -2727,7 +2732,7 @@ dummy_func( } // This is secretly a super-instruction - inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, unused/1, method, self, args[oparg] -- unused)) { + inst(CALL_NO_KW_LIST_APPEND, (unused/1, unused/2, method, self, args[oparg] -- unused)) { assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); @@ -2748,7 +2753,7 @@ dummy_func( DISPATCH(); } - inst(CALL_NO_KW_METHOD_DESCRIPTOR_O, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_O, (unused/1, unused/2, method, unused, args[oparg] -- res)) { assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -2782,7 +2787,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { + inst(CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, (unused/1, unused/2, method, unused, args[oparg] -- res)) { int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -2814,7 +2819,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, (unused/1, unused/2, method, unused, args[oparg] -- res)) { assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -2846,7 +2851,7 @@ dummy_func( CHECK_EVAL_BREAKER(); } - inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, unused/1, method, unused, args[oparg] -- res)) { + inst(CALL_NO_KW_METHOD_DESCRIPTOR_FAST, (unused/1, unused/2, method, unused, args[oparg] -- res)) { assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7df585be0bef50..420d136bb98158 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3301,7 +3301,7 @@ TARGET(CALL) { PREDICTED(CALL); - static_assert(INLINE_CACHE_ENTRIES_CALL == 4, "incorrect cache size"); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); PyObject **args = (stack_pointer - oparg); PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; @@ -3382,7 +3382,7 @@ STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3442,7 +3442,6 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; uint32_t func_version = read_u32(&next_instr[1].cache); - uint16_t min_args = read_u16(&next_instr[3].cache); #line 2469 "Python/bytecodes.c" assert(kwnames == NULL); DEOPT_IF(tstate->interp->eval_frame, CALL); @@ -3457,6 +3456,11 @@ PyFunctionObject *func = (PyFunctionObject *)callable; DEOPT_IF(func->func_version != func_version, CALL); PyCodeObject *code = (PyCodeObject *)func->func_code; + assert(func->func_defaults); + assert(PyTuple_CheckExact(func->func_defaults)); + int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); + assert(defcount <= code->co_argcount); + int min_args = code->co_argcount - defcount; DEOPT_IF(argcount > code->co_argcount, CALL); DEOPT_IF(argcount < min_args, CALL); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); @@ -3473,7 +3477,7 @@ STACK_SHRINK(oparg + 2); JUMPBY(INLINE_CACHE_ENTRIES_CALL); DISPATCH_INLINED(new_frame); - #line 3477 "Python/generated_cases.c.h" + #line 3481 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_TYPE_1) { @@ -3481,7 +3485,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2501 "Python/bytecodes.c" + #line 2506 "Python/bytecodes.c" assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); @@ -3492,11 +3496,11 @@ res = Py_NewRef(Py_TYPE(obj)); Py_DECREF(obj); Py_DECREF(&PyType_Type); // I.e., callable - #line 3496 "Python/generated_cases.c.h" + #line 3500 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; DISPATCH(); } @@ -3505,7 +3509,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2514 "Python/bytecodes.c" + #line 2519 "Python/bytecodes.c" assert(kwnames == NULL); assert(cframe.use_tracing == 0); assert(oparg == 1); @@ -3517,11 +3521,11 @@ Py_DECREF(arg); Py_DECREF(&PyUnicode_Type); // I.e., callable if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3521 "Python/generated_cases.c.h" + #line 3525 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3531,7 +3535,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *null = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2529 "Python/bytecodes.c" + #line 2534 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 1); DEOPT_IF(null != NULL, CALL); @@ -3542,11 +3546,11 @@ Py_DECREF(arg); Py_DECREF(&PyTuple_Type); // I.e., tuple if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3546 "Python/generated_cases.c.h" + #line 3550 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3556,7 +3560,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2543 "Python/bytecodes.c" + #line 2548 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3578,11 +3582,11 @@ } Py_DECREF(tp); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3582 "Python/generated_cases.c.h" + #line 3586 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3592,7 +3596,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2568 "Python/bytecodes.c" + #line 2573 "Python/bytecodes.c" assert(cframe.use_tracing == 0); /* Builtin METH_O functions */ assert(kwnames == NULL); @@ -3621,11 +3625,11 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3625 "Python/generated_cases.c.h" + #line 3629 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3635,7 +3639,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2600 "Python/bytecodes.c" + #line 2605 "Python/bytecodes.c" assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL functions, without keywords */ assert(kwnames == NULL); @@ -3668,11 +3672,11 @@ 'invalid'). In those cases an exception is set, so we must handle it. */ - #line 3672 "Python/generated_cases.c.h" + #line 3676 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3682,7 +3686,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2636 "Python/bytecodes.c" + #line 2641 "Python/bytecodes.c" assert(cframe.use_tracing == 0); /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ int is_meth = method != NULL; @@ -3715,11 +3719,11 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3719 "Python/generated_cases.c.h" + #line 3723 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3729,7 +3733,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2672 "Python/bytecodes.c" + #line 2677 "Python/bytecodes.c" assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* len(o) */ @@ -3755,11 +3759,11 @@ Py_DECREF(callable); Py_DECREF(arg); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3759 "Python/generated_cases.c.h" + #line 3763 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; DISPATCH(); } @@ -3768,7 +3772,7 @@ PyObject *callable = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2700 "Python/bytecodes.c" + #line 2705 "Python/bytecodes.c" assert(cframe.use_tracing == 0); assert(kwnames == NULL); /* isinstance(o, o2) */ @@ -3796,11 +3800,11 @@ Py_DECREF(cls); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3800 "Python/generated_cases.c.h" + #line 3804 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; DISPATCH(); } @@ -3808,7 +3812,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *self = stack_pointer[-(1 + oparg)]; PyObject *method = stack_pointer[-(2 + oparg)]; - #line 2731 "Python/bytecodes.c" + #line 2736 "Python/bytecodes.c" assert(cframe.use_tracing == 0); assert(kwnames == NULL); assert(oparg == 1); @@ -3827,14 +3831,14 @@ JUMPBY(INLINE_CACHE_ENTRIES_CALL + 1); assert(next_instr[-1].op.code == POP_TOP); DISPATCH(); - #line 3831 "Python/generated_cases.c.h" + #line 3835 "Python/generated_cases.c.h" } TARGET(CALL_NO_KW_METHOD_DESCRIPTOR_O) { PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2752 "Python/bytecodes.c" + #line 2757 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -3865,11 +3869,11 @@ Py_DECREF(arg); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3869 "Python/generated_cases.c.h" + #line 3873 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3878,7 +3882,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2786 "Python/bytecodes.c" + #line 2791 "Python/bytecodes.c" int is_meth = method != NULL; int total_args = oparg; if (is_meth) { @@ -3907,11 +3911,11 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3911 "Python/generated_cases.c.h" + #line 3915 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3920,7 +3924,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2818 "Python/bytecodes.c" + #line 2823 "Python/bytecodes.c" assert(kwnames == NULL); assert(oparg == 0 || oparg == 1); int is_meth = method != NULL; @@ -3949,11 +3953,11 @@ Py_DECREF(self); Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3953 "Python/generated_cases.c.h" + #line 3957 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -3962,7 +3966,7 @@ PyObject **args = (stack_pointer - oparg); PyObject *method = stack_pointer[-(2 + oparg)]; PyObject *res; - #line 2850 "Python/bytecodes.c" + #line 2855 "Python/bytecodes.c" assert(kwnames == NULL); int is_meth = method != NULL; int total_args = oparg; @@ -3990,11 +3994,11 @@ } Py_DECREF(callable); if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } - #line 3994 "Python/generated_cases.c.h" + #line 3998 "Python/generated_cases.c.h" STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; - next_instr += 4; + next_instr += 3; CHECK_EVAL_BREAKER(); DISPATCH(); } @@ -4005,7 +4009,7 @@ PyObject *callargs = stack_pointer[-(1 + ((oparg & 1) ? 1 : 0))]; PyObject *func = stack_pointer[-(2 + ((oparg & 1) ? 1 : 0))]; PyObject *result; - #line 2881 "Python/bytecodes.c" + #line 2886 "Python/bytecodes.c" if (oparg & 1) { // DICT_MERGE is called before this opcode if there are kwargs. // It converts all dict subtypes in kwargs into regular dicts. @@ -4024,15 +4028,15 @@ assert(PyTuple_CheckExact(callargs)); result = do_call_core(tstate, func, callargs, kwargs, cframe.use_tracing); - #line 4028 "Python/generated_cases.c.h" + #line 4032 "Python/generated_cases.c.h" Py_DECREF(func); Py_DECREF(callargs); Py_XDECREF(kwargs); - #line 2900 "Python/bytecodes.c" + #line 2905 "Python/bytecodes.c" assert(PEEK(3 + (oparg & 1)) == NULL); if (result == NULL) { STACK_SHRINK(((oparg & 1) ? 1 : 0)); goto pop_3_error; } - #line 4036 "Python/generated_cases.c.h" + #line 4040 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 1) ? 1 : 0)); STACK_SHRINK(2); stack_pointer[-1] = result; @@ -4047,7 +4051,7 @@ PyObject *kwdefaults = (oparg & 0x02) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0))] : NULL; PyObject *defaults = (oparg & 0x01) ? stack_pointer[-(1 + ((oparg & 0x08) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x01) ? 1 : 0))] : NULL; PyObject *func; - #line 2911 "Python/bytecodes.c" + #line 2916 "Python/bytecodes.c" PyFunctionObject *func_obj = (PyFunctionObject *) PyFunction_New(codeobj, GLOBALS()); @@ -4076,14 +4080,14 @@ func_obj->func_version = ((PyCodeObject *)codeobj)->co_version; func = (PyObject *)func_obj; - #line 4080 "Python/generated_cases.c.h" + #line 4084 "Python/generated_cases.c.h" STACK_SHRINK(((oparg & 0x01) ? 1 : 0) + ((oparg & 0x02) ? 1 : 0) + ((oparg & 0x04) ? 1 : 0) + ((oparg & 0x08) ? 1 : 0)); stack_pointer[-1] = func; DISPATCH(); } TARGET(RETURN_GENERATOR) { - #line 2942 "Python/bytecodes.c" + #line 2947 "Python/bytecodes.c" assert(PyFunction_Check(frame->f_funcobj)); PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); @@ -4104,7 +4108,7 @@ frame = cframe.current_frame = prev; _PyFrame_StackPush(frame, (PyObject *)gen); goto resume_frame; - #line 4108 "Python/generated_cases.c.h" + #line 4112 "Python/generated_cases.c.h" } TARGET(BUILD_SLICE) { @@ -4112,15 +4116,15 @@ PyObject *stop = stack_pointer[-(1 + ((oparg == 3) ? 1 : 0))]; PyObject *start = stack_pointer[-(2 + ((oparg == 3) ? 1 : 0))]; PyObject *slice; - #line 2965 "Python/bytecodes.c" + #line 2970 "Python/bytecodes.c" slice = PySlice_New(start, stop, step); - #line 4118 "Python/generated_cases.c.h" + #line 4122 "Python/generated_cases.c.h" Py_DECREF(start); Py_DECREF(stop); Py_XDECREF(step); - #line 2967 "Python/bytecodes.c" + #line 2972 "Python/bytecodes.c" if (slice == NULL) { STACK_SHRINK(((oparg == 3) ? 1 : 0)); goto pop_2_error; } - #line 4124 "Python/generated_cases.c.h" + #line 4128 "Python/generated_cases.c.h" STACK_SHRINK(((oparg == 3) ? 1 : 0)); STACK_SHRINK(1); stack_pointer[-1] = slice; @@ -4131,7 +4135,7 @@ PyObject *fmt_spec = ((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? stack_pointer[-((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))] : NULL; PyObject *value = stack_pointer[-(1 + (((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0))]; PyObject *result; - #line 2971 "Python/bytecodes.c" + #line 2976 "Python/bytecodes.c" /* Handles f-string value formatting. */ PyObject *(*conv_fn)(PyObject *); int which_conversion = oparg & FVC_MASK; @@ -4166,7 +4170,7 @@ Py_DECREF(value); Py_XDECREF(fmt_spec); if (result == NULL) { STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); goto pop_1_error; } - #line 4170 "Python/generated_cases.c.h" + #line 4174 "Python/generated_cases.c.h" STACK_SHRINK((((oparg & FVS_MASK) == FVS_HAVE_SPEC) ? 1 : 0)); stack_pointer[-1] = result; DISPATCH(); @@ -4175,10 +4179,10 @@ TARGET(COPY) { PyObject *bottom = stack_pointer[-(1 + (oparg-1))]; PyObject *top; - #line 3008 "Python/bytecodes.c" + #line 3013 "Python/bytecodes.c" assert(oparg > 0); top = Py_NewRef(bottom); - #line 4182 "Python/generated_cases.c.h" + #line 4186 "Python/generated_cases.c.h" STACK_GROW(1); stack_pointer[-1] = top; DISPATCH(); @@ -4190,7 +4194,7 @@ PyObject *rhs = stack_pointer[-1]; PyObject *lhs = stack_pointer[-2]; PyObject *res; - #line 3013 "Python/bytecodes.c" + #line 3018 "Python/bytecodes.c" #if ENABLE_SPECIALIZATION _PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr; if (ADAPTIVE_COUNTER_IS_ZERO(cache->counter)) { @@ -4206,12 +4210,12 @@ assert((unsigned)oparg < Py_ARRAY_LENGTH(binary_ops)); assert(binary_ops[oparg]); res = binary_ops[oparg](lhs, rhs); - #line 4210 "Python/generated_cases.c.h" + #line 4214 "Python/generated_cases.c.h" Py_DECREF(lhs); Py_DECREF(rhs); - #line 3029 "Python/bytecodes.c" + #line 3034 "Python/bytecodes.c" if (res == NULL) goto pop_2_error; - #line 4215 "Python/generated_cases.c.h" + #line 4219 "Python/generated_cases.c.h" STACK_SHRINK(1); stack_pointer[-1] = res; next_instr += 1; @@ -4221,27 +4225,27 @@ TARGET(SWAP) { PyObject *top = stack_pointer[-1]; PyObject *bottom = stack_pointer[-(2 + (oparg-2))]; - #line 3034 "Python/bytecodes.c" + #line 3039 "Python/bytecodes.c" assert(oparg >= 2); - #line 4227 "Python/generated_cases.c.h" + #line 4231 "Python/generated_cases.c.h" stack_pointer[-1] = bottom; stack_pointer[-(2 + (oparg-2))] = top; DISPATCH(); } TARGET(EXTENDED_ARG) { - #line 3038 "Python/bytecodes.c" + #line 3043 "Python/bytecodes.c" assert(oparg); assert(cframe.use_tracing == 0); opcode = next_instr->op.code; oparg = oparg << 8 | next_instr->op.arg; PRE_DISPATCH_GOTO(); DISPATCH_GOTO(); - #line 4241 "Python/generated_cases.c.h" + #line 4245 "Python/generated_cases.c.h" } TARGET(CACHE) { - #line 3047 "Python/bytecodes.c" + #line 3052 "Python/bytecodes.c" Py_UNREACHABLE(); - #line 4247 "Python/generated_cases.c.h" + #line 4251 "Python/generated_cases.c.h" } diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 5c984eb4ae12e3..f57b76aeb31050 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -695,7 +695,7 @@ _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { } #endif -enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; +enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT_IBC000, INSTR_FMT_IBC00000000, INSTR_FMT_IBIB, INSTR_FMT_IX, INSTR_FMT_IXC, INSTR_FMT_IXC000 }; struct opcode_metadata { bool valid_entry; enum InstructionFormat instr_format; @@ -844,24 +844,24 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[256] = { [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000 }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000 }, [KW_NAMES] = { true, INSTR_FMT_IB }, - [CALL] = { true, INSTR_FMT_IBC000 }, - [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC000 }, - [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC000 }, - [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_TYPE_1] = { true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_STR_1] = { true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_TUPLE_1] = { true, INSTR_FMT_IBC000 }, - [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_BUILTIN_O] = { true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_BUILTIN_FAST] = { true, INSTR_FMT_IBC000 }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_LEN] = { true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_ISINSTANCE] = { true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_LIST_APPEND] = { true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC000 }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC000 }, - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC000 }, + [CALL] = { true, INSTR_FMT_IBC00 }, + [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00 }, + [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00 }, + [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00 }, + [CALL_NO_KW_TYPE_1] = { true, INSTR_FMT_IBC00 }, + [CALL_NO_KW_STR_1] = { true, INSTR_FMT_IBC00 }, + [CALL_NO_KW_TUPLE_1] = { true, INSTR_FMT_IBC00 }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00 }, + [CALL_NO_KW_BUILTIN_O] = { true, INSTR_FMT_IBC00 }, + [CALL_NO_KW_BUILTIN_FAST] = { true, INSTR_FMT_IBC00 }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00 }, + [CALL_NO_KW_LEN] = { true, INSTR_FMT_IBC00 }, + [CALL_NO_KW_ISINSTANCE] = { true, INSTR_FMT_IBC00 }, + [CALL_NO_KW_LIST_APPEND] = { true, INSTR_FMT_IBC00 }, + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00 }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00 }, + [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00 }, + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00 }, [CALL_FUNCTION_EX] = { true, INSTR_FMT_IB }, [MAKE_FUNCTION] = { true, INSTR_FMT_IB }, [RETURN_GENERATOR] = { true, INSTR_FMT_IX }, diff --git a/Python/specialize.c b/Python/specialize.c index 9187438d519b26..a9d3226ee39f5f 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1627,17 +1627,12 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, assert(nargs <= argcount && nargs >= min_args); assert(min_args >= 0 && defcount >= 0); assert(defcount == 0 || func->func_defaults != NULL); - if (min_args > 0xffff) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_RANGE); - return -1; - } int version = _PyFunction_GetVersionForCurrentState(func); if (version == 0) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } write_u32(cache->func_version, version); - cache->min_args = min_args; if (argcount == nargs) { instr->op.code = bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; } From 4ec8dd10bd4682793559c4eccbcf6ae00688c4c3 Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Wed, 5 Apr 2023 17:33:52 -0700 Subject: [PATCH 30/76] gh-93910: [Enum] remove member.member deprecation (GH-103236) i.e. Color.RED.BLUE is now officially supported. --- Doc/howto/enum.rst | 7 ++- Lib/enum.py | 50 +++++++++---------- Lib/test/test_enum.py | 29 +++-------- ...3-04-04-12-43-38.gh-issue-93910.jurMzv.rst | 1 + 4 files changed, 37 insertions(+), 50 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-04-12-43-38.gh-issue-93910.jurMzv.rst diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 38951ed7348f56..9390faded2fb8d 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -988,12 +988,11 @@ but remain normal attributes. """""""""""""""""""" Enum members are instances of their enum class, and are normally accessed as -``EnumClass.member``. In Python versions starting with ``3.5`` you could access -members from other members -- this practice is discouraged, is deprecated -in ``3.12``, and will be removed in ``3.14``. +``EnumClass.member``. In certain situations, such as writing custom enum +behavior, being able to access one member directly from another is useful, +and is supported. .. versionchanged:: 3.5 -.. versionchanged:: 3.12 Creating members that are mixed with other data types diff --git a/Lib/enum.py b/Lib/enum.py index ec698d5fa3c300..0b0629cf920506 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -201,23 +201,13 @@ def __get__(self, instance, ownerclass=None): ) else: if self.fget is None: - if self.member is None: # not sure this can happen, but just in case + # look for a member by this name. + try: + return ownerclass._member_map_[self.name] + except KeyError: raise AttributeError( '%r has no attribute %r' % (ownerclass, self.name) ) - # issue warning deprecating this behavior - import warnings - warnings.warn( - "`member.member` access (e.g. `Color.RED.BLUE`) is " - "deprecated and will be removed in 3.14.", - DeprecationWarning, - stacklevel=2, - ) - return self.member - # XXX: uncomment in 3.14 and remove warning above - # raise AttributeError( - # '%r member has no attribute %r' % (ownerclass, self.name) - # ) else: return self.fget(instance) @@ -314,22 +304,32 @@ def __set_name__(self, enum_class, member_name): ): # no other instances found, record this member in _member_names_ enum_class._member_names_.append(member_name) - # get redirect in place before adding to _member_map_ - # but check for other instances in parent classes first - descriptor = None + # if necessary, get redirect in place and then add it to _member_map_ + found_descriptor = None for base in enum_class.__mro__[1:]: descriptor = base.__dict__.get(member_name) if descriptor is not None: if isinstance(descriptor, (property, DynamicClassAttribute)): + found_descriptor = descriptor break - redirect = property() - redirect.member = enum_member - redirect.__set_name__(enum_class, member_name) - if descriptor: - redirect.fget = getattr(descriptor, 'fget', None) - redirect.fset = getattr(descriptor, 'fset', None) - redirect.fdel = getattr(descriptor, 'fdel', None) - setattr(enum_class, member_name, redirect) + elif ( + hasattr(descriptor, 'fget') and + hasattr(descriptor, 'fset') and + hasattr(descriptor, 'fdel') + ): + found_descriptor = descriptor + continue + if found_descriptor: + redirect = property() + redirect.member = enum_member + redirect.__set_name__(enum_class, member_name) + # earlier descriptor found; copy fget, fset, fdel to this one. + redirect.fget = found_descriptor.fget + redirect.fset = found_descriptor.fset + redirect.fdel = found_descriptor.fdel + setattr(enum_class, member_name, redirect) + else: + setattr(enum_class, member_name, enum_member) # now add to _member_map_ (even aliases) enum_class._member_map_[member_name] = enum_member try: diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index ee5280601be184..e4151bf9e6d9b3 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2686,28 +2686,15 @@ class Private(Enum): self.assertEqual(Private._Private__corporal, 'Radar') self.assertEqual(Private._Private__major_, 'Hoolihan') - @unittest.skipIf( - python_version <= (3, 13), - 'member.member access currently deprecated', - ) - def test_exception_for_member_from_member_access(self): - with self.assertRaisesRegex(AttributeError, " member has no attribute .NO."): - class Di(Enum): - YES = 1 - NO = 0 - nope = Di.YES.NO - - @unittest.skipIf( - python_version > (3, 13), - 'member.member access now raises', - ) - def test_warning_for_member_from_member_access(self): - with self.assertWarnsRegex(DeprecationWarning, '`member.member` access .* is deprecated and will be removed in 3.14'): - class Di(Enum): - YES = 1 - NO = 0 - warn = Di.YES.NO + def test_member_from_member_access(self): + class Di(Enum): + YES = 1 + NO = 0 + name = 3 + warn = Di.YES.NO self.assertIs(warn, Di.NO) + self.assertIs(Di.name, Di['name']) + self.assertEqual(Di.name.name, 'name') def test_dynamic_members_with_static_methods(self): # diff --git a/Misc/NEWS.d/next/Library/2023-04-04-12-43-38.gh-issue-93910.jurMzv.rst b/Misc/NEWS.d/next/Library/2023-04-04-12-43-38.gh-issue-93910.jurMzv.rst new file mode 100644 index 00000000000000..783aefae0770a9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-04-12-43-38.gh-issue-93910.jurMzv.rst @@ -0,0 +1 @@ +Remove deprecation of enum ``memmber.member`` access. From 03089fdccc7dbe3f69227fbd570df92278371e7f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 5 Apr 2023 18:42:02 -0600 Subject: [PATCH 31/76] gh-101659: Add _Py_AtExit() (gh-103298) The function is like Py_AtExit() but for a single interpreter. This is a companion to the atexit module's register() function, taking a C callback instead of a Python one. We also update the _xxinterpchannels module to use _Py_AtExit(), which is the motivating case. (This is inspired by pain points felt while working on gh-101660.) --- Include/cpython/pylifecycle.h | 4 ++ Include/internal/pycore_atexit.h | 56 +++++++++++++++++ Include/internal/pycore_interp.h | 17 +----- Include/internal/pycore_runtime.h | 5 +- Lib/test/test__xxinterpchannels.py | 40 +++++++++---- Makefile.pre.in | 1 + Modules/_testcapimodule.c | 32 ++++++++++ Modules/_xxinterpchannelsmodule.c | 96 ++++++++++++++++++++++++++---- Modules/_xxsubinterpretersmodule.c | 11 +--- Modules/atexitmodule.c | 57 +++++++++++++++--- PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/pylifecycle.c | 14 ++--- 13 files changed, 269 insertions(+), 68 deletions(-) create mode 100644 Include/internal/pycore_atexit.h diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index 821b169d7a1759..79d55711319e55 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -65,3 +65,7 @@ PyAPI_FUNC(char *) _Py_SetLocaleFromEnv(int category); PyAPI_FUNC(PyStatus) _Py_NewInterpreterFromConfig( PyThreadState **tstate_p, const _PyInterpreterConfig *config); + +typedef void (*atexit_datacallbackfunc)(void *); +PyAPI_FUNC(int) _Py_AtExit( + PyInterpreterState *, atexit_datacallbackfunc, void *); diff --git a/Include/internal/pycore_atexit.h b/Include/internal/pycore_atexit.h new file mode 100644 index 00000000000000..b4663b396852f3 --- /dev/null +++ b/Include/internal/pycore_atexit.h @@ -0,0 +1,56 @@ +#ifndef Py_INTERNAL_ATEXIT_H +#define Py_INTERNAL_ATEXIT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + + +//############### +// runtime atexit + +typedef void (*atexit_callbackfunc)(void); + +struct _atexit_runtime_state { +#define NEXITFUNCS 32 + atexit_callbackfunc callbacks[NEXITFUNCS]; + int ncallbacks; +}; + + +//################### +// interpreter atexit + +struct atexit_callback; +typedef struct atexit_callback { + atexit_datacallbackfunc func; + void *data; + struct atexit_callback *next; +} atexit_callback; + +typedef struct { + PyObject *func; + PyObject *args; + PyObject *kwargs; +} atexit_py_callback; + +struct atexit_state { + atexit_callback *ll_callbacks; + atexit_callback *last_ll_callback; + + // XXX The rest of the state could be moved to the atexit module state + // and a low-level callback added for it during module exec. + // For the moment we leave it here. + atexit_py_callback **callbacks; + int ncallbacks; + int callback_len; +}; + + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_ATEXIT_H */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 1f2c0db2eb5f27..d64a68cd2da54e 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -10,8 +10,9 @@ extern "C" { #include -#include "pycore_atomic.h" // _Py_atomic_address #include "pycore_ast_state.h" // struct ast_state +#include "pycore_atexit.h" // struct atexit_state +#include "pycore_atomic.h" // _Py_atomic_address #include "pycore_ceval_state.h" // struct _ceval_state #include "pycore_code.h" // struct callable_cache #include "pycore_context.h" // struct _Py_context_state @@ -32,20 +33,6 @@ extern "C" { #include "pycore_warnings.h" // struct _warnings_runtime_state -// atexit state -typedef struct { - PyObject *func; - PyObject *args; - PyObject *kwargs; -} atexit_callback; - -struct atexit_state { - atexit_callback **callbacks; - int ncallbacks; - int callback_len; -}; - - struct _Py_long_state { int max_str_digits; }; diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 8877b282bc38de..3ebe49926edda6 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_atexit.h" // struct atexit_runtime_state #include "pycore_atomic.h" /* _Py_atomic_address */ #include "pycore_ceval_state.h" // struct _ceval_runtime_state #include "pycore_floatobject.h" // struct _Py_float_runtime_state @@ -131,9 +132,7 @@ typedef struct pyruntimestate { struct _parser_runtime_state parser; -#define NEXITFUNCS 32 - void (*exitfuncs[NEXITFUNCS])(void); - int nexitfuncs; + struct _atexit_runtime_state atexit; struct _import_runtime_state imports; struct _ceval_runtime_state ceval; diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py index 69bda89a1688f5..b65281106f667c 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__xxinterpchannels.py @@ -550,6 +550,7 @@ def test_channel_list_interpreters_closed_send_end(self): import _xxinterpchannels as _channels _channels.close({cid}, force=True) """)) + return # Both ends should raise an error. with self.assertRaises(channels.ChannelClosedError): channels.list_interpreters(cid, send=True) @@ -673,17 +674,34 @@ def test_recv_default(self): self.assertIs(obj6, default) def test_recv_sending_interp_destroyed(self): - cid = channels.create() - interp = interpreters.create() - interpreters.run_string(interp, dedent(f""" - import _xxinterpchannels as _channels - _channels.send({cid}, b'spam') - """)) - interpreters.destroy(interp) - - with self.assertRaisesRegex(RuntimeError, - 'unrecognized interpreter ID'): - channels.recv(cid) + with self.subTest('closed'): + cid1 = channels.create() + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxinterpchannels as _channels + _channels.send({cid1}, b'spam') + """)) + interpreters.destroy(interp) + + with self.assertRaisesRegex(RuntimeError, + f'channel {cid1} is closed'): + channels.recv(cid1) + del cid1 + with self.subTest('still open'): + cid2 = channels.create() + interp = interpreters.create() + interpreters.run_string(interp, dedent(f""" + import _xxinterpchannels as _channels + _channels.send({cid2}, b'spam') + """)) + channels.send(cid2, b'eggs') + interpreters.destroy(interp) + + channels.recv(cid2) + with self.assertRaisesRegex(RuntimeError, + f'channel {cid2} is empty'): + channels.recv(cid2) + del cid2 def test_allowed_types(self): cid = channels.create() diff --git a/Makefile.pre.in b/Makefile.pre.in index 9fdbd8db19bb33..1c1bddcad82475 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1660,6 +1660,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_asdl.h \ $(srcdir)/Include/internal/pycore_ast.h \ $(srcdir)/Include/internal/pycore_ast_state.h \ + $(srcdir)/Include/internal/pycore_atexit.h \ $(srcdir)/Include/internal/pycore_atomic.h \ $(srcdir)/Include/internal/pycore_atomic_funcs.h \ $(srcdir)/Include/internal/pycore_bitutils.h \ diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3d9a2aeeb7cfd5..557a6d46ed4632 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3381,6 +3381,37 @@ test_gc_visit_objects_exit_early(PyObject *Py_UNUSED(self), } +struct atexit_data { + int called; +}; + +static void +callback(void *data) +{ + ((struct atexit_data *)data)->called += 1; +} + +static PyObject * +test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyThreadState *oldts = PyThreadState_Swap(NULL); + PyThreadState *tstate = Py_NewInterpreter(); + + struct atexit_data data = {0}; + int res = _Py_AtExit(tstate->interp, callback, (void *)&data); + Py_EndInterpreter(tstate); + PyThreadState_Swap(oldts); + if (res < 0) { + return NULL; + } + if (data.called == 0) { + PyErr_SetString(PyExc_RuntimeError, "atexit callback not called"); + return NULL; + } + Py_RETURN_NONE; +} + + static PyObject *test_buildvalue_issue38913(PyObject *, PyObject *); static PyMethodDef TestMethods[] = { @@ -3525,6 +3556,7 @@ static PyMethodDef TestMethods[] = { {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, {"test_gc_visit_objects_basic", test_gc_visit_objects_basic, METH_NOARGS, NULL}, {"test_gc_visit_objects_exit_early", test_gc_visit_objects_exit_early, METH_NOARGS, NULL}, + {"test_atexit", test_atexit, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index ef1cdcab952271..13b005eaef9866 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -174,19 +174,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) } int res = _PyCrossInterpreterData_Release(data); if (res < 0) { - // XXX Fix this! - /* The owning interpreter is already destroyed. - * Ideally, this shouldn't ever happen. When an interpreter is - * about to be destroyed, we should clear out all of its objects - * from every channel associated with that interpreter. - * For now we hack around that to resolve refleaks, by decref'ing - * the released object here, even if its the wrong interpreter. - * The owning interpreter has already been destroyed - * so we should be okay, especially since the currently - * shareable types are all very basic, with no GC. - * That said, it becomes much messier once interpreters - * no longer share a GIL, so this needs to be fixed before then. */ - _PyCrossInterpreterData_Clear(NULL, data); + /* The owning interpreter is already destroyed. */ if (ignoreexc) { // XXX Emit a warning? PyErr_Clear(); @@ -489,6 +477,30 @@ _channelqueue_get(_channelqueue *queue) return _channelitem_popped(item); } +static void +_channelqueue_drop_interpreter(_channelqueue *queue, int64_t interp) +{ + _channelitem *prev = NULL; + _channelitem *next = queue->first; + while (next != NULL) { + _channelitem *item = next; + next = item->next; + if (item->data->interp == interp) { + if (prev == NULL) { + queue->first = item->next; + } + else { + prev->next = item->next; + } + _channelitem_free(item); + queue->count -= 1; + } + else { + prev = item; + } + } +} + /* channel-interpreter associations */ struct _channelend; @@ -693,6 +705,20 @@ _channelends_close_interpreter(_channelends *ends, int64_t interp, int which) return 0; } +static void +_channelends_drop_interpreter(_channelends *ends, int64_t interp) +{ + _channelend *end; + end = _channelend_find(ends->send, interp, NULL); + if (end != NULL) { + _channelends_close_end(ends, end, 1); + } + end = _channelend_find(ends->recv, interp, NULL); + if (end != NULL) { + _channelends_close_end(ends, end, 0); + } +} + static void _channelends_close_all(_channelends *ends, int which, int force) { @@ -841,6 +867,18 @@ _channel_close_interpreter(_PyChannelState *chan, int64_t interp, int end) return res; } +static void +_channel_drop_interpreter(_PyChannelState *chan, int64_t interp) +{ + PyThread_acquire_lock(chan->mutex, WAIT_LOCK); + + _channelqueue_drop_interpreter(chan->queue, interp); + _channelends_drop_interpreter(chan->ends, interp); + chan->open = _channelends_is_open(chan->ends); + + PyThread_release_lock(chan->mutex); +} + static int _channel_close_all(_PyChannelState *chan, int end, int force) { @@ -1213,6 +1251,21 @@ _channels_list_all(_channels *channels, int64_t *count) return cids; } +static void +_channels_drop_interpreter(_channels *channels, int64_t interp) +{ + PyThread_acquire_lock(channels->mutex, WAIT_LOCK); + + _channelref *ref = channels->head; + for (; ref != NULL; ref = ref->next) { + if (ref->chan != NULL) { + _channel_drop_interpreter(ref->chan, interp); + } + } + + PyThread_release_lock(channels->mutex); +} + /* support for closing non-empty channels */ struct _channel_closing { @@ -1932,6 +1985,19 @@ _global_channels(void) { } +static void +clear_interpreter(void *data) +{ + if (_globals.module_count == 0) { + return; + } + PyInterpreterState *interp = (PyInterpreterState *)data; + assert(interp == _get_current_interp()); + int64_t id = PyInterpreterState_GetID(interp); + _channels_drop_interpreter(&_globals.channels, id); +} + + static PyObject * channel_create(PyObject *self, PyObject *Py_UNUSED(ignored)) { @@ -2339,6 +2405,10 @@ module_exec(PyObject *mod) goto error; } + // Make sure chnnels drop objects owned by this interpreter + PyInterpreterState *interp = _get_current_interp(); + _Py_AtExit(interp, clear_interpreter, (void *)interp); + return 0; error: diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 11164676c4d107..884fb0d31f2b7f 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -67,16 +67,7 @@ _release_xid_data(_PyCrossInterpreterData *data, int ignoreexc) } int res = _PyCrossInterpreterData_Release(data); if (res < 0) { - // XXX Fix this! - /* The owning interpreter is already destroyed. - * Ideally, this shouldn't ever happen. (It's highly unlikely.) - * For now we hack around that to resolve refleaks, by decref'ing - * the released object here, even if its the wrong interpreter. - * The owning interpreter has already been destroyed - * so we should be okay, especially since the currently - * shareable types are all very basic, with no GC. - * That said, it becomes much messier once interpreters - * no longer share a GIL, so this needs to be fixed before then. */ + /* The owning interpreter is already destroyed. */ _PyCrossInterpreterData_Clear(NULL, data); if (ignoreexc) { // XXX Emit a warning? diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index a1c511e09d704e..47afd7f0751039 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -7,6 +7,7 @@ */ #include "Python.h" +#include "pycore_atexit.h" #include "pycore_initconfig.h" // _PyStatus_NO_MEMORY #include "pycore_interp.h" // PyInterpreterState.atexit #include "pycore_pystate.h" // _PyInterpreterState_GET @@ -22,10 +23,36 @@ get_atexit_state(void) } +int +_Py_AtExit(PyInterpreterState *interp, + atexit_datacallbackfunc func, void *data) +{ + assert(interp == _PyInterpreterState_GET()); + atexit_callback *callback = PyMem_Malloc(sizeof(atexit_callback)); + if (callback == NULL) { + PyErr_NoMemory(); + return -1; + } + callback->func = func; + callback->data = data; + callback->next = NULL; + + struct atexit_state *state = &interp->atexit; + if (state->ll_callbacks == NULL) { + state->ll_callbacks = callback; + state->last_ll_callback = callback; + } + else { + state->last_ll_callback->next = callback; + } + return 0; +} + + static void atexit_delete_cb(struct atexit_state *state, int i) { - atexit_callback *cb = state->callbacks[i]; + atexit_py_callback *cb = state->callbacks[i]; state->callbacks[i] = NULL; Py_DECREF(cb->func); @@ -39,7 +66,7 @@ atexit_delete_cb(struct atexit_state *state, int i) static void atexit_cleanup(struct atexit_state *state) { - atexit_callback *cb; + atexit_py_callback *cb; for (int i = 0; i < state->ncallbacks; i++) { cb = state->callbacks[i]; if (cb == NULL) @@ -60,7 +87,7 @@ _PyAtExit_Init(PyInterpreterState *interp) state->callback_len = 32; state->ncallbacks = 0; - state->callbacks = PyMem_New(atexit_callback*, state->callback_len); + state->callbacks = PyMem_New(atexit_py_callback*, state->callback_len); if (state->callbacks == NULL) { return _PyStatus_NO_MEMORY(); } @@ -75,6 +102,18 @@ _PyAtExit_Fini(PyInterpreterState *interp) atexit_cleanup(state); PyMem_Free(state->callbacks); state->callbacks = NULL; + + atexit_callback *next = state->ll_callbacks; + state->ll_callbacks = NULL; + while (next != NULL) { + atexit_callback *callback = next; + next = callback->next; + atexit_datacallbackfunc exitfunc = callback->func; + void *data = callback->data; + // It was allocated in _PyAtExit_AddCallback(). + PyMem_Free(callback); + exitfunc(data); + } } @@ -88,7 +127,7 @@ atexit_callfuncs(struct atexit_state *state) } for (int i = state->ncallbacks - 1; i >= 0; i--) { - atexit_callback *cb = state->callbacks[i]; + atexit_py_callback *cb = state->callbacks[i]; if (cb == NULL) { continue; } @@ -152,17 +191,17 @@ atexit_register(PyObject *module, PyObject *args, PyObject *kwargs) struct atexit_state *state = get_atexit_state(); if (state->ncallbacks >= state->callback_len) { - atexit_callback **r; + atexit_py_callback **r; state->callback_len += 16; - size_t size = sizeof(atexit_callback*) * (size_t)state->callback_len; - r = (atexit_callback**)PyMem_Realloc(state->callbacks, size); + size_t size = sizeof(atexit_py_callback*) * (size_t)state->callback_len; + r = (atexit_py_callback**)PyMem_Realloc(state->callbacks, size); if (r == NULL) { return PyErr_NoMemory(); } state->callbacks = r; } - atexit_callback *callback = PyMem_Malloc(sizeof(atexit_callback)); + atexit_py_callback *callback = PyMem_Malloc(sizeof(atexit_py_callback)); if (callback == NULL) { return PyErr_NoMemory(); } @@ -233,7 +272,7 @@ atexit_unregister(PyObject *module, PyObject *func) struct atexit_state *state = get_atexit_state(); for (int i = 0; i < state->ncallbacks; i++) { - atexit_callback *cb = state->callbacks[i]; + atexit_py_callback *cb = state->callbacks[i]; if (cb == NULL) { continue; } diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 8fab600334160d..29f32db579fa40 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -197,6 +197,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 6c5d8dd89f5bc7..6a622fd93930ad 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -498,6 +498,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 8110d94ba17526..d6627bc6b7e86b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2937,23 +2937,23 @@ wait_for_thread_shutdown(PyThreadState *tstate) Py_DECREF(threading); } -#define NEXITFUNCS 32 int Py_AtExit(void (*func)(void)) { - if (_PyRuntime.nexitfuncs >= NEXITFUNCS) + if (_PyRuntime.atexit.ncallbacks >= NEXITFUNCS) return -1; - _PyRuntime.exitfuncs[_PyRuntime.nexitfuncs++] = func; + _PyRuntime.atexit.callbacks[_PyRuntime.atexit.ncallbacks++] = func; return 0; } static void call_ll_exitfuncs(_PyRuntimeState *runtime) { - while (runtime->nexitfuncs > 0) { + struct _atexit_runtime_state *state = &runtime->atexit; + while (state->ncallbacks > 0) { /* pop last function from the list */ - runtime->nexitfuncs--; - void (*exitfunc)(void) = runtime->exitfuncs[runtime->nexitfuncs]; - runtime->exitfuncs[runtime->nexitfuncs] = NULL; + state->ncallbacks--; + atexit_callbackfunc exitfunc = state->callbacks[state->ncallbacks]; + state->callbacks[state->ncallbacks] = NULL; exitfunc(); } From 6e3ee049ace60beb97784f10099ae399efd83bac Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Wed, 5 Apr 2023 23:42:14 -0400 Subject: [PATCH 32/76] gh-103207: Fix Welcome formatting issues when macOS Installer is run in dark mode. (GH-103303) --- Mac/BuildScript/resources/Welcome.rtf | 51 ++++++++++++--------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index efdcb12f849b66..1f109ee9f13e0d 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,9 +1,9 @@ {\rtf1\ansi\ansicpg1252\cocoartf2708 \cocoascreenfonts1\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fmodern\fcharset0 CourierNewPSMT; -\f3\fnil\fcharset0 HelveticaNeue;\f4\fnil\fcharset0 HelveticaNeue-Bold;} -{\colortbl;\red255\green255\blue255;\red24\green26\blue30;\red255\green255\blue255;} -{\*\expandedcolortbl;;\cssrgb\c12157\c13725\c15686;\cssrgb\c100000\c100000\c100000;} -\margl1440\margr1440\vieww12820\viewh10620\viewkind0 +} +{\colortbl;\red255\green255\blue255;} +{\*\expandedcolortbl;;} +\margl1440\margr1440\vieww12200\viewh10880\viewkind0 \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 \f0\fs24 \cf0 This package will install @@ -26,32 +26,25 @@ At the end of this install, click on \ \f1\b macOS 13 Ventura users -\f0\b0 : due to an issue with macOS +\f0\b0 : Due to an issue with the macOS \f1\b Installer -\f0\b0 app, installation of some -\f3 \cf2 \cb3 \expnd0\expndtw0\kerning0 -third-party -\f0 \cf0 \cb1 \kerning1\expnd0\expndtw0 packages including this Python package may fail with a vague -\f4\b \cf2 \cb3 \expnd0\expndtw0\kerning0 -\'93The installer encountered an error\'94 -\f3\b0 message if the -\f4\b Installer -\f3\b0 app does not have permission to access the folder containing the downloaded installer file, typically in the -\f4\b Downloads -\f3\b0 folder. Go to -\f4\b System Settings -\f3\b0 -> -\f4\b Privacy & Security -\f3\b0 -> -\f4\b Files and Folders -\f3\b0 , then click the mark in front of -\f4\b Installer -\f3\b0 to expand, and enable -\f4\b Downloads Folder -\f0\b0 \cf0 \cb1 \kerning1\expnd0\expndtw0 by moving the toggle to the right -\f3 \cf2 \cb3 \expnd0\expndtw0\kerning0 -. See {\field{\*\fldinst{HYPERLINK "https://github.com/python/cpython/issues/103207"}}{\fldrslt https://github.com/python/cpython/issues/103207}} for up-to-date information on this issue. -\f0 \cf0 \cb1 \kerning1\expnd0\expndtw0 \ +\f0\b0 app, installation of some third-party packages including this Python package may fail with a vague +\f1\b "The installer encountered an error" +\f0\b0 message if the +\f1\b Installer +\f0\b0 app does not have permission to access the folder containing the downloaded installer file, typically in the +\f1\b Downloads +\f0\b0 folder. Go to +\f1\b System Settings +\f0\b0 -> +\f1\b Privacy & Security +\f0\b0 -> +\f1\b Files and Folders +\f0\b0 , then click the mark in front of +\f1\b Installer +\f0\b0 to expand, and enable +\f1\b Downloads Folder +\f0\b0 by moving the toggle to the right. See {\field{\*\fldinst{HYPERLINK "https://github.com/python/cpython/issues/103207"}}{\fldrslt https://github.com/python/cpython/issues/103207}} for up-to-date information on this issue.\ \ \f1\b NOTE: From a44568b80dfc494759d45db59423ed314bc70f9e Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Wed, 5 Apr 2023 21:30:11 -0700 Subject: [PATCH 33/76] [Enum] unchain exception property.__get__ (GH-103305) --- Lib/enum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/enum.py b/Lib/enum.py index 0b0629cf920506..10902c4b202a2d 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -207,7 +207,7 @@ def __get__(self, instance, ownerclass=None): except KeyError: raise AttributeError( '%r has no attribute %r' % (ownerclass, self.name) - ) + ) from None else: return self.fget(instance) From 482b6eeadcde3e6573f0d243e687de7be7680379 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 6 Apr 2023 11:08:25 +0100 Subject: [PATCH 34/76] gh-102799: use sys.exception() instead of sys.exc_info() in tests (#103293) --- Lib/test/profilee.py | 2 +- Lib/test/support/asyncore.py | 7 +-- Lib/test/test_asyncio/test_tasks.py | 2 +- Lib/test/test_asyncio/utils.py | 2 +- Lib/test/test_class.py | 2 +- Lib/test/test_cprofile.py | 2 +- Lib/test/test_exceptions.py | 66 ++++++++++++++--------------- Lib/test/test_generators.py | 34 +++++++-------- Lib/test/test_logging.py | 3 +- Lib/test/test_profile.py | 2 +- Lib/test/test_ssl.py | 2 +- Lib/test/test_traceback.py | 15 +++---- 12 files changed, 68 insertions(+), 71 deletions(-) diff --git a/Lib/test/profilee.py b/Lib/test/profilee.py index 6ad2c8395634fd..b6a090a2e34613 100644 --- a/Lib/test/profilee.py +++ b/Lib/test/profilee.py @@ -79,7 +79,7 @@ def helper1(): TICKS += 19 lst = [] lst.append(42) # 0 - sys.exc_info() # 0 + sys.exception() # 0 def helper2_indirect(): helper2() # 50 diff --git a/Lib/test/support/asyncore.py b/Lib/test/support/asyncore.py index 401fa60bcf35f2..b397aca5568079 100644 --- a/Lib/test/support/asyncore.py +++ b/Lib/test/support/asyncore.py @@ -537,10 +537,11 @@ def send(self, data): # --------------------------------------------------------------------------- def compact_traceback(): - t, v, tb = sys.exc_info() - tbinfo = [] + exc = sys.exception() + tb = exc.__traceback__ if not tb: # Must have a traceback raise AssertionError("traceback does not exist") + tbinfo = [] while tb: tbinfo.append(( tb.tb_frame.f_code.co_filename, @@ -554,7 +555,7 @@ def compact_traceback(): file, function, line = tbinfo[-1] info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo]) - return (file, function, line), t, v, info + return (file, function, line), type(exc), exc, info def close_all(map=None, ignore_all=False): if map is None: diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 731fa0c5a60b9b..31622c91470bcb 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -606,7 +606,7 @@ def on_timeout(): if ( timed_out and task.uncancel() == 0 - and sys.exc_info()[0] is asyncio.CancelledError + and type(sys.exception()) is asyncio.CancelledError ): # Note the five rules that are needed here to satisfy proper # uncancellation: diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 5b9c86eb9859a0..6dee5bb33b2560 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -577,7 +577,7 @@ def tearDown(self): # Detect CPython bug #23353: ensure that yield/yield-from is not used # in an except block of a generator - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertIsNone(sys.exception()) self.doCleanups() threading_helper.threading_cleanup(*self._thread_cleanup) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 61df81b169775e..d7a48e55b10180 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -457,7 +457,7 @@ def __init__(self): a = A() self.assertEqual(_testcapi.hasattr_string(a, "attr"), True) self.assertEqual(_testcapi.hasattr_string(a, "noattr"), False) - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertIsNone(sys.exception()) def testDel(self): x = [] diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 4ec769885292a8..98648528bc81f2 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -100,7 +100,7 @@ def main(): profilee.py:98(subhelper) <- 8 0.064 0.080 profilee.py:88(helper2) {built-in method builtins.hasattr} <- 4 0.000 0.004 profilee.py:73(helper1) 8 0.000 0.008 profilee.py:88(helper2) -{built-in method sys.exc_info} <- 4 0.000 0.000 profilee.py:73(helper1) +{built-in method sys.exception} <- 4 0.000 0.000 profilee.py:73(helper1) {method 'append' of 'list' objects} <- 4 0.000 0.000 profilee.py:73(helper1)""" _ProfileOutput['print_callees'] = """\ :1() -> 1 0.270 1.000 profilee.py:25(testfunc) diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 74a5884264d577..684e888f08c778 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -334,8 +334,7 @@ def test_capi1(): try: _testcapi.raise_exception(BadException, 1) except TypeError as err: - exc, err, tb = sys.exc_info() - co = tb.tb_frame.f_code + co = err.__traceback__.tb_frame.f_code self.assertEqual(co.co_name, "test_capi1") self.assertTrue(co.co_filename.endswith('test_exceptions.py')) else: @@ -346,8 +345,7 @@ def test_capi2(): try: _testcapi.raise_exception(BadException, 0) except RuntimeError as err: - exc, err, tb = sys.exc_info() - tb = tb.tb_next + tb = err.__traceback__.tb_next co = tb.tb_frame.f_code self.assertEqual(co.co_name, "__init__") self.assertTrue(co.co_filename.endswith('test_exceptions.py')) @@ -888,28 +886,28 @@ def yield_raise(): try: raise KeyError("caught") except KeyError: - yield sys.exc_info()[0] - yield sys.exc_info()[0] - yield sys.exc_info()[0] + yield sys.exception() + yield sys.exception() + yield sys.exception() g = yield_raise() - self.assertEqual(next(g), KeyError) - self.assertEqual(sys.exc_info()[0], None) - self.assertEqual(next(g), KeyError) - self.assertEqual(sys.exc_info()[0], None) - self.assertEqual(next(g), None) + self.assertIsInstance(next(g), KeyError) + self.assertIsNone(sys.exception()) + self.assertIsInstance(next(g), KeyError) + self.assertIsNone(sys.exception()) + self.assertIsNone(next(g)) # Same test, but inside an exception handler try: raise TypeError("foo") except TypeError: g = yield_raise() - self.assertEqual(next(g), KeyError) - self.assertEqual(sys.exc_info()[0], TypeError) - self.assertEqual(next(g), KeyError) - self.assertEqual(sys.exc_info()[0], TypeError) - self.assertEqual(next(g), TypeError) + self.assertIsInstance(next(g), KeyError) + self.assertIsInstance(sys.exception(), TypeError) + self.assertIsInstance(next(g), KeyError) + self.assertIsInstance(sys.exception(), TypeError) + self.assertIsInstance(next(g), TypeError) del g - self.assertEqual(sys.exc_info()[0], TypeError) + self.assertIsInstance(sys.exception(), TypeError) def test_generator_leaking2(self): # See issue 12475. @@ -924,7 +922,7 @@ def g(): next(it) except StopIteration: pass - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertIsNone(sys.exception()) def test_generator_leaking3(self): # See issue #23353. When gen.throw() is called, the caller's @@ -933,17 +931,17 @@ def g(): try: yield except ZeroDivisionError: - yield sys.exc_info()[1] + yield sys.exception() it = g() next(it) try: 1/0 except ZeroDivisionError as e: - self.assertIs(sys.exc_info()[1], e) + self.assertIs(sys.exception(), e) gen_exc = it.throw(e) - self.assertIs(sys.exc_info()[1], e) + self.assertIs(sys.exception(), e) self.assertIs(gen_exc, e) - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertIsNone(sys.exception()) def test_generator_leaking4(self): # See issue #23353. When an exception is raised by a generator, @@ -952,7 +950,7 @@ def g(): try: 1/0 except ZeroDivisionError: - yield sys.exc_info()[0] + yield sys.exception() raise it = g() try: @@ -960,7 +958,7 @@ def g(): except TypeError: # The caller's exception state (TypeError) is temporarily # saved in the generator. - tp = next(it) + tp = type(next(it)) self.assertIs(tp, ZeroDivisionError) try: next(it) @@ -968,15 +966,15 @@ def g(): # with an exception, it shouldn't have restored the old # exception state (TypeError). except ZeroDivisionError as e: - self.assertIs(sys.exc_info()[1], e) + self.assertIs(sys.exception(), e) # We used to find TypeError here. - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertIsNone(sys.exception()) def test_generator_doesnt_retain_old_exc(self): def g(): - self.assertIsInstance(sys.exc_info()[1], RuntimeError) + self.assertIsInstance(sys.exception(), RuntimeError) yield - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertIsNone(sys.exception()) it = g() try: raise RuntimeError @@ -984,7 +982,7 @@ def g(): next(it) self.assertRaises(StopIteration, next, it) - def test_generator_finalizing_and_exc_info(self): + def test_generator_finalizing_and_sys_exception(self): # See #7173 def simple_gen(): yield 1 @@ -996,7 +994,7 @@ def run_gen(): return next(gen) run_gen() gc_collect() - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertIsNone(sys.exception()) def _check_generator_cleanup_exc_state(self, testfunc): # Issue #12791: exception state is cleaned up as soon as a generator @@ -1067,14 +1065,14 @@ def test_3114(self): class MyObject: def __del__(self): nonlocal e - e = sys.exc_info() + e = sys.exception() e = () try: raise Exception(MyObject()) except: pass gc_collect() # For PyPy or other GCs. - self.assertEqual(e, (None, None, None)) + self.assertIsNone(e) def test_raise_does_not_create_context_chain_cycle(self): class A(Exception): @@ -1692,7 +1690,7 @@ def g(): raise ValueError except ValueError: yield 1 - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertIsNone(sys.exception()) yield 2 gen = g() diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 492b77a954d865..a5949dec70d176 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -234,16 +234,16 @@ class ExceptionTest(unittest.TestCase): def test_except_throw(self): def store_raise_exc_generator(): try: - self.assertEqual(sys.exc_info()[0], None) + self.assertIsNone(sys.exception()) yield except Exception as exc: # exception raised by gen.throw(exc) - self.assertEqual(sys.exc_info()[0], ValueError) + self.assertIsInstance(sys.exception(), ValueError) self.assertIsNone(exc.__context__) yield # ensure that the exception is not lost - self.assertEqual(sys.exc_info()[0], ValueError) + self.assertIsInstance(sys.exception(), ValueError) yield # we should be able to raise back the ValueError @@ -265,11 +265,11 @@ def store_raise_exc_generator(): next(make) self.assertIsNone(cm.exception.__context__) - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertIsNone(sys.exception()) def test_except_next(self): def gen(): - self.assertEqual(sys.exc_info()[0], ValueError) + self.assertIsInstance(sys.exception(), ValueError) yield "done" g = gen() @@ -277,23 +277,23 @@ def gen(): raise ValueError except Exception: self.assertEqual(next(g), "done") - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertIsNone(sys.exception()) def test_except_gen_except(self): def gen(): try: - self.assertEqual(sys.exc_info()[0], None) + self.assertIsNone(sys.exception()) yield # we are called from "except ValueError:", TypeError must # inherit ValueError in its context raise TypeError() except TypeError as exc: - self.assertEqual(sys.exc_info()[0], TypeError) + self.assertIsInstance(sys.exception(), TypeError) self.assertEqual(type(exc.__context__), ValueError) # here we are still called from the "except ValueError:" - self.assertEqual(sys.exc_info()[0], ValueError) + self.assertIsInstance(sys.exception(), ValueError) yield - self.assertIsNone(sys.exc_info()[0]) + self.assertIsNone(sys.exception()) yield "done" g = gen() @@ -304,7 +304,7 @@ def gen(): next(g) self.assertEqual(next(g), "done") - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertIsNone(sys.exception()) def test_nested_gen_except_loop(self): def gen(): @@ -330,19 +330,19 @@ def test_except_throw_exception_context(self): def gen(): try: try: - self.assertEqual(sys.exc_info()[0], None) + self.assertIsNone(sys.exception()) yield except ValueError: # we are called from "except ValueError:" - self.assertEqual(sys.exc_info()[0], ValueError) + self.assertIsInstance(sys.exception(), ValueError) raise TypeError() except Exception as exc: - self.assertEqual(sys.exc_info()[0], TypeError) + self.assertIsInstance(sys.exception(), TypeError) self.assertEqual(type(exc.__context__), ValueError) # we are still called from "except ValueError:" - self.assertEqual(sys.exc_info()[0], ValueError) + self.assertIsInstance(sys.exception(), ValueError) yield - self.assertIsNone(sys.exc_info()[0]) + self.assertIsNone(sys.exception()) yield "done" g = gen() @@ -353,7 +353,7 @@ def gen(): g.throw(exc) self.assertEqual(next(g), "done") - self.assertEqual(sys.exc_info(), (None, None, None)) + self.assertIsNone(sys.exception()) def test_except_throw_bad_exception(self): class E(Exception): diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 072056d3722106..c6de34e9dbdc8f 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -5097,8 +5097,7 @@ def test_encoding_errors_none(self): message = [] def dummy_handle_error(record): - _, v, _ = sys.exc_info() - message.append(str(v)) + message.append(str(sys.exception())) handler.handleError = dummy_handle_error logging.debug('The Øresund Bridge joins Copenhagen to Malmö') diff --git a/Lib/test/test_profile.py b/Lib/test/test_profile.py index d97fe447c38b01..a1dfc9abbb8ef7 100644 --- a/Lib/test/test_profile.py +++ b/Lib/test/test_profile.py @@ -178,7 +178,7 @@ def main(): 8 63.976 7.997 79.960 9.995 profilee.py:98(subhelper)""" _ProfileOutput['print_callers'] = """\ :0(append) <- profilee.py:73(helper1)(4) 119.964 -:0(exc_info) <- profilee.py:73(helper1)(4) 119.964 +:0(exception) <- profilee.py:73(helper1)(4) 119.964 :0(hasattr) <- profilee.py:73(helper1)(4) 119.964 profilee.py:88(helper2)(8) 399.912 profilee.py:110(__getattr__) <- :0(hasattr)(12) 11.964 diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index abf024fb89d2f3..d46ce5e60e2141 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -253,7 +253,7 @@ def wrapper(*args, **kw): def handle_error(prefix): - exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) + exc_format = ' '.join(traceback.format_exception(sys.exception())) if support.verbose: sys.stdout.write(prefix + exc_format) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 399c59f8780d8e..a6172ff05eed47 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1211,8 +1211,7 @@ def test_recursive_traceback_python(self): def test_recursive_traceback_cpython_internal(self): from _testcapi import exception_print def render_exc(): - exc_type, exc_value, exc_tb = sys.exc_info() - exception_print(exc_value) + exception_print(sys.exception()) self._check_recursive_traceback_display(render_exc) def test_format_stack(self): @@ -2470,8 +2469,8 @@ def test_cause(self): try: 1/0 finally: - exc_info_context = sys.exc_info() - exc_context = traceback.TracebackException(*exc_info_context) + exc = sys.exception() + exc_context = traceback.TracebackException.from_exception(exc) cause = Exception("cause") raise Exception("uh oh") from cause except Exception as e: @@ -2492,8 +2491,8 @@ def test_context(self): try: 1/0 finally: - exc_info_context = sys.exc_info() - exc_context = traceback.TracebackException(*exc_info_context) + exc = sys.exception() + exc_context = traceback.TracebackException.from_exception(exc) raise Exception("uh oh") except Exception as e: exc_obj = e @@ -2557,8 +2556,8 @@ def test_compact_no_cause(self): try: 1/0 finally: - exc_info_context = sys.exc_info() - exc_context = traceback.TracebackException(*exc_info_context) + exc = sys.exception() + exc_context = traceback.TracebackException.from_exception(exc) raise Exception("uh oh") except Exception as e: exc_obj = e From 52bc2e7b9d451821513a580a9b73c20cfdcf2b21 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Thu, 6 Apr 2023 14:05:23 +0100 Subject: [PATCH 35/76] gh-48330: assert warning is emitted on unittest.TestResult with no addDuration (#103309) --- Lib/test/test_unittest/test_case.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index 05d60a8ad3cf94..dd5ff6d553e61d 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -304,7 +304,8 @@ def defaultTestResult(self): def test(self): pass - Foo('test').run() + with self.assertWarns(RuntimeWarning): + Foo('test').run() def test_deprecation_of_return_val_from_test(self): # Issue 41322 - deprecate return of value that is not None from a test From 23cf1e20a6470588fbc64483031ceeec7614dc56 Mon Sep 17 00:00:00 2001 From: Jeffrey Newman Date: Thu, 6 Apr 2023 10:59:36 -0500 Subject: [PATCH 36/76] gh-99202: Fix extension type from documentation for compiling in C++20 mode (#102518) --- Doc/extending/newtypes_tutorial.rst | 4 ++-- Doc/includes/custom.c | 4 ++-- Doc/includes/custom2.c | 4 ++-- Doc/includes/custom3.c | 4 ++-- Doc/includes/custom4.c | 4 ++-- .../2023-03-07-23-30-29.gh-issue-99202.hhiAJF.rst | 1 + 6 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2023-03-07-23-30-29.gh-issue-99202.hhiAJF.rst diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 54de3fd42437d9..f89934a11f12a8 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -88,7 +88,7 @@ standard Python floats:: The second bit is the definition of the type object. :: static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "custom.Custom", .tp_doc = PyDoc_STR("Custom objects"), .tp_basicsize = sizeof(CustomObject), @@ -109,7 +109,7 @@ common practice to not specify them explicitly unless you need them. We're going to pick it apart, one field at a time:: - PyVarObject_HEAD_INIT(NULL, 0) + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) This line is mandatory boilerplate to initialize the ``ob_base`` field mentioned above. :: diff --git a/Doc/includes/custom.c b/Doc/includes/custom.c index 26ca754964733d..9cfba50ace25db 100644 --- a/Doc/includes/custom.c +++ b/Doc/includes/custom.c @@ -7,7 +7,7 @@ typedef struct { } CustomObject; static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "custom.Custom", .tp_doc = PyDoc_STR("Custom objects"), .tp_basicsize = sizeof(CustomObject), @@ -17,7 +17,7 @@ static PyTypeObject CustomType = { }; static PyModuleDef custommodule = { - PyModuleDef_HEAD_INIT, + .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom", .m_doc = "Example module that creates an extension type.", .m_size = -1, diff --git a/Doc/includes/custom2.c b/Doc/includes/custom2.c index a3b2d6ab78d3c4..a0222b1795209b 100644 --- a/Doc/includes/custom2.c +++ b/Doc/includes/custom2.c @@ -90,7 +90,7 @@ static PyMethodDef Custom_methods[] = { }; static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "custom2.Custom", .tp_doc = PyDoc_STR("Custom objects"), .tp_basicsize = sizeof(CustomObject), @@ -104,7 +104,7 @@ static PyTypeObject CustomType = { }; static PyModuleDef custommodule = { - PyModuleDef_HEAD_INIT, + .m_base =PyModuleDef_HEAD_INIT, .m_name = "custom2", .m_doc = "Example module that creates an extension type.", .m_size = -1, diff --git a/Doc/includes/custom3.c b/Doc/includes/custom3.c index 1a68bc4be8c399..4aeebe0a7507d1 100644 --- a/Doc/includes/custom3.c +++ b/Doc/includes/custom3.c @@ -130,7 +130,7 @@ static PyMethodDef Custom_methods[] = { }; static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "custom3.Custom", .tp_doc = PyDoc_STR("Custom objects"), .tp_basicsize = sizeof(CustomObject), @@ -145,7 +145,7 @@ static PyTypeObject CustomType = { }; static PyModuleDef custommodule = { - PyModuleDef_HEAD_INIT, + .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom3", .m_doc = "Example module that creates an extension type.", .m_size = -1, diff --git a/Doc/includes/custom4.c b/Doc/includes/custom4.c index b932d159d26e93..3998918f68301e 100644 --- a/Doc/includes/custom4.c +++ b/Doc/includes/custom4.c @@ -146,7 +146,7 @@ static PyMethodDef Custom_methods[] = { }; static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "custom4.Custom", .tp_doc = PyDoc_STR("Custom objects"), .tp_basicsize = sizeof(CustomObject), @@ -163,7 +163,7 @@ static PyTypeObject CustomType = { }; static PyModuleDef custommodule = { - PyModuleDef_HEAD_INIT, + .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom4", .m_doc = "Example module that creates an extension type.", .m_size = -1, diff --git a/Misc/NEWS.d/next/Documentation/2023-03-07-23-30-29.gh-issue-99202.hhiAJF.rst b/Misc/NEWS.d/next/Documentation/2023-03-07-23-30-29.gh-issue-99202.hhiAJF.rst new file mode 100644 index 00000000000000..1569e815ee50fa --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2023-03-07-23-30-29.gh-issue-99202.hhiAJF.rst @@ -0,0 +1 @@ +Fix extension type from documentation for compiling in C++20 mode From 52e9b389a8773f485c3b0f1d62bf3cce4812d44d Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 6 Apr 2023 12:00:49 -0600 Subject: [PATCH 37/76] gh-100227: Use an Array for _PyRuntime's Set of Locks During Init (gh-103315) This cleans things up a bit and simplifies adding new granular global locks. --- Python/pystate.c | 108 +++++++++++++++++++++-------------------------- 1 file changed, 48 insertions(+), 60 deletions(-) diff --git a/Python/pystate.c b/Python/pystate.c index 1e59a8c5f89717..d09c1d5743a4c6 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -354,47 +354,29 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS static const _PyRuntimeState initial = _PyRuntimeState_INIT(_PyRuntime); _Py_COMP_DIAG_POP +#define NUMLOCKS 4 + static int -alloc_for_runtime(PyThread_type_lock *plock1, PyThread_type_lock *plock2, - PyThread_type_lock *plock3, PyThread_type_lock *plock4) +alloc_for_runtime(PyThread_type_lock locks[NUMLOCKS]) { /* Force default allocator, since _PyRuntimeState_Fini() must use the same allocator than this function. */ PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - PyThread_type_lock lock1 = PyThread_allocate_lock(); - if (lock1 == NULL) { - return -1; - } - - PyThread_type_lock lock2 = PyThread_allocate_lock(); - if (lock2 == NULL) { - PyThread_free_lock(lock1); - return -1; - } - - PyThread_type_lock lock3 = PyThread_allocate_lock(); - if (lock3 == NULL) { - PyThread_free_lock(lock1); - PyThread_free_lock(lock2); - return -1; - } - - PyThread_type_lock lock4 = PyThread_allocate_lock(); - if (lock4 == NULL) { - PyThread_free_lock(lock1); - PyThread_free_lock(lock2); - PyThread_free_lock(lock3); - return -1; + for (int i = 0; i < NUMLOCKS; i++) { + PyThread_type_lock lock = PyThread_allocate_lock(); + if (lock == NULL) { + for (int j = 0; j < i; j++) { + PyThread_free_lock(locks[j]); + locks[j] = NULL; + } + break; + } + locks[i] = lock; } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - *plock1 = lock1; - *plock2 = lock2; - *plock3 = lock3; - *plock4 = lock4; return 0; } @@ -403,10 +385,7 @@ init_runtime(_PyRuntimeState *runtime, void *open_code_hook, void *open_code_userdata, _Py_AuditHookEntry *audit_hook_head, Py_ssize_t unicode_next_index, - PyThread_type_lock unicode_ids_mutex, - PyThread_type_lock interpreters_mutex, - PyThread_type_lock xidregistry_mutex, - PyThread_type_lock getargs_mutex) + PyThread_type_lock locks[NUMLOCKS]) { if (runtime->_initialized) { Py_FatalError("runtime already initialized"); @@ -424,17 +403,21 @@ init_runtime(_PyRuntimeState *runtime, PyPreConfig_InitPythonConfig(&runtime->preconfig); - runtime->interpreters.mutex = interpreters_mutex; - - runtime->xidregistry.mutex = xidregistry_mutex; - - runtime->getargs.mutex = getargs_mutex; + PyThread_type_lock *lockptrs[NUMLOCKS] = { + &runtime->interpreters.mutex, + &runtime->xidregistry.mutex, + &runtime->getargs.mutex, + &runtime->unicode_state.ids.lock, + }; + for (int i = 0; i < NUMLOCKS; i++) { + assert(locks[i] != NULL); + *lockptrs[i] = locks[i]; + } // Set it to the ID of the main thread of the main interpreter. runtime->main_thread = PyThread_get_thread_ident(); runtime->unicode_state.ids.next_index = unicode_next_index; - runtime->unicode_state.ids.lock = unicode_ids_mutex; runtime->_initialized = 1; } @@ -452,8 +435,8 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) // is called multiple times. Py_ssize_t unicode_next_index = runtime->unicode_state.ids.next_index; - PyThread_type_lock lock1, lock2, lock3, lock4; - if (alloc_for_runtime(&lock1, &lock2, &lock3, &lock4) != 0) { + PyThread_type_lock locks[NUMLOCKS]; + if (alloc_for_runtime(locks) != 0) { return _PyStatus_NO_MEMORY(); } @@ -474,7 +457,7 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime) } init_runtime(runtime, open_code_hook, open_code_userdata, audit_hook_head, - unicode_next_index, lock1, lock2, lock3, lock4); + unicode_next_index, locks); return _PyStatus_OK(); } @@ -504,10 +487,15 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime) LOCK = NULL; \ } - FREE_LOCK(runtime->interpreters.mutex); - FREE_LOCK(runtime->xidregistry.mutex); - FREE_LOCK(runtime->unicode_state.ids.lock); - FREE_LOCK(runtime->getargs.mutex); + PyThread_type_lock *lockptrs[NUMLOCKS] = { + &runtime->interpreters.mutex, + &runtime->xidregistry.mutex, + &runtime->getargs.mutex, + &runtime->unicode_state.ids.lock, + }; + for (int i = 0; i < NUMLOCKS; i++) { + FREE_LOCK(*lockptrs[i]); + } #undef FREE_LOCK PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -527,25 +515,25 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - int reinit_interp = _PyThread_at_fork_reinit(&runtime->interpreters.mutex); - int reinit_xidregistry = _PyThread_at_fork_reinit(&runtime->xidregistry.mutex); - int reinit_unicode_ids = _PyThread_at_fork_reinit(&runtime->unicode_state.ids.lock); - int reinit_getargs = _PyThread_at_fork_reinit(&runtime->getargs.mutex); + PyThread_type_lock *lockptrs[NUMLOCKS] = { + &runtime->interpreters.mutex, + &runtime->xidregistry.mutex, + &runtime->getargs.mutex, + &runtime->unicode_state.ids.lock, + }; + int reinit_err = 0; + for (int i = 0; i < NUMLOCKS; i++) { + reinit_err += _PyThread_at_fork_reinit(lockptrs[i]); + } PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); /* bpo-42540: id_mutex is freed by _PyInterpreterState_Delete, which does * not force the default allocator. */ - int reinit_main_id = _PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex); + reinit_err += _PyThread_at_fork_reinit(&runtime->interpreters.main->id_mutex); - if (reinit_interp < 0 - || reinit_main_id < 0 - || reinit_xidregistry < 0 - || reinit_unicode_ids < 0 - || reinit_getargs < 0) - { + if (reinit_err < 0) { return _PyStatus_ERR("Failed to reinitialize runtime locks"); - } PyStatus status = gilstate_tss_reinit(runtime); From 1724553e6e8baae655901488968a40df981f32da Mon Sep 17 00:00:00 2001 From: Ijtaba Hussain Date: Thu, 6 Apr 2023 23:13:33 +0500 Subject: [PATCH 38/76] gh-103186: assert in tests that UnsafeMailcapInput warnings are provided (#103217) --- Lib/test/test_mailcap.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_mailcap.py b/Lib/test/test_mailcap.py index 819dc80a266433..8a94b0cb1f27c7 100644 --- a/Lib/test/test_mailcap.py +++ b/Lib/test/test_mailcap.py @@ -127,7 +127,6 @@ def test_subst(self): (["", "audio/*", "foo.txt"], ""), (["echo foo", "audio/*", "foo.txt"], "echo foo"), (["echo %s", "audio/*", "foo.txt"], "echo foo.txt"), - (["echo %t", "audio/*", "foo.txt"], None), (["echo %t", "audio/wav", "foo.txt"], "echo audio/wav"), (["echo \\%t", "audio/*", "foo.txt"], "echo %t"), (["echo foo", "audio/*", "foo.txt", plist], "echo foo"), @@ -210,9 +209,6 @@ def test_findmatch(self): ([c, "audio/basic"], {"key": "description", "filename": fname}, ('"An audio fragment"', audio_basic_entry)), - ([c, "audio/*"], - {"filename": fname}, - (None, None)), ([c, "audio/wav"], {"filename": fname}, ("/usr/local/bin/showaudio audio/wav", audio_entry)), @@ -245,6 +241,30 @@ def test_test(self): ] self._run_cases(cases) + def test_unsafe_mailcap_input(self): + with self.assertWarnsRegex(mailcap.UnsafeMailcapInput, + 'Refusing to substitute parameter.*' + 'into a shell command'): + unsafe_param = mailcap.subst("echo %{total}", + "audio/wav", + "foo.txt", + ["total=*"]) + self.assertEqual(unsafe_param, None) + + with self.assertWarnsRegex(mailcap.UnsafeMailcapInput, + 'Refusing to substitute MIME type' + '.*into a shell'): + unsafe_mimetype = mailcap.subst("echo %t", "audio/*", "foo.txt") + self.assertEqual(unsafe_mimetype, None) + + with self.assertWarnsRegex(mailcap.UnsafeMailcapInput, + 'Refusing to use mailcap with filename.*' + 'Use a safe temporary filename.'): + unsafe_filename = mailcap.findmatch(MAILCAPDICT, + "audio/wav", + filename="foo*.txt") + self.assertEqual(unsafe_filename, (None, None)) + def _run_cases(self, cases): for c in cases: self.assertEqual(mailcap.findmatch(*c[0], **c[1]), c[2]) From affedee8bf2ec00c404ffa39342a593a66bf95bd Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 6 Apr 2023 20:17:53 +0100 Subject: [PATCH 39/76] gh-103193: Use LBYL idioms rather than EAFP in `inspect.getattr_static` (#103318) --- Lib/inspect.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index 8739c9c2572643..a317f0ca74884f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1787,11 +1787,8 @@ def _check_instance(obj, attr): def _check_class(klass, attr): for entry in _static_getmro(klass): - if _shadowed_dict(type(entry)) is _sentinel: - try: - return entry.__dict__[attr] - except KeyError: - pass + if _shadowed_dict(type(entry)) is _sentinel and attr in entry.__dict__: + return entry.__dict__[attr] return _sentinel def _is_type(obj): @@ -1803,11 +1800,9 @@ def _is_type(obj): def _shadowed_dict(klass): for entry in _static_getmro(klass): - try: - class_dict = _get_dunder_dict_of_class(entry)["__dict__"] - except KeyError: - pass - else: + dunder_dict = _get_dunder_dict_of_class(entry) + if '__dict__' in dunder_dict: + class_dict = dunder_dict['__dict__'] if not (type(class_dict) is types.GetSetDescriptorType and class_dict.__name__ == "__dict__" and class_dict.__objclass__ is entry): @@ -1850,11 +1845,11 @@ def getattr_static(obj, attr, default=_sentinel): if obj is klass: # for types we check the metaclass too for entry in _static_getmro(type(klass)): - if _shadowed_dict(type(entry)) is _sentinel: - try: - return entry.__dict__[attr] - except KeyError: - pass + if ( + _shadowed_dict(type(entry)) is _sentinel + and attr in entry.__dict__ + ): + return entry.__dict__[attr] if default is not _sentinel: return default raise AttributeError(attr) From dca7d174f1dc3f9e67c7451a27bc92dc5a733008 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Thu, 6 Apr 2023 21:49:24 +0100 Subject: [PATCH 40/76] gh-103193: Speedup and inline `inspect._is_type` (#103321) Improve performance of `inspect.getattr_static` --- Lib/inspect.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/Lib/inspect.py b/Lib/inspect.py index a317f0ca74884f..4242b40c2a08df 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1791,13 +1791,6 @@ def _check_class(klass, attr): return entry.__dict__[attr] return _sentinel -def _is_type(obj): - try: - _static_getmro(obj) - except TypeError: - return False - return True - def _shadowed_dict(klass): for entry in _static_getmro(klass): dunder_dict = _get_dunder_dict_of_class(entry) @@ -1821,8 +1814,10 @@ def getattr_static(obj, attr, default=_sentinel): documentation for details. """ instance_result = _sentinel - if not _is_type(obj): - klass = type(obj) + + objtype = type(obj) + if type not in _static_getmro(objtype): + klass = objtype dict_attr = _shadowed_dict(klass) if (dict_attr is _sentinel or type(dict_attr) is types.MemberDescriptorType): From 5d08c3ff7d89ca11556f18663a372f6c12435504 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 6 Apr 2023 14:19:54 -0700 Subject: [PATCH 41/76] GH-102700: allow built-in modules to be submodules (GH-103162) --- Lib/importlib/_bootstrap.py | 2 -- Lib/test/test_importlib/builtin/test_finder.py | 17 ----------------- ...23-04-01-00-46-31.gh-issue-102700.493NB4.rst | 1 + 3 files changed, 1 insertion(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-01-00-46-31.gh-issue-102700.493NB4.rst diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index bebe7e15cbce67..22fa2469964ab3 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -887,8 +887,6 @@ class BuiltinImporter: @classmethod def find_spec(cls, fullname, path=None, target=None): - if path is not None: - return None if _imp.is_builtin(fullname): return spec_from_loader(fullname, cls, origin=cls._ORIGIN) else: diff --git a/Lib/test/test_importlib/builtin/test_finder.py b/Lib/test/test_importlib/builtin/test_finder.py index a4869e07b9c0c2..81dc5a3699d952 100644 --- a/Lib/test/test_importlib/builtin/test_finder.py +++ b/Lib/test/test_importlib/builtin/test_finder.py @@ -37,13 +37,6 @@ def test_failure(self): spec = self.machinery.BuiltinImporter.find_spec(name) self.assertIsNone(spec) - def test_ignore_path(self): - # The value for 'path' should always trigger a failed import. - with util.uncache(util.BUILTINS.good_name): - spec = self.machinery.BuiltinImporter.find_spec(util.BUILTINS.good_name, - ['pkg']) - self.assertIsNone(spec) - (Frozen_FindSpecTests, Source_FindSpecTests @@ -77,16 +70,6 @@ def test_failure(self): loader = self.machinery.BuiltinImporter.find_module('importlib') self.assertIsNone(loader) - def test_ignore_path(self): - # The value for 'path' should always trigger a failed import. - with util.uncache(util.BUILTINS.good_name): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader = self.machinery.BuiltinImporter.find_module( - util.BUILTINS.good_name, - ['pkg']) - self.assertIsNone(loader) - (Frozen_FinderTests, Source_FinderTests diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-01-00-46-31.gh-issue-102700.493NB4.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-01-00-46-31.gh-issue-102700.493NB4.rst new file mode 100644 index 00000000000000..46951486e4f9c9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-01-00-46-31.gh-issue-102700.493NB4.rst @@ -0,0 +1 @@ +Allow built-in modules to be submodules. This allows submodules to be statically linked into a CPython binary. From a653c32d08abfa34d70186542479e3a7e704f8bd Mon Sep 17 00:00:00 2001 From: AGZain <31054801+AGZain@users.noreply.github.com> Date: Thu, 6 Apr 2023 18:19:11 -0400 Subject: [PATCH 42/76] gh-93121: fix test_mailbox where some test cases were accidentally dropped (#93242) --- Lib/test/test_mailbox.py | 45 ++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 07c2764dfd1b2f..4c592eaf34da23 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -31,7 +31,7 @@ def _check_sample(self, msg): # Inspect a mailbox.Message representation of the sample message self.assertIsInstance(msg, email.message.Message) self.assertIsInstance(msg, mailbox.Message) - for key, value in _sample_headers.items(): + for key, value in _sample_headers: self.assertIn(value, msg.get_all(key)) self.assertTrue(msg.is_multipart()) self.assertEqual(len(msg.get_payload()), len(_sample_payloads)) @@ -2264,30 +2264,31 @@ def test_nonempty_maildir_both(self): _bytes_sample_message = _sample_message.encode('ascii') -_sample_headers = { - "Return-Path":"", - "X-Original-To":"gkj+person@localhost", - "Delivered-To":"gkj+person@localhost", - "Received":"""from localhost (localhost [127.0.0.1]) +_sample_headers = [ + ("Return-Path", ""), + ("X-Original-To", "gkj+person@localhost"), + ("Delivered-To", "gkj+person@localhost"), + ("Received", """from localhost (localhost [127.0.0.1]) by andy.gregorykjohnson.com (Postfix) with ESMTP id 356ED9DD17 - for ; Wed, 13 Jul 2005 17:23:16 -0400 (EDT)""", - "Delivered-To":"gkj@sundance.gregorykjohnson.com", - "Received":"""from localhost [127.0.0.1] + for ; Wed, 13 Jul 2005 17:23:16 -0400 (EDT)"""), + ("Delivered-To", "gkj@sundance.gregorykjohnson.com"), + ("Received", """from localhost [127.0.0.1] by localhost with POP3 (fetchmail-6.2.5) - for gkj+person@localhost (single-drop); Wed, 13 Jul 2005 17:23:16 -0400 (EDT)""", - "Received":"""from andy.gregorykjohnson.com (andy.gregorykjohnson.com [64.32.235.228]) + for gkj+person@localhost (single-drop); Wed, 13 Jul 2005 17:23:16 -0400 (EDT)"""), + ("Received", """from andy.gregorykjohnson.com (andy.gregorykjohnson.com [64.32.235.228]) by sundance.gregorykjohnson.com (Postfix) with ESMTP id 5B056316746 - for ; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)""", - "Received":"""by andy.gregorykjohnson.com (Postfix, from userid 1000) - id 490CD9DD17; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)""", - "Date":"Wed, 13 Jul 2005 17:23:11 -0400", - "From":""""Gregory K. Johnson" """, - "To":"gkj@gregorykjohnson.com", - "Subject":"Sample message", - "Mime-Version":"1.0", - "Content-Type":"""multipart/mixed; boundary="NMuMz9nt05w80d4+\"""", - "Content-Disposition":"inline", - "User-Agent": "Mutt/1.5.9i" } + for ; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)"""), + ("Received", """by andy.gregorykjohnson.com (Postfix, from userid 1000) + id 490CD9DD17; Wed, 13 Jul 2005 17:23:11 -0400 (EDT)"""), + ("Date", "Wed, 13 Jul 2005 17:23:11 -0400"), + ("From", """"Gregory K. Johnson" """), + ("To", "gkj@gregorykjohnson.com"), + ("Subject", "Sample message"), + ("Mime-Version", "1.0"), + ("Content-Type", """multipart/mixed; boundary="NMuMz9nt05w80d4+\""""), + ("Content-Disposition", "inline"), + ("User-Agent", "Mutt/1.5.9i"), +] _sample_payloads = ("""This is a sample message. From f0424ba4b6663d2a4240239266bea08aff46bb6c Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Fri, 7 Apr 2023 04:34:19 +0400 Subject: [PATCH 43/76] gh-103266: Fix a typo in example code for bisect() function (#103267) --- Doc/library/bisect.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index b85564f17866e0..e3c8c801904b61 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -210,10 +210,10 @@ records in a table:: >>> Movie = namedtuple('Movie', ('name', 'released', 'director')) >>> movies = [ - ... Movie('Jaws', 1975, 'Speilberg'), + ... Movie('Jaws', 1975, 'Spielberg'), ... Movie('Titanic', 1997, 'Cameron'), ... Movie('The Birds', 1963, 'Hitchcock'), - ... Movie('Aliens', 1986, 'Scott') + ... Movie('Aliens', 1986, 'Cameron') ... ] >>> # Find the first movie released after 1960 @@ -228,8 +228,8 @@ records in a table:: >>> pprint(movies) [Movie(name='The Birds', released=1963, director='Hitchcock'), Movie(name='Love Story', released=1970, director='Hiller'), - Movie(name='Jaws', released=1975, director='Speilberg'), - Movie(name='Aliens', released=1986, director='Scott'), + Movie(name='Jaws', released=1975, director='Spielberg'), + Movie(name='Aliens', released=1986, director='Cameron'), Movie(name='Titanic', released=1997, director='Cameron')] If the key function is expensive, it is possible to avoid repeated function From efb0a2cf3adf4629cf4669cb558758fb78107319 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Fri, 7 Apr 2023 10:51:29 +0900 Subject: [PATCH 44/76] gh-103256: Fix hmac algorithm to support fallback implementation (gh-103286) Co-authored-by: Gregory P. Smith --- Lib/test/test_hmac.py | 10 ++++++++++ .../2023-04-06-17-28-36.gh-issue-103256.1syxfs.rst | 6 ++++++ Modules/_hashopenssl.c | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-06-17-28-36.gh-issue-103256.1syxfs.rst diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 7cf99735ca39f0..a39a2c45ebc2e2 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -373,6 +373,16 @@ def test_with_digestmod_no_default(self): with self.assertRaisesRegex(TypeError, r'required.*digestmod'): hmac.HMAC(key, msg=data, digestmod='') + def test_with_fallback(self): + cache = getattr(hashlib, '__builtin_constructor_cache') + try: + cache['foo'] = hashlib.sha256 + hexdigest = hmac.digest(b'key', b'message', 'foo').hex() + expected = '6e9ef29b75fffc5b7abae527d58fdadb2fe42e7219011976917343065f58ed4a' + self.assertEqual(hexdigest, expected) + finally: + cache.pop('foo') + class ConstructorTestCase(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2023-04-06-17-28-36.gh-issue-103256.1syxfs.rst b/Misc/NEWS.d/next/Library/2023-04-06-17-28-36.gh-issue-103256.1syxfs.rst new file mode 100644 index 00000000000000..894c046dcdf0fd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-06-17-28-36.gh-issue-103256.1syxfs.rst @@ -0,0 +1,6 @@ +Fixed a bug that caused :mod:`hmac` to raise an exception when the requested +hash algorithm was not available in OpenSSL despite being available +separately as part of ``hashlib`` itself. It now falls back properly to the +built-in. This could happen when, for example, your OpenSSL does not include +SHA3 support and you want to compute ``hmac.digest(b'K', b'M', +'sha3_256')``. diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index ee8c588020118c..7476e5dc7dd61e 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -355,7 +355,7 @@ py_digest_by_name(PyObject *module, const char *name, enum Py_hash_type py_ht) } } if (digest == NULL) { - _setException(PyExc_ValueError, "unsupported hash type %s", name); + _setException(state->unsupported_digestmod_error, "unsupported hash type %s", name); return NULL; } return digest; From 059bb04245a8b3490f93dfd72522a431a113eef1 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 7 Apr 2023 12:22:36 +0300 Subject: [PATCH 45/76] gh-102213: Revert "gh-102213: Optimize the performance of `__getattr__` (GH-102248)" (GH-103332) This reverts commit aa0a73d1bc53dcb6348a869df1e775138991e561. --- Include/internal/pycore_object.h | 1 - Objects/object.c | 6 ------ Objects/typeobject.c | 9 +++------ 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index e18e787449c257..b3d496ed6fc240 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -375,7 +375,6 @@ extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); extern int _PyType_HasSubclasses(PyTypeObject *); extern PyObject* _PyType_GetSubclasses(PyTypeObject *); -extern PyObject* _PyObject_GenericTryGetAttr(PyObject *, PyObject *); // Access macro to the members which are floating "behind" the object static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) { diff --git a/Objects/object.c b/Objects/object.c index 9dd5eb998217f6..71f098eed37f51 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1491,12 +1491,6 @@ PyObject_GenericGetAttr(PyObject *obj, PyObject *name) return _PyObject_GenericGetAttrWithDict(obj, name, NULL, 0); } -PyObject * -_PyObject_GenericTryGetAttr(PyObject *obj, PyObject *name) -{ - return _PyObject_GenericGetAttrWithDict(obj, name, NULL, 1); -} - int _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, PyObject *value, PyObject *dict) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 24541bddbbc33f..995547e7915f77 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -8274,17 +8274,14 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) (Py_IS_TYPE(getattribute, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)getattribute)->d_wrapped == (void *)PyObject_GenericGetAttr)) - /* finding nothing is reasonable when __getattr__ is defined */ - res = _PyObject_GenericTryGetAttr(self, name); + res = PyObject_GenericGetAttr(self, name); else { Py_INCREF(getattribute); res = call_attribute(self, getattribute, name); Py_DECREF(getattribute); } - if (res == NULL) { - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { - PyErr_Clear(); - } + if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); res = call_attribute(self, getattr, name); } Py_DECREF(getattr); From 995386071f96e4cfebfa027a71ca9134e4651d2a Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Fri, 7 Apr 2023 13:43:41 +0300 Subject: [PATCH 46/76] bpo-46523: fix tests rerun when `setUp[Class|Module]` fails (#30895) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jelle Zijlstra Co-authored-by: Łukasz Langa --- Lib/test/libregrtest/main.py | 31 ++++++- Lib/test/support/__init__.py | 2 +- Lib/test/test_regrtest.py | 154 +++++++++++++++++++++++++++++++++++ 3 files changed, 184 insertions(+), 3 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 19ccf2db5e7f06..3c3509d0303371 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -29,6 +29,14 @@ # Must be smaller than buildbot "1200 seconds without output" limit. EXIT_TIMEOUT = 120.0 +# gh-90681: When rerunning tests, we might need to rerun the whole +# class or module suite if some its life-cycle hooks fail. +# Test level hooks are not affected. +_TEST_LIFECYCLE_HOOKS = frozenset(( + 'setUpClass', 'tearDownClass', + 'setUpModule', 'tearDownModule', +)) + EXITCODE_BAD_TEST = 2 EXITCODE_INTERRUPTED = 130 EXITCODE_ENV_CHANGED = 3 @@ -337,8 +345,12 @@ def rerun_failed_tests(self): errors = result.errors or [] failures = result.failures or [] - error_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in errors] - failure_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in failures] + error_names = [ + self.normalize_test_name(test_full_name, is_error=True) + for (test_full_name, *_) in errors] + failure_names = [ + self.normalize_test_name(test_full_name) + for (test_full_name, *_) in failures] self.ns.verbose = True orig_match_tests = self.ns.match_tests if errors or failures: @@ -364,6 +376,21 @@ def rerun_failed_tests(self): self.display_result() + def normalize_test_name(self, test_full_name, *, is_error=False): + short_name = test_full_name.split(" ")[0] + if is_error and short_name in _TEST_LIFECYCLE_HOOKS: + # This means that we have a failure in a life-cycle hook, + # we need to rerun the whole module or class suite. + # Basically the error looks like this: + # ERROR: setUpClass (test.test_reg_ex.RegTest) + # or + # ERROR: setUpModule (test.test_reg_ex) + # So, we need to parse the class / module name. + lpar = test_full_name.index('(') + rpar = test_full_name.index(')') + return test_full_name[lpar + 1: rpar].split('.')[-1] + return short_name + def display_result(self): # If running the test suite for PGO then no one cares about results. if self.ns.pgo: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index c309fd7910e0e6..d063837baee2de 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -1108,7 +1108,7 @@ def _run_suite(suite): if junit_xml_list is not None: junit_xml_list.append(result.get_xml_element()) - if not result.testsRun and not result.skipped: + if not result.testsRun and not result.skipped and not result.errors: raise TestDidNotRun if not result.wasSuccessful(): if len(result.errors) == 1 and not result.failures: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index baae4efc2ad789..ac49fbae847726 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -1120,6 +1120,160 @@ def test_fail_once(self): self.check_executed_tests(output, [testname], rerun={testname: "test_fail_once"}) + def test_rerun_setup_class_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.TestCase): + @classmethod + def setUpClass(self): + raise RuntimeError('Fail') + + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun={testname: "ExampleTests"}) + + def test_rerun_teardown_class_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.TestCase): + @classmethod + def tearDownClass(self): + raise RuntimeError('Fail') + + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun={testname: "ExampleTests"}) + + def test_rerun_setup_module_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + def setUpModule(): + raise RuntimeError('Fail') + + class ExampleTests(unittest.TestCase): + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun={testname: testname}) + + def test_rerun_teardown_module_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + def tearDownModule(): + raise RuntimeError('Fail') + + class ExampleTests(unittest.TestCase): + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun={testname: testname}) + + def test_rerun_setup_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.TestCase): + def setUp(self): + raise RuntimeError('Fail') + + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun={testname: "test_success"}) + + def test_rerun_teardown_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.TestCase): + def tearDown(self): + raise RuntimeError('Fail') + + def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun={testname: "test_success"}) + + def test_rerun_async_setup_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + raise RuntimeError('Fail') + + async def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun={testname: "test_success"}) + + def test_rerun_async_teardown_hook_failure(self): + # FAILURE then FAILURE + code = textwrap.dedent(""" + import unittest + + class ExampleTests(unittest.IsolatedAsyncioTestCase): + async def asyncTearDown(self): + raise RuntimeError('Fail') + + async def test_success(self): + return + """) + testname = self.create_test(code=code) + + output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST) + self.check_executed_tests(output, testname, + failed=[testname], + rerun={testname: "test_success"}) + def test_no_tests_ran(self): code = textwrap.dedent(""" import unittest From 4dc339b4d69195448207e1faecc3e258700daf33 Mon Sep 17 00:00:00 2001 From: AN Long Date: Fri, 7 Apr 2023 19:56:00 +0800 Subject: [PATCH 47/76] GH-88013: Fix TypeError raised by ntpath.realpath in some cases (GH-102813) --- Lib/ntpath.py | 2 +- Lib/test/test_ntpath.py | 7 +++++++ .../Windows/2023-03-18-21-38-00.gh-issue-88013.Z3loxC.rst | 2 ++ 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Windows/2023-03-18-21-38-00.gh-issue-88013.Z3loxC.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index e93a5e69600973..6e2da79c85d3f0 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -670,7 +670,7 @@ def _getfinalpathname_nonstrict(path): # Non-strict algorithm is to find as much of the target directory # as we can and join the rest. - tail = '' + tail = path[:0] while path: try: path = _getfinalpathname(path) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 08c8a7a1f94b95..4e755d15403916 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1,6 +1,7 @@ import inspect import ntpath import os +import string import sys import unittest import warnings @@ -374,6 +375,12 @@ def test_realpath_basic(self): self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1")), os.fsencode(ABSTFN)) + # gh-88013: call ntpath.realpath with binary drive name may raise a + # TypeError. The drive should not exist to reproduce the bug. + drives = {f"{c}:\\" for c in string.ascii_uppercase} - set(os.listdrives()) + d = drives.pop().encode() + self.assertEqual(ntpath.realpath(d), d) + @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_strict(self): diff --git a/Misc/NEWS.d/next/Windows/2023-03-18-21-38-00.gh-issue-88013.Z3loxC.rst b/Misc/NEWS.d/next/Windows/2023-03-18-21-38-00.gh-issue-88013.Z3loxC.rst new file mode 100644 index 00000000000000..4ca3185ea1f65e --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-03-18-21-38-00.gh-issue-88013.Z3loxC.rst @@ -0,0 +1,2 @@ +Fixed a bug where :exc:`TypeError` was raised when calling +:func:`ntpath.realpath` with a bytes parameter in some cases. From 04501ee57e7f6ef216b15b45785285fe4acebc42 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 7 Apr 2023 18:14:50 +0100 Subject: [PATCH 48/76] gh-103193: Celebrate performance improvements to `inspect.getattr_static` in 'What's New in Python 3.12' (#103349) --- Doc/whatsnew/3.12.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 23524ec5d7d452..3a12fb20c43c51 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -255,6 +255,10 @@ inspect for determining the current state of asynchronous generators. (Contributed by Thomas Krennwallner in :issue:`35759`.) +* The performance of :func:`inspect.getattr_static` has been considerably + improved. Most calls to the function should be around 2x faster than they + were in Python 3.11. (Contributed by Alex Waygood in :gh:`103193`.) + pathlib ------- From 800382a2b0980c21dfb2a8ac02aaf1e881f987b9 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 7 Apr 2023 18:21:19 +0100 Subject: [PATCH 49/76] gh-74690: Add more tests for runtime-checkable protocols (#103347) --- Lib/test/test_typing.py | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 7d2e6a6a9f6287..b8eee9a570a301 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -2577,6 +2577,22 @@ def meth(x): ... class PG(Protocol[T]): def meth(x): ... + @runtime_checkable + class WeirdProto(Protocol): + meth = str.maketrans + + @runtime_checkable + class WeirdProto2(Protocol): + meth = lambda *args, **kwargs: None + + class CustomCallable: + def __call__(self, *args, **kwargs): + pass + + @runtime_checkable + class WeirderProto(Protocol): + meth = CustomCallable() + class BadP(Protocol): def meth(x): ... @@ -2586,8 +2602,15 @@ def meth(x): ... class C: def meth(x): ... - self.assertIsInstance(C(), P) - self.assertIsInstance(C(), PG) + class C2: + def __init__(self): + self.meth = lambda: None + + for klass in C, C2: + for proto in P, PG, WeirdProto, WeirdProto2, WeirderProto: + with self.subTest(klass=klass.__name__, proto=proto.__name__): + self.assertIsInstance(klass(), proto) + with self.assertRaises(TypeError): isinstance(C(), PG[T]) with self.assertRaises(TypeError): @@ -2829,6 +2852,20 @@ def __init__(self, x): self.assertIsInstance(C(1), P) self.assertIsInstance(C(1), PG) + def test_protocols_isinstance_monkeypatching(self): + @runtime_checkable + class HasX(Protocol): + x: int + + class Foo: ... + + f = Foo() + self.assertNotIsInstance(f, HasX) + f.x = 42 + self.assertIsInstance(f, HasX) + del f.x + self.assertNotIsInstance(f, HasX) + def test_protocol_checks_after_subscript(self): class P(Protocol[T]): pass class C(P[T]): pass From 2667452945eb0a3b8993bb4298ca8da54dc0155a Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 7 Apr 2023 10:57:46 -0700 Subject: [PATCH 50/76] gh-103225: Fixed zero lineno issue for pdb (#103265) Co-authored-by: Artem Mukhin --- Lib/pdb.py | 14 +++++++++-- Lib/test/test_pdb.py | 25 +++++++++++++++++++ ...-04-05-01-28-53.gh-issue-103225.QD3JVU.rst | 1 + 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-05-01-28-53.gh-issue-103225.QD3JVU.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index 3a06cd00ad2bf1..e043b0d46f7dd0 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1351,7 +1351,7 @@ def do_longlist(self, arg): filename = self.curframe.f_code.co_filename breaklist = self.get_file_breaks(filename) try: - lines, lineno = inspect.getsourcelines(self.curframe) + lines, lineno = self._getsourcelines(self.curframe) except OSError as err: self.error(err) return @@ -1367,7 +1367,7 @@ def do_source(self, arg): except: return try: - lines, lineno = inspect.getsourcelines(obj) + lines, lineno = self._getsourcelines(obj) except (OSError, TypeError) as err: self.error(err) return @@ -1662,6 +1662,16 @@ def _compile_error_message(self, expr): return _rstr(self._format_exc(exc)) return "" + def _getsourcelines(self, obj): + # GH-103319 + # inspect.getsourcelines() returns lineno = 0 for + # module-level frame which breaks our code print line number + # This method should be replaced by inspect.getsourcelines(obj) + # once this bug is fixed in inspect + lines, lineno = inspect.getsourcelines(obj) + lineno = max(1, lineno) + return lines, lineno + # Collect all command help into docstring, if not run with -OO if __doc__ is not None: diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index de2bab46495729..9ad9a1c52ac102 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -1675,6 +1675,31 @@ def test_pdb_issue_gh_101673(): (Pdb) continue """ +def test_pdb_issue_gh_103225(): + """See GH-103225 + + Make sure longlist uses 1-based line numbers in frames that correspond to a module + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'longlist', + ... 'continue' + ... ]): + ... a = 1 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... b = 2 + > (7)() + -> b = 2 + (Pdb) longlist + 1 with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + 2 'longlist', + 3 'continue' + 4 ]): + 5 a = 1 + 6 import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + 7 -> b = 2 + (Pdb) continue + """ + @support.requires_subprocess() class PdbTestCase(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2023-04-05-01-28-53.gh-issue-103225.QD3JVU.rst b/Misc/NEWS.d/next/Library/2023-04-05-01-28-53.gh-issue-103225.QD3JVU.rst new file mode 100644 index 00000000000000..5d1a063acdeb8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-05-01-28-53.gh-issue-103225.QD3JVU.rst @@ -0,0 +1 @@ +Fix a bug in :mod:`pdb` when displaying line numbers of module-level source code. From a90863c993157ae65e040476cf46abd73ae54b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Fri, 7 Apr 2023 18:23:59 +0000 Subject: [PATCH 51/76] gh-100220: Fix error handling in make rules (GH-100328) Set `SHELL = /bin/sh -e` to ensure that complex recipes fail on the first error rather than incorrectly reporting success. Co-authored-by: Zachary Ware --- Makefile.pre.in | 2 +- .../next/Build/2022-12-18-07-24-44.gh-issue-100220.BgSV7C.rst | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2022-12-18-07-24-44.gh-issue-100220.BgSV7C.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index 1c1bddcad82475..fefa5d4e8147bf 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -59,7 +59,7 @@ DSYMUTIL_PATH= @DSYMUTIL_PATH@ GNULD= @GNULD@ # Shell used by make (some versions default to the login shell, which is bad) -SHELL= /bin/sh +SHELL= /bin/sh -e # Use this to make a link between python$(VERSION) and python in $(BINDIR) LN= @LN@ diff --git a/Misc/NEWS.d/next/Build/2022-12-18-07-24-44.gh-issue-100220.BgSV7C.rst b/Misc/NEWS.d/next/Build/2022-12-18-07-24-44.gh-issue-100220.BgSV7C.rst new file mode 100644 index 00000000000000..7135317cd06fa2 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-12-18-07-24-44.gh-issue-100220.BgSV7C.rst @@ -0,0 +1,4 @@ +Changed the default value of the ``SHELL`` Makefile variable from ``/bin/sh`` +to ``/bin/sh -e`` to ensure that complex recipes correctly fail after an error. +Previously, ``make install`` could fail to install some files and yet return +a successful result. From 5d7d86f2fdbbfc23325e7256ee289bf20ce7124e Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Sat, 8 Apr 2023 03:11:11 +0800 Subject: [PATCH 52/76] gh-103272: regression test for getattr exception in property (#103336) --- Lib/test/test_descr.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index cbc020d1d3904a..f17bb1813b9d87 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5003,6 +5003,19 @@ class Child(Parent): gc.collect() self.assertEqual(Parent.__subclasses__(), []) + def test_attr_raise_through_property(self): + # add test case for gh-103272 + class A: + def __getattr__(self, name): + raise ValueError("FOO") + + @property + def foo(self): + return self.__getattr__("asdf") + + with self.assertRaisesRegex(ValueError, "FOO"): + A().foo + class DictProxyTests(unittest.TestCase): def setUp(self): From 644136563da653a93268cb8cae7e25ff691047a3 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Fri, 7 Apr 2023 22:06:37 +0100 Subject: [PATCH 53/76] gh-74690: Document changes made to runtime-checkable protocols in 3.12 (#103348) Co-authored-by: Jelle Zijlstra --- Doc/library/typing.rst | 27 +++++++++----- Doc/whatsnew/3.12.rst | 35 +++++++++++++++++++ ...3-04-07-15-09-26.gh-issue-74690.0f886b.rst | 3 ++ ...3-04-07-15-15-40.gh-issue-74690.un84hh.rst | 8 +++++ 4 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-07-15-09-26.gh-issue-74690.0f886b.rst create mode 100644 Misc/NEWS.d/next/Library/2023-04-07-15-15-40.gh-issue-74690.un84hh.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 8728ca7b6b358c..03ff259bbdf75f 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1598,15 +1598,6 @@ These are not used in annotations. They are building blocks for creating generic import threading assert isinstance(threading.Thread(name='Bob'), Named) - .. versionchanged:: 3.12 - The internal implementation of :func:`isinstance` checks against - runtime-checkable protocols now uses :func:`inspect.getattr_static` - to look up attributes (previously, :func:`hasattr` was used). - As a result, some objects which used to be considered instances - of a runtime-checkable protocol may no longer be considered instances - of that protocol on Python 3.12+, and vice versa. - Most users are unlikely to be affected by this change. - .. note:: :func:`!runtime_checkable` will check only the presence of the required @@ -1628,6 +1619,24 @@ These are not used in annotations. They are building blocks for creating generic .. versionadded:: 3.8 + .. versionchanged:: 3.12 + The internal implementation of :func:`isinstance` checks against + runtime-checkable protocols now uses :func:`inspect.getattr_static` + to look up attributes (previously, :func:`hasattr` was used). + As a result, some objects which used to be considered instances + of a runtime-checkable protocol may no longer be considered instances + of that protocol on Python 3.12+, and vice versa. + Most users are unlikely to be affected by this change. + + .. versionchanged:: 3.12 + The members of a runtime-checkable protocol are now considered "frozen" + at runtime as soon as the class has been created. Monkey-patching + attributes onto a runtime-checkable protocol will still work, but will + have no impact on :func:`isinstance` checks comparing objects to the + protocol. See :ref:`"What's new in Python 3.12" ` + for more details. + + Other special directives """""""""""""""""""""""" diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3a12fb20c43c51..66e40ef7326cc0 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -422,6 +422,8 @@ tempfile The :class:`tempfile.NamedTemporaryFile` function has a new optional parameter *delete_on_close* (Contributed by Evgeny Zorin in :gh:`58451`.) +.. _whatsnew-typing-py312: + typing ------ @@ -441,6 +443,39 @@ typing vice versa. Most users are unlikely to be affected by this change. (Contributed by Alex Waygood in :gh:`102433`.) +* The members of a runtime-checkable protocol are now considered "frozen" at + runtime as soon as the class has been created. Monkey-patching attributes + onto a runtime-checkable protocol will still work, but will have no impact on + :func:`isinstance` checks comparing objects to the protocol. For example:: + + >>> from typing import Protocol, runtime_checkable + >>> @runtime_checkable + ... class HasX(Protocol): + ... x = 1 + ... + >>> class Foo: ... + ... + >>> f = Foo() + >>> isinstance(f, HasX) + False + >>> f.x = 1 + >>> isinstance(f, HasX) + True + >>> HasX.y = 2 + >>> isinstance(f, HasX) # unchanged, even though HasX now also has a "y" attribute + True + + This change was made in order to speed up ``isinstance()`` checks against + runtime-checkable protocols. + +* The performance profile of :func:`isinstance` checks against + :func:`runtime-checkable protocols ` has changed + significantly. Most ``isinstance()`` checks against protocols with only a few + members should be at least 2x faster than in 3.11, and some may be 20x + faster or more. However, ``isinstance()`` checks against protocols with seven + or more members may be slower than in Python 3.11. (Contributed by Alex + Waygood in :gh:`74690` and :gh:`103193`.) + sys --- diff --git a/Misc/NEWS.d/next/Library/2023-04-07-15-09-26.gh-issue-74690.0f886b.rst b/Misc/NEWS.d/next/Library/2023-04-07-15-09-26.gh-issue-74690.0f886b.rst new file mode 100644 index 00000000000000..0a103ae11970d4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-07-15-09-26.gh-issue-74690.0f886b.rst @@ -0,0 +1,3 @@ +The members of a runtime-checkable protocol are now considered "frozen" at +runtime as soon as the class has been created. See +:ref:`"What's new in Python 3.12" ` for more details. diff --git a/Misc/NEWS.d/next/Library/2023-04-07-15-15-40.gh-issue-74690.un84hh.rst b/Misc/NEWS.d/next/Library/2023-04-07-15-15-40.gh-issue-74690.un84hh.rst new file mode 100644 index 00000000000000..48f11aac692ddb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-07-15-15-40.gh-issue-74690.un84hh.rst @@ -0,0 +1,8 @@ +The performance of :func:`isinstance` checks against +:func:`runtime-checkable protocols ` has been +considerably improved for protocols that only have a few members. To achieve +this improvement, several internal implementation details of the +:mod:`typing` module have been refactored, including +``typing._ProtocolMeta.__instancecheck__``, +``typing._is_callable_members_only``, and ``typing._get_protocol_attrs``. +Patches by Alex Waygood. From 91794e587306343619a451473efae22aa9f4b9bd Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sat, 8 Apr 2023 01:43:44 +0200 Subject: [PATCH 54/76] gh-83004: Harden _socket init (GH-103261) Automerge-Triggered-By: GH:erlend-aasland --- Modules/socketmodule.c | 1118 +++++++++++++++++++++------------------- 1 file changed, 581 insertions(+), 537 deletions(-) diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index c2b8283d84e5f8..49342b3d48de0e 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7338,1394 +7338,1427 @@ static struct PyModuleDef socketmodule = { PyMODINIT_FUNC PyInit__socket(void) { - PyObject *m, *has_ipv6; + PyObject *m = NULL; - if (!os_init()) - return NULL; + if (!os_init()) { + goto error; + } Py_SET_TYPE(&sock_type, &PyType_Type); m = PyModule_Create(&socketmodule); - if (m == NULL) - return NULL; + if (m == NULL) { + goto error; + } - PyModule_AddObject(m, "error", Py_NewRef(PyExc_OSError)); - socket_herror = PyErr_NewException("socket.herror", - PyExc_OSError, NULL); - if (socket_herror == NULL) - return NULL; - PyModule_AddObject(m, "herror", Py_NewRef(socket_herror)); - socket_gaierror = PyErr_NewException("socket.gaierror", PyExc_OSError, - NULL); - if (socket_gaierror == NULL) - return NULL; - PyModule_AddObject(m, "gaierror", Py_NewRef(socket_gaierror)); - PyModule_AddObjectRef(m, "timeout", PyExc_TimeoutError); +#define ADD_EXC(MOD, NAME, VAR, BASE) do { \ + VAR = PyErr_NewException("socket." NAME, BASE, NULL); \ + if (VAR == NULL) { \ + goto error; \ + } \ + int rc = PyModule_AddObjectRef(MOD, NAME, VAR); \ + Py_DECREF(VAR); \ + if (rc < 0) { \ + goto error; \ + } \ +} while (0) - if (PyModule_AddObject(m, "SocketType", Py_NewRef(&sock_type)) != 0) - return NULL; - if (PyModule_AddObject(m, "socket", Py_NewRef(&sock_type)) != 0) - return NULL; + ADD_EXC(m, "herror", socket_herror, PyExc_OSError); + ADD_EXC(m, "gaierror", socket_gaierror, PyExc_OSError); +#undef ADD_EXC + + if (PyModule_AddObjectRef(m, "error", PyExc_OSError) < 0) { + goto error; + } + if (PyModule_AddObjectRef(m, "timeout", PyExc_TimeoutError) < 0) { + goto error; + } + if (PyModule_AddObjectRef(m, "SocketType", (PyObject *)&sock_type) < 0) { + goto error; + } + if (PyModule_AddType(m, &sock_type) < 0) { + goto error; + } + + PyObject *has_ipv6; #ifdef ENABLE_IPV6 has_ipv6 = Py_True; #else has_ipv6 = Py_False; #endif - PyModule_AddObject(m, "has_ipv6", Py_NewRef(has_ipv6)); + if (PyModule_AddObjectRef(m, "has_ipv6", has_ipv6) < 0) { + goto error; + } /* Export C API */ PySocketModule_APIObject *capi = sock_get_api(); if (capi == NULL) { - Py_DECREF(m); - return NULL; + goto error; } PyObject *capsule = PyCapsule_New(capi, PySocket_CAPSULE_NAME, sock_destroy_api); if (capsule == NULL) { sock_free_api(capi); - Py_DECREF(m); - return NULL; + goto error; } - if (PyModule_AddObject(m, PySocket_CAPI_NAME, capsule) < 0) { - Py_DECREF(capsule); - Py_DECREF(m); - return NULL; + int rc = PyModule_AddObjectRef(m, PySocket_CAPI_NAME, capsule); + Py_DECREF(capsule); + if (rc < 0) { + goto error; } +#define ADD_INT_MACRO(MOD, INT) do { \ + if (PyModule_AddIntConstant(MOD, #INT, INT) < 0) { \ + goto error; \ + } \ +} while (0) + +#define ADD_INT_CONST(MOD, NAME, INT) do { \ + if (PyModule_AddIntConstant(MOD, NAME, INT) < 0) { \ + goto error; \ + } \ +} while (0) + +#define ADD_STR_CONST(MOD, NAME, STR) do { \ + if (PyModule_AddStringConstant(MOD, NAME, STR) < 0) { \ + goto error; \ + } \ +} while (0) + /* Address families (we only support AF_INET and AF_UNIX) */ #ifdef AF_UNSPEC - PyModule_AddIntMacro(m, AF_UNSPEC); + ADD_INT_MACRO(m, AF_UNSPEC); #endif - PyModule_AddIntMacro(m, AF_INET); + ADD_INT_MACRO(m, AF_INET); #if defined(AF_UNIX) - PyModule_AddIntMacro(m, AF_UNIX); + ADD_INT_MACRO(m, AF_UNIX); #endif /* AF_UNIX */ #ifdef AF_AX25 /* Amateur Radio AX.25 */ - PyModule_AddIntMacro(m, AF_AX25); + ADD_INT_MACRO(m, AF_AX25); #endif #ifdef AF_IPX - PyModule_AddIntMacro(m, AF_IPX); /* Novell IPX */ + ADD_INT_MACRO(m, AF_IPX); /* Novell IPX */ #endif #ifdef AF_APPLETALK /* Appletalk DDP */ - PyModule_AddIntMacro(m, AF_APPLETALK); + ADD_INT_MACRO(m, AF_APPLETALK); #endif #ifdef AF_NETROM /* Amateur radio NetROM */ - PyModule_AddIntMacro(m, AF_NETROM); + ADD_INT_MACRO(m, AF_NETROM); #endif #ifdef AF_BRIDGE /* Multiprotocol bridge */ - PyModule_AddIntMacro(m, AF_BRIDGE); + ADD_INT_MACRO(m, AF_BRIDGE); #endif #ifdef AF_ATMPVC /* ATM PVCs */ - PyModule_AddIntMacro(m, AF_ATMPVC); + ADD_INT_MACRO(m, AF_ATMPVC); #endif #ifdef AF_AAL5 /* Reserved for Werner's ATM */ - PyModule_AddIntMacro(m, AF_AAL5); + ADD_INT_MACRO(m, AF_AAL5); #endif #ifdef HAVE_SOCKADDR_ALG - PyModule_AddIntMacro(m, AF_ALG); /* Linux crypto */ + ADD_INT_MACRO(m, AF_ALG); /* Linux crypto */ #endif #ifdef AF_X25 /* Reserved for X.25 project */ - PyModule_AddIntMacro(m, AF_X25); + ADD_INT_MACRO(m, AF_X25); #endif #ifdef AF_INET6 - PyModule_AddIntMacro(m, AF_INET6); /* IP version 6 */ + ADD_INT_MACRO(m, AF_INET6); /* IP version 6 */ #endif #ifdef AF_ROSE /* Amateur Radio X.25 PLP */ - PyModule_AddIntMacro(m, AF_ROSE); + ADD_INT_MACRO(m, AF_ROSE); #endif #ifdef AF_DECnet /* Reserved for DECnet project */ - PyModule_AddIntMacro(m, AF_DECnet); + ADD_INT_MACRO(m, AF_DECnet); #endif #ifdef AF_NETBEUI /* Reserved for 802.2LLC project */ - PyModule_AddIntMacro(m, AF_NETBEUI); + ADD_INT_MACRO(m, AF_NETBEUI); #endif #ifdef AF_SECURITY /* Security callback pseudo AF */ - PyModule_AddIntMacro(m, AF_SECURITY); + ADD_INT_MACRO(m, AF_SECURITY); #endif #ifdef AF_KEY /* PF_KEY key management API */ - PyModule_AddIntMacro(m, AF_KEY); + ADD_INT_MACRO(m, AF_KEY); #endif #ifdef AF_NETLINK /* */ - PyModule_AddIntMacro(m, AF_NETLINK); - PyModule_AddIntMacro(m, NETLINK_ROUTE); + ADD_INT_MACRO(m, AF_NETLINK); + ADD_INT_MACRO(m, NETLINK_ROUTE); #ifdef NETLINK_SKIP - PyModule_AddIntMacro(m, NETLINK_SKIP); + ADD_INT_MACRO(m, NETLINK_SKIP); #endif #ifdef NETLINK_W1 - PyModule_AddIntMacro(m, NETLINK_W1); + ADD_INT_MACRO(m, NETLINK_W1); #endif - PyModule_AddIntMacro(m, NETLINK_USERSOCK); - PyModule_AddIntMacro(m, NETLINK_FIREWALL); + ADD_INT_MACRO(m, NETLINK_USERSOCK); + ADD_INT_MACRO(m, NETLINK_FIREWALL); #ifdef NETLINK_TCPDIAG - PyModule_AddIntMacro(m, NETLINK_TCPDIAG); + ADD_INT_MACRO(m, NETLINK_TCPDIAG); #endif #ifdef NETLINK_NFLOG - PyModule_AddIntMacro(m, NETLINK_NFLOG); + ADD_INT_MACRO(m, NETLINK_NFLOG); #endif #ifdef NETLINK_XFRM - PyModule_AddIntMacro(m, NETLINK_XFRM); + ADD_INT_MACRO(m, NETLINK_XFRM); #endif #ifdef NETLINK_ARPD - PyModule_AddIntMacro(m, NETLINK_ARPD); + ADD_INT_MACRO(m, NETLINK_ARPD); #endif #ifdef NETLINK_ROUTE6 - PyModule_AddIntMacro(m, NETLINK_ROUTE6); + ADD_INT_MACRO(m, NETLINK_ROUTE6); #endif - PyModule_AddIntMacro(m, NETLINK_IP6_FW); + ADD_INT_MACRO(m, NETLINK_IP6_FW); #ifdef NETLINK_DNRTMSG - PyModule_AddIntMacro(m, NETLINK_DNRTMSG); + ADD_INT_MACRO(m, NETLINK_DNRTMSG); #endif #ifdef NETLINK_TAPBASE - PyModule_AddIntMacro(m, NETLINK_TAPBASE); + ADD_INT_MACRO(m, NETLINK_TAPBASE); #endif #ifdef NETLINK_CRYPTO - PyModule_AddIntMacro(m, NETLINK_CRYPTO); + ADD_INT_MACRO(m, NETLINK_CRYPTO); #endif #endif /* AF_NETLINK */ #ifdef AF_QIPCRTR /* Qualcomm IPCROUTER */ - PyModule_AddIntMacro(m, AF_QIPCRTR); + ADD_INT_MACRO(m, AF_QIPCRTR); #endif #ifdef AF_VSOCK - PyModule_AddIntConstant(m, "AF_VSOCK", AF_VSOCK); - PyModule_AddIntConstant(m, "SO_VM_SOCKETS_BUFFER_SIZE", 0); - PyModule_AddIntConstant(m, "SO_VM_SOCKETS_BUFFER_MIN_SIZE", 1); - PyModule_AddIntConstant(m, "SO_VM_SOCKETS_BUFFER_MAX_SIZE", 2); - PyModule_AddIntConstant(m, "VMADDR_CID_ANY", 0xffffffff); - PyModule_AddIntConstant(m, "VMADDR_PORT_ANY", 0xffffffff); - PyModule_AddIntConstant(m, "VMADDR_CID_HOST", 2); - PyModule_AddIntConstant(m, "VM_SOCKETS_INVALID_VERSION", 0xffffffff); - PyModule_AddIntConstant(m, "IOCTL_VM_SOCKETS_GET_LOCAL_CID", _IO(7, 0xb9)); + ADD_INT_CONST(m, "AF_VSOCK", AF_VSOCK); + ADD_INT_CONST(m, "SO_VM_SOCKETS_BUFFER_SIZE", 0); + ADD_INT_CONST(m, "SO_VM_SOCKETS_BUFFER_MIN_SIZE", 1); + ADD_INT_CONST(m, "SO_VM_SOCKETS_BUFFER_MAX_SIZE", 2); + ADD_INT_CONST(m, "VMADDR_CID_ANY", 0xffffffff); + ADD_INT_CONST(m, "VMADDR_PORT_ANY", 0xffffffff); + ADD_INT_CONST(m, "VMADDR_CID_HOST", 2); + ADD_INT_CONST(m, "VM_SOCKETS_INVALID_VERSION", 0xffffffff); + ADD_INT_CONST(m, "IOCTL_VM_SOCKETS_GET_LOCAL_CID", _IO(7, 0xb9)); #endif #ifdef AF_ROUTE /* Alias to emulate 4.4BSD */ - PyModule_AddIntMacro(m, AF_ROUTE); + ADD_INT_MACRO(m, AF_ROUTE); #endif #ifdef AF_LINK - PyModule_AddIntMacro(m, AF_LINK); + ADD_INT_MACRO(m, AF_LINK); #endif #ifdef AF_ASH /* Ash */ - PyModule_AddIntMacro(m, AF_ASH); + ADD_INT_MACRO(m, AF_ASH); #endif #ifdef AF_ECONET /* Acorn Econet */ - PyModule_AddIntMacro(m, AF_ECONET); + ADD_INT_MACRO(m, AF_ECONET); #endif #ifdef AF_ATMSVC /* ATM SVCs */ - PyModule_AddIntMacro(m, AF_ATMSVC); + ADD_INT_MACRO(m, AF_ATMSVC); #endif #ifdef AF_SNA /* Linux SNA Project (nutters!) */ - PyModule_AddIntMacro(m, AF_SNA); + ADD_INT_MACRO(m, AF_SNA); #endif #ifdef AF_IRDA /* IRDA sockets */ - PyModule_AddIntMacro(m, AF_IRDA); + ADD_INT_MACRO(m, AF_IRDA); #endif #ifdef AF_PPPOX /* PPPoX sockets */ - PyModule_AddIntMacro(m, AF_PPPOX); + ADD_INT_MACRO(m, AF_PPPOX); #endif #ifdef AF_WANPIPE /* Wanpipe API Sockets */ - PyModule_AddIntMacro(m, AF_WANPIPE); + ADD_INT_MACRO(m, AF_WANPIPE); #endif #ifdef AF_LLC /* Linux LLC */ - PyModule_AddIntMacro(m, AF_LLC); + ADD_INT_MACRO(m, AF_LLC); #endif #ifdef HAVE_AF_HYPERV /* Hyper-V sockets */ - PyModule_AddIntMacro(m, AF_HYPERV); + ADD_INT_MACRO(m, AF_HYPERV); /* for proto */ - PyModule_AddIntMacro(m, HV_PROTOCOL_RAW); + ADD_INT_MACRO(m, HV_PROTOCOL_RAW); /* for setsockopt() */ - PyModule_AddIntMacro(m, HVSOCKET_CONNECT_TIMEOUT); - PyModule_AddIntMacro(m, HVSOCKET_CONNECT_TIMEOUT_MAX); - PyModule_AddIntMacro(m, HVSOCKET_CONNECTED_SUSPEND); - PyModule_AddIntMacro(m, HVSOCKET_ADDRESS_FLAG_PASSTHRU); + ADD_INT_MACRO(m, HVSOCKET_CONNECT_TIMEOUT); + ADD_INT_MACRO(m, HVSOCKET_CONNECT_TIMEOUT_MAX); + ADD_INT_MACRO(m, HVSOCKET_CONNECTED_SUSPEND); + ADD_INT_MACRO(m, HVSOCKET_ADDRESS_FLAG_PASSTHRU); /* for bind() or connect() */ - PyModule_AddStringConstant(m, "HV_GUID_ZERO", "00000000-0000-0000-0000-000000000000"); - PyModule_AddStringConstant(m, "HV_GUID_WILDCARD", "00000000-0000-0000-0000-000000000000"); - PyModule_AddStringConstant(m, "HV_GUID_BROADCAST", "FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF"); - PyModule_AddStringConstant(m, "HV_GUID_CHILDREN", "90DB8B89-0D35-4F79-8CE9-49EA0AC8B7CD"); - PyModule_AddStringConstant(m, "HV_GUID_LOOPBACK", "E0E16197-DD56-4A10-9195-5EE7A155A838"); - PyModule_AddStringConstant(m, "HV_GUID_PARENT", "A42E7CDA-D03F-480C-9CC2-A4DE20ABB878"); + ADD_STR_CONST(m, "HV_GUID_ZERO", "00000000-0000-0000-0000-000000000000"); + ADD_STR_CONST(m, "HV_GUID_WILDCARD", "00000000-0000-0000-0000-000000000000"); + ADD_STR_CONST(m, "HV_GUID_BROADCAST", "FFFFFFFF-FFFF-FFFF-FFFF-FFFFFFFFFFFF"); + ADD_STR_CONST(m, "HV_GUID_CHILDREN", "90DB8B89-0D35-4F79-8CE9-49EA0AC8B7CD"); + ADD_STR_CONST(m, "HV_GUID_LOOPBACK", "E0E16197-DD56-4A10-9195-5EE7A155A838"); + ADD_STR_CONST(m, "HV_GUID_PARENT", "A42E7CDA-D03F-480C-9CC2-A4DE20ABB878"); #endif /* HAVE_AF_HYPERV */ #ifdef USE_BLUETOOTH - PyModule_AddIntMacro(m, AF_BLUETOOTH); + ADD_INT_MACRO(m, AF_BLUETOOTH); #ifdef BTPROTO_L2CAP - PyModule_AddIntMacro(m, BTPROTO_L2CAP); + ADD_INT_MACRO(m, BTPROTO_L2CAP); #endif /* BTPROTO_L2CAP */ #ifdef BTPROTO_HCI - PyModule_AddIntMacro(m, BTPROTO_HCI); - PyModule_AddIntMacro(m, SOL_HCI); + ADD_INT_MACRO(m, BTPROTO_HCI); + ADD_INT_MACRO(m, SOL_HCI); #if !defined(__NetBSD__) && !defined(__DragonFly__) - PyModule_AddIntMacro(m, HCI_FILTER); + ADD_INT_MACRO(m, HCI_FILTER); #if !defined(__FreeBSD__) - PyModule_AddIntMacro(m, HCI_TIME_STAMP); - PyModule_AddIntMacro(m, HCI_DATA_DIR); + ADD_INT_MACRO(m, HCI_TIME_STAMP); + ADD_INT_MACRO(m, HCI_DATA_DIR); #endif /* !__FreeBSD__ */ #endif /* !__NetBSD__ && !__DragonFly__ */ #endif /* BTPROTO_HCI */ #ifdef BTPROTO_RFCOMM - PyModule_AddIntMacro(m, BTPROTO_RFCOMM); + ADD_INT_MACRO(m, BTPROTO_RFCOMM); #endif /* BTPROTO_RFCOMM */ - PyModule_AddStringConstant(m, "BDADDR_ANY", "00:00:00:00:00:00"); - PyModule_AddStringConstant(m, "BDADDR_LOCAL", "00:00:00:FF:FF:FF"); + ADD_STR_CONST(m, "BDADDR_ANY", "00:00:00:00:00:00"); + ADD_STR_CONST(m, "BDADDR_LOCAL", "00:00:00:FF:FF:FF"); #ifdef BTPROTO_SCO - PyModule_AddIntMacro(m, BTPROTO_SCO); + ADD_INT_MACRO(m, BTPROTO_SCO); #endif /* BTPROTO_SCO */ #endif /* USE_BLUETOOTH */ #ifdef AF_CAN /* Controller Area Network */ - PyModule_AddIntMacro(m, AF_CAN); + ADD_INT_MACRO(m, AF_CAN); #endif #ifdef PF_CAN /* Controller Area Network */ - PyModule_AddIntMacro(m, PF_CAN); + ADD_INT_MACRO(m, PF_CAN); #endif /* Reliable Datagram Sockets */ #ifdef AF_RDS - PyModule_AddIntMacro(m, AF_RDS); + ADD_INT_MACRO(m, AF_RDS); #endif #ifdef PF_RDS - PyModule_AddIntMacro(m, PF_RDS); + ADD_INT_MACRO(m, PF_RDS); #endif /* Kernel event messages */ #ifdef PF_SYSTEM - PyModule_AddIntMacro(m, PF_SYSTEM); + ADD_INT_MACRO(m, PF_SYSTEM); #endif #ifdef AF_SYSTEM - PyModule_AddIntMacro(m, AF_SYSTEM); + ADD_INT_MACRO(m, AF_SYSTEM); #endif #ifdef AF_PACKET - PyModule_AddIntMacro(m, AF_PACKET); + ADD_INT_MACRO(m, AF_PACKET); #endif #ifdef PF_PACKET - PyModule_AddIntMacro(m, PF_PACKET); + ADD_INT_MACRO(m, PF_PACKET); #endif #ifdef PACKET_HOST - PyModule_AddIntMacro(m, PACKET_HOST); + ADD_INT_MACRO(m, PACKET_HOST); #endif #ifdef PACKET_BROADCAST - PyModule_AddIntMacro(m, PACKET_BROADCAST); + ADD_INT_MACRO(m, PACKET_BROADCAST); #endif #ifdef PACKET_MULTICAST - PyModule_AddIntMacro(m, PACKET_MULTICAST); + ADD_INT_MACRO(m, PACKET_MULTICAST); #endif #ifdef PACKET_OTHERHOST - PyModule_AddIntMacro(m, PACKET_OTHERHOST); + ADD_INT_MACRO(m, PACKET_OTHERHOST); #endif #ifdef PACKET_OUTGOING - PyModule_AddIntMacro(m, PACKET_OUTGOING); + ADD_INT_MACRO(m, PACKET_OUTGOING); #endif #ifdef PACKET_LOOPBACK - PyModule_AddIntMacro(m, PACKET_LOOPBACK); + ADD_INT_MACRO(m, PACKET_LOOPBACK); #endif #ifdef PACKET_FASTROUTE - PyModule_AddIntMacro(m, PACKET_FASTROUTE); + ADD_INT_MACRO(m, PACKET_FASTROUTE); #endif #ifdef HAVE_LINUX_TIPC_H - PyModule_AddIntMacro(m, AF_TIPC); + ADD_INT_MACRO(m, AF_TIPC); /* for addresses */ - PyModule_AddIntMacro(m, TIPC_ADDR_NAMESEQ); - PyModule_AddIntMacro(m, TIPC_ADDR_NAME); - PyModule_AddIntMacro(m, TIPC_ADDR_ID); + ADD_INT_MACRO(m, TIPC_ADDR_NAMESEQ); + ADD_INT_MACRO(m, TIPC_ADDR_NAME); + ADD_INT_MACRO(m, TIPC_ADDR_ID); - PyModule_AddIntMacro(m, TIPC_ZONE_SCOPE); - PyModule_AddIntMacro(m, TIPC_CLUSTER_SCOPE); - PyModule_AddIntMacro(m, TIPC_NODE_SCOPE); + ADD_INT_MACRO(m, TIPC_ZONE_SCOPE); + ADD_INT_MACRO(m, TIPC_CLUSTER_SCOPE); + ADD_INT_MACRO(m, TIPC_NODE_SCOPE); /* for setsockopt() */ - PyModule_AddIntMacro(m, SOL_TIPC); - PyModule_AddIntMacro(m, TIPC_IMPORTANCE); - PyModule_AddIntMacro(m, TIPC_SRC_DROPPABLE); - PyModule_AddIntMacro(m, TIPC_DEST_DROPPABLE); - PyModule_AddIntMacro(m, TIPC_CONN_TIMEOUT); + ADD_INT_MACRO(m, SOL_TIPC); + ADD_INT_MACRO(m, TIPC_IMPORTANCE); + ADD_INT_MACRO(m, TIPC_SRC_DROPPABLE); + ADD_INT_MACRO(m, TIPC_DEST_DROPPABLE); + ADD_INT_MACRO(m, TIPC_CONN_TIMEOUT); - PyModule_AddIntMacro(m, TIPC_LOW_IMPORTANCE); - PyModule_AddIntMacro(m, TIPC_MEDIUM_IMPORTANCE); - PyModule_AddIntMacro(m, TIPC_HIGH_IMPORTANCE); - PyModule_AddIntMacro(m, TIPC_CRITICAL_IMPORTANCE); + ADD_INT_MACRO(m, TIPC_LOW_IMPORTANCE); + ADD_INT_MACRO(m, TIPC_MEDIUM_IMPORTANCE); + ADD_INT_MACRO(m, TIPC_HIGH_IMPORTANCE); + ADD_INT_MACRO(m, TIPC_CRITICAL_IMPORTANCE); /* for subscriptions */ - PyModule_AddIntMacro(m, TIPC_SUB_PORTS); - PyModule_AddIntMacro(m, TIPC_SUB_SERVICE); + ADD_INT_MACRO(m, TIPC_SUB_PORTS); + ADD_INT_MACRO(m, TIPC_SUB_SERVICE); #ifdef TIPC_SUB_CANCEL /* doesn't seem to be available everywhere */ - PyModule_AddIntMacro(m, TIPC_SUB_CANCEL); + ADD_INT_MACRO(m, TIPC_SUB_CANCEL); #endif - PyModule_AddIntMacro(m, TIPC_WAIT_FOREVER); - PyModule_AddIntMacro(m, TIPC_PUBLISHED); - PyModule_AddIntMacro(m, TIPC_WITHDRAWN); - PyModule_AddIntMacro(m, TIPC_SUBSCR_TIMEOUT); - PyModule_AddIntMacro(m, TIPC_CFG_SRV); - PyModule_AddIntMacro(m, TIPC_TOP_SRV); + ADD_INT_MACRO(m, TIPC_WAIT_FOREVER); + ADD_INT_MACRO(m, TIPC_PUBLISHED); + ADD_INT_MACRO(m, TIPC_WITHDRAWN); + ADD_INT_MACRO(m, TIPC_SUBSCR_TIMEOUT); + ADD_INT_MACRO(m, TIPC_CFG_SRV); + ADD_INT_MACRO(m, TIPC_TOP_SRV); #endif #ifdef HAVE_SOCKADDR_ALG /* Socket options */ - PyModule_AddIntMacro(m, ALG_SET_KEY); - PyModule_AddIntMacro(m, ALG_SET_IV); - PyModule_AddIntMacro(m, ALG_SET_OP); - PyModule_AddIntMacro(m, ALG_SET_AEAD_ASSOCLEN); - PyModule_AddIntMacro(m, ALG_SET_AEAD_AUTHSIZE); - PyModule_AddIntMacro(m, ALG_SET_PUBKEY); + ADD_INT_MACRO(m, ALG_SET_KEY); + ADD_INT_MACRO(m, ALG_SET_IV); + ADD_INT_MACRO(m, ALG_SET_OP); + ADD_INT_MACRO(m, ALG_SET_AEAD_ASSOCLEN); + ADD_INT_MACRO(m, ALG_SET_AEAD_AUTHSIZE); + ADD_INT_MACRO(m, ALG_SET_PUBKEY); /* Operations */ - PyModule_AddIntMacro(m, ALG_OP_DECRYPT); - PyModule_AddIntMacro(m, ALG_OP_ENCRYPT); - PyModule_AddIntMacro(m, ALG_OP_SIGN); - PyModule_AddIntMacro(m, ALG_OP_VERIFY); + ADD_INT_MACRO(m, ALG_OP_DECRYPT); + ADD_INT_MACRO(m, ALG_OP_ENCRYPT); + ADD_INT_MACRO(m, ALG_OP_SIGN); + ADD_INT_MACRO(m, ALG_OP_VERIFY); #endif /* IEEE 802.3 protocol numbers required for a standard TCP/IP network stack */ #ifdef ETHERTYPE_ARP - PyModule_AddIntMacro(m, ETHERTYPE_ARP); + ADD_INT_MACRO(m, ETHERTYPE_ARP); #endif #ifdef ETHERTYPE_IP - PyModule_AddIntMacro(m, ETHERTYPE_IP); + ADD_INT_MACRO(m, ETHERTYPE_IP); #endif #ifdef ETHERTYPE_IPV6 - PyModule_AddIntMacro(m, ETHERTYPE_IPV6); + ADD_INT_MACRO(m, ETHERTYPE_IPV6); #endif #ifdef ETHERTYPE_VLAN - PyModule_AddIntMacro(m, ETHERTYPE_VLAN); + ADD_INT_MACRO(m, ETHERTYPE_VLAN); #endif /* Linux pseudo-protocol for sniffing every packet */ #ifdef ETH_P_ALL - PyModule_AddIntMacro(m, ETH_P_ALL); + ADD_INT_MACRO(m, ETH_P_ALL); #endif /* Socket types */ - PyModule_AddIntMacro(m, SOCK_STREAM); - PyModule_AddIntMacro(m, SOCK_DGRAM); + ADD_INT_MACRO(m, SOCK_STREAM); + ADD_INT_MACRO(m, SOCK_DGRAM); /* We have incomplete socket support. */ #ifdef SOCK_RAW /* SOCK_RAW is marked as optional in the POSIX specification */ - PyModule_AddIntMacro(m, SOCK_RAW); + ADD_INT_MACRO(m, SOCK_RAW); #endif #ifdef SOCK_SEQPACKET - PyModule_AddIntMacro(m, SOCK_SEQPACKET); + ADD_INT_MACRO(m, SOCK_SEQPACKET); #endif #if defined(SOCK_RDM) - PyModule_AddIntMacro(m, SOCK_RDM); + ADD_INT_MACRO(m, SOCK_RDM); #endif #ifdef SOCK_CLOEXEC - PyModule_AddIntMacro(m, SOCK_CLOEXEC); + ADD_INT_MACRO(m, SOCK_CLOEXEC); #endif #ifdef SOCK_NONBLOCK - PyModule_AddIntMacro(m, SOCK_NONBLOCK); + ADD_INT_MACRO(m, SOCK_NONBLOCK); #endif #ifdef SO_DEBUG - PyModule_AddIntMacro(m, SO_DEBUG); + ADD_INT_MACRO(m, SO_DEBUG); #endif #ifdef SO_ACCEPTCONN - PyModule_AddIntMacro(m, SO_ACCEPTCONN); + ADD_INT_MACRO(m, SO_ACCEPTCONN); #endif #ifdef SO_REUSEADDR - PyModule_AddIntMacro(m, SO_REUSEADDR); + ADD_INT_MACRO(m, SO_REUSEADDR); #endif #ifdef SO_EXCLUSIVEADDRUSE - PyModule_AddIntMacro(m, SO_EXCLUSIVEADDRUSE); + ADD_INT_MACRO(m, SO_EXCLUSIVEADDRUSE); #endif #ifdef SO_INCOMING_CPU - PyModule_AddIntMacro(m, SO_INCOMING_CPU); + ADD_INT_MACRO(m, SO_INCOMING_CPU); #endif #ifdef SO_KEEPALIVE - PyModule_AddIntMacro(m, SO_KEEPALIVE); + ADD_INT_MACRO(m, SO_KEEPALIVE); #endif #ifdef SO_DONTROUTE - PyModule_AddIntMacro(m, SO_DONTROUTE); + ADD_INT_MACRO(m, SO_DONTROUTE); #endif #ifdef SO_BROADCAST - PyModule_AddIntMacro(m, SO_BROADCAST); + ADD_INT_MACRO(m, SO_BROADCAST); #endif #ifdef SO_USELOOPBACK - PyModule_AddIntMacro(m, SO_USELOOPBACK); + ADD_INT_MACRO(m, SO_USELOOPBACK); #endif #ifdef SO_LINGER - PyModule_AddIntMacro(m, SO_LINGER); + ADD_INT_MACRO(m, SO_LINGER); #endif #ifdef SO_OOBINLINE - PyModule_AddIntMacro(m, SO_OOBINLINE); + ADD_INT_MACRO(m, SO_OOBINLINE); #endif #ifndef __GNU__ #ifdef SO_REUSEPORT - PyModule_AddIntMacro(m, SO_REUSEPORT); + ADD_INT_MACRO(m, SO_REUSEPORT); #endif #endif #ifdef SO_SNDBUF - PyModule_AddIntMacro(m, SO_SNDBUF); + ADD_INT_MACRO(m, SO_SNDBUF); #endif #ifdef SO_RCVBUF - PyModule_AddIntMacro(m, SO_RCVBUF); + ADD_INT_MACRO(m, SO_RCVBUF); #endif #ifdef SO_SNDLOWAT - PyModule_AddIntMacro(m, SO_SNDLOWAT); + ADD_INT_MACRO(m, SO_SNDLOWAT); #endif #ifdef SO_RCVLOWAT - PyModule_AddIntMacro(m, SO_RCVLOWAT); + ADD_INT_MACRO(m, SO_RCVLOWAT); #endif #ifdef SO_SNDTIMEO - PyModule_AddIntMacro(m, SO_SNDTIMEO); + ADD_INT_MACRO(m, SO_SNDTIMEO); #endif #ifdef SO_RCVTIMEO - PyModule_AddIntMacro(m, SO_RCVTIMEO); + ADD_INT_MACRO(m, SO_RCVTIMEO); #endif #ifdef SO_ERROR - PyModule_AddIntMacro(m, SO_ERROR); + ADD_INT_MACRO(m, SO_ERROR); #endif #ifdef SO_TYPE - PyModule_AddIntMacro(m, SO_TYPE); + ADD_INT_MACRO(m, SO_TYPE); #endif #ifdef SO_SETFIB - PyModule_AddIntMacro(m, SO_SETFIB); + ADD_INT_MACRO(m, SO_SETFIB); #endif #ifdef SO_PASSCRED - PyModule_AddIntMacro(m, SO_PASSCRED); + ADD_INT_MACRO(m, SO_PASSCRED); #endif #ifdef SO_PEERCRED - PyModule_AddIntMacro(m, SO_PEERCRED); + ADD_INT_MACRO(m, SO_PEERCRED); #endif #ifdef LOCAL_PEERCRED - PyModule_AddIntMacro(m, LOCAL_PEERCRED); + ADD_INT_MACRO(m, LOCAL_PEERCRED); #endif #ifdef SO_PASSSEC - PyModule_AddIntMacro(m, SO_PASSSEC); + ADD_INT_MACRO(m, SO_PASSSEC); #endif #ifdef SO_PEERSEC - PyModule_AddIntMacro(m, SO_PEERSEC); + ADD_INT_MACRO(m, SO_PEERSEC); #endif #ifdef SO_BINDTODEVICE - PyModule_AddIntMacro(m, SO_BINDTODEVICE); + ADD_INT_MACRO(m, SO_BINDTODEVICE); #endif #ifdef SO_PRIORITY - PyModule_AddIntMacro(m, SO_PRIORITY); + ADD_INT_MACRO(m, SO_PRIORITY); #endif #ifdef SO_MARK - PyModule_AddIntMacro(m, SO_MARK); + ADD_INT_MACRO(m, SO_MARK); #endif #ifdef SO_USER_COOKIE - PyModule_AddIntMacro(m, SO_USER_COOKIE); + ADD_INT_MACRO(m, SO_USER_COOKIE); #endif #ifdef SO_RTABLE - PyModule_AddIntMacro(m, SO_RTABLE); + ADD_INT_MACRO(m, SO_RTABLE); #endif #ifdef SO_DOMAIN - PyModule_AddIntMacro(m, SO_DOMAIN); + ADD_INT_MACRO(m, SO_DOMAIN); #endif #ifdef SO_PROTOCOL - PyModule_AddIntMacro(m, SO_PROTOCOL); + ADD_INT_MACRO(m, SO_PROTOCOL); #endif #ifdef LOCAL_CREDS - PyModule_AddIntMacro(m, LOCAL_CREDS); + ADD_INT_MACRO(m, LOCAL_CREDS); #endif #ifdef LOCAL_CREDS_PERSISTENT - PyModule_AddIntMacro(m, LOCAL_CREDS_PERSISTENT); + ADD_INT_MACRO(m, LOCAL_CREDS_PERSISTENT); #endif /* Maximum number of connections for "listen" */ #ifdef SOMAXCONN - PyModule_AddIntMacro(m, SOMAXCONN); + ADD_INT_MACRO(m, SOMAXCONN); #else - PyModule_AddIntConstant(m, "SOMAXCONN", 5); /* Common value */ + ADD_INT_CONST(m, "SOMAXCONN", 5); /* Common value */ #endif /* Ancillary message types */ #ifdef SCM_RIGHTS - PyModule_AddIntMacro(m, SCM_RIGHTS); + ADD_INT_MACRO(m, SCM_RIGHTS); #endif #ifdef SCM_CREDENTIALS - PyModule_AddIntMacro(m, SCM_CREDENTIALS); + ADD_INT_MACRO(m, SCM_CREDENTIALS); #endif #ifdef SCM_CREDS - PyModule_AddIntMacro(m, SCM_CREDS); + ADD_INT_MACRO(m, SCM_CREDS); #endif #ifdef SCM_CREDS2 - PyModule_AddIntMacro(m, SCM_CREDS2); + ADD_INT_MACRO(m, SCM_CREDS2); #endif /* Flags for send, recv */ #ifdef MSG_OOB - PyModule_AddIntMacro(m, MSG_OOB); + ADD_INT_MACRO(m, MSG_OOB); #endif #ifdef MSG_PEEK - PyModule_AddIntMacro(m, MSG_PEEK); + ADD_INT_MACRO(m, MSG_PEEK); #endif #ifdef MSG_DONTROUTE - PyModule_AddIntMacro(m, MSG_DONTROUTE); + ADD_INT_MACRO(m, MSG_DONTROUTE); #endif #ifdef MSG_DONTWAIT - PyModule_AddIntMacro(m, MSG_DONTWAIT); + ADD_INT_MACRO(m, MSG_DONTWAIT); #endif #ifdef MSG_EOR - PyModule_AddIntMacro(m, MSG_EOR); + ADD_INT_MACRO(m, MSG_EOR); #endif #ifdef MSG_TRUNC // workaround for https://github.com/WebAssembly/wasi-libc/issues/305 #if defined(__wasi__) && !defined(__WASI_RIFLAGS_RECV_DATA_TRUNCATED) # define __WASI_RIFLAGS_RECV_DATA_TRUNCATED 2 #endif - PyModule_AddIntMacro(m, MSG_TRUNC); + ADD_INT_MACRO(m, MSG_TRUNC); #endif #ifdef MSG_CTRUNC - PyModule_AddIntMacro(m, MSG_CTRUNC); + ADD_INT_MACRO(m, MSG_CTRUNC); #endif #ifdef MSG_WAITALL - PyModule_AddIntMacro(m, MSG_WAITALL); + ADD_INT_MACRO(m, MSG_WAITALL); #endif #ifdef MSG_BTAG - PyModule_AddIntMacro(m, MSG_BTAG); + ADD_INT_MACRO(m, MSG_BTAG); #endif #ifdef MSG_ETAG - PyModule_AddIntMacro(m, MSG_ETAG); + ADD_INT_MACRO(m, MSG_ETAG); #endif #ifdef MSG_NOSIGNAL - PyModule_AddIntMacro(m, MSG_NOSIGNAL); + ADD_INT_MACRO(m, MSG_NOSIGNAL); #endif #ifdef MSG_NOTIFICATION - PyModule_AddIntMacro(m, MSG_NOTIFICATION); + ADD_INT_MACRO(m, MSG_NOTIFICATION); #endif #ifdef MSG_CMSG_CLOEXEC - PyModule_AddIntMacro(m, MSG_CMSG_CLOEXEC); + ADD_INT_MACRO(m, MSG_CMSG_CLOEXEC); #endif #ifdef MSG_ERRQUEUE - PyModule_AddIntMacro(m, MSG_ERRQUEUE); + ADD_INT_MACRO(m, MSG_ERRQUEUE); #endif #ifdef MSG_CONFIRM - PyModule_AddIntMacro(m, MSG_CONFIRM); + ADD_INT_MACRO(m, MSG_CONFIRM); #endif #ifdef MSG_MORE - PyModule_AddIntMacro(m, MSG_MORE); + ADD_INT_MACRO(m, MSG_MORE); #endif #ifdef MSG_EOF - PyModule_AddIntMacro(m, MSG_EOF); + ADD_INT_MACRO(m, MSG_EOF); #endif #ifdef MSG_BCAST - PyModule_AddIntMacro(m, MSG_BCAST); + ADD_INT_MACRO(m, MSG_BCAST); #endif #ifdef MSG_MCAST - PyModule_AddIntMacro(m, MSG_MCAST); + ADD_INT_MACRO(m, MSG_MCAST); #endif #ifdef MSG_FASTOPEN - PyModule_AddIntMacro(m, MSG_FASTOPEN); + ADD_INT_MACRO(m, MSG_FASTOPEN); #endif /* Protocol level and numbers, usable for [gs]etsockopt */ #ifdef SOL_SOCKET - PyModule_AddIntMacro(m, SOL_SOCKET); + ADD_INT_MACRO(m, SOL_SOCKET); #endif #ifdef SOL_IP - PyModule_AddIntMacro(m, SOL_IP); + ADD_INT_MACRO(m, SOL_IP); #else - PyModule_AddIntConstant(m, "SOL_IP", 0); + ADD_INT_CONST(m, "SOL_IP", 0); #endif #ifdef SOL_IPX - PyModule_AddIntMacro(m, SOL_IPX); + ADD_INT_MACRO(m, SOL_IPX); #endif #ifdef SOL_AX25 - PyModule_AddIntMacro(m, SOL_AX25); + ADD_INT_MACRO(m, SOL_AX25); #endif #ifdef SOL_ATALK - PyModule_AddIntMacro(m, SOL_ATALK); + ADD_INT_MACRO(m, SOL_ATALK); #endif #ifdef SOL_NETROM - PyModule_AddIntMacro(m, SOL_NETROM); + ADD_INT_MACRO(m, SOL_NETROM); #endif #ifdef SOL_ROSE - PyModule_AddIntMacro(m, SOL_ROSE); + ADD_INT_MACRO(m, SOL_ROSE); #endif #ifdef SOL_TCP - PyModule_AddIntMacro(m, SOL_TCP); + ADD_INT_MACRO(m, SOL_TCP); #else - PyModule_AddIntConstant(m, "SOL_TCP", 6); + ADD_INT_CONST(m, "SOL_TCP", 6); #endif #ifdef SOL_UDP - PyModule_AddIntMacro(m, SOL_UDP); + ADD_INT_MACRO(m, SOL_UDP); #else - PyModule_AddIntConstant(m, "SOL_UDP", 17); + ADD_INT_CONST(m, "SOL_UDP", 17); #endif #ifdef SOL_CAN_BASE - PyModule_AddIntMacro(m, SOL_CAN_BASE); + ADD_INT_MACRO(m, SOL_CAN_BASE); #endif #ifdef SOL_CAN_RAW - PyModule_AddIntMacro(m, SOL_CAN_RAW); - PyModule_AddIntMacro(m, CAN_RAW); + ADD_INT_MACRO(m, SOL_CAN_RAW); + ADD_INT_MACRO(m, CAN_RAW); #endif #if defined(HAVE_LINUX_CAN_H) || defined(HAVE_NETCAN_CAN_H) - PyModule_AddIntMacro(m, CAN_EFF_FLAG); - PyModule_AddIntMacro(m, CAN_RTR_FLAG); - PyModule_AddIntMacro(m, CAN_ERR_FLAG); + ADD_INT_MACRO(m, CAN_EFF_FLAG); + ADD_INT_MACRO(m, CAN_RTR_FLAG); + ADD_INT_MACRO(m, CAN_ERR_FLAG); - PyModule_AddIntMacro(m, CAN_SFF_MASK); - PyModule_AddIntMacro(m, CAN_EFF_MASK); - PyModule_AddIntMacro(m, CAN_ERR_MASK); + ADD_INT_MACRO(m, CAN_SFF_MASK); + ADD_INT_MACRO(m, CAN_EFF_MASK); + ADD_INT_MACRO(m, CAN_ERR_MASK); #ifdef CAN_ISOTP - PyModule_AddIntMacro(m, CAN_ISOTP); + ADD_INT_MACRO(m, CAN_ISOTP); #endif #ifdef CAN_J1939 - PyModule_AddIntMacro(m, CAN_J1939); + ADD_INT_MACRO(m, CAN_J1939); #endif #endif #if defined(HAVE_LINUX_CAN_RAW_H) || defined(HAVE_NETCAN_CAN_H) - PyModule_AddIntMacro(m, CAN_RAW_FILTER); + ADD_INT_MACRO(m, CAN_RAW_FILTER); #ifdef CAN_RAW_ERR_FILTER - PyModule_AddIntMacro(m, CAN_RAW_ERR_FILTER); + ADD_INT_MACRO(m, CAN_RAW_ERR_FILTER); #endif - PyModule_AddIntMacro(m, CAN_RAW_LOOPBACK); - PyModule_AddIntMacro(m, CAN_RAW_RECV_OWN_MSGS); + ADD_INT_MACRO(m, CAN_RAW_LOOPBACK); + ADD_INT_MACRO(m, CAN_RAW_RECV_OWN_MSGS); #endif #ifdef HAVE_LINUX_CAN_RAW_FD_FRAMES - PyModule_AddIntMacro(m, CAN_RAW_FD_FRAMES); + ADD_INT_MACRO(m, CAN_RAW_FD_FRAMES); #endif #ifdef HAVE_LINUX_CAN_RAW_JOIN_FILTERS - PyModule_AddIntMacro(m, CAN_RAW_JOIN_FILTERS); + ADD_INT_MACRO(m, CAN_RAW_JOIN_FILTERS); #endif #ifdef HAVE_LINUX_CAN_BCM_H - PyModule_AddIntMacro(m, CAN_BCM); + ADD_INT_MACRO(m, CAN_BCM); /* BCM opcodes */ - PyModule_AddIntConstant(m, "CAN_BCM_TX_SETUP", TX_SETUP); - PyModule_AddIntConstant(m, "CAN_BCM_TX_DELETE", TX_DELETE); - PyModule_AddIntConstant(m, "CAN_BCM_TX_READ", TX_READ); - PyModule_AddIntConstant(m, "CAN_BCM_TX_SEND", TX_SEND); - PyModule_AddIntConstant(m, "CAN_BCM_RX_SETUP", RX_SETUP); - PyModule_AddIntConstant(m, "CAN_BCM_RX_DELETE", RX_DELETE); - PyModule_AddIntConstant(m, "CAN_BCM_RX_READ", RX_READ); - PyModule_AddIntConstant(m, "CAN_BCM_TX_STATUS", TX_STATUS); - PyModule_AddIntConstant(m, "CAN_BCM_TX_EXPIRED", TX_EXPIRED); - PyModule_AddIntConstant(m, "CAN_BCM_RX_STATUS", RX_STATUS); - PyModule_AddIntConstant(m, "CAN_BCM_RX_TIMEOUT", RX_TIMEOUT); - PyModule_AddIntConstant(m, "CAN_BCM_RX_CHANGED", RX_CHANGED); + ADD_INT_CONST(m, "CAN_BCM_TX_SETUP", TX_SETUP); + ADD_INT_CONST(m, "CAN_BCM_TX_DELETE", TX_DELETE); + ADD_INT_CONST(m, "CAN_BCM_TX_READ", TX_READ); + ADD_INT_CONST(m, "CAN_BCM_TX_SEND", TX_SEND); + ADD_INT_CONST(m, "CAN_BCM_RX_SETUP", RX_SETUP); + ADD_INT_CONST(m, "CAN_BCM_RX_DELETE", RX_DELETE); + ADD_INT_CONST(m, "CAN_BCM_RX_READ", RX_READ); + ADD_INT_CONST(m, "CAN_BCM_TX_STATUS", TX_STATUS); + ADD_INT_CONST(m, "CAN_BCM_TX_EXPIRED", TX_EXPIRED); + ADD_INT_CONST(m, "CAN_BCM_RX_STATUS", RX_STATUS); + ADD_INT_CONST(m, "CAN_BCM_RX_TIMEOUT", RX_TIMEOUT); + ADD_INT_CONST(m, "CAN_BCM_RX_CHANGED", RX_CHANGED); /* BCM flags */ - PyModule_AddIntConstant(m, "CAN_BCM_SETTIMER", SETTIMER); - PyModule_AddIntConstant(m, "CAN_BCM_STARTTIMER", STARTTIMER); - PyModule_AddIntConstant(m, "CAN_BCM_TX_COUNTEVT", TX_COUNTEVT); - PyModule_AddIntConstant(m, "CAN_BCM_TX_ANNOUNCE", TX_ANNOUNCE); - PyModule_AddIntConstant(m, "CAN_BCM_TX_CP_CAN_ID", TX_CP_CAN_ID); - PyModule_AddIntConstant(m, "CAN_BCM_RX_FILTER_ID", RX_FILTER_ID); - PyModule_AddIntConstant(m, "CAN_BCM_RX_CHECK_DLC", RX_CHECK_DLC); - PyModule_AddIntConstant(m, "CAN_BCM_RX_NO_AUTOTIMER", RX_NO_AUTOTIMER); - PyModule_AddIntConstant(m, "CAN_BCM_RX_ANNOUNCE_RESUME", RX_ANNOUNCE_RESUME); - PyModule_AddIntConstant(m, "CAN_BCM_TX_RESET_MULTI_IDX", TX_RESET_MULTI_IDX); - PyModule_AddIntConstant(m, "CAN_BCM_RX_RTR_FRAME", RX_RTR_FRAME); + ADD_INT_CONST(m, "CAN_BCM_SETTIMER", SETTIMER); + ADD_INT_CONST(m, "CAN_BCM_STARTTIMER", STARTTIMER); + ADD_INT_CONST(m, "CAN_BCM_TX_COUNTEVT", TX_COUNTEVT); + ADD_INT_CONST(m, "CAN_BCM_TX_ANNOUNCE", TX_ANNOUNCE); + ADD_INT_CONST(m, "CAN_BCM_TX_CP_CAN_ID", TX_CP_CAN_ID); + ADD_INT_CONST(m, "CAN_BCM_RX_FILTER_ID", RX_FILTER_ID); + ADD_INT_CONST(m, "CAN_BCM_RX_CHECK_DLC", RX_CHECK_DLC); + ADD_INT_CONST(m, "CAN_BCM_RX_NO_AUTOTIMER", RX_NO_AUTOTIMER); + ADD_INT_CONST(m, "CAN_BCM_RX_ANNOUNCE_RESUME", RX_ANNOUNCE_RESUME); + ADD_INT_CONST(m, "CAN_BCM_TX_RESET_MULTI_IDX", TX_RESET_MULTI_IDX); + ADD_INT_CONST(m, "CAN_BCM_RX_RTR_FRAME", RX_RTR_FRAME); #ifdef CAN_FD_FRAME /* CAN_FD_FRAME was only introduced in the 4.8.x kernel series */ - PyModule_AddIntConstant(m, "CAN_BCM_CAN_FD_FRAME", CAN_FD_FRAME); + ADD_INT_CONST(m, "CAN_BCM_CAN_FD_FRAME", CAN_FD_FRAME); #endif #endif #ifdef HAVE_LINUX_CAN_J1939_H - PyModule_AddIntMacro(m, J1939_MAX_UNICAST_ADDR); - PyModule_AddIntMacro(m, J1939_IDLE_ADDR); - PyModule_AddIntMacro(m, J1939_NO_ADDR); - PyModule_AddIntMacro(m, J1939_NO_NAME); - PyModule_AddIntMacro(m, J1939_PGN_REQUEST); - PyModule_AddIntMacro(m, J1939_PGN_ADDRESS_CLAIMED); - PyModule_AddIntMacro(m, J1939_PGN_ADDRESS_COMMANDED); - PyModule_AddIntMacro(m, J1939_PGN_PDU1_MAX); - PyModule_AddIntMacro(m, J1939_PGN_MAX); - PyModule_AddIntMacro(m, J1939_NO_PGN); + ADD_INT_MACRO(m, J1939_MAX_UNICAST_ADDR); + ADD_INT_MACRO(m, J1939_IDLE_ADDR); + ADD_INT_MACRO(m, J1939_NO_ADDR); + ADD_INT_MACRO(m, J1939_NO_NAME); + ADD_INT_MACRO(m, J1939_PGN_REQUEST); + ADD_INT_MACRO(m, J1939_PGN_ADDRESS_CLAIMED); + ADD_INT_MACRO(m, J1939_PGN_ADDRESS_COMMANDED); + ADD_INT_MACRO(m, J1939_PGN_PDU1_MAX); + ADD_INT_MACRO(m, J1939_PGN_MAX); + ADD_INT_MACRO(m, J1939_NO_PGN); /* J1939 socket options */ - PyModule_AddIntMacro(m, SO_J1939_FILTER); - PyModule_AddIntMacro(m, SO_J1939_PROMISC); - PyModule_AddIntMacro(m, SO_J1939_SEND_PRIO); - PyModule_AddIntMacro(m, SO_J1939_ERRQUEUE); + ADD_INT_MACRO(m, SO_J1939_FILTER); + ADD_INT_MACRO(m, SO_J1939_PROMISC); + ADD_INT_MACRO(m, SO_J1939_SEND_PRIO); + ADD_INT_MACRO(m, SO_J1939_ERRQUEUE); - PyModule_AddIntMacro(m, SCM_J1939_DEST_ADDR); - PyModule_AddIntMacro(m, SCM_J1939_DEST_NAME); - PyModule_AddIntMacro(m, SCM_J1939_PRIO); - PyModule_AddIntMacro(m, SCM_J1939_ERRQUEUE); + ADD_INT_MACRO(m, SCM_J1939_DEST_ADDR); + ADD_INT_MACRO(m, SCM_J1939_DEST_NAME); + ADD_INT_MACRO(m, SCM_J1939_PRIO); + ADD_INT_MACRO(m, SCM_J1939_ERRQUEUE); - PyModule_AddIntMacro(m, J1939_NLA_PAD); - PyModule_AddIntMacro(m, J1939_NLA_BYTES_ACKED); + ADD_INT_MACRO(m, J1939_NLA_PAD); + ADD_INT_MACRO(m, J1939_NLA_BYTES_ACKED); - PyModule_AddIntMacro(m, J1939_EE_INFO_NONE); - PyModule_AddIntMacro(m, J1939_EE_INFO_TX_ABORT); + ADD_INT_MACRO(m, J1939_EE_INFO_NONE); + ADD_INT_MACRO(m, J1939_EE_INFO_TX_ABORT); - PyModule_AddIntMacro(m, J1939_FILTER_MAX); + ADD_INT_MACRO(m, J1939_FILTER_MAX); #endif #ifdef SOL_RDS - PyModule_AddIntMacro(m, SOL_RDS); + ADD_INT_MACRO(m, SOL_RDS); #endif #ifdef HAVE_SOCKADDR_ALG - PyModule_AddIntMacro(m, SOL_ALG); + ADD_INT_MACRO(m, SOL_ALG); #endif #ifdef RDS_CANCEL_SENT_TO - PyModule_AddIntMacro(m, RDS_CANCEL_SENT_TO); + ADD_INT_MACRO(m, RDS_CANCEL_SENT_TO); #endif #ifdef RDS_GET_MR - PyModule_AddIntMacro(m, RDS_GET_MR); + ADD_INT_MACRO(m, RDS_GET_MR); #endif #ifdef RDS_FREE_MR - PyModule_AddIntMacro(m, RDS_FREE_MR); + ADD_INT_MACRO(m, RDS_FREE_MR); #endif #ifdef RDS_RECVERR - PyModule_AddIntMacro(m, RDS_RECVERR); + ADD_INT_MACRO(m, RDS_RECVERR); #endif #ifdef RDS_CONG_MONITOR - PyModule_AddIntMacro(m, RDS_CONG_MONITOR); + ADD_INT_MACRO(m, RDS_CONG_MONITOR); #endif #ifdef RDS_GET_MR_FOR_DEST - PyModule_AddIntMacro(m, RDS_GET_MR_FOR_DEST); + ADD_INT_MACRO(m, RDS_GET_MR_FOR_DEST); #endif #ifdef IPPROTO_IP - PyModule_AddIntMacro(m, IPPROTO_IP); + ADD_INT_MACRO(m, IPPROTO_IP); #else - PyModule_AddIntConstant(m, "IPPROTO_IP", 0); + ADD_INT_CONST(m, "IPPROTO_IP", 0); #endif #ifdef IPPROTO_HOPOPTS - PyModule_AddIntMacro(m, IPPROTO_HOPOPTS); + ADD_INT_MACRO(m, IPPROTO_HOPOPTS); #endif #ifdef IPPROTO_ICMP - PyModule_AddIntMacro(m, IPPROTO_ICMP); + ADD_INT_MACRO(m, IPPROTO_ICMP); #else - PyModule_AddIntConstant(m, "IPPROTO_ICMP", 1); + ADD_INT_CONST(m, "IPPROTO_ICMP", 1); #endif #ifdef IPPROTO_IGMP - PyModule_AddIntMacro(m, IPPROTO_IGMP); + ADD_INT_MACRO(m, IPPROTO_IGMP); #endif #ifdef IPPROTO_GGP - PyModule_AddIntMacro(m, IPPROTO_GGP); + ADD_INT_MACRO(m, IPPROTO_GGP); #endif #ifdef IPPROTO_IPV4 - PyModule_AddIntMacro(m, IPPROTO_IPV4); + ADD_INT_MACRO(m, IPPROTO_IPV4); #endif #ifdef IPPROTO_IPV6 - PyModule_AddIntMacro(m, IPPROTO_IPV6); + ADD_INT_MACRO(m, IPPROTO_IPV6); #endif #ifdef IPPROTO_IPIP - PyModule_AddIntMacro(m, IPPROTO_IPIP); + ADD_INT_MACRO(m, IPPROTO_IPIP); #endif #ifdef IPPROTO_TCP - PyModule_AddIntMacro(m, IPPROTO_TCP); + ADD_INT_MACRO(m, IPPROTO_TCP); #else - PyModule_AddIntConstant(m, "IPPROTO_TCP", 6); + ADD_INT_CONST(m, "IPPROTO_TCP", 6); #endif #ifdef IPPROTO_EGP - PyModule_AddIntMacro(m, IPPROTO_EGP); + ADD_INT_MACRO(m, IPPROTO_EGP); #endif #ifdef IPPROTO_PUP - PyModule_AddIntMacro(m, IPPROTO_PUP); + ADD_INT_MACRO(m, IPPROTO_PUP); #endif #ifdef IPPROTO_UDP - PyModule_AddIntMacro(m, IPPROTO_UDP); + ADD_INT_MACRO(m, IPPROTO_UDP); #else - PyModule_AddIntConstant(m, "IPPROTO_UDP", 17); + ADD_INT_CONST(m, "IPPROTO_UDP", 17); #endif #ifdef IPPROTO_UDPLITE - PyModule_AddIntMacro(m, IPPROTO_UDPLITE); + ADD_INT_MACRO(m, IPPROTO_UDPLITE); #ifndef UDPLITE_SEND_CSCOV #define UDPLITE_SEND_CSCOV 10 #endif - PyModule_AddIntMacro(m, UDPLITE_SEND_CSCOV); + ADD_INT_MACRO(m, UDPLITE_SEND_CSCOV); #ifndef UDPLITE_RECV_CSCOV #define UDPLITE_RECV_CSCOV 11 #endif - PyModule_AddIntMacro(m, UDPLITE_RECV_CSCOV); + ADD_INT_MACRO(m, UDPLITE_RECV_CSCOV); #endif #ifdef IPPROTO_IDP - PyModule_AddIntMacro(m, IPPROTO_IDP); + ADD_INT_MACRO(m, IPPROTO_IDP); #endif #ifdef IPPROTO_HELLO - PyModule_AddIntMacro(m, IPPROTO_HELLO); + ADD_INT_MACRO(m, IPPROTO_HELLO); #endif #ifdef IPPROTO_ND - PyModule_AddIntMacro(m, IPPROTO_ND); + ADD_INT_MACRO(m, IPPROTO_ND); #endif #ifdef IPPROTO_TP - PyModule_AddIntMacro(m, IPPROTO_TP); + ADD_INT_MACRO(m, IPPROTO_TP); #endif #ifdef IPPROTO_ROUTING - PyModule_AddIntMacro(m, IPPROTO_ROUTING); + ADD_INT_MACRO(m, IPPROTO_ROUTING); #endif #ifdef IPPROTO_FRAGMENT - PyModule_AddIntMacro(m, IPPROTO_FRAGMENT); + ADD_INT_MACRO(m, IPPROTO_FRAGMENT); #endif #ifdef IPPROTO_RSVP - PyModule_AddIntMacro(m, IPPROTO_RSVP); + ADD_INT_MACRO(m, IPPROTO_RSVP); #endif #ifdef IPPROTO_GRE - PyModule_AddIntMacro(m, IPPROTO_GRE); + ADD_INT_MACRO(m, IPPROTO_GRE); #endif #ifdef IPPROTO_ESP - PyModule_AddIntMacro(m, IPPROTO_ESP); + ADD_INT_MACRO(m, IPPROTO_ESP); #endif #ifdef IPPROTO_AH - PyModule_AddIntMacro(m, IPPROTO_AH); + ADD_INT_MACRO(m, IPPROTO_AH); #endif #ifdef IPPROTO_MOBILE - PyModule_AddIntMacro(m, IPPROTO_MOBILE); + ADD_INT_MACRO(m, IPPROTO_MOBILE); #endif #ifdef IPPROTO_ICMPV6 - PyModule_AddIntMacro(m, IPPROTO_ICMPV6); + ADD_INT_MACRO(m, IPPROTO_ICMPV6); #endif #ifdef IPPROTO_NONE - PyModule_AddIntMacro(m, IPPROTO_NONE); + ADD_INT_MACRO(m, IPPROTO_NONE); #endif #ifdef IPPROTO_DSTOPTS - PyModule_AddIntMacro(m, IPPROTO_DSTOPTS); + ADD_INT_MACRO(m, IPPROTO_DSTOPTS); #endif #ifdef IPPROTO_XTP - PyModule_AddIntMacro(m, IPPROTO_XTP); + ADD_INT_MACRO(m, IPPROTO_XTP); #endif #ifdef IPPROTO_EON - PyModule_AddIntMacro(m, IPPROTO_EON); + ADD_INT_MACRO(m, IPPROTO_EON); #endif #ifdef IPPROTO_PIM - PyModule_AddIntMacro(m, IPPROTO_PIM); + ADD_INT_MACRO(m, IPPROTO_PIM); #endif #ifdef IPPROTO_IPCOMP - PyModule_AddIntMacro(m, IPPROTO_IPCOMP); + ADD_INT_MACRO(m, IPPROTO_IPCOMP); #endif #ifdef IPPROTO_VRRP - PyModule_AddIntMacro(m, IPPROTO_VRRP); + ADD_INT_MACRO(m, IPPROTO_VRRP); #endif #ifdef IPPROTO_SCTP - PyModule_AddIntMacro(m, IPPROTO_SCTP); + ADD_INT_MACRO(m, IPPROTO_SCTP); #endif #ifdef IPPROTO_BIP - PyModule_AddIntMacro(m, IPPROTO_BIP); + ADD_INT_MACRO(m, IPPROTO_BIP); #endif #ifdef IPPROTO_MPTCP - PyModule_AddIntMacro(m, IPPROTO_MPTCP); + ADD_INT_MACRO(m, IPPROTO_MPTCP); #endif /**/ #ifdef IPPROTO_RAW - PyModule_AddIntMacro(m, IPPROTO_RAW); + ADD_INT_MACRO(m, IPPROTO_RAW); #else - PyModule_AddIntConstant(m, "IPPROTO_RAW", 255); + ADD_INT_CONST(m, "IPPROTO_RAW", 255); #endif #ifdef IPPROTO_MAX - PyModule_AddIntMacro(m, IPPROTO_MAX); + ADD_INT_MACRO(m, IPPROTO_MAX); #endif #ifdef MS_WINDOWS - PyModule_AddIntMacro(m, IPPROTO_ICLFXBM); - PyModule_AddIntMacro(m, IPPROTO_ST); - PyModule_AddIntMacro(m, IPPROTO_CBT); - PyModule_AddIntMacro(m, IPPROTO_IGP); - PyModule_AddIntMacro(m, IPPROTO_RDP); - PyModule_AddIntMacro(m, IPPROTO_PGM); - PyModule_AddIntMacro(m, IPPROTO_L2TP); - PyModule_AddIntMacro(m, IPPROTO_SCTP); + ADD_INT_MACRO(m, IPPROTO_ICLFXBM); + ADD_INT_MACRO(m, IPPROTO_ST); + ADD_INT_MACRO(m, IPPROTO_CBT); + ADD_INT_MACRO(m, IPPROTO_IGP); + ADD_INT_MACRO(m, IPPROTO_RDP); + ADD_INT_MACRO(m, IPPROTO_PGM); + ADD_INT_MACRO(m, IPPROTO_L2TP); + ADD_INT_MACRO(m, IPPROTO_SCTP); #endif #ifdef SYSPROTO_CONTROL - PyModule_AddIntMacro(m, SYSPROTO_CONTROL); + ADD_INT_MACRO(m, SYSPROTO_CONTROL); #endif /* Some port configuration */ #ifdef IPPORT_RESERVED - PyModule_AddIntMacro(m, IPPORT_RESERVED); + ADD_INT_MACRO(m, IPPORT_RESERVED); #else - PyModule_AddIntConstant(m, "IPPORT_RESERVED", 1024); + ADD_INT_CONST(m, "IPPORT_RESERVED", 1024); #endif #ifdef IPPORT_USERRESERVED - PyModule_AddIntMacro(m, IPPORT_USERRESERVED); + ADD_INT_MACRO(m, IPPORT_USERRESERVED); #else - PyModule_AddIntConstant(m, "IPPORT_USERRESERVED", 5000); + ADD_INT_CONST(m, "IPPORT_USERRESERVED", 5000); #endif /* Some reserved IP v.4 addresses */ #ifdef INADDR_ANY - PyModule_AddIntMacro(m, INADDR_ANY); + ADD_INT_MACRO(m, INADDR_ANY); #else - PyModule_AddIntConstant(m, "INADDR_ANY", 0x00000000); + ADD_INT_CONST(m, "INADDR_ANY", 0x00000000); #endif #ifdef INADDR_BROADCAST - PyModule_AddIntMacro(m, INADDR_BROADCAST); + ADD_INT_MACRO(m, INADDR_BROADCAST); #else - PyModule_AddIntConstant(m, "INADDR_BROADCAST", 0xffffffff); + ADD_INT_CONST(m, "INADDR_BROADCAST", 0xffffffff); #endif #ifdef INADDR_LOOPBACK - PyModule_AddIntMacro(m, INADDR_LOOPBACK); + ADD_INT_MACRO(m, INADDR_LOOPBACK); #else - PyModule_AddIntConstant(m, "INADDR_LOOPBACK", 0x7F000001); + ADD_INT_CONST(m, "INADDR_LOOPBACK", 0x7F000001); #endif #ifdef INADDR_UNSPEC_GROUP - PyModule_AddIntMacro(m, INADDR_UNSPEC_GROUP); + ADD_INT_MACRO(m, INADDR_UNSPEC_GROUP); #else - PyModule_AddIntConstant(m, "INADDR_UNSPEC_GROUP", 0xe0000000); + ADD_INT_CONST(m, "INADDR_UNSPEC_GROUP", 0xe0000000); #endif #ifdef INADDR_ALLHOSTS_GROUP - PyModule_AddIntConstant(m, "INADDR_ALLHOSTS_GROUP", + ADD_INT_CONST(m, "INADDR_ALLHOSTS_GROUP", INADDR_ALLHOSTS_GROUP); #else - PyModule_AddIntConstant(m, "INADDR_ALLHOSTS_GROUP", 0xe0000001); + ADD_INT_CONST(m, "INADDR_ALLHOSTS_GROUP", 0xe0000001); #endif #ifdef INADDR_MAX_LOCAL_GROUP - PyModule_AddIntMacro(m, INADDR_MAX_LOCAL_GROUP); + ADD_INT_MACRO(m, INADDR_MAX_LOCAL_GROUP); #else - PyModule_AddIntConstant(m, "INADDR_MAX_LOCAL_GROUP", 0xe00000ff); + ADD_INT_CONST(m, "INADDR_MAX_LOCAL_GROUP", 0xe00000ff); #endif #ifdef INADDR_NONE - PyModule_AddIntMacro(m, INADDR_NONE); + ADD_INT_MACRO(m, INADDR_NONE); #else - PyModule_AddIntConstant(m, "INADDR_NONE", 0xffffffff); + ADD_INT_CONST(m, "INADDR_NONE", 0xffffffff); #endif /* IPv4 [gs]etsockopt options */ #ifdef IP_OPTIONS - PyModule_AddIntMacro(m, IP_OPTIONS); + ADD_INT_MACRO(m, IP_OPTIONS); #endif #ifdef IP_HDRINCL - PyModule_AddIntMacro(m, IP_HDRINCL); + ADD_INT_MACRO(m, IP_HDRINCL); #endif #ifdef IP_TOS - PyModule_AddIntMacro(m, IP_TOS); + ADD_INT_MACRO(m, IP_TOS); #endif #ifdef IP_TTL - PyModule_AddIntMacro(m, IP_TTL); + ADD_INT_MACRO(m, IP_TTL); #endif #ifdef IP_RECVOPTS - PyModule_AddIntMacro(m, IP_RECVOPTS); + ADD_INT_MACRO(m, IP_RECVOPTS); #endif #ifdef IP_RECVRETOPTS - PyModule_AddIntMacro(m, IP_RECVRETOPTS); + ADD_INT_MACRO(m, IP_RECVRETOPTS); #endif #ifdef IP_RECVTOS - PyModule_AddIntMacro(m, IP_RECVTOS); + ADD_INT_MACRO(m, IP_RECVTOS); #endif #ifdef IP_RECVDSTADDR - PyModule_AddIntMacro(m, IP_RECVDSTADDR); + ADD_INT_MACRO(m, IP_RECVDSTADDR); #endif #ifdef IP_RETOPTS - PyModule_AddIntMacro(m, IP_RETOPTS); + ADD_INT_MACRO(m, IP_RETOPTS); #endif #ifdef IP_MULTICAST_IF - PyModule_AddIntMacro(m, IP_MULTICAST_IF); + ADD_INT_MACRO(m, IP_MULTICAST_IF); #endif #ifdef IP_MULTICAST_TTL - PyModule_AddIntMacro(m, IP_MULTICAST_TTL); + ADD_INT_MACRO(m, IP_MULTICAST_TTL); #endif #ifdef IP_MULTICAST_LOOP - PyModule_AddIntMacro(m, IP_MULTICAST_LOOP); + ADD_INT_MACRO(m, IP_MULTICAST_LOOP); #endif #ifdef IP_ADD_MEMBERSHIP - PyModule_AddIntMacro(m, IP_ADD_MEMBERSHIP); + ADD_INT_MACRO(m, IP_ADD_MEMBERSHIP); #endif #ifdef IP_DROP_MEMBERSHIP - PyModule_AddIntMacro(m, IP_DROP_MEMBERSHIP); + ADD_INT_MACRO(m, IP_DROP_MEMBERSHIP); #endif #ifdef IP_DEFAULT_MULTICAST_TTL - PyModule_AddIntMacro(m, IP_DEFAULT_MULTICAST_TTL); + ADD_INT_MACRO(m, IP_DEFAULT_MULTICAST_TTL); #endif #ifdef IP_DEFAULT_MULTICAST_LOOP - PyModule_AddIntMacro(m, IP_DEFAULT_MULTICAST_LOOP); + ADD_INT_MACRO(m, IP_DEFAULT_MULTICAST_LOOP); #endif #ifdef IP_MAX_MEMBERSHIPS - PyModule_AddIntMacro(m, IP_MAX_MEMBERSHIPS); + ADD_INT_MACRO(m, IP_MAX_MEMBERSHIPS); #endif #ifdef IP_TRANSPARENT - PyModule_AddIntMacro(m, IP_TRANSPARENT); + ADD_INT_MACRO(m, IP_TRANSPARENT); #endif #ifdef IP_PKTINFO - PyModule_AddIntMacro(m, IP_PKTINFO); + ADD_INT_MACRO(m, IP_PKTINFO); #endif #ifdef IP_BIND_ADDRESS_NO_PORT - PyModule_AddIntMacro(m, IP_BIND_ADDRESS_NO_PORT); + ADD_INT_MACRO(m, IP_BIND_ADDRESS_NO_PORT); #endif /* IPv6 [gs]etsockopt options, defined in RFC2553 */ #ifdef IPV6_JOIN_GROUP - PyModule_AddIntMacro(m, IPV6_JOIN_GROUP); + ADD_INT_MACRO(m, IPV6_JOIN_GROUP); #endif #ifdef IPV6_LEAVE_GROUP - PyModule_AddIntMacro(m, IPV6_LEAVE_GROUP); + ADD_INT_MACRO(m, IPV6_LEAVE_GROUP); #endif #ifdef IPV6_MULTICAST_HOPS - PyModule_AddIntMacro(m, IPV6_MULTICAST_HOPS); + ADD_INT_MACRO(m, IPV6_MULTICAST_HOPS); #endif #ifdef IPV6_MULTICAST_IF - PyModule_AddIntMacro(m, IPV6_MULTICAST_IF); + ADD_INT_MACRO(m, IPV6_MULTICAST_IF); #endif #ifdef IPV6_MULTICAST_LOOP - PyModule_AddIntMacro(m, IPV6_MULTICAST_LOOP); + ADD_INT_MACRO(m, IPV6_MULTICAST_LOOP); #endif #ifdef IPV6_UNICAST_HOPS - PyModule_AddIntMacro(m, IPV6_UNICAST_HOPS); + ADD_INT_MACRO(m, IPV6_UNICAST_HOPS); #endif /* Additional IPV6 socket options, defined in RFC 3493 */ #ifdef IPV6_V6ONLY - PyModule_AddIntMacro(m, IPV6_V6ONLY); + ADD_INT_MACRO(m, IPV6_V6ONLY); #endif /* Advanced IPV6 socket options, from RFC 3542 */ #ifdef IPV6_CHECKSUM - PyModule_AddIntMacro(m, IPV6_CHECKSUM); + ADD_INT_MACRO(m, IPV6_CHECKSUM); #endif #ifdef IPV6_DONTFRAG - PyModule_AddIntMacro(m, IPV6_DONTFRAG); + ADD_INT_MACRO(m, IPV6_DONTFRAG); #endif #ifdef IPV6_DSTOPTS - PyModule_AddIntMacro(m, IPV6_DSTOPTS); + ADD_INT_MACRO(m, IPV6_DSTOPTS); #endif #ifdef IPV6_HOPLIMIT - PyModule_AddIntMacro(m, IPV6_HOPLIMIT); + ADD_INT_MACRO(m, IPV6_HOPLIMIT); #endif #ifdef IPV6_HOPOPTS - PyModule_AddIntMacro(m, IPV6_HOPOPTS); + ADD_INT_MACRO(m, IPV6_HOPOPTS); #endif #ifdef IPV6_NEXTHOP - PyModule_AddIntMacro(m, IPV6_NEXTHOP); + ADD_INT_MACRO(m, IPV6_NEXTHOP); #endif #ifdef IPV6_PATHMTU - PyModule_AddIntMacro(m, IPV6_PATHMTU); + ADD_INT_MACRO(m, IPV6_PATHMTU); #endif #ifdef IPV6_PKTINFO - PyModule_AddIntMacro(m, IPV6_PKTINFO); + ADD_INT_MACRO(m, IPV6_PKTINFO); #endif #ifdef IPV6_RECVDSTOPTS - PyModule_AddIntMacro(m, IPV6_RECVDSTOPTS); + ADD_INT_MACRO(m, IPV6_RECVDSTOPTS); #endif #ifdef IPV6_RECVHOPLIMIT - PyModule_AddIntMacro(m, IPV6_RECVHOPLIMIT); + ADD_INT_MACRO(m, IPV6_RECVHOPLIMIT); #endif #ifdef IPV6_RECVHOPOPTS - PyModule_AddIntMacro(m, IPV6_RECVHOPOPTS); + ADD_INT_MACRO(m, IPV6_RECVHOPOPTS); #endif #ifdef IPV6_RECVPKTINFO - PyModule_AddIntMacro(m, IPV6_RECVPKTINFO); + ADD_INT_MACRO(m, IPV6_RECVPKTINFO); #endif #ifdef IPV6_RECVRTHDR - PyModule_AddIntMacro(m, IPV6_RECVRTHDR); + ADD_INT_MACRO(m, IPV6_RECVRTHDR); #endif #ifdef IPV6_RECVTCLASS - PyModule_AddIntMacro(m, IPV6_RECVTCLASS); + ADD_INT_MACRO(m, IPV6_RECVTCLASS); #endif #ifdef IPV6_RTHDR - PyModule_AddIntMacro(m, IPV6_RTHDR); + ADD_INT_MACRO(m, IPV6_RTHDR); #endif #ifdef IPV6_RTHDRDSTOPTS - PyModule_AddIntMacro(m, IPV6_RTHDRDSTOPTS); + ADD_INT_MACRO(m, IPV6_RTHDRDSTOPTS); #endif #ifdef IPV6_RTHDR_TYPE_0 - PyModule_AddIntMacro(m, IPV6_RTHDR_TYPE_0); + ADD_INT_MACRO(m, IPV6_RTHDR_TYPE_0); #endif #ifdef IPV6_RECVPATHMTU - PyModule_AddIntMacro(m, IPV6_RECVPATHMTU); + ADD_INT_MACRO(m, IPV6_RECVPATHMTU); #endif #ifdef IPV6_TCLASS - PyModule_AddIntMacro(m, IPV6_TCLASS); + ADD_INT_MACRO(m, IPV6_TCLASS); #endif #ifdef IPV6_USE_MIN_MTU - PyModule_AddIntMacro(m, IPV6_USE_MIN_MTU); + ADD_INT_MACRO(m, IPV6_USE_MIN_MTU); #endif /* TCP options */ #ifdef TCP_NODELAY - PyModule_AddIntMacro(m, TCP_NODELAY); + ADD_INT_MACRO(m, TCP_NODELAY); #endif #ifdef TCP_MAXSEG - PyModule_AddIntMacro(m, TCP_MAXSEG); + ADD_INT_MACRO(m, TCP_MAXSEG); #endif #ifdef TCP_CORK - PyModule_AddIntMacro(m, TCP_CORK); + ADD_INT_MACRO(m, TCP_CORK); #endif #ifdef TCP_KEEPIDLE - PyModule_AddIntMacro(m, TCP_KEEPIDLE); + ADD_INT_MACRO(m, TCP_KEEPIDLE); #endif /* TCP_KEEPALIVE is OSX's TCP_KEEPIDLE equivalent */ #if defined(__APPLE__) && defined(TCP_KEEPALIVE) - PyModule_AddIntMacro(m, TCP_KEEPALIVE); + ADD_INT_MACRO(m, TCP_KEEPALIVE); #endif #ifdef TCP_KEEPINTVL - PyModule_AddIntMacro(m, TCP_KEEPINTVL); + ADD_INT_MACRO(m, TCP_KEEPINTVL); #endif #ifdef TCP_KEEPCNT - PyModule_AddIntMacro(m, TCP_KEEPCNT); + ADD_INT_MACRO(m, TCP_KEEPCNT); #endif #ifdef TCP_SYNCNT - PyModule_AddIntMacro(m, TCP_SYNCNT); + ADD_INT_MACRO(m, TCP_SYNCNT); #endif #ifdef TCP_LINGER2 - PyModule_AddIntMacro(m, TCP_LINGER2); + ADD_INT_MACRO(m, TCP_LINGER2); #endif #ifdef TCP_DEFER_ACCEPT - PyModule_AddIntMacro(m, TCP_DEFER_ACCEPT); + ADD_INT_MACRO(m, TCP_DEFER_ACCEPT); #endif #ifdef TCP_WINDOW_CLAMP - PyModule_AddIntMacro(m, TCP_WINDOW_CLAMP); + ADD_INT_MACRO(m, TCP_WINDOW_CLAMP); #endif #ifdef TCP_INFO - PyModule_AddIntMacro(m, TCP_INFO); + ADD_INT_MACRO(m, TCP_INFO); #endif #ifdef TCP_CONNECTION_INFO - PyModule_AddIntMacro(m, TCP_CONNECTION_INFO); + ADD_INT_MACRO(m, TCP_CONNECTION_INFO); #endif #ifdef TCP_QUICKACK - PyModule_AddIntMacro(m, TCP_QUICKACK); + ADD_INT_MACRO(m, TCP_QUICKACK); #endif #ifdef TCP_CONGESTION - PyModule_AddIntMacro(m, TCP_CONGESTION); + ADD_INT_MACRO(m, TCP_CONGESTION); #endif #ifdef TCP_MD5SIG - PyModule_AddIntMacro(m, TCP_MD5SIG); + ADD_INT_MACRO(m, TCP_MD5SIG); #endif #ifdef TCP_THIN_LINEAR_TIMEOUTS - PyModule_AddIntMacro(m, TCP_THIN_LINEAR_TIMEOUTS); + ADD_INT_MACRO(m, TCP_THIN_LINEAR_TIMEOUTS); #endif #ifdef TCP_THIN_DUPACK - PyModule_AddIntMacro(m, TCP_THIN_DUPACK); + ADD_INT_MACRO(m, TCP_THIN_DUPACK); #endif #ifdef TCP_USER_TIMEOUT - PyModule_AddIntMacro(m, TCP_USER_TIMEOUT); + ADD_INT_MACRO(m, TCP_USER_TIMEOUT); #endif #ifdef TCP_REPAIR - PyModule_AddIntMacro(m, TCP_REPAIR); + ADD_INT_MACRO(m, TCP_REPAIR); #endif #ifdef TCP_REPAIR_QUEUE - PyModule_AddIntMacro(m, TCP_REPAIR_QUEUE); + ADD_INT_MACRO(m, TCP_REPAIR_QUEUE); #endif #ifdef TCP_QUEUE_SEQ - PyModule_AddIntMacro(m, TCP_QUEUE_SEQ); + ADD_INT_MACRO(m, TCP_QUEUE_SEQ); #endif #ifdef TCP_REPAIR_OPTIONS - PyModule_AddIntMacro(m, TCP_REPAIR_OPTIONS); + ADD_INT_MACRO(m, TCP_REPAIR_OPTIONS); #endif #ifdef TCP_FASTOPEN - PyModule_AddIntMacro(m, TCP_FASTOPEN); + ADD_INT_MACRO(m, TCP_FASTOPEN); #endif #ifdef TCP_TIMESTAMP - PyModule_AddIntMacro(m, TCP_TIMESTAMP); + ADD_INT_MACRO(m, TCP_TIMESTAMP); #endif #ifdef TCP_NOTSENT_LOWAT - PyModule_AddIntMacro(m, TCP_NOTSENT_LOWAT); + ADD_INT_MACRO(m, TCP_NOTSENT_LOWAT); #endif #ifdef TCP_CC_INFO - PyModule_AddIntMacro(m, TCP_CC_INFO); + ADD_INT_MACRO(m, TCP_CC_INFO); #endif #ifdef TCP_SAVE_SYN - PyModule_AddIntMacro(m, TCP_SAVE_SYN); + ADD_INT_MACRO(m, TCP_SAVE_SYN); #endif #ifdef TCP_SAVED_SYN - PyModule_AddIntMacro(m, TCP_SAVED_SYN); + ADD_INT_MACRO(m, TCP_SAVED_SYN); #endif #ifdef TCP_REPAIR_WINDOW - PyModule_AddIntMacro(m, TCP_REPAIR_WINDOW); + ADD_INT_MACRO(m, TCP_REPAIR_WINDOW); #endif #ifdef TCP_FASTOPEN_CONNECT - PyModule_AddIntMacro(m, TCP_FASTOPEN_CONNECT); + ADD_INT_MACRO(m, TCP_FASTOPEN_CONNECT); #endif #ifdef TCP_ULP - PyModule_AddIntMacro(m, TCP_ULP); + ADD_INT_MACRO(m, TCP_ULP); #endif #ifdef TCP_MD5SIG_EXT - PyModule_AddIntMacro(m, TCP_MD5SIG_EXT); + ADD_INT_MACRO(m, TCP_MD5SIG_EXT); #endif #ifdef TCP_FASTOPEN_KEY - PyModule_AddIntMacro(m, TCP_FASTOPEN_KEY); + ADD_INT_MACRO(m, TCP_FASTOPEN_KEY); #endif #ifdef TCP_FASTOPEN_NO_COOKIE - PyModule_AddIntMacro(m, TCP_FASTOPEN_NO_COOKIE); + ADD_INT_MACRO(m, TCP_FASTOPEN_NO_COOKIE); #endif #ifdef TCP_ZEROCOPY_RECEIVE - PyModule_AddIntMacro(m, TCP_ZEROCOPY_RECEIVE); + ADD_INT_MACRO(m, TCP_ZEROCOPY_RECEIVE); #endif #ifdef TCP_INQ - PyModule_AddIntMacro(m, TCP_INQ); + ADD_INT_MACRO(m, TCP_INQ); #endif #ifdef TCP_TX_DELAY - PyModule_AddIntMacro(m, TCP_TX_DELAY); + ADD_INT_MACRO(m, TCP_TX_DELAY); #endif /* IPX options */ #ifdef IPX_TYPE - PyModule_AddIntMacro(m, IPX_TYPE); + ADD_INT_MACRO(m, IPX_TYPE); #endif /* Reliable Datagram Sockets */ #ifdef RDS_CMSG_RDMA_ARGS - PyModule_AddIntMacro(m, RDS_CMSG_RDMA_ARGS); + ADD_INT_MACRO(m, RDS_CMSG_RDMA_ARGS); #endif #ifdef RDS_CMSG_RDMA_DEST - PyModule_AddIntMacro(m, RDS_CMSG_RDMA_DEST); + ADD_INT_MACRO(m, RDS_CMSG_RDMA_DEST); #endif #ifdef RDS_CMSG_RDMA_MAP - PyModule_AddIntMacro(m, RDS_CMSG_RDMA_MAP); + ADD_INT_MACRO(m, RDS_CMSG_RDMA_MAP); #endif #ifdef RDS_CMSG_RDMA_STATUS - PyModule_AddIntMacro(m, RDS_CMSG_RDMA_STATUS); + ADD_INT_MACRO(m, RDS_CMSG_RDMA_STATUS); #endif #ifdef RDS_CMSG_RDMA_UPDATE - PyModule_AddIntMacro(m, RDS_CMSG_RDMA_UPDATE); + ADD_INT_MACRO(m, RDS_CMSG_RDMA_UPDATE); #endif #ifdef RDS_RDMA_READWRITE - PyModule_AddIntMacro(m, RDS_RDMA_READWRITE); + ADD_INT_MACRO(m, RDS_RDMA_READWRITE); #endif #ifdef RDS_RDMA_FENCE - PyModule_AddIntMacro(m, RDS_RDMA_FENCE); + ADD_INT_MACRO(m, RDS_RDMA_FENCE); #endif #ifdef RDS_RDMA_INVALIDATE - PyModule_AddIntMacro(m, RDS_RDMA_INVALIDATE); + ADD_INT_MACRO(m, RDS_RDMA_INVALIDATE); #endif #ifdef RDS_RDMA_USE_ONCE - PyModule_AddIntMacro(m, RDS_RDMA_USE_ONCE); + ADD_INT_MACRO(m, RDS_RDMA_USE_ONCE); #endif #ifdef RDS_RDMA_DONTWAIT - PyModule_AddIntMacro(m, RDS_RDMA_DONTWAIT); + ADD_INT_MACRO(m, RDS_RDMA_DONTWAIT); #endif #ifdef RDS_RDMA_NOTIFY_ME - PyModule_AddIntMacro(m, RDS_RDMA_NOTIFY_ME); + ADD_INT_MACRO(m, RDS_RDMA_NOTIFY_ME); #endif #ifdef RDS_RDMA_SILENT - PyModule_AddIntMacro(m, RDS_RDMA_SILENT); + ADD_INT_MACRO(m, RDS_RDMA_SILENT); #endif /* get{addr,name}info parameters */ #ifdef EAI_ADDRFAMILY - PyModule_AddIntMacro(m, EAI_ADDRFAMILY); + ADD_INT_MACRO(m, EAI_ADDRFAMILY); #endif #ifdef EAI_AGAIN - PyModule_AddIntMacro(m, EAI_AGAIN); + ADD_INT_MACRO(m, EAI_AGAIN); #endif #ifdef EAI_BADFLAGS - PyModule_AddIntMacro(m, EAI_BADFLAGS); + ADD_INT_MACRO(m, EAI_BADFLAGS); #endif #ifdef EAI_FAIL - PyModule_AddIntMacro(m, EAI_FAIL); + ADD_INT_MACRO(m, EAI_FAIL); #endif #ifdef EAI_FAMILY - PyModule_AddIntMacro(m, EAI_FAMILY); + ADD_INT_MACRO(m, EAI_FAMILY); #endif #ifdef EAI_MEMORY - PyModule_AddIntMacro(m, EAI_MEMORY); + ADD_INT_MACRO(m, EAI_MEMORY); #endif #ifdef EAI_NODATA - PyModule_AddIntMacro(m, EAI_NODATA); + ADD_INT_MACRO(m, EAI_NODATA); #endif #ifdef EAI_NONAME - PyModule_AddIntMacro(m, EAI_NONAME); + ADD_INT_MACRO(m, EAI_NONAME); #endif #ifdef EAI_OVERFLOW - PyModule_AddIntMacro(m, EAI_OVERFLOW); + ADD_INT_MACRO(m, EAI_OVERFLOW); #endif #ifdef EAI_SERVICE - PyModule_AddIntMacro(m, EAI_SERVICE); + ADD_INT_MACRO(m, EAI_SERVICE); #endif #ifdef EAI_SOCKTYPE - PyModule_AddIntMacro(m, EAI_SOCKTYPE); + ADD_INT_MACRO(m, EAI_SOCKTYPE); #endif #ifdef EAI_SYSTEM - PyModule_AddIntMacro(m, EAI_SYSTEM); + ADD_INT_MACRO(m, EAI_SYSTEM); #endif #ifdef EAI_BADHINTS - PyModule_AddIntMacro(m, EAI_BADHINTS); + ADD_INT_MACRO(m, EAI_BADHINTS); #endif #ifdef EAI_PROTOCOL - PyModule_AddIntMacro(m, EAI_PROTOCOL); + ADD_INT_MACRO(m, EAI_PROTOCOL); #endif #ifdef EAI_MAX - PyModule_AddIntMacro(m, EAI_MAX); + ADD_INT_MACRO(m, EAI_MAX); #endif #ifdef AI_PASSIVE - PyModule_AddIntMacro(m, AI_PASSIVE); + ADD_INT_MACRO(m, AI_PASSIVE); #endif #ifdef AI_CANONNAME - PyModule_AddIntMacro(m, AI_CANONNAME); + ADD_INT_MACRO(m, AI_CANONNAME); #endif #ifdef AI_NUMERICHOST - PyModule_AddIntMacro(m, AI_NUMERICHOST); + ADD_INT_MACRO(m, AI_NUMERICHOST); #endif #ifdef AI_NUMERICSERV - PyModule_AddIntMacro(m, AI_NUMERICSERV); + ADD_INT_MACRO(m, AI_NUMERICSERV); #endif #ifdef AI_MASK - PyModule_AddIntMacro(m, AI_MASK); + ADD_INT_MACRO(m, AI_MASK); #endif #ifdef AI_ALL - PyModule_AddIntMacro(m, AI_ALL); + ADD_INT_MACRO(m, AI_ALL); #endif #ifdef AI_V4MAPPED_CFG - PyModule_AddIntMacro(m, AI_V4MAPPED_CFG); + ADD_INT_MACRO(m, AI_V4MAPPED_CFG); #endif #ifdef AI_ADDRCONFIG - PyModule_AddIntMacro(m, AI_ADDRCONFIG); + ADD_INT_MACRO(m, AI_ADDRCONFIG); #endif #ifdef AI_V4MAPPED - PyModule_AddIntMacro(m, AI_V4MAPPED); + ADD_INT_MACRO(m, AI_V4MAPPED); #endif #ifdef AI_DEFAULT - PyModule_AddIntMacro(m, AI_DEFAULT); + ADD_INT_MACRO(m, AI_DEFAULT); #endif #ifdef NI_MAXHOST - PyModule_AddIntMacro(m, NI_MAXHOST); + ADD_INT_MACRO(m, NI_MAXHOST); #endif #ifdef NI_MAXSERV - PyModule_AddIntMacro(m, NI_MAXSERV); + ADD_INT_MACRO(m, NI_MAXSERV); #endif #ifdef NI_NOFQDN - PyModule_AddIntMacro(m, NI_NOFQDN); + ADD_INT_MACRO(m, NI_NOFQDN); #endif #ifdef NI_NUMERICHOST - PyModule_AddIntMacro(m, NI_NUMERICHOST); + ADD_INT_MACRO(m, NI_NUMERICHOST); #endif #ifdef NI_NAMEREQD - PyModule_AddIntMacro(m, NI_NAMEREQD); + ADD_INT_MACRO(m, NI_NAMEREQD); #endif #ifdef NI_NUMERICSERV - PyModule_AddIntMacro(m, NI_NUMERICSERV); + ADD_INT_MACRO(m, NI_NUMERICSERV); #endif #ifdef NI_DGRAM - PyModule_AddIntMacro(m, NI_DGRAM); + ADD_INT_MACRO(m, NI_DGRAM); #endif /* shutdown() parameters */ #ifdef SHUT_RD - PyModule_AddIntMacro(m, SHUT_RD); + ADD_INT_MACRO(m, SHUT_RD); #elif defined(SD_RECEIVE) - PyModule_AddIntConstant(m, "SHUT_RD", SD_RECEIVE); + ADD_INT_CONST(m, "SHUT_RD", SD_RECEIVE); #else - PyModule_AddIntConstant(m, "SHUT_RD", 0); + ADD_INT_CONST(m, "SHUT_RD", 0); #endif #ifdef SHUT_WR - PyModule_AddIntMacro(m, SHUT_WR); + ADD_INT_MACRO(m, SHUT_WR); #elif defined(SD_SEND) - PyModule_AddIntConstant(m, "SHUT_WR", SD_SEND); + ADD_INT_CONST(m, "SHUT_WR", SD_SEND); #else - PyModule_AddIntConstant(m, "SHUT_WR", 1); + ADD_INT_CONST(m, "SHUT_WR", 1); #endif #ifdef SHUT_RDWR - PyModule_AddIntMacro(m, SHUT_RDWR); + ADD_INT_MACRO(m, SHUT_RDWR); #elif defined(SD_BOTH) - PyModule_AddIntConstant(m, "SHUT_RDWR", SD_BOTH); + ADD_INT_CONST(m, "SHUT_RDWR", SD_BOTH); #else - PyModule_AddIntConstant(m, "SHUT_RDWR", 2); + ADD_INT_CONST(m, "SHUT_RDWR", 2); #endif #ifdef SIO_RCVALL @@ -8741,22 +8774,26 @@ PyInit__socket(void) #endif }; int i; - for(i = 0; i Date: Sat, 8 Apr 2023 10:09:00 +0800 Subject: [PATCH 55/76] gh-103329: Add regression test for PropertyMock with side effect (#103358) --- .../test_unittest/testmock/testhelpers.py | 23 ++++++++++++++++++- ...-04-08-00-50-23.gh-issue-103329.M38tqF.rst | 1 + 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-04-08-00-50-23.gh-issue-103329.M38tqF.rst diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index 9e7ec5d62d5da2..dc4d004cda8a75 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -1077,7 +1077,7 @@ def test_propertymock(self): p.stop() - def test_propertymock_returnvalue(self): + def test_propertymock_bare(self): m = MagicMock() p = PropertyMock() type(m).foo = p @@ -1088,6 +1088,27 @@ def test_propertymock_returnvalue(self): self.assertNotIsInstance(returned, PropertyMock) + def test_propertymock_returnvalue(self): + m = MagicMock() + p = PropertyMock(return_value=42) + type(m).foo = p + + returned = m.foo + p.assert_called_once_with() + self.assertEqual(returned, 42) + self.assertNotIsInstance(returned, PropertyMock) + + + def test_propertymock_side_effect(self): + m = MagicMock() + p = PropertyMock(side_effect=ValueError) + type(m).foo = p + + with self.assertRaises(ValueError): + m.foo + p.assert_called_once_with() + + class TestCallablePredicate(unittest.TestCase): def test_type(self): diff --git a/Misc/NEWS.d/next/Tests/2023-04-08-00-50-23.gh-issue-103329.M38tqF.rst b/Misc/NEWS.d/next/Tests/2023-04-08-00-50-23.gh-issue-103329.M38tqF.rst new file mode 100644 index 00000000000000..79448ed728040d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-04-08-00-50-23.gh-issue-103329.M38tqF.rst @@ -0,0 +1 @@ +Regression tests for the behaviour of ``unittest.mock.PropertyMock`` were added. From 13774969f78de0435da4efbdf21f1130a7a71964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Sat, 8 Apr 2023 09:04:23 +0200 Subject: [PATCH 56/76] gh-102809: Remove gdbinit mention in Misc/README (#103269) --- Misc/README | 1 - 1 file changed, 1 deletion(-) diff --git a/Misc/README b/Misc/README index e4dd2005411a8d..3dab768ba1a7a4 100644 --- a/Misc/README +++ b/Misc/README @@ -8,7 +8,6 @@ Files found here ---------------- ACKS Acknowledgements -gdbinit Handy stuff to put in your .gdbinit file, if you use gdb HISTORY News from previous releases -- oldest last indent.pro GNU indent profile approximating my C style NEWS News for this release (for some meaning of "this") From a34c79623877a7ed0c86a6d48ddccffa0de76836 Mon Sep 17 00:00:00 2001 From: "C.A.M. Gerlach" Date: Sat, 8 Apr 2023 02:37:23 -0500 Subject: [PATCH 57/76] gh-75729: Fix os.spawn tests not handling spaces on Windows (#99150) * Quote paths in os.spawn tests on Windows so they work with spaces * Add NEWS entry for os spawn test fix * Fix code style to avoid double negative in os.spawn tests Co-authored-by: Jelle Zijlstra --------- Co-authored-by: Jelle Zijlstra --- Lib/test/test_os.py | 102 ++++++++++-------- ...2-11-06-18-42-38.gh-issue-75729.uGYJrv.rst | 2 + 2 files changed, 59 insertions(+), 45 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2022-11-06-18-42-38.gh-issue-75729.uGYJrv.rst diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 74ece3ffb4ed17..584cc05ca82a55 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3214,6 +3214,14 @@ def kill_process(pid): @support.requires_subprocess() class SpawnTests(unittest.TestCase): + @staticmethod + def quote_args(args): + # On Windows, os.spawn* simply joins arguments with spaces: + # arguments need to be quoted + if os.name != 'nt': + return args + return [f'"{arg}"' if " " in arg.strip() else arg for arg in args] + def create_args(self, *, with_env=False, use_bytes=False): self.exitcode = 17 @@ -3234,115 +3242,118 @@ def create_args(self, *, with_env=False, use_bytes=False): with open(filename, "w", encoding="utf-8") as fp: fp.write(code) - args = [sys.executable, filename] + program = sys.executable + args = self.quote_args([program, filename]) if use_bytes: + program = os.fsencode(program) args = [os.fsencode(a) for a in args] self.env = {os.fsencode(k): os.fsencode(v) for k, v in self.env.items()} - return args + return program, args @requires_os_func('spawnl') def test_spawnl(self): - args = self.create_args() - exitcode = os.spawnl(os.P_WAIT, args[0], *args) + program, args = self.create_args() + exitcode = os.spawnl(os.P_WAIT, program, *args) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnle') def test_spawnle(self): - args = self.create_args(with_env=True) - exitcode = os.spawnle(os.P_WAIT, args[0], *args, self.env) + program, args = self.create_args(with_env=True) + exitcode = os.spawnle(os.P_WAIT, program, *args, self.env) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnlp') def test_spawnlp(self): - args = self.create_args() - exitcode = os.spawnlp(os.P_WAIT, args[0], *args) + program, args = self.create_args() + exitcode = os.spawnlp(os.P_WAIT, program, *args) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnlpe') def test_spawnlpe(self): - args = self.create_args(with_env=True) - exitcode = os.spawnlpe(os.P_WAIT, args[0], *args, self.env) + program, args = self.create_args(with_env=True) + exitcode = os.spawnlpe(os.P_WAIT, program, *args, self.env) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnv') def test_spawnv(self): - args = self.create_args() - exitcode = os.spawnv(os.P_WAIT, args[0], args) + program, args = self.create_args() + exitcode = os.spawnv(os.P_WAIT, program, args) self.assertEqual(exitcode, self.exitcode) # Test for PyUnicode_FSConverter() - exitcode = os.spawnv(os.P_WAIT, FakePath(args[0]), args) + exitcode = os.spawnv(os.P_WAIT, FakePath(program), args) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnve') def test_spawnve(self): - args = self.create_args(with_env=True) - exitcode = os.spawnve(os.P_WAIT, args[0], args, self.env) + program, args = self.create_args(with_env=True) + exitcode = os.spawnve(os.P_WAIT, program, args, self.env) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnvp') def test_spawnvp(self): - args = self.create_args() - exitcode = os.spawnvp(os.P_WAIT, args[0], args) + program, args = self.create_args() + exitcode = os.spawnvp(os.P_WAIT, program, args) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnvpe') def test_spawnvpe(self): - args = self.create_args(with_env=True) - exitcode = os.spawnvpe(os.P_WAIT, args[0], args, self.env) + program, args = self.create_args(with_env=True) + exitcode = os.spawnvpe(os.P_WAIT, program, args, self.env) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnv') def test_nowait(self): - args = self.create_args() - pid = os.spawnv(os.P_NOWAIT, args[0], args) + program, args = self.create_args() + pid = os.spawnv(os.P_NOWAIT, program, args) support.wait_process(pid, exitcode=self.exitcode) @requires_os_func('spawnve') def test_spawnve_bytes(self): # Test bytes handling in parse_arglist and parse_envlist (#28114) - args = self.create_args(with_env=True, use_bytes=True) - exitcode = os.spawnve(os.P_WAIT, args[0], args, self.env) + program, args = self.create_args(with_env=True, use_bytes=True) + exitcode = os.spawnve(os.P_WAIT, program, args, self.env) self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnl') def test_spawnl_noargs(self): - args = self.create_args() - self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, args[0]) - self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, args[0], '') + program, __ = self.create_args() + self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, program) + self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, program, '') @requires_os_func('spawnle') def test_spawnle_noargs(self): - args = self.create_args() - self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, args[0], {}) - self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, args[0], '', {}) + program, __ = self.create_args() + self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, program, {}) + self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, program, '', {}) @requires_os_func('spawnv') def test_spawnv_noargs(self): - args = self.create_args() - self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], ()) - self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], []) - self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], ('',)) - self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], ['']) + program, __ = self.create_args() + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, program, ()) + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, program, []) + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, program, ('',)) + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, program, ['']) @requires_os_func('spawnve') def test_spawnve_noargs(self): - args = self.create_args() - self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], (), {}) - self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], [], {}) - self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], ('',), {}) - self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], [''], {}) + program, __ = self.create_args() + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, program, (), {}) + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, program, [], {}) + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, program, ('',), {}) + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, program, [''], {}) def _test_invalid_env(self, spawn): - args = [sys.executable, '-c', 'pass'] + program = sys.executable + args = self.quote_args([program, '-c', 'pass']) # null character in the environment variable name newenv = os.environ.copy() newenv["FRUIT\0VEGETABLE"] = "cabbage" try: - exitcode = spawn(os.P_WAIT, args[0], args, newenv) + exitcode = spawn(os.P_WAIT, program, args, newenv) except ValueError: pass else: @@ -3352,7 +3363,7 @@ def _test_invalid_env(self, spawn): newenv = os.environ.copy() newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" try: - exitcode = spawn(os.P_WAIT, args[0], args, newenv) + exitcode = spawn(os.P_WAIT, program, args, newenv) except ValueError: pass else: @@ -3362,7 +3373,7 @@ def _test_invalid_env(self, spawn): newenv = os.environ.copy() newenv["FRUIT=ORANGE"] = "lemon" try: - exitcode = spawn(os.P_WAIT, args[0], args, newenv) + exitcode = spawn(os.P_WAIT, program, args, newenv) except ValueError: pass else: @@ -3375,10 +3386,11 @@ def _test_invalid_env(self, spawn): fp.write('import sys, os\n' 'if os.getenv("FRUIT") != "orange=lemon":\n' ' raise AssertionError') - args = [sys.executable, filename] + + args = self.quote_args([program, filename]) newenv = os.environ.copy() newenv["FRUIT"] = "orange=lemon" - exitcode = spawn(os.P_WAIT, args[0], args, newenv) + exitcode = spawn(os.P_WAIT, program, args, newenv) self.assertEqual(exitcode, 0) @requires_os_func('spawnve') diff --git a/Misc/NEWS.d/next/Tests/2022-11-06-18-42-38.gh-issue-75729.uGYJrv.rst b/Misc/NEWS.d/next/Tests/2022-11-06-18-42-38.gh-issue-75729.uGYJrv.rst new file mode 100644 index 00000000000000..8baecdfc31881f --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2022-11-06-18-42-38.gh-issue-75729.uGYJrv.rst @@ -0,0 +1,2 @@ +Fix the :func:`os.spawn* ` tests failing on Windows +when the working directory or interpreter path contains spaces. From 1e9dfdacefa2c8c27762ba6491b0f570147ee355 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Sat, 8 Apr 2023 08:56:20 +0100 Subject: [PATCH 58/76] Docs: use Node.findall to avoid a deprecation warning (#99403) --- Doc/tools/extensions/c_annotations.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index 5af56433f41573..3551bfa4c0f133 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -20,6 +20,7 @@ """ from os import path +import docutils from docutils import nodes from docutils.parsers.rst import directives from docutils.parsers.rst import Directive @@ -41,6 +42,16 @@ } +# Monkeypatch nodes.Node.findall for forwards compatability +# This patch can be dropped when the minimum Sphinx version is 4.4.0 +# or the minimum Docutils version is 0.18.1. +if docutils.__version_info__ < (0, 18, 1): + def findall(self, *args, **kwargs): + return iter(self.traverse(*args, **kwargs)) + + nodes.Node.findall = findall + + class RCEntry: def __init__(self, name): self.name = name @@ -87,7 +98,7 @@ def __init__(self, refcount_filename, stable_abi_file): self.stable_abi_data[name] = record def add_annotations(self, app, doctree): - for node in doctree.traverse(addnodes.desc_content): + for node in doctree.findall(addnodes.desc_content): par = node.parent if par['domain'] != 'c': continue From 3310b94d3db2f477cf2b8789c30ac0f22f82d2dd Mon Sep 17 00:00:00 2001 From: Ezio Melotti Date: Sat, 8 Apr 2023 16:46:47 +0800 Subject: [PATCH 59/76] gh-100574: add examples/links to the `strptime`/`strftime` docs (#100575) --- Doc/library/datetime.rst | 47 ++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 50827b27ebea04..761f5f04b9b288 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -737,18 +737,16 @@ Instance methods: .. method:: date.strftime(format) Return a string representing the date, controlled by an explicit format string. - Format codes referring to hours, minutes or seconds will see 0 values. For a - complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. + Format codes referring to hours, minutes or seconds will see 0 values. + See also :ref:`strftime-strptime-behavior` and :meth:`date.isoformat`. .. method:: date.__format__(format) Same as :meth:`.date.strftime`. This makes it possible to specify a format string for a :class:`.date` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. For a - complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. + literals ` and when using :meth:`str.format`. + See also :ref:`strftime-strptime-behavior` and :meth:`date.isoformat`. Examples of Usage: :class:`date` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1051,8 +1049,8 @@ Other constructors, all class methods: :exc:`ValueError` is raised if the date_string and format can't be parsed by :func:`time.strptime` or if it returns a value which isn't a - time tuple. For a complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. + time tuple. See also :ref:`strftime-strptime-behavior` and + :meth:`datetime.fromisoformat`. @@ -1510,20 +1508,21 @@ Instance methods: (which :func:`time.ctime` invokes, but which :meth:`datetime.ctime` does not invoke) conforms to the C standard. + .. method:: datetime.strftime(format) - Return a string representing the date and time, controlled by an explicit format - string. For a complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. + Return a string representing the date and time, + controlled by an explicit format string. + See also :ref:`strftime-strptime-behavior` and :meth:`datetime.isoformat`. .. method:: datetime.__format__(format) Same as :meth:`.datetime.strftime`. This makes it possible to specify a format string for a :class:`.datetime` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. For a - complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. + literals ` and when using :meth:`str.format`. + See also :ref:`strftime-strptime-behavior` and :meth:`datetime.isoformat`. + Examples of Usage: :class:`.datetime` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1868,17 +1867,15 @@ Instance methods: .. method:: time.strftime(format) Return a string representing the time, controlled by an explicit format - string. For a complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. + string. See also :ref:`strftime-strptime-behavior` and :meth:`time.isoformat`. .. method:: time.__format__(format) - Same as :meth:`.time.strftime`. This makes it possible to specify a format string - for a :class:`.time` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. For a - complete list of formatting directives, see - :ref:`strftime-strptime-behavior`. + Same as :meth:`.time.strftime`. This makes it possible to specify + a format string for a :class:`.time` object in :ref:`formatted string + literals ` and when using :meth:`str.format`. + See also :ref:`strftime-strptime-behavior` and :meth:`time.isoformat`. .. method:: time.utcoffset() @@ -2320,6 +2317,14 @@ versus :meth:`strptime`: :meth:`strftime` and :meth:`strptime` Format Codes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +These methods accept format codes that can be used to parse and format dates:: + + >>> datetime.strptime('31/01/22 23:59:59.999999', + ... '%d/%m/%y %H:%M:%S.%f') + datetime.datetime(2022, 1, 31, 23, 59, 59, 999999) + >>> _.strftime('%a %d %b %Y, %I:%M%p') + 'Mon 31 Jan 2022, 11:59PM' + The following is a list of all the format codes that the 1989 C standard requires, and these work on all platforms with a standard C implementation. From b22d021ee6309f9b0d43e683056db84cfb6fd3c0 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sat, 8 Apr 2023 12:04:47 +0300 Subject: [PATCH 60/76] gh-100176: Tools/iobench: Remove redundant compat code for Python <= 3.2 (#100197) Co-authored-by: Alex Waygood --- Tools/iobench/iobench.py | 96 ++++++++++++++++++++++------------------ 1 file changed, 54 insertions(+), 42 deletions(-) diff --git a/Tools/iobench/iobench.py b/Tools/iobench/iobench.py index b0a7feb92e4f92..4017149ec91630 100644 --- a/Tools/iobench/iobench.py +++ b/Tools/iobench/iobench.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- -# This file should be kept compatible with both Python 2.6 and Python >= 3.0. - import itertools import os import platform @@ -14,39 +11,37 @@ TEXT_ENCODING = 'utf8' NEWLINES = 'lf' -# Compatibility -try: - xrange -except NameError: - xrange = range def text_open(fn, mode, encoding=None): try: return open(fn, mode, encoding=encoding or TEXT_ENCODING) except TypeError: - if 'r' in mode: - mode += 'U' # 'U' mode is needed only in Python 2.x return open(fn, mode) + def get_file_sizes(): for s in ['20 KiB', '400 KiB', '10 MiB']: size, unit = s.split() size = int(size) * {'KiB': 1024, 'MiB': 1024 ** 2}[unit] yield s.replace(' ', ''), size + def get_binary_files(): return ((name + ".bin", size) for name, size in get_file_sizes()) + def get_text_files(): - return (("%s-%s-%s.txt" % (name, TEXT_ENCODING, NEWLINES), size) + return ((f"{name}-{TEXT_ENCODING}-{NEWLINES}.txt", size) for name, size in get_file_sizes()) + def with_open_mode(mode): def decorate(f): f.file_open_mode = mode return f return decorate + def with_sizes(*sizes): def decorate(f): f.file_sizes = sizes @@ -64,6 +59,7 @@ def read_bytewise(f): while f.read(1): pass + @with_open_mode("r") @with_sizes("medium") def read_small_chunks(f): @@ -72,6 +68,7 @@ def read_small_chunks(f): while f.read(20): pass + @with_open_mode("r") @with_sizes("medium") def read_big_chunks(f): @@ -80,6 +77,7 @@ def read_big_chunks(f): while f.read(4096): pass + @with_open_mode("r") @with_sizes("small", "medium", "large") def read_whole_file(f): @@ -88,6 +86,7 @@ def read_whole_file(f): while f.read(): pass + @with_open_mode("rt") @with_sizes("medium") def read_lines(f): @@ -96,6 +95,7 @@ def read_lines(f): for line in f: pass + @with_open_mode("r") @with_sizes("medium") def seek_forward_bytewise(f): @@ -103,9 +103,10 @@ def seek_forward_bytewise(f): f.seek(0, 2) size = f.tell() f.seek(0, 0) - for i in xrange(0, size - 1): + for i in range(0, size - 1): f.seek(i, 0) + @with_open_mode("r") @with_sizes("medium") def seek_forward_blockwise(f): @@ -113,9 +114,10 @@ def seek_forward_blockwise(f): f.seek(0, 2) size = f.tell() f.seek(0, 0) - for i in xrange(0, size - 1, 1000): + for i in range(0, size - 1, 1000): f.seek(i, 0) + @with_open_mode("rb") @with_sizes("medium") def read_seek_bytewise(f): @@ -124,6 +126,7 @@ def read_seek_bytewise(f): while f.read(1): f.seek(1, 1) + @with_open_mode("rb") @with_sizes("medium") def read_seek_blockwise(f): @@ -137,28 +140,31 @@ def read_seek_blockwise(f): @with_sizes("small") def write_bytewise(f, source): """ write one unit at a time """ - for i in xrange(0, len(source)): + for i in range(0, len(source)): f.write(source[i:i+1]) + @with_open_mode("w") @with_sizes("medium") def write_small_chunks(f, source): """ write 20 units at a time """ - for i in xrange(0, len(source), 20): + for i in range(0, len(source), 20): f.write(source[i:i+20]) + @with_open_mode("w") @with_sizes("medium") def write_medium_chunks(f, source): """ write 4096 units at a time """ - for i in xrange(0, len(source), 4096): + for i in range(0, len(source), 4096): f.write(source[i:i+4096]) + @with_open_mode("w") @with_sizes("large") def write_large_chunks(f, source): """ write 1e6 units at a time """ - for i in xrange(0, len(source), 1000000): + for i in range(0, len(source), 1000000): f.write(source[i:i+1000000]) @@ -167,59 +173,65 @@ def write_large_chunks(f, source): def modify_bytewise(f, source): """ modify one unit at a time """ f.seek(0) - for i in xrange(0, len(source)): + for i in range(0, len(source)): f.write(source[i:i+1]) + @with_open_mode("w+") @with_sizes("medium") def modify_small_chunks(f, source): """ modify 20 units at a time """ f.seek(0) - for i in xrange(0, len(source), 20): + for i in range(0, len(source), 20): f.write(source[i:i+20]) + @with_open_mode("w+") @with_sizes("medium") def modify_medium_chunks(f, source): """ modify 4096 units at a time """ f.seek(0) - for i in xrange(0, len(source), 4096): + for i in range(0, len(source), 4096): f.write(source[i:i+4096]) + @with_open_mode("wb+") @with_sizes("medium") def modify_seek_forward_bytewise(f, source): """ alternate write & seek one unit """ f.seek(0) - for i in xrange(0, len(source), 2): + for i in range(0, len(source), 2): f.write(source[i:i+1]) f.seek(i+2) + @with_open_mode("wb+") @with_sizes("medium") def modify_seek_forward_blockwise(f, source): """ alternate write & seek 1000 units """ f.seek(0) - for i in xrange(0, len(source), 2000): + for i in range(0, len(source), 2000): f.write(source[i:i+1000]) f.seek(i+2000) + # XXX the 2 following tests don't work with py3k's text IO @with_open_mode("wb+") @with_sizes("medium") def read_modify_bytewise(f, source): """ alternate read & write one unit """ f.seek(0) - for i in xrange(0, len(source), 2): + for i in range(0, len(source), 2): f.read(1) f.write(source[i+1:i+2]) + @with_open_mode("wb+") @with_sizes("medium") def read_modify_blockwise(f, source): """ alternate read & write 1000 units """ f.seek(0) - for i in xrange(0, len(source), 2000): + for i in range(0, len(source), 2000): f.read(1000) f.write(source[i+1000:i+2000]) @@ -242,6 +254,7 @@ def read_modify_blockwise(f, source): read_modify_bytewise, read_modify_blockwise, ] + def run_during(duration, func): _t = time.time n = 0 @@ -257,6 +270,7 @@ def run_during(duration, func): real = (end[4] if start[4] else time.time()) - real_start return n, real, sum(end[0:2]) - sum(start[0:2]) + def warm_cache(filename): with open(filename, "rb") as f: f.read() @@ -266,9 +280,7 @@ def run_all_tests(options): def print_label(filename, func): name = re.split(r'[-.]', filename)[0] out.write( - ("[%s] %s... " - % (name.center(7), func.__doc__.strip()) - ).ljust(52)) + f"[{name.center(7)}] {func.__doc__.strip()}... ".ljust(52)) out.flush() def print_results(size, n, real, cpu): @@ -276,8 +288,9 @@ def print_results(size, n, real, cpu): bw = ("%4d MiB/s" if bw > 100 else "%.3g MiB/s") % bw out.write(bw.rjust(12) + "\n") if cpu < 0.90 * real: - out.write(" warning: test above used only %d%% CPU, " - "result may be flawed!\n" % (100.0 * cpu / real)) + out.write(" warning: test above used only " + f"{cpu / real:%} CPU, " + "result may be flawed!\n") def run_one_test(name, size, open_func, test_func, *args): mode = test_func.file_open_mode @@ -308,22 +321,15 @@ def run_test_family(tests, mode_filter, files, open_func, *make_args): "large": 2, } - print("Python %s" % sys.version) - if sys.version_info < (3, 3): - if sys.maxunicode > 0xffff: - text = "UCS-4 (wide build)" - else: - text = "UTF-16 (narrow build)" - else: - text = "PEP 393" - print("Unicode: %s" % text) + print(f"Python {sys.version}") + print("Unicode: PEP 393") print(platform.platform()) binary_files = list(get_binary_files()) text_files = list(get_text_files()) if "b" in options: print("Binary unit = one byte") if "t" in options: - print("Text unit = one character (%s-decoded)" % TEXT_ENCODING) + print(f"Text unit = one character ({TEXT_ENCODING}-decoded)") # Binary reads if "b" in options and "r" in options: @@ -338,6 +344,7 @@ def run_test_family(tests, mode_filter, files, open_func, *make_args): # Binary writes if "b" in options and "w" in options: print("\n** Binary append **\n") + def make_test_source(name, size): with open(name, "rb") as f: return f.read() @@ -347,6 +354,7 @@ def make_test_source(name, size): # Text writes if "t" in options and "w" in options: print("\n** Text append **\n") + def make_test_source(name, size): with text_open(name, "r") as f: return f.read() @@ -356,6 +364,7 @@ def make_test_source(name, size): # Binary overwrites if "b" in options and "w" in options: print("\n** Binary overwrite **\n") + def make_test_source(name, size): with open(name, "rb") as f: return f.read() @@ -365,6 +374,7 @@ def make_test_source(name, size): # Text overwrites if "t" in options and "w" in options: print("\n** Text overwrite **\n") + def make_test_source(name, size): with text_open(name, "r") as f: return f.read() @@ -388,7 +398,7 @@ def prepare_files(): break else: raise RuntimeError( - "Couldn't find chunk marker in %s !" % __file__) + f"Couldn't find chunk marker in {__file__} !") if NEWLINES == "all": it = itertools.cycle(["\n", "\r", "\r\n"]) else: @@ -414,6 +424,7 @@ def prepare_files(): f.write(head) f.write(tail) + def main(): global TEXT_ENCODING, NEWLINES @@ -433,7 +444,7 @@ def main(): help="run write & modify tests") parser.add_option("-E", "--encoding", action="store", dest="encoding", default=None, - help="encoding for text tests (default: %s)" % TEXT_ENCODING) + help=f"encoding for text tests (default: {TEXT_ENCODING})") parser.add_option("-N", "--newlines", action="store", dest="newlines", default='lf', help="line endings for text tests " @@ -446,7 +457,7 @@ def main(): parser.error("unexpected arguments") NEWLINES = options.newlines.lower() if NEWLINES not in ('lf', 'cr', 'crlf', 'all'): - parser.error("invalid 'newlines' option: %r" % NEWLINES) + parser.error(f"invalid 'newlines' option: {NEWLINES!r}") test_options = "" if options.read: @@ -471,6 +482,7 @@ def main(): prepare_files() run_all_tests(test_options) + if __name__ == "__main__": main() From 0ba0ca05d2b56afa0b055db02233e703fe138918 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 8 Apr 2023 15:09:00 +0100 Subject: [PATCH 61/76] gh-103373: Improve documentation for `__mro_entries__` (#103374) --- Doc/reference/datamodel.rst | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index a09d5529c8c6f9..ed9ebaba784bc2 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2086,11 +2086,15 @@ When a class definition is executed, the following steps occur: Resolving MRO entries ^^^^^^^^^^^^^^^^^^^^^ -If a base that appears in class definition is not an instance of :class:`type`, -then an ``__mro_entries__`` method is searched on it. If found, it is called -with the original bases tuple. This method must return a tuple of classes that -will be used instead of this base. The tuple may be empty, in such case -the original base is ignored. +.. method:: object.__mro_entries__(self, bases) + + If a base that appears in a class definition is not an instance of + :class:`type`, then an ``__mro_entries__`` method is searched on the base. + If an ``__mro_entries__`` method is found, the base is substituted with the + result of a call to ``__mro_entries__`` when creating the class. + The method is called with the original bases tuple, and must return a tuple + of classes that will be used instead of the base. The returned tuple may be + empty: in these cases, the original base is ignored. .. seealso:: From 35167043e3a21055a94cf3de6ceccd1585554cb8 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sun, 9 Apr 2023 02:56:42 +0900 Subject: [PATCH 62/76] gh-103242: Migrate SSLContext.set_ecdh_curve not to use deprecated APIs (#103378) Migrate `SSLContext.set_ecdh_curve()` not to use deprecated OpenSSL APIs. --- .../2023-04-08-17-13-07.gh-issue-103242.ysI1b3.rst | 2 ++ Modules/_ssl.c | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-08-17-13-07.gh-issue-103242.ysI1b3.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-08-17-13-07.gh-issue-103242.ysI1b3.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-08-17-13-07.gh-issue-103242.ysI1b3.rst new file mode 100644 index 00000000000000..38b107f3be174f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-08-17-13-07.gh-issue-103242.ysI1b3.rst @@ -0,0 +1,2 @@ +Migrate :meth:`~ssl.SSLContext.set_ecdh_curve` method not to use deprecated +OpenSSL APIs. Patch by Dong-hee Na. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 5f17cd502d4598..c9e2f24d66cc00 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -4336,8 +4336,6 @@ _ssl__SSLContext_set_ecdh_curve(PySSLContext *self, PyObject *name) { PyObject *name_bytes; int nid; - EC_KEY *key; - if (!PyUnicode_FSConverter(name, &name_bytes)) return NULL; assert(PyBytes_Check(name_bytes)); @@ -4348,13 +4346,20 @@ _ssl__SSLContext_set_ecdh_curve(PySSLContext *self, PyObject *name) "unknown elliptic curve name %R", name); return NULL; } - key = EC_KEY_new_by_curve_name(nid); +#if OPENSSL_VERSION_MAJOR < 3 + EC_KEY *key = EC_KEY_new_by_curve_name(nid); if (key == NULL) { _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); return NULL; } SSL_CTX_set_tmp_ecdh(self->ctx, key); EC_KEY_free(key); +#else + if (!SSL_CTX_set1_groups(self->ctx, &nid, 1)) { + _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); + return NULL; + } +#endif Py_RETURN_NONE; } From f329a8bc1e57e454852f8887df6267b42047cd1b Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Sun, 9 Apr 2023 03:03:52 +0200 Subject: [PATCH 63/76] gh-103092: Isolate `socket` module (#103094) --- Lib/test/test_socket.py | 7 + ...-04-03-23-43-12.gh-issue-103092.3xqk4y.rst | 1 + Modules/socketmodule.c | 394 +++++++++++------- Modules/socketmodule.h | 1 + Tools/c-analyzer/cpython/globals-to-fix.tsv | 6 - 5 files changed, 242 insertions(+), 167 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-03-23-43-12.gh-issue-103092.3xqk4y.rst diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 32252f7b741fda..bb7bf436d2d721 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -8,6 +8,7 @@ import array import contextlib import errno +import gc import io import itertools import math @@ -836,6 +837,12 @@ def requireSocket(*args): class GeneralModuleTests(unittest.TestCase): + @unittest.skipUnless(_socket is not None, 'need _socket module') + def test_socket_type(self): + self.assertTrue(gc.is_tracked(_socket.socket)) + with self.assertRaisesRegex(TypeError, "immutable"): + _socket.socket.foo = 1 + def test_SocketType_is_socketobject(self): import _socket self.assertTrue(socket.SocketType is _socket.socket) diff --git a/Misc/NEWS.d/next/Library/2023-04-03-23-43-12.gh-issue-103092.3xqk4y.rst b/Misc/NEWS.d/next/Library/2023-04-03-23-43-12.gh-issue-103092.3xqk4y.rst new file mode 100644 index 00000000000000..e7586a223c1415 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-03-23-43-12.gh-issue-103092.3xqk4y.rst @@ -0,0 +1 @@ +Isolate :mod:`!_socket` (apply :pep:`687`). Patch by Erlend E. Aasland. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 49342b3d48de0e..656cd546d46d31 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -108,6 +108,7 @@ Local naming conventions: #define PY_SSIZE_T_CLEAN #include "Python.h" #include "pycore_fileutils.h" // _Py_set_inheritable() +#include "pycore_moduleobject.h" // _PyModule_GetState #include "structmember.h" // PyMemberDef #ifdef _Py_MEMORY_SANITIZER @@ -337,9 +338,9 @@ static FlagRuntimeInfo win_runtime_flags[] = { /*[clinic input] module _socket -class _socket.socket "PySocketSockObject *" "&sock_type" +class _socket.socket "PySocketSockObject *" "clinic_state()->sock_type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7a8313d9b7f51988]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=2db2489bd2219fd8]*/ static int remove_unusable_flags(PyObject *m) @@ -541,22 +542,59 @@ remove_unusable_flags(PyObject *m) #define INADDR_NONE (-1) #endif +typedef struct _socket_state { + /* The sock_type variable contains pointers to various functions, + some of which call new_sockobject(), which uses sock_type, so + there has to be a circular reference. */ + PyTypeObject *sock_type; + + /* Global variable holding the exception type for errors detected + by this module (but not argument type or memory errors, etc.). */ + PyObject *socket_herror; + PyObject *socket_gaierror; + + /* Default timeout for new sockets */ + _PyTime_t defaulttimeout; + +#if defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4) +#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) + /* accept4() is available on Linux 2.6.28+ and glibc 2.10 */ + int accept4_works; +#endif +#endif + +#ifdef SOCK_CLOEXEC + /* socket() and socketpair() fail with EINVAL on Linux kernel older + * than 2.6.27 if SOCK_CLOEXEC flag is set in the socket type. */ + int sock_cloexec_works; +#endif +} socket_state; + +static inline socket_state * +get_module_state(PyObject *mod) +{ + void *state = _PyModule_GetState(mod); + assert(state != NULL); + return (socket_state *)state; +} + +static struct PyModuleDef socketmodule; + +static inline socket_state * +find_module_state_by_def(PyTypeObject *type) +{ + PyObject *mod = PyType_GetModuleByDef(type, &socketmodule); + assert(mod != NULL); + return get_module_state(mod); +} + +#define clinic_state() (find_module_state_by_def(type)) #include "clinic/socketmodule.c.h" +#undef clinic_state /* XXX There's a problem here: *static* functions are not supposed to have a Py prefix (or use CapitalizedWords). Later... */ -/* Global variable holding the exception type for errors detected - by this module (but not argument type or memory errors, etc.). */ -static PyObject *socket_herror; -static PyObject *socket_gaierror; - -/* A forward reference to the socket type object. - The sock_type variable contains pointers to various functions, - some of which call new_sockobject(), which uses sock_type, so - there has to be a circular reference. */ -static PyTypeObject sock_type; - #if defined(HAVE_POLL_H) #include #elif defined(HAVE_SYS_POLL_H) @@ -641,7 +679,7 @@ set_error(void) #if defined(HAVE_GETHOSTBYNAME_R) || defined (HAVE_GETHOSTBYNAME) || defined (HAVE_GETHOSTBYADDR) static PyObject * -set_herror(int h_error) +set_herror(socket_state *state, int h_error) { PyObject *v; @@ -651,7 +689,7 @@ set_herror(int h_error) v = Py_BuildValue("(is)", h_error, "host not found"); #endif if (v != NULL) { - PyErr_SetObject(socket_herror, v); + PyErr_SetObject(state->socket_herror, v); Py_DECREF(v); } @@ -662,7 +700,7 @@ set_herror(int h_error) #ifdef HAVE_GETADDRINFO static PyObject * -set_gaierror(int error) +set_gaierror(socket_state *state, int error) { PyObject *v; @@ -678,7 +716,7 @@ set_gaierror(int error) v = Py_BuildValue("(is)", error, "getaddrinfo failed"); #endif if (v != NULL) { - PyErr_SetObject(socket_gaierror, v); + PyErr_SetObject(state->socket_gaierror, v); Py_DECREF(v); } @@ -991,11 +1029,8 @@ sock_call(PySocketSockObject *s, /* Initialize a new socket object. */ -/* Default timeout for new sockets */ -static _PyTime_t defaulttimeout = _PYTIME_FROMSECONDS(-1); - static int -init_sockobject(PySocketSockObject *s, +init_sockobject(socket_state *state, PySocketSockObject *s, SOCKET_T fd, int family, int type, int proto) { s->sock_fd = fd; @@ -1025,13 +1060,14 @@ init_sockobject(PySocketSockObject *s, else #endif { - s->sock_timeout = defaulttimeout; - if (defaulttimeout >= 0) { + s->sock_timeout = state->defaulttimeout; + if (state->defaulttimeout >= 0) { if (internal_setblocking(s, 0) == -1) { return -1; } } } + s->state = state; return 0; } @@ -1043,14 +1079,15 @@ init_sockobject(PySocketSockObject *s, in NEWOBJ()). */ static PySocketSockObject * -new_sockobject(SOCKET_T fd, int family, int type, int proto) +new_sockobject(socket_state *state, SOCKET_T fd, int family, int type, + int proto) { - PySocketSockObject *s; - s = (PySocketSockObject *) - PyType_GenericNew(&sock_type, NULL, NULL); - if (s == NULL) + PyTypeObject *tp = state->sock_type; + PySocketSockObject *s = (PySocketSockObject *)tp->tp_alloc(tp, 0); + if (s == NULL) { return NULL; - if (init_sockobject(s, fd, family, type, proto) == -1) { + } + if (init_sockobject(state, s, fd, family, type, proto) == -1) { Py_DECREF(s); return NULL; } @@ -1074,7 +1111,8 @@ static PyThread_type_lock netdb_lock; an error occurred; then an exception is raised. */ static int -setipaddr(const char *name, struct sockaddr *addr_ret, size_t addr_ret_size, int af) +setipaddr(socket_state *state, const char *name, struct sockaddr *addr_ret, + size_t addr_ret_size, int af) { struct addrinfo hints, *res; int error; @@ -1095,7 +1133,7 @@ setipaddr(const char *name, struct sockaddr *addr_ret, size_t addr_ret_size, int outcome of the first call. */ if (error) { res = NULL; // no-op, remind us that it is invalid; gh-100795 - set_gaierror(error); + set_gaierror(state, error); return -1; } switch (res->ai_family) { @@ -1206,7 +1244,7 @@ setipaddr(const char *name, struct sockaddr *addr_ret, size_t addr_ret_size, int Py_END_ALLOW_THREADS if (error) { res = NULL; // no-op, remind us that it is invalid; gh-100795 - set_gaierror(error); + set_gaierror(state, error); return -1; } if (res->ai_addrlen < addr_ret_size) @@ -1889,7 +1927,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, return 0; } struct sockaddr_in* addr = &addrbuf->in; - result = setipaddr(host.buf, (struct sockaddr *)addr, + result = setipaddr(s->state, host.buf, (struct sockaddr *)addr, sizeof(*addr), AF_INET); idna_cleanup(&host); if (result < 0) @@ -1934,7 +1972,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, return 0; } struct sockaddr_in6* addr = &addrbuf->in6; - result = setipaddr(host.buf, (struct sockaddr *)addr, + result = setipaddr(s->state, host.buf, (struct sockaddr *)addr, sizeof(*addr), AF_INET6); idna_cleanup(&host); if (result < 0) @@ -2813,10 +2851,6 @@ struct sock_accept { }; #if defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4) -#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) -/* accept4() is available on Linux 2.6.28+ and glibc 2.10 */ -static int accept4_works = -1; -#endif static int sock_accept_impl(PySocketSockObject *s, void *data) @@ -2835,15 +2869,16 @@ sock_accept_impl(PySocketSockObject *s, void *data) #endif #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) - if (accept4_works != 0) { + socket_state *state = s->state; + if (state->accept4_works != 0) { ctx->result = accept4(s->sock_fd, addr, paddrlen, SOCK_CLOEXEC); - if (ctx->result == INVALID_SOCKET && accept4_works == -1) { + if (ctx->result == INVALID_SOCKET && state->accept4_works == -1) { /* On Linux older than 2.6.28, accept4() fails with ENOSYS */ - accept4_works = (errno != ENOSYS); + state->accept4_works = (errno != ENOSYS); } } - if (accept4_works == 0) + if (state->accept4_works == 0) ctx->result = accept(s->sock_fd, addr, paddrlen); #else ctx->result = accept(s->sock_fd, addr, paddrlen); @@ -2896,7 +2931,8 @@ sock_accept(PySocketSockObject *s, PyObject *Py_UNUSED(ignored)) #else #if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) - if (!accept4_works) + socket_state *state = s->state; + if (!state->accept4_works) #endif { if (_Py_set_inheritable(newfd, 0, NULL) < 0) { @@ -5219,13 +5255,23 @@ sock_finalize(PySocketSockObject *s) PyErr_SetRaisedException(exc); } +static int +sock_traverse(PySocketSockObject *s, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(s)); + return 0; +} + static void sock_dealloc(PySocketSockObject *s) { - if (PyObject_CallFinalizerFromDealloc((PyObject *)s) < 0) + if (PyObject_CallFinalizerFromDealloc((PyObject *)s) < 0) { return; - - Py_TYPE(s)->tp_free((PyObject *)s); + } + PyTypeObject *tp = Py_TYPE(s); + PyObject_GC_UnTrack(s); + tp->tp_free((PyObject *)s); + Py_DECREF(tp); } @@ -5277,12 +5323,6 @@ sock_new(PyTypeObject *type, PyObject *args, PyObject *kwds) /* Initialize a new socket object. */ -#ifdef SOCK_CLOEXEC -/* socket() and socketpair() fail with EINVAL on Linux kernel older - * than 2.6.27 if SOCK_CLOEXEC flag is set in the socket type. */ -static int sock_cloexec_works = -1; -#endif - /*ARGSUSED*/ #ifndef HAVE_SOCKET @@ -5310,10 +5350,11 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, { SOCKET_T fd = INVALID_SOCKET; + socket_state *state = find_module_state_by_def(Py_TYPE(self)); #ifndef MS_WINDOWS #ifdef SOCK_CLOEXEC - int *atomic_flag_works = &sock_cloexec_works; + int *atomic_flag_works = &state->sock_cloexec_works; #else int *atomic_flag_works = NULL; #endif @@ -5468,15 +5509,15 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, /* UNIX */ Py_BEGIN_ALLOW_THREADS #ifdef SOCK_CLOEXEC - if (sock_cloexec_works != 0) { + if (state->sock_cloexec_works != 0) { fd = socket(family, type | SOCK_CLOEXEC, proto); - if (sock_cloexec_works == -1) { + if (state->sock_cloexec_works == -1) { if (fd >= 0) { - sock_cloexec_works = 1; + state->sock_cloexec_works = 1; } else if (errno == EINVAL) { /* Linux older than 2.6.27 does not support SOCK_CLOEXEC */ - sock_cloexec_works = 0; + state->sock_cloexec_works = 0; fd = socket(family, type, proto); } } @@ -5499,7 +5540,7 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, } #endif } - if (init_sockobject(self, fd, family, type, proto) == -1) { + if (init_sockobject(state, self, fd, family, type, proto) == -1) { SOCKETCLOSE(fd); return -1; } @@ -5511,55 +5552,26 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, /* Type object for socket objects. */ -static PyTypeObject sock_type = { - PyVarObject_HEAD_INIT(0, 0) /* Must fill in type value later */ - "_socket.socket", /* tp_name */ - sizeof(PySocketSockObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)sock_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)sock_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - sock_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - sock_methods, /* tp_methods */ - sock_memberlist, /* tp_members */ - sock_getsetlist, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - sock_initobj, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - sock_new, /* tp_new */ - PyObject_Del, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - (destructor)sock_finalize, /* tp_finalize */ +static PyType_Slot sock_slots[] = { + {Py_tp_dealloc, sock_dealloc}, + {Py_tp_traverse, sock_traverse}, + {Py_tp_repr, sock_repr}, + {Py_tp_doc, (void *)sock_doc}, + {Py_tp_methods, sock_methods}, + {Py_tp_members, sock_memberlist}, + {Py_tp_getset, sock_getsetlist}, + {Py_tp_init, sock_initobj}, + {Py_tp_new, sock_new}, + {Py_tp_finalize, sock_finalize}, + {0, NULL}, +}; + +static PyType_Spec sock_spec = { + .name = "_socket.socket", + .basicsize = sizeof(PySocketSockObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | + Py_TPFLAGS_IMMUTABLETYPE), + .slots = sock_slots, }; @@ -5687,8 +5699,12 @@ socket_gethostbyname(PyObject *self, PyObject *args) if (PySys_Audit("socket.gethostbyname", "O", args) < 0) { goto finally; } - if (setipaddr(name, (struct sockaddr *)&addrbuf, sizeof(addrbuf), AF_INET) < 0) + socket_state *state = get_module_state(self); + int rc = setipaddr(state, name, (struct sockaddr *)&addrbuf, + sizeof(addrbuf), AF_INET); + if (rc < 0) { goto finally; + } ret = make_ipv4_addr(&addrbuf); finally: PyMem_Free(name); @@ -5719,7 +5735,8 @@ sock_decode_hostname(const char *name) /* Convenience function common to gethostbyname_ex and gethostbyaddr */ static PyObject * -gethost_common(struct hostent *h, struct sockaddr *addr, size_t alen, int af) +gethost_common(socket_state *state, struct hostent *h, struct sockaddr *addr, + size_t alen, int af) { char **pch; PyObject *rtn_tuple = (PyObject *)NULL; @@ -5730,7 +5747,7 @@ gethost_common(struct hostent *h, struct sockaddr *addr, size_t alen, int af) if (h == NULL) { /* Let's get real error message to return */ - set_herror(h_errno); + set_herror(state, h_errno); return NULL; } @@ -5877,8 +5894,10 @@ socket_gethostbyname_ex(PyObject *self, PyObject *args) if (PySys_Audit("socket.gethostbyname", "O", args) < 0) { goto finally; } - if (setipaddr(name, SAS2SA(&addr), sizeof(addr), AF_INET) < 0) + socket_state *state = get_module_state(self); + if (setipaddr(state, name, SAS2SA(&addr), sizeof(addr), AF_INET) < 0) { goto finally; + } Py_BEGIN_ALLOW_THREADS #ifdef HAVE_GETHOSTBYNAME_R #if defined(HAVE_GETHOSTBYNAME_R_6_ARG) @@ -5904,7 +5923,7 @@ socket_gethostbyname_ex(PyObject *self, PyObject *args) Therefore, we cast the sockaddr_storage into sockaddr to access sa_family. */ sa = SAS2SA(&addr); - ret = gethost_common(h, SAS2SA(&addr), sizeof(addr), + ret = gethost_common(state, h, SAS2SA(&addr), sizeof(addr), sa->sa_family); #ifdef USE_GETHOSTBYNAME_LOCK PyThread_release_lock(netdb_lock); @@ -5960,8 +5979,10 @@ socket_gethostbyaddr(PyObject *self, PyObject *args) goto finally; } af = AF_UNSPEC; - if (setipaddr(ip_num, sa, sizeof(addr), af) < 0) + socket_state *state = get_module_state(self); + if (setipaddr(state, ip_num, sa, sizeof(addr), af) < 0) { goto finally; + } af = sa->sa_family; ap = NULL; /* al = 0; */ @@ -6002,7 +6023,7 @@ socket_gethostbyaddr(PyObject *self, PyObject *args) h = gethostbyaddr(ap, al, af); #endif /* HAVE_GETHOSTBYNAME_R */ Py_END_ALLOW_THREADS - ret = gethost_common(h, SAS2SA(&addr), sizeof(addr), af); + ret = gethost_common(state, h, SAS2SA(&addr), sizeof(addr), af); #ifdef USE_GETHOSTBYNAME_LOCK PyThread_release_lock(netdb_lock); #endif @@ -6221,8 +6242,9 @@ socket_socketpair(PyObject *self, PyObject *args) SOCKET_T sv[2]; int family, type = SOCK_STREAM, proto = 0; PyObject *res = NULL; + socket_state *state = get_module_state(self); #ifdef SOCK_CLOEXEC - int *atomic_flag_works = &sock_cloexec_works; + int *atomic_flag_works = &state->sock_cloexec_works; #else int *atomic_flag_works = NULL; #endif @@ -6240,15 +6262,15 @@ socket_socketpair(PyObject *self, PyObject *args) /* Create a pair of socket fds */ Py_BEGIN_ALLOW_THREADS #ifdef SOCK_CLOEXEC - if (sock_cloexec_works != 0) { + if (state->sock_cloexec_works != 0) { ret = socketpair(family, type | SOCK_CLOEXEC, proto, sv); - if (sock_cloexec_works == -1) { + if (state->sock_cloexec_works == -1) { if (ret >= 0) { - sock_cloexec_works = 1; + state->sock_cloexec_works = 1; } else if (errno == EINVAL) { /* Linux older than 2.6.27 does not support SOCK_CLOEXEC */ - sock_cloexec_works = 0; + state->sock_cloexec_works = 0; ret = socketpair(family, type, proto, sv); } } @@ -6268,10 +6290,10 @@ socket_socketpair(PyObject *self, PyObject *args) if (_Py_set_inheritable(sv[1], 0, atomic_flag_works) < 0) goto finally; - s0 = new_sockobject(sv[0], family, type, proto); + s0 = new_sockobject(state, sv[0], family, type, proto); if (s0 == NULL) goto finally; - s1 = new_sockobject(sv[1], family, type, proto); + s1 = new_sockobject(state, sv[1], family, type, proto); if (s1 == NULL) goto finally; res = PyTuple_Pack(2, s0, s1); @@ -6726,7 +6748,8 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) Py_END_ALLOW_THREADS if (error) { res0 = NULL; // gh-100795 - set_gaierror(error); + socket_state *state = get_module_state(self); + set_gaierror(state, error); goto err; } @@ -6825,7 +6848,8 @@ socket_getnameinfo(PyObject *self, PyObject *args) Py_END_ALLOW_THREADS if (error) { res = NULL; // gh-100795 - set_gaierror(error); + socket_state *state = get_module_state(self); + set_gaierror(state, error); goto fail; } if (res->ai_next) { @@ -6857,7 +6881,8 @@ socket_getnameinfo(PyObject *self, PyObject *args) error = getnameinfo(res->ai_addr, (socklen_t) res->ai_addrlen, hbuf, sizeof(hbuf), pbuf, sizeof(pbuf), flags); if (error) { - set_gaierror(error); + socket_state *state = get_module_state(self); + set_gaierror(state, error); goto fail; } @@ -6883,11 +6908,12 @@ Get host and port for a sockaddr."); static PyObject * socket_getdefaulttimeout(PyObject *self, PyObject *Py_UNUSED(ignored)) { - if (defaulttimeout < 0) { + socket_state *state = get_module_state(self); + if (state->defaulttimeout < 0) { Py_RETURN_NONE; } else { - double seconds = _PyTime_AsSecondsDouble(defaulttimeout); + double seconds = _PyTime_AsSecondsDouble(state->defaulttimeout); return PyFloat_FromDouble(seconds); } } @@ -6907,7 +6933,8 @@ socket_setdefaulttimeout(PyObject *self, PyObject *arg) if (socket_parse_timeout(&timeout, arg) < 0) return NULL; - defaulttimeout = timeout; + socket_state *state = get_module_state(self); + state->defaulttimeout = timeout; Py_RETURN_NONE; } @@ -7293,7 +7320,7 @@ sock_destroy_api(PyObject *capsule) } static PySocketModule_APIObject * -sock_get_api(void) +sock_get_api(socket_state *state) { PySocketModule_APIObject *capi = PyMem_Malloc(sizeof(PySocketModule_APIObject)); if (capi == NULL) { @@ -7301,7 +7328,7 @@ sock_get_api(void) return NULL; } - capi->Sock_Type = (PyTypeObject *)Py_NewRef(&sock_type); + capi->Sock_Type = (PyTypeObject *)Py_NewRef(state->sock_type); capi->error = Py_NewRef(PyExc_OSError); capi->timeout_error = Py_NewRef(PyExc_TimeoutError); return capi; @@ -7323,47 +7350,38 @@ PyDoc_STRVAR(socket_doc, \n\ See the socket module for documentation."); -static struct PyModuleDef socketmodule = { - PyModuleDef_HEAD_INIT, - PySocket_MODULE_NAME, - socket_doc, - -1, - socket_methods, - NULL, - NULL, - NULL, - NULL -}; - -PyMODINIT_FUNC -PyInit__socket(void) +static int +socket_exec(PyObject *m) { - PyObject *m = NULL; - if (!os_init()) { goto error; } - Py_SET_TYPE(&sock_type, &PyType_Type); - m = PyModule_Create(&socketmodule); - if (m == NULL) { - goto error; - } + socket_state *state = get_module_state(m); + state->defaulttimeout = _PYTIME_FROMSECONDS(-1); + +#if defined(HAVE_ACCEPT) || defined(HAVE_ACCEPT4) +#if defined(HAVE_ACCEPT4) && defined(SOCK_CLOEXEC) + state->accept4_works = -1; +#endif +#endif + +#ifdef SOCK_CLOEXEC + state->sock_cloexec_works = -1; +#endif #define ADD_EXC(MOD, NAME, VAR, BASE) do { \ VAR = PyErr_NewException("socket." NAME, BASE, NULL); \ if (VAR == NULL) { \ goto error; \ } \ - int rc = PyModule_AddObjectRef(MOD, NAME, VAR); \ - Py_DECREF(VAR); \ - if (rc < 0) { \ + if (PyModule_AddObjectRef(MOD, NAME, VAR) < 0) { \ goto error; \ } \ } while (0) - ADD_EXC(m, "herror", socket_herror, PyExc_OSError); - ADD_EXC(m, "gaierror", socket_gaierror, PyExc_OSError); + ADD_EXC(m, "herror", state->socket_herror, PyExc_OSError); + ADD_EXC(m, "gaierror", state->socket_gaierror, PyExc_OSError); #undef ADD_EXC @@ -7373,10 +7391,16 @@ PyInit__socket(void) if (PyModule_AddObjectRef(m, "timeout", PyExc_TimeoutError) < 0) { goto error; } - if (PyModule_AddObjectRef(m, "SocketType", (PyObject *)&sock_type) < 0) { + + PyObject *sock_type = PyType_FromMetaclass(NULL, m, &sock_spec, NULL); + if (sock_type == NULL) { + goto error; + } + state->sock_type = (PyTypeObject *)sock_type; + if (PyModule_AddObjectRef(m, "SocketType", sock_type) < 0) { goto error; } - if (PyModule_AddType(m, &sock_type) < 0) { + if (PyModule_AddType(m, state->sock_type) < 0) { goto error; } @@ -7391,7 +7415,7 @@ PyInit__socket(void) } /* Export C API */ - PySocketModule_APIObject *capi = sock_get_api(); + PySocketModule_APIObject *capi = sock_get_api(state); if (capi == NULL) { goto error; } @@ -8813,9 +8837,57 @@ PyInit__socket(void) #undef ADD_INT_CONST #undef ADD_STR_CONST - return m; + return 0; error: - Py_XDECREF(m); - return NULL; + return -1; +} + +static struct PyModuleDef_Slot socket_slots[] = { + {Py_mod_exec, socket_exec}, + {0, NULL}, +}; + +static int +socket_traverse(PyObject *mod, visitproc visit, void *arg) +{ + socket_state *state = get_module_state(mod); + Py_VISIT(state->sock_type); + Py_VISIT(state->socket_herror); + Py_VISIT(state->socket_gaierror); + return 0; +} + +static int +socket_clear(PyObject *mod) +{ + socket_state *state = get_module_state(mod); + Py_CLEAR(state->sock_type); + Py_CLEAR(state->socket_herror); + Py_CLEAR(state->socket_gaierror); + return 0; +} + +static void +socket_free(void *mod) +{ + (void)socket_clear((PyObject *)mod); +} + +static struct PyModuleDef socketmodule = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = PySocket_MODULE_NAME, + .m_doc = socket_doc, + .m_size = sizeof(socket_state), + .m_methods = socket_methods, + .m_slots = socket_slots, + .m_traverse = socket_traverse, + .m_clear = socket_clear, + .m_free = socket_free, +}; + +PyMODINIT_FUNC +PyInit__socket(void) +{ + return PyModuleDef_Init(&socketmodule); } diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index f31ba532a6c60d..f5ca00450ee92a 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -322,6 +322,7 @@ typedef struct { sets a Python exception */ _PyTime_t sock_timeout; /* Operation timeout in seconds; 0.0 means non-blocking */ + struct _socket_state *state; } PySocketSockObject; /* --- C API ----------------------------------------------------*/ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 0620c7e13925b5..e36024ad16c271 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -392,7 +392,6 @@ Modules/_decimal/_decimal.c - PyDecSignalDictMixin_Type - Modules/_decimal/_decimal.c - PyDec_Type - Modules/ossaudiodev.c - OSSAudioType - Modules/ossaudiodev.c - OSSMixerType - -Modules/socketmodule.c - sock_type - Modules/xxmodule.c - Null_Type - Modules/xxmodule.c - Str_Type - Modules/xxmodule.c - Xxo_Type - @@ -416,8 +415,6 @@ Modules/_cursesmodule.c - PyCursesError - Modules/_decimal/_decimal.c - DecimalException - Modules/_tkinter.c - Tkinter_TclError - Modules/ossaudiodev.c - OSSAudioError - -Modules/socketmodule.c - socket_herror - -Modules/socketmodule.c - socket_gaierror - Modules/xxlimited_35.c - ErrorObject - Modules/xxmodule.c - ErrorObject - @@ -514,8 +511,6 @@ Modules/cjkcodecs/cjkcodecs.h - mapping_list - Modules/readline.c - libedit_append_replace_history_offset - Modules/readline.c - using_libedit_emulation - Modules/readline.c - libedit_history_start - -Modules/socketmodule.c - accept4_works - -Modules/socketmodule.c - sock_cloexec_works - ##----------------------- ## state @@ -541,4 +536,3 @@ Modules/readline.c - sigwinch_ohandler - Modules/readline.c - completed_input_string - Modules/rotatingtree.c - random_stream - Modules/rotatingtree.c - random_value - -Modules/socketmodule.c - defaulttimeout - From 63dc969ec6129704c3b23b384ecfa8e485d6c4cb Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Sun, 9 Apr 2023 07:29:08 +0530 Subject: [PATCH 64/76] Document `asyncio` performance improvement in What's New (#103370) Co-authored-by: Alex Waygood --- Doc/whatsnew/3.12.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 66e40ef7326cc0..651caed864fef7 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -210,6 +210,11 @@ array asyncio ------- +* The performance of writing to sockets in :mod:`asyncio` has been + significantly improved. ``asyncio`` now avoids unnecessary copying when + writing to sockets and uses :meth:`~socket.socket.sendmsg` if the platform + supports it. (Contributed by Kumar Aditya in :gh:`91166`.) + * On Linux, :mod:`asyncio` uses :class:`~asyncio.PidfdChildWatcher` by default if :func:`os.pidfd_open` is available and functional instead of :class:`~asyncio.ThreadedChildWatcher`. From 83af8f268629139b9628ff53617671469167146a Mon Sep 17 00:00:00 2001 From: yuki Date: Sun, 9 Apr 2023 11:02:16 +0900 Subject: [PATCH 65/76] Docs: Fix broken reference `__getitem__` in `string.rst` (#103371) --- Doc/library/string.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 5ada827328188d..f55074cc582718 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -235,7 +235,7 @@ dictionary keys (e.g., the strings ``'10'`` or ``':-]'``) within a format string The *arg_name* can be followed by any number of index or attribute expressions. An expression of the form ``'.name'`` selects the named attribute using :func:`getattr`, while an expression of the form ``'[index]'`` -does an index lookup using :func:`__getitem__`. +does an index lookup using :meth:`~object.__getitem__`. .. versionchanged:: 3.1 The positional argument specifiers can be omitted for :meth:`str.format`, From 264b87f7fd0634485b9b4d21df49bec3401d7b2a Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Sun, 9 Apr 2023 03:05:50 +0100 Subject: [PATCH 66/76] gh-102799: use `sys.exception()` instead of `sys.exc_info()` in pdb (#103294) --- Lib/pdb.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Lib/pdb.py b/Lib/pdb.py index e043b0d46f7dd0..e03142e9b5d85c 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1258,7 +1258,7 @@ def _getval_except(self, arg, frame=None): return _rstr('** raised %s **' % self._format_exc(exc)) def _error_exc(self): - exc = sys.exc_info()[1] + exc = sys.exception() self.error(self._format_exc(exc)) def _msg_val_func(self, arg, func): @@ -1755,9 +1755,10 @@ def post_mortem(t=None): """ # handling the default if t is None: - # sys.exc_info() returns (type, value, traceback) if an exception is - # being handled, otherwise it returns None - t = sys.exc_info()[2] + exc = sys.exception() + if exc is not None: + t = exc.__traceback__ + if t is None: raise ValueError("A valid traceback must be passed if no " "exception is being handled") @@ -1841,18 +1842,18 @@ def main(): except Restart: print("Restarting", target, "with arguments:") print("\t" + " ".join(sys.argv[1:])) - except SystemExit: + except SystemExit as e: # In most cases SystemExit does not warrant a post-mortem session. print("The program exited via sys.exit(). Exit status:", end=' ') - print(sys.exc_info()[1]) + print(e) except SyntaxError: traceback.print_exc() sys.exit(1) - except: + except BaseException as e: traceback.print_exc() print("Uncaught exception. Entering post mortem debugging") print("Running 'cont' or 'step' will restart the program") - t = sys.exc_info()[2] + t = e.__traceback__ pdb.interaction(None, t) print("Post mortem debugger finished. The " + target + " will be restarted") From 090e26ea807aa414d6a6a01d9365b0288c10a5db Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 9 Apr 2023 09:39:03 +0300 Subject: [PATCH 67/76] CI: Do not allow merge if labelled DO-NOT-MERGE (#103337) Co-authored-by: C.A.M. Gerlach --- .github/workflows/require-pr-label.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/require-pr-label.yml diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml new file mode 100644 index 00000000000000..e847bae155e216 --- /dev/null +++ b/.github/workflows/require-pr-label.yml @@ -0,0 +1,17 @@ +name: Check labels + +on: + pull_request: + types: [opened, reopened, labeled, unlabeled, synchronize] + +jobs: + label: + name: DO-NOT-MERGE + runs-on: ubuntu-latest + + steps: + - uses: mheap/github-action-required-labels@v4 + with: + mode: exactly + count: 0 + labels: "DO-NOT-MERGE" From c330b4a3e7b34308ad97d51de9ab8f4e51a0805c Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sun, 9 Apr 2023 10:43:01 +0300 Subject: [PATCH 68/76] Add hugovk as a .github code owner (GH-103394) Automerge-Triggered-By: GH:hugovk --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fc1bb3388976d5..60af0519cdb743 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,7 +5,7 @@ # https://git-scm.com/docs/gitignore#_pattern_format # GitHub -.github/** @ezio-melotti +.github/** @ezio-melotti @hugovk # Build system configure* @erlend-aasland @corona10 From 5d4afc45b9567d227ea9a9988d58404662fce58c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 9 Apr 2023 13:42:43 +0530 Subject: [PATCH 69/76] build(deps): bump actions/stale from 7 to 8 (#103169) Bumps [actions/stale](https://github.com/actions/stale) from 7 to 8. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v7...v8) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 07dbcfe31d6563..d79e856c87e78d 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -15,7 +15,7 @@ jobs: steps: - name: "Check PRs" - uses: actions/stale@v7 + uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' From d9305f8e9d3e0f6267286c2da4b24e97b7a569f2 Mon Sep 17 00:00:00 2001 From: JakobDev Date: Sun, 9 Apr 2023 10:13:40 +0200 Subject: [PATCH 70/76] Fix typos in test_tempfile.py (#102841) --- Lib/test/test_tempfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 90155487cff585..11a43aca17e88a 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1016,7 +1016,7 @@ def use_closed(): self.assertRaises(ValueError, use_closed) def test_context_man_not_del_on_close_if_delete_on_close_false(self): - # Issue gh-58451: tempfile.NamedTemporaryFile is not particulary useful + # Issue gh-58451: tempfile.NamedTemporaryFile is not particularly useful # on Windows # A NamedTemporaryFile is NOT deleted when closed if # delete_on_close=False, but is deleted on context manager exit @@ -1608,7 +1608,7 @@ def test_explicit_cleanup(self): finally: os.rmdir(dir) - def test_explict_cleanup_ignore_errors(self): + def test_explicit_cleanup_ignore_errors(self): """Test that cleanup doesn't return an error when ignoring them.""" with tempfile.TemporaryDirectory() as working_dir: temp_dir = self.do_create( From 86d20441557bedbea3dadd5d0818a492148335bd Mon Sep 17 00:00:00 2001 From: Oleg Iarygin Date: Sun, 9 Apr 2023 12:18:53 +0400 Subject: [PATCH 71/76] gh-103300: Fix `Popen.wait()` deadlock in patchcheck.py (#103301) --- Tools/patchcheck/patchcheck.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/patchcheck/patchcheck.py b/Tools/patchcheck/patchcheck.py index 6dcf612066199c..44a6fb8c660cd2 100755 --- a/Tools/patchcheck/patchcheck.py +++ b/Tools/patchcheck/patchcheck.py @@ -130,9 +130,10 @@ def changed_files(base_branch=None): with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, cwd=SRCDIR) as st: - if st.wait() != 0: + git_file_status, _ = st.communicate() + if st.returncode != 0: sys.exit(f'error running {cmd}') - for line in st.stdout: + for line in git_file_status.splitlines(): line = line.decode().rstrip() status_text, filename = line.split(maxsplit=1) status = set(status_text) From 975d220bbed0e7a15b62f1d2d03557740a55f68d Mon Sep 17 00:00:00 2001 From: mara004 Date: Sun, 9 Apr 2023 10:26:52 +0200 Subject: [PATCH 72/76] ctypes docs: fix missing `not` in variadic functions section (#102611) --- Doc/library/ctypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index d49d702e9e7999..81509c0920bb6e 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -390,7 +390,7 @@ regular, non-variadic, function arguments: libc.printf.argtypes = [ctypes.c_char_p] -Because specifying the attribute does inhibit portability it is advised to always +Because specifying the attribute does not inhibit portability it is advised to always specify ``argtypes`` for all variadic functions. From 45b4b37fc26f288fabb27dc08b253121697c517b Mon Sep 17 00:00:00 2001 From: Stepfen Shawn Date: Sun, 9 Apr 2023 16:43:21 +0800 Subject: [PATCH 73/76] Remove useless symbol in pystats.h (#101864) --- Include/pystats.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/Include/pystats.h b/Include/pystats.h index 25ed4bddc7240c..4b961bad2a43e4 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -72,8 +72,6 @@ typedef struct _object_stats { uint64_t type_cache_collisions; } ObjectStats; -# - typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; From 8317d51996e68d8bb205385c1d47a9edcd128e7b Mon Sep 17 00:00:00 2001 From: Furkan Onder Date: Sun, 9 Apr 2023 11:44:49 +0300 Subject: [PATCH 74/76] Gh-68586: use run_python_until_end in test_capi (GH-102729) Co-authored-by: Aidin Gharibnavaz Automerge-Triggered-By: GH:kumaraditya303 --- Lib/test/test_capi/test_misc.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index c34ee578b5c83f..637adc01a331ce 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -21,7 +21,7 @@ from test.support import import_helper from test.support import threading_helper from test.support import warnings_helper -from test.support.script_helper import assert_python_failure, assert_python_ok +from test.support.script_helper import assert_python_failure, assert_python_ok, run_python_until_end try: import _posixsubprocess except ImportError: @@ -69,21 +69,17 @@ def test_instancemethod(self): @support.requires_subprocess() def test_no_FatalError_infinite_loop(self): - with support.SuppressCrashReport(): - p = subprocess.Popen([sys.executable, "-c", - 'import _testcapi;' - '_testcapi.crash_no_current_thread()'], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True) - (out, err) = p.communicate() - self.assertEqual(out, '') + run_result, _cmd_line = run_python_until_end( + '-c', 'import _testcapi; _testcapi.crash_no_current_thread()', + ) + _rc, out, err = run_result + self.assertEqual(out, b'') # This used to cause an infinite loop. msg = ("Fatal Python error: PyThreadState_Get: " "the function must be called with the GIL held, " "after Python initialization and before Python finalization, " "but the GIL is released " - "(the current Python thread state is NULL)") + "(the current Python thread state is NULL)").encode() self.assertTrue(err.rstrip().startswith(msg), err) From 0a675f4bb57d01a5e69f8f58ae934ad7ca501a8d Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 9 Apr 2023 16:48:45 +0100 Subject: [PATCH 75/76] GH-103379: Fix up old tests for `pathlib.PurePath._parse_path` (GH-103380) Co-authored-by: Terry Jan Reedy --- Lib/test/test_pathlib.py | 207 ++++++++++++++++++--------------------- 1 file changed, 97 insertions(+), 110 deletions(-) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 8b6e012b730d75..fe75f1c2580bc4 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -24,116 +24,6 @@ grp = pwd = None -class _BaseFlavourTest(object): - - def _check_parse_parts(self, arg, expected): - def f(parts): - path = self.cls(*parts)._raw_path - return self.cls._parse_path(path) - sep = self.flavour.sep - altsep = self.flavour.altsep - actual = f([x.replace('/', sep) for x in arg]) - self.assertEqual(actual, expected) - if altsep: - actual = f([x.replace('/', altsep) for x in arg]) - self.assertEqual(actual, expected) - - def test_parse_parts_common(self): - check = self._check_parse_parts - sep = self.flavour.sep - # Unanchored parts. - check([], ('', '', [])) - check(['a'], ('', '', ['a'])) - check(['a/'], ('', '', ['a'])) - check(['a', 'b'], ('', '', ['a', 'b'])) - # Expansion. - check(['a/b'], ('', '', ['a', 'b'])) - check(['a/b/'], ('', '', ['a', 'b'])) - check(['a', 'b/c', 'd'], ('', '', ['a', 'b', 'c', 'd'])) - # Collapsing and stripping excess slashes. - check(['a', 'b//c', 'd'], ('', '', ['a', 'b', 'c', 'd'])) - check(['a', 'b/c/', 'd'], ('', '', ['a', 'b', 'c', 'd'])) - # Eliminating standalone dots. - check(['.'], ('', '', [])) - check(['.', '.', 'b'], ('', '', ['b'])) - check(['a', '.', 'b'], ('', '', ['a', 'b'])) - check(['a', '.', '.'], ('', '', ['a'])) - # The first part is anchored. - check(['/a/b'], ('', sep, [sep, 'a', 'b'])) - check(['/a', 'b'], ('', sep, [sep, 'a', 'b'])) - check(['/a/', 'b'], ('', sep, [sep, 'a', 'b'])) - # Ignoring parts before an anchored part. - check(['a', '/b', 'c'], ('', sep, [sep, 'b', 'c'])) - check(['a', '/b', '/c'], ('', sep, [sep, 'c'])) - - -class PosixFlavourTest(_BaseFlavourTest, unittest.TestCase): - cls = pathlib.PurePosixPath - flavour = pathlib.PurePosixPath._flavour - - def test_parse_parts(self): - check = self._check_parse_parts - # Collapsing of excess leading slashes, except for the double-slash - # special case. - check(['//a', 'b'], ('', '//', ['//', 'a', 'b'])) - check(['///a', 'b'], ('', '/', ['/', 'a', 'b'])) - check(['////a', 'b'], ('', '/', ['/', 'a', 'b'])) - # Paths which look like NT paths aren't treated specially. - check(['c:a'], ('', '', ['c:a'])) - check(['c:\\a'], ('', '', ['c:\\a'])) - check(['\\a'], ('', '', ['\\a'])) - - -class NTFlavourTest(_BaseFlavourTest, unittest.TestCase): - cls = pathlib.PureWindowsPath - flavour = pathlib.PureWindowsPath._flavour - - def test_parse_parts(self): - check = self._check_parse_parts - # First part is anchored. - check(['c:'], ('c:', '', ['c:'])) - check(['c:/'], ('c:', '\\', ['c:\\'])) - check(['/'], ('', '\\', ['\\'])) - check(['c:a'], ('c:', '', ['c:', 'a'])) - check(['c:/a'], ('c:', '\\', ['c:\\', 'a'])) - check(['/a'], ('', '\\', ['\\', 'a'])) - # UNC paths. - check(['//a/b'], ('\\\\a\\b', '\\', ['\\\\a\\b\\'])) - check(['//a/b/'], ('\\\\a\\b', '\\', ['\\\\a\\b\\'])) - check(['//a/b/c'], ('\\\\a\\b', '\\', ['\\\\a\\b\\', 'c'])) - # Second part is anchored, so that the first part is ignored. - check(['a', 'Z:b', 'c'], ('Z:', '', ['Z:', 'b', 'c'])) - check(['a', 'Z:/b', 'c'], ('Z:', '\\', ['Z:\\', 'b', 'c'])) - # UNC paths. - check(['a', '//b/c', 'd'], ('\\\\b\\c', '\\', ['\\\\b\\c\\', 'd'])) - # Collapsing and stripping excess slashes. - check(['a', 'Z://b//c/', 'd/'], ('Z:', '\\', ['Z:\\', 'b', 'c', 'd'])) - # UNC paths. - check(['a', '//b/c//', 'd'], ('\\\\b\\c', '\\', ['\\\\b\\c\\', 'd'])) - # Extended paths. - check(['//?/c:/'], ('\\\\?\\c:', '\\', ['\\\\?\\c:\\'])) - check(['//?/c:/a'], ('\\\\?\\c:', '\\', ['\\\\?\\c:\\', 'a'])) - check(['//?/c:/a', '/b'], ('\\\\?\\c:', '\\', ['\\\\?\\c:\\', 'b'])) - # Extended UNC paths (format is "\\?\UNC\server\share"). - check(['//?/UNC/b/c'], ('\\\\?\\UNC\\b\\c', '\\', ['\\\\?\\UNC\\b\\c\\'])) - check(['//?/UNC/b/c/d'], ('\\\\?\\UNC\\b\\c', '\\', ['\\\\?\\UNC\\b\\c\\', 'd'])) - # Second part has a root but not drive. - check(['a', '/b', 'c'], ('', '\\', ['\\', 'b', 'c'])) - check(['Z:/a', '/b', 'c'], ('Z:', '\\', ['Z:\\', 'b', 'c'])) - check(['//?/Z:/a', '/b', 'c'], ('\\\\?\\Z:', '\\', ['\\\\?\\Z:\\', 'b', 'c'])) - # Joining with the same drive => the first path is appended to if - # the second path is relative. - check(['c:/a/b', 'c:x/y'], ('c:', '\\', ['c:\\', 'a', 'b', 'x', 'y'])) - check(['c:/a/b', 'c:/x/y'], ('c:', '\\', ['c:\\', 'x', 'y'])) - # Paths to files with NTFS alternate data streams - check(['./c:s'], ('', '', ['c:s'])) - check(['cc:s'], ('', '', ['cc:s'])) - check(['C:c:s'], ('C:', '', ['C:', 'c:s'])) - check(['C:/c:s'], ('C:', '\\', ['C:\\', 'c:s'])) - check(['D:a', './c:b'], ('D:', '', ['D:', 'a', 'c:b'])) - check(['D:/a', './c:b'], ('D:', '\\', ['D:\\', 'a', 'c:b'])) - - # # Tests for the pure classes. # @@ -246,6 +136,46 @@ class P(_BasePurePathSubclass, self.cls): for parent in p.parents: self.assertTrue(parent.init_called) + def _get_drive_root_parts(self, parts): + path = self.cls(*parts) + return path.drive, path.root, path.parts + + def _check_drive_root_parts(self, arg, *expected): + sep = self.flavour.sep + actual = self._get_drive_root_parts([x.replace('/', sep) for x in arg]) + self.assertEqual(actual, expected) + if altsep := self.flavour.altsep: + actual = self._get_drive_root_parts([x.replace('/', altsep) for x in arg]) + self.assertEqual(actual, expected) + + def test_drive_root_parts_common(self): + check = self._check_drive_root_parts + sep = self.flavour.sep + # Unanchored parts. + check((), '', '', ()) + check(('a',), '', '', ('a',)) + check(('a/',), '', '', ('a',)) + check(('a', 'b'), '', '', ('a', 'b')) + # Expansion. + check(('a/b',), '', '', ('a', 'b')) + check(('a/b/',), '', '', ('a', 'b')) + check(('a', 'b/c', 'd'), '', '', ('a', 'b', 'c', 'd')) + # Collapsing and stripping excess slashes. + check(('a', 'b//c', 'd'), '', '', ('a', 'b', 'c', 'd')) + check(('a', 'b/c/', 'd'), '', '', ('a', 'b', 'c', 'd')) + # Eliminating standalone dots. + check(('.',), '', '', ()) + check(('.', '.', 'b'), '', '', ('b',)) + check(('a', '.', 'b'), '', '', ('a', 'b')) + check(('a', '.', '.'), '', '', ('a',)) + # The first part is anchored. + check(('/a/b',), '', sep, (sep, 'a', 'b')) + check(('/a', 'b'), '', sep, (sep, 'a', 'b')) + check(('/a/', 'b'), '', sep, (sep, 'a', 'b')) + # Ignoring parts before an anchored part. + check(('a', '/b', 'c'), '', sep, (sep, 'b', 'c')) + check(('a', '/b', '/c'), '', sep, (sep, 'c')) + def test_join_common(self): P = self.cls p = P('a/b') @@ -770,6 +700,18 @@ def test_pickling_common(self): class PurePosixPathTest(_BasePurePathTest, unittest.TestCase): cls = pathlib.PurePosixPath + def test_drive_root_parts(self): + check = self._check_drive_root_parts + # Collapsing of excess leading slashes, except for the double-slash + # special case. + check(('//a', 'b'), '', '//', ('//', 'a', 'b')) + check(('///a', 'b'), '', '/', ('/', 'a', 'b')) + check(('////a', 'b'), '', '/', ('/', 'a', 'b')) + # Paths which look like NT paths aren't treated specially. + check(('c:a',), '', '', ('c:a',)) + check(('c:\\a',), '', '', ('c:\\a',)) + check(('\\a',), '', '', ('\\a',)) + def test_root(self): P = self.cls self.assertEqual(P('/a/b').root, '/') @@ -860,6 +802,51 @@ class PureWindowsPathTest(_BasePurePathTest, unittest.TestCase): ], }) + def test_drive_root_parts(self): + check = self._check_drive_root_parts + # First part is anchored. + check(('c:',), 'c:', '', ('c:',)) + check(('c:/',), 'c:', '\\', ('c:\\',)) + check(('/',), '', '\\', ('\\',)) + check(('c:a',), 'c:', '', ('c:', 'a')) + check(('c:/a',), 'c:', '\\', ('c:\\', 'a')) + check(('/a',), '', '\\', ('\\', 'a')) + # UNC paths. + check(('//a/b',), '\\\\a\\b', '\\', ('\\\\a\\b\\',)) + check(('//a/b/',), '\\\\a\\b', '\\', ('\\\\a\\b\\',)) + check(('//a/b/c',), '\\\\a\\b', '\\', ('\\\\a\\b\\', 'c')) + # Second part is anchored, so that the first part is ignored. + check(('a', 'Z:b', 'c'), 'Z:', '', ('Z:', 'b', 'c')) + check(('a', 'Z:/b', 'c'), 'Z:', '\\', ('Z:\\', 'b', 'c')) + # UNC paths. + check(('a', '//b/c', 'd'), '\\\\b\\c', '\\', ('\\\\b\\c\\', 'd')) + # Collapsing and stripping excess slashes. + check(('a', 'Z://b//c/', 'd/'), 'Z:', '\\', ('Z:\\', 'b', 'c', 'd')) + # UNC paths. + check(('a', '//b/c//', 'd'), '\\\\b\\c', '\\', ('\\\\b\\c\\', 'd')) + # Extended paths. + check(('//?/c:/',), '\\\\?\\c:', '\\', ('\\\\?\\c:\\',)) + check(('//?/c:/a',), '\\\\?\\c:', '\\', ('\\\\?\\c:\\', 'a')) + check(('//?/c:/a', '/b'), '\\\\?\\c:', '\\', ('\\\\?\\c:\\', 'b')) + # Extended UNC paths (format is "\\?\UNC\server\share"). + check(('//?/UNC/b/c',), '\\\\?\\UNC\\b\\c', '\\', ('\\\\?\\UNC\\b\\c\\',)) + check(('//?/UNC/b/c/d',), '\\\\?\\UNC\\b\\c', '\\', ('\\\\?\\UNC\\b\\c\\', 'd')) + # Second part has a root but not drive. + check(('a', '/b', 'c'), '', '\\', ('\\', 'b', 'c')) + check(('Z:/a', '/b', 'c'), 'Z:', '\\', ('Z:\\', 'b', 'c')) + check(('//?/Z:/a', '/b', 'c'), '\\\\?\\Z:', '\\', ('\\\\?\\Z:\\', 'b', 'c')) + # Joining with the same drive => the first path is appended to if + # the second path is relative. + check(('c:/a/b', 'c:x/y'), 'c:', '\\', ('c:\\', 'a', 'b', 'x', 'y')) + check(('c:/a/b', 'c:/x/y'), 'c:', '\\', ('c:\\', 'x', 'y')) + # Paths to files with NTFS alternate data streams + check(('./c:s',), '', '', ('c:s',)) + check(('cc:s',), '', '', ('cc:s',)) + check(('C:c:s',), 'C:', '', ('C:', 'c:s')) + check(('C:/c:s',), 'C:', '\\', ('C:\\', 'c:s')) + check(('D:a', './c:b'), 'D:', '', ('D:', 'a', 'c:b')) + check(('D:/a', './c:b'), 'D:', '\\', ('D:\\', 'a', 'c:b')) + def test_str(self): p = self.cls('a/b/c') self.assertEqual(str(p), 'a\\b\\c') From 2c673d5e93cfe2779f27c4e742d7e50f7a94f356 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 9 Apr 2023 18:40:03 +0100 Subject: [PATCH 76/76] GH-101362: Omit path anchor from `pathlib.PurePath()._parts` (GH-102476) Improve performance of path construction by skipping the addition of the path anchor (`drive + root`) to the internal `_parts` list. Rename this attribute to `_tail` for clarity. --- Lib/pathlib.py | 171 +++++++++++------- Lib/test/test_pathlib.py | 2 - ...-03-06-18-49-57.gh-issue-101362.eSSy6L.rst | 2 + 3 files changed, 108 insertions(+), 67 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-06-18-49-57.gh-issue-101362.eSSy6L.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 490f89f39d26d1..4ae1fae6f4b358 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -210,20 +210,17 @@ def _select_from(self, parent_path, is_dir, exists, scandir, normcase): class _PathParents(Sequence): """This object provides sequence-like access to the logical ancestors of a path. Don't try to construct it yourself.""" - __slots__ = ('_pathcls', '_drv', '_root', '_parts') + __slots__ = ('_pathcls', '_drv', '_root', '_tail') def __init__(self, path): # We don't store the instance to avoid reference cycles self._pathcls = type(path) self._drv = path.drive self._root = path.root - self._parts = path._parts + self._tail = path._tail def __len__(self): - if self._drv or self._root: - return len(self._parts) - 1 - else: - return len(self._parts) + return len(self._tail) def __getitem__(self, idx): if isinstance(idx, slice): @@ -234,7 +231,7 @@ def __getitem__(self, idx): if idx < 0: idx += len(self) return self._pathcls._from_parsed_parts(self._drv, self._root, - self._parts[:-idx - 1]) + self._tail[:-idx - 1]) def __repr__(self): return "<{}.parents>".format(self._pathcls.__name__) @@ -249,9 +246,41 @@ class PurePath(object): PureWindowsPath object. You can also instantiate either of these classes directly, regardless of your system. """ + __slots__ = ( - '_raw_path', '_drv', '_root', '_parts_cached', - '_str', '_hash', '_parts_tuple', '_parts_normcase_cached', + # The `_raw_path` slot stores an unnormalized string path. This is set + # in the `__init__()` method. + '_raw_path', + + # The `_drv`, `_root` and `_tail_cached` slots store parsed and + # normalized parts of the path. They are set when any of the `drive`, + # `root` or `_tail` properties are accessed for the first time. The + # three-part division corresponds to the result of + # `os.path.splitroot()`, except that the tail is further split on path + # separators (i.e. it is a list of strings), and that the root and + # tail are normalized. + '_drv', '_root', '_tail_cached', + + # The `_str` slot stores the string representation of the path, + # computed from the drive, root and tail when `__str__()` is called + # for the first time. It's used to implement `_str_normcase` + '_str', + + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', ) _flavour = os.path @@ -277,10 +306,7 @@ def __init__(self, *args): path = os.fspath(args[0]) else: path = self._flavour.join(*args) - if isinstance(path, str): - # Force-cast str subclasses to str (issue #21127) - path = str(path) - else: + if not isinstance(path, str): raise TypeError( "argument should be a str or an os.PathLike " "object where __fspath__ returns a str, " @@ -299,33 +325,32 @@ def _parse_path(cls, path): if drv.startswith(sep): # pathlib assumes that UNC paths always have a root. root = sep - unfiltered_parsed = [drv + root] + rel.split(sep) - parsed = [sys.intern(x) for x in unfiltered_parsed if x and x != '.'] + parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] return drv, root, parsed def _load_parts(self): - drv, root, parts = self._parse_path(self._raw_path) + drv, root, tail = self._parse_path(self._raw_path) self._drv = drv self._root = root - self._parts_cached = parts + self._tail_cached = tail @classmethod - def _from_parsed_parts(cls, drv, root, parts): - path = cls._format_parsed_parts(drv, root, parts) + def _from_parsed_parts(cls, drv, root, tail): + path = cls._format_parsed_parts(drv, root, tail) self = cls(path) self._str = path or '.' self._drv = drv self._root = root - self._parts_cached = parts + self._tail_cached = tail return self @classmethod - def _format_parsed_parts(cls, drv, root, parts): + def _format_parsed_parts(cls, drv, root, tail): if drv or root: - return drv + root + cls._flavour.sep.join(parts[1:]) - elif parts and cls._flavour.splitdrive(parts[0])[0]: - parts = ['.'] + parts - return cls._flavour.sep.join(parts) + return drv + root + cls._flavour.sep.join(tail) + elif tail and cls._flavour.splitdrive(tail[0])[0]: + tail = ['.'] + tail + return cls._flavour.sep.join(tail) def __str__(self): """Return the string representation of the path, suitable for @@ -334,7 +359,7 @@ def __str__(self): return self._str except AttributeError: self._str = self._format_parsed_parts(self.drive, self.root, - self._parts) or '.' + self._tail) or '.' return self._str def __fspath__(self): @@ -374,25 +399,34 @@ def as_uri(self): path = str(self) return prefix + urlquote_from_bytes(os.fsencode(path)) + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + self._str_normcase_cached = self._flavour.normcase(str(self)) + return self._str_normcase_cached + @property def _parts_normcase(self): - # Cached parts with normalized case, for hashing and comparison. + # Cached parts with normalized case, for comparisons. try: return self._parts_normcase_cached except AttributeError: - self._parts_normcase_cached = [self._flavour.normcase(p) for p in self._parts] + self._parts_normcase_cached = self._str_normcase.split(self._flavour.sep) return self._parts_normcase_cached def __eq__(self, other): if not isinstance(other, PurePath): return NotImplemented - return self._parts_normcase == other._parts_normcase and self._flavour is other._flavour + return self._str_normcase == other._str_normcase and self._flavour is other._flavour def __hash__(self): try: return self._hash except AttributeError: - self._hash = hash(tuple(self._parts_normcase)) + self._hash = hash(self._str_normcase) return self._hash def __lt__(self, other): @@ -434,12 +468,12 @@ def root(self): return self._root @property - def _parts(self): + def _tail(self): try: - return self._parts_cached + return self._tail_cached except AttributeError: self._load_parts() - return self._parts_cached + return self._tail_cached @property def anchor(self): @@ -450,10 +484,10 @@ def anchor(self): @property def name(self): """The final path component, if any.""" - parts = self._parts - if len(parts) == (1 if (self.drive or self.root) else 0): + tail = self._tail + if not tail: return '' - return parts[-1] + return tail[-1] @property def suffix(self): @@ -501,7 +535,7 @@ def with_name(self, name): if drv or root or not tail or f.sep in tail or (f.altsep and f.altsep in tail): raise ValueError("Invalid name %r" % (name)) return self._from_parsed_parts(self.drive, self.root, - self._parts[:-1] + [name]) + self._tail[:-1] + [name]) def with_stem(self, stem): """Return a new path with the stem changed.""" @@ -526,7 +560,7 @@ def with_suffix(self, suffix): else: name = name[:-len(old_suffix)] + suffix return self._from_parsed_parts(self.drive, self.root, - self._parts[:-1] + [name]) + self._tail[:-1] + [name]) def relative_to(self, other, /, *_deprecated, walk_up=False): """Return the relative path to another path identified by the passed @@ -551,7 +585,7 @@ def relative_to(self, other, /, *_deprecated, walk_up=False): raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") if step and not walk_up: raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") - parts = ('..',) * step + self.parts[len(path.parts):] + parts = ['..'] * step + self._tail[len(path._tail):] return path_cls(*parts) def is_relative_to(self, other, /, *_deprecated): @@ -570,13 +604,10 @@ def is_relative_to(self, other, /, *_deprecated): def parts(self): """An object providing sequence-like access to the components in the filesystem path.""" - # We cache the tuple to avoid building a new one each time .parts - # is accessed. XXX is this necessary? - try: - return self._parts_tuple - except AttributeError: - self._parts_tuple = tuple(self._parts) - return self._parts_tuple + if self.drive or self.root: + return (self.drive + self.root,) + tuple(self._tail) + else: + return tuple(self._tail) def joinpath(self, *args): """Combine this path with one or several arguments, and return a @@ -603,10 +634,10 @@ def parent(self): """The logical parent of the path.""" drv = self.drive root = self.root - parts = self._parts - if len(parts) == 1 and (drv or root): + tail = self._tail + if not tail: return self - return self._from_parsed_parts(drv, root, parts[:-1]) + return self._from_parsed_parts(drv, root, tail[:-1]) @property def parents(self): @@ -624,29 +655,29 @@ def is_absolute(self): def is_reserved(self): """Return True if the path contains one of the special names reserved by the system, if any.""" - if self._flavour is posixpath or not self._parts: + if self._flavour is posixpath or not self._tail: return False # NOTE: the rules for reserved names seem somewhat complicated # (e.g. r"..\NUL" is reserved but not r"foo\NUL" if "foo" does not # exist). We err on the side of caution and return True for paths # which are not considered reserved by Windows. - if self._parts[0].startswith('\\\\'): + if self.drive.startswith('\\\\'): # UNC paths are never reserved. return False - name = self._parts[-1].partition('.')[0].partition(':')[0].rstrip(' ') + name = self._tail[-1].partition('.')[0].partition(':')[0].rstrip(' ') return name.upper() in _WIN_RESERVED_NAMES def match(self, path_pattern): """ Return True if this path matches the given pattern. """ - path_pattern = self._flavour.normcase(path_pattern) - drv, root, pat_parts = self._parse_path(path_pattern) - if not pat_parts: + pat = type(self)(path_pattern) + if not pat.parts: raise ValueError("empty pattern") + pat_parts = pat._parts_normcase parts = self._parts_normcase - if drv or root: + if pat.drive or pat.root: if len(pat_parts) != len(parts): return False elif len(pat_parts) > len(parts): @@ -707,11 +738,21 @@ def __new__(cls, *args, **kwargs): cls = WindowsPath if os.name == 'nt' else PosixPath return object.__new__(cls) - def _make_child_relpath(self, part): - # This is an optimization used for dir walking. `part` must be - # a single part relative to this path. - parts = self._parts + [part] - return self._from_parsed_parts(self.drive, self.root, parts) + def _make_child_relpath(self, name): + path_str = str(self) + tail = self._tail + if tail: + path_str = f'{path_str}{self._flavour.sep}{name}' + elif path_str != '.': + path_str = f'{path_str}{name}' + else: + path_str = name + path = type(self)(path_str) + path._str = path_str + path._drv = self.drive + path._root = self.root + path._tail_cached = tail + [name] + return path def __enter__(self): # In previous versions of pathlib, __exit__() marked this path as @@ -1196,12 +1237,12 @@ def expanduser(self): (as returned by os.path.expanduser) """ if (not (self.drive or self.root) and - self._parts and self._parts[0][:1] == '~'): - homedir = self._flavour.expanduser(self._parts[0]) + self._tail and self._tail[0][:1] == '~'): + homedir = self._flavour.expanduser(self._tail[0]) if homedir[:1] == "~": raise RuntimeError("Could not determine home directory.") - drv, root, parts = self._parse_path(homedir) - return self._from_parsed_parts(drv, root, parts + self._parts[1:]) + drv, root, tail = self._parse_path(homedir) + return self._from_parsed_parts(drv, root, tail + self._tail[1:]) return self diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index fe75f1c2580bc4..3c6da94d094610 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -346,8 +346,6 @@ def test_parts_common(self): p = P('a/b') parts = p.parts self.assertEqual(parts, ('a', 'b')) - # The object gets reused. - self.assertIs(parts, p.parts) # When the path is absolute, the anchor is a separate part. p = P('/a/b') parts = p.parts diff --git a/Misc/NEWS.d/next/Library/2023-03-06-18-49-57.gh-issue-101362.eSSy6L.rst b/Misc/NEWS.d/next/Library/2023-03-06-18-49-57.gh-issue-101362.eSSy6L.rst new file mode 100644 index 00000000000000..87617a503c0dba --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-06-18-49-57.gh-issue-101362.eSSy6L.rst @@ -0,0 +1,2 @@ +Speed up :class:`pathlib.Path` construction by omitting the path anchor from +the internal list of path parts.