From 587f2f018051049cf5d9de3e12ed5aa7644404dc Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 2 May 2023 11:13:47 -0700 Subject: [PATCH 01/65] gh-65022: Fix description of tuple return value in copyreg (#103892) --- Doc/library/copyreg.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 2107215c0c1967..afc3e66f0bf7ac 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -29,7 +29,7 @@ Such constructors may be factory functions or class instances. Declares that *function* should be used as a "reduction" function for objects of type *type*. *function* must return either a string or a tuple - containing two or five elements. See the :attr:`~pickle.Pickler.dispatch_table` + containing between two and six elements. See the :attr:`~pickle.Pickler.dispatch_table` for more details on the interface of *function*. The *constructor_ob* parameter is a legacy feature and is now ignored, but if From 1f5384434dce013b5dcf7e7ea3ec5312d13bba72 Mon Sep 17 00:00:00 2001 From: Prince Roshan Date: Wed, 3 May 2023 01:43:31 +0530 Subject: [PATCH 02/65] gh-103822: [Calendar] change return value to enum for day and month APIs (GH-103827) --- Lib/calendar.py | 3 +-- .../Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst diff --git a/Lib/calendar.py b/Lib/calendar.py index bbd4fea3b88ca4..ea56f12ccc41d0 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -83,7 +83,6 @@ class Day(IntEnum): SUNDAY = 6 - # Number of days per month (except for February in leap years) mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] @@ -156,7 +155,7 @@ def weekday(year, month, day): """Return weekday (0-6 ~ Mon-Sun) for year, month (1-12), day (1-31).""" if not datetime.MINYEAR <= year <= datetime.MAXYEAR: year = 2000 + year % 400 - return datetime.date(year, month, day).weekday() + return Day(datetime.date(year, month, day).weekday()) def monthrange(year, month): diff --git a/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst b/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst new file mode 100644 index 00000000000000..3daf9cc093807b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-02-04-49-45.gh-issue-103822.m0QdAO.rst @@ -0,0 +1 @@ +Update the return type of ``weekday`` to the newly added Day attribute From 47770a1e91d096fd1c689eb0c78b0f9e76b43639 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Tue, 2 May 2023 22:51:18 +0100 Subject: [PATCH 03/65] GH-104104: Optimize `pathlib.Path.glob()` by avoiding repeated calls to `os.path.normcase()` (GH-104105) Use `re.IGNORECASE` to implement case-insensitive matching. This restores behaviour from before GH-31691. --- Lib/pathlib.py | 25 +++++++++++-------- ...-05-02-21-05-30.gh-issue-104104.9tjplT.rst | 2 ++ 2 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 8eb08949fa9b43..61e7f3e4430cae 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -59,6 +59,9 @@ def _is_wildcard_pattern(pat): # be looked up directly as a file. return "*" in pat or "?" in pat or "[" in pat +def _is_case_sensitive(flavour): + return flavour.normcase('Aa') == 'Aa' + # # Globbing helpers # @@ -100,15 +103,14 @@ def select_from(self, parent_path): is_dir = path_cls.is_dir exists = path_cls.exists scandir = path_cls._scandir - normcase = path_cls._flavour.normcase if not is_dir(parent_path): return iter([]) - return self._select_from(parent_path, is_dir, exists, scandir, normcase) + return self._select_from(parent_path, is_dir, exists, scandir) class _TerminatingSelector: - def _select_from(self, parent_path, is_dir, exists, scandir, normcase): + def _select_from(self, parent_path, is_dir, exists, scandir): yield parent_path @@ -118,11 +120,11 @@ def __init__(self, name, child_parts, flavour): self.name = name _Selector.__init__(self, child_parts, flavour) - def _select_from(self, parent_path, is_dir, exists, scandir, normcase): + def _select_from(self, parent_path, is_dir, exists, scandir): try: path = parent_path._make_child_relpath(self.name) if (is_dir if self.dironly else exists)(path): - for p in self.successor._select_from(path, is_dir, exists, scandir, normcase): + for p in self.successor._select_from(path, is_dir, exists, scandir): yield p except PermissionError: return @@ -131,10 +133,11 @@ def _select_from(self, parent_path, is_dir, exists, scandir, normcase): class _WildcardSelector(_Selector): def __init__(self, pat, child_parts, flavour): - self.match = re.compile(fnmatch.translate(flavour.normcase(pat))).fullmatch + flags = re.NOFLAG if _is_case_sensitive(flavour) else re.IGNORECASE + self.match = re.compile(fnmatch.translate(pat), flags=flags).fullmatch _Selector.__init__(self, child_parts, flavour) - def _select_from(self, parent_path, is_dir, exists, scandir, normcase): + def _select_from(self, parent_path, is_dir, exists, scandir): try: # We must close the scandir() object before proceeding to # avoid exhausting file descriptors when globbing deep trees. @@ -153,9 +156,9 @@ def _select_from(self, parent_path, is_dir, exists, scandir, normcase): raise continue name = entry.name - if self.match(normcase(name)): + if self.match(name): path = parent_path._make_child_relpath(name) - for p in self.successor._select_from(path, is_dir, exists, scandir, normcase): + for p in self.successor._select_from(path, is_dir, exists, scandir): yield p except PermissionError: return @@ -187,13 +190,13 @@ def _iterate_directories(self, parent_path, is_dir, scandir): except PermissionError: return - def _select_from(self, parent_path, is_dir, exists, scandir, normcase): + def _select_from(self, parent_path, is_dir, exists, scandir): try: yielded = set() try: successor_select = self.successor._select_from for starting_point in self._iterate_directories(parent_path, is_dir, scandir): - for p in successor_select(starting_point, is_dir, exists, scandir, normcase): + for p in successor_select(starting_point, is_dir, exists, scandir): if p not in yielded: yield p yielded.add(p) diff --git a/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst b/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst new file mode 100644 index 00000000000000..935a0e2a2bff18 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-02-21-05-30.gh-issue-104104.9tjplT.rst @@ -0,0 +1,2 @@ +Improve performance of :meth:`pathlib.Path.glob` by using +:data:`re.IGNORECASE` to implement case-insensitive matching. From 65a49c6553a27cc36eebb4b79f409c3cb4450d8c Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Wed, 3 May 2023 00:16:04 +0100 Subject: [PATCH 04/65] GH-104102: Optimize `pathlib.Path.glob()` handling of `../` pattern segments (GH-104103) These segments do not require a `stat()` call, as the selector's `_select_from()` method is called after we've established that the parent is a directory. --- Lib/pathlib.py | 12 ++++++++++++ Lib/test/test_pathlib.py | 5 +++++ .../2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst | 2 ++ 3 files changed, 19 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 61e7f3e4430cae..c69089f4e1bc5d 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -74,6 +74,8 @@ def _make_selector(pattern_parts, flavour): return _TerminatingSelector() if pat == '**': cls = _RecursiveWildcardSelector + elif pat == '..': + cls = _ParentSelector elif '**' in pat: raise ValueError("Invalid pattern: '**' can only be an entire path component") elif _is_wildcard_pattern(pat): @@ -114,6 +116,16 @@ def _select_from(self, parent_path, is_dir, exists, scandir): yield parent_path +class _ParentSelector(_Selector): + def __init__(self, name, child_parts, flavour): + _Selector.__init__(self, child_parts, flavour) + + def _select_from(self, parent_path, is_dir, exists, scandir): + path = parent_path._make_child_relpath('..') + for p in self.successor._select_from(path, is_dir, exists, scandir): + yield p + + class _PreciseSelector(_Selector): def __init__(self, name, child_parts, flavour): diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 8b5b61a818bbbc..9902b7242205f3 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1892,8 +1892,13 @@ def test_glob_dotdot(self): P = self.cls p = P(BASE) self.assertEqual(set(p.glob("..")), { P(BASE, "..") }) + self.assertEqual(set(p.glob("../..")), { P(BASE, "..", "..") }) + self.assertEqual(set(p.glob("dirA/..")), { P(BASE, "dirA", "..") }) self.assertEqual(set(p.glob("dirA/../file*")), { P(BASE, "dirA/../fileA") }) + self.assertEqual(set(p.glob("dirA/../file*/..")), set()) self.assertEqual(set(p.glob("../xyzzy")), set()) + self.assertEqual(set(p.glob("xyzzy/..")), set()) + self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(BASE, *[".."] * 50)}) @os_helper.skip_unless_symlink def test_glob_permissions(self): diff --git a/Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst b/Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst new file mode 100644 index 00000000000000..7101de908a5004 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-02-20-43-03.gh-issue-104102.vgSdEJ.rst @@ -0,0 +1,2 @@ +Improve performance of :meth:`pathlib.Path.glob` when evaluating patterns +that contain ``'../'`` segments. From 872cbc613245db7a1fc5e6656ed0135d2e115f50 Mon Sep 17 00:00:00 2001 From: Juhi Chandalia Date: Tue, 2 May 2023 19:00:17 -0700 Subject: [PATCH 05/65] GH-103963: Make dis display names of args for intrinsics opcodes (#104029) --- Include/internal/pycore_intrinsics.h | 26 ++++++------ Lib/dis.py | 8 ++++ Lib/opcode.py | 15 +++++++ Lib/test/test_dis.py | 41 ++++++++++++++++++- ...-05-01-17-58-28.gh-issue-103963.XWlHx7.rst | 1 + Tools/build/generate_opcode_h.py | 36 +++++++++++++++- 6 files changed, 110 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 46a52740eb8a0c..3902059a04b9da 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -1,26 +1,24 @@ +// Auto-generated by Tools/build/generate_opcode_h.py from Lib/opcode.py /* Unary Functions: */ +#define INTRINSIC_1_INVALID 0 +#define INTRINSIC_PRINT 1 +#define INTRINSIC_IMPORT_STAR 2 +#define INTRINSIC_STOPITERATION_ERROR 3 +#define INTRINSIC_ASYNC_GEN_WRAP 4 +#define INTRINSIC_UNARY_POSITIVE 5 +#define INTRINSIC_LIST_TO_TUPLE 6 -#define INTRINSIC_PRINT 1 -#define INTRINSIC_IMPORT_STAR 2 -#define INTRINSIC_STOPITERATION_ERROR 3 -#define INTRINSIC_ASYNC_GEN_WRAP 4 -#define INTRINSIC_UNARY_POSITIVE 5 -#define INTRINSIC_LIST_TO_TUPLE 6 - -#define MAX_INTRINSIC_1 6 +#define MAX_INTRINSIC_1 6 /* Binary Functions: */ +#define INTRINSIC_2_INVALID 0 +#define INTRINSIC_PREP_RERAISE_STAR 1 -#define INTRINSIC_PREP_RERAISE_STAR 1 - -#define MAX_INTRINSIC_2 1 - +#define MAX_INTRINSIC_2 1 typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); - extern const instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; extern const instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; - diff --git a/Lib/dis.py b/Lib/dis.py index 85c109584bf94f..3a8e6ac3bf5ace 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -11,6 +11,8 @@ _cache_format, _inline_cache_entries, _nb_ops, + _intrinsic_1_descs, + _intrinsic_2_descs, _specializations, _specialized_instructions, ) @@ -42,6 +44,8 @@ SEND = opmap['SEND'] LOAD_ATTR = opmap['LOAD_ATTR'] LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] +CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] +CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] CACHE = opmap["CACHE"] @@ -506,6 +510,10 @@ def _get_instructions_bytes(code, varname_from_oparg=None, if arg & (1<> COPY 3 POP_EXCEPT RERAISE 1 - >> CALL_INTRINSIC_1 3 + >> CALL_INTRINSIC_1 3 (INTRINSIC_STOPITERATION_ERROR) RERAISE 1 ExceptionTable: 12 rows @@ -942,6 +971,16 @@ def test_kw_names(self): # Test that value is displayed for KW_NAMES self.do_disassembly_test(wrap_func_w_kwargs, dis_kw_names) + def test_intrinsic_1(self): + # Test that argrepr is displayed for CALL_INTRINSIC_1 + self.do_disassembly_test("from math import *", dis_intrinsic_1_2) + self.do_disassembly_test("+a", dis_intrinsic_1_5) + self.do_disassembly_test("(*a,)", dis_intrinsic_1_6) + + def test_intrinsic_2(self): + self.assertIn("CALL_INTRINSIC_2 1 (INTRINSIC_PREP_RERAISE_STAR)", + self.get_disassembly("try: pass\nexcept* Exception: x")) + def test_big_linenos(self): def func(count): namespace = {} diff --git a/Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst b/Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst new file mode 100644 index 00000000000000..cb06ad5d22e8a9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-01-17-58-28.gh-issue-103963.XWlHx7.rst @@ -0,0 +1 @@ +Make :mod:`dis` display the names of the args for :opcode:`CALL_INTRINSIC_*`. diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 645b9f1de1170b..adcbaf2b8e08bf 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -52,6 +52,18 @@ #endif // !Py_INTERNAL_OPCODE_H """ +intrinsic_header = f""" +// Auto-generated by {SCRIPT_NAME} from {PYTHON_OPCODE} + +""".lstrip() + +intrinsic_footer = """ +typedef PyObject *(*instrinsic_func1)(PyThreadState* tstate, PyObject *value); +typedef PyObject *(*instrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); +extern const instrinsic_func1 _PyIntrinsics_UnaryFunctions[]; +extern const instrinsic_func2 _PyIntrinsics_BinaryFunctions[]; +""" + DEFINE = "#define {:<38} {:>3}\n" UINT32_MASK = (1<<32)-1 @@ -67,7 +79,9 @@ def write_int_array_from_ops(name, ops, out): assert bits == 0 out.write(f"}};\n") -def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/internal/pycore_opcode.h'): +def main(opcode_py, outfile='Include/opcode.h', + internaloutfile='Include/internal/pycore_opcode.h', + intrinsicoutfile='Include/internal/pycore_intrinsics.h'): opcode = {} if hasattr(tokenize, 'open'): fp = tokenize.open(opcode_py) # Python 3.2+ @@ -107,9 +121,11 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna opname_including_specialized[next_op] = name used[next_op] = True - with open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj: + with open(outfile, 'w') as fobj, open(internaloutfile, 'w') as iobj, open( + intrinsicoutfile, "w") as nobj: fobj.write(header) iobj.write(internal_header) + nobj.write(intrinsic_header) for name in opname: if name in opmap: @@ -172,6 +188,22 @@ def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/interna for i, (op, _) in enumerate(opcode["_nb_ops"]): fobj.write(DEFINE.format(op, i)) + nobj.write("/* Unary Functions: */") + nobj.write("\n") + for i, op in enumerate(opcode["_intrinsic_1_descs"]): + nobj.write(DEFINE.format(op, i)) + nobj.write("\n") + nobj.write(DEFINE.format("MAX_INTRINSIC_1", i)) + + nobj.write("\n\n") + nobj.write("/* Binary Functions: */\n") + for i, op in enumerate(opcode["_intrinsic_2_descs"]): + nobj.write(DEFINE.format(op, i)) + nobj.write("\n") + nobj.write(DEFINE.format("MAX_INTRINSIC_2", i)) + + nobj.write(intrinsic_footer) + fobj.write("\n") fobj.write("/* Defined in Lib/opcode.py */\n") fobj.write(f"#define ENABLE_SPECIALIZATION {int(ENABLE_SPECIALIZATION)}") From de64e7561680fdc5358001e9488091e75d4174a3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 May 2023 21:30:03 -0600 Subject: [PATCH 06/65] gh-94673: More Per-Interpreter Fields for Builtin Static Types (gh-103912) his involves moving tp_dict, tp_bases, and tp_mro to PyInterpreterState, in the same way we did for tp_subclasses. Those three fields are effectively const for builtin static types (unlike tp_subclasses). In theory we only need to make their values immortal, along with their contents. However, that isn't such a simple proposition. (See gh-103823.) In the meantime the simplest solution is to move the fields into the interpreter. One alternative is to statically allocate the values, but that's its own can of worms. --- Include/internal/pycore_typeobject.h | 7 + Modules/_abc.c | 1 + Modules/gcmodule.c | 31 +--- Objects/structseq.c | 9 +- Objects/typeobject.c | 223 +++++++++++++++++++++------ 5 files changed, 185 insertions(+), 86 deletions(-) diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index f865e51aeba503..6a5ab7e63f85c3 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -44,6 +44,13 @@ struct type_cache { typedef struct { PyTypeObject *type; + int readying; + int ready; + // XXX tp_dict, tp_bases, and tp_mro can probably be statically + // allocated, instead of dynamically and stored on the interpreter. + PyObject *tp_dict; + PyObject *tp_bases; + PyObject *tp_mro; PyObject *tp_subclasses; /* We never clean up weakrefs for static builtin types since they will effectively never get triggered. However, there diff --git a/Modules/_abc.c b/Modules/_abc.c index 997b618d557ab2..9694331339aa78 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -7,6 +7,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_object.h" // _PyType_GetSubclasses() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_typeobject.h" // _PyType_GetMRO() #include "clinic/_abc.c.h" /*[clinic input] diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 8a4d1a4398281e..f4d5186ff1552e 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -2174,23 +2174,6 @@ _PyGC_DumpShutdownStats(PyInterpreterState *interp) } -static void -gc_fini_untrack(PyGC_Head *list) -{ - PyGC_Head *gc; - for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(list)) { - PyObject *op = FROM_GC(gc); - _PyObject_GC_UNTRACK(op); - // gh-92036: If a deallocator function expect the object to be tracked - // by the GC (ex: func_dealloc()), it can crash if called on an object - // which is no longer tracked by the GC. Leak one strong reference on - // purpose so the object is never deleted and its deallocator is not - // called. - Py_INCREF(op); - } -} - - void _PyGC_Fini(PyInterpreterState *interp) { @@ -2198,17 +2181,9 @@ _PyGC_Fini(PyInterpreterState *interp) Py_CLEAR(gcstate->garbage); Py_CLEAR(gcstate->callbacks); - if (!_Py_IsMainInterpreter(interp)) { - // bpo-46070: Explicitly untrack all objects currently tracked by the - // GC. Otherwise, if an object is used later by another interpreter, - // calling PyObject_GC_UnTrack() on the object crashs if the previous - // or the next object of the PyGC_Head structure became a dangling - // pointer. - for (int i = 0; i < NUM_GENERATIONS; i++) { - PyGC_Head *gen = GEN_HEAD(gcstate, i); - gc_fini_untrack(gen); - } - } + /* We expect that none of this interpreters objects are shared + with other interpreters. + See https://github.com/python/cpython/issues/90228. */ } /* for debugging */ diff --git a/Objects/structseq.c b/Objects/structseq.c index f63660acb639c3..8b1895957101a4 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -511,7 +511,6 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, Py_ssize_t n_members = count_members(desc, &n_unnamed_members); PyMemberDef *members = NULL; - int initialized = 1; if ((type->tp_flags & Py_TPFLAGS_READY) == 0) { assert(type->tp_name == NULL); assert(type->tp_members == NULL); @@ -524,7 +523,6 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, initialize_static_fields(type, desc, members, tp_flags); _Py_SetImmortal(type); - initialized = 0; } #ifndef NDEBUG else { @@ -543,13 +541,10 @@ _PyStructSequence_InitBuiltinWithFlags(PyInterpreterState *interp, desc->name); goto error; } - // This should be dropped if tp_dict is made per-interpreter. - if (initialized) { - return 0; - } if (initialize_structseq_dict( - desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) { + desc, _PyType_GetDict(type), n_members, n_unnamed_members) < 0) + { goto error; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a9d3a69263fb40..cf0efe199b2828 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -167,29 +167,93 @@ static_builtin_state_clear(PyInterpreterState *interp, PyTypeObject *self) /* end static builtin helpers */ +static inline void +start_readying(PyTypeObject *type) +{ + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = static_builtin_state_get(interp, type); + assert(state != NULL); + assert(!state->readying); + state->readying = 1; + return; + } + assert((type->tp_flags & Py_TPFLAGS_READYING) == 0); + type->tp_flags |= Py_TPFLAGS_READYING; +} + +static inline void +stop_readying(PyTypeObject *type) +{ + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = static_builtin_state_get(interp, type); + assert(state != NULL); + assert(state->readying); + state->readying = 0; + return; + } + assert(type->tp_flags & Py_TPFLAGS_READYING); + type->tp_flags &= ~Py_TPFLAGS_READYING; +} + +static inline int +is_readying(PyTypeObject *type) +{ + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = static_builtin_state_get(interp, type); + assert(state != NULL); + return state->readying; + } + return (type->tp_flags & Py_TPFLAGS_READYING) != 0; +} + + /* accessors for objects stored on PyTypeObject */ static inline PyObject * lookup_tp_dict(PyTypeObject *self) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + return state->tp_dict; + } return self->tp_dict; } PyObject * _PyType_GetDict(PyTypeObject *self) { + /* It returns a borrowed reference. */ return lookup_tp_dict(self); } static inline void set_tp_dict(PyTypeObject *self, PyObject *dict) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + state->tp_dict = dict; + return; + } self->tp_dict = dict; } static inline void clear_tp_dict(PyTypeObject *self) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + Py_CLEAR(state->tp_dict); + return; + } Py_CLEAR(self->tp_dict); } @@ -197,24 +261,45 @@ clear_tp_dict(PyTypeObject *self) static inline PyObject * lookup_tp_bases(PyTypeObject *self) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + return state->tp_bases; + } return self->tp_bases; } PyObject * _PyType_GetBases(PyTypeObject *self) { + /* It returns a borrowed reference. */ return lookup_tp_bases(self); } static inline void set_tp_bases(PyTypeObject *self, PyObject *bases) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + state->tp_bases = bases; + return; + } self->tp_bases = bases; } static inline void clear_tp_bases(PyTypeObject *self) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + Py_CLEAR(state->tp_bases); + return; + } Py_CLEAR(self->tp_bases); } @@ -222,24 +307,45 @@ clear_tp_bases(PyTypeObject *self) static inline PyObject * lookup_tp_mro(PyTypeObject *self) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + return state->tp_mro; + } return self->tp_mro; } PyObject * _PyType_GetMRO(PyTypeObject *self) { + /* It returns a borrowed reference. */ return lookup_tp_mro(self); } static inline void set_tp_mro(PyTypeObject *self, PyObject *mro) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + state->tp_mro = mro; + return; + } self->tp_mro = mro; } static inline void clear_tp_mro(PyTypeObject *self) { + if (self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + static_builtin_state *state = _PyStaticType_GetState(interp, self); + assert(state != NULL); + Py_CLEAR(state->tp_mro); + return; + } Py_CLEAR(self->tp_mro); } @@ -408,7 +514,7 @@ _PyType_CheckConsistency(PyTypeObject *type) CHECK(Py_REFCNT(type) >= 1); CHECK(PyType_Check(type)); - CHECK(!(type->tp_flags & Py_TPFLAGS_READYING)); + CHECK(!is_readying(type)); CHECK(lookup_tp_dict(type) != NULL); if (type->tp_flags & Py_TPFLAGS_HAVE_GC) { @@ -809,7 +915,6 @@ static PyMemberDef type_members[] = { {"__base__", T_OBJECT, offsetof(PyTypeObject, tp_base), READONLY}, {"__dictoffset__", T_PYSSIZET, offsetof(PyTypeObject, tp_dictoffset), READONLY}, - {"__mro__", T_OBJECT, offsetof(PyTypeObject, tp_mro), READONLY}, {0} }; @@ -1023,7 +1128,21 @@ type_set_abstractmethods(PyTypeObject *type, PyObject *value, void *context) static PyObject * type_get_bases(PyTypeObject *type, void *context) { - return Py_NewRef(lookup_tp_bases(type)); + PyObject *bases = lookup_tp_bases(type); + if (bases == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(bases); +} + +static PyObject * +type_get_mro(PyTypeObject *type, void *context) +{ + PyObject *mro = lookup_tp_mro(type); + if (mro == NULL) { + Py_RETURN_NONE; + } + return Py_NewRef(mro); } static PyTypeObject *best_base(PyObject *); @@ -1402,6 +1521,7 @@ static PyGetSetDef type_getsets[] = { {"__name__", (getter)type_name, (setter)type_set_name, NULL}, {"__qualname__", (getter)type_qualname, (setter)type_set_qualname, NULL}, {"__bases__", (getter)type_get_bases, (setter)type_set_bases, NULL}, + {"__mro__", (getter)type_get_mro, NULL, NULL}, {"__module__", (getter)type_module, (setter)type_set_module, NULL}, {"__abstractmethods__", (getter)type_abstractmethods, (setter)type_set_abstractmethods, NULL}, @@ -4342,7 +4462,7 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) /* Look in tp_dict of types in MRO */ PyObject *mro = lookup_tp_mro(type); if (mro == NULL) { - if ((type->tp_flags & Py_TPFLAGS_READYING) == 0) { + if (!is_readying(type)) { if (PyType_Ready(type) < 0) { *error = -1; return NULL; @@ -4692,11 +4812,11 @@ static void clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type) { if (_Py_IsMainInterpreter(interp)) { - clear_tp_dict(type); - clear_tp_bases(type); - clear_tp_mro(type); Py_CLEAR(type->tp_cache); } + clear_tp_dict(type); + clear_tp_bases(type); + clear_tp_mro(type); clear_static_tp_subclasses(type); } @@ -6684,6 +6804,10 @@ type_ready_pre_checks(PyTypeObject *type) static int type_ready_set_bases(PyTypeObject *type) { + if (lookup_tp_bases(type) != NULL) { + return 0; + } + /* Initialize tp_base (defaults to BaseObject unless that's us) */ PyTypeObject *base = type->tp_base; if (base == NULL && type != &PyBaseObject_Type) { @@ -6997,7 +7121,7 @@ type_ready_add_subclasses(PyTypeObject *type) // Set tp_new and the "__new__" key in the type dictionary. // Use the Py_TPFLAGS_DISALLOW_INSTANTIATION flag. static int -type_ready_set_new(PyTypeObject *type) +type_ready_set_new(PyTypeObject *type, int rerunbuiltin) { PyTypeObject *base = type->tp_base; /* The condition below could use some explanation. @@ -7019,10 +7143,12 @@ type_ready_set_new(PyTypeObject *type) if (!(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION)) { if (type->tp_new != NULL) { - // If "__new__" key does not exists in the type dictionary, - // set it to tp_new_wrapper(). - if (add_tp_new_wrapper(type) < 0) { - return -1; + if (!rerunbuiltin || base == NULL || type->tp_new != base->tp_new) { + // If "__new__" key does not exists in the type dictionary, + // set it to tp_new_wrapper(). + if (add_tp_new_wrapper(type) < 0) { + return -1; + } } } else { @@ -7096,11 +7222,10 @@ type_ready_post_checks(PyTypeObject *type) static int -type_ready(PyTypeObject *type) +type_ready(PyTypeObject *type, int rerunbuiltin) { - _PyObject_ASSERT((PyObject *)type, - (type->tp_flags & Py_TPFLAGS_READYING) == 0); - type->tp_flags |= Py_TPFLAGS_READYING; + _PyObject_ASSERT((PyObject *)type, !is_readying(type)); + start_readying(type); if (type_ready_pre_checks(type) < 0) { goto error; @@ -7125,17 +7250,19 @@ type_ready(PyTypeObject *type) if (type_ready_mro(type) < 0) { goto error; } - if (type_ready_set_new(type) < 0) { + if (type_ready_set_new(type, rerunbuiltin) < 0) { goto error; } if (type_ready_fill_dict(type) < 0) { goto error; } - if (type_ready_inherit(type) < 0) { - goto error; - } - if (type_ready_preheader(type) < 0) { - goto error; + if (!rerunbuiltin) { + if (type_ready_inherit(type) < 0) { + goto error; + } + if (type_ready_preheader(type) < 0) { + goto error; + } } if (type_ready_set_hash(type) < 0) { goto error; @@ -7143,21 +7270,24 @@ type_ready(PyTypeObject *type) if (type_ready_add_subclasses(type) < 0) { goto error; } - if (type_ready_managed_dict(type) < 0) { - goto error; - } - if (type_ready_post_checks(type) < 0) { - goto error; + if (!rerunbuiltin) { + if (type_ready_managed_dict(type) < 0) { + goto error; + } + if (type_ready_post_checks(type) < 0) { + goto error; + } } /* All done -- set the ready flag */ - type->tp_flags = (type->tp_flags & ~Py_TPFLAGS_READYING) | Py_TPFLAGS_READY; + type->tp_flags = type->tp_flags | Py_TPFLAGS_READY; + stop_readying(type); assert(_PyType_CheckConsistency(type)); return 0; error: - type->tp_flags &= ~Py_TPFLAGS_READYING; + stop_readying(type); return -1; } @@ -7175,7 +7305,7 @@ PyType_Ready(PyTypeObject *type) type->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; } - return type_ready(type); + return type_ready(type, 0); } int @@ -7186,35 +7316,26 @@ _PyStaticType_InitBuiltin(PyInterpreterState *interp, PyTypeObject *self) assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); -#ifndef NDEBUG int ismain = _Py_IsMainInterpreter(interp); -#endif - if (self->tp_flags & Py_TPFLAGS_READY) { + if ((self->tp_flags & Py_TPFLAGS_READY) == 0) { + assert(ismain); + + self->tp_flags |= _Py_TPFLAGS_STATIC_BUILTIN; + self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; + + assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); + self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; + self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; + } + else { assert(!ismain); assert(self->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN); assert(self->tp_flags & Py_TPFLAGS_VALID_VERSION_TAG); - - static_builtin_state_init(interp, self); - - /* Per-interpreter tp_subclasses is done lazily. - Otherwise we would initialize it here. */ - - assert(_PyType_CheckConsistency(self)); - return 0; } - assert(ismain); - - self->tp_flags |= _Py_TPFLAGS_STATIC_BUILTIN; - self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; - - assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); - self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; - self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; - static_builtin_state_init(interp, self); - int res = type_ready(self); + int res = type_ready(self, !ismain); if (res < 0) { static_builtin_state_clear(interp, self); } From 292076a9aa29aba1023340a0d24252a7b27a454e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 2 May 2023 21:40:00 -0600 Subject: [PATCH 07/65] gh-104109: Expose Py_NewInterpreterFromConfig() in the Public C-API (gh-104110) We also expose PyInterpreterConfig. This is part of the PEP 684 (per-interpreter GIL) implementation. We will add docs as soon as we can. FYI, I'm adding the new config field for per-interpreter GIL in gh-99114. --- Include/cpython/initconfig.h | 2 +- Include/cpython/pylifecycle.h | 4 ++-- ...2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst | 5 +++++ Modules/_testcapimodule.c | 4 ++-- Modules/_xxsubinterpretersmodule.c | 8 ++++---- Python/pylifecycle.c | 15 ++++++++------- 6 files changed, 22 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 79c1023baa9a0f..9c1783d272f1cd 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -252,7 +252,7 @@ typedef struct { int allow_threads; int allow_daemon_threads; int check_multi_interp_extensions; -} _PyInterpreterConfig; +} PyInterpreterConfig; #define _PyInterpreterConfig_INIT \ { \ diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index 79d55711319e55..08569ee683ce0d 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -62,9 +62,9 @@ PyAPI_FUNC(int) _Py_CoerceLegacyLocale(int warn); PyAPI_FUNC(int) _Py_LegacyLocaleDetected(int warn); PyAPI_FUNC(char *) _Py_SetLocaleFromEnv(int category); -PyAPI_FUNC(PyStatus) _Py_NewInterpreterFromConfig( +PyAPI_FUNC(PyStatus) Py_NewInterpreterFromConfig( PyThreadState **tstate_p, - const _PyInterpreterConfig *config); + const PyInterpreterConfig *config); typedef void (*atexit_datacallbackfunc)(void *); PyAPI_FUNC(int) _Py_AtExit( diff --git a/Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst b/Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst new file mode 100644 index 00000000000000..2ffc0fa81c014a --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-05-02-21-05-54.gh-issue-104109.0tnDZV.rst @@ -0,0 +1,5 @@ +We've added ``Py_NewInterpreterFromConfig()`` and ``PyInterpreterConfig`` to +the public C-API (but not the stable ABI; not yet at least). The new +function may be used to create a new interpreter with various features +configured. The function was added to support PEP 684 (per-interpreter +GIL). diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 30b8b6c6b3a87b..47e0ed9be8e709 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1538,7 +1538,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) PyThreadState_Swap(NULL); - const _PyInterpreterConfig config = { + const PyInterpreterConfig config = { .use_main_obmalloc = use_main_obmalloc, .allow_fork = allow_fork, .allow_exec = allow_exec, @@ -1546,7 +1546,7 @@ run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) .allow_daemon_threads = allow_daemon_threads, .check_multi_interp_extensions = check_multi_interp_extensions, }; - PyStatus status = _Py_NewInterpreterFromConfig(&substate, &config); + PyStatus status = Py_NewInterpreterFromConfig(&substate, &config); if (PyStatus_Exception(status)) { /* Since no new thread state was created, there is no exception to propagate; raise a fresh one after swapping in the old thread diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 884fb0d31f2b7f..95273ab278d996 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -513,12 +513,12 @@ interp_create(PyObject *self, PyObject *args, PyObject *kwds) // Create and initialize the new interpreter. PyThreadState *save_tstate = _PyThreadState_GET(); - const _PyInterpreterConfig config = isolated - ? (_PyInterpreterConfig)_PyInterpreterConfig_INIT - : (_PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; + const PyInterpreterConfig config = isolated + ? (PyInterpreterConfig)_PyInterpreterConfig_INIT + : (PyInterpreterConfig)_PyInterpreterConfig_LEGACY_INIT; // XXX Possible GILState issues? PyThreadState *tstate = NULL; - PyStatus status = _Py_NewInterpreterFromConfig(&tstate, &config); + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); PyThreadState_Swap(save_tstate); if (PyStatus_Exception(status)) { /* Since no new thread state was created, there is no exception to diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index b8a115236900b9..b9add89b9c6c52 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -546,7 +546,8 @@ pycore_init_runtime(_PyRuntimeState *runtime, static PyStatus -init_interp_settings(PyInterpreterState *interp, const _PyInterpreterConfig *config) +init_interp_settings(PyInterpreterState *interp, + const PyInterpreterConfig *config) { assert(interp->feature_flags == 0); @@ -631,7 +632,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return status; } - const _PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; + const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; status = init_interp_settings(interp, &config); if (_PyStatus_EXCEPTION(status)) { return status; @@ -1991,7 +1992,7 @@ Py_Finalize(void) */ static PyStatus -new_interpreter(PyThreadState **tstate_p, const _PyInterpreterConfig *config) +new_interpreter(PyThreadState **tstate_p, const PyInterpreterConfig *config) { PyStatus status; @@ -2079,8 +2080,8 @@ new_interpreter(PyThreadState **tstate_p, const _PyInterpreterConfig *config) } PyStatus -_Py_NewInterpreterFromConfig(PyThreadState **tstate_p, - const _PyInterpreterConfig *config) +Py_NewInterpreterFromConfig(PyThreadState **tstate_p, + const PyInterpreterConfig *config) { return new_interpreter(tstate_p, config); } @@ -2089,8 +2090,8 @@ PyThreadState * Py_NewInterpreter(void) { PyThreadState *tstate = NULL; - const _PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; - PyStatus status = _Py_NewInterpreterFromConfig(&tstate, &config); + const PyInterpreterConfig config = _PyInterpreterConfig_LEGACY_INIT; + PyStatus status = new_interpreter(&tstate, &config); if (_PyStatus_EXCEPTION(status)) { Py_ExitStatusException(status); } From c7c3a60c88de61a79ded9fdaf6bc6a29da4efb9a Mon Sep 17 00:00:00 2001 From: Ethan Furman Date: Tue, 2 May 2023 20:42:00 -0700 Subject: [PATCH 08/65] gh-104049: do not expose on-disk location from SimpleHTTPRequestHandler (#104067) Do not expose the local server's on-disk location from `SimpleHTTPRequestHandler` when generating a directory index. (unnecessary information disclosure) --------- Co-authored-by: Gregory P. Smith Co-authored-by: Jelle Zijlstra --- Lib/http/server.py | 2 +- Lib/test/test_httpservers.py | 8 ++++++++ .../2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst | 2 ++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst diff --git a/Lib/http/server.py b/Lib/http/server.py index 971f08046d50b5..a245ffb307860a 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -791,7 +791,7 @@ def list_directory(self, path): displaypath = urllib.parse.unquote(self.path, errors='surrogatepass') except UnicodeDecodeError: - displaypath = urllib.parse.unquote(path) + displaypath = urllib.parse.unquote(self.path) displaypath = html.escape(displaypath, quote=False) enc = sys.getfilesystemencoding() title = f'Directory listing for {displaypath}' diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index cbcf94136ac4eb..0382b5ec448d57 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -418,6 +418,14 @@ def test_undecodable_filename(self): self.check_status_and_reason(response, HTTPStatus.OK, data=os_helper.TESTFN_UNDECODABLE) + def test_undecodable_parameter(self): + # sanity check using a valid parameter + response = self.request(self.base_url + '/?x=123').read() + self.assertRegex(response, f'listing for {self.base_url}/\?x=123'.encode('latin1')) + # now the bogus encoding + response = self.request(self.base_url + '/?x=%bb').read() + self.assertRegex(response, f'listing for {self.base_url}/\?x=\xef\xbf\xbd'.encode('latin1')) + def test_get_dir_redirect_location_domain_injection_bug(self): """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location. diff --git a/Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst b/Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst new file mode 100644 index 00000000000000..969deb26bfeb95 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2023-05-01-15-03-25.gh-issue-104049.b01Y3g.rst @@ -0,0 +1,2 @@ +Do not expose the local on-disk location in directory indexes +produced by :class:`http.client.SimpleHTTPRequestHandler`. From af886ffa0612124598b5e8174c3b1be0f60eab38 Mon Sep 17 00:00:00 2001 From: andrei kulakov Date: Tue, 2 May 2023 23:50:10 -0400 Subject: [PATCH 09/65] GH-89769: `pathlib.Path.glob()`: do not follow symlinks when checking for precise match (GH-29655) Co-authored-by: Barney Gale --- Doc/library/pathlib.rst | 15 +++++++++------ Lib/pathlib.py | 10 +++++++--- Lib/test/test_pathlib.py | 4 ++++ .../2021-11-19-23-37-18.bpo-45606.UW5XE1.rst | 5 +++++ 4 files changed, 25 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 8e91936680fab8..4847ac24c77513 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -819,9 +819,14 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.10 The *follow_symlinks* parameter was added. -.. method:: Path.exists() +.. method:: Path.exists(*, follow_symlinks=True) - Whether the path points to an existing file or directory:: + Return ``True`` if the path points to an existing file or directory. + + This method normally follows symlinks; to check if a symlink exists, add + the argument ``follow_symlinks=False``. + + :: >>> Path('.').exists() True @@ -832,10 +837,8 @@ call fails (for example because the path doesn't exist). >>> Path('nonexistentfile').exists() False - .. note:: - If the path points to a symlink, :meth:`exists` returns whether the - symlink *points to* an existing file or directory. - + .. versionchanged:: 3.12 + The *follow_symlinks* parameter was added. .. method:: Path.expanduser() diff --git a/Lib/pathlib.py b/Lib/pathlib.py index c69089f4e1bc5d..dee19d1f89ad84 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -135,7 +135,8 @@ def __init__(self, name, child_parts, flavour): def _select_from(self, parent_path, is_dir, exists, scandir): try: path = parent_path._make_child_relpath(self.name) - if (is_dir if self.dironly else exists)(path): + follow = is_dir(path) if self.dironly else exists(path, follow_symlinks=False) + if follow: for p in self.successor._select_from(path, is_dir, exists, scandir): yield p except PermissionError: @@ -1122,12 +1123,15 @@ def hardlink_to(self, target): # Convenience functions for querying the stat results - def exists(self): + def exists(self, *, follow_symlinks=True): """ Whether this path exists. + + This method normally follows symlinks; to check whether a symlink exists, + add the argument follow_symlinks=False. """ try: - self.stat() + self.stat(follow_symlinks=follow_symlinks) except OSError as e: if not _ignore_error(e): raise diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 9902b7242205f3..620d480e37e2ba 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1700,6 +1700,8 @@ def test_exists(self): self.assertIs(True, (p / 'linkB').exists()) self.assertIs(True, (p / 'linkB' / 'fileB').exists()) self.assertIs(False, (p / 'linkA' / 'bah').exists()) + self.assertIs(False, (p / 'brokenLink').exists()) + self.assertIs(True, (p / 'brokenLink').exists(follow_symlinks=False)) self.assertIs(False, (p / 'foo').exists()) self.assertIs(False, P('/xyzzy').exists()) self.assertIs(False, P(BASE + '\udfff').exists()) @@ -1806,6 +1808,8 @@ def _check(glob, expected): _check(p.glob("*/fileB"), ['dirB/fileB']) else: _check(p.glob("*/fileB"), ['dirB/fileB', 'linkB/fileB']) + if os_helper.can_symlink(): + _check(p.glob("brokenLink"), ['brokenLink']) if not os_helper.can_symlink(): _check(p.glob("*/"), ["dirA", "dirB", "dirC", "dirE"]) diff --git a/Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst b/Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst new file mode 100644 index 00000000000000..531f4729220036 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-11-19-23-37-18.bpo-45606.UW5XE1.rst @@ -0,0 +1,5 @@ +Fixed the bug in :meth:`pathlib.Path.glob` -- previously a dangling symlink +would not be found by this method when the pattern is an exact match, but +would be found when the pattern contains a wildcard or the recursive +wildcard (``**``). With this change, a dangling symlink will be found in +both cases. From 5b05b013ff13032ffc4db07108a507c08b3a604d Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Wed, 3 May 2023 08:09:04 +0300 Subject: [PATCH 10/65] gh-101100: Fix Sphinx warnings in `curses` and `curses.ascii` modules (#103457) Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/howto/curses.rst | 44 +- Doc/library/curses.ascii.rst | 150 +++---- Doc/library/curses.rst | 774 ++++++++++++++++++----------------- 3 files changed, 500 insertions(+), 468 deletions(-) diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 83d80471ffc8ee..a3068d86d85bc4 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -4,6 +4,8 @@ Curses Programming with Python ********************************** +.. currentmodule:: curses + :Author: A.M. Kuchling, Eric S. Raymond :Release: 2.04 @@ -65,7 +67,7 @@ The Python module is a fairly simple wrapper over the C functions provided by curses; if you're already familiar with curses programming in C, it's really easy to transfer that knowledge to Python. The biggest difference is that the Python interface makes things simpler by merging different C functions such as -:c:func:`addstr`, :c:func:`mvaddstr`, and :c:func:`mvwaddstr` into a single +:c:func:`!addstr`, :c:func:`!mvaddstr`, and :c:func:`!mvwaddstr` into a single :meth:`~curses.window.addstr` method. You'll see this covered in more detail later. @@ -82,7 +84,7 @@ Before doing anything, curses must be initialized. This is done by calling the :func:`~curses.initscr` function, which will determine the terminal type, send any required setup codes to the terminal, and create various internal data structures. If successful, -:func:`initscr` returns a window object representing the entire +:func:`!initscr` returns a window object representing the entire screen; this is usually called ``stdscr`` after the name of the corresponding C variable. :: @@ -151,8 +153,8 @@ importing the :func:`curses.wrapper` function and using it like this:: The :func:`~curses.wrapper` function takes a callable object and does the initializations described above, also initializing colors if color -support is present. :func:`wrapper` then runs your provided callable. -Once the callable returns, :func:`wrapper` will restore the original +support is present. :func:`!wrapper` then runs your provided callable. +Once the callable returns, :func:`!wrapper` will restore the original state of the terminal. The callable is called inside a :keyword:`try`...\ :keyword:`except` that catches exceptions, restores the state of the terminal, and then re-raises the exception. Therefore @@ -200,7 +202,7 @@ This is because curses was originally written with slow 300-baud terminal connections in mind; with these terminals, minimizing the time required to redraw the screen was very important. Instead curses accumulates changes to the screen and displays them in the most -efficient manner when you call :meth:`refresh`. For example, if your +efficient manner when you call :meth:`!refresh`. For example, if your program displays some text in a window and then clears the window, there's no need to send the original text because they're never visible. @@ -210,7 +212,7 @@ really complicate programming with curses much. Most programs go into a flurry of activity, and then pause waiting for a keypress or some other action on the part of the user. All you have to do is to be sure that the screen has been redrawn before pausing to wait for user input, by first calling -``stdscr.refresh()`` or the :meth:`refresh` method of some other relevant +:meth:`!stdscr.refresh` or the :meth:`!refresh` method of some other relevant window. A pad is a special case of a window; it can be larger than the actual display @@ -234,7 +236,7 @@ displayed. :: # : filled with pad content. pad.refresh( 0,0, 5,5, 20,75) -The :meth:`refresh` call displays a section of the pad in the rectangle +The :meth:`!refresh` call displays a section of the pad in the rectangle extending from coordinate (5,5) to coordinate (20,75) on the screen; the upper left corner of the displayed section is coordinate (0,0) on the pad. Beyond that difference, pads are exactly like ordinary windows and support the same @@ -242,7 +244,7 @@ methods. If you have multiple windows and pads on screen there is a more efficient way to update the screen and prevent annoying screen flicker -as each part of the screen gets updated. :meth:`refresh` actually +as each part of the screen gets updated. :meth:`!refresh` actually does two things: 1) Calls the :meth:`~curses.window.noutrefresh` method of each window @@ -251,8 +253,8 @@ does two things: 2) Calls the function :func:`~curses.doupdate` function to change the physical screen to match the desired state recorded in the data structure. -Instead you can call :meth:`noutrefresh` on a number of windows to -update the data structure, and then call :func:`doupdate` to update +Instead you can call :meth:`!noutrefresh` on a number of windows to +update the data structure, and then call :func:`!doupdate` to update the screen. @@ -261,11 +263,11 @@ Displaying Text From a C programmer's point of view, curses may sometimes look like a twisty maze of functions, all subtly different. For example, -:c:func:`addstr` displays a string at the current cursor location in -the ``stdscr`` window, while :c:func:`mvaddstr` moves to a given y,x -coordinate first before displaying the string. :c:func:`waddstr` is just -like :c:func:`addstr`, but allows specifying a window to use instead of -using ``stdscr`` by default. :c:func:`mvwaddstr` allows specifying both +:c:func:`!addstr` displays a string at the current cursor location in +the ``stdscr`` window, while :c:func:`!mvaddstr` moves to a given y,x +coordinate first before displaying the string. :c:func:`!waddstr` is just +like :c:func:`!addstr`, but allows specifying a window to use instead of +using ``stdscr`` by default. :c:func:`!mvwaddstr` allows specifying both a window and a coordinate. Fortunately the Python interface hides all these details. ``stdscr`` @@ -298,7 +300,7 @@ the next subsection. The :meth:`~curses.window.addstr` method takes a Python string or bytestring as the value to be displayed. The contents of bytestrings are sent to the terminal as-is. Strings are encoded to bytes using -the value of the window's :attr:`encoding` attribute; this defaults to +the value of the window's :attr:`~window.encoding` attribute; this defaults to the default system encoding as returned by :func:`locale.getencoding`. The :meth:`~curses.window.addch` methods take a character, which can be @@ -444,15 +446,15 @@ There are two methods for getting input from a window: It's possible to not wait for the user using the :meth:`~curses.window.nodelay` window method. After ``nodelay(True)``, -:meth:`getch` and :meth:`getkey` for the window become -non-blocking. To signal that no input is ready, :meth:`getch` returns -``curses.ERR`` (a value of -1) and :meth:`getkey` raises an exception. +:meth:`!getch` and :meth:`!getkey` for the window become +non-blocking. To signal that no input is ready, :meth:`!getch` returns +``curses.ERR`` (a value of -1) and :meth:`!getkey` raises an exception. There's also a :func:`~curses.halfdelay` function, which can be used to (in -effect) set a timer on each :meth:`getch`; if no input becomes +effect) set a timer on each :meth:`!getch`; if no input becomes available within a specified delay (measured in tenths of a second), curses raises an exception. -The :meth:`getch` method returns an integer; if it's between 0 and 255, it +The :meth:`!getch` method returns an integer; if it's between 0 and 255, it represents the ASCII code of the key pressed. Values greater than 255 are special keys such as Page Up, Home, or the cursor keys. You can compare the value returned to constants such as :const:`curses.KEY_PPAGE`, diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index e1d1171927c9e2..410b76e77c025b 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -15,81 +15,81 @@ The :mod:`curses.ascii` module supplies name constants for ASCII characters and functions to test membership in various ASCII character classes. The constants supplied are names for control characters as follows: -+--------------+----------------------------------------------+ -| Name | Meaning | -+==============+==============================================+ -| :const:`NUL` | | -+--------------+----------------------------------------------+ -| :const:`SOH` | Start of heading, console interrupt | -+--------------+----------------------------------------------+ -| :const:`STX` | Start of text | -+--------------+----------------------------------------------+ -| :const:`ETX` | End of text | -+--------------+----------------------------------------------+ -| :const:`EOT` | End of transmission | -+--------------+----------------------------------------------+ -| :const:`ENQ` | Enquiry, goes with :const:`ACK` flow control | -+--------------+----------------------------------------------+ -| :const:`ACK` | Acknowledgement | -+--------------+----------------------------------------------+ -| :const:`BEL` | Bell | -+--------------+----------------------------------------------+ -| :const:`BS` | Backspace | -+--------------+----------------------------------------------+ -| :const:`TAB` | Tab | -+--------------+----------------------------------------------+ -| :const:`HT` | Alias for :const:`TAB`: "Horizontal tab" | -+--------------+----------------------------------------------+ -| :const:`LF` | Line feed | -+--------------+----------------------------------------------+ -| :const:`NL` | Alias for :const:`LF`: "New line" | -+--------------+----------------------------------------------+ -| :const:`VT` | Vertical tab | -+--------------+----------------------------------------------+ -| :const:`FF` | Form feed | -+--------------+----------------------------------------------+ -| :const:`CR` | Carriage return | -+--------------+----------------------------------------------+ -| :const:`SO` | Shift-out, begin alternate character set | -+--------------+----------------------------------------------+ -| :const:`SI` | Shift-in, resume default character set | -+--------------+----------------------------------------------+ -| :const:`DLE` | Data-link escape | -+--------------+----------------------------------------------+ -| :const:`DC1` | XON, for flow control | -+--------------+----------------------------------------------+ -| :const:`DC2` | Device control 2, block-mode flow control | -+--------------+----------------------------------------------+ -| :const:`DC3` | XOFF, for flow control | -+--------------+----------------------------------------------+ -| :const:`DC4` | Device control 4 | -+--------------+----------------------------------------------+ -| :const:`NAK` | Negative acknowledgement | -+--------------+----------------------------------------------+ -| :const:`SYN` | Synchronous idle | -+--------------+----------------------------------------------+ -| :const:`ETB` | End transmission block | -+--------------+----------------------------------------------+ -| :const:`CAN` | Cancel | -+--------------+----------------------------------------------+ -| :const:`EM` | End of medium | -+--------------+----------------------------------------------+ -| :const:`SUB` | Substitute | -+--------------+----------------------------------------------+ -| :const:`ESC` | Escape | -+--------------+----------------------------------------------+ -| :const:`FS` | File separator | -+--------------+----------------------------------------------+ -| :const:`GS` | Group separator | -+--------------+----------------------------------------------+ -| :const:`RS` | Record separator, block-mode terminator | -+--------------+----------------------------------------------+ -| :const:`US` | Unit separator | -+--------------+----------------------------------------------+ -| :const:`SP` | Space | -+--------------+----------------------------------------------+ -| :const:`DEL` | Delete | -+--------------+----------------------------------------------+ ++---------------+----------------------------------------------+ +| Name | Meaning | ++===============+==============================================+ +| .. data:: NUL | | ++---------------+----------------------------------------------+ +| .. data:: SOH | Start of heading, console interrupt | ++---------------+----------------------------------------------+ +| .. data:: STX | Start of text | ++---------------+----------------------------------------------+ +| .. data:: ETX | End of text | ++---------------+----------------------------------------------+ +| .. data:: EOT | End of transmission | ++---------------+----------------------------------------------+ +| .. data:: ENQ | Enquiry, goes with :const:`ACK` flow control | ++---------------+----------------------------------------------+ +| .. data:: ACK | Acknowledgement | ++---------------+----------------------------------------------+ +| .. data:: BEL | Bell | ++---------------+----------------------------------------------+ +| .. data:: BS | Backspace | ++---------------+----------------------------------------------+ +| .. data:: TAB | Tab | ++---------------+----------------------------------------------+ +| .. data:: HT | Alias for :const:`TAB`: "Horizontal tab" | ++---------------+----------------------------------------------+ +| .. data:: LF | Line feed | ++---------------+----------------------------------------------+ +| .. data:: NL | Alias for :const:`LF`: "New line" | ++---------------+----------------------------------------------+ +| .. data:: VT | Vertical tab | ++---------------+----------------------------------------------+ +| .. data:: FF | Form feed | ++---------------+----------------------------------------------+ +| .. data:: CR | Carriage return | ++---------------+----------------------------------------------+ +| .. data:: SO | Shift-out, begin alternate character set | ++---------------+----------------------------------------------+ +| .. data:: SI | Shift-in, resume default character set | ++---------------+----------------------------------------------+ +| .. data:: DLE | Data-link escape | ++---------------+----------------------------------------------+ +| .. data:: DC1 | XON, for flow control | ++---------------+----------------------------------------------+ +| .. data:: DC2 | Device control 2, block-mode flow control | ++---------------+----------------------------------------------+ +| .. data:: DC3 | XOFF, for flow control | ++---------------+----------------------------------------------+ +| .. data:: DC4 | Device control 4 | ++---------------+----------------------------------------------+ +| .. data:: NAK | Negative acknowledgement | ++---------------+----------------------------------------------+ +| .. data:: SYN | Synchronous idle | ++---------------+----------------------------------------------+ +| .. data:: ETB | End transmission block | ++---------------+----------------------------------------------+ +| .. data:: CAN | Cancel | ++---------------+----------------------------------------------+ +| .. data:: EM | End of medium | ++---------------+----------------------------------------------+ +| .. data:: SUB | Substitute | ++---------------+----------------------------------------------+ +| .. data:: ESC | Escape | ++---------------+----------------------------------------------+ +| .. data:: FS | File separator | ++---------------+----------------------------------------------+ +| .. data:: GS | Group separator | ++---------------+----------------------------------------------+ +| .. data:: RS | Record separator, block-mode terminator | ++---------------+----------------------------------------------+ +| .. data:: US | Unit separator | ++---------------+----------------------------------------------+ +| .. data:: SP | Space | ++---------------+----------------------------------------------+ +| .. data:: DEL | Delete | ++---------------+----------------------------------------------+ Note that many of these have little practical significance in modern usage. The mnemonics derive from teleprinter conventions that predate digital computers. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index f50b51c3780ef0..cf208f3ba0db36 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -107,7 +107,7 @@ The module :mod:`curses` defines the following functions: Return the attribute value for displaying text in the specified color pair. Only the first 256 color pairs are supported. This attribute value can be combined with :const:`A_STANDOUT`, :const:`A_REVERSE`, - and the other :const:`A_\*` attributes. :func:`pair_number` is the counterpart + and the other :const:`!A_\*` attributes. :func:`pair_number` is the counterpart to this function. @@ -223,7 +223,7 @@ The module :mod:`curses` defines the following functions: .. function:: getwin(file) - Read window related data stored in the file by an earlier :func:`putwin` call. + Read window related data stored in the file by an earlier :func:`window.putwin` call. The routine then creates and initializes a new window using that data, returning the new window object. @@ -1323,9 +1323,9 @@ The :mod:`curses` module defines the following data members: .. data:: version +.. data:: __version__ - A bytes object representing the current version of the module. Also available as - :const:`__version__`. + A bytes object representing the current version of the module. .. data:: ncurses_version @@ -1339,51 +1339,55 @@ The :mod:`curses` module defines the following data members: .. versionadded:: 3.8 +.. data:: COLORS + + The maximum number of colors the terminal can support. + +.. data:: COLOR_PAIRS + + The maximum number of color pairs the terminal can support. Some constants are available to specify character cell attributes. The exact constants available are system dependent. -+------------------+-------------------------------+ -| Attribute | Meaning | -+==================+===============================+ -| ``A_ALTCHARSET`` | Alternate character set mode | -+------------------+-------------------------------+ -| ``A_BLINK`` | Blink mode | -+------------------+-------------------------------+ -| ``A_BOLD`` | Bold mode | -+------------------+-------------------------------+ -| ``A_DIM`` | Dim mode | -+------------------+-------------------------------+ -| ``A_INVIS`` | Invisible or blank mode | -+------------------+-------------------------------+ -| ``A_ITALIC`` | Italic mode | -+------------------+-------------------------------+ -| ``A_NORMAL`` | Normal attribute | -+------------------+-------------------------------+ -| ``A_PROTECT`` | Protected mode | -+------------------+-------------------------------+ -| ``A_REVERSE`` | Reverse background and | -| | foreground colors | -+------------------+-------------------------------+ -| ``A_STANDOUT`` | Standout mode | -+------------------+-------------------------------+ -| ``A_UNDERLINE`` | Underline mode | -+------------------+-------------------------------+ -| ``A_HORIZONTAL`` | Horizontal highlight | -+------------------+-------------------------------+ -| ``A_LEFT`` | Left highlight | -+------------------+-------------------------------+ -| ``A_LOW`` | Low highlight | -+------------------+-------------------------------+ -| ``A_RIGHT`` | Right highlight | -+------------------+-------------------------------+ -| ``A_TOP`` | Top highlight | -+------------------+-------------------------------+ -| ``A_VERTICAL`` | Vertical highlight | -+------------------+-------------------------------+ -| ``A_CHARTEXT`` | Bit-mask to extract a | -| | character | -+------------------+-------------------------------+ ++------------------------+-------------------------------+ +| Attribute | Meaning | ++========================+===============================+ +| .. data:: A_ALTCHARSET | Alternate character set mode | ++------------------------+-------------------------------+ +| .. data:: A_BLINK | Blink mode | ++------------------------+-------------------------------+ +| .. data:: A_BOLD | Bold mode | ++------------------------+-------------------------------+ +| .. data:: A_DIM | Dim mode | ++------------------------+-------------------------------+ +| .. data:: A_INVIS | Invisible or blank mode | ++------------------------+-------------------------------+ +| .. data:: A_ITALIC | Italic mode | ++------------------------+-------------------------------+ +| .. data:: A_NORMAL | Normal attribute | ++------------------------+-------------------------------+ +| .. data:: A_PROTECT | Protected mode | ++------------------------+-------------------------------+ +| .. data:: A_REVERSE | Reverse background and | +| | foreground colors | ++------------------------+-------------------------------+ +| .. data:: A_STANDOUT | Standout mode | ++------------------------+-------------------------------+ +| .. data:: A_UNDERLINE | Underline mode | ++------------------------+-------------------------------+ +| .. data:: A_HORIZONTAL | Horizontal highlight | ++------------------------+-------------------------------+ +| .. data:: A_LEFT | Left highlight | ++------------------------+-------------------------------+ +| .. data:: A_LOW | Low highlight | ++------------------------+-------------------------------+ +| .. data:: A_RIGHT | Right highlight | ++------------------------+-------------------------------+ +| .. data:: A_TOP | Top highlight | ++------------------------+-------------------------------+ +| .. data:: A_VERTICAL | Vertical highlight | ++------------------------+-------------------------------+ .. versionadded:: 3.7 ``A_ITALIC`` was added. @@ -1391,220 +1395,220 @@ The exact constants available are system dependent. Several constants are available to extract corresponding attributes returned by some methods. -+------------------+-------------------------------+ -| Bit-mask | Meaning | -+==================+===============================+ -| ``A_ATTRIBUTES`` | Bit-mask to extract | -| | attributes | -+------------------+-------------------------------+ -| ``A_CHARTEXT`` | Bit-mask to extract a | -| | character | -+------------------+-------------------------------+ -| ``A_COLOR`` | Bit-mask to extract | -| | color-pair field information | -+------------------+-------------------------------+ ++-------------------------+-------------------------------+ +| Bit-mask | Meaning | ++=========================+===============================+ +| .. data:: A_ATTRIBUTES | Bit-mask to extract | +| | attributes | ++-------------------------+-------------------------------+ +| .. data:: A_CHARTEXT | Bit-mask to extract a | +| | character | ++-------------------------+-------------------------------+ +| .. data:: A_COLOR | Bit-mask to extract | +| | color-pair field information | ++-------------------------+-------------------------------+ Keys are referred to by integer constants with names starting with ``KEY_``. The exact keycaps available are system dependent. .. XXX this table is far too large! should it be alphabetized? -+-------------------+--------------------------------------------+ -| Key constant | Key | -+===================+============================================+ -| ``KEY_MIN`` | Minimum key value | -+-------------------+--------------------------------------------+ -| ``KEY_BREAK`` | Break key (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_DOWN`` | Down-arrow | -+-------------------+--------------------------------------------+ -| ``KEY_UP`` | Up-arrow | -+-------------------+--------------------------------------------+ -| ``KEY_LEFT`` | Left-arrow | -+-------------------+--------------------------------------------+ -| ``KEY_RIGHT`` | Right-arrow | -+-------------------+--------------------------------------------+ -| ``KEY_HOME`` | Home key (upward+left arrow) | -+-------------------+--------------------------------------------+ -| ``KEY_BACKSPACE`` | Backspace (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_F0`` | Function keys. Up to 64 function keys are | -| | supported. | -+-------------------+--------------------------------------------+ -| ``KEY_Fn`` | Value of function key *n* | -+-------------------+--------------------------------------------+ -| ``KEY_DL`` | Delete line | -+-------------------+--------------------------------------------+ -| ``KEY_IL`` | Insert line | -+-------------------+--------------------------------------------+ -| ``KEY_DC`` | Delete character | -+-------------------+--------------------------------------------+ -| ``KEY_IC`` | Insert char or enter insert mode | -+-------------------+--------------------------------------------+ -| ``KEY_EIC`` | Exit insert char mode | -+-------------------+--------------------------------------------+ -| ``KEY_CLEAR`` | Clear screen | -+-------------------+--------------------------------------------+ -| ``KEY_EOS`` | Clear to end of screen | -+-------------------+--------------------------------------------+ -| ``KEY_EOL`` | Clear to end of line | -+-------------------+--------------------------------------------+ -| ``KEY_SF`` | Scroll 1 line forward | -+-------------------+--------------------------------------------+ -| ``KEY_SR`` | Scroll 1 line backward (reverse) | -+-------------------+--------------------------------------------+ -| ``KEY_NPAGE`` | Next page | -+-------------------+--------------------------------------------+ -| ``KEY_PPAGE`` | Previous page | -+-------------------+--------------------------------------------+ -| ``KEY_STAB`` | Set tab | -+-------------------+--------------------------------------------+ -| ``KEY_CTAB`` | Clear tab | -+-------------------+--------------------------------------------+ -| ``KEY_CATAB`` | Clear all tabs | -+-------------------+--------------------------------------------+ -| ``KEY_ENTER`` | Enter or send (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_SRESET`` | Soft (partial) reset (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_RESET`` | Reset or hard reset (unreliable) | -+-------------------+--------------------------------------------+ -| ``KEY_PRINT`` | Print | -+-------------------+--------------------------------------------+ -| ``KEY_LL`` | Home down or bottom (lower left) | -+-------------------+--------------------------------------------+ -| ``KEY_A1`` | Upper left of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_A3`` | Upper right of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_B2`` | Center of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_C1`` | Lower left of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_C3`` | Lower right of keypad | -+-------------------+--------------------------------------------+ -| ``KEY_BTAB`` | Back tab | -+-------------------+--------------------------------------------+ -| ``KEY_BEG`` | Beg (beginning) | -+-------------------+--------------------------------------------+ -| ``KEY_CANCEL`` | Cancel | -+-------------------+--------------------------------------------+ -| ``KEY_CLOSE`` | Close | -+-------------------+--------------------------------------------+ -| ``KEY_COMMAND`` | Cmd (command) | -+-------------------+--------------------------------------------+ -| ``KEY_COPY`` | Copy | -+-------------------+--------------------------------------------+ -| ``KEY_CREATE`` | Create | -+-------------------+--------------------------------------------+ -| ``KEY_END`` | End | -+-------------------+--------------------------------------------+ -| ``KEY_EXIT`` | Exit | -+-------------------+--------------------------------------------+ -| ``KEY_FIND`` | Find | -+-------------------+--------------------------------------------+ -| ``KEY_HELP`` | Help | -+-------------------+--------------------------------------------+ -| ``KEY_MARK`` | Mark | -+-------------------+--------------------------------------------+ -| ``KEY_MESSAGE`` | Message | -+-------------------+--------------------------------------------+ -| ``KEY_MOVE`` | Move | -+-------------------+--------------------------------------------+ -| ``KEY_NEXT`` | Next | -+-------------------+--------------------------------------------+ -| ``KEY_OPEN`` | Open | -+-------------------+--------------------------------------------+ -| ``KEY_OPTIONS`` | Options | -+-------------------+--------------------------------------------+ -| ``KEY_PREVIOUS`` | Prev (previous) | -+-------------------+--------------------------------------------+ -| ``KEY_REDO`` | Redo | -+-------------------+--------------------------------------------+ -| ``KEY_REFERENCE`` | Ref (reference) | -+-------------------+--------------------------------------------+ -| ``KEY_REFRESH`` | Refresh | -+-------------------+--------------------------------------------+ -| ``KEY_REPLACE`` | Replace | -+-------------------+--------------------------------------------+ -| ``KEY_RESTART`` | Restart | -+-------------------+--------------------------------------------+ -| ``KEY_RESUME`` | Resume | -+-------------------+--------------------------------------------+ -| ``KEY_SAVE`` | Save | -+-------------------+--------------------------------------------+ -| ``KEY_SBEG`` | Shifted Beg (beginning) | -+-------------------+--------------------------------------------+ -| ``KEY_SCANCEL`` | Shifted Cancel | -+-------------------+--------------------------------------------+ -| ``KEY_SCOMMAND`` | Shifted Command | -+-------------------+--------------------------------------------+ -| ``KEY_SCOPY`` | Shifted Copy | -+-------------------+--------------------------------------------+ -| ``KEY_SCREATE`` | Shifted Create | -+-------------------+--------------------------------------------+ -| ``KEY_SDC`` | Shifted Delete char | -+-------------------+--------------------------------------------+ -| ``KEY_SDL`` | Shifted Delete line | -+-------------------+--------------------------------------------+ -| ``KEY_SELECT`` | Select | -+-------------------+--------------------------------------------+ -| ``KEY_SEND`` | Shifted End | -+-------------------+--------------------------------------------+ -| ``KEY_SEOL`` | Shifted Clear line | -+-------------------+--------------------------------------------+ -| ``KEY_SEXIT`` | Shifted Exit | -+-------------------+--------------------------------------------+ -| ``KEY_SFIND`` | Shifted Find | -+-------------------+--------------------------------------------+ -| ``KEY_SHELP`` | Shifted Help | -+-------------------+--------------------------------------------+ -| ``KEY_SHOME`` | Shifted Home | -+-------------------+--------------------------------------------+ -| ``KEY_SIC`` | Shifted Input | -+-------------------+--------------------------------------------+ -| ``KEY_SLEFT`` | Shifted Left arrow | -+-------------------+--------------------------------------------+ -| ``KEY_SMESSAGE`` | Shifted Message | -+-------------------+--------------------------------------------+ -| ``KEY_SMOVE`` | Shifted Move | -+-------------------+--------------------------------------------+ -| ``KEY_SNEXT`` | Shifted Next | -+-------------------+--------------------------------------------+ -| ``KEY_SOPTIONS`` | Shifted Options | -+-------------------+--------------------------------------------+ -| ``KEY_SPREVIOUS`` | Shifted Prev | -+-------------------+--------------------------------------------+ -| ``KEY_SPRINT`` | Shifted Print | -+-------------------+--------------------------------------------+ -| ``KEY_SREDO`` | Shifted Redo | -+-------------------+--------------------------------------------+ -| ``KEY_SREPLACE`` | Shifted Replace | -+-------------------+--------------------------------------------+ -| ``KEY_SRIGHT`` | Shifted Right arrow | -+-------------------+--------------------------------------------+ -| ``KEY_SRSUME`` | Shifted Resume | -+-------------------+--------------------------------------------+ -| ``KEY_SSAVE`` | Shifted Save | -+-------------------+--------------------------------------------+ -| ``KEY_SSUSPEND`` | Shifted Suspend | -+-------------------+--------------------------------------------+ -| ``KEY_SUNDO`` | Shifted Undo | -+-------------------+--------------------------------------------+ -| ``KEY_SUSPEND`` | Suspend | -+-------------------+--------------------------------------------+ -| ``KEY_UNDO`` | Undo | -+-------------------+--------------------------------------------+ -| ``KEY_MOUSE`` | Mouse event has occurred | -+-------------------+--------------------------------------------+ -| ``KEY_RESIZE`` | Terminal resize event | -+-------------------+--------------------------------------------+ -| ``KEY_MAX`` | Maximum key value | -+-------------------+--------------------------------------------+ ++-------------------------+--------------------------------------------+ +| Key constant | Key | ++=========================+============================================+ +| .. data:: KEY_MIN | Minimum key value | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_BREAK | Break key (unreliable) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_DOWN | Down-arrow | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_UP | Up-arrow | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_LEFT | Left-arrow | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_RIGHT | Right-arrow | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_HOME | Home key (upward+left arrow) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_BACKSPACE | Backspace (unreliable) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_F0 | Function keys. Up to 64 function keys are | +| | supported. | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_Fn | Value of function key *n* | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_DL | Delete line | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_IL | Insert line | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_DC | Delete character | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_IC | Insert char or enter insert mode | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_EIC | Exit insert char mode | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_CLEAR | Clear screen | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_EOS | Clear to end of screen | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_EOL | Clear to end of line | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SF | Scroll 1 line forward | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SR | Scroll 1 line backward (reverse) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_NPAGE | Next page | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_PPAGE | Previous page | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_STAB | Set tab | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_CTAB | Clear tab | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_CATAB | Clear all tabs | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_ENTER | Enter or send (unreliable) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SRESET | Soft (partial) reset (unreliable) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_RESET | Reset or hard reset (unreliable) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_PRINT | Print | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_LL | Home down or bottom (lower left) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_A1 | Upper left of keypad | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_A3 | Upper right of keypad | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_B2 | Center of keypad | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_C1 | Lower left of keypad | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_C3 | Lower right of keypad | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_BTAB | Back tab | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_BEG | Beg (beginning) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_CANCEL | Cancel | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_CLOSE | Close | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_COMMAND | Cmd (command) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_COPY | Copy | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_CREATE | Create | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_END | End | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_EXIT | Exit | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_FIND | Find | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_HELP | Help | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_MARK | Mark | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_MESSAGE | Message | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_MOVE | Move | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_NEXT | Next | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_OPEN | Open | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_OPTIONS | Options | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_PREVIOUS | Prev (previous) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_REDO | Redo | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_REFERENCE | Ref (reference) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_REFRESH | Refresh | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_REPLACE | Replace | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_RESTART | Restart | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_RESUME | Resume | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SAVE | Save | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SBEG | Shifted Beg (beginning) | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SCANCEL | Shifted Cancel | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SCOMMAND | Shifted Command | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SCOPY | Shifted Copy | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SCREATE | Shifted Create | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SDC | Shifted Delete char | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SDL | Shifted Delete line | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SELECT | Select | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SEND | Shifted End | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SEOL | Shifted Clear line | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SEXIT | Shifted Exit | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SFIND | Shifted Find | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SHELP | Shifted Help | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SHOME | Shifted Home | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SIC | Shifted Input | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SLEFT | Shifted Left arrow | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SMESSAGE | Shifted Message | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SMOVE | Shifted Move | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SNEXT | Shifted Next | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SOPTIONS | Shifted Options | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SPREVIOUS | Shifted Prev | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SPRINT | Shifted Print | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SREDO | Shifted Redo | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SREPLACE | Shifted Replace | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SRIGHT | Shifted Right arrow | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SRSUME | Shifted Resume | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SSAVE | Shifted Save | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SSUSPEND | Shifted Suspend | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SUNDO | Shifted Undo | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_SUSPEND | Suspend | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_UNDO | Undo | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_MOUSE | Mouse event has occurred | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_RESIZE | Terminal resize event | ++-------------------------+--------------------------------------------+ +| .. data:: KEY_MAX | Maximum key value | ++-------------------------+--------------------------------------------+ On VT100s and their software emulations, such as X terminal emulators, there are -normally at least four function keys (:const:`KEY_F1`, :const:`KEY_F2`, -:const:`KEY_F3`, :const:`KEY_F4`) available, and the arrow keys mapped to +normally at least four function keys (:const:`KEY_F1 `, :const:`KEY_F2 `, +:const:`KEY_F3 `, :const:`KEY_F4 `) available, and the arrow keys mapped to :const:`KEY_UP`, :const:`KEY_DOWN`, :const:`KEY_LEFT` and :const:`KEY_RIGHT` in the obvious way. If your machine has a PC keyboard, it is safe to expect arrow keys and twelve function keys (older PC keyboards may have only ten function @@ -1635,117 +1639,143 @@ falls back on a crude printable ASCII approximation. These are available only after :func:`initscr` has been called. -+------------------+------------------------------------------+ -| ACS code | Meaning | -+==================+==========================================+ -| ``ACS_BBSS`` | alternate name for upper right corner | -+------------------+------------------------------------------+ -| ``ACS_BLOCK`` | solid square block | -+------------------+------------------------------------------+ -| ``ACS_BOARD`` | board of squares | -+------------------+------------------------------------------+ -| ``ACS_BSBS`` | alternate name for horizontal line | -+------------------+------------------------------------------+ -| ``ACS_BSSB`` | alternate name for upper left corner | -+------------------+------------------------------------------+ -| ``ACS_BSSS`` | alternate name for top tee | -+------------------+------------------------------------------+ -| ``ACS_BTEE`` | bottom tee | -+------------------+------------------------------------------+ -| ``ACS_BULLET`` | bullet | -+------------------+------------------------------------------+ -| ``ACS_CKBOARD`` | checker board (stipple) | -+------------------+------------------------------------------+ -| ``ACS_DARROW`` | arrow pointing down | -+------------------+------------------------------------------+ -| ``ACS_DEGREE`` | degree symbol | -+------------------+------------------------------------------+ -| ``ACS_DIAMOND`` | diamond | -+------------------+------------------------------------------+ -| ``ACS_GEQUAL`` | greater-than-or-equal-to | -+------------------+------------------------------------------+ -| ``ACS_HLINE`` | horizontal line | -+------------------+------------------------------------------+ -| ``ACS_LANTERN`` | lantern symbol | -+------------------+------------------------------------------+ -| ``ACS_LARROW`` | left arrow | -+------------------+------------------------------------------+ -| ``ACS_LEQUAL`` | less-than-or-equal-to | -+------------------+------------------------------------------+ -| ``ACS_LLCORNER`` | lower left-hand corner | -+------------------+------------------------------------------+ -| ``ACS_LRCORNER`` | lower right-hand corner | -+------------------+------------------------------------------+ -| ``ACS_LTEE`` | left tee | -+------------------+------------------------------------------+ -| ``ACS_NEQUAL`` | not-equal sign | -+------------------+------------------------------------------+ -| ``ACS_PI`` | letter pi | -+------------------+------------------------------------------+ -| ``ACS_PLMINUS`` | plus-or-minus sign | -+------------------+------------------------------------------+ -| ``ACS_PLUS`` | big plus sign | -+------------------+------------------------------------------+ -| ``ACS_RARROW`` | right arrow | -+------------------+------------------------------------------+ -| ``ACS_RTEE`` | right tee | -+------------------+------------------------------------------+ -| ``ACS_S1`` | scan line 1 | -+------------------+------------------------------------------+ -| ``ACS_S3`` | scan line 3 | -+------------------+------------------------------------------+ -| ``ACS_S7`` | scan line 7 | -+------------------+------------------------------------------+ -| ``ACS_S9`` | scan line 9 | -+------------------+------------------------------------------+ -| ``ACS_SBBS`` | alternate name for lower right corner | -+------------------+------------------------------------------+ -| ``ACS_SBSB`` | alternate name for vertical line | -+------------------+------------------------------------------+ -| ``ACS_SBSS`` | alternate name for right tee | -+------------------+------------------------------------------+ -| ``ACS_SSBB`` | alternate name for lower left corner | -+------------------+------------------------------------------+ -| ``ACS_SSBS`` | alternate name for bottom tee | -+------------------+------------------------------------------+ -| ``ACS_SSSB`` | alternate name for left tee | -+------------------+------------------------------------------+ -| ``ACS_SSSS`` | alternate name for crossover or big plus | -+------------------+------------------------------------------+ -| ``ACS_STERLING`` | pound sterling | -+------------------+------------------------------------------+ -| ``ACS_TTEE`` | top tee | -+------------------+------------------------------------------+ -| ``ACS_UARROW`` | up arrow | -+------------------+------------------------------------------+ -| ``ACS_ULCORNER`` | upper left corner | -+------------------+------------------------------------------+ -| ``ACS_URCORNER`` | upper right corner | -+------------------+------------------------------------------+ -| ``ACS_VLINE`` | vertical line | -+------------------+------------------------------------------+ ++------------------------+------------------------------------------+ +| ACS code | Meaning | ++========================+==========================================+ +| .. data:: ACS_BBSS | alternate name for upper right corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_BLOCK | solid square block | ++------------------------+------------------------------------------+ +| .. data:: ACS_BOARD | board of squares | ++------------------------+------------------------------------------+ +| .. data:: ACS_BSBS | alternate name for horizontal line | ++------------------------+------------------------------------------+ +| .. data:: ACS_BSSB | alternate name for upper left corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_BSSS | alternate name for top tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_BTEE | bottom tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_BULLET | bullet | ++------------------------+------------------------------------------+ +| .. data:: ACS_CKBOARD | checker board (stipple) | ++------------------------+------------------------------------------+ +| .. data:: ACS_DARROW | arrow pointing down | ++------------------------+------------------------------------------+ +| .. data:: ACS_DEGREE | degree symbol | ++------------------------+------------------------------------------+ +| .. data:: ACS_DIAMOND | diamond | ++------------------------+------------------------------------------+ +| .. data:: ACS_GEQUAL | greater-than-or-equal-to | ++------------------------+------------------------------------------+ +| .. data:: ACS_HLINE | horizontal line | ++------------------------+------------------------------------------+ +| .. data:: ACS_LANTERN | lantern symbol | ++------------------------+------------------------------------------+ +| .. data:: ACS_LARROW | left arrow | ++------------------------+------------------------------------------+ +| .. data:: ACS_LEQUAL | less-than-or-equal-to | ++------------------------+------------------------------------------+ +| .. data:: ACS_LLCORNER | lower left-hand corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_LRCORNER | lower right-hand corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_LTEE | left tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_NEQUAL | not-equal sign | ++------------------------+------------------------------------------+ +| .. data:: ACS_PI | letter pi | ++------------------------+------------------------------------------+ +| .. data:: ACS_PLMINUS | plus-or-minus sign | ++------------------------+------------------------------------------+ +| .. data:: ACS_PLUS | big plus sign | ++------------------------+------------------------------------------+ +| .. data:: ACS_RARROW | right arrow | ++------------------------+------------------------------------------+ +| .. data:: ACS_RTEE | right tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_S1 | scan line 1 | ++------------------------+------------------------------------------+ +| .. data:: ACS_S3 | scan line 3 | ++------------------------+------------------------------------------+ +| .. data:: ACS_S7 | scan line 7 | ++------------------------+------------------------------------------+ +| .. data:: ACS_S9 | scan line 9 | ++------------------------+------------------------------------------+ +| .. data:: ACS_SBBS | alternate name for lower right corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_SBSB | alternate name for vertical line | ++------------------------+------------------------------------------+ +| .. data:: ACS_SBSS | alternate name for right tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_SSBB | alternate name for lower left corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_SSBS | alternate name for bottom tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_SSSB | alternate name for left tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_SSSS | alternate name for crossover or big plus | ++------------------------+------------------------------------------+ +| .. data:: ACS_STERLING | pound sterling | ++------------------------+------------------------------------------+ +| .. data:: ACS_TTEE | top tee | ++------------------------+------------------------------------------+ +| .. data:: ACS_UARROW | up arrow | ++------------------------+------------------------------------------+ +| .. data:: ACS_ULCORNER | upper left corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_URCORNER | upper right corner | ++------------------------+------------------------------------------+ +| .. data:: ACS_VLINE | vertical line | ++------------------------+------------------------------------------+ + +The following table lists mouse button constants used by :meth:`getmouse`: + ++----------------------------------+---------------------------------------------+ +| Mouse button constant | Meaning | ++==================================+=============================================+ +| .. data:: BUTTONn_PRESSED | Mouse button *n* pressed | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTONn_RELEASED | Mouse button *n* released | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTONn_CLICKED | Mouse button *n* clicked | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTONn_DOUBLE_CLICKED | Mouse button *n* double clicked | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTONn_TRIPLE_CLICKED | Mouse button *n* triple clicked | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTON_SHIFT | Shift was down during button state change | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTON_CTRL | Control was down during button state change | ++----------------------------------+---------------------------------------------+ +| .. data:: BUTTON_ALT | Control was down during button state change | ++----------------------------------+---------------------------------------------+ + + .. versionchanged:: 3.10 + The ``BUTTON5_*`` constants are now exposed if they are provided by the + underlying curses library. The following table lists the predefined colors: -+-------------------+----------------------------+ -| Constant | Color | -+===================+============================+ -| ``COLOR_BLACK`` | Black | -+-------------------+----------------------------+ -| ``COLOR_BLUE`` | Blue | -+-------------------+----------------------------+ -| ``COLOR_CYAN`` | Cyan (light greenish blue) | -+-------------------+----------------------------+ -| ``COLOR_GREEN`` | Green | -+-------------------+----------------------------+ -| ``COLOR_MAGENTA`` | Magenta (purplish red) | -+-------------------+----------------------------+ -| ``COLOR_RED`` | Red | -+-------------------+----------------------------+ -| ``COLOR_WHITE`` | White | -+-------------------+----------------------------+ -| ``COLOR_YELLOW`` | Yellow | -+-------------------+----------------------------+ ++-------------------------+----------------------------+ +| Constant | Color | ++=========================+============================+ +| .. data:: COLOR_BLACK | Black | ++-------------------------+----------------------------+ +| .. data:: COLOR_BLUE | Blue | ++-------------------------+----------------------------+ +| .. data:: COLOR_CYAN | Cyan (light greenish blue) | ++-------------------------+----------------------------+ +| .. data:: COLOR_GREEN | Green | ++-------------------------+----------------------------+ +| .. data:: COLOR_MAGENTA | Magenta (purplish red) | ++-------------------------+----------------------------+ +| .. data:: COLOR_RED | Red | ++-------------------------+----------------------------+ +| .. data:: COLOR_WHITE | White | ++-------------------------+----------------------------+ +| .. data:: COLOR_YELLOW | Yellow | ++-------------------------+----------------------------+ :mod:`curses.textpad` --- Text input widget for curses programs @@ -1851,19 +1881,19 @@ You can instantiate a :class:`Textbox` object as follows: Move operations do nothing if the cursor is at an edge where the movement is not possible. The following synonyms are supported where possible: - +------------------------+------------------+ - | Constant | Keystroke | - +========================+==================+ - | :const:`KEY_LEFT` | :kbd:`Control-B` | - +------------------------+------------------+ - | :const:`KEY_RIGHT` | :kbd:`Control-F` | - +------------------------+------------------+ - | :const:`KEY_UP` | :kbd:`Control-P` | - +------------------------+------------------+ - | :const:`KEY_DOWN` | :kbd:`Control-N` | - +------------------------+------------------+ - | :const:`KEY_BACKSPACE` | :kbd:`Control-h` | - +------------------------+------------------+ + +--------------------------------+------------------+ + | Constant | Keystroke | + +================================+==================+ + | :const:`~curses.KEY_LEFT` | :kbd:`Control-B` | + +--------------------------------+------------------+ + | :const:`~curses.KEY_RIGHT` | :kbd:`Control-F` | + +--------------------------------+------------------+ + | :const:`~curses.KEY_UP` | :kbd:`Control-P` | + +--------------------------------+------------------+ + | :const:`~curses.KEY_DOWN` | :kbd:`Control-N` | + +--------------------------------+------------------+ + | :const:`~curses.KEY_BACKSPACE` | :kbd:`Control-h` | + +--------------------------------+------------------+ All other keystrokes are treated as a command to insert the given character and move right (with line wrapping). From fdb3ef8c0f94c7e55870a585dc6499aca46f9f90 Mon Sep 17 00:00:00 2001 From: Tim Hoffmann <2836374+timhoffm@users.noreply.github.com> Date: Wed, 3 May 2023 09:00:42 +0200 Subject: [PATCH 11/65] gh-82012: Deprecate bitwise inversion (~) of bool (#103487) The bitwise inversion operator on bool returns the bitwise inversion of the underlying int value; i.e. `~True == -2` such that `bool(~True) == True`. It's a common pitfall that users mistake `~` as negation operator and actually want `not`. Supporting `~` is an artifact of bool inheriting from int. Since there is no real use-case for the current behavior, let's deprecate `~` on bool and later raise an error. This removes a potential source errors for users. Full reasoning: https://github.com/python/cpython/issues/82012#issuecomment-1258705971 Co-authored-by: Jelle Zijlstra Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/library/functions.rst | 2 +- Doc/library/stdtypes.rst | 54 +++++++++++-------- Doc/whatsnew/3.12.rst | 6 +++ Lib/test/test_bool.py | 18 ++++++- ...3-04-12-19-55-24.gh-issue-82012.FlcJAh.rst | 5 ++ Objects/boolobject.c | 18 ++++++- 6 files changed, 78 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index a5e86ef0f9eb59..085a11c3caa708 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -147,7 +147,7 @@ are always available. They are listed here in alphabetical order. or omitted, this returns ``False``; otherwise, it returns ``True``. The :class:`bool` class is a subclass of :class:`int` (see :ref:`typesnumeric`). It cannot be subclassed further. Its only instances are ``False`` and - ``True`` (see :ref:`bltin-boolean-values`). + ``True`` (see :ref:`typebool`). .. index:: pair: Boolean; type diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 2360472b31f175..f6662b4336c276 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -802,6 +802,39 @@ number, :class:`float`, or :class:`complex`:: hash_value = -2 return hash_value +.. _typebool: + +Boolean Type - :class:`bool` +============================ + +Booleans represent truth values. The :class:`bool` type has exactly two +constant instances: ``True`` and ``False``. + +.. index:: + single: False + single: True + pair: Boolean; values + +The built-in function :func:`bool` converts any value to a boolean, if the +value can be interpreted as a truth value (see section :ref:`truth` above). + +For logical operations, use the :ref:`boolean operators ` ``and``, +``or`` and ``not``. +When applying the bitwise operators ``&``, ``|``, ``^`` to two booleans, they +return a bool equivalent to the logical operations "and", "or", "xor". However, +the logical operators ``and``, ``or`` and ``!=`` should be preferred +over ``&``, ``|`` and ``^``. + +.. deprecated:: 3.12 + + The use of the bitwise inversion operator ``~`` is deprecated and will + raise an error in Python 3.14. + +:class:`bool` is a subclass of :class:`int` (see :ref:`typesnumeric`). In +many numeric contexts, ``False`` and ``True`` behave like the integers 0 and 1, respectively. +However, relying on this is discouraged; explicitly convert using :func:`int` +instead. + .. _typeiter: Iterator Types @@ -5394,27 +5427,6 @@ information. There is exactly one ``NotImplemented`` object. It is written as ``NotImplemented``. -.. _bltin-boolean-values: - -Boolean Values --------------- - -Boolean values are the two constant objects ``False`` and ``True``. They are -used to represent truth values (although other values can also be considered -false or true). In numeric contexts (for example when used as the argument to -an arithmetic operator), they behave like the integers 0 and 1, respectively. -The built-in function :func:`bool` can be used to convert any value to a -Boolean, if the value can be interpreted as a truth value (see section -:ref:`truth` above). - -.. index:: - single: False - single: True - pair: Boolean; values - -They are written as ``False`` and ``True``, respectively. - - .. _typesinternal: Internal Objects diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index a3fce7ccacf7c1..5f8a1f08026d0b 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -710,6 +710,12 @@ Deprecated replaced by :data:`calendar.Month.JANUARY` and :data:`calendar.Month.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) +* The bitwise inversion operator (``~``) on bool is deprecated. It will throw an + error in Python 3.14. Use ``not`` for logical negation of bools instead. + In the rare case that you really need the bitwise inversion of the underlying + ``int``, convert to int explicitly with ``~int(x)``. (Contributed by Tim Hoffmann + in :gh:`103487`.) + Pending Removal in Python 3.13 ------------------------------ diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index 916e22a527a8e0..34ecb45f161dfe 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -58,8 +58,22 @@ def test_math(self): self.assertEqual(-True, -1) self.assertEqual(abs(True), 1) self.assertIsNot(abs(True), True) - self.assertEqual(~False, -1) - self.assertEqual(~True, -2) + with self.assertWarns(DeprecationWarning): + # We need to put the bool in a variable, because the constant + # ~False is evaluated at compile time due to constant folding; + # consequently the DeprecationWarning would be issued during + # module loading and not during test execution. + false = False + self.assertEqual(~false, -1) + with self.assertWarns(DeprecationWarning): + # also check that the warning is issued in case of constant + # folding at compile time + self.assertEqual(eval("~False"), -1) + with self.assertWarns(DeprecationWarning): + true = True + self.assertEqual(~true, -2) + with self.assertWarns(DeprecationWarning): + self.assertEqual(eval("~True"), -2) self.assertEqual(False+2, 2) self.assertEqual(True+2, 3) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst new file mode 100644 index 00000000000000..819a2359bf6fae --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-12-19-55-24.gh-issue-82012.FlcJAh.rst @@ -0,0 +1,5 @@ +The bitwise inversion operator (``~``) on bool is deprecated. +It returns the bitwise inversion of the underlying ``int`` representation such that +``bool(~True) == True``, which can be confusing. Use ``not`` for logical negation +of bools. In the rare case that you really need the bitwise inversion of the underlying ``int``, +convert to int explicitly ``~int(x)``. diff --git a/Objects/boolobject.c b/Objects/boolobject.c index 597a76fa5cb162..0300f7bb4e3dc0 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -73,6 +73,22 @@ bool_vectorcall(PyObject *type, PyObject * const*args, /* Arithmetic operations redefined to return bool if both args are bool. */ +static PyObject * +bool_invert(PyObject *v) +{ + if (PyErr_WarnEx(PyExc_DeprecationWarning, + "Bitwise inversion '~' on bool is deprecated. This " + "returns the bitwise inversion of the underlying int " + "object and is usually not what you expect from negating " + "a bool. Use the 'not' operator for boolean negation or " + "~int(x) if you really want the bitwise inversion of the " + "underlying int.", + 1) < 0) { + return NULL; + } + return PyLong_Type.tp_as_number->nb_invert(v); +} + static PyObject * bool_and(PyObject *a, PyObject *b) { @@ -119,7 +135,7 @@ static PyNumberMethods bool_as_number = { 0, /* nb_positive */ 0, /* nb_absolute */ 0, /* nb_bool */ - 0, /* nb_invert */ + (unaryfunc)bool_invert, /* nb_invert */ 0, /* nb_lshift */ 0, /* nb_rshift */ bool_and, /* nb_and */ From 8d34031068ece75667260f6526d3165efe34e054 Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Wed, 3 May 2023 00:20:00 -0700 Subject: [PATCH 12/65] gh-104078: Improve performance of PyObject_HasAttrString (#104079) --- ...-05-01-21-05-47.gh-issue-104078.vRaBsU.rst | 1 + Objects/object.c | 23 ++++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst new file mode 100644 index 00000000000000..6f24529bac3e0c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-21-05-47.gh-issue-104078.vRaBsU.rst @@ -0,0 +1 @@ +Improve the performance of :c:func:`PyObject_HasAttrString` diff --git a/Objects/object.c b/Objects/object.c index ee8690101d3cc0..c6ef5928164824 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -918,13 +918,24 @@ PyObject_GetAttrString(PyObject *v, const char *name) int PyObject_HasAttrString(PyObject *v, const char *name) { - PyObject *res = PyObject_GetAttrString(v, name); - if (res != NULL) { - Py_DECREF(res); - return 1; + if (Py_TYPE(v)->tp_getattr != NULL) { + PyObject *res = (*Py_TYPE(v)->tp_getattr)(v, (char*)name); + if (res != NULL) { + Py_DECREF(res); + return 1; + } + PyErr_Clear(); + return 0; } - PyErr_Clear(); - return 0; + + PyObject *attr_name = PyUnicode_FromString(name); + if (attr_name == NULL) { + PyErr_Clear(); + return 0; + } + int ok = PyObject_HasAttr(v, attr_name); + Py_DECREF(attr_name); + return ok; } int From 65c4a2b326086875ecbedf032204d1ff24ba74d7 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Thu, 23 Mar 2023 10:36:47 -0400 Subject: [PATCH 13/65] GH-84976: Move Lib/datetime.py to Lib/_pydatetime This breaks the tests, but we are keeping it as a separate commit so that the move operation and editing of the moved files are separate, for a cleaner history. --- Lib/{datetime.py => _pydatetime.py} | 0 Lib/test/datetimetester.py | 6 +++++- Lib/test/test_datetime.py | 12 ++++++++---- Python/stdlib_module_names.h | 1 + 4 files changed, 14 insertions(+), 5 deletions(-) rename Lib/{datetime.py => _pydatetime.py} (100%) diff --git a/Lib/datetime.py b/Lib/_pydatetime.py similarity index 100% rename from Lib/datetime.py rename to Lib/_pydatetime.py diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index c5eb6e7f1643ee..fb07d2a5ad9b06 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -39,6 +39,10 @@ # Needed by test_datetime import _strptime +try: + import _pydatetime +except ImportError: + pass # pickle_loads = {pickle.loads, pickle._loads} @@ -92,7 +96,7 @@ def test_divide_and_round(self): if '_Fast' in self.__class__.__name__: self.skipTest('Only run for Pure Python implementation') - dar = datetime_module._divide_and_round + dar = _pydatetime._divide_and_round self.assertEqual(dar(-10, -3), 3) self.assertEqual(dar(5, -2), -2) diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index 7f9094fa7bd4e6..3859733a4fe65b 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -8,10 +8,12 @@ def load_tests(loader, tests, pattern): try: - pure_tests = import_fresh_module(TESTS, fresh=['datetime', '_strptime'], - blocked=['_datetime']) - fast_tests = import_fresh_module(TESTS, fresh=['datetime', - '_datetime', '_strptime']) + pure_tests = import_fresh_module(TESTS, + fresh=['datetime', '_pydatetime', '_strptime'], + blocked=['_datetime']) + fast_tests = import_fresh_module(TESTS, + fresh=['datetime', '_strptime'], + blocked=['_pydatetime']) finally: # XXX: import_fresh_module() is supposed to leave sys.module cache untouched, # XXX: but it does not, so we have to cleanup ourselves. @@ -42,6 +44,8 @@ def setUpClass(cls_, module=module): cls_._save_sys_modules = sys.modules.copy() sys.modules[TESTS] = module sys.modules['datetime'] = module.datetime_module + if hasattr(module, '_pydatetime'): + sys.modules['_pydatetime'] = module._pydatetime sys.modules['_strptime'] = module._strptime @classmethod def tearDownClass(cls_): diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 27f42e5202e571..ed4a0ac2dd32de 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -56,6 +56,7 @@ static const char* _Py_stdlib_module_names[] = { "_posixshmem", "_posixsubprocess", "_py_abc", +"_pydatetime", "_pydecimal", "_pyio", "_pylong", From 0a5cd984b215f28d3c205eadf0daf201b3388c90 Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Thu, 23 Mar 2023 11:36:22 -0400 Subject: [PATCH 14/65] GH-84976: Re-introduce `datetime.py` and fix reprs Without the change to the reprs, pure-python classes would have a repr of `datetime._pydatetime.time`, etc. --- Lib/_pydatetime.py | 42 ++++++------------- Lib/datetime.py | 9 ++++ ...3-04-19-16-08-53.gh-issue-84976.HwbzlD.rst | 5 +++ 3 files changed, 27 insertions(+), 29 deletions(-) create mode 100644 Lib/datetime.py create mode 100644 Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index b0eb1c216a689d..f4fc2c58e5e293 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -16,6 +16,13 @@ def _cmp(x, y): return 0 if x == y else 1 if x > y else -1 +def _get_class_module(self): + module_name = self.__class__.__module__ + if module_name == '_pydatetime': + return 'datetime' + else: + return module_name + MINYEAR = 1 MAXYEAR = 9999 _MAXORDINAL = 3652059 # date.max.toordinal() @@ -706,7 +713,7 @@ def __repr__(self): args.append("microseconds=%d" % self._microseconds) if not args: args.append('0') - return "%s.%s(%s)" % (self.__class__.__module__, + return "%s.%s(%s)" % (_get_class_module(self), self.__class__.__qualname__, ', '.join(args)) @@ -1016,7 +1023,7 @@ def __repr__(self): >>> repr(dt) 'datetime.datetime(2010, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)' """ - return "%s.%s(%d, %d, %d)" % (self.__class__.__module__, + return "%s.%s(%d, %d, %d)" % (_get_class_module(self), self.__class__.__qualname__, self._year, self._month, @@ -1510,7 +1517,7 @@ def __repr__(self): s = ", %d" % self._second else: s = "" - s= "%s.%s(%d, %d%s)" % (self.__class__.__module__, + s= "%s.%s(%d, %d%s)" % (_get_class_module(self), self.__class__.__qualname__, self._hour, self._minute, s) if self._tzinfo is not None: @@ -2065,7 +2072,7 @@ def __repr__(self): del L[-1] if L[-1] == 0: del L[-1] - s = "%s.%s(%s)" % (self.__class__.__module__, + s = "%s.%s(%s)" % (_get_class_module(self), self.__class__.__qualname__, ", ".join(map(str, L))) if self._tzinfo is not None: @@ -2372,10 +2379,10 @@ def __repr__(self): if self is self.utc: return 'datetime.timezone.utc' if self._name is None: - return "%s.%s(%r)" % (self.__class__.__module__, + return "%s.%s(%r)" % (_get_class_module(self), self.__class__.__qualname__, self._offset) - return "%s.%s(%r, %r)" % (self.__class__.__module__, + return "%s.%s(%r, %r)" % (_get_class_module(self), self.__class__.__qualname__, self._offset, self._name) @@ -2638,26 +2645,3 @@ def _name_from_offset(delta): # small dst() may get within its bounds; and it doesn't even matter if some # perverse time zone returns a negative dst()). So a breaking case must be # pretty bizarre, and a tzinfo subclass can override fromutc() if it is. - -try: - from _datetime import * -except ImportError: - pass -else: - # Clean up unused names - del (_DAYNAMES, _DAYS_BEFORE_MONTH, _DAYS_IN_MONTH, _DI100Y, _DI400Y, - _DI4Y, _EPOCH, _MAXORDINAL, _MONTHNAMES, _build_struct_time, - _check_date_fields, _check_time_fields, - _check_tzinfo_arg, _check_tzname, _check_utc_offset, _cmp, _cmperror, - _date_class, _days_before_month, _days_before_year, _days_in_month, - _format_time, _format_offset, _index, _is_leap, _isoweek1monday, _math, - _ord2ymd, _time, _time_class, _tzinfo_class, _wrap_strftime, _ymd2ord, - _divide_and_round, _parse_isoformat_date, _parse_isoformat_time, - _parse_hh_mm_ss_ff, _IsoCalendarDate, _isoweek_to_gregorian, - _find_isoformat_datetime_separator, _FRACTION_CORRECTION, - _is_ascii_digit) - # XXX Since import * above excludes names that start with _, - # docstring does not get overwritten. In the future, it may be - # appropriate to maintain a single module level docstring and - # remove the following line. - from _datetime import __doc__ diff --git a/Lib/datetime.py b/Lib/datetime.py new file mode 100644 index 00000000000000..bad8beb4f6b026 --- /dev/null +++ b/Lib/datetime.py @@ -0,0 +1,9 @@ +try: + from _datetime import * + from _datetime import __doc__ +except ImportError: + from _pydatetime import * + from _pydatetime import __doc__ + +__all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo", + "MINYEAR", "MAXYEAR") diff --git a/Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst b/Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst new file mode 100644 index 00000000000000..8658627aeba434 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-19-16-08-53.gh-issue-84976.HwbzlD.rst @@ -0,0 +1,5 @@ +Create a new ``Lib/_pydatetime.py`` file that defines the Python version of +the ``datetime`` module, and make ``datetime`` import the contents of the +new library only if the C implementation is missing. Currently, the full +Python implementation is defined and then deleted if the C implementation is +not available, slowing down ``import datetime`` unnecessarily. From bcea36f8db9ad4fd542b38997e065987e829cb9f Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 3 May 2023 02:51:47 -0700 Subject: [PATCH 15/65] gh-103845: Remove line & instruction instrumentations before adding them back (GH-103851) --- Lib/test/test_monitoring.py | 36 ++++++++++++++++ ...-04-25-20-56-01.gh-issue-103845.V7NYFn.rst | 1 + Python/instrumentation.c | 41 +++++++++++++++---- 3 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 738ace923cc523..a493bb54d70d38 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -876,6 +876,42 @@ def func3(): ('instruction', 'func3', 34), ('line', 'check_events', 11)]) + def test_with_restart(self): + def func1(): + line1 = 1 + line2 = 2 + line3 = 3 + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 4), + ('line', 'func1', 2), + ('instruction', 'func1', 6), + ('instruction', 'func1', 8), + ('line', 'func1', 3), + ('instruction', 'func1', 10), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), + ('line', 'check_events', 11)]) + + sys.monitoring.restart_events() + + self.check_events(func1, recorders = LINE_AND_INSTRUCTION_RECORDERS, expected = [ + ('line', 'check_events', 10), + ('line', 'func1', 1), + ('instruction', 'func1', 2), + ('instruction', 'func1', 4), + ('line', 'func1', 2), + ('instruction', 'func1', 6), + ('instruction', 'func1', 8), + ('line', 'func1', 3), + ('instruction', 'func1', 10), + ('instruction', 'func1', 12), + ('instruction', 'func1', 14), + ('line', 'check_events', 11)]) + class TestInstallIncrementallly(MonitoringTestBase, unittest.TestCase): def check_events(self, func, must_include, tool=TEST_TOOL, recorders=(ExceptionRecorder,)): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst new file mode 100644 index 00000000000000..e8434854cde632 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-25-20-56-01.gh-issue-103845.V7NYFn.rst @@ -0,0 +1 @@ +Remove both line and instruction instrumentation before adding new ones for monitoring, to avoid newly added instrumentation being removed immediately. diff --git a/Python/instrumentation.c b/Python/instrumentation.c index c5bbbdacbb851e..a1423240609699 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1477,25 +1477,25 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) } } } - uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; + + // GH-103845: We need to remove both the line and instruction instrumentation before + // adding new ones, otherwise we may remove the newly added instrumentation. + uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; - if (new_line_tools | removed_line_tools) { + uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; + + if (removed_line_tools) { _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { if (line_data[i].original_opcode) { if (removed_line_tools) { remove_line_tools(code, i, removed_line_tools); } - if (new_line_tools) { - add_line_tools(code, i, new_line_tools); - } } i += instruction_length(code, i); } } - uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; - uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; - if (new_per_instruction_tools | removed_per_instruction_tools) { + if (removed_per_instruction_tools) { for (int i = code->_co_firsttraceable; i < code_len;) { int opcode = _Py_GetBaseOpcode(code, i); if (opcode == RESUME || opcode == END_FOR) { @@ -1505,6 +1505,31 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) if (removed_per_instruction_tools) { remove_per_instruction_tools(code, i, removed_per_instruction_tools); } + i += instruction_length(code, i); + } + } + + uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; + uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; + + if (new_line_tools) { + _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; + for (int i = code->_co_firsttraceable; i < code_len;) { + if (line_data[i].original_opcode) { + if (new_line_tools) { + add_line_tools(code, i, new_line_tools); + } + } + i += instruction_length(code, i); + } + } + if (new_per_instruction_tools) { + for (int i = code->_co_firsttraceable; i < code_len;) { + int opcode = _Py_GetBaseOpcode(code, i); + if (opcode == RESUME || opcode == END_FOR) { + i += instruction_length(code, i); + continue; + } if (new_per_instruction_tools) { add_per_instruction_tools(code, i, new_per_instruction_tools); } From 326997829d02458246dfd5b6d03297e2418bde52 Mon Sep 17 00:00:00 2001 From: Barry Warsaw Date: Wed, 3 May 2023 04:55:22 -0700 Subject: [PATCH 16/65] gh-98040: Remove find_loader, find_module and other deprecated APIs (#98059) * Remove deprecated classes from pkgutil * Remove some other PEP 302 obsolescence * Use find_spec instead of load_module * Remove more tests of PEP 302 obsolete APIs * Remove another bunch of tests using obsolete load_modules() * Remove deleted names from __all__ * Remove obsolete footnote * imp is removed * Remove `imp` from generated stdlib names * What's new and blurb * Update zipimport documentation for the removed methods * Fix some Windows tests * Remove any test (or part of a test) that references `find_module()`. * Use assertIsNone() / assertIsNotNone() consistently. * Update Doc/reference/import.rst * We don't need pkgutil._get_spec() any more either * test.test_importlib.fixtures.NullFinder * ...BadLoaderFinder.find_module * ...test_api.InvalidatingNullFinder.find_module * ...test.test_zipimport test of z.find_module * Suppress cross-references to find_loader and find_module * Suppress cross-references to Finder * Suppress cross-references to pkgutil.ImpImporter and pkgutil.ImpLoader --------- Co-authored-by: Oleg Iarygin Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com> --- Doc/library/importlib.rst | 112 +----------------- Doc/library/pkgutil.rst | 27 ----- Doc/library/sys.rst | 2 +- Doc/library/zipimport.rst | 27 +---- Doc/reference/import.rst | 34 +++--- Doc/whatsnew/2.3.rst | 2 +- Doc/whatsnew/3.10.rst | 38 +++--- Doc/whatsnew/3.11.rst | 14 +-- Doc/whatsnew/3.12.rst | 16 ++- Doc/whatsnew/3.3.rst | 4 +- Doc/whatsnew/3.4.rst | 18 +-- Doc/whatsnew/3.7.rst | 4 +- Lib/importlib/__init__.py | 34 ------ Lib/importlib/_bootstrap.py | 41 +------ Lib/importlib/_bootstrap_external.py | 95 +-------------- Lib/importlib/abc.py | 92 +------------- Lib/pkgutil.py | 14 --- Lib/pydoc.py | 2 +- .../test_importlib/builtin/test_finder.py | 33 ------ .../extension/test_path_hook.py | 2 +- Lib/test/test_importlib/fixtures.py | 5 - Lib/test/test_importlib/frozen/test_finder.py | 40 ------- Lib/test/test_importlib/frozen/test_loader.py | 82 ------------- .../test_importlib/import_/test___loader__.py | 43 ------- .../import_/test___package__.py | 36 ------ Lib/test/test_importlib/import_/test_api.py | 5 - .../test_importlib/import_/test_caching.py | 25 ++-- .../test_importlib/import_/test_meta_path.py | 10 -- Lib/test/test_importlib/import_/test_path.py | 73 +----------- .../source/test_case_sensitivity.py | 13 -- Lib/test/test_importlib/source/test_finder.py | 29 ++--- .../test_importlib/source/test_path_hook.py | 9 -- Lib/test/test_importlib/test_abc.py | 78 +----------- Lib/test/test_importlib/test_api.py | 67 ++++------- Lib/test/test_importlib/test_windows.py | 18 +-- Lib/test/test_importlib/util.py | 27 +---- Lib/test/test_pkgutil.py | 10 +- Lib/test/test_zipimport.py | 1 - ...2-10-09-14-47-42.gh-issue-98040.IN3qab.rst | 2 + 39 files changed, 137 insertions(+), 1047 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 89efa64c6b5203..65aaad0df9ee66 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -127,28 +127,6 @@ Functions .. versionchanged:: 3.3 Parent packages are automatically imported. -.. function:: find_loader(name, path=None) - - Find the loader for a module, optionally within the specified *path*. If the - module is in :attr:`sys.modules`, then ``sys.modules[name].__loader__`` is - returned (unless the loader would be ``None`` or is not set, in which case - :exc:`ValueError` is raised). Otherwise a search using :attr:`sys.meta_path` - is done. ``None`` is returned if no loader is found. - - A dotted name does not have its parents implicitly imported as that requires - loading them and that may not be desired. To properly import a submodule you - will need to import all parent packages of the submodule and use the correct - argument to *path*. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - If ``__loader__`` is not set, raise :exc:`ValueError`, just like when the - attribute is set to ``None``. - - .. deprecated:: 3.4 - Use :func:`importlib.util.find_spec` instead. - .. function:: invalidate_caches() Invalidate the internal caches of finders stored at @@ -247,7 +225,6 @@ are also provided to help in implementing the core ABCs. ABC hierarchy:: object - +-- Finder (deprecated) +-- MetaPathFinder +-- PathEntryFinder +-- Loader @@ -258,28 +235,6 @@ ABC hierarchy:: +-- SourceLoader -.. class:: Finder - - An abstract base class representing a :term:`finder`. - - .. deprecated:: 3.3 - Use :class:`MetaPathFinder` or :class:`PathEntryFinder` instead. - - .. abstractmethod:: find_module(fullname, path=None) - - An abstract method for finding a :term:`loader` for the specified - module. Originally specified in :pep:`302`, this method was meant - for use in :data:`sys.meta_path` and in the path-based import subsystem. - - .. versionchanged:: 3.4 - Returns ``None`` when called instead of raising - :exc:`NotImplementedError`. - - .. deprecated:: 3.10 - Implement :meth:`MetaPathFinder.find_spec` or - :meth:`PathEntryFinder.find_spec` instead. - - .. class:: MetaPathFinder An abstract base class representing a :term:`meta path finder`. @@ -287,7 +242,7 @@ ABC hierarchy:: .. versionadded:: 3.3 .. versionchanged:: 3.10 - No longer a subclass of :class:`Finder`. + No longer a subclass of :class:`!Finder`. .. method:: find_spec(fullname, path, target=None) @@ -303,25 +258,6 @@ ABC hierarchy:: .. versionadded:: 3.4 - .. method:: find_module(fullname, path) - - A legacy method for finding a :term:`loader` for the specified - module. If this is a top-level import, *path* will be ``None``. - Otherwise, this is a search for a subpackage or module and *path* - will be the value of :attr:`__path__` from the parent - package. If a loader cannot be found, ``None`` is returned. - - If :meth:`find_spec` is defined, backwards-compatible functionality is - provided. - - .. versionchanged:: 3.4 - Returns ``None`` when called instead of raising - :exc:`NotImplementedError`. Can use :meth:`find_spec` to provide - functionality. - - .. deprecated:: 3.4 - Use :meth:`find_spec` instead. - .. method:: invalidate_caches() An optional method which, when called, should invalidate any internal @@ -342,7 +278,7 @@ ABC hierarchy:: .. versionadded:: 3.3 .. versionchanged:: 3.10 - No longer a subclass of :class:`Finder`. + No longer a subclass of :class:`!Finder`. .. method:: find_spec(fullname, target=None) @@ -356,36 +292,6 @@ ABC hierarchy:: .. versionadded:: 3.4 - .. method:: find_loader(fullname) - - A legacy method for finding a :term:`loader` for the specified - module. Returns a 2-tuple of ``(loader, portion)`` where ``portion`` - is a sequence of file system locations contributing to part of a namespace - package. The loader may be ``None`` while specifying ``portion`` to - signify the contribution of the file system locations to a namespace - package. An empty list can be used for ``portion`` to signify the loader - is not part of a namespace package. If ``loader`` is ``None`` and - ``portion`` is the empty list then no loader or location for a namespace - package were found (i.e. failure to find anything for the module). - - If :meth:`find_spec` is defined then backwards-compatible functionality is - provided. - - .. versionchanged:: 3.4 - Returns ``(None, [])`` instead of raising :exc:`NotImplementedError`. - Uses :meth:`find_spec` when available to provide functionality. - - .. deprecated:: 3.4 - Use :meth:`find_spec` instead. - - .. method:: find_module(fullname) - - A concrete implementation of :meth:`Finder.find_module` which is - equivalent to ``self.find_loader(fullname)[0]``. - - .. deprecated:: 3.4 - Use :meth:`find_spec` instead. - .. method:: invalidate_caches() An optional method which, when called, should invalidate any internal @@ -881,13 +787,6 @@ find and load modules. is no longer valid then ``None`` is returned but no value is cached in :data:`sys.path_importer_cache`. - .. classmethod:: find_module(fullname, path=None) - - A legacy wrapper around :meth:`find_spec`. - - .. deprecated:: 3.4 - Use :meth:`find_spec` instead. - .. classmethod:: invalidate_caches() Calls :meth:`importlib.abc.PathEntryFinder.invalidate_caches` on all @@ -938,13 +837,6 @@ find and load modules. .. versionadded:: 3.4 - .. method:: find_loader(fullname) - - Attempt to find the loader to handle *fullname* within :attr:`path`. - - .. deprecated:: 3.10 - Use :meth:`find_spec` instead. - .. method:: invalidate_caches() Clear out the internal cache. diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 64e617b82b48bc..98e6e294af0c5c 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -48,33 +48,6 @@ support. this function to raise an exception (in line with :func:`os.path.isdir` behavior). - -.. class:: ImpImporter(dirname=None) - - :pep:`302` Finder that wraps Python's "classic" import algorithm. - - If *dirname* is a string, a :pep:`302` finder is created that searches that - directory. If *dirname* is ``None``, a :pep:`302` finder is created that - searches the current :data:`sys.path`, plus any modules that are frozen or - built-in. - - Note that :class:`ImpImporter` does not currently support being used by - placement on :data:`sys.meta_path`. - - .. deprecated:: 3.3 - This emulation is no longer needed, as the standard import mechanism - is now fully :pep:`302` compliant and available in :mod:`importlib`. - - -.. class:: ImpLoader(fullname, file, filename, etc) - - :term:`Loader ` that wraps Python's "classic" import algorithm. - - .. deprecated:: 3.3 - This emulation is no longer needed, as the standard import mechanism - is now fully :pep:`302` compliant and available in :mod:`importlib`. - - .. function:: find_loader(fullname) Retrieve a module :term:`loader` for the given *fullname*. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 95ad243bdde398..57a0d0a1258c2a 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1177,7 +1177,7 @@ always available. :term:`Module specs ` were introduced in Python 3.4, by :pep:`451`. Earlier versions of Python looked for a method called - :meth:`~importlib.abc.MetaPathFinder.find_module`. + :meth:`!find_module`. This is still called as a fallback if a :data:`meta_path` entry doesn't have a :meth:`~importlib.abc.MetaPathFinder.find_spec` method. diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst index fe1adcae163c23..11d19e8c863e9f 100644 --- a/Doc/library/zipimport.rst +++ b/Doc/library/zipimport.rst @@ -74,6 +74,11 @@ zipimporter Objects :exc:`ZipImportError` is raised if *archivepath* doesn't point to a valid ZIP archive. + .. versionchanged:: 3.12 + + Methods ``find_loader()`` and ``find_module()``, deprecated in 3.10 are + now removed. Use :meth:`find_spec` instead. + .. method:: create_module(spec) Implementation of :meth:`importlib.abc.Loader.create_module` that returns @@ -89,28 +94,6 @@ zipimporter Objects .. versionadded:: 3.10 - .. method:: find_loader(fullname, path=None) - - An implementation of :meth:`importlib.abc.PathEntryFinder.find_loader`. - - .. deprecated:: 3.10 - - Use :meth:`find_spec` instead. - - - .. method:: find_module(fullname, path=None) - - Search for a module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. It returns the zipimporter instance itself - if the module was found, or :const:`None` if it wasn't. The optional - *path* argument is ignored---it's there for compatibility with the - importer protocol. - - .. deprecated:: 3.10 - - Use :meth:`find_spec` instead. - - .. method:: find_spec(fullname, target=None) An implementation of :meth:`importlib.abc.PathEntryFinder.find_spec`. diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 57eb5403243eef..8f30181ae0a10d 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -324,15 +324,18 @@ modules, and one that knows how to import modules from an :term:`import path` .. versionchanged:: 3.4 The :meth:`~importlib.abc.MetaPathFinder.find_spec` method of meta path - finders replaced :meth:`~importlib.abc.MetaPathFinder.find_module`, which + finders replaced :meth:`!find_module`, which is now deprecated. While it will continue to work without change, the import machinery will try it only if the finder does not implement ``find_spec()``. .. versionchanged:: 3.10 - Use of :meth:`~importlib.abc.MetaPathFinder.find_module` by the import system + Use of :meth:`!find_module` by the import system now raises :exc:`ImportWarning`. +.. versionchanged:: 3.12 + ``find_module()`` has been removed. Use :meth:`find_spec` instead. + Loading ======= @@ -837,7 +840,7 @@ stores finder objects rather than being limited to :term:`importer` objects). In this way, the expensive search for a particular :term:`path entry` location's :term:`path entry finder` need only be done once. User code is free to remove cache entries from :data:`sys.path_importer_cache` forcing -the path based finder to perform the path entry search again [#fnpic]_. +the path based finder to perform the path entry search again. If the path entry is not present in the cache, the path based finder iterates over every callable in :data:`sys.path_hooks`. Each of the :term:`path entry @@ -887,13 +890,13 @@ module. ``find_spec()`` returns a fully populated spec for the module. This spec will always have "loader" set (with one exception). To indicate to the import machinery that the spec represents a namespace -:term:`portion`, the path entry finder sets "submodule_search_locations" to +:term:`portion`, the path entry finder sets ``submodule_search_locations`` to a list containing the portion. .. versionchanged:: 3.4 :meth:`~importlib.abc.PathEntryFinder.find_spec` replaced - :meth:`~importlib.abc.PathEntryFinder.find_loader` and - :meth:`~importlib.abc.PathEntryFinder.find_module`, both of which + :meth:`!find_loader` and + :meth:`!find_module`, both of which are now deprecated, but will be used if ``find_spec()`` is not defined. Older path entry finders may implement one of these two deprecated methods @@ -901,7 +904,7 @@ a list containing the portion. sake of backward compatibility. However, if ``find_spec()`` is implemented on the path entry finder, the legacy methods are ignored. - :meth:`~importlib.abc.PathEntryFinder.find_loader` takes one argument, the + :meth:`!find_loader` takes one argument, the fully qualified name of the module being imported. ``find_loader()`` returns a 2-tuple where the first item is the loader and the second item is a namespace :term:`portion`. @@ -920,10 +923,13 @@ a list containing the portion. ``find_loader()`` in preference to ``find_module()``. .. versionchanged:: 3.10 - Calls to :meth:`~importlib.abc.PathEntryFinder.find_module` and - :meth:`~importlib.abc.PathEntryFinder.find_loader` by the import + Calls to :meth:`!find_module` and + :meth:`!find_loader` by the import system will raise :exc:`ImportWarning`. +.. versionchanged:: 3.12 + ``find_module()`` and ``find_loader()`` have been removed. + Replacing the standard import system ==================================== @@ -1045,8 +1051,8 @@ The original specification for :data:`sys.meta_path` was :pep:`302`, with subsequent extension in :pep:`420`. :pep:`420` introduced :term:`namespace packages ` for -Python 3.3. :pep:`420` also introduced the :meth:`find_loader` protocol as an -alternative to :meth:`find_module`. +Python 3.3. :pep:`420` also introduced the :meth:`!find_loader` protocol as an +alternative to :meth:`!find_module`. :pep:`366` describes the addition of the ``__package__`` attribute for explicit relative imports in main modules. @@ -1073,9 +1079,3 @@ methods to finders and loaders. module may replace itself in :data:`sys.modules`. This is implementation-specific behavior that is not guaranteed to work in other Python implementations. - -.. [#fnpic] In legacy code, it is possible to find instances of - :class:`imp.NullImporter` in the :data:`sys.path_importer_cache`. It - is recommended that code be changed to use ``None`` instead. See - :ref:`portingpythoncode` for more details. Note that the ``imp`` module - was removed in Python 3.12. diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index af489d7cb45c2a..4eb864f5092d30 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -728,7 +728,7 @@ module: Importer objects must have a single method, ``find_module(fullname, path=None)``. *fullname* will be a module or package name, e.g. ``string`` or -``distutils.core``. :meth:`find_module` must return a loader object that has a +``distutils.core``. :meth:`!find_module` must return a loader object that has a single method, ``load_module(fullname)``, that creates and returns the corresponding module object. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index f6a48ed2680c14..44c8fa1e9eb2c4 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1608,7 +1608,7 @@ Deprecated * Starting in this release, there will be a concerted effort to begin cleaning up old import semantics that were kept for Python 2.7 compatibility. Specifically, - :meth:`~importlib.abc.PathEntryFinder.find_loader`/:meth:`~importlib.abc.Finder.find_module` + :meth:`!find_loader`/:meth:`!find_module` (superseded by :meth:`~importlib.abc.Finder.find_spec`), :meth:`~importlib.abc.Loader.load_module` (superseded by :meth:`~importlib.abc.Loader.exec_module`), @@ -1645,8 +1645,8 @@ Deprecated :meth:`~importlib.abc.Loader.exec_module` is preferred. (Contributed by Brett Cannon in :issue:`26131`.) -* The use of :meth:`importlib.abc.MetaPathFinder.find_module` and - :meth:`importlib.abc.PathEntryFinder.find_module` by the import system now +* The use of :meth:`!importlib.abc.MetaPathFinder.find_module` and + :meth:`!importlib.abc.PathEntryFinder.find_module` by the import system now trigger an :exc:`ImportWarning` as :meth:`importlib.abc.MetaPathFinder.find_spec` and :meth:`importlib.abc.PathEntryFinder.find_spec` @@ -1654,40 +1654,40 @@ Deprecated :func:`importlib.util.spec_from_loader` to help in porting. (Contributed by Brett Cannon in :issue:`42134`.) -* The use of :meth:`importlib.abc.PathEntryFinder.find_loader` by the import +* The use of :meth:`!importlib.abc.PathEntryFinder.find_loader` by the import system now triggers an :exc:`ImportWarning` as :meth:`importlib.abc.PathEntryFinder.find_spec` is preferred. You can use :func:`importlib.util.spec_from_loader` to help in porting. (Contributed by Brett Cannon in :issue:`43672`.) * The various implementations of - :meth:`importlib.abc.MetaPathFinder.find_module` ( - :meth:`importlib.machinery.BuiltinImporter.find_module`, - :meth:`importlib.machinery.FrozenImporter.find_module`, - :meth:`importlib.machinery.WindowsRegistryFinder.find_module`, - :meth:`importlib.machinery.PathFinder.find_module`, - :meth:`importlib.abc.MetaPathFinder.find_module` ), - :meth:`importlib.abc.PathEntryFinder.find_module` ( - :meth:`importlib.machinery.FileFinder.find_module` ), and - :meth:`importlib.abc.PathEntryFinder.find_loader` ( - :meth:`importlib.machinery.FileFinder.find_loader` ) + :meth:`!importlib.abc.MetaPathFinder.find_module` ( + :meth:`!importlib.machinery.BuiltinImporter.find_module`, + :meth:`!importlib.machinery.FrozenImporter.find_module`, + :meth:`!importlib.machinery.WindowsRegistryFinder.find_module`, + :meth:`!importlib.machinery.PathFinder.find_module`, + :meth:`!importlib.abc.MetaPathFinder.find_module` ), + :meth:`!importlib.abc.PathEntryFinder.find_module` ( + :meth:`!importlib.machinery.FileFinder.find_module` ), and + :meth:`!importlib.abc.PathEntryFinder.find_loader` ( + :meth:`!importlib.machinery.FileFinder.find_loader` ) now raise :exc:`DeprecationWarning` and are slated for removal in Python 3.12 (previously they were documented as deprecated in Python 3.4). (Contributed by Brett Cannon in :issue:`42135`.) -* :class:`importlib.abc.Finder` is deprecated (including its sole method, - :meth:`~importlib.abc.Finder.find_module`). Both +* :class:`!importlib.abc.Finder` is deprecated (including its sole method, + :meth:`!find_module`). Both :class:`importlib.abc.MetaPathFinder` and :class:`importlib.abc.PathEntryFinder` no longer inherit from the class. Users should inherit from one of these two classes as appropriate instead. (Contributed by Brett Cannon in :issue:`42135`.) -* The deprecations of :mod:`imp`, :func:`importlib.find_loader`, +* The deprecations of :mod:`imp`, :func:`!importlib.find_loader`, :func:`importlib.util.set_package_wrapper`, :func:`importlib.util.set_loader_wrapper`, :func:`importlib.util.module_for_loader`, - :class:`pkgutil.ImpImporter`, and - :class:`pkgutil.ImpLoader` have all been updated to list Python 3.12 as the + :class:`!pkgutil.ImpImporter`, and + :class:`!pkgutil.ImpLoader` have all been updated to list Python 3.12 as the slated version of removal (they began raising :exc:`DeprecationWarning` in previous versions of Python). (Contributed by Brett Cannon in :issue:`43720`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 6b591d5e184ea7..918a6824618c1e 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1876,24 +1876,24 @@ C APIs pending removal are * The :class:`typing.io ` namespace * The :class:`typing.re ` namespace * :func:`!cgi.log` -* :func:`importlib.find_loader` +* :func:`!importlib.find_loader` * :meth:`importlib.abc.Loader.module_repr` -* :meth:`importlib.abc.MetaPathFinder.find_module` -* :meth:`importlib.abc.PathEntryFinder.find_loader` -* :meth:`importlib.abc.PathEntryFinder.find_module` +* :meth:`!importlib.abc.MetaPathFinder.find_module` +* :meth:`!importlib.abc.PathEntryFinder.find_loader` +* :meth:`!importlib.abc.PathEntryFinder.find_module` * :meth:`!importlib.machinery.BuiltinImporter.find_module` * :meth:`!importlib.machinery.BuiltinLoader.module_repr` * :meth:`!importlib.machinery.FileFinder.find_loader` * :meth:`!importlib.machinery.FileFinder.find_module` * :meth:`!importlib.machinery.FrozenImporter.find_module` * :meth:`!importlib.machinery.FrozenLoader.module_repr` -* :meth:`importlib.machinery.PathFinder.find_module` +* :meth:`!importlib.machinery.PathFinder.find_module` * :meth:`!importlib.machinery.WindowsRegistryFinder.find_module` * :func:`importlib.util.module_for_loader` * :func:`!importlib.util.set_loader_wrapper` * :func:`!importlib.util.set_package_wrapper` -* :class:`pkgutil.ImpImporter` -* :class:`pkgutil.ImpLoader` +* :class:`!pkgutil.ImpImporter` +* :class:`!pkgutil.ImpLoader` * :meth:`pathlib.Path.link_to` * :func:`!sqlite3.enable_shared_cache` * :func:`!sqlite3.OptimizedUnicode` diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 5f8a1f08026d0b..1139bb89f5cf0c 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -983,14 +983,20 @@ Removed * Many previously deprecated cleanups in :mod:`importlib` have now been completed: - * References to, and support for ``module_repr()`` has been eradicated. + * References to, and support for ``module_repr()`` has been removed. (Contributed by Barry Warsaw in :gh:`97850`.) -* ``importlib.util.set_package`` has been removed. - (Contributed by Brett Cannon in :gh:`65961`.) + * ``importlib.util.set_package`` has been removed. (Contributed by Brett + Cannon in :gh:`65961`.) + + * Support for ``find_loader()`` and ``find_module()`` APIs have been + removed. (Contributed by Barry Warsaw in :gh:`98040`.) + + * ``importlib.abc.Finder``, ``pkg.ImpImporter``, and ``pkg.ImpLoader`` have + been removed. (Contributed by Barry Warsaw in :gh:`98040`.) -* The ``imp`` module has been removed. (Contributed by Barry Warsaw in - :gh:`98040`.) + * The ``imp`` module has been removed. (Contributed by Barry Warsaw in + :gh:`98040`.) * Removed the ``suspicious`` rule from the documentation Makefile, and removed ``Doc/tools/rstlint.py``, both in favor of `sphinx-lint diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 5b6c3dcd45c6f5..df1f2ab775b047 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -714,7 +714,7 @@ to properly delineate between :term:`meta path finders ` and :term:`path entry finders ` by introducing :class:`importlib.abc.MetaPathFinder` and :class:`importlib.abc.PathEntryFinder`, respectively. The old ABC of -:class:`importlib.abc.Finder` is now only provided for backwards-compatibility +:class:`!importlib.abc.Finder` is now only provided for backwards-compatibility and does not enforce any method requirements. In terms of finders, :class:`importlib.machinery.FileFinder` exposes the @@ -2390,7 +2390,7 @@ Porting Python code :attr:`sys.path_importer_cache` where it represents the use of implicit finders, but semantically it should not change anything. -* :class:`importlib.abc.Finder` no longer specifies a ``find_module()`` abstract +* :class:`!importlib.abc.Finder` no longer specifies a ``find_module()`` abstract method that must be implemented. If you were relying on subclasses to implement that method, make sure to check for the method's existence first. You will probably want to check for ``find_loader()`` first, though, in the diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index b7bb505a818482..53f78e3621cfbd 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2077,19 +2077,19 @@ Deprecations in the Python API ------------------------------ * As mentioned in :ref:`whatsnew-pep-451`, a number of :mod:`importlib` - methods and functions are deprecated: :meth:`importlib.find_loader` is + methods and functions are deprecated: :meth:`!importlib.find_loader` is replaced by :func:`importlib.util.find_spec`; - :meth:`importlib.machinery.PathFinder.find_module` is replaced by + :meth:`!importlib.machinery.PathFinder.find_module` is replaced by :meth:`importlib.machinery.PathFinder.find_spec`; - :meth:`importlib.abc.MetaPathFinder.find_module` is replaced by + :meth:`!importlib.abc.MetaPathFinder.find_module` is replaced by :meth:`importlib.abc.MetaPathFinder.find_spec`; - :meth:`importlib.abc.PathEntryFinder.find_loader` and - :meth:`~importlib.abc.PathEntryFinder.find_module` are replaced by + :meth:`!importlib.abc.PathEntryFinder.find_loader` and + :meth:`!find_module` are replaced by :meth:`importlib.abc.PathEntryFinder.find_spec`; all of the ``xxxLoader`` ABC - ``load_module`` methods (:meth:`importlib.abc.Loader.load_module`, - :meth:`importlib.abc.InspectLoader.load_module`, - :meth:`importlib.abc.FileLoader.load_module`, - :meth:`importlib.abc.SourceLoader.load_module`) should no longer be + ``load_module`` methods (:meth:`!importlib.abc.Loader.load_module`, + :meth:`!importlib.abc.InspectLoader.load_module`, + :meth:`!importlib.abc.FileLoader.load_module`, + :meth:`!importlib.abc.SourceLoader.load_module`) should no longer be implemented, instead loaders should implement an ``exec_module`` method (:meth:`importlib.abc.Loader.exec_module`, diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index df3b636cb9ec46..28f22836d8d09e 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -2004,11 +2004,11 @@ importlib --------- Methods -:meth:`MetaPathFinder.find_module() ` +:meth:`MetaPathFinder.find_module() ` (replaced by :meth:`MetaPathFinder.find_spec() `) and -:meth:`PathEntryFinder.find_loader() ` +:meth:`PathEntryFinder.find_loader() ` (replaced by :meth:`PathEntryFinder.find_spec() `) both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index 21d9dee652b3df..707c081cb2c5b6 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -70,40 +70,6 @@ def invalidate_caches(): finder.invalidate_caches() -def find_loader(name, path=None): - """Return the loader for the specified module. - - This is a backward-compatible wrapper around find_spec(). - - This function is deprecated in favor of importlib.util.find_spec(). - - """ - warnings.warn('Deprecated since Python 3.4 and slated for removal in ' - 'Python 3.12; use importlib.util.find_spec() instead', - DeprecationWarning, stacklevel=2) - try: - loader = sys.modules[name].__loader__ - if loader is None: - raise ValueError(f'{name}.__loader__ is None') - else: - return loader - except KeyError: - pass - except AttributeError: - raise ValueError(f'{name}.__loader__ is not set') from None - - spec = _bootstrap._find_spec(name, path) - # We won't worry about malformed specs (missing attributes). - if spec is None: - return None - if spec.loader is None: - if spec.submodule_search_locations is None: - raise ImportError(f'spec for {name} missing loader', name=name) - raise ImportError('namespace packages do not have loaders', - name=name) - return spec.loader - - def import_module(name, package=None): """Import a module. diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index e4fcaa61e6de29..c48fd506a0e4eb 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -892,21 +892,6 @@ def find_spec(cls, fullname, path=None, target=None): else: return None - @classmethod - def find_module(cls, fullname, path=None): - """Find the built-in module. - - If 'path' is ever specified then the search is considered a failure. - - This method is deprecated. Use find_spec() instead. - - """ - _warnings.warn("BuiltinImporter.find_module() is deprecated and " - "slated for removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - spec = cls.find_spec(fullname, path) - return spec.loader if spec is not None else None - @staticmethod def create_module(spec): """Create a built-in module""" @@ -1076,18 +1061,6 @@ def find_spec(cls, fullname, path=None, target=None): spec.submodule_search_locations.insert(0, pkgdir) return spec - @classmethod - def find_module(cls, fullname, path=None): - """Find a frozen module. - - This method is deprecated. Use find_spec() instead. - - """ - _warnings.warn("FrozenImporter.find_module() is deprecated and " - "slated for removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - return cls if _imp.is_frozen(fullname) else None - @staticmethod def create_module(spec): """Set __file__, if able.""" @@ -1170,16 +1143,6 @@ def _resolve_name(name, package, level): return f'{base}.{name}' if name else base -def _find_spec_legacy(finder, name, path): - msg = (f"{_object_name(finder)}.find_spec() not found; " - "falling back to find_module()") - _warnings.warn(msg, ImportWarning) - loader = finder.find_module(name, path) - if loader is None: - return None - return spec_from_loader(name, loader) - - def _find_spec(name, path, target=None): """Find a module's spec.""" meta_path = sys.meta_path @@ -1200,9 +1163,7 @@ def _find_spec(name, path, target=None): try: find_spec = finder.find_spec except AttributeError: - spec = _find_spec_legacy(finder, name, path) - if spec is None: - continue + continue else: spec = find_spec(name, path, target) if spec is not None: diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index cb227373ca2fd4..7a3fdbaebdf24e 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -659,26 +659,6 @@ def _wrap(new, old): return _check_name_wrapper -def _find_module_shim(self, fullname): - """Try to find a loader for the specified module by delegating to - self.find_loader(). - - This method is deprecated in favor of finder.find_spec(). - - """ - _warnings.warn("find_module() is deprecated and " - "slated for removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - # Call find_loader(). If it returns a string (indicating this - # is a namespace package portion), generate a warning and - # return None. - loader, portions = self.find_loader(fullname) - if loader is None and len(portions): - msg = f'Not importing directory {portions[0]}: missing __init__' - _warnings.warn(msg, ImportWarning) - return loader - - def _classify_pyc(data, name, exc_details): """Perform basic validity checking of a pyc header and return the flags field, which determines how the pyc should be further validated against the source. @@ -985,22 +965,6 @@ def find_spec(cls, fullname, path=None, target=None): origin=filepath) return spec - @classmethod - def find_module(cls, fullname, path=None): - """Find module named in the registry. - - This method is deprecated. Use find_spec() instead. - - """ - _warnings.warn("WindowsRegistryFinder.find_module() is deprecated and " - "slated for removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - spec = cls.find_spec(fullname, path) - if spec is not None: - return spec.loader - else: - return None - class _LoaderBasics: @@ -1517,27 +1481,6 @@ def _path_importer_cache(cls, path): sys.path_importer_cache[path] = finder return finder - @classmethod - def _legacy_get_spec(cls, fullname, finder): - # This would be a good place for a DeprecationWarning if - # we ended up going that route. - if hasattr(finder, 'find_loader'): - msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; " - "falling back to find_loader()") - _warnings.warn(msg, ImportWarning) - loader, portions = finder.find_loader(fullname) - else: - msg = (f"{_bootstrap._object_name(finder)}.find_spec() not found; " - "falling back to find_module()") - _warnings.warn(msg, ImportWarning) - loader = finder.find_module(fullname) - portions = [] - if loader is not None: - return _bootstrap.spec_from_loader(fullname, loader) - spec = _bootstrap.ModuleSpec(fullname, None) - spec.submodule_search_locations = portions - return spec - @classmethod def _get_spec(cls, fullname, path, target=None): """Find the loader or namespace_path for this module/package name.""" @@ -1549,10 +1492,7 @@ def _get_spec(cls, fullname, path, target=None): continue finder = cls._path_importer_cache(entry) if finder is not None: - if hasattr(finder, 'find_spec'): - spec = finder.find_spec(fullname, target) - else: - spec = cls._legacy_get_spec(fullname, finder) + spec = finder.find_spec(fullname, target) if spec is None: continue if spec.loader is not None: @@ -1594,22 +1534,6 @@ def find_spec(cls, fullname, path=None, target=None): else: return spec - @classmethod - def find_module(cls, fullname, path=None): - """find the module on sys.path or 'path' based on sys.path_hooks and - sys.path_importer_cache. - - This method is deprecated. Use find_spec() instead. - - """ - _warnings.warn("PathFinder.find_module() is deprecated and " - "slated for removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - spec = cls.find_spec(fullname, path) - if spec is None: - return None - return spec.loader - @staticmethod def find_distributions(*args, **kwargs): """ @@ -1654,23 +1578,6 @@ def invalidate_caches(self): """Invalidate the directory mtime.""" self._path_mtime = -1 - find_module = _find_module_shim - - def find_loader(self, fullname): - """Try to find a loader for the specified module, or the namespace - package portions. Returns (loader, list-of-portions). - - This method is deprecated. Use find_spec() instead. - - """ - _warnings.warn("FileFinder.find_loader() is deprecated and " - "slated for removal in Python 3.12; use find_spec() instead", - DeprecationWarning) - spec = self.find_spec(fullname) - if spec is None: - return None, [] - return spec.loader, spec.submodule_search_locations or [] - def _get_spec(self, loader_class, fullname, path, smsl, target): loader = loader_class(fullname, path) return spec_from_file_location(fullname, path, loader=loader, diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 8fa9a0f3bc1e4b..b56fa94eb9c135 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -19,7 +19,7 @@ __all__ = [ - 'Loader', 'Finder', 'MetaPathFinder', 'PathEntryFinder', + 'Loader', 'MetaPathFinder', 'PathEntryFinder', 'ResourceLoader', 'InspectLoader', 'ExecutionLoader', 'FileLoader', 'SourceLoader', ] @@ -49,38 +49,6 @@ def _register(abstract_cls, *classes): abstract_cls.register(frozen_cls) -class Finder(metaclass=abc.ABCMeta): - - """Legacy abstract base class for import finders. - - It may be subclassed for compatibility with legacy third party - reimplementations of the import system. Otherwise, finder - implementations should derive from the more specific MetaPathFinder - or PathEntryFinder ABCs. - - Deprecated since Python 3.3 - """ - - def __init__(self): - warnings.warn("the Finder ABC is deprecated and " - "slated for removal in Python 3.12; use MetaPathFinder " - "or PathEntryFinder instead", - DeprecationWarning) - - @abc.abstractmethod - def find_module(self, fullname, path=None): - """An abstract method that should find a module. - The fullname is a str and the optional path is a str or None. - Returns a Loader object or None. - """ - warnings.warn("importlib.abc.Finder along with its find_module() " - "method are deprecated and " - "slated for removal in Python 3.12; use " - "MetaPathFinder.find_spec() or " - "PathEntryFinder.find_spec() instead", - DeprecationWarning) - - class MetaPathFinder(metaclass=abc.ABCMeta): """Abstract base class for import finders on sys.meta_path.""" @@ -88,27 +56,6 @@ class MetaPathFinder(metaclass=abc.ABCMeta): # We don't define find_spec() here since that would break # hasattr checks we do to support backward compatibility. - def find_module(self, fullname, path): - """Return a loader for the module. - - If no module is found, return None. The fullname is a str and - the path is a list of strings or None. - - This method is deprecated since Python 3.4 in favor of - finder.find_spec(). If find_spec() exists then backwards-compatible - functionality is provided for this method. - - """ - warnings.warn("MetaPathFinder.find_module() is deprecated since Python " - "3.4 in favor of MetaPathFinder.find_spec() and is " - "slated for removal in Python 3.12", - DeprecationWarning, - stacklevel=2) - if not hasattr(self, 'find_spec'): - return None - found = self.find_spec(fullname, path) - return found.loader if found is not None else None - def invalidate_caches(self): """An optional method for clearing the finder's cache, if any. This method is used by importlib.invalidate_caches(). @@ -122,43 +69,6 @@ class PathEntryFinder(metaclass=abc.ABCMeta): """Abstract base class for path entry finders used by PathFinder.""" - # We don't define find_spec() here since that would break - # hasattr checks we do to support backward compatibility. - - def find_loader(self, fullname): - """Return (loader, namespace portion) for the path entry. - - The fullname is a str. The namespace portion is a sequence of - path entries contributing to part of a namespace package. The - sequence may be empty. If loader is not None, the portion will - be ignored. - - The portion will be discarded if another path entry finder - locates the module as a normal module or package. - - This method is deprecated since Python 3.4 in favor of - finder.find_spec(). If find_spec() is provided than backwards-compatible - functionality is provided. - """ - warnings.warn("PathEntryFinder.find_loader() is deprecated since Python " - "3.4 in favor of PathEntryFinder.find_spec() " - "(available since 3.4)", - DeprecationWarning, - stacklevel=2) - if not hasattr(self, 'find_spec'): - return None, [] - found = self.find_spec(fullname) - if found is not None: - if not found.submodule_search_locations: - portions = [] - else: - portions = found.submodule_search_locations - return found.loader, portions - else: - return None, [] - - find_module = _bootstrap_external._find_module_shim - def invalidate_caches(self): """An optional method for clearing the finder's cache, if any. This method is used by PathFinder.invalidate_caches(). diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index fb977eaaa05767..f62eccb974d6f5 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -23,20 +23,6 @@ ModuleInfo.__doc__ = 'A namedtuple with minimal info about a module.' -def _get_spec(finder, name): - """Return the finder-specific module spec.""" - # Works with legacy finders. - try: - find_spec = finder.find_spec - except AttributeError: - loader = finder.find_module(name) - if loader is None: - return None - return importlib.util.spec_from_loader(name, loader) - else: - return find_spec(name) - - def read_code(stream): # This helper is needed in order for the PEP 302 emulation to # correctly handle compiled files diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 1c3443fa8469f7..b10a5da99402b5 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -2242,7 +2242,7 @@ def run(self, callback, key=None, completer=None, onerror=None): callback(None, modname, '') else: try: - spec = pkgutil._get_spec(importer, modname) + spec = importer.find_spec(modname) except SyntaxError: # raised by tests for bad coding cookies or BOM continue diff --git a/Lib/test/test_importlib/builtin/test_finder.py b/Lib/test/test_importlib/builtin/test_finder.py index 81dc5a3699d952..111c4af1ea7cfe 100644 --- a/Lib/test/test_importlib/builtin/test_finder.py +++ b/Lib/test/test_importlib/builtin/test_finder.py @@ -43,38 +43,5 @@ def test_failure(self): ) = util.test_both(FindSpecTests, machinery=machinery) -@unittest.skipIf(util.BUILTINS.good_name is None, 'no reasonable builtin module') -class FinderTests(abc.FinderTests): - - """Test find_module() for built-in modules.""" - - def test_module(self): - # Common case. - with util.uncache(util.BUILTINS.good_name): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - found = self.machinery.BuiltinImporter.find_module(util.BUILTINS.good_name) - self.assertTrue(found) - self.assertTrue(hasattr(found, 'load_module')) - - # Built-in modules cannot be a package. - test_package = test_package_in_package = test_package_over_module = None - - # Built-in modules cannot be in a package. - test_module_in_package = None - - def test_failure(self): - assert 'importlib' not in sys.builtin_module_names - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader = self.machinery.BuiltinImporter.find_module('importlib') - self.assertIsNone(loader) - - -(Frozen_FinderTests, - Source_FinderTests - ) = util.test_both(FinderTests, machinery=machinery) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/extension/test_path_hook.py b/Lib/test/test_importlib/extension/test_path_hook.py index a0adc70ad1ec4d..ec9644dc520534 100644 --- a/Lib/test/test_importlib/extension/test_path_hook.py +++ b/Lib/test/test_importlib/extension/test_path_hook.py @@ -19,7 +19,7 @@ def hook(self, entry): def test_success(self): # Path hook should handle a directory where a known extension module # exists. - self.assertTrue(hasattr(self.hook(util.EXTENSIONS.path), 'find_module')) + self.assertTrue(hasattr(self.hook(util.EXTENSIONS.path), 'find_spec')) (Frozen_PathHooksTests, diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py index a364a977bce781..73e5da2ba92279 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/fixtures.py @@ -350,11 +350,6 @@ def DALS(str): return textwrap.dedent(str).lstrip() -class NullFinder: - def find_module(self, name): - pass - - @requires_zlib() class ZipFixtures: root = 'test.test_importlib.data' diff --git a/Lib/test/test_importlib/frozen/test_finder.py b/Lib/test/test_importlib/frozen/test_finder.py index 069755606b40af..469dcdbd09eaf7 100644 --- a/Lib/test/test_importlib/frozen/test_finder.py +++ b/Lib/test/test_importlib/frozen/test_finder.py @@ -182,45 +182,5 @@ def test_not_using_frozen(self): ) = util.test_both(FindSpecTests, machinery=machinery) -class FinderTests(abc.FinderTests): - - """Test finding frozen modules.""" - - def find(self, name, path=None): - finder = self.machinery.FrozenImporter - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - with import_helper.frozen_modules(): - return finder.find_module(name, path) - - def test_module(self): - name = '__hello__' - loader = self.find(name) - self.assertTrue(hasattr(loader, 'load_module')) - - def test_package(self): - loader = self.find('__phello__') - self.assertTrue(hasattr(loader, 'load_module')) - - def test_module_in_package(self): - loader = self.find('__phello__.spam', ['__phello__']) - self.assertTrue(hasattr(loader, 'load_module')) - - # No frozen package within another package to test with. - test_package_in_package = None - - # No easy way to test. - test_package_over_module = None - - def test_failure(self): - loader = self.find('') - self.assertIsNone(loader) - - -(Frozen_FinderTests, - Source_FinderTests - ) = util.test_both(FinderTests, machinery=machinery) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/frozen/test_loader.py b/Lib/test/test_importlib/frozen/test_loader.py index da1569e3d0681e..4f1af454b52c71 100644 --- a/Lib/test/test_importlib/frozen/test_loader.py +++ b/Lib/test/test_importlib/frozen/test_loader.py @@ -125,88 +125,6 @@ def test_unloadable(self): ) = util.test_both(ExecModuleTests, machinery=machinery) -class LoaderTests(abc.LoaderTests): - - def load_module(self, name): - with fresh(name, oldapi=True): - module = self.machinery.FrozenImporter.load_module(name) - with captured_stdout() as stdout: - module.main() - return module, stdout - - def test_module(self): - module, stdout = self.load_module('__hello__') - filename = resolve_stdlib_file('__hello__') - check = {'__name__': '__hello__', - '__package__': '', - '__loader__': self.machinery.FrozenImporter, - '__file__': filename, - } - for attr, value in check.items(): - self.assertEqual(getattr(module, attr, None), value) - self.assertEqual(stdout.getvalue(), 'Hello world!\n') - - def test_package(self): - module, stdout = self.load_module('__phello__') - filename = resolve_stdlib_file('__phello__', ispkg=True) - pkgdir = os.path.dirname(filename) - check = {'__name__': '__phello__', - '__package__': '__phello__', - '__path__': [pkgdir], - '__loader__': self.machinery.FrozenImporter, - '__file__': filename, - } - for attr, value in check.items(): - attr_value = getattr(module, attr, None) - self.assertEqual(attr_value, value, - "for __phello__.%s, %r != %r" % - (attr, attr_value, value)) - self.assertEqual(stdout.getvalue(), 'Hello world!\n') - - def test_lacking_parent(self): - with util.uncache('__phello__'): - module, stdout = self.load_module('__phello__.spam') - filename = resolve_stdlib_file('__phello__.spam') - check = {'__name__': '__phello__.spam', - '__package__': '__phello__', - '__loader__': self.machinery.FrozenImporter, - '__file__': filename, - } - for attr, value in check.items(): - attr_value = getattr(module, attr) - self.assertEqual(attr_value, value, - "for __phello__.spam.%s, %r != %r" % - (attr, attr_value, value)) - self.assertEqual(stdout.getvalue(), 'Hello world!\n') - - def test_module_reuse(self): - with fresh('__hello__', oldapi=True): - module1 = self.machinery.FrozenImporter.load_module('__hello__') - module2 = self.machinery.FrozenImporter.load_module('__hello__') - with captured_stdout() as stdout: - module1.main() - module2.main() - self.assertIs(module1, module2) - self.assertEqual(stdout.getvalue(), - 'Hello world!\nHello world!\n') - - # No way to trigger an error in a frozen module. - test_state_after_failure = None - - def test_unloadable(self): - with import_helper.frozen_modules(): - with deprecated(): - assert self.machinery.FrozenImporter.find_module('_not_real') is None - with self.assertRaises(ImportError) as cm: - self.load_module('_not_real') - self.assertEqual(cm.exception.name, '_not_real') - - -(Frozen_LoaderTests, - Source_LoaderTests - ) = util.test_both(LoaderTests, machinery=machinery) - - class InspectLoaderTests: """Tests for the InspectLoader methods for FrozenImporter.""" diff --git a/Lib/test/test_importlib/import_/test___loader__.py b/Lib/test/test_importlib/import_/test___loader__.py index eaf665a6f5b5af..a14163919af677 100644 --- a/Lib/test/test_importlib/import_/test___loader__.py +++ b/Lib/test/test_importlib/import_/test___loader__.py @@ -33,48 +33,5 @@ def test___loader__(self): ) = util.test_both(SpecLoaderAttributeTests, __import__=util.__import__) -class LoaderMock: - - def find_module(self, fullname, path=None): - return self - - def load_module(self, fullname): - sys.modules[fullname] = self.module - return self.module - - -class LoaderAttributeTests: - - def test___loader___missing(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - module = types.ModuleType('blah') - try: - del module.__loader__ - except AttributeError: - pass - loader = LoaderMock() - loader.module = module - with util.uncache('blah'), util.import_state(meta_path=[loader]): - module = self.__import__('blah') - self.assertEqual(loader, module.__loader__) - - def test___loader___is_None(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - module = types.ModuleType('blah') - module.__loader__ = None - loader = LoaderMock() - loader.module = module - with util.uncache('blah'), util.import_state(meta_path=[loader]): - returned_module = self.__import__('blah') - self.assertEqual(loader, module.__loader__) - - -(Frozen_Tests, - Source_Tests - ) = util.test_both(LoaderAttributeTests, __import__=util.__import__) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index ab1b35ee3c1a4e..7130c99a6fc171 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -95,25 +95,6 @@ def __init__(self, parent): self.parent = parent -class Using__package__PEP302(Using__package__): - mock_modules = util.mock_modules - - def test_using___package__(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_using___package__() - - def test_spec_fallback(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_spec_fallback() - - -(Frozen_UsingPackagePEP302, - Source_UsingPackagePEP302 - ) = util.test_both(Using__package__PEP302, __import__=util.__import__) - - class Using__package__PEP451(Using__package__): mock_modules = util.mock_spec @@ -162,23 +143,6 @@ def test_submodule(self): module = getattr(pkg, 'mod') self.assertEqual(module.__package__, 'pkg') -class Setting__package__PEP302(Setting__package__, unittest.TestCase): - mock_modules = util.mock_modules - - def test_top_level(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_top_level() - - def test_package(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_package() - - def test_submodule(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - super().test_submodule() class Setting__package__PEP451(Setting__package__, unittest.TestCase): mock_modules = util.mock_spec diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py index 0ee032b0206df9..d6ad590b3d46a0 100644 --- a/Lib/test/test_importlib/import_/test_api.py +++ b/Lib/test/test_importlib/import_/test_api.py @@ -28,11 +28,6 @@ def exec_module(module): class BadLoaderFinder: - @classmethod - def find_module(cls, fullname, path): - if fullname == SUBMOD_NAME: - return cls - @classmethod def load_module(cls, fullname): if fullname == SUBMOD_NAME: diff --git a/Lib/test/test_importlib/import_/test_caching.py b/Lib/test/test_importlib/import_/test_caching.py index 3ca765fb4ada97..aedf0fd4f9db02 100644 --- a/Lib/test/test_importlib/import_/test_caching.py +++ b/Lib/test/test_importlib/import_/test_caching.py @@ -52,12 +52,11 @@ class ImportlibUseCache(UseCache, unittest.TestCase): __import__ = util.__import__['Source'] def create_mock(self, *names, return_=None): - mock = util.mock_modules(*names) - original_load = mock.load_module - def load_module(self, fullname): - original_load(fullname) - return return_ - mock.load_module = MethodType(load_module, mock) + mock = util.mock_spec(*names) + original_spec = mock.find_spec + def find_spec(self, fullname, path, target=None): + return original_spec(fullname) + mock.find_spec = MethodType(find_spec, mock) return mock # __import__ inconsistent between loaders and built-in import when it comes @@ -86,14 +85,12 @@ def test_using_cache_for_assigning_to_attribute(self): # See test_using_cache_after_loader() for reasoning. def test_using_cache_for_fromlist(self): # [from cache for fromlist] - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - with self.create_mock('pkg.__init__', 'pkg.module') as importer: - with util.import_state(meta_path=[importer]): - module = self.__import__('pkg', fromlist=['module']) - self.assertTrue(hasattr(module, 'module')) - self.assertEqual(id(module.module), - id(sys.modules['pkg.module'])) + with self.create_mock('pkg.__init__', 'pkg.module') as importer: + with util.import_state(meta_path=[importer]): + module = self.__import__('pkg', fromlist=['module']) + self.assertTrue(hasattr(module, 'module')) + self.assertEqual(id(module.module), + id(sys.modules['pkg.module'])) if __name__ == '__main__': diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py index c8b898ec237850..8689017ba43112 100644 --- a/Lib/test/test_importlib/import_/test_meta_path.py +++ b/Lib/test/test_importlib/import_/test_meta_path.py @@ -113,16 +113,6 @@ def test_with_path(self): super().test_no_path() -class CallSignaturePEP302(CallSignoreSuppressImportWarning): - mock_modules = util.mock_modules - finder_name = 'find_module' - - -(Frozen_CallSignaturePEP302, - Source_CallSignaturePEP302 - ) = util.test_both(CallSignaturePEP302, __import__=util.__import__) - - class CallSignaturePEP451(CallSignature): mock_modules = util.mock_spec finder_name = 'find_spec' diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py index de620842bbc52b..89b52fbd1e1aff 100644 --- a/Lib/test/test_importlib/import_/test_path.py +++ b/Lib/test/test_importlib/import_/test_path.py @@ -116,46 +116,6 @@ def test_None_on_sys_path(self): if email is not missing: sys.modules['email'] = email - def test_finder_with_find_module(self): - class TestFinder: - def find_module(self, fullname): - return self.to_return - failing_finder = TestFinder() - failing_finder.to_return = None - path = 'testing path' - with util.import_state(path_importer_cache={path: failing_finder}): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - self.assertIsNone( - self.machinery.PathFinder.find_spec('whatever', [path])) - success_finder = TestFinder() - success_finder.to_return = __loader__ - with util.import_state(path_importer_cache={path: success_finder}): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - spec = self.machinery.PathFinder.find_spec('whatever', [path]) - self.assertEqual(spec.loader, __loader__) - - def test_finder_with_find_loader(self): - class TestFinder: - loader = None - portions = [] - def find_loader(self, fullname): - return self.loader, self.portions - path = 'testing path' - with util.import_state(path_importer_cache={path: TestFinder()}): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - self.assertIsNone( - self.machinery.PathFinder.find_spec('whatever', [path])) - success_finder = TestFinder() - success_finder.loader = __loader__ - with util.import_state(path_importer_cache={path: success_finder}): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - spec = self.machinery.PathFinder.find_spec('whatever', [path]) - self.assertEqual(spec.loader, __loader__) - def test_finder_with_find_spec(self): class TestFinder: spec = None @@ -228,9 +188,9 @@ def invalidate_caches(self): class FindModuleTests(FinderTests): def find(self, *args, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return self.machinery.PathFinder.find_module(*args, **kwargs) + spec = self.machinery.PathFinder.find_spec(*args, **kwargs) + return None if spec is None else spec.loader + def check_found(self, found, importer): self.assertIs(found, importer) @@ -255,16 +215,14 @@ def check_found(self, found, importer): class PathEntryFinderTests: def test_finder_with_failing_find_spec(self): - # PathEntryFinder with find_module() defined should work. - # Issue #20763. class Finder: - path_location = 'test_finder_with_find_module' + path_location = 'test_finder_with_find_spec' def __init__(self, path): if path != self.path_location: raise ImportError @staticmethod - def find_module(fullname): + def find_spec(fullname, target=None): return None @@ -274,27 +232,6 @@ def find_module(fullname): warnings.simplefilter("ignore", ImportWarning) self.machinery.PathFinder.find_spec('importlib') - def test_finder_with_failing_find_module(self): - # PathEntryFinder with find_module() defined should work. - # Issue #20763. - class Finder: - path_location = 'test_finder_with_find_module' - def __init__(self, path): - if path != self.path_location: - raise ImportError - - @staticmethod - def find_module(fullname): - return None - - - with util.import_state(path=[Finder.path_location]+sys.path[:], - path_hooks=[Finder]): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", ImportWarning) - warnings.simplefilter("ignore", DeprecationWarning) - self.machinery.PathFinder.find_module('importlib') - (Frozen_PEFTests, Source_PEFTests diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py index 9d472707abe840..6a06313319dbcd 100644 --- a/Lib/test/test_importlib/source/test_case_sensitivity.py +++ b/Lib/test/test_importlib/source/test_case_sensitivity.py @@ -63,19 +63,6 @@ def test_insensitive(self): self.assertIn(self.name, insensitive.get_filename(self.name)) -class CaseSensitivityTestPEP302(CaseSensitivityTest): - def find(self, finder): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return finder.find_module(self.name) - - -(Frozen_CaseSensitivityTestPEP302, - Source_CaseSensitivityTestPEP302 - ) = util.test_both(CaseSensitivityTestPEP302, importlib=importlib, - machinery=machinery) - - class CaseSensitivityTestPEP451(CaseSensitivityTest): def find(self, finder): found = finder.find_spec(self.name) diff --git a/Lib/test/test_importlib/source/test_finder.py b/Lib/test/test_importlib/source/test_finder.py index bed9d56dca84ee..12db7c7d352a2f 100644 --- a/Lib/test/test_importlib/source/test_finder.py +++ b/Lib/test/test_importlib/source/test_finder.py @@ -120,7 +120,7 @@ def test_package_over_module(self): def test_failure(self): with util.create_modules('blah') as mapping: nothing = self.import_(mapping['.root'], 'sdfsadsadf') - self.assertIsNone(nothing) + self.assertEqual(nothing, self.NOT_FOUND) def test_empty_string_for_dir(self): # The empty string from sys.path means to search in the cwd. @@ -150,7 +150,7 @@ def test_dir_removal_handling(self): found = self._find(finder, 'mod', loader_only=True) self.assertIsNotNone(found) found = self._find(finder, 'mod', loader_only=True) - self.assertIsNone(found) + self.assertEqual(found, self.NOT_FOUND) @unittest.skipUnless(sys.platform != 'win32', 'os.chmod() does not support the needed arguments under Windows') @@ -196,10 +196,12 @@ class FinderTestsPEP420(FinderTests): NOT_FOUND = (None, []) def _find(self, finder, name, loader_only=False): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader_portions = finder.find_loader(name) - return loader_portions[0] if loader_only else loader_portions + spec = finder.find_spec(name) + if spec is None: + return self.NOT_FOUND + if loader_only: + return spec.loader + return spec.loader, spec.submodule_search_locations (Frozen_FinderTestsPEP420, @@ -207,20 +209,5 @@ def _find(self, finder, name, loader_only=False): ) = util.test_both(FinderTestsPEP420, machinery=machinery) -class FinderTestsPEP302(FinderTests): - - NOT_FOUND = None - - def _find(self, finder, name, loader_only=False): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - return finder.find_module(name) - - -(Frozen_FinderTestsPEP302, - Source_FinderTestsPEP302 - ) = util.test_both(FinderTestsPEP302, machinery=machinery) - - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/source/test_path_hook.py b/Lib/test/test_importlib/source/test_path_hook.py index ead62f5e945e2a..f274330e0b333b 100644 --- a/Lib/test/test_importlib/source/test_path_hook.py +++ b/Lib/test/test_importlib/source/test_path_hook.py @@ -18,19 +18,10 @@ def test_success(self): self.assertTrue(hasattr(self.path_hook()(mapping['.root']), 'find_spec')) - def test_success_legacy(self): - with util.create_modules('dummy') as mapping: - self.assertTrue(hasattr(self.path_hook()(mapping['.root']), - 'find_module')) - def test_empty_string(self): # The empty string represents the cwd. self.assertTrue(hasattr(self.path_hook()(''), 'find_spec')) - def test_empty_string_legacy(self): - # The empty string represents the cwd. - self.assertTrue(hasattr(self.path_hook()(''), 'find_module')) - (Frozen_PathHookTest, Source_PathHooktest diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index 3c9149c4e45a92..603125f6d926f6 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -147,20 +147,13 @@ def ins(self): class MetaPathFinder: - def find_module(self, fullname, path): - return super().find_module(fullname, path) + pass class MetaPathFinderDefaultsTests(ABCTestHarness): SPLIT = make_abc_subclasses(MetaPathFinder) - def test_find_module(self): - # Default should return None. - with self.assertWarns(DeprecationWarning): - found = self.ins.find_module('something', None) - self.assertIsNone(found) - def test_invalidate_caches(self): # Calling the method is a no-op. self.ins.invalidate_caches() @@ -173,22 +166,13 @@ def test_invalidate_caches(self): class PathEntryFinder: - def find_loader(self, fullname): - return super().find_loader(fullname) + pass class PathEntryFinderDefaultsTests(ABCTestHarness): SPLIT = make_abc_subclasses(PathEntryFinder) - def test_find_loader(self): - with self.assertWarns(DeprecationWarning): - found = self.ins.find_loader('something') - self.assertEqual(found, (None, [])) - - def find_module(self): - self.assertEqual(None, self.ins.find_module('something')) - def test_invalidate_caches(self): # Should be a no-op. self.ins.invalidate_caches() @@ -201,8 +185,7 @@ def test_invalidate_caches(self): class Loader: - def load_module(self, fullname): - return super().load_module(fullname) + pass class LoaderDefaultsTests(ABCTestHarness): @@ -333,14 +316,6 @@ def find_spec(self, fullname, path, target=None): return MetaPathSpecFinder() - def test_find_module(self): - finder = self.finder(None) - path = ['a', 'b', 'c'] - name = 'blah' - with self.assertWarns(DeprecationWarning): - found = finder.find_module(name, path) - self.assertIsNone(found) - def test_find_spec_with_explicit_target(self): loader = object() spec = self.util.spec_from_loader('blah', loader) @@ -370,53 +345,6 @@ def test_spec(self): ) = test_util.test_both(MetaPathFinderFindModuleTests, abc=abc, util=util) -##### PathEntryFinder concrete methods ######################################### -class PathEntryFinderFindLoaderTests: - - @classmethod - def finder(cls, spec): - class PathEntrySpecFinder(cls.abc.PathEntryFinder): - - def find_spec(self, fullname, target=None): - self.called_for = fullname - return spec - - return PathEntrySpecFinder() - - def test_no_spec(self): - finder = self.finder(None) - name = 'blah' - with self.assertWarns(DeprecationWarning): - found = finder.find_loader(name) - self.assertIsNone(found[0]) - self.assertEqual([], found[1]) - self.assertEqual(name, finder.called_for) - - def test_spec_with_loader(self): - loader = object() - spec = self.util.spec_from_loader('blah', loader) - finder = self.finder(spec) - with self.assertWarns(DeprecationWarning): - found = finder.find_loader('blah') - self.assertIs(found[0], spec.loader) - - def test_spec_with_portions(self): - spec = self.machinery.ModuleSpec('blah', None) - paths = ['a', 'b', 'c'] - spec.submodule_search_locations = paths - finder = self.finder(spec) - with self.assertWarns(DeprecationWarning): - found = finder.find_loader('blah') - self.assertIsNone(found[0]) - self.assertEqual(paths, found[1]) - - -(Frozen_PEFFindLoaderTests, - Source_PEFFindLoaderTests - ) = test_util.test_both(PathEntryFinderFindLoaderTests, abc=abc, util=util, - machinery=machinery) - - ##### Loader concrete methods ################################################## class LoaderLoadModuleTests: diff --git a/Lib/test/test_importlib/test_api.py b/Lib/test/test_importlib/test_api.py index b3a99dc2dd5731..ecf2c47c462e23 100644 --- a/Lib/test/test_importlib/test_api.py +++ b/Lib/test/test_importlib/test_api.py @@ -95,7 +95,8 @@ def load_b(): (Frozen_ImportModuleTests, Source_ImportModuleTests - ) = test_util.test_both(ImportModuleTests, init=init) + ) = test_util.test_both( + ImportModuleTests, init=init, util=util, machinery=machinery) class FindLoaderTests: @@ -103,29 +104,26 @@ class FindLoaderTests: FakeMetaFinder = None def test_sys_modules(self): - # If a module with __loader__ is in sys.modules, then return it. + # If a module with __spec__.loader is in sys.modules, then return it. name = 'some_mod' with test_util.uncache(name): module = types.ModuleType(name) loader = 'a loader!' - module.__loader__ = loader + module.__spec__ = self.machinery.ModuleSpec(name, loader) sys.modules[name] = module - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - found = self.init.find_loader(name) - self.assertEqual(loader, found) + spec = self.util.find_spec(name) + self.assertIsNotNone(spec) + self.assertEqual(spec.loader, loader) def test_sys_modules_loader_is_None(self): - # If sys.modules[name].__loader__ is None, raise ValueError. + # If sys.modules[name].__spec__.loader is None, raise ValueError. name = 'some_mod' with test_util.uncache(name): module = types.ModuleType(name) module.__loader__ = None sys.modules[name] = module with self.assertRaises(ValueError): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.init.find_loader(name) + self.util.find_spec(name) def test_sys_modules_loader_is_not_set(self): # Should raise ValueError @@ -134,24 +132,20 @@ def test_sys_modules_loader_is_not_set(self): with test_util.uncache(name): module = types.ModuleType(name) try: - del module.__loader__ + del module.__spec__.loader except AttributeError: pass sys.modules[name] = module with self.assertRaises(ValueError): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.init.find_loader(name) + self.util.find_spec(name) def test_success(self): # Return the loader found on sys.meta_path. name = 'some_mod' with test_util.uncache(name): with test_util.import_state(meta_path=[self.FakeMetaFinder]): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - warnings.simplefilter('ignore', ImportWarning) - self.assertEqual((name, None), self.init.find_loader(name)) + spec = self.util.find_spec(name) + self.assertEqual((name, (name, None)), (spec.name, spec.loader)) def test_success_path(self): # Searching on a path should work. @@ -159,17 +153,12 @@ def test_success_path(self): path = 'path to some place' with test_util.uncache(name): with test_util.import_state(meta_path=[self.FakeMetaFinder]): - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - warnings.simplefilter('ignore', ImportWarning) - self.assertEqual((name, path), - self.init.find_loader(name, path)) + spec = self.util.find_spec(name, path) + self.assertEqual(name, spec.name) def test_nothing(self): # None is returned upon failure to find a loader. - with warnings.catch_warnings(): - warnings.simplefilter('ignore', DeprecationWarning) - self.assertIsNone(self.init.find_loader('nevergoingtofindthismodule')) + self.assertIsNone(self.util.find_spec('nevergoingtofindthismodule')) class FindLoaderPEP451Tests(FindLoaderTests): @@ -182,20 +171,8 @@ def find_spec(name, path=None, target=None): (Frozen_FindLoaderPEP451Tests, Source_FindLoaderPEP451Tests - ) = test_util.test_both(FindLoaderPEP451Tests, init=init) - - -class FindLoaderPEP302Tests(FindLoaderTests): - - class FakeMetaFinder: - @staticmethod - def find_module(name, path=None): - return name, path - - -(Frozen_FindLoaderPEP302Tests, - Source_FindLoaderPEP302Tests - ) = test_util.test_both(FindLoaderPEP302Tests, init=init) + ) = test_util.test_both( + FindLoaderPEP451Tests, init=init, util=util, machinery=machinery) class ReloadTests: @@ -380,7 +357,8 @@ def test_module_missing_spec(self): (Frozen_ReloadTests, Source_ReloadTests - ) = test_util.test_both(ReloadTests, init=init, util=util) + ) = test_util.test_both( + ReloadTests, init=init, util=util, machinery=machinery) class InvalidateCacheTests: @@ -390,8 +368,6 @@ def test_method_called(self): class InvalidatingNullFinder: def __init__(self, *ignored): self.called = False - def find_module(self, *args): - return None def invalidate_caches(self): self.called = True @@ -416,7 +392,8 @@ def test_method_lacking(self): (Frozen_InvalidateCacheTests, Source_InvalidateCacheTests - ) = test_util.test_both(InvalidateCacheTests, init=init) + ) = test_util.test_both( + InvalidateCacheTests, init=init, util=util, machinery=machinery) class FrozenImportlibTests(unittest.TestCase): diff --git a/Lib/test/test_importlib/test_windows.py b/Lib/test/test_importlib/test_windows.py index b7dfe865a03a97..40b8aa1787fe08 100644 --- a/Lib/test/test_importlib/test_windows.py +++ b/Lib/test/test_importlib/test_windows.py @@ -92,30 +92,16 @@ class WindowsRegistryFinderTests: def test_find_spec_missing(self): spec = self.machinery.WindowsRegistryFinder.find_spec('spam') - self.assertIs(spec, None) - - def test_find_module_missing(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader = self.machinery.WindowsRegistryFinder.find_module('spam') - self.assertIs(loader, None) + self.assertIsNone(spec) def test_module_found(self): with setup_module(self.machinery, self.test_module): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader = self.machinery.WindowsRegistryFinder.find_module(self.test_module) spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) - self.assertIsNot(loader, None) - self.assertIsNot(spec, None) + self.assertIsNotNone(spec) def test_module_not_found(self): with setup_module(self.machinery, self.test_module, path="."): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - loader = self.machinery.WindowsRegistryFinder.find_module(self.test_module) spec = self.machinery.WindowsRegistryFinder.find_spec(self.test_module) - self.assertIsNone(loader) self.assertIsNone(spec) (Frozen_WindowsRegistryFinderTests, diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index e348733f6ce3c3..c25be096e52874 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -194,8 +194,7 @@ def import_state(**kwargs): new_value = default setattr(sys, attr, new_value) if len(kwargs): - raise ValueError( - 'unrecognized arguments: {0}'.format(kwargs.keys())) + raise ValueError('unrecognized arguments: {}'.format(kwargs)) yield finally: for attr, value in originals.items(): @@ -243,30 +242,6 @@ def __exit__(self, *exc_info): self._uncache.__exit__(None, None, None) -class mock_modules(_ImporterMock): - - """Importer mock using PEP 302 APIs.""" - - def find_module(self, fullname, path=None): - if fullname not in self.modules: - return None - else: - return self - - def load_module(self, fullname): - if fullname not in self.modules: - raise ImportError - else: - sys.modules[fullname] = self.modules[fullname] - if fullname in self.module_code: - try: - self.module_code[fullname]() - except Exception: - del sys.modules[fullname] - raise - return self.modules[fullname] - - class mock_spec(_ImporterMock): """Importer mock using PEP 451 APIs.""" diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 4d9f5db3c6b3cf..902627088fc291 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -429,7 +429,7 @@ def test_iter_importers(self): importers = list(iter_importers(fullname)) expected_importer = get_importer(pathitem) for finder in importers: - spec = pkgutil._get_spec(finder, fullname) + spec = finder.find_spec(fullname) loader = spec.loader try: loader = loader.loader @@ -441,7 +441,7 @@ def test_iter_importers(self): self.assertEqual(finder, expected_importer) self.assertIsInstance(loader, importlib.machinery.SourceFileLoader) - self.assertIsNone(pkgutil._get_spec(finder, pkgname)) + self.assertIsNone(finder.find_spec(pkgname)) with self.assertRaises(ImportError): list(iter_importers('invalid.module')) @@ -535,12 +535,6 @@ class ImportlibMigrationTests(unittest.TestCase): # PEP 302 emulation in this module is in the process of being # deprecated in favour of importlib proper - def check_deprecated(self): - return check_warnings( - ("This emulation is deprecated and slated for removal in " - "Python 3.12; use 'importlib' instead", - DeprecationWarning)) - def test_get_loader_avoids_emulation(self): with check_warnings() as w: self.assertIsNotNone(pkgutil.get_loader("sys")) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 52d43bdead67f8..14c19719e260c4 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -836,7 +836,6 @@ def _testBogusZipFile(self): self.assertRaises(TypeError, z.get_source, None) error = zipimport.ZipImportError - self.assertIsNone(z.find_module('abc')) self.assertIsNone(z.find_spec('abc')) with warnings.catch_warnings(): diff --git a/Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst b/Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst new file mode 100644 index 00000000000000..ac185406844153 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-09-14-47-42.gh-issue-98040.IN3qab.rst @@ -0,0 +1,2 @@ +Remove more deprecated importlib APIs: ``find_loader()``, ``find_module()``, +``importlib.abc.Finder``, ``pkgutil.ImpImporter``, ``pkgutil.ImpLoader``. From 328435ed42d9d2d0aab7024540c745e730b9b9b1 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 3 May 2023 13:29:42 +0100 Subject: [PATCH 17/65] GH-98040: Suppress cross-references to the removed ``imp`` module (#104131) Suppress cross-references to imp --- Doc/c-api/import.rst | 4 ++-- Doc/library/functions.rst | 1 + Doc/whatsnew/3.0.rst | 2 +- Doc/whatsnew/3.10.rst | 2 +- Doc/whatsnew/3.11.rst | 2 +- Doc/whatsnew/3.12.rst | 2 +- Doc/whatsnew/3.2.rst | 4 ++-- Doc/whatsnew/3.3.rst | 6 +++--- Doc/whatsnew/3.4.rst | 12 ++++++------ Doc/whatsnew/3.6.rst | 2 +- Misc/NEWS.d/3.8.0a1.rst | 2 +- 11 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 474a64800044d0..8e5af32b65e02c 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -186,10 +186,10 @@ Importing Modules .. versionadded:: 3.2 .. versionchanged:: 3.3 - Uses :func:`imp.source_from_cache()` in calculating the source path if + Uses :func:`!imp.source_from_cache()` in calculating the source path if only the bytecode path is provided. .. versionchanged:: 3.12 - No longer uses the removed ``imp`` module. + No longer uses the removed :mod:`!imp` module. .. c:function:: long PyImport_GetMagicNumber() diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 085a11c3caa708..47e3880129595e 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1987,6 +1987,7 @@ are always available. They are listed here in alphabetical order. .. index:: statement: import + module: builtins .. note:: diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index 63b24748d8aab6..f9ac13036cbc8d 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -840,7 +840,7 @@ Builtins need it; however, 99 percent of the time an explicit :keyword:`for` loop is more readable. -* Removed :func:`reload`. Use :func:`imp.reload`. +* Removed :func:`reload`. Use :func:`!imp.reload`. * Removed. :meth:`dict.has_key` -- use the :keyword:`in` operator instead. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 44c8fa1e9eb2c4..3d7bcf89694477 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1682,7 +1682,7 @@ Deprecated classes as appropriate instead. (Contributed by Brett Cannon in :issue:`42135`.) -* The deprecations of :mod:`imp`, :func:`!importlib.find_loader`, +* The deprecations of :mod:`!imp`, :func:`!importlib.find_loader`, :func:`importlib.util.set_package_wrapper`, :func:`importlib.util.set_loader_wrapper`, :func:`importlib.util.module_for_loader`, diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 918a6824618c1e..6df30ad6c19c2c 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1872,7 +1872,7 @@ C APIs pending removal are * The :mod:`asynchat` module * The :mod:`asyncore` module * The :ref:`entire distutils package ` -* The :mod:`imp` module +* The :mod:`!imp` module * The :class:`typing.io ` namespace * The :class:`typing.re ` namespace * :func:`!cgi.log` diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 1139bb89f5cf0c..4ed08f3ca3b3d3 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -995,7 +995,7 @@ Removed * ``importlib.abc.Finder``, ``pkg.ImpImporter``, and ``pkg.ImpLoader`` have been removed. (Contributed by Barry Warsaw in :gh:`98040`.) - * The ``imp`` module has been removed. (Contributed by Barry Warsaw in + * The :mod:`!imp` module has been removed. (Contributed by Barry Warsaw in :gh:`98040`.) * Removed the ``suspicious`` rule from the documentation Makefile, and diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 8dbe2a1d828b4e..7af0c0288376fd 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -319,7 +319,7 @@ aspects that are visible to the programmer: >>> collections.__cached__ # doctest: +SKIP 'c:/py32/lib/__pycache__/collections.cpython-32.pyc' -* The tag that is unique to each interpreter is accessible from the :mod:`imp` +* The tag that is unique to each interpreter is accessible from the :mod:`!imp` module: >>> import imp # doctest: +SKIP @@ -328,7 +328,7 @@ aspects that are visible to the programmer: * Scripts that try to deduce source filename from the imported file now need to be smarter. It is no longer sufficient to simply strip the "c" from a ".pyc" - filename. Instead, use the new functions in the :mod:`imp` module: + filename. Instead, use the new functions in the :mod:`!imp` module: >>> imp.source_from_cache('c:/py32/lib/__pycache__/collections.cpython-32.pyc') # doctest: +SKIP 'c:/py32/lib/collections.py' diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index df1f2ab775b047..f121652ba51cbc 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -685,7 +685,7 @@ through normal attribute access. Using importlib as the Implementation of Import =============================================== :issue:`2377` - Replace __import__ w/ importlib.__import__ -:issue:`13959` - Re-implement parts of :mod:`imp` in pure Python +:issue:`13959` - Re-implement parts of :mod:`!imp` in pure Python :issue:`14605` - Make import machinery explicit :issue:`14646` - Require loaders set __loader__ and __package__ @@ -762,7 +762,7 @@ Loaders are also now expected to set the ``__package__`` attribute from from :mod:`importlib` and import itself is setting the attribute post-load. ``None`` is now inserted into :attr:`sys.path_importer_cache` when no finder -can be found on :attr:`sys.path_hooks`. Since :class:`imp.NullImporter` is not +can be found on :attr:`sys.path_hooks`. Since :class:`!imp.NullImporter` is not directly exposed on :attr:`sys.path_hooks` it could no longer be relied upon to always be available to use as a value representing no finder found. @@ -2385,7 +2385,7 @@ Porting Python code * Because ``None`` is now inserted into :attr:`sys.path_importer_cache`, if you are clearing out entries in the dictionary of paths that do not have a finder, you will need to remove keys paired with values of ``None`` **and** - :class:`imp.NullImporter` to be backwards-compatible. This will lead to extra + :class:`!imp.NullImporter` to be backwards-compatible. This will lead to extra overhead on older versions of Python that re-insert ``None`` into :attr:`sys.path_importer_cache` where it represents the use of implicit finders, but semantically it should not change anything. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 53f78e3621cfbd..dabfdaa230ff08 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -991,18 +991,18 @@ for the :meth:`~importlib.abc.InspectLoader.get_code` method. However, it will normally be desirable to override the default implementation for performance reasons. (Contributed by Brett Cannon in :issue:`18072`.) -The :func:`~importlib.reload` function has been moved from :mod:`imp` to -:mod:`importlib` as part of the :mod:`imp` module deprecation. (Contributed by +The :func:`~importlib.reload` function has been moved from :mod:`!imp` to +:mod:`importlib` as part of the :mod:`!imp` module deprecation. (Contributed by Berker Peksag in :issue:`18193`.) :mod:`importlib.util` now has a :data:`~importlib.util.MAGIC_NUMBER` attribute providing access to the bytecode version number. This replaces the -:func:`~imp.get_magic` function in the deprecated :mod:`imp` module. +:func:`!get_magic` function in the deprecated :mod:`!imp` module. (Contributed by Brett Cannon in :issue:`18192`.) New :mod:`importlib.util` functions :func:`~importlib.util.cache_from_source` and :func:`~importlib.util.source_from_cache` replace the same-named functions -in the deprecated :mod:`imp` module. (Contributed by Brett Cannon in +in the deprecated :mod:`!imp` module. (Contributed by Brett Cannon in :issue:`18194`.) The :mod:`importlib` bootstrap :class:`.NamespaceLoader` now conforms to @@ -2101,7 +2101,7 @@ Deprecations in the Python API and :meth:`importlib.util.set_package` are no longer needed because their functions are now handled automatically by the import system. -* The :mod:`imp` module is pending deprecation. To keep compatibility with +* The :mod:`!imp` module is pending deprecation. To keep compatibility with Python 2/3 code bases, the module's removal is currently not scheduled. * The :mod:`formatter` module is pending deprecation and is slated for removal @@ -2300,7 +2300,7 @@ Changes in the Python API then you can see if the module's ``__spec__.location`` is set to ``'frozen'``, check if the loader is a subclass of :class:`importlib.machinery.FrozenImporter`, - or if Python 2 compatibility is necessary you can use :func:`imp.is_frozen`. + or if Python 2 compatibility is necessary you can use :func:`!imp.is_frozen`. * :func:`py_compile.compile` now raises :exc:`FileExistsError` if the file path it would write to is a symlink or a non-regular file. This is to act as a diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index c7faaebfed62b3..3a681754e25dd7 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2180,7 +2180,7 @@ Changes in the Python API now raises :exc:`ValueError` for out-of-range values, rather than returning :const:`None`. See :issue:`20059`. -* The :mod:`imp` module now raises a :exc:`DeprecationWarning` instead of +* The :mod:`!imp` module now raises a :exc:`DeprecationWarning` instead of :exc:`PendingDeprecationWarning`. * The following modules have had missing APIs added to their :attr:`__all__` diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index db2eba32e6ea34..854458f2d1a994 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -3818,7 +3818,7 @@ user. .. section: Library The :2to3fixer:`reload` fixer now uses :func:`importlib.reload` instead of -deprecated :func:`imp.reload`. +deprecated :func:`!imp.reload`. .. From 8b03e5ff94d494109a84a106c1471175d42dd8fe Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 3 May 2023 13:57:23 +0100 Subject: [PATCH 18/65] GH-97850: Suppress cross-references to the removed ``module_repr`` method (#104133) Suppress cross-references to ``module_repr`` --- Doc/reference/import.rst | 2 +- Doc/whatsnew/3.10.rst | 10 +++++----- Doc/whatsnew/3.11.rst | 2 +- Doc/whatsnew/3.12.rst | 2 +- Doc/whatsnew/3.4.rst | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 8f30181ae0a10d..0f416a5c583f85 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -706,7 +706,7 @@ Here are the exact rules used: * Otherwise, just use the module's ``__name__`` in the repr. .. versionchanged:: 3.12 - Use of :meth:`module_repr`, having been deprecated since Python 3.4, was + Use of :meth:`!module_repr`, having been deprecated since Python 3.4, was removed in Python 3.12 and is no longer called during the resolution of a module's repr. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 3d7bcf89694477..f689930ff0caca 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1612,7 +1612,7 @@ Deprecated (superseded by :meth:`~importlib.abc.Finder.find_spec`), :meth:`~importlib.abc.Loader.load_module` (superseded by :meth:`~importlib.abc.Loader.exec_module`), - :meth:`~importlib.abc.Loader.module_repr` (which the import system + :meth:`!module_repr` (which the import system takes care of for you), the ``__package__`` attribute (superseded by ``__spec__.parent``), the ``__loader__`` attribute (superseded by ``__spec__.loader``), and the ``__cached__`` attribute @@ -1693,14 +1693,14 @@ Deprecated (Contributed by Brett Cannon in :issue:`43720`.) * The import system now uses the ``__spec__`` attribute on modules before - falling back on :meth:`~importlib.abc.Loader.module_repr` for a module's + falling back on :meth:`!module_repr` for a module's ``__repr__()`` method. Removal of the use of ``module_repr()`` is scheduled for Python 3.12. (Contributed by Brett Cannon in :issue:`42137`.) -* :meth:`importlib.abc.Loader.module_repr`, - :meth:`importlib.machinery.FrozenLoader.module_repr`, and - :meth:`importlib.machinery.BuiltinLoader.module_repr` are deprecated and +* :meth:`!importlib.abc.Loader.module_repr`, + :meth:`!importlib.machinery.FrozenLoader.module_repr`, and + :meth:`!importlib.machinery.BuiltinLoader.module_repr` are deprecated and slated for removal in Python 3.12. (Contributed by Brett Cannon in :issue:`42136`.) diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 6df30ad6c19c2c..7d3639f67952f6 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1877,7 +1877,7 @@ C APIs pending removal are * The :class:`typing.re ` namespace * :func:`!cgi.log` * :func:`!importlib.find_loader` -* :meth:`importlib.abc.Loader.module_repr` +* :meth:`!importlib.abc.Loader.module_repr` * :meth:`!importlib.abc.MetaPathFinder.find_module` * :meth:`!importlib.abc.PathEntryFinder.find_loader` * :meth:`!importlib.abc.PathEntryFinder.find_module` diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 4ed08f3ca3b3d3..3381ce7b6b0d56 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -983,7 +983,7 @@ Removed * Many previously deprecated cleanups in :mod:`importlib` have now been completed: - * References to, and support for ``module_repr()`` has been removed. + * References to, and support for :meth:`!module_repr()` has been removed. (Contributed by Barry Warsaw in :gh:`97850`.) * ``importlib.util.set_package`` has been removed. (Contributed by Brett diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index dabfdaa230ff08..38f9449b9afe3a 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2096,7 +2096,7 @@ Deprecations in the Python API :meth:`importlib.abc.InspectLoader.exec_module` :meth:`importlib.abc.SourceLoader.exec_module`) and let the import system take care of the rest; and - :meth:`importlib.abc.Loader.module_repr`, + :meth:`!importlib.abc.Loader.module_repr`, :meth:`importlib.util.module_for_loader`, :meth:`importlib.util.set_loader`, and :meth:`importlib.util.set_package` are no longer needed because their functions are now handled automatically by the import system. From 423d7faeb37b6c13b3ebbf9255165fefc651983e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 3 May 2023 14:13:03 +0100 Subject: [PATCH 19/65] GH-97850: Suppress cross-references to removed ``importlib.util`` functions (#104134) `importlib.utils` -> `importlib.util` in a few places --------- Co-authored-by: Alex Waygood --- Doc/whatsnew/3.10.rst | 6 +++--- Doc/whatsnew/3.11.rst | 2 +- Doc/whatsnew/3.4.rst | 6 +++--- Misc/NEWS.d/3.10.0a5.rst | 2 +- Misc/NEWS.d/3.12.0a1.rst | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index f689930ff0caca..661eeaedbfc0d0 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -1683,9 +1683,9 @@ Deprecated (Contributed by Brett Cannon in :issue:`42135`.) * The deprecations of :mod:`!imp`, :func:`!importlib.find_loader`, - :func:`importlib.util.set_package_wrapper`, - :func:`importlib.util.set_loader_wrapper`, - :func:`importlib.util.module_for_loader`, + :func:`!importlib.util.set_package_wrapper`, + :func:`!importlib.util.set_loader_wrapper`, + :func:`!importlib.util.module_for_loader`, :class:`!pkgutil.ImpImporter`, and :class:`!pkgutil.ImpLoader` have all been updated to list Python 3.12 as the slated version of removal (they began raising :exc:`DeprecationWarning` in diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 7d3639f67952f6..7a479c6e56a9f0 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1889,7 +1889,7 @@ C APIs pending removal are * :meth:`!importlib.machinery.FrozenLoader.module_repr` * :meth:`!importlib.machinery.PathFinder.find_module` * :meth:`!importlib.machinery.WindowsRegistryFinder.find_module` -* :func:`importlib.util.module_for_loader` +* :func:`!importlib.util.module_for_loader` * :func:`!importlib.util.set_loader_wrapper` * :func:`!importlib.util.set_package_wrapper` * :class:`!pkgutil.ImpImporter` diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 38f9449b9afe3a..45bb91833a352b 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2097,8 +2097,8 @@ Deprecations in the Python API :meth:`importlib.abc.SourceLoader.exec_module`) and let the import system take care of the rest; and :meth:`!importlib.abc.Loader.module_repr`, - :meth:`importlib.util.module_for_loader`, :meth:`importlib.util.set_loader`, - and :meth:`importlib.util.set_package` are no longer needed because their + :meth:`!importlib.util.module_for_loader`, :meth:`!importlib.util.set_loader`, + and :meth:`!importlib.util.set_package` are no longer needed because their functions are now handled automatically by the import system. * The :mod:`!imp` module is pending deprecation. To keep compatibility with @@ -2277,7 +2277,7 @@ Changes in the Python API in a backwards-compatible fashion, use e.g. ``getattr(module, '__loader__', None) is not None``. (:issue:`17115`.) -* :meth:`importlib.util.module_for_loader` now sets ``__loader__`` and +* :meth:`!importlib.util.module_for_loader` now sets ``__loader__`` and ``__package__`` unconditionally to properly support reloading. If this is not desired then you will need to set these attributes manually. You can use :func:`importlib.util.module_to_load` for module management. diff --git a/Misc/NEWS.d/3.10.0a5.rst b/Misc/NEWS.d/3.10.0a5.rst index 1c7c7447cae065..497e3849171831 100644 --- a/Misc/NEWS.d/3.10.0a5.rst +++ b/Misc/NEWS.d/3.10.0a5.rst @@ -499,7 +499,7 @@ Araujo. .. nonce: HY2beA .. section: Documentation -Updated importlib.utils.resolve_name() doc to use __spec__.parent instead of +Updated importlib.util.resolve_name() doc to use __spec__.parent instead of __package__. (Thanks Yair Frid.) .. diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index 075e8da825a331..ff5064f89d8dd8 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -2028,8 +2028,8 @@ resources. .. nonce: NzdREm .. section: Library -Remove deprecated :func:`importlib.utils.set_loader` and -:func:`importlib.utils.module_for_loader` from :mod:`importlib.utils`. +Remove deprecated :func:`!importlib.util.set_loader` and +:func:`!importlib.util.module_for_loader` from :mod:`importlib.util`. .. From 524a7f77fd8244835e382f076dd4a76404580bb3 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 3 May 2023 15:17:14 +0200 Subject: [PATCH 20/65] gh-103968: Deprecate creating heap types whose metaclass has custom tp_new. (GH-103972) (That's a mouthful of an edge case!) Co-authored-by: Barney Gale --- Doc/c-api/type.rst | 26 ++++++++++++- Doc/whatsnew/3.12.rst | 20 ++++++++++ Lib/test/test_capi/test_misc.py | 14 +++++++ ...-04-28-18-04-38.gh-issue-103968.EnVvOx.rst | 4 ++ Modules/_testcapi/heaptype.c | 16 +++++++- Objects/typeobject.c | 38 ++++++++++++++----- 6 files changed, 106 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 69b15296993301..9fd40e1008c4a8 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -256,8 +256,13 @@ The following functions and structs are used to create The metaclass *metaclass* is used to construct the resulting type object. When *metaclass* is ``NULL``, the metaclass is derived from *bases* (or *Py_tp_base[s]* slots if *bases* is ``NULL``, see below). - Note that metaclasses that override - :c:member:`~PyTypeObject.tp_new` are not supported. + + Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not + supported. + (For backwards compatibility, other ``PyType_From*`` functions allow + such metaclasses. They ignore ``tp_new``, which may result in incomplete + initialization. This is deprecated and in Python 3.14+ such metaclasses will + not be supported.) The *bases* argument can be used to specify base classes; it can either be only one class or a tuple of classes. @@ -305,6 +310,11 @@ The following functions and structs are used to create The function now finds and uses a metaclass corresponding to the provided base classes. Previously, only :class:`type` instances were returned. + The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. + which may result in incomplete initialization. + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it + will be no longer allowed. .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) @@ -317,6 +327,12 @@ The following functions and structs are used to create The function now finds and uses a metaclass corresponding to the provided base classes. Previously, only :class:`type` instances were returned. + The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. + which may result in incomplete initialization. + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it + will be no longer allowed. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``. @@ -327,6 +343,12 @@ The following functions and structs are used to create base classes provided in *Py_tp_base[s]* slots. Previously, only :class:`type` instances were returned. + The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. + which may result in incomplete initialization. + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it + will be no longer allowed. + .. c:type:: PyType_Spec Structure defining a type's behavior. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3381ce7b6b0d56..63db5d3f89be19 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1320,6 +1320,21 @@ Porting to Python 3.12 available on debug builds. If you happen to be using it then you'll need to start using ``_Py_GetGlobalRefTotal()``. +* The following functions now select an appropriate metaclass for the newly + created type: + + * :c:func:`PyType_FromSpec` + * :c:func:`PyType_FromSpecWithBases` + * :c:func:`PyType_FromModuleAndSpec` + + Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` + is deprecated, and in Python 3.14+ it will be disallowed. + Note that these functions ignore ``tp_new`` of the metaclass, possibly + allowing incomplete initialization. + + Note that :c:func:`PyType_FromMetaclass` (added in Python 3.12) + already disallows creating classes whose metaclass overrides ``tp_new``. + Deprecated ---------- @@ -1396,6 +1411,11 @@ Deprecated * ``_PyErr_ChainExceptions`` is deprecated. Use ``_PyErr_ChainExceptions1`` instead. (Contributed by Irit Katriel in :gh:`102192`.) +* Using :c:func:`PyType_FromSpec`, :c:func:`PyType_FromSpecWithBases` + or :c:func:`PyType_FromModuleAndSpec` to create a class whose metaclass + overrides :c:member:`~PyTypeObject.tp_new` is deprecated. + Call the metaclass instead. + Removed ------- diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 9d5d1ca6e7dce2..1d426d0f8f825c 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -681,6 +681,20 @@ def test_heaptype_with_custom_metaclass(self): with self.assertRaisesRegex(TypeError, msg): t = _testcapi.pytype_fromspec_meta(_testcapi.HeapCTypeMetaclassCustomNew) + def test_heaptype_with_custom_metaclass_deprecation(self): + # gh-103968: a metaclass with custom tp_new is deprecated, but still + # allowed for functions that existed in 3.11 + # (PyType_FromSpecWithBases is used here). + class Base(metaclass=_testcapi.HeapCTypeMetaclassCustomNew): + pass + + with warnings_helper.check_warnings( + ('.*custom tp_new.*in Python 3.14.*', DeprecationWarning), + ): + sub = _testcapi.make_type_with_base(Base) + self.assertTrue(issubclass(sub, Base)) + self.assertIsInstance(sub, _testcapi.HeapCTypeMetaclassCustomNew) + def test_multiple_inheritance_ctypes_with_weakref_or_dict(self): with self.assertRaises(TypeError): diff --git a/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst b/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst new file mode 100644 index 00000000000000..5e4270f82afd84 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-04-28-18-04-38.gh-issue-103968.EnVvOx.rst @@ -0,0 +1,4 @@ +:c:func:`PyType_FromSpec` and its variants now allow creating classes whose +metaclass overrides :c:member:`~PyTypeObject.tp_new`. The ``tp_new`` is +ignored. This behavior is deprecated and will be disallowed in 3.14+. The +new :c:func:`PyType_FromMetaclass` already disallows it. diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 209cc182c0698d..6384fbc485fbdf 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -22,7 +22,7 @@ static PyObject *pytype_fromspec_meta(PyObject* self, PyObject *meta) "_testcapi.HeapCTypeViaMetaclass", sizeof(PyObject), 0, - Py_TPFLAGS_DEFAULT, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, HeapCTypeViaMetaclass_slots }; @@ -385,6 +385,19 @@ make_immutable_type_with_base(PyObject *self, PyObject *base) return PyType_FromSpecWithBases(&ImmutableSubclass_spec, base); } +static PyObject * +make_type_with_base(PyObject *self, PyObject *base) +{ + assert(PyType_Check(base)); + PyType_Spec ImmutableSubclass_spec = { + .name = "_testcapi.Subclass", + .basicsize = (int)((PyTypeObject*)base)->tp_basicsize, + .slots = empty_type_slots, + .flags = Py_TPFLAGS_DEFAULT, + }; + return PyType_FromSpecWithBases(&ImmutableSubclass_spec, base); +} + static PyMethodDef TestMethods[] = { {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O}, @@ -397,6 +410,7 @@ static PyMethodDef TestMethods[] = { test_from_spec_invalid_metatype_inheritance, METH_NOARGS}, {"make_immutable_type_with_base", make_immutable_type_with_base, METH_O}, + {"make_type_with_base", make_type_with_base, METH_O}, {NULL}, }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index cf0efe199b2828..4ced04b0bde966 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3950,9 +3950,10 @@ check_basicsize_includes_size_and_offsets(PyTypeObject* type) return 1; } -PyObject * -PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, - PyType_Spec *spec, PyObject *bases_in) +static PyObject * +_PyType_FromMetaclass_impl( + PyTypeObject *metaclass, PyObject *module, + PyType_Spec *spec, PyObject *bases_in, int _allow_tp_new) { /* Invariant: A non-NULL value in one of these means this function holds * a strong reference or owns allocated memory. @@ -4127,9 +4128,21 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, goto finally; } if (metaclass->tp_new != PyType_Type.tp_new) { - PyErr_SetString(PyExc_TypeError, - "Metaclasses with custom tp_new are not supported."); - goto finally; + if (_allow_tp_new) { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Using PyType_Spec with metaclasses that have custom " + "tp_new is deprecated and will no longer be allowed in " + "Python 3.14.") < 0) { + goto finally; + } + } + else { + PyErr_SetString( + PyExc_TypeError, + "Metaclasses with custom tp_new are not supported."); + goto finally; + } } /* Calculate best base, and check that all bases are type objects */ @@ -4316,22 +4329,29 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, return (PyObject*)res; } +PyObject * +PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, + PyType_Spec *spec, PyObject *bases_in) +{ + return _PyType_FromMetaclass_impl(metaclass, module, spec, bases_in, 0); +} + PyObject * PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) { - return PyType_FromMetaclass(NULL, module, spec, bases); + return _PyType_FromMetaclass_impl(NULL, module, spec, bases, 1); } PyObject * PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) { - return PyType_FromMetaclass(NULL, NULL, spec, bases); + return _PyType_FromMetaclass_impl(NULL, NULL, spec, bases, 1); } PyObject * PyType_FromSpec(PyType_Spec *spec) { - return PyType_FromMetaclass(NULL, NULL, spec, NULL); + return _PyType_FromMetaclass_impl(NULL, NULL, spec, NULL, 1); } PyObject * From 0fc58c66bafbd20f02c206c801cf9ab939853164 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 3 May 2023 07:04:50 -0700 Subject: [PATCH 21/65] gh-103693: Add convenience variable feature to `pdb` (#103694) --- Doc/library/pdb.rst | 15 ++++ Doc/whatsnew/3.12.rst | 8 ++ Lib/pdb.py | 17 ++++ Lib/test/test_pdb.py | 78 +++++++++++++++++++ ...-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst | 1 + 5 files changed, 119 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 5bc48a6d5f77fd..8a386aa77368f2 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -263,6 +263,21 @@ the commands; the input is split at the first ``;;`` pair, even if it is in the middle of a quoted string. A workaround for strings with double semicolons is to use implicit string concatenation ``';'';'`` or ``";"";"``. +To set a temporary global variable, use a *convenience variable*. A *convenience +variable* is a variable whose name starts with ``$``. For example, ``$foo = 1`` +sets a global variable ``$foo`` which you can use in the debugger session. The +*convenience variables* are cleared when the program resumes execution so it's +less likely to interfere with your program compared to using normal variables +like ``foo = 1``. + +There are three preset *convenience variables*: + +* ``$_frame``: the current frame you are debugging +* ``$_retval``: the return value if the frame is returning +* ``$_exception``: the exception if the frame is raising an exception + +.. versionadded:: 3.12 + .. index:: pair: .pdbrc; file triple: debugger; configuration; file diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 63db5d3f89be19..94b44ed9ce0953 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -408,6 +408,14 @@ os.path * Add :func:`os.path.splitroot` to split a path into a triad ``(drive, root, tail)``. (Contributed by Barney Gale in :gh:`101000`.) +pdb +--- + +* Add convenience variables to hold values temporarily for debug session + and provide quick access to values like the current frame or the return + value. + (Contributed by Tian Gao in :gh:`103693`.) + shutil ------ diff --git a/Lib/pdb.py b/Lib/pdb.py index 645cbf518e58e3..b3dc5a455e56b9 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -270,6 +270,8 @@ def forget(self): self.lineno = None self.stack = [] self.curindex = 0 + if hasattr(self, 'curframe') and self.curframe: + self.curframe.f_globals.pop('__pdb_convenience_variables', None) self.curframe = None self.tb_lineno.clear() @@ -288,6 +290,7 @@ def setup(self, f, tb): # locals whenever the .f_locals accessor is called, so we # cache it here to ensure that modifications are not overwritten. self.curframe_locals = self.curframe.f_locals + self.set_convenience_variable(self.curframe, '_frame', self.curframe) return self.execRcLines() # Can be executed earlier than 'setup' if desired @@ -359,6 +362,7 @@ def user_return(self, frame, return_value): if self._wait_for_mainpyfile: return frame.f_locals['__return__'] = return_value + self.set_convenience_variable(frame, '_retval', return_value) self.message('--Return--') self.interaction(frame, None) @@ -369,6 +373,7 @@ def user_exception(self, frame, exc_info): return exc_type, exc_value, exc_traceback = exc_info frame.f_locals['__exception__'] = exc_type, exc_value + self.set_convenience_variable(frame, '_exception', exc_value) # An 'Internal StopIteration' exception is an exception debug event # issued by the interpreter when handling a subgenerator run with @@ -394,6 +399,7 @@ def _cmdloop(self): self.message('--KeyboardInterrupt--') # Called before loop, handles display expressions + # Set up convenience variable containers def preloop(self): displaying = self.displaying.get(self.curframe) if displaying: @@ -477,6 +483,9 @@ def precmd(self, line): next = line[marker+2:].lstrip() self.cmdqueue.append(next) line = line[:marker].rstrip() + + # Replace all the convenience variables + line = re.sub(r'\$([a-zA-Z_][a-zA-Z0-9_]*)', r'__pdb_convenience_variables["\1"]', line) return line def onecmd(self, line): @@ -527,6 +536,13 @@ def message(self, msg): def error(self, msg): print('***', msg, file=self.stdout) + # convenience variables + + def set_convenience_variable(self, frame, name, value): + if '__pdb_convenience_variables' not in frame.f_globals: + frame.f_globals['__pdb_convenience_variables'] = {} + frame.f_globals['__pdb_convenience_variables'][name] = value + # Generic completion functions. Individual complete_foo methods can be # assigned below to one of these functions. @@ -1018,6 +1034,7 @@ def _select_frame(self, number): self.curindex = number self.curframe = self.stack[self.curindex][0] self.curframe_locals = self.curframe.f_locals + self.set_convenience_variable(self.curframe, '_frame', self.curframe) self.print_stack_entry(self.stack[self.curindex]) self.lineno = None diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 2f712a10257984..482c92dbf1f6a0 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -746,6 +746,84 @@ def test_pdb_where_command(): (Pdb) continue """ +def test_convenience_variables(): + """Test convenience variables + + >>> def util_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... try: + ... raise Exception('test') + ... except: + ... pass + ... return 1 + + >>> def test_function(): + ... util_function() + + >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... '$_frame.f_lineno', # Check frame convenience variable + ... '$a = 10', # Set a convenience variable + ... '$a', # Print its value + ... 'p $a + 2', # Do some calculation + ... 'u', # Switch frame + ... '$_frame.f_lineno', # Make sure the frame changed + ... '$a', # Make sure the value persists + ... 'd', # Go back to the original frame + ... 'next', + ... '$a', # The value should be gone + ... 'next', + ... '$_exception', # Check exception convenience variable + ... 'next', + ... '$_exception', # Exception should be gone + ... 'return', + ... '$_retval', # Check return convenience variable + ... 'continue', + ... ]): + ... test_function() + > (3)util_function() + -> try: + (Pdb) $_frame.f_lineno + 3 + (Pdb) $a = 10 + (Pdb) $a + 10 + (Pdb) p $a + 2 + 12 + (Pdb) u + > (2)test_function() + -> util_function() + (Pdb) $_frame.f_lineno + 2 + (Pdb) $a + 10 + (Pdb) d + > (3)util_function() + -> try: + (Pdb) next + > (4)util_function() + -> raise Exception('test') + (Pdb) $a + *** KeyError: 'a' + (Pdb) next + Exception: test + > (4)util_function() + -> raise Exception('test') + (Pdb) $_exception + Exception('test') + (Pdb) next + > (5)util_function() + -> except: + (Pdb) $_exception + *** KeyError: '_exception' + (Pdb) return + --Return-- + > (7)util_function()->1 + -> return 1 + (Pdb) $_retval + 1 + (Pdb) continue + """ + def test_post_mortem(): """Test post mortem traceback debugging. diff --git a/Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst b/Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst new file mode 100644 index 00000000000000..52c68bfc9ceea4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-22-21-34-13.gh-issue-103693.SBtuLQ.rst @@ -0,0 +1 @@ +Add convenience variable feature to :mod:`pdb` From 38dc3f28dde92b4fa6284a57f4e554991a3d9276 Mon Sep 17 00:00:00 2001 From: Paul Ganssle <1377457+pganssle@users.noreply.github.com> Date: Wed, 3 May 2023 11:17:27 -0400 Subject: [PATCH 22/65] GH-103944: Remove last use of `utcfromtimestamp` (#103995) * Remove last use of `utcfromtimestamp` This was a weirdly valid use of `utcfromtimestamp` in the sense that the "timestamps" in TZif files are not epoch times, but actually something more properly thought of as "number of seconds since 1970 in the local time zone", so even though we didn't want UTC time, `utcfromtimestamp` was still a good way to get the thing we wanted. Since we're deprecating `utcfromtimestamp`, it's just as valid to use `timedelta` arithmetic here. We may be able to avoid the question entirely by switching these tests over to using `ZoneInfo` in the future. * Fix a few missing DeprecationWarnings in tests In one test, we simply turn off DeprecationWarning rather than asserting about it, because whether the error condition happens before or after the warning seems to differ between the Python and C versions. --- Lib/test/datetimetester.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index fb07d2a5ad9b06..55e061950ff280 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2,18 +2,19 @@ See https://www.zope.dev/Members/fdrake/DateTimeWiki/TestCases """ -import io -import itertools import bisect import copy import decimal -import sys +import io +import itertools import os import pickle import random import re import struct +import sys import unittest +import warnings from array import array @@ -51,11 +52,12 @@ for proto in range(pickle.HIGHEST_PROTOCOL + 1)] assert len(pickle_choices) == pickle.HIGHEST_PROTOCOL + 1 +EPOCH_NAIVE = datetime(1970, 1, 1, 0, 0) # For calculating transitions + # An arbitrary collection of objects of non-datetime types, for testing # mixed-type comparisons. OTHERSTUFF = (10, 34.5, "abc", {}, [], ()) - # XXX Copied from test_float. INF = float("inf") NAN = float("nan") @@ -2626,9 +2628,10 @@ def test_utcfromtimestamp_limits(self): for test_name, ts in test_cases: with self.subTest(test_name, ts=ts): with self.assertRaises((ValueError, OverflowError)): - # converting a Python int to C time_t can raise a - # OverflowError, especially on 32-bit platforms. - self.theclass.utcfromtimestamp(ts) + with self.assertWarns(DeprecationWarning): + # converting a Python int to C time_t can raise a + # OverflowError, especially on 32-bit platforms. + self.theclass.utcfromtimestamp(ts) def test_insane_fromtimestamp(self): # It's possible that some platform maps time_t to double, @@ -2645,8 +2648,9 @@ def test_insane_utcfromtimestamp(self): # exempt such platforms (provided they return reasonable # results!). for insane in -1e200, 1e200: - self.assertRaises(OverflowError, self.theclass.utcfromtimestamp, - insane) + with self.assertWarns(DeprecationWarning): + self.assertRaises(OverflowError, self.theclass.utcfromtimestamp, + insane) @unittest.skipIf(sys.platform == "win32", "Windows doesn't accept negative timestamps") def test_negative_float_fromtimestamp(self): @@ -3005,7 +3009,7 @@ def __new__(cls, *args, **kwargs): for name, meth_name, kwargs in test_cases: with self.subTest(name): constr = getattr(DateTimeSubclass, meth_name) - if constr == "utcnow": + if meth_name == "utcnow": with self.assertWarns(DeprecationWarning): dt = constr(**kwargs) else: @@ -4733,8 +4737,10 @@ def test_tzinfo_utcfromtimestamp(self): # Try with and without naming the keyword; for whatever reason, # utcfromtimestamp() doesn't accept a tzinfo argument. off42 = FixedOffset(42, "42") - self.assertRaises(TypeError, meth, ts, off42) - self.assertRaises(TypeError, meth, ts, tzinfo=off42) + with warnings.catch_warnings(category=DeprecationWarning): + warnings.simplefilter("ignore", category=DeprecationWarning) + self.assertRaises(TypeError, meth, ts, off42) + self.assertRaises(TypeError, meth, ts, tzinfo=off42) def test_tzinfo_timetuple(self): # TestDateTime tested most of this. datetime adds a twist to the @@ -6102,15 +6108,14 @@ def stats(cls, start_year=1): def transitions(self): for (_, prev_ti), (t, ti) in pairs(zip(self.ut, self.ti)): shift = ti[0] - prev_ti[0] - # TODO: Remove this use of utcfromtimestamp - yield datetime.utcfromtimestamp(t), shift + yield (EPOCH_NAIVE + timedelta(seconds=t)), shift def nondst_folds(self): """Find all folds with the same value of isdst on both sides of the transition.""" for (_, prev_ti), (t, ti) in pairs(zip(self.ut, self.ti)): shift = ti[0] - prev_ti[0] if shift < ZERO and ti[1] == prev_ti[1]: - yield datetime.utcfromtimestamp(t), -shift, prev_ti[2], ti[2] + yield _utcfromtimestamp(datetime, t,), -shift, prev_ti[2], ti[2] @classmethod def print_all_nondst_folds(cls, same_abbr=False, start_year=1): From da1980afcb8820ffaa0574df735bc39b1a276a76 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Wed, 3 May 2023 17:28:44 +0100 Subject: [PATCH 23/65] GH-104114: Fix `pathlib.WindowsPath.glob()` use of literal pattern segment case (GH-104116) We now use `_WildcardSelector` to evaluate literal pattern segments, which allows us to retrieve the real filesystem case. This change is necessary in order to implement a *case_sensitive* argument (see GH-81079) and a *follow_symlinks* argument (see GH-77609). --- Lib/pathlib.py | 52 +++++-------------- Lib/test/test_pathlib.py | 4 +- ...-05-03-03-14-33.gh-issue-104114.RG26RD.rst | 3 ++ 3 files changed, 18 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst diff --git a/Lib/pathlib.py b/Lib/pathlib.py index dee19d1f89ad84..8a1651c23d7f4d 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -54,11 +54,6 @@ def _ignore_error(exception): getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) -def _is_wildcard_pattern(pat): - # Whether this pattern needs actual matching using fnmatch, or can - # be looked up directly as a file. - return "*" in pat or "?" in pat or "[" in pat - def _is_case_sensitive(flavour): return flavour.normcase('Aa') == 'Aa' @@ -78,10 +73,8 @@ def _make_selector(pattern_parts, flavour): cls = _ParentSelector elif '**' in pat: raise ValueError("Invalid pattern: '**' can only be an entire path component") - elif _is_wildcard_pattern(pat): - cls = _WildcardSelector else: - cls = _PreciseSelector + cls = _WildcardSelector return cls(pat, child_parts, flavour) @@ -102,17 +95,15 @@ def select_from(self, parent_path): """Iterate over all child paths of `parent_path` matched by this selector. This can contain parent_path itself.""" path_cls = type(parent_path) - is_dir = path_cls.is_dir - exists = path_cls.exists scandir = path_cls._scandir - if not is_dir(parent_path): + if not parent_path.is_dir(): return iter([]) - return self._select_from(parent_path, is_dir, exists, scandir) + return self._select_from(parent_path, scandir) class _TerminatingSelector: - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, scandir): yield parent_path @@ -120,29 +111,12 @@ class _ParentSelector(_Selector): def __init__(self, name, child_parts, flavour): _Selector.__init__(self, child_parts, flavour) - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, scandir): path = parent_path._make_child_relpath('..') - for p in self.successor._select_from(path, is_dir, exists, scandir): + for p in self.successor._select_from(path, scandir): yield p -class _PreciseSelector(_Selector): - - def __init__(self, name, child_parts, flavour): - self.name = name - _Selector.__init__(self, child_parts, flavour) - - def _select_from(self, parent_path, is_dir, exists, scandir): - try: - path = parent_path._make_child_relpath(self.name) - follow = is_dir(path) if self.dironly else exists(path, follow_symlinks=False) - if follow: - for p in self.successor._select_from(path, is_dir, exists, scandir): - yield p - except PermissionError: - return - - class _WildcardSelector(_Selector): def __init__(self, pat, child_parts, flavour): @@ -150,7 +124,7 @@ def __init__(self, pat, child_parts, flavour): self.match = re.compile(fnmatch.translate(pat), flags=flags).fullmatch _Selector.__init__(self, child_parts, flavour) - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, scandir): try: # We must close the scandir() object before proceeding to # avoid exhausting file descriptors when globbing deep trees. @@ -171,7 +145,7 @@ def _select_from(self, parent_path, is_dir, exists, scandir): name = entry.name if self.match(name): path = parent_path._make_child_relpath(name) - for p in self.successor._select_from(path, is_dir, exists, scandir): + for p in self.successor._select_from(path, scandir): yield p except PermissionError: return @@ -182,7 +156,7 @@ class _RecursiveWildcardSelector(_Selector): def __init__(self, pat, child_parts, flavour): _Selector.__init__(self, child_parts, flavour) - def _iterate_directories(self, parent_path, is_dir, scandir): + def _iterate_directories(self, parent_path, scandir): yield parent_path try: # We must close the scandir() object before proceeding to @@ -198,18 +172,18 @@ def _iterate_directories(self, parent_path, is_dir, scandir): raise if entry_is_dir and not entry.is_symlink(): path = parent_path._make_child_relpath(entry.name) - for p in self._iterate_directories(path, is_dir, scandir): + for p in self._iterate_directories(path, scandir): yield p except PermissionError: return - def _select_from(self, parent_path, is_dir, exists, scandir): + def _select_from(self, parent_path, scandir): try: yielded = set() try: successor_select = self.successor._select_from - for starting_point in self._iterate_directories(parent_path, is_dir, scandir): - for p in successor_select(starting_point, is_dir, exists, scandir): + for starting_point in self._iterate_directories(parent_path, scandir): + for p in successor_select(starting_point, scandir): if p not in yielded: yield p yielded.add(p) diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 620d480e37e2ba..424bb92a87d112 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -3122,7 +3122,7 @@ def test_glob(self): self.assertEqual(set(p.glob("FILEa")), { P(BASE, "fileA") }) self.assertEqual(set(p.glob("*a\\")), { P(BASE, "dirA") }) self.assertEqual(set(p.glob("F*a")), { P(BASE, "fileA") }) - self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\FILEa"}) + self.assertEqual(set(map(str, p.glob("FILEa"))), {f"{p}\\fileA"}) self.assertEqual(set(map(str, p.glob("F*a"))), {f"{p}\\fileA"}) def test_rglob(self): @@ -3130,7 +3130,7 @@ def test_rglob(self): p = P(BASE, "dirC") self.assertEqual(set(p.rglob("FILEd")), { P(BASE, "dirC/dirD/fileD") }) self.assertEqual(set(p.rglob("*\\")), { P(BASE, "dirC/dirD") }) - self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\FILEd"}) + self.assertEqual(set(map(str, p.rglob("FILEd"))), {f"{p}\\dirD\\fileD"}) def test_expanduser(self): P = self.cls diff --git a/Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst b/Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst new file mode 100644 index 00000000000000..e705fea8326e7a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-03-03-14-33.gh-issue-104114.RG26RD.rst @@ -0,0 +1,3 @@ +Fix issue where :meth:`pathlib.Path.glob` returns paths using the case of +non-wildcard segments for corresponding path segments, rather than the real +filesystem case. From 9f9e001ab2ab67acdb2e0383a25ab4c164608a47 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 3 May 2023 21:55:19 +0100 Subject: [PATCH 24/65] gh-103590: mention that the change is included in 3.11.4 and clarify except* documentation (#104095) --- Doc/reference/compound_stmts.rst | 6 ++++-- Doc/whatsnew/3.12.rst | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index f0a8936c35bf4a..0731589e4cadb8 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -362,8 +362,10 @@ one :keyword:`!except*` clause, the first that matches it. :: Any remaining exceptions that were not handled by any :keyword:`!except*` -clause are re-raised at the end, combined into an exception group along with -all exceptions that were raised from within :keyword:`!except*` clauses. +clause are re-raised at the end, along with all exceptions that were +raised from within the :keyword:`!except*` clauses. If this list contains +more than one exception to reraise, they are combined into an exception +group. If the raised exception is not an exception group and its type matches one of the :keyword:`!except*` clauses, it is caught and wrapped by an diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 94b44ed9ce0953..49f9bd28248dac 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -239,7 +239,8 @@ Other Language Changes * When a ``try-except*`` construct handles the entire :exc:`ExceptionGroup` and raises one other exception, that exception is no longer wrapped in an - :exc:`ExceptionGroup`. (Contributed by Irit Katriel in :gh:`103590`.) + :exc:`ExceptionGroup`. Also changed in version 3.11.4. (Contributed by Irit + Katriel in :gh:`103590`.) New Modules From d6e83fbf30fb25996b547d8a2444814437e228e5 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Thu, 4 May 2023 02:11:54 +0300 Subject: [PATCH 25/65] gh-97850: Deprecate `find_loader` and `get_loader` in `pkgutil` (GH-98520) Co-authored-by: C.A.M. Gerlach Co-authored-by: Brett Cannon Co-authored-by: Oleg Iarygin --- Doc/library/pkgutil.rst | 7 ++++ Doc/whatsnew/3.12.rst | 5 +++ Lib/pkgutil.py | 8 ++++ Lib/test/test_pkgutil.py | 39 ++++++++++--------- ...2-10-21-16-23-31.gh-issue-97850.N46coo.rst | 2 + 5 files changed, 43 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 98e6e294af0c5c..891a867d1ceb68 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -64,6 +64,10 @@ support. .. versionchanged:: 3.4 Updated to be based on :pep:`451` + .. deprecated-removed:: 3.12 3.14 + Use :func:`importlib.util.find_spec` instead. + + .. function:: get_importer(path_item) Retrieve a :term:`finder` for the given *path_item*. @@ -96,6 +100,9 @@ support. .. versionchanged:: 3.4 Updated to be based on :pep:`451` + .. deprecated-removed:: 3.12 3.14 + Use :func:`importlib.util.find_spec` instead. + .. function:: iter_importers(fullname='') diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 49f9bd28248dac..3dfd787e3df837 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -813,6 +813,11 @@ Pending Removal in Python 3.14 * The *onerror* argument of :func:`shutil.rmtree` is deprecated in 3.12, and will be removed in 3.14. +* :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` + now raise :exc:`DeprecationWarning`; + use :func:`importlib.util.find_spec` instead. + (Contributed by Nikita Sobolev in :gh:`97850`.) + Pending Removal in Future Versions ---------------------------------- diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index f62eccb974d6f5..dccbec52aa731e 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -270,6 +270,10 @@ def get_loader(module_or_name): If the named module is not already imported, its containing package (if any) is imported, in order to establish the package __path__. """ + warnings._deprecated("pkgutil.get_loader", + f"{warnings._DEPRECATED_MSG}; " + "use importlib.util.find_spec() instead", + remove=(3, 14)) if module_or_name in sys.modules: module_or_name = sys.modules[module_or_name] if module_or_name is None: @@ -294,6 +298,10 @@ def find_loader(fullname): importlib.util.find_spec that converts most failures to ImportError and only returns the loader rather than the full spec """ + warnings._deprecated("pkgutil.find_loader", + f"{warnings._DEPRECATED_MSG}; " + "use importlib.util.find_spec() instead", + remove=(3, 14)) if fullname.startswith('.'): msg = "Relative module name {!r} not supported".format(fullname) raise ImportError(msg) diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 902627088fc291..6fcd726345eeac 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -1,6 +1,6 @@ from pathlib import Path from test.support.import_helper import unload, CleanImport -from test.support.warnings_helper import check_warnings +from test.support.warnings_helper import check_warnings, ignore_warnings import unittest import sys import importlib @@ -535,25 +535,18 @@ class ImportlibMigrationTests(unittest.TestCase): # PEP 302 emulation in this module is in the process of being # deprecated in favour of importlib proper - def test_get_loader_avoids_emulation(self): - with check_warnings() as w: - self.assertIsNotNone(pkgutil.get_loader("sys")) - self.assertIsNotNone(pkgutil.get_loader("os")) - self.assertIsNotNone(pkgutil.get_loader("test.support")) - self.assertEqual(len(w.warnings), 0) - @unittest.skipIf(__name__ == '__main__', 'not compatible with __main__') + @ignore_warnings(category=DeprecationWarning) def test_get_loader_handles_missing_loader_attribute(self): global __loader__ this_loader = __loader__ del __loader__ try: - with check_warnings() as w: - self.assertIsNotNone(pkgutil.get_loader(__name__)) - self.assertEqual(len(w.warnings), 0) + self.assertIsNotNone(pkgutil.get_loader(__name__)) finally: __loader__ = this_loader + @ignore_warnings(category=DeprecationWarning) def test_get_loader_handles_missing_spec_attribute(self): name = 'spam' mod = type(sys)(name) @@ -563,6 +556,7 @@ def test_get_loader_handles_missing_spec_attribute(self): loader = pkgutil.get_loader(name) self.assertIsNone(loader) + @ignore_warnings(category=DeprecationWarning) def test_get_loader_handles_spec_attribute_none(self): name = 'spam' mod = type(sys)(name) @@ -572,6 +566,7 @@ def test_get_loader_handles_spec_attribute_none(self): loader = pkgutil.get_loader(name) self.assertIsNone(loader) + @ignore_warnings(category=DeprecationWarning) def test_get_loader_None_in_sys_modules(self): name = 'totally bogus' sys.modules[name] = None @@ -581,18 +576,26 @@ def test_get_loader_None_in_sys_modules(self): del sys.modules[name] self.assertIsNone(loader) + def test_get_loader_is_deprecated(self): + with check_warnings( + (r".*\bpkgutil.get_loader\b.*", DeprecationWarning), + ): + res = pkgutil.get_loader("sys") + self.assertIsNotNone(res) + + def test_find_loader_is_deprecated(self): + with check_warnings( + (r".*\bpkgutil.find_loader\b.*", DeprecationWarning), + ): + res = pkgutil.find_loader("sys") + self.assertIsNotNone(res) + + @ignore_warnings(category=DeprecationWarning) def test_find_loader_missing_module(self): name = 'totally bogus' loader = pkgutil.find_loader(name) self.assertIsNone(loader) - def test_find_loader_avoids_emulation(self): - with check_warnings() as w: - self.assertIsNotNone(pkgutil.find_loader("sys")) - self.assertIsNotNone(pkgutil.find_loader("os")) - self.assertIsNotNone(pkgutil.find_loader("test.support")) - self.assertEqual(len(w.warnings), 0) - def test_get_importer_avoids_emulation(self): # We use an illegal path so *none* of the path hooks should fire with check_warnings() as w: diff --git a/Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst b/Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst new file mode 100644 index 00000000000000..e3297d164fff6d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-21-16-23-31.gh-issue-97850.N46coo.rst @@ -0,0 +1,2 @@ +Deprecate :func:`pkgutil.find_loader` and :func:`pkgutil.get_loader` +in favor of :func:`importlib.util.find_spec`. From e95dd40aff35775efce4c03bec7d82f03711310b Mon Sep 17 00:00:00 2001 From: Yuxin Wu Date: Thu, 4 May 2023 07:26:39 +0800 Subject: [PATCH 26/65] pydoc.safeimport: Use importlib.import_module instead of __import__ (GH-103118) --- Lib/pydoc.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/pydoc.py b/Lib/pydoc.py index b10a5da99402b5..84e673a7f87f90 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -448,7 +448,7 @@ def safeimport(path, forceload=0, cache={}): # Prevent garbage collection. cache[key] = sys.modules[key] del sys.modules[key] - module = __import__(path) + module = importlib.import_module(path) except BaseException as err: # Did the error occur before or after the module was found? if path in sys.modules: @@ -463,9 +463,6 @@ def safeimport(path, forceload=0, cache={}): else: # Some other error occurred during the importing process. raise ErrorDuringImport(path, err) - for part in path.split('.')[1:]: - try: module = getattr(module, part) - except AttributeError: return None return module # ---------------------------------------------------- formatter base class From 9885677b0494e9be3eb8d7d69bebca0e79d8abcc Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 4 May 2023 02:11:29 +0200 Subject: [PATCH 27/65] gh-104146: Remove unused 'second_pass_replacements' from clinic.py (#104147) The code that manipulated 'second_pass_replacements' was removed in 2015 with commit 0759f84 (gh-67688, bpo-23500). --- Tools/clinic/clinic.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index b8b2b75c749152..d3e120c0e7a8ee 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -2112,8 +2112,6 @@ def parse(self, input): traceback.format_exc().rstrip()) printer.print_block(block) - second_pass_replacements = {} - # these are destinations not buffers for name, destination in self.destinations.items(): if destination.type == 'suppress': @@ -2155,23 +2153,8 @@ def parse(self, input): printer_2.print_block(block, core_includes=True) write_file(destination.filename, printer_2.f.getvalue()) continue - text = printer.f.getvalue() - - if second_pass_replacements: - printer_2 = BlockPrinter(self.language) - parser_2 = BlockParser(text, self.language) - changed = False - for block in parser_2: - if block.dsl_name: - for id, replacement in second_pass_replacements.items(): - if id in block.output: - changed = True - block.output = block.output.replace(id, replacement) - printer_2.print_block(block) - if changed: - text = printer_2.f.getvalue() - - return text + + return printer.f.getvalue() def _module_and_class(self, fields): From 35d273825abc319d0ecbd69110e847f6040d0cd7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 08:11:09 +0100 Subject: [PATCH 28/65] GH-97950: Allow translation of index directive content (#104000) --- Doc/conf.py | 5 +++++ Doc/tools/extensions/pyspecific.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/Doc/conf.py b/Doc/conf.py index 6a3c01cd91a77b..485c0bdf84df2e 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -91,6 +91,11 @@ # Avoid a warning with Sphinx >= 2.0 master_doc = 'contents' +# Allow translation of index directives +gettext_additional_targets = [ + 'index', +] + # Options for HTML output # ----------------------- diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index d659a4a54b9d11..39c7c42e66f9a0 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -674,6 +674,34 @@ def process_audit_events(app, doctree, fromdocname): node.replace_self(table) +def patch_pairindextypes(app) -> None: + if app.builder.name != 'gettext': + return + + # allow translating deprecated index entries + try: + from sphinx.domains.python import pairindextypes + except ImportError: + pass + else: + # Sphinx checks if a 'pair' type entry on an index directive is one of + # the Sphinx-translated pairindextypes values. As we intend to move + # away from this, we need Sphinx to believe that these values don't + # exist, by deleting them when using the gettext builder. + + # pairindextypes.pop('module', None) + # pairindextypes.pop('keyword', None) + # pairindextypes.pop('operator', None) + # pairindextypes.pop('object', None) + # pairindextypes.pop('exception', None) + # pairindextypes.pop('statement', None) + # pairindextypes.pop('builtin', None) + + # there needs to be at least one statement in this block, will be + # removed when the first of the below is uncommented. + pass + + def setup(app): app.add_role('issue', issue_role) app.add_role('gh', gh_issue_role) @@ -695,6 +723,7 @@ def setup(app): app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) app.add_directive('miscnews', MiscNews) + app.connect('builder-inited', patch_pairindextypes) app.connect('doctree-resolved', process_audit_events) app.connect('env-merge-info', audit_events_merge) app.connect('env-purge-doc', audit_events_purge) From cd9a56c2b0e14f56f2e83dd4db43c5c69a74b232 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 4 May 2023 09:56:53 +0200 Subject: [PATCH 29/65] gh-103509: PEP 697 -- Limited C API for Extending Opaque Types (GH-103511) Co-authored-by: Oleg Iarygin Co-authored-by: Erlend E. Aasland --- Doc/c-api/object.rst | 39 ++ Doc/c-api/structures.rst | 16 + Doc/c-api/type.rst | 48 ++- Doc/c-api/typeobj.rst | 20 + Doc/data/stable_abi.dat | 2 + Doc/whatsnew/3.12.rst | 15 + Include/cpython/object.h | 1 + Include/descrobject.h | 1 + Include/internal/pycore_object.h | 5 - Include/object.h | 5 + Include/pyport.h | 11 + Lib/test/test_capi/test_misc.py | 158 +++++++- Lib/test/test_stable_abi_ctypes.py | 2 + ...-04-13-16-54-00.gh-issue-103509.A26Qu8.rst | 5 + Misc/stable_abi.toml | 9 + Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/heaptype.c | 131 ++++++- Modules/_testcapi/heaptype_relative.c | 343 ++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 3 + Objects/descrobject.c | 6 + Objects/typeobject.c | 107 +++++- PC/pyconfig.h | 2 + PC/python3dll.c | 2 + PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 + Python/structmember.c | 12 + configure | 35 ++ configure.ac | 1 + pyconfig.h.in | 3 + 30 files changed, 970 insertions(+), 19 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst create mode 100644 Modules/_testcapi/heaptype_relative.c diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 0a12bb9e8c54f0..a0c3194ab0fb90 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -395,3 +395,42 @@ Object Protocol returns ``NULL`` if the object cannot be iterated. .. versionadded:: 3.10 + +.. c:function:: void *PyObject_GetTypeData(PyObject *o, PyTypeObject *cls) + + Get a pointer to subclass-specific data reserved for *cls*. + + The object *o* must be an instance of *cls*, and *cls* must have been + created using negative :c:member:`PyType_Spec.basicsize`. + Python does not check this. + + On error, set an exception and return ``NULL``. + + .. versionadded:: 3.12 + +.. c:function:: Py_ssize_t PyType_GetTypeDataSize(PyTypeObject *cls) + + Return the size of the instance memory space reserved for *cls*, i.e. the size of the + memory :c:func:`PyObject_GetTypeData` returns. + + This may be larger than requested using :c:member:`-PyType_Spec.basicsize `; + it is safe to use this larger size (e.g. with :c:func:`!memset`). + + The type *cls* **must** have been created using + negative :c:member:`PyType_Spec.basicsize`. + Python does not check this. + + On error, set an exception and return a negative value. + + .. versionadded:: 3.12 + +.. c:function:: void *PyObject_GetItemData(PyObject *o) + + Get a pointer to per-item data for a class with + :const:`Py_TPFLAGS_ITEMS_AT_END`. + + On error, set an exception and return ``NULL``. + :py:exc:`TypeError` is raised if *o* does not have + :const:`Py_TPFLAGS_ITEMS_AT_END` set. + + .. versionadded:: 3.12 diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 9618a0cf676972..338db6378d240e 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -486,6 +486,22 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: Emit an ``object.__getattr__`` :ref:`audit event ` before reading. +.. c:macro:: Py_RELATIVE_OFFSET + + Indicates that the :c:member:`~PyMemberDef.offset` of this ``PyMemberDef`` + entry indicates an offset from the subclass-specific data, rather than + from ``PyObject``. + + Can only be used as part of :c:member:`Py_tp_members ` + :c:type:`slot ` when creating a class using negative + :c:member:`~PyTypeDef.basicsize`. + It is mandatory in that case. + + This flag is only used in :c:type:`PyTypeSlot`. + When setting :c:member:`~PyTypeObject.tp_members` during + class creation, Python clears it and sets + :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. + .. index:: single: READ_RESTRICTED single: WRITE_RESTRICTED diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 9fd40e1008c4a8..c21fd9255d28ad 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -353,25 +353,57 @@ The following functions and structs are used to create Structure defining a type's behavior. - .. c:member:: const char* PyType_Spec.name + .. c:member:: const char* name Name of the type, used to set :c:member:`PyTypeObject.tp_name`. - .. c:member:: int PyType_Spec.basicsize - .. c:member:: int PyType_Spec.itemsize + .. c:member:: int basicsize - Size of the instance in bytes, used to set - :c:member:`PyTypeObject.tp_basicsize` and - :c:member:`PyTypeObject.tp_itemsize`. + If positive, specifies the size of the instance in bytes. + It is used to set :c:member:`PyTypeObject.tp_basicsize`. - .. c:member:: int PyType_Spec.flags + If zero, specifies that :c:member:`~PyTypeObject.tp_basicsize` + should be inherited. + + If negative, the absolute value specifies how much space instances of the + class need *in addition* to the superclass. + Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific + memory reserved this way. + + .. versionchanged:: 3.12 + + Previously, this field could not be negative. + + .. c:member:: int itemsize + + Size of one element of a variable-size type, in bytes. + Used to set :c:member:`PyTypeObject.tp_itemsize`. + See ``tp_itemsize`` documentation for caveats. + + If zero, :c:member:`~PyTypeObject.tp_itemsize` is inherited. + Extending arbitrary variable-sized classes is dangerous, + since some types use a fixed offset for variable-sized memory, + which can then overlap fixed-sized memory used by a subclass. + To help prevent mistakes, inheriting ``itemsize`` is only possible + in the following situations: + + - The base is not variable-sized (its + :c:member:`~PyTypeObject.tp_itemsize`). + - The requested :c:member:`PyType_Spec.basicsize` is positive, + suggesting that the memory layout of the base class is known. + - The requested :c:member:`PyType_Spec.basicsize` is zero, + suggesting that the subclass does not access the instance's memory + directly. + - With the :const:`Py_TPFLAGS_ITEMS_AT_END` flag. + + .. c:member:: unsigned int flags Type flags, used to set :c:member:`PyTypeObject.tp_flags`. If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set, :c:func:`PyType_FromSpecWithBases` sets it automatically. - .. c:member:: PyType_Slot *PyType_Spec.slots + .. c:member:: PyType_Slot *slots Array of :c:type:`PyType_Slot` structures. Terminated by the special slot value ``{0, NULL}``. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index e963b90628aa49..e13db3fb2211e2 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1171,6 +1171,26 @@ and :c:type:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_weaklistoffset` field is set in a superclass. + .. data:: Py_TPFLAGS_ITEMS_AT_END + + Only usable with variable-size types, i.e. ones with non-zero + :c:member:`~PyObject.tp_itemsize`. + + Indicates that the variable-sized portion of an instance of this type is + at the end of the instance's memory area, at an offset of + :c:expr:`Py_TYPE(obj)->tp_basicsize` (which may be different in each + subclass). + + When setting this flag, be sure that all superclasses either + use this memory layout, or are not variable-sized. + Python does not check this. + + .. versionadded:: 3.12 + + **Inheritance:** + + This flag is inherited. + .. XXX Document more flags here? diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 4cc06d22baaa93..f112d268129fd1 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -521,6 +521,7 @@ function,PyObject_GetAttrString,3.2,, function,PyObject_GetBuffer,3.11,, function,PyObject_GetItem,3.2,, function,PyObject_GetIter,3.2,, +function,PyObject_GetTypeData,3.12,, function,PyObject_HasAttr,3.2,, function,PyObject_HasAttrString,3.2,, function,PyObject_Hash,3.2,, @@ -675,6 +676,7 @@ function,PyType_GetModuleState,3.10,, function,PyType_GetName,3.11,, function,PyType_GetQualName,3.11,, function,PyType_GetSlot,3.4,, +function,PyType_GetTypeDataSize,3.12,, function,PyType_IsSubtype,3.2,, function,PyType_Modified,3.2,, function,PyType_Ready,3.2,, diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3dfd787e3df837..edbf921467553c 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1159,6 +1159,21 @@ New Features (Contributed by Petr Viktorin in :gh:`101101`.) +* :pep:`697`: Added API for extending types whose instance memory layout is + opaque: + + - :c:member:`PyType_Spec.basicsize` can be zero or negative to specify + inheriting or extending the base class size. + - :c:func:`PyObject_GetTypeData` and :c:func:`PyType_GetTypeDataSize` + added to allow access to subclass-specific instance data. + - :const:`Py_TPFLAGS_ITEMS_AT_END` and :c:func:`PyObject_GetItemData` + added to allow safely extending certain variable-sized types, including + :c:var:`PyType_Type`. + - :c:macro:`Py_RELATIVE_OFFSET` added to allow defining + :c:type:`members ` in terms of a subclass-specific struct. + + (Contributed by Petr Viktorin in :gh:`103509`.) + * Added the new limited C API function :c:func:`PyType_FromMetaclass`, which generalizes the existing :c:func:`PyType_FromModuleAndSpec` using an additional metaclass argument. diff --git a/Include/cpython/object.h b/Include/cpython/object.h index ce4d13cd9c28fe..d8eff691039d24 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -553,6 +553,7 @@ Py_DEPRECATED(3.11) typedef int UsingDeprecatedTrashcanMacro; Py_TRASHCAN_END; \ } while(0); +PyAPI_FUNC(void *) PyObject_GetItemData(PyObject *obj); PyAPI_FUNC(int) _PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg); PyAPI_FUNC(void) _PyObject_ClearManagedDict(PyObject *obj); diff --git a/Include/descrobject.h b/Include/descrobject.h index 0a420b865dfd1b..fd66d17b497a31 100644 --- a/Include/descrobject.h +++ b/Include/descrobject.h @@ -83,6 +83,7 @@ struct PyMemberDef { #define Py_READONLY 1 #define Py_AUDIT_READ 2 // Added in 3.10, harmless no-op before that #define _Py_WRITE_RESTRICTED 4 // Deprecated, no-op. Do not reuse the value. +#define Py_RELATIVE_OFFSET 8 PyAPI_FUNC(PyObject *) PyMember_GetOne(const char *, PyMemberDef *); PyAPI_FUNC(int) PyMember_SetOne(char *, PyMemberDef *, PyObject *); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 91853ad0525b55..b9e700ea280cbf 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -389,11 +389,6 @@ extern PyObject ** _PyObject_ComputedDictPointer(PyObject *); extern void _PyObject_FreeInstanceAttributes(PyObject *obj); extern int _PyObject_IsInstanceDictEmpty(PyObject *); -// Access macro to the members which are floating "behind" the object -static inline PyMemberDef* _PyHeapType_GET_MEMBERS(PyHeapTypeObject *etype) { - return (PyMemberDef*)((char*)etype + Py_TYPE(etype)->tp_basicsize); -} - PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, PyObject *); /* C function call trampolines to mitigate bad function pointer casts. diff --git a/Include/object.h b/Include/object.h index 66c3df0d7f780a..81aeb2d8bd5a69 100644 --- a/Include/object.h +++ b/Include/object.h @@ -355,6 +355,8 @@ PyAPI_FUNC(PyObject *) PyType_GetQualName(PyTypeObject *); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030C0000 PyAPI_FUNC(PyObject *) PyType_FromMetaclass(PyTypeObject*, PyObject*, PyType_Spec*, PyObject*); +PyAPI_FUNC(void *) PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls); +PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls); #endif /* Generic type check */ @@ -521,6 +523,9 @@ given type object has a specified feature. // subject itself (rather than a mapped attribute on it): #define _Py_TPFLAGS_MATCH_SELF (1UL << 22) +/* Items (ob_size*tp_itemsize) are found at the end of an instance's memory */ +#define Py_TPFLAGS_ITEMS_AT_END (1UL << 23) + /* These flags are used to determine if a type is a subclass. */ #define Py_TPFLAGS_LONG_SUBCLASS (1UL << 24) #define Py_TPFLAGS_LIST_SUBCLASS (1UL << 25) diff --git a/Include/pyport.h b/Include/pyport.h index bd0ba6d0681b21..d7c6ae64f2bf2f 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -765,4 +765,15 @@ extern char * _getpty(int *, int, mode_t, int); #undef __bool__ #endif +// Make sure we have maximum alignment, even if the current compiler +// does not support max_align_t. Note that: +// - Autoconf reports alignment of unknown types to 0. +// - 'long double' has maximum alignment on *most* platforms, +// looks like the best we can do for pre-C11 compilers. +// - The value is tested, see test_alignof_max_align_t +#if !defined(ALIGNOF_MAX_ALIGN_T) || ALIGNOF_MAX_ALIGN_T == 0 +# undef ALIGNOF_MAX_ALIGN_T +# define ALIGNOF_MAX_ALIGN_T _Alignof(long double) +#endif + #endif /* Py_PYPORT_H */ diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 1d426d0f8f825c..22be3c0814278e 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -16,11 +16,13 @@ import unittest import warnings import weakref +import operator from test import support from test.support import MISSING_C_DOCSTRINGS from test.support import import_helper from test.support import threading_helper from test.support import warnings_helper +from test.support import requires_limited_api from test.support.script_helper import assert_python_failure, assert_python_ok, run_python_until_end try: import _posixsubprocess @@ -770,7 +772,6 @@ def meth(self): MutableBase.meth = lambda self: 'changed' self.assertEqual(instance.meth(), 'changed') - def test_pynumber_tobase(self): from _testcapi import pynumber_tobase small_number = 123 @@ -1072,6 +1073,161 @@ class Data(_testcapi.ObjExtraData): self.assertIsNone(d.extra) +@requires_limited_api +class TestHeapTypeRelative(unittest.TestCase): + """Test API for extending opaque types (PEP 697)""" + + @requires_limited_api + def test_heaptype_relative_sizes(self): + # Test subclassing using "relative" basicsize, see PEP 697 + def check(extra_base_size, extra_size): + Base, Sub, instance, data_ptr, data_offset, data_size = ( + _testcapi.make_sized_heaptypes( + extra_base_size, -extra_size)) + + # no alignment shenanigans when inheriting directly + if extra_size == 0: + self.assertEqual(Base.__basicsize__, Sub.__basicsize__) + self.assertEqual(data_size, 0) + + else: + # The following offsets should be in increasing order: + offsets = [ + (0, 'start of object'), + (Base.__basicsize__, 'end of base data'), + (data_offset, 'subclass data'), + (data_offset + extra_size, 'end of requested subcls data'), + (data_offset + data_size, 'end of reserved subcls data'), + (Sub.__basicsize__, 'end of object'), + ] + ordered_offsets = sorted(offsets, key=operator.itemgetter(0)) + self.assertEqual( + offsets, ordered_offsets, + msg=f'Offsets not in expected order, got: {ordered_offsets}') + + # end of reserved subcls data == end of object + self.assertEqual(Sub.__basicsize__, data_offset + data_size) + + # we don't reserve (requested + alignment) or more data + self.assertLess(data_size - extra_size, + _testcapi.ALIGNOF_MAX_ALIGN_T) + + # The offsets/sizes we calculated should be aligned. + self.assertEqual(data_offset % _testcapi.ALIGNOF_MAX_ALIGN_T, 0) + self.assertEqual(data_size % _testcapi.ALIGNOF_MAX_ALIGN_T, 0) + + sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, + object.__basicsize__, + object.__basicsize__-1, + object.__basicsize__+1}) + for extra_base_size in sizes: + for extra_size in sizes: + args = dict(extra_base_size=extra_base_size, + extra_size=extra_size) + with self.subTest(**args): + check(**args) + + def test_HeapCCollection(self): + """Make sure HeapCCollection works properly by itself""" + collection = _testcapi.HeapCCollection(1, 2, 3) + self.assertEqual(list(collection), [1, 2, 3]) + + def test_heaptype_inherit_itemsize(self): + """Test HeapCCollection subclasses work properly""" + sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, + object.__basicsize__, + object.__basicsize__-1, + object.__basicsize__+1}) + for extra_size in sizes: + with self.subTest(extra_size=extra_size): + Sub = _testcapi.subclass_var_heaptype( + _testcapi.HeapCCollection, -extra_size, 0, 0) + collection = Sub(1, 2, 3) + collection.set_data_to_3s() + + self.assertEqual(list(collection), [1, 2, 3]) + mem = collection.get_data() + self.assertGreaterEqual(len(mem), extra_size) + self.assertTrue(set(mem) <= {3}, f'got {mem!r}') + + def test_heaptype_invalid_inheritance(self): + with self.assertRaises(SystemError, + msg="Cannot extend variable-size class without " + + "Py_TPFLAGS_ITEMS_AT_END"): + _testcapi.subclass_heaptype(int, -8, 0) + + def test_heaptype_relative_members(self): + """Test HeapCCollection subclasses work properly""" + sizes = sorted({0, 1, 2, 3, 4, 7, 8, 123, + object.__basicsize__, + object.__basicsize__-1, + object.__basicsize__+1}) + for extra_base_size in sizes: + for extra_size in sizes: + for offset in sizes: + with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size, offset=offset): + if offset < extra_size: + Sub = _testcapi.make_heaptype_with_member( + extra_base_size, -extra_size, offset, True) + Base = Sub.mro()[1] + instance = Sub() + self.assertEqual(instance.memb, instance.get_memb()) + instance.set_memb(13) + self.assertEqual(instance.memb, instance.get_memb()) + self.assertEqual(instance.get_memb(), 13) + instance.memb = 14 + self.assertEqual(instance.memb, instance.get_memb()) + self.assertEqual(instance.get_memb(), 14) + self.assertGreaterEqual(instance.get_memb_offset(), Base.__basicsize__) + self.assertLess(instance.get_memb_offset(), Sub.__basicsize__) + with self.assertRaises(SystemError): + instance.get_memb_relative() + with self.assertRaises(SystemError): + instance.set_memb_relative(0) + else: + with self.assertRaises(SystemError): + Sub = _testcapi.make_heaptype_with_member( + extra_base_size, -extra_size, offset, True) + with self.assertRaises(SystemError): + Sub = _testcapi.make_heaptype_with_member( + extra_base_size, extra_size, offset, True) + with self.subTest(extra_base_size=extra_base_size, extra_size=extra_size): + with self.assertRaises(SystemError): + Sub = _testcapi.make_heaptype_with_member( + extra_base_size, -extra_size, -1, True) + + def test_heaptype_relative_members_errors(self): + with self.assertRaisesRegex( + SystemError, + r"With Py_RELATIVE_OFFSET, basicsize must be negative"): + _testcapi.make_heaptype_with_member(0, 1234, 0, True) + with self.assertRaisesRegex( + SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): + _testcapi.make_heaptype_with_member(0, -8, 1234, True) + with self.assertRaisesRegex( + SystemError, r"Member offset out of range \(0\.\.-basicsize\)"): + _testcapi.make_heaptype_with_member(0, -8, -1, True) + + Sub = _testcapi.make_heaptype_with_member(0, -8, 0, True) + instance = Sub() + with self.assertRaisesRegex( + SystemError, r"PyMember_GetOne used with Py_RELATIVE_OFFSET"): + instance.get_memb_relative() + with self.assertRaisesRegex( + SystemError, r"PyMember_SetOne used with Py_RELATIVE_OFFSET"): + instance.set_memb_relative(0) + + def test_pyobject_getitemdata_error(self): + """Test PyObject_GetItemData fails on unsupported types""" + with self.assertRaises(TypeError): + # None is not variable-length + _testcapi.pyobject_getitemdata(None) + with self.assertRaises(TypeError): + # int is variable-length, but doesn't have the + # Py_TPFLAGS_ITEMS_AT_END layout (and flag) + _testcapi.pyobject_getitemdata(0) + + class TestPendingCalls(unittest.TestCase): def pendingcalls_submit(self, l, n): diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 2feaaf8603b831..4ca39d85e5460c 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -529,6 +529,7 @@ def test_windows_feature_macros(self): "PyObject_GetBuffer", "PyObject_GetItem", "PyObject_GetIter", + "PyObject_GetTypeData", "PyObject_HasAttr", "PyObject_HasAttrString", "PyObject_Hash", @@ -679,6 +680,7 @@ def test_windows_feature_macros(self): "PyType_GetName", "PyType_GetQualName", "PyType_GetSlot", + "PyType_GetTypeDataSize", "PyType_IsSubtype", "PyType_Modified", "PyType_Ready", diff --git a/Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst b/Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst new file mode 100644 index 00000000000000..af630c3aafa940 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2023-04-13-16-54-00.gh-issue-103509.A26Qu8.rst @@ -0,0 +1,5 @@ +Added C API for extending types whose instance memory layout is opaque: +:c:member:`PyType_Spec.basicsize` can now be zero or negative, +:c:func:`PyObject_GetTypeData` can be used to get subclass-specific data, +and :c:macro:`Py_TPFLAGS_ITEMS_AT_END` can be used to safely extend +variable-size objects. See :pep:`697` for details. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 23baeeeae79193..48299e9b35ff97 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2397,3 +2397,12 @@ added = '3.12' # Before 3.12, available in "structmember.h" w/o Py_ prefix [const.Py_AUDIT_READ] added = '3.12' # Before 3.12, available in "structmember.h" + +[function.PyObject_GetTypeData] + added = '3.12' +[function.PyType_GetTypeDataSize] + added = '3.12' +[const.Py_RELATIVE_OFFSET] + added = '3.12' +[const.Py_TPFLAGS_ITEMS_AT_END] + added = '3.12' diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index a90c1e96ef0231..6b483341953711 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c _testcapi/immortal.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 6384fbc485fbdf..3488e35922c5ac 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -371,7 +371,6 @@ create_type_from_repeated_slots(PyObject *self, PyObject *variant_obj) } - static PyObject * make_immutable_type_with_base(PyObject *self, PyObject *base) { @@ -399,6 +398,17 @@ make_type_with_base(PyObject *self, PyObject *base) } +static PyObject * +pyobject_getitemdata(PyObject *self, PyObject *o) +{ + void *pointer = PyObject_GetItemData(o); + if (pointer == NULL) { + return NULL; + } + return PyLong_FromVoidPtr(pointer); +} + + static PyMethodDef TestMethods[] = { {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O}, {"test_type_from_ephemeral_spec", test_type_from_ephemeral_spec, METH_NOARGS}, @@ -411,6 +421,7 @@ static PyMethodDef TestMethods[] = { METH_NOARGS}, {"make_immutable_type_with_base", make_immutable_type_with_base, METH_O}, {"make_type_with_base", make_type_with_base, METH_O}, + {"pyobject_getitemdata", pyobject_getitemdata, METH_O}, {NULL}, }; @@ -987,6 +998,113 @@ static PyType_Spec HeapCTypeSetattr_spec = { HeapCTypeSetattr_slots }; +PyDoc_STRVAR(HeapCCollection_doc, +"Tuple-like heap type that uses PyObject_GetItemData for items."); + +static PyObject* +HeapCCollection_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds) +{ + PyObject *self = NULL; + PyObject *result = NULL; + + Py_ssize_t size = PyTuple_GET_SIZE(args); + self = subtype->tp_alloc(subtype, size); + if (!self) { + goto finally; + } + PyObject **data = PyObject_GetItemData(self); + if (!data) { + goto finally; + } + + for (Py_ssize_t i = 0; i < size; i++) { + data[i] = Py_NewRef(PyTuple_GET_ITEM(args, i)); + } + + result = self; + self = NULL; + finally: + Py_XDECREF(self); + return result; +} + +static Py_ssize_t +HeapCCollection_length(PyVarObject *self) +{ + return Py_SIZE(self); +} + +static PyObject* +HeapCCollection_item(PyObject *self, Py_ssize_t i) +{ + if (i < 0 || i >= Py_SIZE(self)) { + return PyErr_Format(PyExc_IndexError, "index %zd out of range", i); + } + PyObject **data = PyObject_GetItemData(self); + if (!data) { + return NULL; + } + return Py_NewRef(data[i]); +} + +static int +HeapCCollection_traverse(PyObject *self, visitproc visit, void *arg) +{ + PyObject **data = PyObject_GetItemData(self); + if (!data) { + return -1; + } + for (Py_ssize_t i = 0; i < Py_SIZE(self); i++) { + Py_VISIT(data[i]); + } + return 0; +} + +static int +HeapCCollection_clear(PyObject *self) +{ + PyObject **data = PyObject_GetItemData(self); + if (!data) { + return -1; + } + Py_ssize_t size = Py_SIZE(self); + Py_SET_SIZE(self, 0); + for (Py_ssize_t i = 0; i < size; i++) { + Py_CLEAR(data[i]); + } + return 0; +} + +static void +HeapCCollection_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + HeapCCollection_clear(self); + PyObject_GC_UnTrack(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot HeapCCollection_slots[] = { + {Py_tp_new, HeapCCollection_new}, + {Py_sq_length, HeapCCollection_length}, + {Py_sq_item, HeapCCollection_item}, + {Py_tp_traverse, HeapCCollection_traverse}, + {Py_tp_clear, HeapCCollection_clear}, + {Py_tp_dealloc, HeapCCollection_dealloc}, + {Py_tp_doc, (void *)HeapCCollection_doc}, + {0, 0}, +}; + +static PyType_Spec HeapCCollection_spec = { + .name = "_testcapi.HeapCCollection", + .basicsize = sizeof(PyVarObject), + .itemsize = sizeof(PyObject*), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_ITEMS_AT_END), + .slots = HeapCCollection_slots, +}; + int _PyTestCapi_Init_Heaptype(PyObject *m) { _testcapimodule = PyModule_GetDef(m); @@ -1110,5 +1228,16 @@ _PyTestCapi_Init_Heaptype(PyObject *m) { } PyModule_AddObject(m, "HeapCTypeMetaclassCustomNew", HeapCTypeMetaclassCustomNew); + PyObject *HeapCCollection = PyType_FromMetaclass( + NULL, m, &HeapCCollection_spec, NULL); + if (HeapCCollection == NULL) { + return -1; + } + int rc = PyModule_AddType(m, (PyTypeObject *)HeapCCollection); + Py_DECREF(HeapCCollection); + if (rc < 0) { + return -1; + } + return 0; } diff --git a/Modules/_testcapi/heaptype_relative.c b/Modules/_testcapi/heaptype_relative.c new file mode 100644 index 00000000000000..c247ca33b33708 --- /dev/null +++ b/Modules/_testcapi/heaptype_relative.c @@ -0,0 +1,343 @@ +#define Py_LIMITED_API 0x030c0000 // 3.12 +#include "parts.h" +#include // max_align_t +#include // memset + +#ifdef LIMITED_API_AVAILABLE + +static PyType_Slot empty_slots[] = { + {0, NULL}, +}; + +static PyObject * +make_sized_heaptypes(PyObject *module, PyObject *args) +{ + PyObject *base = NULL; + PyObject *sub = NULL; + PyObject *instance = NULL; + PyObject *result = NULL; + + int extra_base_size, basicsize; + + int r = PyArg_ParseTuple(args, "ii", &extra_base_size, &basicsize); + if (!r) { + goto finally; + } + + PyType_Spec base_spec = { + .name = "_testcapi.Base", + .basicsize = sizeof(PyObject) + extra_base_size, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = empty_slots, + }; + PyType_Spec sub_spec = { + .name = "_testcapi.Sub", + .basicsize = basicsize, + .flags = Py_TPFLAGS_DEFAULT, + .slots = empty_slots, + }; + + base = PyType_FromMetaclass(NULL, module, &base_spec, NULL); + if (!base) { + goto finally; + } + sub = PyType_FromMetaclass(NULL, module, &sub_spec, base); + if (!sub) { + goto finally; + } + instance = PyObject_CallNoArgs(sub); + if (!instance) { + goto finally; + } + char *data_ptr = PyObject_GetTypeData(instance, (PyTypeObject *)sub); + if (!data_ptr) { + goto finally; + } + Py_ssize_t data_size = PyType_GetTypeDataSize((PyTypeObject *)sub); + if (data_size < 0) { + goto finally; + } + + result = Py_BuildValue("OOOKnn", base, sub, instance, + (unsigned long long)data_ptr, + (Py_ssize_t)(data_ptr - (char*)instance), + data_size); + finally: + Py_XDECREF(base); + Py_XDECREF(sub); + Py_XDECREF(instance); + return result; +} + +static PyObject * +var_heaptype_set_data_to_3s( + PyObject *self, PyTypeObject *defining_class, + PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + void *data_ptr = PyObject_GetTypeData(self, defining_class); + if (!data_ptr) { + return NULL; + } + Py_ssize_t data_size = PyType_GetTypeDataSize(defining_class); + if (data_size < 0) { + return NULL; + } + memset(data_ptr, 3, data_size); + Py_RETURN_NONE; +} + +static PyObject * +var_heaptype_get_data(PyObject *self, PyTypeObject *defining_class, + PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + void *data_ptr = PyObject_GetTypeData(self, defining_class); + if (!data_ptr) { + return NULL; + } + Py_ssize_t data_size = PyType_GetTypeDataSize(defining_class); + if (data_size < 0) { + return NULL; + } + return PyBytes_FromStringAndSize(data_ptr, data_size); +} + +static PyMethodDef var_heaptype_methods[] = { + {"set_data_to_3s", _PyCFunction_CAST(var_heaptype_set_data_to_3s), + METH_METHOD | METH_FASTCALL | METH_KEYWORDS}, + {"get_data", _PyCFunction_CAST(var_heaptype_get_data), + METH_METHOD | METH_FASTCALL | METH_KEYWORDS}, + {NULL}, +}; + +static PyObject * +subclass_var_heaptype(PyObject *module, PyObject *args) +{ + PyObject *result = NULL; + + PyObject *base; // borrowed from args + int basicsize, itemsize; + long pfunc; + + int r = PyArg_ParseTuple(args, "Oiil", &base, &basicsize, &itemsize, &pfunc); + if (!r) { + goto finally; + } + + PyType_Slot slots[] = { + {Py_tp_methods, var_heaptype_methods}, + {0, NULL}, + }; + + PyType_Spec sub_spec = { + .name = "_testcapi.Sub", + .basicsize = basicsize, + .itemsize = itemsize, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_ITEMS_AT_END, + .slots = slots, + }; + + result = PyType_FromMetaclass(NULL, module, &sub_spec, base); + finally: + return result; +} + +static PyObject * +subclass_heaptype(PyObject *module, PyObject *args) +{ + PyObject *result = NULL; + + PyObject *base; // borrowed from args + int basicsize, itemsize; + + int r = PyArg_ParseTuple(args, "Oii", &base, &basicsize, &itemsize); + if (!r) { + goto finally; + } + + PyType_Slot slots[] = { + {Py_tp_methods, var_heaptype_methods}, + {0, NULL}, + }; + + PyType_Spec sub_spec = { + .name = "_testcapi.Sub", + .basicsize = basicsize, + .itemsize = itemsize, + .flags = Py_TPFLAGS_DEFAULT, + .slots = slots, + }; + + result = PyType_FromMetaclass(NULL, module, &sub_spec, base); + finally: + return result; +} + +static PyMemberDef * +heaptype_with_member_extract_and_check_memb(PyObject *self) +{ + PyMemberDef *def = PyType_GetSlot(Py_TYPE(self), Py_tp_members); + if (!def) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ValueError, "tp_members is NULL"); + } + return NULL; + } + if (!def[0].name) { + PyErr_SetString(PyExc_ValueError, "tp_members[0] is NULL"); + return NULL; + } + if (def[1].name) { + PyErr_SetString(PyExc_ValueError, "tp_members[1] is not NULL"); + return NULL; + } + if (strcmp(def[0].name, "memb")) { + PyErr_SetString(PyExc_ValueError, "tp_members[0] is not for `memb`"); + return NULL; + } + if (def[0].flags) { + PyErr_SetString(PyExc_ValueError, "tp_members[0] has flags set"); + return NULL; + } + return def; +} + +static PyObject * +heaptype_with_member_get_memb(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyMemberDef *def = heaptype_with_member_extract_and_check_memb(self); + return PyMember_GetOne((const char *)self, def); +} + +static PyObject * +heaptype_with_member_set_memb(PyObject *self, PyObject *value) +{ + PyMemberDef *def = heaptype_with_member_extract_and_check_memb(self); + int r = PyMember_SetOne((char *)self, def, value); + if (r < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject * +get_memb_offset(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyMemberDef *def = heaptype_with_member_extract_and_check_memb(self); + return PyLong_FromSsize_t(def->offset); +} + +static PyObject * +heaptype_with_member_get_memb_relative(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyMemberDef def = {"memb", Py_T_BYTE, sizeof(PyObject), Py_RELATIVE_OFFSET}; + return PyMember_GetOne((const char *)self, &def); +} + +static PyObject * +heaptype_with_member_set_memb_relative(PyObject *self, PyObject *value) +{ + PyMemberDef def = {"memb", Py_T_BYTE, sizeof(PyObject), Py_RELATIVE_OFFSET}; + int r = PyMember_SetOne((char *)self, &def, value); + if (r < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyMethodDef heaptype_with_member_methods[] = { + {"get_memb", heaptype_with_member_get_memb, METH_NOARGS}, + {"set_memb", heaptype_with_member_set_memb, METH_O}, + {"get_memb_offset", get_memb_offset, METH_NOARGS}, + {"get_memb_relative", heaptype_with_member_get_memb_relative, METH_NOARGS}, + {"set_memb_relative", heaptype_with_member_set_memb_relative, METH_O}, + {NULL}, +}; + +static PyObject * +make_heaptype_with_member(PyObject *module, PyObject *args) +{ + PyObject *base = NULL; + PyObject *result = NULL; + + int extra_base_size, basicsize, offset, add_flag; + + int r = PyArg_ParseTuple(args, "iiip", &extra_base_size, &basicsize, &offset, &add_flag); + if (!r) { + goto finally; + } + + PyType_Spec base_spec = { + .name = "_testcapi.Base", + .basicsize = sizeof(PyObject) + extra_base_size, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = empty_slots, + }; + base = PyType_FromMetaclass(NULL, module, &base_spec, NULL); + if (!base) { + goto finally; + } + + PyMemberDef members[] = { + {"memb", Py_T_BYTE, offset, add_flag ? Py_RELATIVE_OFFSET : 0}, + {0}, + }; + PyType_Slot slots[] = { + {Py_tp_members, members}, + {Py_tp_methods, heaptype_with_member_methods}, + {0, NULL}, + }; + + PyType_Spec sub_spec = { + .name = "_testcapi.Sub", + .basicsize = basicsize, + .flags = Py_TPFLAGS_DEFAULT, + .slots = slots, + }; + + result = PyType_FromMetaclass(NULL, module, &sub_spec, base); + finally: + Py_XDECREF(base); + return result; +} + + +static PyObject * +test_alignof_max_align_t(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + // We define ALIGNOF_MAX_ALIGN_T even if the compiler doesn't support + // max_align_t. Double-check that it's correct. + assert(ALIGNOF_MAX_ALIGN_T > 0); + assert(ALIGNOF_MAX_ALIGN_T >= _Alignof(long long)); + assert(ALIGNOF_MAX_ALIGN_T >= _Alignof(long double)); + assert(ALIGNOF_MAX_ALIGN_T >= _Alignof(void*)); + assert(ALIGNOF_MAX_ALIGN_T >= _Alignof(void (*)(void))); + + // Ensure it's a power of two + assert((ALIGNOF_MAX_ALIGN_T & (ALIGNOF_MAX_ALIGN_T - 1)) == 0); + + Py_RETURN_NONE; +} + +static PyMethodDef TestMethods[] = { + {"make_sized_heaptypes", make_sized_heaptypes, METH_VARARGS}, + {"subclass_var_heaptype", subclass_var_heaptype, METH_VARARGS}, + {"subclass_heaptype", subclass_heaptype, METH_VARARGS}, + {"make_heaptype_with_member", make_heaptype_with_member, METH_VARARGS}, + {"test_alignof_max_align_t", test_alignof_max_align_t, METH_NOARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_HeaptypeRelative(PyObject *m) { + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + + if (PyModule_AddIntMacro(m, ALIGNOF_MAX_ALIGN_T) < 0) { + return -1; + } + + return 0; +} + +#endif // LIMITED_API_AVAILABLE diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 4d2d6832a827ae..d75412d5116063 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -43,6 +43,7 @@ int _PyTestCapi_Init_Immortal(PyObject *module); #ifdef LIMITED_API_AVAILABLE int _PyTestCapi_Init_VectorcallLimited(PyObject *module); +int _PyTestCapi_Init_HeaptypeRelative(PyObject *module); #endif // LIMITED_API_AVAILABLE #endif diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 47e0ed9be8e709..1ecc442058088a 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4324,6 +4324,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_VectorcallLimited(m) < 0) { return NULL; } + if (_PyTestCapi_Init_HeaptypeRelative(m) < 0) { + return NULL; + } #endif PyState_AddModule(m, &_testcapimodule); diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 334be75e8df9df..17c0c85a06c4b8 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -978,6 +978,12 @@ PyDescr_NewMember(PyTypeObject *type, PyMemberDef *member) { PyMemberDescrObject *descr; + if (member->flags & Py_RELATIVE_OFFSET) { + PyErr_SetString( + PyExc_SystemError, + "PyDescr_NewMember used with Py_RELATIVE_OFFSET"); + return NULL; + } descr = (PyMemberDescrObject *)descr_new(&PyMemberDescr_Type, type, member->name); if (descr != NULL) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 4ced04b0bde966..171c76a59a5541 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -18,6 +18,7 @@ #include "structmember.h" // PyMemberDef #include +#include // ptrdiff_t /*[clinic input] class type "PyTypeObject *" "&PyType_Type" @@ -1686,6 +1687,12 @@ PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) /* Helpers for subtyping */ +static inline PyMemberDef * +_PyHeapType_GET_MEMBERS(PyHeapTypeObject* type) +{ + return PyObject_GetItemData((PyObject *)type); +} + static int traverse_slots(PyTypeObject *type, PyObject *self, visitproc visit, void *arg) { @@ -3873,6 +3880,15 @@ static const PySlot_Offset pyslot_offsets[] = { #include "typeslots.inc" }; +/* Align up to the nearest multiple of alignof(max_align_t) + * (like _Py_ALIGN_UP, but for a size rather than pointer) + */ +static Py_ssize_t +_align_up(Py_ssize_t size) +{ + return (size + ALIGNOF_MAX_ALIGN_T - 1) & ~(ALIGNOF_MAX_ALIGN_T - 1); +} + /* Given a PyType_FromMetaclass `bases` argument (NULL, type, or tuple of * types), return a tuple of types. */ @@ -4013,6 +4029,20 @@ _PyType_FromMetaclass_impl( assert(memb->flags == READONLY); vectorcalloffset = memb->offset; } + if (memb->flags & Py_RELATIVE_OFFSET) { + if (spec->basicsize > 0) { + PyErr_SetString( + PyExc_SystemError, + "With Py_RELATIVE_OFFSET, basicsize must be negative."); + goto finally; + } + if (memb->offset < 0 || memb->offset >= -spec->basicsize) { + PyErr_SetString( + PyExc_SystemError, + "Member offset out of range (0..-basicsize)"); + goto finally; + } + } } break; case Py_tp_doc: @@ -4154,6 +4184,32 @@ _PyType_FromMetaclass_impl( // here we just check its work assert(_PyType_HasFeature(base, Py_TPFLAGS_BASETYPE)); + /* Calculate sizes */ + + Py_ssize_t basicsize = spec->basicsize; + Py_ssize_t type_data_offset = spec->basicsize; + if (basicsize == 0) { + /* Inherit */ + basicsize = base->tp_basicsize; + } + else if (basicsize < 0) { + /* Extend */ + type_data_offset = _align_up(base->tp_basicsize); + basicsize = type_data_offset + _align_up(-spec->basicsize); + + /* Inheriting variable-sized types is limited */ + if (base->tp_itemsize + && !((base->tp_flags | spec->flags) & Py_TPFLAGS_ITEMS_AT_END)) + { + PyErr_SetString( + PyExc_SystemError, + "Cannot extend variable-size class without Py_TPFLAGS_ITEMS_AT_END."); + goto finally; + } + } + + Py_ssize_t itemsize = spec->itemsize; + /* Allocate the new type * * Between here and PyType_Ready, we should limit: @@ -4201,8 +4257,8 @@ _PyType_FromMetaclass_impl( /* Copy the sizes */ - type->tp_basicsize = spec->basicsize; - type->tp_itemsize = spec->itemsize; + type->tp_basicsize = basicsize; + type->tp_itemsize = itemsize; /* Copy all the ordinary slots */ @@ -4219,6 +4275,16 @@ _PyType_FromMetaclass_impl( size_t len = Py_TYPE(type)->tp_itemsize * nmembers; memcpy(_PyHeapType_GET_MEMBERS(res), slot->pfunc, len); type->tp_members = _PyHeapType_GET_MEMBERS(res); + PyMemberDef *memb; + Py_ssize_t i; + for (memb = _PyHeapType_GET_MEMBERS(res), i = nmembers; + i > 0; ++memb, --i) + { + if (memb->flags & Py_RELATIVE_OFFSET) { + memb->flags &= ~Py_RELATIVE_OFFSET; + memb->offset += type_data_offset; + } + } } break; default: @@ -4227,6 +4293,7 @@ _PyType_FromMetaclass_impl( PySlot_Offset slotoffsets = pyslot_offsets[slot->slot]; short slot_offset = slotoffsets.slot_offset; if (slotoffsets.subslot_offset == -1) { + /* Set a slot in the main PyTypeObject */ *(void**)((char*)res_start + slot_offset) = slot->pfunc; } else { @@ -4461,6 +4528,34 @@ PyType_GetModuleByDef(PyTypeObject *type, PyModuleDef *def) return NULL; } +void * +PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) +{ + assert(PyObject_TypeCheck(obj, cls)); + return (char *)obj + _align_up(cls->tp_base->tp_basicsize); +} + +Py_ssize_t +PyType_GetTypeDataSize(PyTypeObject *cls) +{ + ptrdiff_t result = cls->tp_basicsize - _align_up(cls->tp_base->tp_basicsize); + if (result < 0) { + return 0; + } + return result; +} + +void * +PyObject_GetItemData(PyObject *obj) +{ + if (!PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_ITEMS_AT_END)) { + PyErr_Format(PyExc_TypeError, + "type '%s' does not have Py_TPFLAGS_ITEMS_AT_END", + Py_TYPE(obj)->tp_name); + return NULL; + } + return (char *)obj + Py_TYPE(obj)->tp_basicsize; +} /* Internal API to look for a name through the MRO, bypassing the method cache. This returns a borrowed reference, and might set an exception. @@ -5158,7 +5253,8 @@ PyTypeObject PyType_Type = { 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_TYPE_SUBCLASS | - Py_TPFLAGS_HAVE_VECTORCALL, /* tp_flags */ + Py_TPFLAGS_HAVE_VECTORCALL | + Py_TPFLAGS_ITEMS_AT_END, /* tp_flags */ type_doc, /* tp_doc */ (traverseproc)type_traverse, /* tp_traverse */ (inquiry)type_clear, /* tp_clear */ @@ -6572,9 +6668,14 @@ inherit_special(PyTypeObject *type, PyTypeObject *base) else if (PyType_IsSubtype(base, &PyDict_Type)) { type->tp_flags |= Py_TPFLAGS_DICT_SUBCLASS; } + + /* Setup some inheritable flags */ if (PyType_HasFeature(base, _Py_TPFLAGS_MATCH_SELF)) { type->tp_flags |= _Py_TPFLAGS_MATCH_SELF; } + if (PyType_HasFeature(base, Py_TPFLAGS_ITEMS_AT_END)) { + type->tp_flags |= Py_TPFLAGS_ITEMS_AT_END; + } } static int diff --git a/PC/pyconfig.h b/PC/pyconfig.h index 8a3bf8968ce29d..3415efe2dea117 100644 --- a/PC/pyconfig.h +++ b/PC/pyconfig.h @@ -330,6 +330,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ # define SIZEOF_HKEY 8 # define SIZEOF_SIZE_T 8 # define ALIGNOF_SIZE_T 8 +# define ALIGNOF_MAX_ALIGN_T 8 /* configure.ac defines HAVE_LARGEFILE_SUPPORT iff sizeof(off_t) > sizeof(long), and sizeof(long long) >= sizeof(off_t). On Win64 the second condition is not true, but if fpos_t replaces off_t @@ -351,6 +352,7 @@ Py_NO_ENABLE_SHARED to find out. Also support MS_NO_COREDLL for b/w compat */ # else # define SIZEOF_TIME_T 4 # endif +# define ALIGNOF_MAX_ALIGN_T 8 #endif #ifdef _DEBUG diff --git a/PC/python3dll.c b/PC/python3dll.c index 706affa18351b3..7e848abccfd1fa 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -467,6 +467,7 @@ EXPORT_FUNC(PyObject_GetAttrString) EXPORT_FUNC(PyObject_GetBuffer) EXPORT_FUNC(PyObject_GetItem) EXPORT_FUNC(PyObject_GetIter) +EXPORT_FUNC(PyObject_GetTypeData) EXPORT_FUNC(PyObject_HasAttr) EXPORT_FUNC(PyObject_HasAttrString) EXPORT_FUNC(PyObject_Hash) @@ -618,6 +619,7 @@ EXPORT_FUNC(PyType_GetModuleState) EXPORT_FUNC(PyType_GetName) EXPORT_FUNC(PyType_GetQualName) EXPORT_FUNC(PyType_GetSlot) +EXPORT_FUNC(PyType_GetTypeDataSize) EXPORT_FUNC(PyType_IsSubtype) EXPORT_FUNC(PyType_Modified) EXPORT_FUNC(PyType_Ready) diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 21941247eb9692..56448b6ee7d492 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -98,6 +98,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 0e42e4982c21ff..297c9ce799bea1 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -24,6 +24,9 @@ Source Files + + Source Files + Source Files diff --git a/Python/structmember.c b/Python/structmember.c index 1b8be28dcf2eb2..19a75224a0f32e 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -8,6 +8,12 @@ PyObject * PyMember_GetOne(const char *obj_addr, PyMemberDef *l) { PyObject *v; + if (l->flags & Py_RELATIVE_OFFSET) { + PyErr_SetString( + PyExc_SystemError, + "PyMember_GetOne used with Py_RELATIVE_OFFSET"); + return NULL; + } const char* addr = obj_addr + l->offset; switch (l->type) { @@ -103,6 +109,12 @@ int PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) { PyObject *oldv; + if (l->flags & Py_RELATIVE_OFFSET) { + PyErr_SetString( + PyExc_SystemError, + "PyMember_SetOne used with Py_RELATIVE_OFFSET"); + return -1; + } addr += l->offset; diff --git a/configure b/configure index 8133d47f61355b..b8fa9d66e735dd 100755 --- a/configure +++ b/configure @@ -10729,6 +10729,41 @@ cat >>confdefs.h <<_ACEOF _ACEOF +# The cast to long int works around a bug in the HP C Compiler, +# see AC_CHECK_SIZEOF for more information. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking alignment of max_align_t" >&5 +$as_echo_n "checking alignment of max_align_t... " >&6; } +if ${ac_cv_alignof_max_align_t+:} false; then : + $as_echo_n "(cached) " >&6 +else + if ac_fn_c_compute_int "$LINENO" "(long int) offsetof (ac__type_alignof_, y)" "ac_cv_alignof_max_align_t" "$ac_includes_default +#ifndef offsetof +# define offsetof(type, member) ((char *) &((type *) 0)->member - (char *) 0) +#endif +typedef struct { char x; max_align_t y; } ac__type_alignof_;"; then : + +else + if test "$ac_cv_type_max_align_t" = yes; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error 77 "cannot compute alignment of max_align_t +See \`config.log' for more details" "$LINENO" 5; } + else + ac_cv_alignof_max_align_t=0 + fi +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_alignof_max_align_t" >&5 +$as_echo "$ac_cv_alignof_max_align_t" >&6; } + + + +cat >>confdefs.h <<_ACEOF +#define ALIGNOF_MAX_ALIGN_T $ac_cv_alignof_max_align_t +_ACEOF + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for long double" >&5 diff --git a/configure.ac b/configure.ac index 3f20d8980d8abc..0940b93c25f7bf 100644 --- a/configure.ac +++ b/configure.ac @@ -2921,6 +2921,7 @@ AC_CHECK_SIZEOF(size_t, 4) AC_CHECK_ALIGNOF(size_t) AC_CHECK_SIZEOF(pid_t, 4) AC_CHECK_SIZEOF(uintptr_t) +AC_CHECK_ALIGNOF(max_align_t) AC_TYPE_LONG_DOUBLE AC_CHECK_SIZEOF(long double, 16) diff --git a/pyconfig.h.in b/pyconfig.h.in index 236cee6588d49b..2c22b27af65ea3 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -19,6 +19,9 @@ /* The normal alignment of `long', in bytes. */ #undef ALIGNOF_LONG +/* The normal alignment of `max_align_t', in bytes. */ +#undef ALIGNOF_MAX_ALIGN_T + /* The normal alignment of `size_t', in bytes. */ #undef ALIGNOF_SIZE_T From d0122372f2acb4cc56b89ab8c577ff9039d17d89 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 09:17:12 +0100 Subject: [PATCH 30/65] GH-97950: Use new-style index directive ('module') (#103996) * Use new-style index directive ('module') - C API * Use new-style index directive ('module') - Library * Use new-style index directive ('module') - Reference * Use new-style index directive ('module') - Tutorial * Uncomment module removal in pairindextypes * Use new-style index directive ('module') - C API * Use new-style index directive ('module') - Library * Use new-style index directive ('module') - Reference --- Doc/c-api/exceptions.rst | 6 +++--- Doc/c-api/init.rst | 14 +++++++------- Doc/c-api/intro.rst | 6 +++--- Doc/library/_thread.rst | 2 +- Doc/library/binascii.rst | 4 ++-- Doc/library/cmath.rst | 2 +- Doc/library/copy.rst | 2 +- Doc/library/copyreg.rst | 4 ++-- Doc/library/exceptions.rst | 2 +- Doc/library/fnmatch.rst | 4 ++-- Doc/library/functions.rst | 4 ++-- Doc/library/http.client.rst | 2 +- Doc/library/internet.rst | 2 +- Doc/library/locale.rst | 4 ++-- Doc/library/marshal.rst | 4 ++-- Doc/library/os.path.rst | 2 +- Doc/library/os.rst | 4 ++-- Doc/library/pdb.rst | 4 ++-- Doc/library/posix.rst | 2 +- Doc/library/pwd.rst | 2 +- Doc/library/pyexpat.rst | 2 +- Doc/library/runpy.rst | 4 ++-- Doc/library/shelve.rst | 6 +++--- Doc/library/site.rst | 4 ++-- Doc/library/socket.rst | 2 +- Doc/library/stdtypes.rst | 8 ++++---- Doc/reference/compound_stmts.rst | 2 +- Doc/reference/datamodel.rst | 8 ++++---- Doc/reference/executionmodel.rst | 2 +- Doc/reference/toplevel_components.rst | 8 ++++---- Doc/tools/extensions/pyspecific.py | 6 +----- Doc/tutorial/inputoutput.rst | 2 +- Doc/tutorial/modules.rst | 4 ++-- 33 files changed, 65 insertions(+), 69 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 49d2f18d4573b0..4ed96f01dbbc3e 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -602,7 +602,7 @@ Signal Handling .. c:function:: int PyErr_CheckSignals() .. index:: - module: signal + pair: module; signal single: SIGINT single: KeyboardInterrupt (built-in exception) @@ -633,7 +633,7 @@ Signal Handling .. c:function:: void PyErr_SetInterrupt() .. index:: - module: signal + pair: module; signal single: SIGINT single: KeyboardInterrupt (built-in exception) @@ -648,7 +648,7 @@ Signal Handling .. c:function:: int PyErr_SetInterruptEx(int signum) .. index:: - module: signal + pair: module; signal single: KeyboardInterrupt (built-in exception) Simulate the effect of a signal arriving. The next time diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 38e324fb6409bc..26762969ef8eba 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -336,9 +336,9 @@ Initializing and finalizing the interpreter single: PyEval_InitThreads() single: modules (in module sys) single: path (in module sys) - module: builtins - module: __main__ - module: sys + pair: module; builtins + pair: module; __main__ + pair: module; sys triple: module; search; path single: PySys_SetArgv() single: PySys_SetArgvEx() @@ -1051,7 +1051,7 @@ code, or when embedding the Python interpreter: .. deprecated:: 3.9 - .. index:: module: _thread + .. index:: pair: module; _thread .. c:function:: int PyEval_ThreadsInitialized() @@ -1494,9 +1494,9 @@ function. You can create and destroy them using the following functions: .. c:function:: PyThreadState* Py_NewInterpreter() .. index:: - module: builtins - module: __main__ - module: sys + pair: module; builtins + pair: module; __main__ + pair: module; sys single: stdout (in module sys) single: stderr (in module sys) single: stdin (in module sys) diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index acd4e033dfbc4b..1a200b919f0e75 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -705,9 +705,9 @@ interpreter can only be used after the interpreter has been initialized. .. index:: single: Py_Initialize() - module: builtins - module: __main__ - module: sys + pair: module; builtins + pair: module; __main__ + pair: module; sys triple: module; search; path single: path (in module sys) diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 122692a428594f..ba9314e46ab6ea 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -208,7 +208,7 @@ In addition to these methods, lock objects can also be used via the **Caveats:** - .. index:: module: signal + .. index:: pair: module; signal * Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` exception will be received by an arbitrary thread. (When the :mod:`signal` diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 5a0815faa38eac..21960cb7972e6e 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -6,8 +6,8 @@ representations. .. index:: - module: uu - module: base64 + pair: module; uu + pair: module; base64 -------------- diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 5ed7a09b3e9db2..b17d58e1cc0ce1 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -301,7 +301,7 @@ Constants .. versionadded:: 3.6 -.. index:: module: math +.. index:: pair: module; math Note that the selection of functions is similar, but not identical, to that in module :mod:`math`. The reason for having two modules is that some users aren't diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index a8bc2fa55ea8c3..8f32477ed508c3 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -68,7 +68,7 @@ Shallow copies of dictionaries can be made using :meth:`dict.copy`, and of lists by assigning a slice of the entire list, for example, ``copied_list = original_list[:]``. -.. index:: module: pickle +.. index:: pair: module; pickle Classes can use the same interfaces to control copying that they use to control pickling. See the description of module :mod:`pickle` for information on these diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index afc3e66f0bf7ac..2a28c043f80723 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -7,8 +7,8 @@ **Source code:** :source:`Lib/copyreg.py` .. index:: - module: pickle - module: copy + pair: module; pickle + pair: module; copy -------------- diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 18c3f47dddc079..aee1cb5cc6a475 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -318,7 +318,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: OSError([arg]) OSError(errno, strerror[, filename[, winerror[, filename2]]]) - .. index:: module: errno + .. index:: pair: module; errno This exception is raised when a system function returns a system-related error, including I/O failures such as "file not found" or "disk full" diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index 46bf0fc2848058..aed8991d44772f 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -8,7 +8,7 @@ .. index:: single: filenames; wildcard expansion -.. index:: module: re +.. index:: pair: module; re -------------- @@ -38,7 +38,7 @@ special characters used in shell-style wildcards are: For a literal match, wrap the meta-characters in brackets. For example, ``'[?]'`` matches the character ``'?'``. -.. index:: module: glob +.. index:: pair: module; glob Note that the filename separator (``'/'`` on Unix) is *not* special to this module. See module :mod:`glob` for pathname expansion (:mod:`glob` uses diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 47e3880129595e..ddb918de115fcf 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1340,7 +1340,7 @@ are always available. They are listed here in alphabetical order. single: I/O control; buffering single: binary mode single: text mode - module: sys + pair: module; sys See also the file handling modules, such as :mod:`fileinput`, :mod:`io` (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, @@ -1987,7 +1987,7 @@ are always available. They are listed here in alphabetical order. .. index:: statement: import - module: builtins + pair: module; builtins .. note:: diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index abdc6b447a8b61..eb8c1e198e2b09 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -10,7 +10,7 @@ pair: HTTP; protocol single: HTTP; http.client (standard module) -.. index:: module: urllib.request +.. index:: pair: module; urllib.request -------------- diff --git a/Doc/library/internet.rst b/Doc/library/internet.rst index ff58dcf4d89c36..681769a4820dba 100644 --- a/Doc/library/internet.rst +++ b/Doc/library/internet.rst @@ -9,7 +9,7 @@ Internet Protocols and Support single: Internet single: World Wide Web -.. index:: module: socket +.. index:: pair: module; socket The modules described in this chapter implement internet protocols and support for related technology. They are all implemented in Python. Most of these diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index f726f8397c9648..f2abb3638a141f 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -16,7 +16,7 @@ functionality. The POSIX locale mechanism allows programmers to deal with certain cultural issues in an application, without requiring the programmer to know all the specifics of each country where the software is executed. -.. index:: module: _locale +.. index:: pair: module; _locale The :mod:`locale` module is implemented on top of the :mod:`_locale` module, which in turn uses an ANSI C locale implementation if available. @@ -464,7 +464,7 @@ The :mod:`locale` module defines the following exception and functions: .. data:: LC_CTYPE - .. index:: module: string + .. index:: pair: module; string Locale category for the character type functions. Depending on the settings of this category, the functions of module :mod:`string` dealing with case change diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index 24f9dc1689da4a..0556f19699dc15 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -15,8 +15,8 @@ undocumented on purpose; it may change between Python versions (although it rarely does). [#]_ .. index:: - module: pickle - module: shelve + pair: module; pickle + pair: module; shelve This is not a general "persistence" module. For general persistence and transfer of Python objects through RPC calls, see the modules :mod:`pickle` and diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 96bcb48ad7d126..7881c52db87090 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -159,7 +159,7 @@ the :mod:`glob` module.) On Unix and Windows, return the argument with an initial component of ``~`` or ``~user`` replaced by that *user*'s home directory. - .. index:: module: pwd + .. index:: pair: module; pwd On Unix, an initial ``~`` is replaced by the environment variable :envvar:`HOME` if it is set; otherwise the current user's home directory is looked up in the diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 76623c6305432a..641e289e77c518 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1284,7 +1284,7 @@ or `the MSDN `_ on Windo .. function:: openpty() - .. index:: module: pty + .. index:: pair: module; pty Open a new pseudo-terminal pair. Return a pair of file descriptors ``(master, slave)`` for the pty and the tty, respectively. The new file @@ -2890,7 +2890,7 @@ features: possible and call :func:`lstat` on the result. This does not apply to dangling symlinks or junction points, which will raise the usual exceptions. - .. index:: module: stat + .. index:: pair: module; stat Example:: diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 8a386aa77368f2..74bffef5562ae1 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -20,8 +20,8 @@ supports post-mortem debugging and can be called under program control. .. index:: single: Pdb (class in pdb) - module: bdb - module: cmd + pair: module; bdb + pair: module; cmd The debugger is extensible -- it is actually defined as the class :class:`Pdb`. This is currently undocumented but easily understood by reading the source. The diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index ec04b0dcfc162f..0413f9d02a8d57 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -11,7 +11,7 @@ This module provides access to operating system functionality that is standardized by the C Standard and the POSIX standard (a thinly disguised Unix interface). -.. index:: module: os +.. index:: pair: module; os **Do not import this module directly.** Instead, import the module :mod:`os`, which provides a *portable* version of this interface. On Unix, the :mod:`os` diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst index 98f3c45e29cbcb..7cafc66fd7e93c 100644 --- a/Doc/library/pwd.rst +++ b/Doc/library/pwd.rst @@ -39,7 +39,7 @@ raised if the entry asked for cannot be found. .. note:: - .. index:: module: crypt + .. index:: pair: module; crypt In traditional Unix the field ``pw_passwd`` usually contains a password encrypted with a DES derived algorithm (see module :mod:`crypt`). However most diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index d6581e21b01c0e..935e872480efda 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -33,7 +33,7 @@ can be set to handler functions. When an XML document is then fed to the parser, the handler functions are called for the character data and markup in the XML document. -.. index:: module: pyexpat +.. index:: pair: module; pyexpat This module uses the :mod:`pyexpat` module to provide access to the Expat parser. Direct use of the :mod:`pyexpat` module is deprecated. diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 501f4ddf5a3e3f..42ed8c253b8027 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -30,7 +30,7 @@ The :mod:`runpy` module provides two functions: .. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) .. index:: - module: __main__ + pair: module; __main__ Execute the code of the specified module and return the resulting module globals dictionary. The module's code is first located using the standard @@ -101,7 +101,7 @@ The :mod:`runpy` module provides two functions: .. function:: run_path(path_name, init_globals=None, run_name=None) .. index:: - module: __main__ + pair: module; __main__ Execute the code at the named filesystem location and return the resulting module globals dictionary. As with a script name supplied to the CPython diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index a50fc6f0bf77b2..dc87af398ed757 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -6,7 +6,7 @@ **Source code:** :source:`Lib/shelve.py` -.. index:: module: pickle +.. index:: pair: module; pickle -------------- @@ -95,8 +95,8 @@ Restrictions ------------ .. index:: - module: dbm.ndbm - module: dbm.gnu + pair: module; dbm.ndbm + pair: module; dbm.gnu * The choice of which database package will be used (such as :mod:`dbm.ndbm` or :mod:`dbm.gnu`) depends on which interface is available. Therefore it is not diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 4a88013f1d6ed2..a4efefd587a609 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -109,7 +109,7 @@ directory precedes the :file:`foo` directory because :file:`bar.pth` comes alphabetically before :file:`foo.pth`; and :file:`spam` is omitted because it is not mentioned in either path configuration file. -.. index:: module: sitecustomize +.. index:: pair: module; sitecustomize After these path manipulations, an attempt is made to import a module named :mod:`sitecustomize`, which can perform arbitrary site-specific customizations. @@ -121,7 +121,7 @@ with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), attempted output from :mod:`sitecustomize` is ignored. Any other exception causes a silent and perhaps mysterious failure of the process. -.. index:: module: usercustomize +.. index:: pair: module; usercustomize After this, an attempt is made to import a module named :mod:`usercustomize`, which can perform arbitrary user-specific customizations, if diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index c8ca555700a3c9..150c1f93afd967 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -1916,7 +1916,7 @@ to sockets. .. method:: socket.setsockopt(level, optname, None, optlen: int) :noindex: - .. index:: module: struct + .. index:: pair: module; struct Set the value of the given socket option (see the Unix manual page :manpage:`setsockopt(2)`). The needed symbolic constants are defined in the diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index f6662b4336c276..4c0a213dcbea26 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -330,7 +330,7 @@ Notes: (3) .. index:: - module: math + pair: module; math single: floor() (in module math) single: ceil() (in module math) single: trunc() (in module math) @@ -1600,7 +1600,7 @@ String Methods -------------- .. index:: - module: re + pair: module; re Strings implement all of the :ref:`common ` sequence operations, along with the additional methods described below. @@ -2511,7 +2511,7 @@ Binary Sequence Types --- :class:`bytes`, :class:`bytearray`, :class:`memoryview object: bytes object: bytearray object: memoryview - module: array + pair: module; array The core built-in types for manipulating binary data are :class:`bytes` and :class:`bytearray`. They are supported by :class:`memoryview` which uses @@ -5378,7 +5378,7 @@ Type Objects .. index:: builtin: type - module: types + pair: module; types Type objects represent the various object types. An object's type is accessed by the built-in function :func:`type`. There are no special operations on diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 0731589e4cadb8..4bdecbf243650f 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -297,7 +297,7 @@ traceback attached to them, they form a reference cycle with the stack frame, keeping all locals in that frame alive until the next garbage collection occurs. .. index:: - module: sys + pair: module; sys object: traceback Before an :keyword:`!except` clause's suite is executed, diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index c35bf4016a28d2..9d3ea3f78de16f 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -377,7 +377,7 @@ Sequences (and hence unhashable), byte arrays otherwise provide the same interface and functionality as immutable :class:`bytes` objects. - .. index:: module: array + .. index:: pair: module; array The extension module :mod:`array` provides an additional example of a mutable sequence type, as does the :mod:`collections` module. @@ -451,8 +451,8 @@ Mappings section :ref:`dict`). .. index:: - module: dbm.ndbm - module: dbm.gnu + pair: module; dbm.ndbm + pair: module; dbm.gnu The extension modules :mod:`dbm.ndbm` and :mod:`dbm.gnu` provide additional examples of mapping types, as does the :mod:`collections` @@ -909,7 +909,7 @@ Class instances I/O objects (also known as file objects) .. index:: builtin: open - module: io + pair: module; io single: popen() (in module os) single: makefile() (socket method) single: sys.stdin diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index a264015cbf4049..8917243999d399 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -151,7 +151,7 @@ to previously bound variables in the nearest enclosing function scope. :exc:`SyntaxError` is raised at compile time if the given name does not exist in any enclosing function scope. -.. index:: module: __main__ +.. index:: pair: module; __main__ The namespace for a module is automatically created the first time a module is imported. The main module for a script is always called :mod:`__main__`. diff --git a/Doc/reference/toplevel_components.rst b/Doc/reference/toplevel_components.rst index 319c9de484241e..ee472ace6e2129 100644 --- a/Doc/reference/toplevel_components.rst +++ b/Doc/reference/toplevel_components.rst @@ -21,9 +21,9 @@ Complete Python programs .. index:: single: program .. index:: - module: sys - module: __main__ - module: builtins + pair: module; sys + pair: module; __main__ + pair: module; builtins While a language specification need not prescribe how the language interpreter is invoked, it is useful to have a notion of a complete Python program. A @@ -38,7 +38,7 @@ the next section. .. index:: single: interactive mode - module: __main__ + pair: module; __main__ The interpreter may also be invoked in interactive mode; in this case, it does not read and execute a complete program but reads and executes one statement diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 39c7c42e66f9a0..12d3e382c87318 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -689,7 +689,7 @@ def patch_pairindextypes(app) -> None: # away from this, we need Sphinx to believe that these values don't # exist, by deleting them when using the gettext builder. - # pairindextypes.pop('module', None) + pairindextypes.pop('module', None) # pairindextypes.pop('keyword', None) # pairindextypes.pop('operator', None) # pairindextypes.pop('object', None) @@ -697,10 +697,6 @@ def patch_pairindextypes(app) -> None: # pairindextypes.pop('statement', None) # pairindextypes.pop('builtin', None) - # there needs to be at least one statement in this block, will be - # removed when the first of the below is uncommented. - pass - def setup(app): app.add_role('issue', issue_role) diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 3581b3727a53ea..aeacc1559a228c 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -466,7 +466,7 @@ Reference for a complete guide to file objects. Saving structured data with :mod:`json` --------------------------------------- -.. index:: module: json +.. index:: pair: module; json Strings can easily be written to and read from a file. Numbers take a bit more effort, since the :meth:`read` method only returns strings, which will have to diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 4daafa49a34d2e..3bd034bcc9703f 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -264,7 +264,7 @@ Some tips for experts: Standard Modules ================ -.. index:: module: sys +.. index:: pair: module; sys Python comes with a library of standard modules, described in a separate document, the Python Library Reference ("Library Reference" hereafter). Some @@ -345,7 +345,7 @@ Without arguments, :func:`dir` lists the names you have defined currently:: Note that it lists all types of names: variables, modules, functions, etc. -.. index:: module: builtins +.. index:: pair: module; builtins :func:`dir` does not list the names of built-in functions and variables. If you want a list of those, they are defined in the standard module From 33ca322c50baa5152afe388fb3b8b7d63dc5a9b9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 09:44:12 +0100 Subject: [PATCH 31/65] GH-97950: Use new-style index directive ('keyword') (#104153) * Uncomment keyword removal in pairindextypes * Use new-style index directive ('keyword') - Reference --- Doc/reference/compound_stmts.rst | 36 +++++++++++++++--------------- Doc/reference/expressions.rst | 6 ++--- Doc/reference/simple_stmts.rst | 12 +++++----- Doc/tools/extensions/pyspecific.py | 2 +- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 4bdecbf243650f..1f03180f1565ba 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -85,8 +85,8 @@ The :keyword:`!if` statement .. index:: ! statement: if - keyword: elif - keyword: else + pair: keyword; elif + pair: keyword; else single: : (colon); compound statement The :keyword:`if` statement is used for conditional execution: @@ -110,7 +110,7 @@ The :keyword:`!while` statement .. index:: ! statement: while - keyword: else + pair: keyword; else pair: loop; statement single: : (colon); compound statement @@ -143,8 +143,8 @@ The :keyword:`!for` statement .. index:: ! statement: for - keyword: in - keyword: else + pair: keyword; in + pair: keyword; else pair: target; list pair: loop; statement object: sequence @@ -206,10 +206,10 @@ The :keyword:`!try` statement .. index:: ! statement: try - keyword: except - keyword: finally - keyword: else - keyword: as + pair: keyword; except + pair: keyword; finally + pair: keyword; else + pair: keyword; as single: : (colon); compound statement The :keyword:`!try` statement specifies exception handlers and/or cleanup code @@ -326,7 +326,7 @@ stored in the :mod:`sys` module is reset to its previous value:: .. index:: - keyword: except_star + pair: keyword; except_star .. _except_star: @@ -387,7 +387,7 @@ cannot appear in an :keyword:`!except*` clause. .. index:: - keyword: else + pair: keyword; else statement: return statement: break statement: continue @@ -404,7 +404,7 @@ the :keyword:`!else` clause are not handled by the preceding :keyword:`except` clauses. -.. index:: keyword: finally +.. index:: pair: keyword; finally .. _finally: @@ -469,7 +469,7 @@ The :keyword:`!with` statement .. index:: ! statement: with - keyword: as + pair: keyword; as single: as; with statement single: , (comma); with statement single: : (colon); compound statement @@ -586,10 +586,10 @@ The :keyword:`!match` statement .. index:: ! statement: match - ! keyword: case + ! pair: keyword; case ! single: pattern matching - keyword: if - keyword: as + pair: keyword; if + pair: keyword; as pair: match; case single: as; match statement single: : (colon); compound statement @@ -1474,8 +1474,8 @@ Coroutine function definition : ["->" `expression`] ":" `suite` .. index:: - keyword: async - keyword: await + pair: keyword; async + pair: keyword; await Execution of Python coroutines can be suspended and resumed at many points (see :term:`coroutine`). :keyword:`await` expressions, :keyword:`async for` and diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 1e4a13fbd6a3ce..fe2b090b23c614 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -415,8 +415,8 @@ Yield expressions ----------------- .. index:: - keyword: yield - keyword: from + pair: keyword; yield + pair: keyword; from pair: yield; expression pair: generator; function @@ -1149,7 +1149,7 @@ a class instance: if that method was called. -.. index:: keyword: await +.. index:: pair: keyword; await .. _await: Await expression diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index c98ac81e415b72..5c7b5d7efaa2b5 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -495,7 +495,7 @@ If an expression list is present, it is evaluated, else ``None`` is substituted. :keyword:`return` leaves the current function call with the expression list (or ``None``) as return value. -.. index:: keyword: finally +.. index:: pair: keyword; finally When :keyword:`return` passes control out of a :keyword:`try` statement with a :keyword:`finally` clause, that :keyword:`!finally` clause is executed before @@ -679,7 +679,7 @@ The :keyword:`!break` statement :keyword:`while` loop, but not nested in a function or class definition within that loop. -.. index:: keyword: else +.. index:: pair: keyword; else pair: loop control; target It terminates the nearest enclosing loop, skipping the optional :keyword:`!else` @@ -688,7 +688,7 @@ clause if the loop has one. If a :keyword:`for` loop is terminated by :keyword:`break`, the loop control target keeps its current value. -.. index:: keyword: finally +.. index:: pair: keyword; finally When :keyword:`break` passes control out of a :keyword:`try` statement with a :keyword:`finally` clause, that :keyword:`!finally` clause is executed before @@ -705,7 +705,7 @@ The :keyword:`!continue` statement statement: for statement: while pair: loop; statement - keyword: finally + pair: keyword; finally .. productionlist:: python-grammar continue_stmt: "continue" @@ -729,8 +729,8 @@ The :keyword:`!import` statement ! statement: import single: module; importing pair: name; binding - keyword: from - keyword: as + pair: keyword; from + pair: keyword; as exception: ImportError single: , (comma); import statement diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 12d3e382c87318..c347a327e0910e 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -690,7 +690,7 @@ def patch_pairindextypes(app) -> None: # exist, by deleting them when using the gettext builder. pairindextypes.pop('module', None) - # pairindextypes.pop('keyword', None) + pairindextypes.pop('keyword', None) # pairindextypes.pop('operator', None) # pairindextypes.pop('object', None) # pairindextypes.pop('exception', None) From b0ce2db1184124575d5934d205344a9cf5debb79 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 10:03:11 +0100 Subject: [PATCH 32/65] GH-97950: Use new-style index directive ('operator') (#104156) --- Doc/library/stdtypes.rst | 56 +++++++++++++++--------------- Doc/reference/expressions.rst | 50 +++++++++++++------------- Doc/tools/extensions/pyspecific.py | 2 +- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 4c0a213dcbea26..46b0389adaa88f 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -61,8 +61,8 @@ objects considered false: ``range(0)`` .. index:: - operator: or - operator: and + pair: operator; or + pair: operator; and single: False single: True @@ -95,9 +95,9 @@ These are the Boolean operations, ordered by ascending priority: +-------------+---------------------------------+-------+ .. index:: - operator: and - operator: or - operator: not + pair: operator; and + pair: operator; or + pair: operator; not Notes: @@ -122,14 +122,14 @@ Comparisons .. index:: pair: chaining; comparisons pair: operator; comparison - operator: == - operator: < (less) - operator: <= - operator: > (greater) - operator: >= - operator: != - operator: is - operator: is not + pair: operator; == + pair: operator; < (less) + pair: operator; <= + pair: operator; > (greater) + pair: operator; >= + pair: operator; != + pair: operator; is + pair: operator; is not There are eight comparison operations in Python. They all have the same priority (which is higher than that of the Boolean operations). Comparisons can @@ -192,8 +192,8 @@ customized; also they can be applied to any two objects and never raise an exception. .. index:: - operator: in - operator: not in + pair: operator; in + pair: operator; not in Two more operations with the same syntactic priority, :keyword:`in` and :keyword:`not in`, are supported by types that are :term:`iterable` or @@ -253,11 +253,11 @@ and imaginary parts. single: operator; - (minus) single: - (minus); unary operator single: - (minus); binary operator - operator: * (asterisk) - operator: / (slash) - operator: // - operator: % (percent) - operator: ** + pair: operator; * (asterisk) + pair: operator; / (slash) + pair: operator; // + pair: operator; % (percent) + pair: operator; ** Python fully supports mixed arithmetic: when a binary arithmetic operator has operands of different numeric types, the operand with the "narrower" type is @@ -392,12 +392,12 @@ Bitwise Operations on Integer Types pair: bitwise; operations pair: shifting; operations pair: masking; operations - operator: | (vertical bar) - operator: ^ (caret) - operator: & (ampersand) - operator: << - operator: >> - operator: ~ (tilde) + pair: operator; | (vertical bar) + pair: operator; ^ (caret) + pair: operator; & (ampersand) + pair: operator; << + pair: operator; >> + pair: operator; ~ (tilde) Bitwise operations only make sense for integers. The result of bitwise operations is calculated as though carried out in two's complement with an @@ -952,8 +952,8 @@ operations have the same priority as the corresponding numeric operations. [3]_ pair: repetition; operation pair: subscript; operation pair: slice; operation - operator: in - operator: not in + pair: operator; in + pair: operator; not in single: count() (sequence method) single: index() (sequence method) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index fe2b090b23c614..b1bb162f62e1ec 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1171,7 +1171,7 @@ The power operator .. index:: pair: power; operation - operator: ** + pair: operator; ** The power operator binds more tightly than unary operators on its left; it binds less tightly than unary operators on its right. The syntax is: @@ -1232,7 +1232,7 @@ operation can be overridden with the :meth:`__pos__` special method. .. index:: single: inversion - operator: ~ (tilde) + pair: operator; ~ (tilde) The unary ``~`` (invert) operator yields the bitwise inversion of its integer argument. The bitwise inversion of ``x`` is defined as ``-(x+1)``. It only @@ -1267,7 +1267,7 @@ operators and one for additive operators: .. index:: single: multiplication - operator: * (asterisk) + pair: operator; * (asterisk) The ``*`` (multiplication) operator yields the product of its arguments. The arguments must either both be numbers, or one argument must be an integer and @@ -1280,7 +1280,7 @@ This operation can be customized using the special :meth:`__mul__` and .. index:: single: matrix multiplication - operator: @ (at) + pair: operator; @ (at) The ``@`` (at) operator is intended to be used for matrix multiplication. No builtin Python types implement this operator. @@ -1290,8 +1290,8 @@ builtin Python types implement this operator. .. index:: exception: ZeroDivisionError single: division - operator: / (slash) - operator: // + pair: operator; / (slash) + pair: operator; // The ``/`` (division) and ``//`` (floor division) operators yield the quotient of their arguments. The numeric arguments are first converted to a common type. @@ -1305,7 +1305,7 @@ This operation can be customized using the special :meth:`__truediv__` and .. index:: single: modulo - operator: % (percent) + pair: operator; % (percent) The ``%`` (modulo) operator yields the remainder from the division of the first argument by the second. The numeric arguments are first converted to a common @@ -1363,8 +1363,8 @@ Shifting operations .. index:: pair: shifting; operation - operator: << - operator: >> + pair: operator; << + pair: operator; >> The shifting operations have lower priority than the arithmetic operations: @@ -1399,7 +1399,7 @@ Each of the three bitwise operations has a different priority level: .. index:: pair: bitwise; and - operator: & (ampersand) + pair: operator; & (ampersand) The ``&`` operator yields the bitwise AND of its arguments, which must be integers or one of them must be a custom object overriding :meth:`__and__` or @@ -1408,7 +1408,7 @@ integers or one of them must be a custom object overriding :meth:`__and__` or .. index:: pair: bitwise; xor pair: exclusive; or - operator: ^ (caret) + pair: operator; ^ (caret) The ``^`` operator yields the bitwise XOR (exclusive OR) of its arguments, which must be integers or one of them must be a custom object overriding :meth:`__xor__` or @@ -1417,7 +1417,7 @@ must be integers or one of them must be a custom object overriding :meth:`__xor_ .. index:: pair: bitwise; or pair: inclusive; or - operator: | (vertical bar) + pair: operator; | (vertical bar) The ``|`` operator yields the bitwise (inclusive) OR of its arguments, which must be integers or one of them must be a custom object overriding :meth:`__or__` or @@ -1432,12 +1432,12 @@ Comparisons .. index:: single: comparison pair: C; language - operator: < (less) - operator: > (greater) - operator: <= - operator: >= - operator: == - operator: != + pair: operator; < (less) + pair: operator; > (greater) + pair: operator; <= + pair: operator; >= + pair: operator; == + pair: operator; != Unlike C, all comparison operations in Python have the same priority, which is lower than that of any arithmetic, shifting or bitwise operation. Also unlike @@ -1669,8 +1669,8 @@ raises the :exc:`IndexError` exception. (If any other exception is raised, it i if :keyword:`in` raised that exception). .. index:: - operator: in - operator: not in + pair: operator; in + pair: operator; not in pair: membership; test object: sequence @@ -1678,8 +1678,8 @@ The operator :keyword:`not in` is defined to have the inverse truth value of :keyword:`in`. .. index:: - operator: is - operator: is not + pair: operator; is + pair: operator; is not pair: identity; test @@ -1719,17 +1719,17 @@ control flow statements, the following values are interpreted as false: other values are interpreted as true. User-defined objects can customize their truth value by providing a :meth:`__bool__` method. -.. index:: operator: not +.. index:: pair: operator; not The operator :keyword:`not` yields ``True`` if its argument is false, ``False`` otherwise. -.. index:: operator: and +.. index:: pair: operator; and The expression ``x and y`` first evaluates *x*; if *x* is false, its value is returned; otherwise, *y* is evaluated and the resulting value is returned. -.. index:: operator: or +.. index:: pair: operator; or The expression ``x or y`` first evaluates *x*; if *x* is true, its value is returned; otherwise, *y* is evaluated and the resulting value is returned. diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index c347a327e0910e..8ca3ef04523a82 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -691,7 +691,7 @@ def patch_pairindextypes(app) -> None: pairindextypes.pop('module', None) pairindextypes.pop('keyword', None) - # pairindextypes.pop('operator', None) + pairindextypes.pop('operator', None) # pairindextypes.pop('object', None) # pairindextypes.pop('exception', None) # pairindextypes.pop('statement', None) From 6ab463684b9d79880d98cd1f1406aa86af65985e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 11:04:41 +0100 Subject: [PATCH 33/65] GH-97950: Use new-style index directive ('object') (#104158) * Uncomment object removal in pairindextypes * Use new-style index directive ('object') - C API * Use new-style index directive ('object') - Library * Use new-style index directive ('object') - Reference * Use new-style index directive ('object') - Tutorial --- Doc/c-api/bytearray.rst | 2 +- Doc/c-api/bytes.rst | 2 +- Doc/c-api/capsule.rst | 2 +- Doc/c-api/complex.rst | 2 +- Doc/c-api/concrete.rst | 6 +- Doc/c-api/dict.rst | 2 +- Doc/c-api/file.rst | 2 +- Doc/c-api/float.rst | 2 +- Doc/c-api/function.rst | 2 +- Doc/c-api/intro.rst | 2 +- Doc/c-api/list.rst | 2 +- Doc/c-api/long.rst | 4 +- Doc/c-api/memoryview.rst | 2 +- Doc/c-api/method.rst | 4 +- Doc/c-api/module.rst | 2 +- Doc/c-api/none.rst | 2 +- Doc/c-api/set.rst | 4 +- Doc/c-api/tuple.rst | 2 +- Doc/c-api/type.rst | 2 +- Doc/library/functions.rst | 2 +- Doc/library/socket.rst | 2 +- Doc/library/stdtypes.rst | 50 ++++++++--------- Doc/library/sys.rst | 2 +- Doc/library/traceback.rst | 2 +- Doc/reference/compound_stmts.rst | 10 ++-- Doc/reference/datamodel.rst | 88 +++++++++++++++--------------- Doc/reference/expressions.rst | 60 ++++++++++---------- Doc/reference/simple_stmts.rst | 16 +++--- Doc/tools/extensions/pyspecific.py | 2 +- Doc/tutorial/classes.rst | 2 +- Doc/tutorial/inputoutput.rst | 2 +- 31 files changed, 143 insertions(+), 143 deletions(-) diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 4bf3cfe100cd01..456f7d89bca03c 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -5,7 +5,7 @@ Byte Array Objects ------------------ -.. index:: object: bytearray +.. index:: pair: object; bytearray .. c:type:: PyByteArrayObject diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d62962cab45f6b..9f48f2ffafe170 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -8,7 +8,7 @@ Bytes Objects These functions raise :exc:`TypeError` when expecting a bytes parameter and called with a non-bytes parameter. -.. index:: object: bytes +.. index:: pair: object; bytes .. c:type:: PyBytesObject diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 1c8f432505ef68..427ed959c58568 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -5,7 +5,7 @@ Capsules -------- -.. index:: object: Capsule +.. index:: pair: object; Capsule Refer to :ref:`using-capsules` for more information on using these objects. diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 9228ce85200023..344da903da4c1a 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -5,7 +5,7 @@ Complex Number Objects ---------------------- -.. index:: object: complex number +.. index:: pair: object; complex number Python's complex number objects are implemented as two distinct types when viewed from the C API: one is the Python object exposed to Python programs, and diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 8d3124a12fa9d2..880f7b15ce68e8 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -40,7 +40,7 @@ This section describes Python type objects and the singleton object ``None``. Numeric Objects =============== -.. index:: object: numeric +.. index:: pair: object; numeric .. toctree:: @@ -55,7 +55,7 @@ Numeric Objects Sequence Objects ================ -.. index:: object: sequence +.. index:: pair: object; sequence Generic operations on sequence objects were discussed in the previous chapter; this section deals with the specific kinds of sequence objects that are @@ -77,7 +77,7 @@ intrinsic to the Python language. Container Objects ================= -.. index:: object: mapping +.. index:: pair: object; mapping .. toctree:: diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index b9f84cea785644..f02abb01f0220b 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -5,7 +5,7 @@ Dictionary Objects ------------------ -.. index:: object: dictionary +.. index:: pair: object; dictionary .. c:type:: PyDictObject diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index 58ed58e5466859..f32ecba9f27029 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -5,7 +5,7 @@ File Objects ------------ -.. index:: object: file +.. index:: pair: object; file These APIs are a minimal emulation of the Python 2 C API for built-in file objects, which used to rely on the buffered I/O (:c:expr:`FILE*`) support diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 023b12c20b7c83..05b2d100d575cb 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -5,7 +5,7 @@ Floating Point Objects ---------------------- -.. index:: object: floating point +.. index:: pair: object; floating point .. c:type:: PyFloatObject diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 947ed70404081b..5857dba82c11c6 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -5,7 +5,7 @@ Function Objects ---------------- -.. index:: object: function +.. index:: pair: object; function There are a few functions specific to Python functions. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 1a200b919f0e75..8de76e55cd0586 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -261,7 +261,7 @@ complete listing. Objects, Types and Reference Counts =================================== -.. index:: object: type +.. index:: pair: object; type Most Python/C API functions have one or more arguments as well as a return value of type :c:expr:`PyObject*`. This type is a pointer to an opaque data type diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index f9e65354a259f4..317421f0db8453 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -5,7 +5,7 @@ List Objects ------------ -.. index:: object: list +.. index:: pair: object; list .. c:type:: PyListObject diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 41b5632d23003f..4a71c89ad85d31 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -5,8 +5,8 @@ Integer Objects --------------- -.. index:: object: long integer - object: integer +.. index:: pair: object; long integer + pair: object; integer All integers are implemented as "long" integer objects of arbitrary size. diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index ebd5c7760437bf..2aa43318e7a455 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -3,7 +3,7 @@ .. _memoryview-objects: .. index:: - object: memoryview + pair: object; memoryview MemoryView objects ------------------ diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst index 6e7e1e21aa93f2..93ad30cd4f7a8d 100644 --- a/Doc/c-api/method.rst +++ b/Doc/c-api/method.rst @@ -5,7 +5,7 @@ Instance Method Objects ----------------------- -.. index:: object: instancemethod +.. index:: pair: object; instancemethod An instance method is a wrapper for a :c:data:`PyCFunction` and the new way to bind a :c:data:`PyCFunction` to a class object. It replaces the former call @@ -47,7 +47,7 @@ to bind a :c:data:`PyCFunction` to a class object. It replaces the former call Method Objects -------------- -.. index:: object: method +.. index:: pair: object; method Methods are bound function objects. Methods are always bound to an instance of a user-defined class. Unbound methods (methods bound to a class object) are diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index c0351c8a6c72aa..230b471d473be7 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -5,7 +5,7 @@ Module Objects -------------- -.. index:: object: module +.. index:: pair: object; module .. c:var:: PyTypeObject PyModule_Type diff --git a/Doc/c-api/none.rst b/Doc/c-api/none.rst index 26d2b7aab201ba..b84a16a28ead56 100644 --- a/Doc/c-api/none.rst +++ b/Doc/c-api/none.rst @@ -5,7 +5,7 @@ The ``None`` Object ------------------- -.. index:: object: None +.. index:: pair: object; None Note that the :c:type:`PyTypeObject` for ``None`` is not directly exposed in the Python/C API. Since ``None`` is a singleton, testing for object identity (using diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index f0d905bae8ae44..8e8af60252250e 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -9,8 +9,8 @@ Set Objects .. index:: - object: set - object: frozenset + pair: object; set + pair: object; frozenset This section details the public API for :class:`set` and :class:`frozenset` objects. Any functionality not listed below is best accessed using either diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 5acddf7849aa33..ac62058676eeeb 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -5,7 +5,7 @@ Tuple Objects ------------- -.. index:: object: tuple +.. index:: pair: object; tuple .. c:type:: PyTupleObject diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index c21fd9255d28ad..fb38935e003336 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -5,7 +5,7 @@ Type Objects ------------ -.. index:: object: type +.. index:: pair: object; type .. c:type:: PyTypeObject diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index ddb918de115fcf..5a022edb75b7d1 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1830,7 +1830,7 @@ are always available. They are listed here in alphabetical order. .. class:: type(object) type(name, bases, dict, **kwds) - .. index:: object: type + .. index:: pair: object; type With one argument, return the type of an *object*. The return value is a type object and generally the same object as returned by diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 150c1f93afd967..4ee0897db94063 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -19,7 +19,7 @@ all modern Unix systems, Windows, MacOS, and probably additional platforms. .. include:: ../includes/wasm-notavail.rst -.. index:: object: socket +.. index:: pair: object; socket The Python interface is a straightforward transliteration of the Unix system call and library interface for sockets to Python's object-oriented style: the diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 46b0389adaa88f..2f549f2e97c1de 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -205,11 +205,11 @@ Numeric Types --- :class:`int`, :class:`float`, :class:`complex` ================================================================ .. index:: - object: numeric - object: Boolean - object: integer - object: floating point - object: complex number + pair: object; numeric + pair: object; Boolean + pair: object; integer + pair: object; floating point + pair: object; complex number pair: C; language There are three distinct numeric types: :dfn:`integers`, :dfn:`floating @@ -927,7 +927,7 @@ described in dedicated sections. Common Sequence Operations -------------------------- -.. index:: object: sequence +.. index:: pair: object; sequence The operations in the following table are supported by most sequence types, both mutable and immutable. The :class:`collections.abc.Sequence` ABC is @@ -1112,7 +1112,7 @@ Immutable Sequence Types .. index:: triple: immutable; sequence; types - object: tuple + pair: object; tuple builtin: hash The only operation that immutable sequence types generally implement that is @@ -1134,8 +1134,8 @@ Mutable Sequence Types .. index:: triple: mutable; sequence; types - object: list - object: bytearray + pair: object; list + pair: object; bytearray The operations in the following table are defined on mutable sequence types. The :class:`collections.abc.MutableSequence` ABC is provided to make it @@ -1252,7 +1252,7 @@ Notes: Lists ----- -.. index:: object: list +.. index:: pair: object; list Lists are mutable sequences, typically used to store collections of homogeneous items (where the precise degree of similarity will vary by @@ -1331,7 +1331,7 @@ application). Tuples ------ -.. index:: object: tuple +.. index:: pair: object; tuple Tuples are immutable sequences, typically used to store collections of heterogeneous data (such as the 2-tuples produced by the :func:`enumerate` @@ -1375,7 +1375,7 @@ choice than a simple tuple object. Ranges ------ -.. index:: object: range +.. index:: pair: object; range The :class:`range` type represents an immutable sequence of numbers and is commonly used for looping a specific number of times in :keyword:`for` @@ -1500,7 +1500,7 @@ objects that compare equal might have different :attr:`~range.start`, .. index:: single: string; text sequence type single: str (built-in class); (see also string) - object: string + pair: object; string .. _textseq: @@ -1534,7 +1534,7 @@ Since there is no separate "character" type, indexing a string produces strings of length 1. That is, for a non-empty string *s*, ``s[0] == s[0:1]``. .. index:: - object: io.StringIO + pair: object; io.StringIO There is also no mutable string type, but :meth:`str.join` or :class:`io.StringIO` can be used to efficiently construct strings from @@ -2508,9 +2508,9 @@ Binary Sequence Types --- :class:`bytes`, :class:`bytearray`, :class:`memoryview ================================================================================= .. index:: - object: bytes - object: bytearray - object: memoryview + pair: object; bytes + pair: object; bytearray + pair: object; memoryview pair: module; array The core built-in types for manipulating binary data are :class:`bytes` and @@ -2526,7 +2526,7 @@ The :mod:`array` module supports efficient storage of basic data types like Bytes Objects ------------- -.. index:: object: bytes +.. index:: pair: object; bytes Bytes objects are immutable sequences of single bytes. Since many major binary protocols are based on the ASCII text encoding, bytes objects offer @@ -2633,7 +2633,7 @@ always convert a bytes object into a list of integers using ``list(b)``. Bytearray Objects ----------------- -.. index:: object: bytearray +.. index:: pair: object; bytearray :class:`bytearray` objects are a mutable counterpart to :class:`bytes` objects. @@ -4212,7 +4212,7 @@ copying. Set Types --- :class:`set`, :class:`frozenset` ============================================== -.. index:: object: set +.. index:: pair: object; set A :dfn:`set` object is an unordered collection of distinct :term:`hashable` objects. Common uses include membership testing, removing duplicates from a sequence, and @@ -4414,8 +4414,8 @@ Mapping Types --- :class:`dict` =============================== .. index:: - object: mapping - object: dictionary + pair: object; mapping + pair: object; dictionary triple: operations on; mapping; types triple: operations on; dictionary; type statement: del @@ -4889,7 +4889,7 @@ Generic Alias Type ------------------ .. index:: - object: GenericAlias + pair: object; GenericAlias pair: Generic; Alias ``GenericAlias`` objects are generally created by @@ -5144,7 +5144,7 @@ Union Type ---------- .. index:: - object: Union + pair: object; Union pair: union; type A union object holds the value of the ``|`` (bitwise or) operation on @@ -5301,7 +5301,7 @@ See :ref:`function` for more information. Methods ------- -.. index:: object: method +.. index:: pair: object; method Methods are functions that are called using the attribute notation. There are two flavors: built-in methods (such as :meth:`append` on lists) and class diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 57a0d0a1258c2a..bacf8ceac5041e 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -444,7 +444,7 @@ always available. object ` which typically encapsulates the call stack at the point where the exception last occurred. - .. index:: object: traceback + .. index:: pair: object; traceback If no exception is being handled anywhere on the stack, this function return a tuple containing three ``None`` values. diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 561c85290463ef..5c0e261b90763c 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -14,7 +14,7 @@ interpreter when it prints a stack trace. This is useful when you want to print stack traces under program control, such as in a "wrapper" around the interpreter. -.. index:: object: traceback +.. index:: pair: object; traceback The module uses traceback objects --- these are objects of type :class:`types.TracebackType`, which are assigned to the ``__traceback__`` field of :class:`BaseException` instances. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 1f03180f1565ba..5750d3149c8e95 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -147,7 +147,7 @@ The :keyword:`!for` statement pair: keyword; else pair: target; list pair: loop; statement - object: sequence + pair: object; sequence single: : (colon); compound statement The :keyword:`for` statement is used to iterate over the elements of a sequence @@ -298,7 +298,7 @@ keeping all locals in that frame alive until the next garbage collection occurs. .. index:: pair: module; sys - object: traceback + pair: object; traceback Before an :keyword:`!except` clause's suite is executed, the exception is stored in the :mod:`sys` module, where it can be accessed @@ -1194,8 +1194,8 @@ Function definitions pair: function; definition pair: function; name pair: name; binding - object: user-defined function - object: function + pair: object; user-defined function + pair: object; function pair: function; name pair: name; binding single: () (parentheses); function definition @@ -1363,7 +1363,7 @@ Class definitions ================= .. index:: - object: class + pair: object; class statement: class pair: class; definition pair: class; name diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 9d3ea3f78de16f..18906a005b2761 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -142,7 +142,7 @@ attributes.' These are attributes that provide access to the implementation and are not intended for general use. Their definition may change in the future. None - .. index:: object: None + .. index:: pair: object; None This type has a single value. There is a single object with this value. This object is accessed through the built-in name ``None``. It is used to signify the @@ -150,7 +150,7 @@ None don't explicitly return anything. Its truth value is false. NotImplemented - .. index:: object: NotImplemented + .. index:: pair: object; NotImplemented This type has a single value. There is a single object with this value. This object is accessed through the built-in name ``NotImplemented``. Numeric methods @@ -171,7 +171,7 @@ NotImplemented Ellipsis .. index:: - object: Ellipsis + pair: object; Ellipsis single: ...; ellipsis literal This type has a single value. There is a single object with this value. This @@ -179,7 +179,7 @@ Ellipsis ``Ellipsis``. Its truth value is true. :class:`numbers.Number` - .. index:: object: numeric + .. index:: pair: object; numeric These are created by numeric literals and returned as results by arithmetic operators and arithmetic built-in functions. Numeric objects are immutable; @@ -209,7 +209,7 @@ Ellipsis numbers: :class:`numbers.Integral` - .. index:: object: integer + .. index:: pair: object; integer These represent elements from the mathematical set of integers (positive and negative). @@ -225,7 +225,7 @@ Ellipsis Booleans (:class:`bool`) .. index:: - object: Boolean + pair: object; Boolean single: False single: True @@ -242,7 +242,7 @@ Ellipsis :class:`numbers.Real` (:class:`float`) .. index:: - object: floating point + pair: object; floating point pair: floating point; number pair: C; language pair: Java; language @@ -257,7 +257,7 @@ Ellipsis :class:`numbers.Complex` (:class:`complex`) .. index:: - object: complex + pair: object; complex pair: complex; number These represent complex numbers as a pair of machine-level double precision @@ -268,7 +268,7 @@ Ellipsis Sequences .. index:: builtin: len - object: sequence + pair: object; sequence single: index operation single: item selection single: subscription @@ -293,8 +293,8 @@ Sequences Immutable sequences .. index:: - object: immutable sequence - object: immutable + pair: object; immutable sequence + pair: object; immutable An object of an immutable sequence type cannot change once it is created. (If the object contains references to other objects, these other objects may be @@ -328,7 +328,7 @@ Sequences Tuples .. index:: - object: tuple + pair: object; tuple pair: singleton; tuple pair: empty; tuple @@ -350,8 +350,8 @@ Sequences Mutable sequences .. index:: - object: mutable sequence - object: mutable + pair: object; mutable sequence + pair: object; mutable pair: assignment; statement single: subscription single: slicing @@ -363,7 +363,7 @@ Sequences There are currently two intrinsic mutable sequence types: Lists - .. index:: object: list + .. index:: pair: object; list The items of a list are arbitrary Python objects. Lists are formed by placing a comma-separated list of expressions in square brackets. (Note @@ -385,7 +385,7 @@ Sequences Set types .. index:: builtin: len - object: set type + pair: object; set type These represent unordered, finite sets of unique, immutable objects. As such, they cannot be indexed by any subscript. However, they can be iterated over, and @@ -402,14 +402,14 @@ Set types There are currently two intrinsic set types: Sets - .. index:: object: set + .. index:: pair: object; set These represent a mutable set. They are created by the built-in :func:`set` constructor and can be modified afterwards by several methods, such as :meth:`~set.add`. Frozen sets - .. index:: object: frozenset + .. index:: pair: object; frozenset These represent an immutable set. They are created by the built-in :func:`frozenset` constructor. As a frozenset is immutable and @@ -420,7 +420,7 @@ Mappings .. index:: builtin: len single: subscription - object: mapping + pair: object; mapping These represent finite sets of objects indexed by arbitrary index sets. The subscript notation ``a[k]`` selects the item indexed by ``k`` from the mapping @@ -431,7 +431,7 @@ Mappings There is currently a single intrinsic mapping type: Dictionaries - .. index:: object: dictionary + .. index:: pair: object; dictionary These represent finite sets of objects indexed by nearly arbitrary values. The only types of values not acceptable as keys are values containing lists or @@ -465,7 +465,7 @@ Mappings Callable types .. index:: - object: callable + pair: object; callable pair: function; call single: invocation pair: function; argument @@ -476,8 +476,8 @@ Callable types User-defined functions .. index:: pair: user-defined; function - object: function - object: user-defined function + pair: object; function + pair: object; user-defined function A user-defined function object is created by a function definition (see section :ref:`function`). It should be called with an argument list @@ -580,8 +580,8 @@ Callable types Instance methods .. index:: - object: method - object: user-defined method + pair: object; method + pair: object; user-defined method pair: user-defined; method An instance method object combines a class, a class instance and any @@ -688,8 +688,8 @@ Callable types Built-in functions .. index:: - object: built-in function - object: function + pair: object; built-in function + pair: object; function pair: C; language A built-in function object is a wrapper around a C function. Examples of @@ -703,8 +703,8 @@ Callable types Built-in methods .. index:: - object: built-in method - object: method + pair: object; built-in method + pair: object; method pair: built-in; method This is really a different disguise of a built-in function, this time containing @@ -728,7 +728,7 @@ Callable types Modules .. index:: statement: import - object: module + pair: object; module Modules are a basic organizational unit of Python code, and are created by the :ref:`import system ` as invoked either by the @@ -805,12 +805,12 @@ Custom classes .. XXX: Could we add that MRO doc as an appendix to the language ref? .. index:: - object: class - object: class instance - object: instance + pair: object; class + pair: object; class instance + pair: object; instance pair: class object; call single: container - object: dictionary + pair: object; dictionary pair: class; attribute When a class attribute reference (for class :class:`C`, say) would yield a @@ -865,8 +865,8 @@ Custom classes Class instances .. index:: - object: class instance - object: instance + pair: object; class instance + pair: object; instance pair: class; instance pair: class instance; attribute @@ -892,9 +892,9 @@ Class instances dictionary directly. .. index:: - object: numeric - object: sequence - object: mapping + pair: object; numeric + pair: object; sequence + pair: object; mapping Class instances can pretend to be numbers, sequences, or mappings if they have methods with certain special names. See section :ref:`specialnames`. @@ -996,7 +996,7 @@ Internal types required stack size; :attr:`co_flags` is an integer encoding a number of flags for the interpreter. - .. index:: object: generator + .. index:: pair: object; generator The following flag bits are defined for :attr:`co_flags`: bit ``0x04`` is set if the function uses the ``*arguments`` syntax to accept an arbitrary number of @@ -1053,7 +1053,7 @@ Internal types .. _frame-objects: Frame objects - .. index:: object: frame + .. index:: pair: object; frame Frame objects represent execution frames. They may occur in traceback objects (see below), and are also passed to registered trace functions. @@ -1116,7 +1116,7 @@ Internal types Traceback objects .. index:: - object: traceback + pair: object; traceback pair: stack; trace pair: exception; handler pair: execution; stack @@ -1498,7 +1498,7 @@ Basic customization .. method:: object.__hash__(self) .. index:: - object: dictionary + pair: object; dictionary builtin: hash Called by built-in function :func:`hash` and for operations on members of @@ -2506,7 +2506,7 @@ through the object's keys; for sequences, it should iterate through the values. .. versionadded:: 3.4 -.. index:: object: slice +.. index:: pair: object; slice .. note:: diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index b1bb162f62e1ec..78185657bf822e 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -245,7 +245,7 @@ List displays pair: list; display pair: list; comprehensions pair: empty; list - object: list + pair: object; list single: [] (square brackets); list expression single: , (comma); expression list @@ -270,7 +270,7 @@ Set displays .. index:: pair: set; display pair: set; comprehensions - object: set + pair: object; set single: {} (curly brackets); set expression single: , (comma); expression list @@ -299,7 +299,7 @@ Dictionary displays pair: dictionary; display pair: dictionary; comprehensions key, datum, key/datum pair - object: dictionary + pair: object; dictionary single: {} (curly brackets); dictionary expression single: : (colon); in dictionary expressions single: , (comma); in dictionary displays @@ -361,7 +361,7 @@ Generator expressions .. index:: pair: generator; expression - object: generator + pair: object; generator single: () (parentheses); generator expression A generator expression is a compact generator notation in parentheses: @@ -522,7 +522,7 @@ on the right hand side of an assignment statement. The proposal that expanded on :pep:`492` by adding generator capabilities to coroutine functions. -.. index:: object: generator +.. index:: pair: object; generator .. _generator-methods: Generator-iterator methods @@ -701,7 +701,7 @@ of a *finalizer* method see the implementation of The expression ``yield from `` is a syntax error when used in an asynchronous generator function. -.. index:: object: asynchronous-generator +.. index:: pair: object; asynchronous-generator .. _asynchronous-generator-methods: Asynchronous generator-iterator methods @@ -811,8 +811,8 @@ An attribute reference is a primary followed by a period and a name: .. index:: exception: AttributeError - object: module - object: list + pair: object; module + pair: object; list The primary must evaluate to an object of a type that supports attribute references, which most objects do. This object is then asked to produce the @@ -833,12 +833,12 @@ Subscriptions single: [] (square brackets); subscription .. index:: - object: sequence - object: mapping - object: string - object: tuple - object: list - object: dictionary + pair: object; sequence + pair: object; mapping + pair: object; string + pair: object; tuple + pair: object; list + pair: object; dictionary pair: sequence; item The subscription of an instance of a :ref:`container class ` @@ -906,10 +906,10 @@ Slicings single: , (comma); slicing .. index:: - object: sequence - object: string - object: tuple - object: list + pair: object; sequence + pair: object; string + pair: object; tuple + pair: object; list A slicing selects a range of items in a sequence object (e.g., a string, tuple or list). Slicings may be used as expressions or as targets in assignment or @@ -950,7 +950,7 @@ substituting ``None`` for missing expressions. .. index:: - object: callable + pair: object; callable single: call single: argument; call semantics single: () (parentheses); call @@ -1100,8 +1100,8 @@ a user-defined function: .. index:: pair: function; call triple: user-defined; function; call - object: user-defined function - object: function + pair: object; user-defined function + pair: object; function The code block for the function is executed, passing it the argument list. The first thing the code block will do is bind the formal parameters to the @@ -1115,25 +1115,25 @@ a built-in function or method: pair: built-in function; call pair: method; call pair: built-in method; call - object: built-in method - object: built-in function - object: method - object: function + pair: object; built-in method + pair: object; built-in function + pair: object; method + pair: object; function The result is up to the interpreter; see :ref:`built-in-funcs` for the descriptions of built-in functions and methods. a class object: .. index:: - object: class + pair: object; class pair: class object; call A new instance of that class is returned. a class instance method: .. index:: - object: class instance - object: instance + pair: object; class instance + pair: object; instance pair: class instance; call The corresponding user-defined function is called, with an argument list that is @@ -1672,7 +1672,7 @@ if :keyword:`in` raised that exception). pair: operator; in pair: operator; not in pair: membership; test - object: sequence + pair: object; sequence The operator :keyword:`not in` is defined to have the inverse truth value of :keyword:`in`. @@ -1854,7 +1854,7 @@ Expression lists starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] starred_item: `assignment_expression` | "*" `or_expr` -.. index:: object: tuple +.. index:: pair: object; tuple Except when part of a list or set display, an expression list containing at least one comma yields a tuple. The length of diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 5c7b5d7efaa2b5..acc3998e09be22 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -54,7 +54,7 @@ expression). .. index:: builtin: repr - object: None + pair: object; None pair: string; conversion single: output pair: standard; output @@ -76,7 +76,7 @@ Assignment statements pair: assignment; statement pair: binding; name pair: rebinding; name - object: mutable + pair: object; mutable pair: attribute; assignment Assignment statements are used to (re)bind names to values and to modify @@ -185,7 +185,7 @@ Assignment of an object to a single target is recursively defined as follows. .. index:: pair: subscription; assignment - object: mutable + pair: object; mutable * If the target is a subscription: The primary expression in the reference is evaluated. It should yield either a mutable sequence object (such as a list) @@ -193,8 +193,8 @@ Assignment of an object to a single target is recursively defined as follows. evaluated. .. index:: - object: sequence - object: list + pair: object; sequence + pair: object; list If the primary is a mutable sequence object (such as a list), the subscript must yield an integer. If it is negative, the sequence's length is added to @@ -204,8 +204,8 @@ Assignment of an object to a single target is recursively defined as follows. raised (assignment to a subscripted sequence cannot add new items to a list). .. index:: - object: mapping - object: dictionary + pair: object; mapping + pair: object; dictionary If the primary is a mapping object (such as a dictionary), the subscript must have a type compatible with the mapping's key type, and the mapping is then @@ -574,7 +574,7 @@ instantiating the class with no arguments. The :dfn:`type` of the exception is the exception instance's class, the :dfn:`value` is the instance itself. -.. index:: object: traceback +.. index:: pair: object; traceback A traceback object is normally created automatically when an exception is raised and attached to it as the :attr:`__traceback__` attribute, which is writable. diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 8ca3ef04523a82..41016113d25732 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -692,7 +692,7 @@ def patch_pairindextypes(app) -> None: pairindextypes.pop('module', None) pairindextypes.pop('keyword', None) pairindextypes.pop('operator', None) - # pairindextypes.pop('object', None) + pairindextypes.pop('object', None) # pairindextypes.pop('exception', None) # pairindextypes.pop('statement', None) # pairindextypes.pop('builtin', None) diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 116801177a3add..06445e000c1ef6 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -344,7 +344,7 @@ list objects have methods called append, insert, remove, sort, and so on. However, in the following discussion, we'll use the term method exclusively to mean methods of class instance objects, unless explicitly stated otherwise.) -.. index:: object: method +.. index:: pair: object; method Valid method names of an instance object depend on its class. By definition, all attributes of a class that are function objects define corresponding diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index aeacc1559a228c..3fcf4e3f43a300 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -286,7 +286,7 @@ Reading and Writing Files .. index:: builtin: open - object: file + pair: object; file :func:`open` returns a :term:`file object`, and is most commonly used with two positional arguments and one keyword argument: From 83751bbd142c23ca3f6af34ec617630dc3173b2a Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Thu, 4 May 2023 18:20:20 +0800 Subject: [PATCH 34/65] gh-104089: catch DeprecationWarning in `test_fstring` (#104137) Co-authored-by: Alex Waygood --- Lib/test/test_fstring.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index be71fde5aaba54..58e2550715cecf 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -980,11 +980,18 @@ def test_roundtrip_raw_quotes(self): self.assertEqual(fr'\"\'\"\'', '\\"\\\'\\"\\\'') def test_fstring_backslash_before_double_bracket(self): - self.assertEqual(f'\{{\}}', '\\{\\}') - self.assertEqual(f'\{{', '\\{') - self.assertEqual(f'\{{{1+1}', '\\{2') - self.assertEqual(f'\}}{1+1}', '\\}2') - self.assertEqual(f'{1+1}\}}', '2\\}') + deprecated_cases = [ + (r"f'\{{\}}'", '\\{\\}'), + (r"f'\{{'", '\\{'), + (r"f'\{{{1+1}'", '\\{2'), + (r"f'\}}{1+1}'", '\\}2'), + (r"f'{1+1}\}}'", '2\\}') + ] + for case, expected_result in deprecated_cases: + with self.subTest(case=case, expected_result=expected_result): + with self.assertWarns(DeprecationWarning): + result = eval(case) + self.assertEqual(result, expected_result) self.assertEqual(fr'\{{\}}', '\\{\\}') self.assertEqual(fr'\{{', '\\{') self.assertEqual(fr'\{{{1+1}', '\\{2') From 3e7995ca119d4253d05a520f35ab3efca793ee49 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 11:30:24 +0100 Subject: [PATCH 35/65] GH-97950: Use new-style index directive ('exception') (#104160) --- Doc/reference/expressions.rst | 18 +++++++++--------- Doc/reference/simple_stmts.rst | 6 +++--- Doc/tools/extensions/pyspecific.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 78185657bf822e..b97a08f25d92a2 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -71,7 +71,7 @@ An identifier occurring as an atom is a name. See section :ref:`identifiers` for lexical definition and section :ref:`naming` for documentation of naming and binding. -.. index:: exception: NameError +.. index:: pair: exception; NameError When the name is bound to an object, evaluation of the atom yields that object. When a name is not bound, an attempt to evaluate it raises a :exc:`NameError` @@ -534,7 +534,7 @@ be used to control the execution of a generator function. Note that calling any of the generator methods below when the generator is already executing raises a :exc:`ValueError` exception. -.. index:: exception: StopIteration +.. index:: pair: exception; StopIteration .. method:: generator.__next__() @@ -589,7 +589,7 @@ is already executing raises a :exc:`ValueError` exception. The second signature \(type\[, value\[, traceback\]\]\) is deprecated and may be removed in a future version of Python. -.. index:: exception: GeneratorExit +.. index:: pair: exception; GeneratorExit .. method:: generator.close() @@ -711,7 +711,7 @@ This subsection describes the methods of an asynchronous generator iterator, which are used to control the execution of a generator function. -.. index:: exception: StopAsyncIteration +.. index:: pair: exception; StopAsyncIteration .. coroutinemethod:: agen.__anext__() @@ -763,7 +763,7 @@ which are used to control the execution of a generator function. The second signature \(type\[, value\[, traceback\]\]\) is deprecated and may be removed in a future version of Python. -.. index:: exception: GeneratorExit +.. index:: pair: exception; GeneratorExit .. coroutinemethod:: agen.aclose() @@ -810,7 +810,7 @@ An attribute reference is a primary followed by a period and a name: attributeref: `primary` "." `identifier` .. index:: - exception: AttributeError + pair: exception; AttributeError pair: object; module pair: object; list @@ -1241,7 +1241,7 @@ applies to integral numbers or to custom objects that override the -.. index:: exception: TypeError +.. index:: pair: exception; TypeError In all three cases, if the argument does not have the proper type, a :exc:`TypeError` exception is raised. @@ -1288,7 +1288,7 @@ builtin Python types implement this operator. .. versionadded:: 3.5 .. index:: - exception: ZeroDivisionError + pair: exception; ZeroDivisionError single: division pair: operator; / (slash) pair: operator; // @@ -1377,7 +1377,7 @@ the left or right by the number of bits given by the second argument. This operation can be customized using the special :meth:`__lshift__` and :meth:`__rshift__` methods. -.. index:: exception: ValueError +.. index:: pair: exception; ValueError A right shift by *n* bits is defined as floor division by ``pow(2,n)``. A left shift by *n* bits is defined as multiplication with ``pow(2,n)``. diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index acc3998e09be22..7cf096019631da 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -398,7 +398,7 @@ The extended form, ``assert expression1, expression2``, is equivalent to :: .. index:: single: __debug__ - exception: AssertionError + pair: exception; AssertionError These equivalences assume that :const:`__debug__` and :exc:`AssertionError` refer to the built-in variables with those names. In the current implementation, the @@ -521,7 +521,7 @@ The :keyword:`!yield` statement single: generator; function single: generator; iterator single: function; generator - exception: StopIteration + pair: exception; StopIteration .. productionlist:: python-grammar yield_stmt: `yield_expression` @@ -731,7 +731,7 @@ The :keyword:`!import` statement pair: name; binding pair: keyword; from pair: keyword; as - exception: ImportError + pair: exception; ImportError single: , (comma); import statement .. productionlist:: python-grammar diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 41016113d25732..9a62ae748e0744 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -693,7 +693,7 @@ def patch_pairindextypes(app) -> None: pairindextypes.pop('keyword', None) pairindextypes.pop('operator', None) pairindextypes.pop('object', None) - # pairindextypes.pop('exception', None) + pairindextypes.pop('exception', None) # pairindextypes.pop('statement', None) # pairindextypes.pop('builtin', None) From 55d50d147c953fab37b273bca9ab010f40e067d3 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Thu, 4 May 2023 11:48:45 +0100 Subject: [PATCH 36/65] GH-97950: Use new-style index directive ('statement') (#104162) --- Doc/library/exceptions.rst | 8 +++--- Doc/library/functions.rst | 2 +- Doc/library/site.rst | 2 +- Doc/library/stdtypes.rst | 8 +++--- Doc/reference/compound_stmts.rst | 42 +++++++++++++++--------------- Doc/reference/datamodel.rst | 8 +++--- Doc/reference/simple_stmts.rst | 32 +++++++++++------------ Doc/tools/extensions/pyspecific.py | 2 +- Doc/tutorial/controlflow.rst | 2 +- 9 files changed, 53 insertions(+), 53 deletions(-) diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index aee1cb5cc6a475..4c84e5f855431a 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -4,8 +4,8 @@ Built-in Exceptions =================== .. index:: - statement: try - statement: except + pair: statement; try + pair: statement; except In Python, all exceptions must be instances of a class that derives from :class:`BaseException`. In a :keyword:`try` statement with an :keyword:`except` @@ -14,7 +14,7 @@ classes derived from that class (but not exception classes from which *it* is derived). Two exception classes that are not related via subclassing are never equivalent, even if they have the same name. -.. index:: statement: raise +.. index:: pair: statement; raise The built-in exceptions listed below can be generated by the interpreter or built-in functions. Except where mentioned, they have an "associated value" @@ -175,7 +175,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: AssertionError - .. index:: statement: assert + .. index:: pair: statement; assert Raised when an :keyword:`assert` statement fails. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 5a022edb75b7d1..80b7b8b4f4ed07 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1986,7 +1986,7 @@ are always available. They are listed here in alphabetical order. .. function:: __import__(name, globals=None, locals=None, fromlist=(), level=0) .. index:: - statement: import + pair: statement; import pair: module; builtins .. note:: diff --git a/Doc/library/site.rst b/Doc/library/site.rst index a4efefd587a609..44f90a3b9e496f 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -51,7 +51,7 @@ searched for site-packages; otherwise they will. .. index:: single: # (hash); comment - statement: import + pair: statement; import A path configuration file is a file whose name has the form :file:`{name}.pth` and exists in one of the four directories mentioned above; its contents are diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 2f549f2e97c1de..c739485c5564da 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -32,8 +32,8 @@ Truth Value Testing =================== .. index:: - statement: if - statement: while + pair: statement; if + pair: statement; while pair: truth; value pair: Boolean; operations single: false @@ -1152,7 +1152,7 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). triple: operations on; list; type pair: subscript; assignment pair: slice; assignment - statement: del + pair: statement; del single: append() (sequence method) single: clear() (sequence method) single: copy() (sequence method) @@ -4418,7 +4418,7 @@ Mapping Types --- :class:`dict` pair: object; dictionary triple: operations on; mapping; types triple: operations on; dictionary; type - statement: del + pair: statement; del builtin: len A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 5750d3149c8e95..e9c1c493ae4278 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -84,7 +84,7 @@ The :keyword:`!if` statement ============================ .. index:: - ! statement: if + ! pair: statement; if pair: keyword; elif pair: keyword; else single: : (colon); compound statement @@ -109,7 +109,7 @@ The :keyword:`!while` statement =============================== .. index:: - ! statement: while + ! pair: statement; while pair: keyword; else pair: loop; statement single: : (colon); compound statement @@ -127,8 +127,8 @@ suite of the :keyword:`!else` clause, if present, is executed and the loop terminates. .. index:: - statement: break - statement: continue + pair: statement; break + pair: statement; continue A :keyword:`break` statement executed in the first suite terminates the loop without executing the :keyword:`!else` clause's suite. A :keyword:`continue` @@ -142,7 +142,7 @@ The :keyword:`!for` statement ============================= .. index:: - ! statement: for + ! pair: statement; for pair: keyword; in pair: keyword; else pair: target; list @@ -167,8 +167,8 @@ the suite in the :keyword:`!else` clause, if present, is executed, and the loop terminates. .. index:: - statement: break - statement: continue + pair: statement; break + pair: statement; continue A :keyword:`break` statement executed in the first suite terminates the loop without executing the :keyword:`!else` clause's suite. A :keyword:`continue` @@ -205,7 +205,7 @@ The :keyword:`!try` statement ============================= .. index:: - ! statement: try + ! pair: statement; try pair: keyword; except pair: keyword; finally pair: keyword; else @@ -388,9 +388,9 @@ cannot appear in an :keyword:`!except*` clause. .. index:: pair: keyword; else - statement: return - statement: break - statement: continue + pair: statement; return + pair: statement; break + pair: statement; continue .. _except_else: @@ -434,9 +434,9 @@ The exception information is not available to the program during execution of the :keyword:`!finally` clause. .. index:: - statement: return - statement: break - statement: continue + pair: statement; return + pair: statement; break + pair: statement; continue When a :keyword:`return`, :keyword:`break` or :keyword:`continue` statement is executed in the :keyword:`try` suite of a :keyword:`!try`...\ :keyword:`!finally` @@ -468,7 +468,7 @@ The :keyword:`!with` statement ============================== .. index:: - ! statement: with + ! pair: statement; with pair: keyword; as single: as; with statement single: , (comma); with statement @@ -585,7 +585,7 @@ The :keyword:`!match` statement =============================== .. index:: - ! statement: match + ! pair: statement; match ! pair: keyword; case ! single: pattern matching pair: keyword; if @@ -1190,7 +1190,7 @@ Function definitions ==================== .. index:: - statement: def + pair: statement; def pair: function; definition pair: function; name pair: name; binding @@ -1364,7 +1364,7 @@ Class definitions .. index:: pair: object; class - statement: class + pair: statement; class pair: class; definition pair: class; name pair: name; binding @@ -1463,7 +1463,7 @@ Coroutines .. versionadded:: 3.5 -.. index:: statement: async def +.. index:: pair: statement; async def .. _`async def`: Coroutine function definition @@ -1497,7 +1497,7 @@ An example of a coroutine function:: ``await`` and ``async`` are now keywords; previously they were only treated as such inside the body of a coroutine function. -.. index:: statement: async for +.. index:: pair: statement; async for .. _`async for`: The :keyword:`!async for` statement @@ -1542,7 +1542,7 @@ It is a :exc:`SyntaxError` to use an ``async for`` statement outside the body of a coroutine function. -.. index:: statement: async with +.. index:: pair: statement; async with .. _`async with`: The :keyword:`!async with` statement diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 18906a005b2761..1294d683e268a0 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -727,7 +727,7 @@ Callable types Modules .. index:: - statement: import + pair: statement; import pair: object; module Modules are a basic organizational unit of Python code, and are created by @@ -1151,7 +1151,7 @@ Internal types single: tb_frame (traceback attribute) single: tb_lineno (traceback attribute) single: tb_lasti (traceback attribute) - statement: try + pair: statement; try Special read-only attributes: :attr:`tb_frame` points to the execution frame of the current level; @@ -1310,7 +1310,7 @@ Basic customization .. index:: single: destructor single: finalizer - statement: del + pair: statement; del Called when the instance is about to be destroyed. This is also called a finalizer or (improperly) a destructor. If a base class has a @@ -2796,7 +2796,7 @@ execution of the block of code. Context managers are normally invoked using the used by directly invoking their methods. .. index:: - statement: with + pair: statement; with single: context manager Typical uses of context managers include saving and restoring various kinds of diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 7cf096019631da..ca1aa744389bda 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -376,7 +376,7 @@ The :keyword:`!assert` statement ================================ .. index:: - ! statement: assert + ! pair: statement; assert pair: debugging; assertions single: , (comma); expression list @@ -419,7 +419,7 @@ The :keyword:`!pass` statement ============================== .. index:: - statement: pass + pair: statement; pass pair: null; operation pair: null; operation @@ -441,7 +441,7 @@ The :keyword:`!del` statement ============================= .. index:: - ! statement: del + ! pair: statement; del pair: deletion; target triple: deletion; target; list @@ -454,7 +454,7 @@ Rather than spelling it out in full details, here are some hints. Deletion of a target list recursively deletes each target, from left to right. .. index:: - statement: global + pair: statement; global pair: unbinding; name Deletion of a name removes the binding of that name from the local or global @@ -480,7 +480,7 @@ The :keyword:`!return` statement ================================ .. index:: - ! statement: return + ! pair: statement; return pair: function; definition pair: class; definition @@ -517,7 +517,7 @@ The :keyword:`!yield` statement =============================== .. index:: - statement: yield + pair: statement; yield single: generator; function single: generator; iterator single: function; generator @@ -553,7 +553,7 @@ The :keyword:`!raise` statement =============================== .. index:: - ! statement: raise + ! pair: statement; raise single: exception pair: raising; exception single: __traceback__ (exception attribute) @@ -667,9 +667,9 @@ The :keyword:`!break` statement =============================== .. index:: - ! statement: break - statement: for - statement: while + ! pair: statement; break + pair: statement; for + pair: statement; while pair: loop; statement .. productionlist:: python-grammar @@ -701,9 +701,9 @@ The :keyword:`!continue` statement ================================== .. index:: - ! statement: continue - statement: for - statement: while + ! pair: statement; continue + pair: statement; for + pair: statement; while pair: loop; statement pair: keyword; finally @@ -726,7 +726,7 @@ The :keyword:`!import` statement ================================ .. index:: - ! statement: import + ! pair: statement; import single: module; importing pair: name; binding pair: keyword; from @@ -942,7 +942,7 @@ The :keyword:`!global` statement ================================ .. index:: - ! statement: global + ! pair: statement; global triple: global; name; binding single: , (comma); identifier list @@ -988,7 +988,7 @@ call. The same applies to the :func:`eval` and :func:`compile` functions. The :keyword:`!nonlocal` statement ================================== -.. index:: statement: nonlocal +.. index:: pair: statement; nonlocal single: , (comma); identifier list .. productionlist:: python-grammar diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 9a62ae748e0744..8fc44cab44455a 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -694,7 +694,7 @@ def patch_pairindextypes(app) -> None: pairindextypes.pop('operator', None) pairindextypes.pop('object', None) pairindextypes.pop('exception', None) - # pairindextypes.pop('statement', None) + pairindextypes.pop('statement', None) # pairindextypes.pop('builtin', None) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 52db51e84cd5fc..c9b3d982c31c9a 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -46,7 +46,7 @@ details see :ref:`tut-match`. ========================== .. index:: - statement: for + pair: statement; for The :keyword:`for` statement in Python differs a bit from what you may be used to in C or Pascal. Rather than always iterating over an arithmetic progression From ef0df5284f929719b2ef3955b1b569ade0a5193c Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Thu, 4 May 2023 14:26:23 +0200 Subject: [PATCH 37/65] gh-97556: Raise null bytes syntax error upon null in multiline string (GH-104136) --- Lib/test/test_cmd_line_script.py | 13 +++++++++++++ Parser/tokenizer.c | 9 ++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index d98e23855e0c19..8bf299382e9ca4 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -669,6 +669,19 @@ def test_syntaxerror_null_bytes(self): ], ) + def test_syntaxerror_null_bytes_in_multiline_string(self): + scripts = ["\n'''\nmultilinestring\0\n'''", "\nf'''\nmultilinestring\0\n'''"] # Both normal and f-strings + with os_helper.temp_dir() as script_dir: + for script in scripts: + script_name = _make_test_script(script_dir, 'script', script) + _, _, stderr = assert_python_failure(script_name) + self.assertEqual( + stderr.splitlines()[-2:], + [ b" multilinestring", + b'SyntaxError: source code cannot contain null bytes' + ] + ) + def test_consistent_sys_path_for_direct_execution(self): # This test case ensures that the following all give the same # sys.path configuration: diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index d2f9fee110ebf5..7c07d2011fda61 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -2301,8 +2301,12 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t /* Get rest of string */ while (end_quote_size != quote_size) { c = tok_nextc(tok); - if (tok->done == E_DECODE) + if (tok->done == E_ERROR) { + return MAKE_TOKEN(ERRORTOKEN); + } + if (tok->done == E_DECODE) { break; + } if (c == EOF || (quote_size == 1 && c == '\n')) { assert(tok->multi_line_start != NULL); // shift the tok_state's location into @@ -2554,6 +2558,9 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct while (end_quote_size != current_tok->f_string_quote_size) { int c = tok_nextc(tok); + if (tok->done == E_ERROR) { + return MAKE_TOKEN(ERRORTOKEN); + } if (c == EOF || (current_tok->f_string_quote_size == 1 && c == '\n')) { if (tok->decoding_erred) { return MAKE_TOKEN(ERRORTOKEN); From 4f524da4843fb71b3ba43d89cd0265e4ecb29798 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Thu, 4 May 2023 15:03:24 +0200 Subject: [PATCH 38/65] gh-103092: Port _ctypes.COMError to heap type (#104020) --- Modules/_ctypes/_ctypes.c | 97 +++++++++++++++++--------------------- Modules/_ctypes/callproc.c | 3 +- Modules/_ctypes/ctypes.h | 7 ++- 3 files changed, 47 insertions(+), 60 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index c7ed6bd2229c79..f6cda45eaeac27 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -139,10 +139,6 @@ static PyTypeObject Simple_Type; strong reference to _ctypes._unpickle() function */ static PyObject *_unpickle; -#ifdef MS_WIN32 -PyObject *ComError; // Borrowed reference to: &PyComError_Type -#endif - /****************************************************************/ @@ -5480,46 +5476,38 @@ comerror_init(PyObject *self, PyObject *args, PyObject *kwds) return 0; } -static PyTypeObject PyComError_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_ctypes.COMError", /* tp_name */ - sizeof(PyBaseExceptionObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - PyDoc_STR(comerror_doc), /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)comerror_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ +static int +comerror_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} + +static void +comerror_dealloc(PyObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); + tp->tp_free(self); + Py_DECREF(tp); +} + +static PyType_Slot comerror_slots[] = { + {Py_tp_doc, (void *)PyDoc_STR(comerror_doc)}, + {Py_tp_init, comerror_init}, + {Py_tp_traverse, comerror_traverse}, + {Py_tp_dealloc, comerror_dealloc}, + {0, NULL}, }; + +static PyType_Spec comerror_spec = { + .name = "_ctypes.COMError", + .basicsize = sizeof(PyBaseExceptionObject), + .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_IMMUTABLETYPE), + .slots = comerror_slots, +}; + #endif // MS_WIN32 static PyObject * @@ -5661,8 +5649,9 @@ _ctypes_add_types(PyObject *mod) } \ } while (0) -#define CREATE_TYPE(MOD, TP, SPEC) do { \ - PyObject *type = PyType_FromMetaclass(NULL, MOD, SPEC, NULL); \ +#define CREATE_TYPE(MOD, TP, SPEC, BASE) do { \ + PyObject *type = PyType_FromMetaclass(NULL, MOD, SPEC, \ + (PyObject *)BASE); \ if (type == NULL) { \ return -1; \ } \ @@ -5675,8 +5664,8 @@ _ctypes_add_types(PyObject *mod) ob_type is the metatype (the 'type'), defaults to PyType_Type, tp_base is the base type, defaults to 'object' aka PyBaseObject_Type. */ - CREATE_TYPE(mod, st->PyCArg_Type, &carg_spec); - CREATE_TYPE(mod, st->PyCThunk_Type, &cthunk_spec); + CREATE_TYPE(mod, st->PyCArg_Type, &carg_spec, NULL); + CREATE_TYPE(mod, st->PyCThunk_Type, &cthunk_spec, NULL); TYPE_READY(&PyCData_Type); /* StgDict is derived from PyDict_Type */ TYPE_READY_BASE(&PyCStgDict_Type, &PyDict_Type); @@ -5709,18 +5698,18 @@ _ctypes_add_types(PyObject *mod) * Simple classes */ - CREATE_TYPE(mod, st->PyCField_Type, &cfield_spec); + CREATE_TYPE(mod, st->PyCField_Type, &cfield_spec, NULL); /************************************************* * * Other stuff */ - CREATE_TYPE(mod, st->DictRemover_Type, &dictremover_spec); - CREATE_TYPE(mod, st->StructParam_Type, &structparam_spec); + CREATE_TYPE(mod, st->DictRemover_Type, &dictremover_spec, NULL); + CREATE_TYPE(mod, st->StructParam_Type, &structparam_spec, NULL); #ifdef MS_WIN32 - TYPE_READY_BASE(&PyComError_Type, (PyTypeObject*)PyExc_Exception); + CREATE_TYPE(mod, st->PyComError_Type, &comerror_spec, PyExc_Exception); #endif #undef TYPE_READY @@ -5750,7 +5739,8 @@ _ctypes_add_objects(PyObject *mod) MOD_ADD("_pointer_type_cache", Py_NewRef(_ctypes_ptrtype_cache)); #ifdef MS_WIN32 - MOD_ADD("COMError", Py_NewRef(ComError)); + ctypes_state *st = GLOBAL_STATE(); + MOD_ADD("COMError", Py_NewRef(st->PyComError_Type)); MOD_ADD("FUNCFLAG_HRESULT", PyLong_FromLong(FUNCFLAG_HRESULT)); MOD_ADD("FUNCFLAG_STDCALL", PyLong_FromLong(FUNCFLAG_STDCALL)); #endif @@ -5807,9 +5797,6 @@ _ctypes_mod_exec(PyObject *mod) if (_ctypes_add_types(mod) < 0) { return -1; } -#ifdef MS_WIN32 - ComError = (PyObject*)&PyComError_Type; -#endif if (_ctypes_add_objects(mod) < 0) { return -1; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index f10cf58216acf7..d2fe525dd4d396 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1115,7 +1115,8 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) descr, source, helpfile, helpcontext, progid); if (obj) { - PyErr_SetObject(ComError, obj); + ctypes_state *st = GLOBAL_STATE(); + PyErr_SetObject((PyObject *)st->PyComError_Type, obj); Py_DECREF(obj); } LocalFree(text); diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 252d9da7dbb56d..8891a0a741de7b 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -37,6 +37,9 @@ typedef struct { PyTypeObject *PyCArg_Type; PyTypeObject *PyCField_Type; PyTypeObject *PyCThunk_Type; +#ifdef MS_WIN32 + PyTypeObject *PyComError_Type; +#endif PyTypeObject *StructParam_Type; } ctypes_state; @@ -392,10 +395,6 @@ extern int _ctypes_simple_instance(PyObject *obj); extern PyObject *_ctypes_ptrtype_cache; PyObject *_ctypes_get_errobj(int **pspace); -#ifdef MS_WIN32 -extern PyObject *ComError; -#endif - #ifdef USING_MALLOC_CLOSURE_DOT_C void Py_ffi_closure_free(void *p); void *Py_ffi_closure_alloc(size_t size, void** codeloc); From e2ef5015d1b6cb56f1a7988583f2fb8c0e6d65fc Mon Sep 17 00:00:00 2001 From: Christopher Chavez Date: Thu, 4 May 2023 08:03:52 -0500 Subject: [PATCH 39/65] =?UTF-8?q?Fix=20typo=20"involvin"=20=E2=86=92=20"in?= =?UTF-8?q?volving"=20(#104166)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/test/test_idle.py | 2 +- Lib/test/test_tix.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index b94b18a541a701..90cff9002b75b2 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -3,7 +3,7 @@ from test.support import check_sanitizer if check_sanitizer(address=True, memory=True): - raise unittest.SkipTest("Tests involvin libX11 can SEGFAULT on ASAN/MSAN builds") + raise unittest.SkipTest("Tests involving libX11 can SEGFAULT on ASAN/MSAN builds") # Skip test_idle if _tkinter wasn't built, if tkinter is missing, # if tcl/tk is not the 8.5+ needed for ttk widgets, diff --git a/Lib/test/test_tix.py b/Lib/test/test_tix.py index 454baeb38a9342..d0d2a164ad2c67 100644 --- a/Lib/test/test_tix.py +++ b/Lib/test/test_tix.py @@ -5,7 +5,7 @@ from test.support import check_sanitizer if check_sanitizer(address=True, memory=True): - raise unittest.SkipTest("Tests involvin libX11 can SEGFAULT on ASAN/MSAN builds") + raise unittest.SkipTest("Tests involving libX11 can SEGFAULT on ASAN/MSAN builds") # Skip this test if the _tkinter module wasn't built. From eba64d2afb4c429e80d863dc0dd7808bdbef30d3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 4 May 2023 15:15:26 +0100 Subject: [PATCH 40/65] gh-104169: Ensure the tokenizer doesn't overwrite previous errors (#104170) --- Parser/tokenizer.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 7c07d2011fda61..52d0d9a534cb6a 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1277,6 +1277,12 @@ _syntaxerror_range(struct tok_state *tok, const char *format, int col_offset, int end_col_offset, va_list vargs) { + // In release builds, we don't want to overwrite a previous error, but in debug builds we + // want to fail if we are not doing it so we can fix it. + assert(tok->done != E_ERROR); + if (tok->done == E_ERROR) { + return ERRORTOKEN; + } PyObject *errmsg, *errtext, *args; errmsg = PyUnicode_FromFormatV(format, vargs); if (!errmsg) { From 45398ad51220b63b8df08fb5551c6b736205daed Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 4 May 2023 16:21:01 +0200 Subject: [PATCH 41/65] gh-103323: Remove PyRuntimeState_GetThreadState() (#104171) This function no longer makes sense, since its runtime parameter is no longer used. Use directly _PyThreadState_GET() and _PyInterpreterState_GET() instead. --- Include/internal/pycore_pystate.h | 10 ++-------- Python/ceval_gil.c | 5 ++--- Python/pylifecycle.c | 7 +++---- Python/pystate.c | 2 +- Python/sysmodule.c | 2 +- 5 files changed, 9 insertions(+), 17 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 180ea676bc22eb..daa40cf4bcd855 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -68,7 +68,7 @@ _Py_ThreadCanHandlePendingCalls(void) } -/* Variable and macro for in-line access to current thread +/* Variable and static inline functions for in-line access to current thread and interpreter state */ #if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) @@ -93,12 +93,6 @@ _PyThreadState_GET(void) #endif } -static inline PyThreadState* -_PyRuntimeState_GetThreadState(_PyRuntimeState *Py_UNUSED(runtime)) -{ - return _PyThreadState_GET(); -} - static inline void _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) @@ -118,7 +112,7 @@ _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) /* Get the current interpreter state. - The macro is unsafe: it does not check for error and it can return NULL. + The function is unsafe: it does not check for error and it can return NULL. The caller must hold the GIL. diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 29796be4b80e93..4d501c89661743 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -546,8 +546,7 @@ _PyEval_Fini(void) void PyEval_AcquireLock(void) { - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); _Py_EnsureTstateNotNULL(tstate); take_gil(tstate); @@ -557,7 +556,7 @@ void PyEval_ReleaseLock(void) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); /* This function must succeed when the current thread state is NULL. We therefore avoid PyThreadState_Get() which dumps a fatal error in debug mode. */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index b9add89b9c6c52..97957d3f17e6db 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1303,8 +1303,7 @@ _Py_InitializeMain(void) if (_PyStatus_EXCEPTION(status)) { return status; } - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); return pyinit_main(tstate); } @@ -1755,7 +1754,7 @@ Py_FinalizeEx(void) } /* Get current thread state and interpreter pointer */ - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); // XXX assert(_Py_IsMainInterpreter(tstate->interp)); // XXX assert(_Py_IsMainThread()); @@ -2800,7 +2799,7 @@ fatal_error(int fd, int header, const char *prefix, const char *msg, tss_tstate != tstate if the current Python thread does not hold the GIL. */ - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + PyThreadState *tstate = _PyThreadState_GET(); PyInterpreterState *interp = NULL; PyThreadState *tss_tstate = PyGILState_GetThisThreadState(); if (tstate != NULL) { diff --git a/Python/pystate.c b/Python/pystate.c index f103a059f0f369..f09d37657f9c27 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1809,7 +1809,7 @@ int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) { _PyRuntimeState *runtime = &_PyRuntime; - PyInterpreterState *interp = _PyRuntimeState_GetThreadState(runtime)->interp; + PyInterpreterState *interp = _PyInterpreterState_GET(); /* Although the GIL is held, a few C API functions can be called * without the GIL held, and in particular some that create and diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 781588b0df4ead..894a3e8a98fd8a 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -365,7 +365,7 @@ PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) _PyRuntimeState *runtime = &_PyRuntime; PyThreadState *tstate; if (runtime->initialized) { - tstate = _PyRuntimeState_GetThreadState(runtime); + tstate = _PyThreadState_GET(); } else { tstate = NULL; From c9ecd3ee75b472bb0a7538e0288c5cfea146da83 Mon Sep 17 00:00:00 2001 From: Kevin Krakauer Date: Thu, 4 May 2023 07:38:20 -0700 Subject: [PATCH 42/65] gh-102795: Fix use of poll in test_epoll's test_control_and_wait (#102796) This test can fail unnecessarily. In the test we wait for events on two file descriptors. This is done in a single call to select.epoll's poll() function. However, it is valid for the OS to return only one event via poll() and the next via a subsequent call to poll(). This rarely happens, but it can cause the test to fail despite properly functioning polling. Instead, we poll a second time when necessary. --- Lib/test/test_epoll.py | 15 +++++++++++---- ...2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst diff --git a/Lib/test/test_epoll.py b/Lib/test/test_epoll.py index b623852f9eb4ee..c94946a6ae6b7c 100644 --- a/Lib/test/test_epoll.py +++ b/Lib/test/test_epoll.py @@ -27,6 +27,7 @@ import socket import time import unittest +from test import support if not hasattr(select, "epoll"): raise unittest.SkipTest("test works only on Linux 2.6") @@ -186,10 +187,16 @@ def test_control_and_wait(self): client.sendall(b"Hello!") server.sendall(b"world!!!") - now = time.monotonic() - events = ep.poll(1.0, 4) - then = time.monotonic() - self.assertFalse(then - now > 0.01) + # we might receive events one at a time, necessitating multiple calls to + # poll + events = [] + for _ in support.busy_retry(support.SHORT_TIMEOUT): + now = time.monotonic() + events += ep.poll(1.0, 4) + then = time.monotonic() + self.assertFalse(then - now > 0.01) + if len(events) >= 2: + break expected = [(client.fileno(), select.EPOLLIN | select.EPOLLOUT), (server.fileno(), select.EPOLLIN | select.EPOLLOUT)] diff --git a/Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst b/Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst new file mode 100644 index 00000000000000..fe2afff91ece7a --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-03-17-22-00-47.gh-issue-102795.z21EoC.rst @@ -0,0 +1 @@ +fix use of poll in test_epoll's test_control_and_wait From fdcb49c36b2ed8347d8d9f2dcd7052cc90207beb Mon Sep 17 00:00:00 2001 From: Itamar Ostricher Date: Thu, 4 May 2023 07:50:26 -0700 Subject: [PATCH 43/65] gh-104066: Improve performance of hasattr for module objects (#104063) --- Include/internal/pycore_moduleobject.h | 3 + ...-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst | 2 + Objects/moduleobject.c | 91 +++++++++++++------ Objects/object.c | 11 +++ 4 files changed, 78 insertions(+), 29 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 76361b8dff113a..15a1bcb6ae5163 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -36,6 +36,9 @@ static inline PyObject* _PyModule_GetDict(PyObject *mod) { return dict; } +PyObject* _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress); +PyObject* _Py_module_getattro(PyModuleObject *m, PyObject *name); + #ifdef __cplusplus } #endif diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst new file mode 100644 index 00000000000000..97e0c01689cb6f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-01-14-48-29.gh-issue-104066.pzoUZQ.rst @@ -0,0 +1,2 @@ +Improve the performance of :func:`hasattr` for module objects with a missing +attribute. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index a0be19a3ca8ac8..ffcd90ed122e77 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -702,7 +702,11 @@ int _PyModuleSpec_IsInitializing(PyObject *spec) { if (spec != NULL) { - PyObject *value = PyObject_GetAttr(spec, &_Py_ID(_initializing)); + PyObject *value; + int ok = _PyObject_LookupAttr(spec, &_Py_ID(_initializing), &value); + if (ok == 0) { + return 0; + } if (value != NULL) { int initializing = PyObject_IsTrue(value); Py_DECREF(value); @@ -738,19 +742,37 @@ _PyModuleSpec_IsUninitializedSubmodule(PyObject *spec, PyObject *name) return is_uninitialized; } -static PyObject* -module_getattro(PyModuleObject *m, PyObject *name) +PyObject* +_Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) { + // When suppress=1, this function suppresses AttributeError. PyObject *attr, *mod_name, *getattr; - attr = PyObject_GenericGetAttr((PyObject *)m, name); - if (attr || !PyErr_ExceptionMatches(PyExc_AttributeError)) { + attr = _PyObject_GenericGetAttrWithDict((PyObject *)m, name, NULL, suppress); + if (attr) { return attr; } - PyErr_Clear(); + if (suppress == 1) { + if (PyErr_Occurred()) { + // pass up non-AttributeError exception + return NULL; + } + } + else { + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + // pass up non-AttributeError exception + return NULL; + } + PyErr_Clear(); + } assert(m->md_dict != NULL); getattr = PyDict_GetItemWithError(m->md_dict, &_Py_ID(__getattr__)); if (getattr) { - return PyObject_CallOneArg(getattr, name); + PyObject *result = PyObject_CallOneArg(getattr, name); + if (result == NULL && suppress == 1 && PyErr_ExceptionMatches(PyExc_AttributeError)) { + // suppress AttributeError + PyErr_Clear(); + } + return result; } if (PyErr_Occurred()) { return NULL; @@ -763,37 +785,48 @@ module_getattro(PyModuleObject *m, PyObject *name) Py_DECREF(mod_name); return NULL; } - Py_XINCREF(spec); - if (_PyModuleSpec_IsInitializing(spec)) { - PyErr_Format(PyExc_AttributeError, - "partially initialized " - "module '%U' has no attribute '%U' " - "(most likely due to a circular import)", - mod_name, name); - } - else if (_PyModuleSpec_IsUninitializedSubmodule(spec, name)) { - PyErr_Format(PyExc_AttributeError, - "cannot access submodule '%U' of module '%U' " - "(most likely due to a circular import)", - name, mod_name); - } - else { - PyErr_Format(PyExc_AttributeError, - "module '%U' has no attribute '%U'", - mod_name, name); + if (suppress != 1) { + Py_XINCREF(spec); + if (_PyModuleSpec_IsInitializing(spec)) { + PyErr_Format(PyExc_AttributeError, + "partially initialized " + "module '%U' has no attribute '%U' " + "(most likely due to a circular import)", + mod_name, name); + } + else if (_PyModuleSpec_IsUninitializedSubmodule(spec, name)) { + PyErr_Format(PyExc_AttributeError, + "cannot access submodule '%U' of module '%U' " + "(most likely due to a circular import)", + name, mod_name); + } + else { + PyErr_Format(PyExc_AttributeError, + "module '%U' has no attribute '%U'", + mod_name, name); + } + Py_XDECREF(spec); } - Py_XDECREF(spec); Py_DECREF(mod_name); return NULL; } else if (PyErr_Occurred()) { return NULL; } - PyErr_Format(PyExc_AttributeError, - "module has no attribute '%U'", name); + if (suppress != 1) { + PyErr_Format(PyExc_AttributeError, + "module has no attribute '%U'", name); + } return NULL; } + +PyObject* +_Py_module_getattro(PyModuleObject *m, PyObject *name) +{ + return _Py_module_getattro_impl(m, name, 0); +} + static int module_traverse(PyModuleObject *m, visitproc visit, void *arg) { @@ -951,7 +984,7 @@ PyTypeObject PyModule_Type = { 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ - (getattrofunc)module_getattro, /* tp_getattro */ + (getattrofunc)_Py_module_getattro, /* tp_getattro */ PyObject_GenericSetAttr, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | diff --git a/Objects/object.c b/Objects/object.c index c6ef5928164824..41c52e21045a21 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1085,6 +1085,17 @@ _PyObject_LookupAttr(PyObject *v, PyObject *name, PyObject **result) return 0; } } + else if (tp->tp_getattro == (getattrofunc)_Py_module_getattro) { + // optimization: suppress attribute error from module getattro method + *result = _Py_module_getattro_impl((PyModuleObject*)v, name, 1); + if (*result != NULL) { + return 1; + } + if (PyErr_Occurred()) { + return -1; + } + return 0; + } else if (tp->tp_getattro != NULL) { *result = (*tp->tp_getattro)(v, name); } From b17d32c1142d16a5fea0c95bce185bf9be696491 Mon Sep 17 00:00:00 2001 From: Gleb Smirnoff Date: Thu, 4 May 2023 07:57:05 -0700 Subject: [PATCH 44/65] gh-96534: socketmodule: support FreeBSD divert(4) socket (#96536) --- Doc/library/socket.rst | 11 +++++++++++ .../2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst | 1 + Modules/socketmodule.c | 13 +++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 4ee0897db94063..13a82cf82d5908 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -509,6 +509,17 @@ Constants .. versionadded:: 3.9 +.. data:: AF_DIVERT + PF_DIVERT + + These two constants, documented in the FreeBSD divert(4) manual page, are + also defined in the socket module. + + .. availability:: FreeBSD >= 14.0. + + .. versionadded:: 3.12 + + .. data:: AF_PACKET PF_PACKET PACKET_* diff --git a/Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst b/Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst new file mode 100644 index 00000000000000..0497d9eb69163e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-09-03-09-24-02.gh-issue-96534.EU4Oxv.rst @@ -0,0 +1 @@ +Support divert(4) added in FreeBSD 14. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index f11d4b1a6e0591..e5478382e11f89 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -1903,6 +1903,11 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, /* RDS sockets use sockaddr_in: fall-through */ #endif /* AF_RDS */ +#ifdef AF_DIVERT + case AF_DIVERT: + /* FreeBSD divert(4) sockets use sockaddr_in: fall-through */ +#endif /* AF_DIVERT */ + case AF_INET: { struct maybe_idna host = {NULL, NULL}; @@ -7683,6 +7688,14 @@ socket_exec(PyObject *m) ADD_INT_MACRO(m, AF_SYSTEM); #endif +/* FreeBSD divert(4) */ +#ifdef PF_DIVERT + PyModule_AddIntMacro(m, PF_DIVERT); +#endif +#ifdef AF_DIVERT + PyModule_AddIntMacro(m, AF_DIVERT); +#endif + #ifdef AF_PACKET ADD_INT_MACRO(m, AF_PACKET); #endif From 04f673327530f47f002e784459037231de478412 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 07:59:46 -0700 Subject: [PATCH 45/65] gh-102500: Implement PEP 688 (#102521) Co-authored-by: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> --- .../pycore_global_objects_fini_generated.h | 3 + Include/internal/pycore_global_strings.h | 3 + Include/internal/pycore_memoryobject.h | 17 ++ .../internal/pycore_runtime_init_generated.h | 3 + Include/internal/pycore_typeobject.h | 2 + .../internal/pycore_unicodeobject_generated.h | 9 + Include/pybuffer.h | 2 +- Lib/_collections_abc.py | 17 +- Lib/inspect.py | 23 ++ Lib/test/test_buffer.py | 142 +++++++++++++ Lib/test/test_collections.py | 11 +- Lib/test/test_doctest.py | 2 +- ...-03-07-17-37-00.gh-issue-102500.RUSQhz.rst | 3 + Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/buffer.c | 102 +++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 4 +- Objects/clinic/memoryobject.c.h | 62 +++++- Objects/memoryobject.c | 40 +++- Objects/object.c | 2 + Objects/typeobject.c | 197 +++++++++++++++++- PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 + Tools/build/generate_global_objects.py | 2 + Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + Tools/c-analyzer/cpython/ignored.tsv | 1 + 26 files changed, 640 insertions(+), 15 deletions(-) create mode 100644 Include/internal/pycore_memoryobject.h create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst create mode 100644 Modules/_testcapi/buffer.c diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index f0740b68dd1114..9377fd8526e3a2 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -593,6 +593,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__await__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bases__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bool__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__buffer__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__build_class__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__builtins__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__bytes__)); @@ -692,6 +693,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__rdivmod__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__reduce__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__reduce_ex__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__release_buffer__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__repr__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__reversed__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__rfloordiv__)); @@ -1122,6 +1124,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reducer_override)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(registry)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rel_tol)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(release)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reload)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(replace)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 234d5e2a09892c..ed9b2bb44ddffc 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -81,6 +81,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__await__) STRUCT_FOR_ID(__bases__) STRUCT_FOR_ID(__bool__) + STRUCT_FOR_ID(__buffer__) STRUCT_FOR_ID(__build_class__) STRUCT_FOR_ID(__builtins__) STRUCT_FOR_ID(__bytes__) @@ -180,6 +181,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__rdivmod__) STRUCT_FOR_ID(__reduce__) STRUCT_FOR_ID(__reduce_ex__) + STRUCT_FOR_ID(__release_buffer__) STRUCT_FOR_ID(__repr__) STRUCT_FOR_ID(__reversed__) STRUCT_FOR_ID(__rfloordiv__) @@ -610,6 +612,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(reducer_override) STRUCT_FOR_ID(registry) STRUCT_FOR_ID(rel_tol) + STRUCT_FOR_ID(release) STRUCT_FOR_ID(reload) STRUCT_FOR_ID(repl) STRUCT_FOR_ID(replace) diff --git a/Include/internal/pycore_memoryobject.h b/Include/internal/pycore_memoryobject.h new file mode 100644 index 00000000000000..acc12c9275172c --- /dev/null +++ b/Include/internal/pycore_memoryobject.h @@ -0,0 +1,17 @@ +#ifndef Py_INTERNAL_MEMORYOBJECT_H +#define Py_INTERNAL_MEMORYOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +PyObject * +PyMemoryView_FromObjectAndFlags(PyObject *v, int flags); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_MEMORYOBJECT_H */ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 16f2147aa8e93b..6ade8fb6eade03 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -587,6 +587,7 @@ extern "C" { INIT_ID(__await__), \ INIT_ID(__bases__), \ INIT_ID(__bool__), \ + INIT_ID(__buffer__), \ INIT_ID(__build_class__), \ INIT_ID(__builtins__), \ INIT_ID(__bytes__), \ @@ -686,6 +687,7 @@ extern "C" { INIT_ID(__rdivmod__), \ INIT_ID(__reduce__), \ INIT_ID(__reduce_ex__), \ + INIT_ID(__release_buffer__), \ INIT_ID(__repr__), \ INIT_ID(__reversed__), \ INIT_ID(__rfloordiv__), \ @@ -1116,6 +1118,7 @@ extern "C" { INIT_ID(reducer_override), \ INIT_ID(registry), \ INIT_ID(rel_tol), \ + INIT_ID(release), \ INIT_ID(reload), \ INIT_ID(repl), \ INIT_ID(replace), \ diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 6a5ab7e63f85c3..f42f8f62de2c1d 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -138,6 +138,8 @@ _Py_type_getattro(PyTypeObject *type, PyObject *name); PyObject *_Py_slot_tp_getattro(PyObject *self, PyObject *name); PyObject *_Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); +PyAPI_DATA(PyTypeObject) _PyBufferWrapper_Type; + PyObject * _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); PyObject * diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index cd41b731537f0e..0b33ea187e60ff 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -96,6 +96,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__bool__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__buffer__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__build_class__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -393,6 +396,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__reduce_ex__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__release_buffer__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__repr__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); @@ -1683,6 +1689,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(rel_tol); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(release); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(reload); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Include/pybuffer.h b/Include/pybuffer.h index bbac60972f5127..ca1c6058d9052c 100644 --- a/Include/pybuffer.h +++ b/Include/pybuffer.h @@ -104,7 +104,7 @@ PyAPI_FUNC(void) PyBuffer_Release(Py_buffer *view); /* Maximum number of dimensions */ #define PyBUF_MAX_NDIM 64 -/* Flags for getting buffers */ +/* Flags for getting buffers. Keep these in sync with inspect.BufferFlags. */ #define PyBUF_SIMPLE 0 #define PyBUF_WRITABLE 0x0001 diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 9d7724c33474cc..2117190cf8b64a 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -49,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "ByteString", + "ByteString", "Buffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -439,6 +439,21 @@ def __subclasshook__(cls, C): return NotImplemented +class Buffer(metaclass=ABCMeta): + + __slots__ = () + + @abstractmethod + def __buffer__(self, flags: int, /) -> memoryview: + raise NotImplementedError + + @classmethod + def __subclasshook__(cls, C): + if cls is Buffer: + return _check_methods(C, "__buffer__") + return NotImplemented + + class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. diff --git a/Lib/inspect.py b/Lib/inspect.py index 92c2675cfd7d32..95da7fb71a3997 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -43,6 +43,7 @@ "Attribute", "BlockFinder", "BoundArguments", + "BufferFlags", "CORO_CLOSED", "CORO_CREATED", "CORO_RUNNING", @@ -3312,6 +3313,28 @@ def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=F globals=globals, locals=locals, eval_str=eval_str) +class BufferFlags(enum.IntFlag): + SIMPLE = 0x0 + WRITABLE = 0x1 + FORMAT = 0x4 + ND = 0x8 + STRIDES = 0x10 | ND + C_CONTIGUOUS = 0x20 | STRIDES + F_CONTIGUOUS = 0x40 | STRIDES + ANY_CONTIGUOUS = 0x80 | STRIDES + INDIRECT = 0x100 | STRIDES + CONTIG = ND | WRITABLE + CONTIG_RO = ND + STRIDED = STRIDES | WRITABLE + STRIDED_RO = STRIDES + RECORDS = STRIDES | WRITABLE | FORMAT + RECORDS_RO = STRIDES | FORMAT + FULL = INDIRECT | WRITABLE | FORMAT + FULL_RO = INDIRECT | FORMAT + READ = 0x100 + WRITE = 0x200 + + def _main(): """ Logic for inspecting an object given at command line """ import argparse diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 098d2d999643cb..b6e82ad4db266a 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -17,6 +17,7 @@ import unittest from test import support from test.support import os_helper +import inspect from itertools import permutations, product from random import randrange, sample, choice import warnings @@ -4438,5 +4439,146 @@ def test_pybuffer_size_from_format(self): struct.calcsize(format)) +class TestPythonBufferProtocol(unittest.TestCase): + def test_basic(self): + class MyBuffer: + def __buffer__(self, flags): + return memoryview(b"hello") + + mv = memoryview(MyBuffer()) + self.assertEqual(mv.tobytes(), b"hello") + self.assertEqual(bytes(MyBuffer()), b"hello") + + def test_bad_buffer_method(self): + class MustReturnMV: + def __buffer__(self, flags): + return 42 + + self.assertRaises(TypeError, memoryview, MustReturnMV()) + + class NoBytesEither: + def __buffer__(self, flags): + return b"hello" + + self.assertRaises(TypeError, memoryview, NoBytesEither()) + + class WrongArity: + def __buffer__(self): + return memoryview(b"hello") + + self.assertRaises(TypeError, memoryview, WrongArity()) + + def test_release_buffer(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + return memoryview(self.ba) + + def __release_buffer__(self, buffer): + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + + def test_same_buffer_returned(self): + class WhatToRelease: + def __init__(self): + self.held = False + self.ba = bytearray(b"hello") + self.created_mv = None + + def __buffer__(self, flags): + if self.held: + raise TypeError("already held") + self.held = True + self.created_mv = memoryview(self.ba) + return self.created_mv + + def __release_buffer__(self, buffer): + assert buffer is self.created_mv + self.held = False + + wr = WhatToRelease() + self.assertFalse(wr.held) + with memoryview(wr) as mv: + self.assertTrue(wr.held) + self.assertEqual(mv.tobytes(), b"hello") + self.assertFalse(wr.held) + + def test_buffer_flags(self): + class PossiblyMutable: + def __init__(self, data, mutable) -> None: + self._data = bytearray(data) + self._mutable = mutable + + def __buffer__(self, flags): + if flags & inspect.BufferFlags.WRITABLE: + if not self._mutable: + raise RuntimeError("not mutable") + return memoryview(self._data) + else: + return memoryview(bytes(self._data)) + + mutable = PossiblyMutable(b"hello", True) + immutable = PossiblyMutable(b"hello", False) + with memoryview._from_flags(mutable, inspect.BufferFlags.WRITABLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"xello") + with memoryview._from_flags(mutable, inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"xello") + with self.assertRaises(TypeError): + mv[0] = ord(b'h') + self.assertEqual(mv.tobytes(), b"xello") + with memoryview._from_flags(immutable, inspect.BufferFlags.SIMPLE) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + + with self.assertRaises(RuntimeError): + memoryview._from_flags(immutable, inspect.BufferFlags.WRITABLE) + with memoryview(immutable) as mv: + self.assertEqual(mv.tobytes(), b"hello") + with self.assertRaises(TypeError): + mv[0] = ord(b'x') + self.assertEqual(mv.tobytes(), b"hello") + + def test_call_builtins(self): + ba = bytearray(b"hello") + mv = ba.__buffer__(0) + self.assertEqual(mv.tobytes(), b"hello") + ba.__release_buffer__(mv) + with self.assertRaises(OverflowError): + ba.__buffer__(sys.maxsize + 1) + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_c_buffer(self): + buf = _testcapi.testBuf() + self.assertEqual(buf.references, 0) + mv = buf.__buffer__(0) + self.assertIsInstance(mv, memoryview) + self.assertEqual(mv.tobytes(), b"test") + self.assertEqual(buf.references, 1) + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + with self.assertRaises(ValueError): + mv.tobytes() + # Calling it again doesn't cause issues + with self.assertRaises(ValueError): + buf.__release_buffer__(mv) + self.assertEqual(buf.references, 0) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index fb568a48396498..8fc28a6bf98e8f 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -25,7 +25,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import ByteString +from collections.abc import ByteString, Buffer class TestUserObjects(unittest.TestCase): @@ -1949,6 +1949,15 @@ def test_ByteString(self): self.assertFalse(issubclass(memoryview, ByteString)) self.validate_abstract_methods(ByteString, '__getitem__', '__len__') + def test_Buffer(self): + for sample in [bytes, bytearray, memoryview]: + self.assertIsInstance(sample(b"x"), Buffer) + self.assertTrue(issubclass(sample, Buffer)) + for sample in [str, list, tuple]: + self.assertNotIsInstance(sample(), Buffer) + self.assertFalse(issubclass(sample, Buffer)) + self.validate_abstract_methods(Buffer, '__buffer__') + def test_MutableSequence(self): for sample in [tuple, str, bytes]: self.assertNotIsInstance(sample(), MutableSequence) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 3491d4cdb1c18b..542fcdb5cf6f66 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -707,7 +707,7 @@ def non_Python_modules(): r""" >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> 830 < len(tests) < 850 # approximate number of objects with docstrings + >>> 830 < len(tests) < 860 # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] >>> len(real_tests) # objects that actually have doctests diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst new file mode 100644 index 00000000000000..e03113ba05cd7d --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-03-07-17-37-00.gh-issue-102500.RUSQhz.rst @@ -0,0 +1,3 @@ +Make the buffer protocol accessible in Python code using the new +``__buffer__`` and ``__release_buffer__`` magic methods. See :pep:`688` for +details. Patch by Jelle Zijlstra. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 6b483341953711..a7803cf7c00e68 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -169,7 +169,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/getargs.c _testcapi/pytime.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyos.c _testcapi/immortal.c _testcapi/heaptype_relative.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c # Some testing modules MUST be built as shared libraries. diff --git a/Modules/_testcapi/buffer.c b/Modules/_testcapi/buffer.c new file mode 100644 index 00000000000000..aff9a477eff57e --- /dev/null +++ b/Modules/_testcapi/buffer.c @@ -0,0 +1,102 @@ +/* Test PEP 688 - Buffers */ + +#include "parts.h" + +#include "structmember.h" // PyMemberDef +#include // offsetof + +typedef struct { + PyObject_HEAD + PyObject *obj; + Py_ssize_t references; +} testBufObject; + +static PyObject * +testbuf_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *obj = PyBytes_FromString("test"); + if (obj == NULL) { + return NULL; + } + testBufObject *self = (testBufObject *)type->tp_alloc(type, 0); + if (self == NULL) { + Py_DECREF(obj); + return NULL; + } + self->obj = obj; + self->references = 0; + return (PyObject *)self; +} + +static int +testbuf_traverse(testBufObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->obj); + return 0; +} + +static int +testbuf_clear(testBufObject *self) +{ + Py_CLEAR(self->obj); + return 0; +} + +static void +testbuf_dealloc(testBufObject *self) +{ + PyObject_GC_UnTrack(self); + Py_XDECREF(self->obj); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static int +testbuf_getbuf(testBufObject *self, Py_buffer *view, int flags) +{ + int buf = PyObject_GetBuffer(self->obj, view, flags); + Py_SETREF(view->obj, Py_NewRef(self)); + self->references++; + return buf; +} + +static void +testbuf_releasebuf(testBufObject *self, Py_buffer *view) +{ + self->references--; + assert(self->references >= 0); +} + +static PyBufferProcs testbuf_as_buffer = { + .bf_getbuffer = (getbufferproc) testbuf_getbuf, + .bf_releasebuffer = (releasebufferproc) testbuf_releasebuf, +}; + +static struct PyMemberDef testbuf_members[] = { + {"references", T_PYSSIZET, offsetof(testBufObject, references), READONLY}, + {NULL}, +}; + +static PyTypeObject testBufType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "testBufType", + .tp_basicsize = sizeof(testBufObject), + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_new = testbuf_new, + .tp_dealloc = (destructor) testbuf_dealloc, + .tp_traverse = (traverseproc) testbuf_traverse, + .tp_clear = (inquiry) testbuf_clear, + .tp_as_buffer = &testbuf_as_buffer, + .tp_members = testbuf_members +}; + +int +_PyTestCapi_Init_Buffer(PyObject *m) { + if (PyType_Ready(&testBufType) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "testBuf", (PyObject *)&testBufType)) { + return -1; + } + + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index d75412d5116063..663d4f2255de88 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -38,6 +38,7 @@ int _PyTestCapi_Init_Float(PyObject *module); int _PyTestCapi_Init_Structmember(PyObject *module); int _PyTestCapi_Init_Exceptions(PyObject *module); int _PyTestCapi_Init_Code(PyObject *module); +int _PyTestCapi_Init_Buffer(PyObject *module); int _PyTestCapi_Init_PyOS(PyObject *module); int _PyTestCapi_Init_Immortal(PyObject *module); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 1ecc442058088a..38f4758e6575d8 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3959,7 +3959,6 @@ static PyTypeObject MyList_Type = { MyList_new, /* tp_new */ }; - /* Test PEP 560 */ typedef struct { @@ -4310,6 +4309,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Code(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Buffer(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_PyOS(m) < 0) { return NULL; } diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index ff7b50bb114b05..25a22341185903 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -62,6 +62,66 @@ memoryview(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } +PyDoc_STRVAR(memoryview__from_flags__doc__, +"_from_flags($type, /, object, flags)\n" +"--\n" +"\n" +"Create a new memoryview object which references the given object."); + +#define MEMORYVIEW__FROM_FLAGS_METHODDEF \ + {"_from_flags", _PyCFunction_CAST(memoryview__from_flags), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, memoryview__from_flags__doc__}, + +static PyObject * +memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int flags); + +static PyObject * +memoryview__from_flags(PyTypeObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(object), &_Py_ID(flags), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"object", "flags", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_from_flags", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject *object; + int flags; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); + if (!args) { + goto exit; + } + object = args[0]; + flags = _PyLong_AsInt(args[1]); + if (flags == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = memoryview__from_flags_impl(type, object, flags); + +exit: + return return_value; +} + PyDoc_STRVAR(memoryview_release__doc__, "release($self, /)\n" "--\n" @@ -356,4 +416,4 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=a832f2fc44e4794c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=01613814112cedd7 input=a9049054013a1b77]*/ diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 34cc797b404cda..f008a8cc3e0474 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -85,7 +85,7 @@ mbuf_alloc(void) } static PyObject * -_PyManagedBuffer_FromObject(PyObject *base) +_PyManagedBuffer_FromObject(PyObject *base, int flags) { _PyManagedBufferObject *mbuf; @@ -93,7 +93,7 @@ _PyManagedBuffer_FromObject(PyObject *base) if (mbuf == NULL) return NULL; - if (PyObject_GetBuffer(base, &mbuf->master, PyBUF_FULL_RO) < 0) { + if (PyObject_GetBuffer(base, &mbuf->master, flags) < 0) { mbuf->master.obj = NULL; Py_DECREF(mbuf); return NULL; @@ -777,11 +777,12 @@ PyMemoryView_FromBuffer(const Py_buffer *info) return mv; } -/* Create a memoryview from an object that implements the buffer protocol. +/* Create a memoryview from an object that implements the buffer protocol, + using the given flags. If the object is a memoryview, the new memoryview must be registered with the same managed buffer. Otherwise, a new managed buffer is created. */ PyObject * -PyMemoryView_FromObject(PyObject *v) +PyMemoryView_FromObjectAndFlags(PyObject *v, int flags) { _PyManagedBufferObject *mbuf; @@ -792,7 +793,7 @@ PyMemoryView_FromObject(PyObject *v) } else if (PyObject_CheckBuffer(v)) { PyObject *ret; - mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(v); + mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(v, flags); if (mbuf == NULL) return NULL; ret = mbuf_add_view(mbuf, NULL); @@ -805,6 +806,14 @@ PyMemoryView_FromObject(PyObject *v) Py_TYPE(v)->tp_name); return NULL; } +/* Create a memoryview from an object that implements the buffer protocol. + If the object is a memoryview, the new memoryview must be registered + with the same managed buffer. Otherwise, a new managed buffer is created. */ +PyObject * +PyMemoryView_FromObject(PyObject *v) +{ + return PyMemoryView_FromObjectAndFlags(v, PyBUF_FULL_RO); +} /* Copy the format string from a base object that might vanish. */ static int @@ -851,7 +860,7 @@ memory_from_contiguous_copy(const Py_buffer *src, char order) if (bytes == NULL) return NULL; - mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(bytes); + mbuf = (_PyManagedBufferObject *)_PyManagedBuffer_FromObject(bytes, PyBUF_FULL_RO); Py_DECREF(bytes); if (mbuf == NULL) return NULL; @@ -968,6 +977,24 @@ memoryview_impl(PyTypeObject *type, PyObject *object) } +/*[clinic input] +@classmethod +memoryview._from_flags + + object: object + flags: int + +Create a new memoryview object which references the given object. +[clinic start generated code]*/ + +static PyObject * +memoryview__from_flags_impl(PyTypeObject *type, PyObject *object, int flags) +/*[clinic end generated code: output=bf71f9906c266ee2 input=f5f82fd0e744356b]*/ +{ + return PyMemoryView_FromObjectAndFlags(object, flags); +} + + /****************************************************************************/ /* Previously in abstract.c */ /****************************************************************************/ @@ -3184,6 +3211,7 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_TOLIST_METHODDEF MEMORYVIEW_CAST_METHODDEF MEMORYVIEW_TOREADONLY_METHODDEF + MEMORYVIEW__FROM_FLAGS_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, {"__exit__", memory_exit, METH_VARARGS, NULL}, {NULL, NULL} diff --git a/Objects/object.c b/Objects/object.c index 41c52e21045a21..a7c79c673d5fd3 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -14,6 +14,7 @@ #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_symtable.h" // PySTEntry_Type +#include "pycore_typeobject.h" // _PyBufferWrapper_Type #include "pycore_unionobject.h" // _PyUnion_Type #include "pycore_interpreteridobject.h" // _PyInterpreterID_Type @@ -2084,6 +2085,7 @@ static PyTypeObject* static_types[] = { &_PyAsyncGenASend_Type, &_PyAsyncGenAThrow_Type, &_PyAsyncGenWrappedValue_Type, + &_PyBufferWrapper_Type, &_PyContextTokenMissing_Type, &_PyCoroWrapper_Type, &_Py_GenericAliasIterType, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 171c76a59a5541..456b10ee01d6bc 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6,6 +6,7 @@ #include "pycore_symtable.h" // _Py_Mangle() #include "pycore_dict.h" // _PyDict_KeysSize() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_memoryobject.h" // PyMemoryView_FromObjectAndFlags() #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_long.h" // _PyLong_IsNegative() @@ -8059,6 +8060,58 @@ wrap_descr_delete(PyObject *self, PyObject *args, void *wrapped) Py_RETURN_NONE; } +static PyObject * +wrap_buffer(PyObject *self, PyObject *args, void *wrapped) +{ + PyObject *arg = NULL; + + if (!PyArg_UnpackTuple(args, "", 1, 1, &arg)) { + return NULL; + } + Py_ssize_t flags = PyNumber_AsSsize_t(arg, PyExc_OverflowError); + if (flags == -1 && PyErr_Occurred()) { + return NULL; + } + if (flags > INT_MAX) { + PyErr_SetString(PyExc_OverflowError, + "buffer flags too large"); + return NULL; + } + + return PyMemoryView_FromObjectAndFlags(self, Py_SAFE_DOWNCAST(flags, Py_ssize_t, int)); +} + +static PyObject * +wrap_releasebuffer(PyObject *self, PyObject *args, void *wrapped) +{ + PyObject *arg = NULL; + if (!PyArg_UnpackTuple(args, "", 1, 1, &arg)) { + return NULL; + } + if (!PyMemoryView_Check(arg)) { + PyErr_SetString(PyExc_TypeError, + "expected a memoryview object"); + return NULL; + } + PyMemoryViewObject *mview = (PyMemoryViewObject *)arg; + if (mview->view.obj != self) { + PyErr_SetString(PyExc_ValueError, + "memoryview's buffer is not this object"); + return NULL; + } + if (mview->flags & _Py_MEMORYVIEW_RELEASED) { + PyErr_SetString(PyExc_ValueError, + "memoryview's buffer has already been released"); + return NULL; + } + PyObject *res = PyObject_CallMethodNoArgs((PyObject *)mview, &_Py_ID(release)); + if (res == NULL) { + return NULL; + } + Py_DECREF(res); + Py_RETURN_NONE; +} + static PyObject * wrap_init(PyObject *self, PyObject *args, void *wrapped, PyObject *kwds) { @@ -8895,6 +8948,132 @@ slot_tp_finalize(PyObject *self) PyErr_SetRaisedException(exc); } +typedef struct _PyBufferWrapper { + PyObject_HEAD + PyObject *mv; + PyObject *obj; +} PyBufferWrapper; + +static int +bufferwrapper_traverse(PyBufferWrapper *self, visitproc visit, void *arg) +{ + Py_VISIT(self->mv); + Py_VISIT(self->obj); + return 0; +} + +static void +bufferwrapper_dealloc(PyObject *self) +{ + PyBufferWrapper *bw = (PyBufferWrapper *)self; + + _PyObject_GC_UNTRACK(self); + Py_XDECREF(bw->mv); + Py_XDECREF(bw->obj); + Py_TYPE(self)->tp_free(self); +} + +static void +bufferwrapper_releasebuf(PyObject *self, Py_buffer *view) +{ + PyBufferWrapper *bw = (PyBufferWrapper *)self; + + assert(PyMemoryView_Check(bw->mv)); + Py_TYPE(bw->mv)->tp_as_buffer->bf_releasebuffer(bw->mv, view); + if (Py_TYPE(bw->obj)->tp_as_buffer != NULL + && Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer != NULL) { + Py_TYPE(bw->obj)->tp_as_buffer->bf_releasebuffer(bw->obj, view); + } +} + +static PyBufferProcs bufferwrapper_as_buffer = { + .bf_releasebuffer = bufferwrapper_releasebuf, +}; + + +PyTypeObject _PyBufferWrapper_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "_buffer_wrapper", + .tp_basicsize = sizeof(PyBufferWrapper), + .tp_alloc = PyType_GenericAlloc, + .tp_free = PyObject_GC_Del, + .tp_traverse = (traverseproc)bufferwrapper_traverse, + .tp_dealloc = bufferwrapper_dealloc, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_as_buffer = &bufferwrapper_as_buffer, +}; + +static int +slot_bf_getbuffer(PyObject *self, Py_buffer *buffer, int flags) +{ + PyObject *flags_obj = PyLong_FromLong(flags); + if (flags_obj == NULL) { + return -1; + } + PyBufferWrapper *wrapper = NULL; + PyObject *stack[2] = {self, flags_obj}; + PyObject *ret = vectorcall_method(&_Py_ID(__buffer__), stack, 2); + if (ret == NULL) { + goto fail; + } + if (!PyMemoryView_Check(ret)) { + PyErr_Format(PyExc_TypeError, + "__buffer__ returned non-memoryview object"); + goto fail; + } + + if (PyObject_GetBuffer(ret, buffer, flags) < 0) { + goto fail; + } + assert(buffer->obj == ret); + + wrapper = PyObject_GC_New(PyBufferWrapper, &_PyBufferWrapper_Type); + if (wrapper == NULL) { + goto fail; + } + wrapper->mv = ret; + wrapper->obj = Py_NewRef(self); + _PyObject_GC_TRACK(wrapper); + + buffer->obj = (PyObject *)wrapper; + Py_DECREF(ret); + Py_DECREF(flags_obj); + return 0; + +fail: + Py_XDECREF(wrapper); + Py_XDECREF(ret); + Py_DECREF(flags_obj); + return -1; +} + +static void +slot_bf_releasebuffer(PyObject *self, Py_buffer *buffer) +{ + PyObject *mv; + if (Py_TYPE(buffer->obj) == &_PyBufferWrapper_Type) { + // Make sure we pass the same memoryview to + // __release_buffer__() that __buffer__() returned. + mv = Py_NewRef(((PyBufferWrapper *)buffer->obj)->mv); + } + else { + mv = PyMemoryView_FromBuffer(buffer); + if (mv == NULL) { + PyErr_WriteUnraisable(self); + return; + } + } + PyObject *stack[2] = {self, mv}; + PyObject *ret = vectorcall_method(&_Py_ID(__release_buffer__), stack, 2); + Py_DECREF(mv); + if (ret == NULL) { + PyErr_WriteUnraisable(self); + } + else { + Py_DECREF(ret); + } +} + static PyObject * slot_am_await(PyObject *self) { @@ -8962,6 +9141,7 @@ an all-zero entry. #undef TPSLOT #undef FLSLOT +#undef BUFSLOT #undef AMSLOT #undef ETSLOT #undef SQSLOT @@ -8981,6 +9161,8 @@ an all-zero entry. #define ETSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ {#NAME, offsetof(PyHeapTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, \ PyDoc_STR(DOC), .name_strobj = &_Py_ID(NAME) } +#define BUFSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ + ETSLOT(NAME, as_buffer.SLOT, FUNCTION, WRAPPER, DOC) #define AMSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ ETSLOT(NAME, as_async.SLOT, FUNCTION, WRAPPER, DOC) #define SQSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ @@ -9062,6 +9244,13 @@ static pytype_slotdef slotdefs[] = { "Create and return new object. See help(type) for accurate signature."), TPSLOT(__del__, tp_finalize, slot_tp_finalize, (wrapperfunc)wrap_del, ""), + BUFSLOT(__buffer__, bf_getbuffer, slot_bf_getbuffer, wrap_buffer, + "__buffer__($self, flags, /)\n--\n\n" + "Return a buffer object that exposes the underlying memory of the object."), + BUFSLOT(__release_buffer__, bf_releasebuffer, slot_bf_releasebuffer, wrap_releasebuffer, + "__release_buffer__($self, /)\n--\n\n" + "Release the buffer object that exposes the underlying memory of the object."), + AMSLOT(__await__, am_await, slot_am_await, wrap_unaryfunc, "__await__($self, /)\n--\n\nReturn an iterator to be used in await expression."), AMSLOT(__aiter__, am_aiter, slot_am_aiter, wrap_unaryfunc, @@ -9208,8 +9397,12 @@ slotptr(PyTypeObject *type, int ioffset) /* Note: this depends on the order of the members of PyHeapTypeObject! */ assert(offset >= 0); - assert((size_t)offset < offsetof(PyHeapTypeObject, as_buffer)); - if ((size_t)offset >= offsetof(PyHeapTypeObject, as_sequence)) { + assert((size_t)offset < offsetof(PyHeapTypeObject, ht_name)); + if ((size_t)offset >= offsetof(PyHeapTypeObject, as_buffer)) { + ptr = (char *)type->tp_as_buffer; + offset -= offsetof(PyHeapTypeObject, as_buffer); + } + else if ((size_t)offset >= offsetof(PyHeapTypeObject, as_sequence)) { ptr = (char *)type->tp_as_sequence; offset -= offsetof(PyHeapTypeObject, as_sequence); } diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index 56448b6ee7d492..350f97f8ff41aa 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -110,6 +110,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 297c9ce799bea1..af80f1eebb3c4d 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -60,6 +60,9 @@ Source Files + + Source Files + Source Files diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index c27817702bf97d..ded19ee489e79b 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -121,6 +121,8 @@ '__xor__', '__divmod__', '__rdivmod__', + '__buffer__', + '__release_buffer__', ] NON_GENERATED_IMMORTAL_OBJECTS = [ diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 4dfbbe72df56a0..165bd74587d7c7 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -86,6 +86,7 @@ Objects/sliceobject.c - PyEllipsis_Type - Objects/sliceobject.c - PySlice_Type - Objects/tupleobject.c - PyTupleIter_Type - Objects/tupleobject.c - PyTuple_Type - +Objects/typeobject.c - _PyBufferWrapper_Type - Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - Objects/typeobject.c - PyType_Type - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 7a5d7d45f5184b..fee493ff7f1666 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -404,6 +404,7 @@ Modules/_testbuffer.c ndarray_memoryview_from_buffer strides - Modules/_testbuffer.c ndarray_memoryview_from_buffer suboffsets - Modules/_testbuffer.c ndarray_push kwlist - Modules/_testbuffer.c staticarray_init kwlist - +Modules/_testcapi/buffer.c - testBufType - Modules/_testcapi/code.c get_code_extra_index key - Modules/_testcapi/datetime.c - test_run_counter - Modules/_testcapi/exceptions.c - PyRecursingInfinitelyError_Type - From b7a0a521960a6e9ea46b79b51cbcc3c4ffcc7057 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Thu, 4 May 2023 08:23:40 -0700 Subject: [PATCH 46/65] gh-102500: Document PEP 688 (#102571) Co-authored-by: Hugo van Kemenade Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/library/collections.abc.rst | 8 +++++++ Doc/library/inspect.rst | 33 ++++++++++++++++++++++++++ Doc/reference/datamodel.rst | 41 +++++++++++++++++++++++++++++++++ Doc/whatsnew/3.12.rst | 14 ++++++++++- 4 files changed, 95 insertions(+), 1 deletion(-) diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 1ada0d352a0cc6..669b7345499a78 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -177,6 +177,7 @@ ABC Inherits from Abstract Methods Mi :class:`AsyncIterable` [1]_ ``__aiter__`` :class:`AsyncIterator` [1]_ :class:`AsyncIterable` ``__anext__`` ``__aiter__`` :class:`AsyncGenerator` [1]_ :class:`AsyncIterator` ``asend``, ``athrow`` ``aclose``, ``__aiter__``, ``__anext__`` +:class:`Buffer` [1]_ ``__buffer__`` ============================== ====================== ======================= ==================================================== @@ -346,6 +347,13 @@ Collections Abstract Base Classes -- Detailed Descriptions .. versionadded:: 3.6 +.. class:: Buffer + + ABC for classes that provide the :meth:`~object.__buffer__` method, + implementing the :ref:`buffer protocol `. See :pep:`688`. + + .. versionadded:: 3.12 + Examples and Recipes -------------------- diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index 88f843c03b1d5a..7884308a333020 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -1603,6 +1603,39 @@ the following flags: for any introspection needs. +Buffer flags +------------ + +.. class:: BufferFlags + + This is an :class:`enum.IntFlag` that represents the flags that + can be passed to the :meth:`~object.__buffer__` method of objects + implementing the :ref:`buffer protocol `. + + The meaning of the flags is explained at :ref:`buffer-request-types`. + + .. attribute:: BufferFlags.SIMPLE + .. attribute:: BufferFlags.WRITABLE + .. attribute:: BufferFlags.FORMAT + .. attribute:: BufferFlags.ND + .. attribute:: BufferFlags.STRIDES + .. attribute:: BufferFlags.C_CONTIGUOUS + .. attribute:: BufferFlags.F_CONTIGUOUS + .. attribute:: BufferFlags.ANY_CONTIGUOUS + .. attribute:: BufferFlags.INDIRECT + .. attribute:: BufferFlags.CONTIG + .. attribute:: BufferFlags.CONTIG_RO + .. attribute:: BufferFlags.STRIDED + .. attribute:: BufferFlags.STRIDED_RO + .. attribute:: BufferFlags.RECORDS + .. attribute:: BufferFlags.RECORDS_RO + .. attribute:: BufferFlags.FULL + .. attribute:: BufferFlags.FULL_RO + .. attribute:: BufferFlags.READ + .. attribute:: BufferFlags.WRITE + + .. versionadded:: 3.12 + .. _inspect-module-cli: Command Line Interface diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 1294d683e268a0..0a9cabc158b9e4 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2865,6 +2865,47 @@ a :exc:`TypeError`. The specification for the Python ``match`` statement. +.. _python-buffer-protocol: + +Emulating buffer types +---------------------- + +The :ref:`buffer protocol ` provides a way for Python +objects to expose efficient access to a low-level memory array. This protocol +is implemented by builtin types such as :class:`bytes` and :class:`memoryview`, +and third-party libraries may define additional buffer types. + +While buffer types are usually implemented in C, it is also possible to +implement the protocol in Python. + +.. method:: object.__buffer__(self, flags) + + Called when a buffer is requested from *self* (for example, by the + :class:`memoryview` constructor). The *flags* argument is an integer + representing the kind of buffer requested, affecting for example whether + the returned buffer is read-only or writable. :class:`inspect.BufferFlags` + provides a convenient way to interpret the flags. The method must return + a :class:`memoryview` object. + +.. method:: object.__release_buffer__(self, buffer) + + Called when a buffer is no longer needed. The *buffer* argument is a + :class:`memoryview` object that was previously returned by + :meth:`~object.__buffer__`. The method must release any resources associated + with the buffer. This method should return ``None``. + Buffer objects that do not need to perform any cleanup are not required + to implement this method. + +.. versionadded:: 3.12 + +.. seealso:: + + :pep:`688` - Making the buffer protocol accessible in Python + Introduces the Python ``__buffer__`` and ``__release_buffer__`` methods. + + :class:`collections.abc.Buffer` + ABC for buffer types. + .. _special-lookup: Special method lookup diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index edbf921467553c..3fe3310a26e59b 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -149,6 +149,19 @@ New Features In Python 3.14, the default will switch to ``'data'``. (Contributed by Petr Viktorin in :pep:`706`.) +PEP 688: Making the buffer protocol accessible in Python +-------------------------------------------------------- + +:pep:`688` introduces a way to use the :ref:`buffer protocol ` +from Python code. Classes that implement the :meth:`~object.__buffer__` method +are now usable as buffer types. + +The new :class:`collections.abc.Buffer` ABC provides a standard +way to represent buffer objects, for example in type annotations. +The new :class:`inspect.BufferFlags` enum represents the flags that +can be used to customize buffer creation. +(Contributed by Jelle Zijlstra in :gh:`102500`.) + New Features Related to Type Hints ================================== @@ -179,7 +192,6 @@ See :pep:`692` for more details. (PEP written by Franek Magiera) - Other Language Changes ====================== From 2ba931ff727395cf89b290ed313a8e15db0bfcf1 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 4 May 2023 18:25:09 +0300 Subject: [PATCH 47/65] gh-99593: Add tests for Unicode C API (part 2) (#99868) Add tests for lower-level functions. Co-authored-by: Oleg Iarygin --- Lib/test/test_capi/test_unicode.py | 497 ++++++++++++++++++++++++++-- Modules/_testcapi/unicode.c | 506 +++++++++++++++++++++++++++-- Modules/_testcapimodule.c | 1 + 3 files changed, 944 insertions(+), 60 deletions(-) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 857579f758386f..00807d968a7c43 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -17,6 +17,287 @@ class Str(str): class CAPITest(unittest.TestCase): + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_new(self): + """Test PyUnicode_New()""" + from _testcapi import unicode_new as new + + for maxchar in 0, 0x61, 0xa1, 0x4f60, 0x1f600, 0x10ffff: + self.assertEqual(new(0, maxchar), '') + self.assertEqual(new(5, maxchar), chr(maxchar)*5) + self.assertEqual(new(0, 0x110000), '') + self.assertRaises(SystemError, new, 5, 0x110000) + self.assertRaises(SystemError, new, -1, 0) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fill(self): + """Test PyUnicode_Fill()""" + from _testcapi import unicode_fill as fill + + strings = [ + # all strings have exactly 5 characters + 'abcde', '\xa1\xa2\xa3\xa4\xa5', + '\u4f60\u597d\u4e16\u754c\uff01', + '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' + ] + chars = [0x78, 0xa9, 0x20ac, 0x1f638] + + for idx, fill_char in enumerate(chars): + # wide -> narrow: exceed maxchar limitation + for to in strings[:idx]: + self.assertRaises(ValueError, fill, to, 0, 0, fill_char) + for to in strings[idx:]: + for start in range(7): + for length in range(-1, 7 - start): + filled = max(min(length, 5 - start), 0) + if filled == 5 and to != strings[idx]: + # narrow -> wide + # Tests omitted since this creates invalid strings. + continue + expected = to[:start] + chr(fill_char) * filled + to[start + filled:] + self.assertEqual(fill(to, start, length, fill_char), + (expected, filled)) + + s = strings[0] + self.assertRaises(IndexError, fill, s, -1, 0, 0x78) + self.assertRaises(ValueError, fill, s, 0, 0, 0x110000) + self.assertRaises(SystemError, fill, b'abc', 0, 0, 0x78) + self.assertRaises(SystemError, fill, [], 0, 0, 0x78) + # CRASHES fill(s, 0, NULL, 0, 0) + # CRASHES fill(NULL, 0, 0, 0x78) + # TODO: Test PyUnicode_Fill() with non-modifiable unicode. + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_writechar(self): + """Test PyUnicode_ReadChar()""" + from _testcapi import unicode_writechar as writechar + + strings = [ + # one string for every kind + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602' + ] + # one character for every kind + out of range code + chars = [0x78, 0xa9, 0x20ac, 0x1f638, 0x110000] + for i, s in enumerate(strings): + for j, c in enumerate(chars): + if j <= i: + self.assertEqual(writechar(s, 1, c), + (s[:1] + chr(c) + s[2:], 0)) + else: + self.assertRaises(ValueError, writechar, s, 1, c) + + self.assertRaises(IndexError, writechar, 'abc', 3, 0x78) + self.assertRaises(IndexError, writechar, 'abc', -1, 0x78) + self.assertRaises(TypeError, writechar, b'abc', 0, 0x78) + self.assertRaises(TypeError, writechar, [], 0, 0x78) + # CRASHES writechar(NULL, 0, 0x78) + # TODO: Test PyUnicode_CopyCharacters() with non-modifiable and legacy + # unicode. + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_resize(self): + """Test PyUnicode_Resize()""" + from _testcapi import unicode_resize as resize + + strings = [ + # all strings have exactly 3 characters + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602' + ] + for s in strings: + self.assertEqual(resize(s, 3), (s, 0)) + self.assertEqual(resize(s, 2), (s[:2], 0)) + self.assertEqual(resize(s, 4), (s + '\0', 0)) + self.assertEqual(resize(s, 0), ('', 0)) + self.assertRaises(SystemError, resize, b'abc', 0) + self.assertRaises(SystemError, resize, [], 0) + self.assertRaises(SystemError, resize, NULL, 0) + # TODO: Test PyUnicode_Resize() with non-modifiable and legacy unicode + # and with NULL as the address. + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_append(self): + """Test PyUnicode_Append()""" + from _testcapi import unicode_append as append + + strings = [ + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602' + ] + for left in strings: + left = left[::-1] + for right in strings: + expected = left + right + self.assertEqual(append(left, right), expected) + + self.assertRaises(SystemError, append, 'abc', b'abc') + self.assertRaises(SystemError, append, b'abc', 'abc') + self.assertRaises(SystemError, append, b'abc', b'abc') + self.assertRaises(SystemError, append, 'abc', []) + self.assertRaises(SystemError, append, [], 'abc') + self.assertRaises(SystemError, append, [], []) + self.assertRaises(SystemError, append, NULL, 'abc') + self.assertRaises(SystemError, append, 'abc', NULL) + # TODO: Test PyUnicode_Append() with modifiable unicode + # and with NULL as the address. + # TODO: Check reference counts. + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_appendanddel(self): + """Test PyUnicode_AppendAndDel()""" + from _testcapi import unicode_appendanddel as appendanddel + + strings = [ + 'abc', '\xa1\xa2\xa3', '\u4f60\u597d\u4e16', + '\U0001f600\U0001f601\U0001f602' + ] + for left in strings: + left = left[::-1] + for right in strings: + self.assertEqual(appendanddel(left, right), left + right) + + self.assertRaises(SystemError, appendanddel, 'abc', b'abc') + self.assertRaises(SystemError, appendanddel, b'abc', 'abc') + self.assertRaises(SystemError, appendanddel, b'abc', b'abc') + self.assertRaises(SystemError, appendanddel, 'abc', []) + self.assertRaises(SystemError, appendanddel, [], 'abc') + self.assertRaises(SystemError, appendanddel, [], []) + self.assertRaises(SystemError, appendanddel, NULL, 'abc') + self.assertRaises(SystemError, appendanddel, 'abc', NULL) + # TODO: Test PyUnicode_AppendAndDel() with modifiable unicode + # and with NULL as the address. + # TODO: Check reference counts. + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fromstringandsize(self): + """Test PyUnicode_FromStringAndSize()""" + from _testcapi import unicode_fromstringandsize as fromstringandsize + + self.assertEqual(fromstringandsize(b'abc'), 'abc') + self.assertEqual(fromstringandsize(b'abc', 2), 'ab') + self.assertEqual(fromstringandsize(b'abc\0def'), 'abc\0def') + self.assertEqual(fromstringandsize(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') + self.assertEqual(fromstringandsize(b'\xe4\xbd\xa0'), '\u4f60') + self.assertEqual(fromstringandsize(b'\xf0\x9f\x98\x80'), '\U0001f600') + self.assertRaises(UnicodeDecodeError, fromstringandsize, b'\xc2\xa1', 1) + self.assertRaises(UnicodeDecodeError, fromstringandsize, b'\xa1', 1) + self.assertEqual(fromstringandsize(b'', 0), '') + self.assertEqual(fromstringandsize(NULL, 0), '') + + self.assertRaises(SystemError, fromstringandsize, b'abc', -1) + # TODO: Test PyUnicode_FromStringAndSize(NULL, size) for size != 0 + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fromstring(self): + """Test PyUnicode_FromString()""" + from _testcapi import unicode_fromstring as fromstring + + self.assertEqual(fromstring(b'abc'), 'abc') + self.assertEqual(fromstring(b'\xc2\xa1\xc2\xa2'), '\xa1\xa2') + self.assertEqual(fromstring(b'\xe4\xbd\xa0'), '\u4f60') + self.assertEqual(fromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') + self.assertRaises(UnicodeDecodeError, fromstring, b'\xc2') + self.assertRaises(UnicodeDecodeError, fromstring, b'\xa1') + self.assertEqual(fromstring(b''), '') + + # CRASHES fromstring(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fromkindanddata(self): + """Test PyUnicode_FromKindAndData()""" + from _testcapi import unicode_fromkindanddata as fromkindanddata + + strings = [ + 'abcde', '\xa1\xa2\xa3\xa4\xa5', + '\u4f60\u597d\u4e16\u754c\uff01', + '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' + ] + enc1 = 'latin1' + for s in strings[:2]: + self.assertEqual(fromkindanddata(1, s.encode(enc1)), s) + enc2 = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' + for s in strings[:3]: + self.assertEqual(fromkindanddata(2, s.encode(enc2)), s) + enc4 = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' + for s in strings: + self.assertEqual(fromkindanddata(4, s.encode(enc4)), s) + self.assertEqual(fromkindanddata(2, '\U0001f600'.encode(enc2)), + '\ud83d\ude00') + for kind in 1, 2, 4: + self.assertEqual(fromkindanddata(kind, b''), '') + self.assertEqual(fromkindanddata(kind, b'\0'*kind), '\0') + self.assertEqual(fromkindanddata(kind, NULL, 0), '') + + for kind in -1, 0, 3, 5, 8: + self.assertRaises(SystemError, fromkindanddata, kind, b'') + self.assertRaises(ValueError, fromkindanddata, 1, b'abc', -1) + self.assertRaises(ValueError, fromkindanddata, 1, NULL, -1) + # CRASHES fromkindanddata(1, NULL, 1) + # CRASHES fromkindanddata(4, b'\xff\xff\xff\xff') + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_substring(self): + """Test PyUnicode_Substring()""" + from _testcapi import unicode_substring as substring + + strings = [ + 'ab', 'ab\xa1\xa2', + 'ab\xa1\xa2\u4f60\u597d', + 'ab\xa1\xa2\u4f60\u597d\U0001f600\U0001f601' + ] + for s in strings: + for start in range(0, len(s) + 2): + for end in range(max(start-1, 0), len(s) + 2): + self.assertEqual(substring(s, start, end), s[start:end]) + + self.assertRaises(IndexError, substring, 'abc', -1, 0) + self.assertRaises(IndexError, substring, 'abc', 0, -1) + # CRASHES substring(b'abc', 0, 0) + # CRASHES substring([], 0, 0) + # CRASHES substring(NULL, 0, 0) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_getlength(self): + """Test PyUnicode_GetLength()""" + from _testcapi import unicode_getlength as getlength + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + self.assertEqual(getlength(s), len(s)) + + self.assertRaises(TypeError, getlength, b'abc') + self.assertRaises(TypeError, getlength, []) + # CRASHES getlength(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_readchar(self): + """Test PyUnicode_ReadChar()""" + from _testcapi import unicode_readchar as readchar + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + for i, c in enumerate(s): + self.assertEqual(readchar(s, i), ord(c)) + self.assertRaises(IndexError, readchar, s, len(s)) + self.assertRaises(IndexError, readchar, s, -1) + + self.assertRaises(TypeError, readchar, b'abc', 0) + self.assertRaises(TypeError, readchar, [], 0) + # CRASHES readchar(NULL, 0) + @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_fromobject(self): @@ -293,13 +574,70 @@ def check_format(expected, format, *args): self.assertRaisesRegex(SystemError, 'invalid format string', PyUnicode_FromFormat, b'%+i', c_int(10)) + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_interninplace(self): + """Test PyUnicode_InternInPlace()""" + from _testcapi import unicode_interninplace as interninplace + + s = b'abc'.decode() + r = interninplace(s) + self.assertEqual(r, 'abc') + + # CRASHES interninplace(b'abc') + # CRASHES interninplace(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_internfromstring(self): + """Test PyUnicode_InternFromString()""" + from _testcapi import unicode_internfromstring as internfromstring + + self.assertEqual(internfromstring(b'abc'), 'abc') + self.assertEqual(internfromstring(b'\xf0\x9f\x98\x80'), '\U0001f600') + self.assertRaises(UnicodeDecodeError, internfromstring, b'\xc2') + self.assertRaises(UnicodeDecodeError, internfromstring, b'\xa1') + self.assertEqual(internfromstring(b''), '') + + # CRASHES internfromstring(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fromwidechar(self): + """Test PyUnicode_FromWideChar()""" + from _testcapi import unicode_fromwidechar as fromwidechar + from _testcapi import SIZEOF_WCHAR_T + + if SIZEOF_WCHAR_T == 2: + encoding = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' + elif SIZEOF_WCHAR_T == 4: + encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' + + for s in '', 'abc', '\xa1\xa2', '\u4f60', '\U0001f600': + b = s.encode(encoding) + self.assertEqual(fromwidechar(b), s) + self.assertEqual(fromwidechar(b + b'\0'*SIZEOF_WCHAR_T, -1), s) + for s in '\ud83d', '\ude00': + b = s.encode(encoding, 'surrogatepass') + self.assertEqual(fromwidechar(b), s) + self.assertEqual(fromwidechar(b + b'\0'*SIZEOF_WCHAR_T, -1), s) + + self.assertEqual(fromwidechar('abc'.encode(encoding), 2), 'ab') + if SIZEOF_WCHAR_T == 2: + self.assertEqual(fromwidechar('a\U0001f600'.encode(encoding), 2), 'a\ud83d') + + self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, -2) + self.assertEqual(fromwidechar(NULL, 0), '') + self.assertRaises(SystemError, fromwidechar, NULL, 1) + self.assertRaises(SystemError, fromwidechar, NULL, -1) + @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_aswidechar(self): """Test PyUnicode_AsWideChar()""" from _testcapi import unicode_aswidechar - import_helper.import_module('ctypes') - from ctypes import c_wchar, sizeof + from _testcapi import unicode_aswidechar_null + from _testcapi import SIZEOF_WCHAR_T wchar, size = unicode_aswidechar('abcdef', 2) self.assertEqual(size, 2) @@ -308,6 +646,8 @@ def test_aswidechar(self): wchar, size = unicode_aswidechar('abc', 3) self.assertEqual(size, 3) self.assertEqual(wchar, 'abc') + self.assertEqual(unicode_aswidechar_null('abc', 10), 4) + self.assertEqual(unicode_aswidechar_null('abc', 0), 4) wchar, size = unicode_aswidechar('abc', 4) self.assertEqual(size, 3) @@ -320,60 +660,113 @@ def test_aswidechar(self): wchar, size = unicode_aswidechar('abc\0def', 20) self.assertEqual(size, 7) self.assertEqual(wchar, 'abc\0def\0') + self.assertEqual(unicode_aswidechar_null('abc\0def', 20), 8) nonbmp = chr(0x10ffff) - if sizeof(c_wchar) == 2: - buflen = 3 + if SIZEOF_WCHAR_T == 2: nchar = 2 - else: # sizeof(c_wchar) == 4 - buflen = 2 + else: # SIZEOF_WCHAR_T == 4 nchar = 1 - wchar, size = unicode_aswidechar(nonbmp, buflen) + wchar, size = unicode_aswidechar(nonbmp, 10) self.assertEqual(size, nchar) self.assertEqual(wchar, nonbmp + '\0') + self.assertEqual(unicode_aswidechar_null(nonbmp, 10), nchar + 1) + + self.assertRaises(TypeError, unicode_aswidechar, b'abc', 10) + self.assertRaises(TypeError, unicode_aswidechar, [], 10) + self.assertRaises(SystemError, unicode_aswidechar, NULL, 10) + self.assertRaises(TypeError, unicode_aswidechar_null, b'abc', 10) + self.assertRaises(TypeError, unicode_aswidechar_null, [], 10) + self.assertRaises(SystemError, unicode_aswidechar_null, NULL, 10) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_aswidecharstring(self): """Test PyUnicode_AsWideCharString()""" from _testcapi import unicode_aswidecharstring - import_helper.import_module('ctypes') - from ctypes import c_wchar, sizeof + from _testcapi import unicode_aswidecharstring_null + from _testcapi import SIZEOF_WCHAR_T wchar, size = unicode_aswidecharstring('abc') self.assertEqual(size, 3) self.assertEqual(wchar, 'abc\0') + self.assertEqual(unicode_aswidecharstring_null('abc'), 'abc') wchar, size = unicode_aswidecharstring('abc\0def') self.assertEqual(size, 7) self.assertEqual(wchar, 'abc\0def\0') + self.assertRaises(ValueError, unicode_aswidecharstring_null, 'abc\0def') nonbmp = chr(0x10ffff) - if sizeof(c_wchar) == 2: + if SIZEOF_WCHAR_T == 2: nchar = 2 - else: # sizeof(c_wchar) == 4 + else: # SIZEOF_WCHAR_T == 4 nchar = 1 wchar, size = unicode_aswidecharstring(nonbmp) self.assertEqual(size, nchar) self.assertEqual(wchar, nonbmp + '\0') + self.assertEqual(unicode_aswidecharstring_null(nonbmp), nonbmp) + + self.assertRaises(TypeError, unicode_aswidecharstring, b'abc') + self.assertRaises(TypeError, unicode_aswidecharstring, []) + self.assertRaises(SystemError, unicode_aswidecharstring, NULL) + self.assertRaises(TypeError, unicode_aswidecharstring_null, b'abc') + self.assertRaises(TypeError, unicode_aswidecharstring_null, []) + self.assertRaises(SystemError, unicode_aswidecharstring_null, NULL) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_asucs4(self): """Test PyUnicode_AsUCS4()""" from _testcapi import unicode_asucs4 + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', 'a\ud800b\udfffc', '\ud834\udd1e']: l = len(s) - self.assertEqual(unicode_asucs4(s, l, True), s+'\0') - self.assertEqual(unicode_asucs4(s, l, False), s+'\uffff') - self.assertEqual(unicode_asucs4(s, l+1, True), s+'\0\uffff') - self.assertEqual(unicode_asucs4(s, l+1, False), s+'\0\uffff') - self.assertRaises(SystemError, unicode_asucs4, s, l-1, True) - self.assertRaises(SystemError, unicode_asucs4, s, l-2, False) + self.assertEqual(unicode_asucs4(s, l, 1), s+'\0') + self.assertEqual(unicode_asucs4(s, l, 0), s+'\uffff') + self.assertEqual(unicode_asucs4(s, l+1, 1), s+'\0\uffff') + self.assertEqual(unicode_asucs4(s, l+1, 0), s+'\0\uffff') + self.assertRaises(SystemError, unicode_asucs4, s, l-1, 1) + self.assertRaises(SystemError, unicode_asucs4, s, l-2, 0) + s = '\0'.join([s, s]) + self.assertEqual(unicode_asucs4(s, len(s), 1), s+'\0') + self.assertEqual(unicode_asucs4(s, len(s), 0), s+'\uffff') + + # CRASHES unicode_asucs4(b'abc', 1, 0) + # CRASHES unicode_asucs4(b'abc', 1, 1) + # CRASHES unicode_asucs4([], 1, 1) + # CRASHES unicode_asucs4(NULL, 1, 0) + # CRASHES unicode_asucs4(NULL, 1, 1) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_asucs4copy(self): + """Test PyUnicode_AsUCS4Copy()""" + from _testcapi import unicode_asucs4copy as asucs4copy + + for s in ['abc', '\xa1\xa2', '\u4f60\u597d', 'a\U0001f600', + 'a\ud800b\udfffc', '\ud834\udd1e']: + self.assertEqual(asucs4copy(s), s+'\0') s = '\0'.join([s, s]) - self.assertEqual(unicode_asucs4(s, len(s), True), s+'\0') - self.assertEqual(unicode_asucs4(s, len(s), False), s+'\uffff') + self.assertEqual(asucs4copy(s), s+'\0') + + # CRASHES asucs4copy(b'abc') + # CRASHES asucs4copy([]) + # CRASHES asucs4copy(NULL) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_fromordinal(self): + """Test PyUnicode_FromOrdinal()""" + from _testcapi import unicode_fromordinal as fromordinal + + self.assertEqual(fromordinal(0x61), 'a') + self.assertEqual(fromordinal(0x20ac), '\u20ac') + self.assertEqual(fromordinal(0x1f600), '\U0001f600') + + self.assertRaises(ValueError, fromordinal, 0x110000) + self.assertRaises(ValueError, fromordinal, -1) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') @@ -381,29 +774,62 @@ def test_asutf8(self): """Test PyUnicode_AsUTF8()""" from _testcapi import unicode_asutf8 - bmp = '\u0100' - bmp2 = '\uffff' - nonbmp = chr(0x10ffff) + self.assertEqual(unicode_asutf8('abc', 4), b'abc\0') + self.assertEqual(unicode_asutf8('абв', 7), b'\xd0\xb0\xd0\xb1\xd0\xb2\0') + self.assertEqual(unicode_asutf8('\U0001f600', 5), b'\xf0\x9f\x98\x80\0') + self.assertEqual(unicode_asutf8('abc\0def', 8), b'abc\0def\0') - self.assertEqual(unicode_asutf8(bmp), b'\xc4\x80') - self.assertEqual(unicode_asutf8(bmp2), b'\xef\xbf\xbf') - self.assertEqual(unicode_asutf8(nonbmp), b'\xf4\x8f\xbf\xbf') - self.assertRaises(UnicodeEncodeError, unicode_asutf8, 'a\ud800b\udfffc') + self.assertRaises(UnicodeEncodeError, unicode_asutf8, '\ud8ff', 0) + self.assertRaises(TypeError, unicode_asutf8, b'abc', 0) + self.assertRaises(TypeError, unicode_asutf8, [], 0) + # CRASHES unicode_asutf8(NULL, 0) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') def test_asutf8andsize(self): """Test PyUnicode_AsUTF8AndSize()""" from _testcapi import unicode_asutf8andsize + from _testcapi import unicode_asutf8andsize_null - bmp = '\u0100' - bmp2 = '\uffff' - nonbmp = chr(0x10ffff) + self.assertEqual(unicode_asutf8andsize('abc', 4), (b'abc\0', 3)) + self.assertEqual(unicode_asutf8andsize('абв', 7), (b'\xd0\xb0\xd0\xb1\xd0\xb2\0', 6)) + self.assertEqual(unicode_asutf8andsize('\U0001f600', 5), (b'\xf0\x9f\x98\x80\0', 4)) + self.assertEqual(unicode_asutf8andsize('abc\0def', 8), (b'abc\0def\0', 7)) + self.assertEqual(unicode_asutf8andsize_null('abc', 4), b'abc\0') + self.assertEqual(unicode_asutf8andsize_null('abc\0def', 8), b'abc\0def\0') - self.assertEqual(unicode_asutf8andsize(bmp), (b'\xc4\x80', 2)) - self.assertEqual(unicode_asutf8andsize(bmp2), (b'\xef\xbf\xbf', 3)) - self.assertEqual(unicode_asutf8andsize(nonbmp), (b'\xf4\x8f\xbf\xbf', 4)) - self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, 'a\ud800b\udfffc') + self.assertRaises(UnicodeEncodeError, unicode_asutf8andsize, '\ud8ff', 0) + self.assertRaises(TypeError, unicode_asutf8andsize, b'abc', 0) + self.assertRaises(TypeError, unicode_asutf8andsize, [], 0) + # CRASHES unicode_asutf8andsize(NULL, 0) + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_getdefaultencoding(self): + """Test PyUnicode_GetDefaultEncoding()""" + from _testcapi import unicode_getdefaultencoding as getdefaultencoding + + self.assertEqual(getdefaultencoding(), b'utf-8') + + @support.cpython_only + @unittest.skipIf(_testcapi is None, 'need _testcapi module') + def test_transform_decimal_and_space(self): + """Test _PyUnicode_TransformDecimalAndSpaceToASCII()""" + from _testcapi import unicode_transformdecimalandspacetoascii as transform_decimal + + self.assertEqual(transform_decimal('123'), + '123') + self.assertEqual(transform_decimal('\u0663.\u0661\u0664'), + '3.14') + self.assertEqual(transform_decimal("\N{EM SPACE}3.14\N{EN SPACE}"), + " 3.14 ") + self.assertEqual(transform_decimal('12\u20ac3'), + '12?') + self.assertEqual(transform_decimal(''), '') + + self.assertRaises(SystemError, transform_decimal, b'123') + self.assertRaises(SystemError, transform_decimal, []) + # CRASHES transform_decimal(NULL) @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') @@ -858,6 +1284,7 @@ def test_copycharacters(self): from _testcapi import unicode_copycharacters strings = [ + # all strings have exactly 5 characters 'abcde', '\xa1\xa2\xa3\xa4\xa5', '\u4f60\u597d\u4e16\u754c\uff01', '\U0001f600\U0001f601\U0001f602\U0001f603\U0001f604' @@ -894,6 +1321,10 @@ def test_copycharacters(self): self.assertRaises(SystemError, unicode_copycharacters, s, 1, s, 0, 5) self.assertRaises(SystemError, unicode_copycharacters, s, 0, s, 0, -1) self.assertRaises(SystemError, unicode_copycharacters, s, 0, b'', 0, 0) + self.assertRaises(SystemError, unicode_copycharacters, s, 0, [], 0, 0) + # CRASHES unicode_copycharacters(s, 0, NULL, 0, 0) + # TODO: Test PyUnicode_CopyCharacters() with non-unicode and + # non-modifiable unicode as "to". @support.cpython_only @unittest.skipIf(_testcapi is None, 'need _testcapi module') diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index 2d23993ce420b3..7dd3b9c0c03e50 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -102,6 +102,278 @@ test_widechar(PyObject *self, PyObject *Py_UNUSED(ignored)) #define NULLABLE(x) do { if (x == Py_None) x = NULL; } while (0); +static PyObject * +unicode_copy(PyObject *unicode) +{ + PyObject *copy; + + if (!unicode) { + return NULL; + } + if (!PyUnicode_Check(unicode)) { + Py_INCREF(unicode); + return unicode; + } + + copy = PyUnicode_New(PyUnicode_GET_LENGTH(unicode), + PyUnicode_MAX_CHAR_VALUE(unicode)); + if (!copy) { + return NULL; + } + if (PyUnicode_CopyCharacters(copy, 0, unicode, + 0, PyUnicode_GET_LENGTH(unicode)) < 0) + { + Py_DECREF(copy); + return NULL; + } + return copy; +} + +/* Test PyUnicode_New() */ +static PyObject * +unicode_new(PyObject *self, PyObject *args) +{ + Py_ssize_t size; + unsigned int maxchar; + PyObject *result; + + if (!PyArg_ParseTuple(args, "nI", &size, &maxchar)) { + return NULL; + } + + result = PyUnicode_New(size, (Py_UCS4)maxchar); + if (!result) { + return NULL; + } + if (size > 0 && maxchar <= 0x10ffff && + PyUnicode_Fill(result, 0, size, (Py_UCS4)maxchar) < 0) + { + Py_DECREF(result); + return NULL; + } + return result; +} + +/* Test PyUnicode_Fill() */ +static PyObject * +unicode_fill(PyObject *self, PyObject *args) +{ + PyObject *to, *to_copy; + Py_ssize_t start, length, filled; + unsigned int fill_char; + + if (!PyArg_ParseTuple(args, "OnnI", &to, &start, &length, &fill_char)) { + return NULL; + } + + NULLABLE(to); + if (!(to_copy = unicode_copy(to)) && to) { + return NULL; + } + + filled = PyUnicode_Fill(to_copy, start, length, (Py_UCS4)fill_char); + if (filled == -1 && PyErr_Occurred()) { + Py_DECREF(to_copy); + return NULL; + } + return Py_BuildValue("(Nn)", to_copy, filled); +} + +/* Test PyUnicode_WriteChar() */ +static PyObject * +unicode_writechar(PyObject *self, PyObject *args) +{ + PyObject *to, *to_copy; + Py_ssize_t index; + unsigned int character; + int result; + + if (!PyArg_ParseTuple(args, "OnI", &to, &index, &character)) { + return NULL; + } + + NULLABLE(to); + if (!(to_copy = unicode_copy(to)) && to) { + return NULL; + } + + result = PyUnicode_WriteChar(to_copy, index, (Py_UCS4)character); + if (result == -1 && PyErr_Occurred()) { + Py_DECREF(to_copy); + return NULL; + } + return Py_BuildValue("(Ni)", to_copy, result); +} + +/* Test PyUnicode_Resize() */ +static PyObject * +unicode_resize(PyObject *self, PyObject *args) +{ + PyObject *obj, *copy; + Py_ssize_t length; + int result; + + if (!PyArg_ParseTuple(args, "On", &obj, &length)) { + return NULL; + } + + NULLABLE(obj); + if (!(copy = unicode_copy(obj)) && obj) { + return NULL; + } + result = PyUnicode_Resize(©, length); + if (result == -1 && PyErr_Occurred()) { + Py_XDECREF(copy); + return NULL; + } + if (obj && PyUnicode_Check(obj) && length > PyUnicode_GET_LENGTH(obj)) { + if (PyUnicode_Fill(copy, PyUnicode_GET_LENGTH(obj), length, 0U) < 0) { + Py_DECREF(copy); + return NULL; + } + } + return Py_BuildValue("(Ni)", copy, result); +} + +/* Test PyUnicode_Append() */ +static PyObject * +unicode_append(PyObject *self, PyObject *args) +{ + PyObject *left, *right, *left_copy; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (!(left_copy = unicode_copy(left)) && left) { + return NULL; + } + PyUnicode_Append(&left_copy, right); + return left_copy; +} + +/* Test PyUnicode_AppendAndDel() */ +static PyObject * +unicode_appendanddel(PyObject *self, PyObject *args) +{ + PyObject *left, *right, *left_copy; + + if (!PyArg_ParseTuple(args, "OO", &left, &right)) + return NULL; + + NULLABLE(left); + NULLABLE(right); + if (!(left_copy = unicode_copy(left)) && left) { + return NULL; + } + Py_XINCREF(right); + PyUnicode_AppendAndDel(&left_copy, right); + return left_copy; +} + +/* Test PyUnicode_FromStringAndSize() */ +static PyObject * +unicode_fromstringandsize(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + + if (size == -100) { + size = bsize; + } + return PyUnicode_FromStringAndSize(s, size); +} + +/* Test PyUnicode_FromString() */ +static PyObject * +unicode_fromstring(PyObject *self, PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyUnicode_FromString(s); +} + +/* Test PyUnicode_FromKindAndData() */ +static PyObject * +unicode_fromkindanddata(PyObject *self, PyObject *args) +{ + int kind; + void *buffer; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "iz#|n", &kind, &buffer, &bsize, &size)) { + return NULL; + } + + if (size == -100) { + size = bsize; + } + if (kind && size % kind) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in unicode_fromkindanddata()"); + return NULL; + } + return PyUnicode_FromKindAndData(kind, buffer, kind ? size / kind : 0); +} + +/* Test PyUnicode_Substring() */ +static PyObject * +unicode_substring(PyObject *self, PyObject *args) +{ + PyObject *str; + Py_ssize_t start, end; + + if (!PyArg_ParseTuple(args, "Onn", &str, &start, &end)) { + return NULL; + } + + NULLABLE(str); + return PyUnicode_Substring(str, start, end); +} + +/* Test PyUnicode_GetLength() */ +static PyObject * +unicode_getlength(PyObject *self, PyObject *arg) +{ + Py_ssize_t result; + + NULLABLE(arg); + result = PyUnicode_GetLength(arg); + if (result == -1) + return NULL; + return PyLong_FromSsize_t(result); +} + +/* Test PyUnicode_ReadChar() */ +static PyObject * +unicode_readchar(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t index; + Py_UCS4 result; + + if (!PyArg_ParseTuple(args, "On", &unicode, &index)) { + return NULL; + } + + NULLABLE(unicode); + result = PyUnicode_ReadChar(unicode, index); + if (result == (Py_UCS4)-1) + return NULL; + return PyLong_FromUnsignedLong(result); +} + /* Test PyUnicode_FromObject() */ static PyObject * unicode_fromobject(PyObject *self, PyObject *arg) @@ -110,6 +382,51 @@ unicode_fromobject(PyObject *self, PyObject *arg) return PyUnicode_FromObject(arg); } +/* Test PyUnicode_InternInPlace() */ +static PyObject * +unicode_interninplace(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + Py_XINCREF(arg); + PyUnicode_InternInPlace(&arg); + return arg; +} + +/* Test PyUnicode_InternFromString() */ +static PyObject * +unicode_internfromstring(PyObject *self, PyObject *arg) +{ + const char *s; + Py_ssize_t size; + + if (!PyArg_Parse(arg, "z#", &s, &size)) { + return NULL; + } + return PyUnicode_InternFromString(s); +} + +/* Test PyUnicode_FromWideChar() */ +static PyObject * +unicode_fromwidechar(PyObject *self, PyObject *args) +{ + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { + return NULL; + } + if (size == -100) { + if (bsize % SIZEOF_WCHAR_T) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in unicode_fromwidechar()"); + return NULL; + } + size = bsize / SIZEOF_WCHAR_T; + } + return PyUnicode_FromWideChar((const wchar_t *)s, size); +} + /* Test PyUnicode_AsWideChar() */ static PyObject * unicode_aswidechar(PyObject *self, PyObject *args) @@ -118,8 +435,9 @@ unicode_aswidechar(PyObject *self, PyObject *args) Py_ssize_t buflen, size; wchar_t *buffer; - if (!PyArg_ParseTuple(args, "Un", &unicode, &buflen)) + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) return NULL; + NULLABLE(unicode); buffer = PyMem_New(wchar_t, buflen); if (buffer == NULL) return PyErr_NoMemory(); @@ -142,17 +460,35 @@ unicode_aswidechar(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, size); } +/* Test PyUnicode_AsWideCharString() with NULL as buffer */ +static PyObject * +unicode_aswidechar_null(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen, size; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + NULLABLE(unicode); + size = PyUnicode_AsWideChar(unicode, NULL, buflen); + if (size == -1) { + return NULL; + } + return PyLong_FromSsize_t(size); +} + /* Test PyUnicode_AsWideCharString() */ static PyObject * unicode_aswidecharstring(PyObject *self, PyObject *args) { PyObject *unicode, *result; - Py_ssize_t size; + Py_ssize_t size = 100; wchar_t *buffer; - if (!PyArg_ParseTuple(args, "U", &unicode)) + if (!PyArg_ParseTuple(args, "O", &unicode)) return NULL; + NULLABLE(unicode); buffer = PyUnicode_AsWideCharString(unicode, &size); if (buffer == NULL) return NULL; @@ -164,6 +500,28 @@ unicode_aswidecharstring(PyObject *self, PyObject *args) return Py_BuildValue("(Nn)", result, size); } +/* Test PyUnicode_AsWideCharString() with NULL as the size address */ +static PyObject * +unicode_aswidecharstring_null(PyObject *self, PyObject *args) +{ + PyObject *unicode, *result; + wchar_t *buffer; + + if (!PyArg_ParseTuple(args, "O", &unicode)) + return NULL; + + NULLABLE(unicode); + buffer = PyUnicode_AsWideCharString(unicode, NULL); + if (buffer == NULL) + return NULL; + + result = PyUnicode_FromWideChar(buffer, -1); + PyMem_Free(buffer); + if (result == NULL) + return NULL; + return result; +} + /* Test PyUnicode_AsUCS4() */ static PyObject * unicode_asucs4(PyObject *self, PyObject *args) @@ -173,10 +531,11 @@ unicode_asucs4(PyObject *self, PyObject *args) int copy_null; Py_ssize_t str_len, buf_len; - if (!PyArg_ParseTuple(args, "Unp:unicode_asucs4", &unicode, &str_len, ©_null)) { + if (!PyArg_ParseTuple(args, "Onp:unicode_asucs4", &unicode, &str_len, ©_null)) { return NULL; } + NULLABLE(unicode); buf_len = str_len + 1; buffer = PyMem_NEW(Py_UCS4, buf_len); if (buffer == NULL) { @@ -195,48 +554,117 @@ unicode_asucs4(PyObject *self, PyObject *args) return result; } -/* Test PyUnicode_AsUTF8() */ +/* Test PyUnicode_AsUCS4Copy() */ static PyObject * -unicode_asutf8(PyObject *self, PyObject *args) +unicode_asucs4copy(PyObject *self, PyObject *args) { PyObject *unicode; - const char *buffer; + Py_UCS4 *buffer; + PyObject *result; - if (!PyArg_ParseTuple(args, "U", &unicode)) { + if (!PyArg_ParseTuple(args, "O", &unicode)) { return NULL; } - buffer = PyUnicode_AsUTF8(unicode); + NULLABLE(unicode); + buffer = PyUnicode_AsUCS4Copy(unicode); if (buffer == NULL) { return NULL; } + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buffer, + PyUnicode_GET_LENGTH(unicode) + 1); + PyMem_FREE(buffer); + return result; +} + +/* Test PyUnicode_FromOrdinal() */ +static PyObject * +unicode_fromordinal(PyObject *self, PyObject *args) +{ + int ordinal; + + if (!PyArg_ParseTuple(args, "i", &ordinal)) + return NULL; - return PyBytes_FromString(buffer); + return PyUnicode_FromOrdinal(ordinal); +} + +/* Test PyUnicode_AsUTF8() */ +static PyObject * +unicode_asutf8(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) + return NULL; + + NULLABLE(unicode); + s = PyUnicode_AsUTF8(unicode); + if (s == NULL) + return NULL; + + return PyBytes_FromStringAndSize(s, buflen); } /* Test PyUnicode_AsUTF8AndSize() */ static PyObject * unicode_asutf8andsize(PyObject *self, PyObject *args) { - PyObject *unicode, *result; - const char *buffer; - Py_ssize_t utf8_len; + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + Py_ssize_t size = -100; - if(!PyArg_ParseTuple(args, "U", &unicode)) { + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) return NULL; - } - buffer = PyUnicode_AsUTF8AndSize(unicode, &utf8_len); - if (buffer == NULL) { + NULLABLE(unicode); + s = PyUnicode_AsUTF8AndSize(unicode, &size); + if (s == NULL) return NULL; - } - result = PyBytes_FromString(buffer); - if (result == NULL) { + return Py_BuildValue("(y#n)", s, buflen, size); +} + +/* Test PyUnicode_AsUTF8AndSize() with NULL as the size address */ +static PyObject * +unicode_asutf8andsize_null(PyObject *self, PyObject *args) +{ + PyObject *unicode; + Py_ssize_t buflen; + const char *s; + + if (!PyArg_ParseTuple(args, "On", &unicode, &buflen)) return NULL; - } - return Py_BuildValue("(Nn)", result, utf8_len); + NULLABLE(unicode); + s = PyUnicode_AsUTF8AndSize(unicode, NULL); + if (s == NULL) + return NULL; + + return PyBytes_FromStringAndSize(s, buflen); +} + +/* Test PyUnicode_GetDefaultEncoding() */ +static PyObject * +unicode_getdefaultencoding(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + const char *s = PyUnicode_GetDefaultEncoding(); + if (s == NULL) + return NULL; + + return PyBytes_FromString(s); +} + +/* Test _PyUnicode_TransformDecimalAndSpaceToASCII() */ +static PyObject * +unicode_transformdecimalandspacetoascii(PyObject *self, PyObject *arg) +{ + NULLABLE(arg); + return _PyUnicode_TransformDecimalAndSpaceToASCII(arg); } /* Test PyUnicode_DecodeUTF8() */ @@ -470,11 +898,11 @@ unicode_findchar(PyObject *self, PyObject *args) Py_ssize_t result; Py_ssize_t start, end; - if (!PyArg_ParseTuple(args, "UInni:unicode_findchar", &str, &ch, + if (!PyArg_ParseTuple(args, "OInni:unicode_findchar", &str, &ch, &start, &end, &direction)) { return NULL; } - + NULLABLE(str); result = PyUnicode_FindChar(str, (Py_UCS4)ch, start, end, direction); if (result == -2) return NULL; @@ -612,11 +1040,12 @@ unicode_copycharacters(PyObject *self, PyObject *args) PyObject *from, *to, *to_copy; Py_ssize_t from_start, to_start, how_many, copied; - if (!PyArg_ParseTuple(args, "UnOnn:unicode_copycharacters", &to, &to_start, + if (!PyArg_ParseTuple(args, "UnOnn", &to, &to_start, &from, &from_start, &how_many)) { return NULL; } + NULLABLE(from); if (!(to_copy = PyUnicode_New(PyUnicode_GET_LENGTH(to), PyUnicode_MAX_CHAR_VALUE(to)))) { return NULL; @@ -626,8 +1055,9 @@ unicode_copycharacters(PyObject *self, PyObject *args) return NULL; } - if ((copied = PyUnicode_CopyCharacters(to_copy, to_start, from, - from_start, how_many)) < 0) { + copied = PyUnicode_CopyCharacters(to_copy, to_start, from, + from_start, how_many); + if (copied == -1 && PyErr_Occurred()) { Py_DECREF(to_copy); return NULL; } @@ -1053,14 +1483,36 @@ static PyMethodDef TestMethods[] = { test_unicode_compare_with_ascii, METH_NOARGS}, {"test_string_from_format", test_string_from_format, METH_NOARGS}, {"test_widechar", test_widechar, METH_NOARGS}, + {"unicode_new", unicode_new, METH_VARARGS}, + {"unicode_fill", unicode_fill, METH_VARARGS}, + {"unicode_writechar", unicode_writechar, METH_VARARGS}, + {"unicode_resize", unicode_resize, METH_VARARGS}, + {"unicode_append", unicode_append, METH_VARARGS}, + {"unicode_appendanddel", unicode_appendanddel, METH_VARARGS}, + {"unicode_fromstringandsize",unicode_fromstringandsize, METH_VARARGS}, + {"unicode_fromstring", unicode_fromstring, METH_O}, + {"unicode_fromkindanddata", unicode_fromkindanddata, METH_VARARGS}, + {"unicode_substring", unicode_substring, METH_VARARGS}, + {"unicode_getlength", unicode_getlength, METH_O}, + {"unicode_readchar", unicode_readchar, METH_VARARGS}, {"unicode_fromobject", unicode_fromobject, METH_O}, + {"unicode_interninplace", unicode_interninplace, METH_O}, + {"unicode_internfromstring", unicode_internfromstring, METH_O}, + {"unicode_fromwidechar", unicode_fromwidechar, METH_VARARGS}, {"unicode_aswidechar", unicode_aswidechar, METH_VARARGS}, + {"unicode_aswidechar_null", unicode_aswidechar_null, METH_VARARGS}, {"unicode_aswidecharstring", unicode_aswidecharstring, METH_VARARGS}, + {"unicode_aswidecharstring_null",unicode_aswidecharstring_null,METH_VARARGS}, {"unicode_asucs4", unicode_asucs4, METH_VARARGS}, + {"unicode_asucs4copy", unicode_asucs4copy, METH_VARARGS}, + {"unicode_fromordinal", unicode_fromordinal, METH_VARARGS}, {"unicode_asutf8", unicode_asutf8, METH_VARARGS}, {"unicode_asutf8andsize", unicode_asutf8andsize, METH_VARARGS}, + {"unicode_asutf8andsize_null",unicode_asutf8andsize_null, METH_VARARGS}, {"unicode_decodeutf8", unicode_decodeutf8, METH_VARARGS}, {"unicode_decodeutf8stateful",unicode_decodeutf8stateful, METH_VARARGS}, + {"unicode_getdefaultencoding",unicode_getdefaultencoding, METH_NOARGS}, + {"unicode_transformdecimalandspacetoascii", unicode_transformdecimalandspacetoascii, METH_O}, {"unicode_concat", unicode_concat, METH_VARARGS}, {"unicode_splitlines", unicode_splitlines, METH_VARARGS}, {"unicode_split", unicode_split, METH_VARARGS}, diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 38f4758e6575d8..ae70a33eb4d1ed 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4247,6 +4247,7 @@ PyInit__testcapi(void) PyModule_AddObject(m, "ULLONG_MAX", PyLong_FromUnsignedLongLong(ULLONG_MAX)); PyModule_AddObject(m, "PY_SSIZE_T_MAX", PyLong_FromSsize_t(PY_SSIZE_T_MAX)); PyModule_AddObject(m, "PY_SSIZE_T_MIN", PyLong_FromSsize_t(PY_SSIZE_T_MIN)); + PyModule_AddObject(m, "SIZEOF_WCHAR_T", PyLong_FromSsize_t(sizeof(wchar_t))); PyModule_AddObject(m, "SIZEOF_TIME_T", PyLong_FromSsize_t(sizeof(time_t))); PyModule_AddObject(m, "Py_Version", PyLong_FromUnsignedLong(Py_Version)); Py_INCREF(&PyInstanceMethod_Type); From 09b7695f12f2b44d2df6898407d7e68dd9493ed4 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Thu, 4 May 2023 09:39:33 -0700 Subject: [PATCH 48/65] gh-91896: Deprecate collections.abc.ByteString (#102096) Co-authored-by: Alex Waygood Co-authored-by: Hugo van Kemenade --- Doc/library/collections.abc.rst | 6 +++++ Doc/library/typing.rst | 3 +-- Doc/whatsnew/3.12.rst | 5 ++++ Lib/_collections_abc.py | 23 +++++++++++++++++-- Lib/test/test_collections.py | 19 +++++++++++---- ...3-03-08-02-45-46.gh-issue-91896.kgON_a.rst | 1 + 6 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 669b7345499a78..43a3286ba832cf 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -273,6 +273,12 @@ Collections Abstract Base Classes -- Detailed Descriptions The index() method added support for *stop* and *start* arguments. + .. deprecated-removed:: 3.12 3.14 + The :class:`ByteString` ABC has been deprecated. + For use in typing, prefer a union, like ``bytes | bytearray``, or + :class:`collections.abc.Buffer`. + For use as an ABC, prefer :class:`Sequence` or :class:`collections.abc.Buffer`. + .. class:: Set MutableSet diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index c22fc0b28a50d0..162041fc7a846e 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -2139,8 +2139,7 @@ Corresponding to collections in :mod:`collections.abc` annotate arguments of any of the types mentioned above. .. deprecated:: 3.9 - :class:`collections.abc.ByteString` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. + Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``. .. class:: Collection(Sized, Iterable[T_co], Container[T_co]) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 3fe3310a26e59b..4a988bf4b993da 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -792,6 +792,11 @@ Pending Removal in Python 3.14 (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) +* Deprecated :class:`collections.abc.ByteString`. + Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. + For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. + (Contributed by Shantanu Jain in :gh:`91896`.) + * Creating immutable types (:data:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable bases using the C API. diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 2117190cf8b64a..601107d2d86771 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -1071,8 +1071,27 @@ def count(self, value): Sequence.register(range) Sequence.register(memoryview) - -class ByteString(Sequence): +class _DeprecateByteStringMeta(ABCMeta): + def __new__(cls, name, bases, namespace, **kwargs): + if name != "ByteString": + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 14), + ) + return super().__new__(cls, name, bases, namespace, **kwargs) + + def __instancecheck__(cls, instance): + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 14), + ) + return super().__instancecheck__(instance) + +class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): """This unifies bytes and bytearray. XXX Should add all their methods. diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 8fc28a6bf98e8f..bb8b352518ef3e 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1940,14 +1940,25 @@ def assert_index_same(seq1, seq2, index_args): def test_ByteString(self): for sample in [bytes, bytearray]: - self.assertIsInstance(sample(), ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(sample(), ByteString) self.assertTrue(issubclass(sample, ByteString)) for sample in [str, list, tuple]: - self.assertNotIsInstance(sample(), ByteString) + with self.assertWarns(DeprecationWarning): + self.assertNotIsInstance(sample(), ByteString) self.assertFalse(issubclass(sample, ByteString)) - self.assertNotIsInstance(memoryview(b""), ByteString) + with self.assertWarns(DeprecationWarning): + self.assertNotIsInstance(memoryview(b""), ByteString) self.assertFalse(issubclass(memoryview, ByteString)) - self.validate_abstract_methods(ByteString, '__getitem__', '__len__') + with self.assertWarns(DeprecationWarning): + self.validate_abstract_methods(ByteString, '__getitem__', '__len__') + + with self.assertWarns(DeprecationWarning): + class X(ByteString): pass + + with self.assertWarns(DeprecationWarning): + # No metaclass conflict + class Z(ByteString, Awaitable): pass def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: diff --git a/Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst b/Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst new file mode 100644 index 00000000000000..b5282d3d612916 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-08-02-45-46.gh-issue-91896.kgON_a.rst @@ -0,0 +1 @@ +Deprecate :class:`collections.abc.ByteString` From 8100be5535073a5442c2b8c68dcb2093ee69433d Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Thu, 4 May 2023 17:44:36 +0100 Subject: [PATCH 49/65] GH-81079: Add case_sensitive argument to `pathlib.Path.glob()` (GH-102710) This argument allows case-sensitive matching to be enabled on Windows, and case-insensitive matching to be enabled on Posix. Co-authored-by: Steve Dower --- Doc/library/pathlib.rst | 20 +++++++++-- Lib/pathlib.py | 34 +++++++++++-------- Lib/test/test_pathlib.py | 12 +++++++ ...3-03-15-00-37-43.gh-issue-81079.heTAod.rst | 2 ++ 4 files changed, 51 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 4847ac24c77513..14118127835bbe 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -855,7 +855,7 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.5 -.. method:: Path.glob(pattern) +.. method:: Path.glob(pattern, *, case_sensitive=None) Glob the given relative *pattern* in the directory represented by this path, yielding all matching files (of any kind):: @@ -876,6 +876,11 @@ call fails (for example because the path doesn't exist). PosixPath('setup.py'), PosixPath('test_pathlib.py')] + By default, or when the *case_sensitive* keyword-only argument is set to + ``None``, this method matches paths using platform-specific casing rules: + typically, case-sensitive on POSIX, and case-insensitive on Windows. + Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. + .. note:: Using the "``**``" pattern in large directory trees may consume an inordinate amount of time. @@ -886,6 +891,9 @@ call fails (for example because the path doesn't exist). Return only directories if *pattern* ends with a pathname components separator (:data:`~os.sep` or :data:`~os.altsep`). + .. versionadded:: 3.12 + The *case_sensitive* argument. + .. method:: Path.group() Return the name of the group owning the file. :exc:`KeyError` is raised @@ -1271,7 +1279,7 @@ call fails (for example because the path doesn't exist). .. versionadded:: 3.6 The *strict* argument (pre-3.6 behavior is strict). -.. method:: Path.rglob(pattern) +.. method:: Path.rglob(pattern, *, case_sensitive=None) Glob the given relative *pattern* recursively. This is like calling :func:`Path.glob` with "``**/``" added in front of the *pattern*, where @@ -1284,12 +1292,20 @@ call fails (for example because the path doesn't exist). PosixPath('setup.py'), PosixPath('test_pathlib.py')] + By default, or when the *case_sensitive* keyword-only argument is set to + ``None``, this method matches paths using platform-specific casing rules: + typically, case-sensitive on POSIX, and case-insensitive on Windows. + Set *case_sensitive* to ``True`` or ``False`` to override this behaviour. + .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob .. versionchanged:: 3.11 Return only directories if *pattern* ends with a pathname components separator (:data:`~os.sep` or :data:`~os.altsep`). + .. versionadded:: 3.12 + The *case_sensitive* argument. + .. method:: Path.rmdir() Remove this directory. The directory must be empty. diff --git a/Lib/pathlib.py b/Lib/pathlib.py index 8a1651c23d7f4d..f32e1e2d822834 100644 --- a/Lib/pathlib.py +++ b/Lib/pathlib.py @@ -62,7 +62,7 @@ def _is_case_sensitive(flavour): # @functools.lru_cache() -def _make_selector(pattern_parts, flavour): +def _make_selector(pattern_parts, flavour, case_sensitive): pat = pattern_parts[0] child_parts = pattern_parts[1:] if not pat: @@ -75,17 +75,17 @@ def _make_selector(pattern_parts, flavour): raise ValueError("Invalid pattern: '**' can only be an entire path component") else: cls = _WildcardSelector - return cls(pat, child_parts, flavour) + return cls(pat, child_parts, flavour, case_sensitive) class _Selector: """A selector matches a specific glob pattern part against the children of a given path.""" - def __init__(self, child_parts, flavour): + def __init__(self, child_parts, flavour, case_sensitive): self.child_parts = child_parts if child_parts: - self.successor = _make_selector(child_parts, flavour) + self.successor = _make_selector(child_parts, flavour, case_sensitive) self.dironly = True else: self.successor = _TerminatingSelector() @@ -108,8 +108,9 @@ def _select_from(self, parent_path, scandir): class _ParentSelector(_Selector): - def __init__(self, name, child_parts, flavour): - _Selector.__init__(self, child_parts, flavour) + + def __init__(self, name, child_parts, flavour, case_sensitive): + _Selector.__init__(self, child_parts, flavour, case_sensitive) def _select_from(self, parent_path, scandir): path = parent_path._make_child_relpath('..') @@ -119,10 +120,13 @@ def _select_from(self, parent_path, scandir): class _WildcardSelector(_Selector): - def __init__(self, pat, child_parts, flavour): - flags = re.NOFLAG if _is_case_sensitive(flavour) else re.IGNORECASE + def __init__(self, pat, child_parts, flavour, case_sensitive): + _Selector.__init__(self, child_parts, flavour, case_sensitive) + if case_sensitive is None: + # TODO: evaluate case-sensitivity of each directory in _select_from() + case_sensitive = _is_case_sensitive(flavour) + flags = re.NOFLAG if case_sensitive else re.IGNORECASE self.match = re.compile(fnmatch.translate(pat), flags=flags).fullmatch - _Selector.__init__(self, child_parts, flavour) def _select_from(self, parent_path, scandir): try: @@ -153,8 +157,8 @@ def _select_from(self, parent_path, scandir): class _RecursiveWildcardSelector(_Selector): - def __init__(self, pat, child_parts, flavour): - _Selector.__init__(self, child_parts, flavour) + def __init__(self, pat, child_parts, flavour, case_sensitive): + _Selector.__init__(self, child_parts, flavour, case_sensitive) def _iterate_directories(self, parent_path, scandir): yield parent_path @@ -819,7 +823,7 @@ def _scandir(self): # includes scandir(), which is used to implement glob(). return os.scandir(self) - def glob(self, pattern): + def glob(self, pattern, *, case_sensitive=None): """Iterate over this subtree and yield all existing files (of any kind, including directories) matching the given relative pattern. """ @@ -831,11 +835,11 @@ def glob(self, pattern): raise NotImplementedError("Non-relative patterns are unsupported") if pattern[-1] in (self._flavour.sep, self._flavour.altsep): pattern_parts.append('') - selector = _make_selector(tuple(pattern_parts), self._flavour) + selector = _make_selector(tuple(pattern_parts), self._flavour, case_sensitive) for p in selector.select_from(self): yield p - def rglob(self, pattern): + def rglob(self, pattern, *, case_sensitive=None): """Recursively yield all existing files (of any kind, including directories) matching the given relative pattern, anywhere in this subtree. @@ -846,7 +850,7 @@ def rglob(self, pattern): raise NotImplementedError("Non-relative patterns are unsupported") if pattern and pattern[-1] in (self._flavour.sep, self._flavour.altsep): pattern_parts.append('') - selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour) + selector = _make_selector(("**",) + tuple(pattern_parts), self._flavour, case_sensitive) for p in selector.select_from(self): yield p diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py index 424bb92a87d112..a932e03df4236d 100644 --- a/Lib/test/test_pathlib.py +++ b/Lib/test/test_pathlib.py @@ -1816,6 +1816,18 @@ def _check(glob, expected): else: _check(p.glob("*/"), ["dirA", "dirB", "dirC", "dirE", "linkB"]) + def test_glob_case_sensitive(self): + P = self.cls + def _check(path, pattern, case_sensitive, expected): + actual = {str(q) for q in path.glob(pattern, case_sensitive=case_sensitive)} + expected = {str(P(BASE, q)) for q in expected} + self.assertEqual(actual, expected) + path = P(BASE) + _check(path, "DIRB/FILE*", True, []) + _check(path, "DIRB/FILE*", False, ["dirB/fileB"]) + _check(path, "dirb/file*", True, []) + _check(path, "dirb/file*", False, ["dirB/fileB"]) + def test_rglob_common(self): def _check(glob, expected): self.assertEqual(set(glob), { P(BASE, q) for q in expected }) diff --git a/Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst b/Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst new file mode 100644 index 00000000000000..ef5690533985d5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-15-00-37-43.gh-issue-81079.heTAod.rst @@ -0,0 +1,2 @@ +Add *case_sensitive* keyword-only argument to :meth:`pathlib.Path.glob` and +:meth:`~pathlib.Path.rglob`. From d47cddfa27cbcbeb4dddb9263976d85c41241672 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Thu, 4 May 2023 19:55:48 +0300 Subject: [PATCH 50/65] gh-102500: Add PEP 688 and 698 to the 3.12 release highlights (#104174) --- Doc/whatsnew/3.12.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 4a988bf4b993da..4f952e2a37ef2f 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -68,13 +68,17 @@ Summary -- Release highlights New typing features: +* :pep:`688`: Making the buffer protocol accessible in Python + * :ref:`whatsnew312-pep692` +* :pep:`698`: Override Decorator for Static Typing + Important deprecations, removals or restrictions: -* :pep:`623`, Remove wstr from Unicode +* :pep:`623`: Remove wstr from Unicode -* :pep:`632`, Remove the ``distutils`` package. +* :pep:`632`: Remove the ``distutils`` package Improved Error Messages ======================= From f5c38382f9c40f0017cef086896a8160e313ac9e Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 4 May 2023 11:45:56 -0600 Subject: [PATCH 51/65] gh-103963: fix 'make regen-opcode' in out-of-tree builds (#104177) --- Makefile.pre.in | 4 +++- PCbuild/regen.targets | 2 +- Tools/build/generate_opcode_h.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index b285ef9e832db5..736a520d0e8fb6 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1376,9 +1376,11 @@ regen-opcode: $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ $(srcdir)/Lib/opcode.py \ $(srcdir)/Include/opcode.h.new \ - $(srcdir)/Include/internal/pycore_opcode.h.new + $(srcdir)/Include/internal/pycore_opcode.h.new \ + $(srcdir)/Include/internal/pycore_intrinsics.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_intrinsics.h $(srcdir)/Include/internal/pycore_intrinsics.h.new .PHONY: regen-token regen-token: diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index aeb7e2e185d9f8..107066817ba6b0 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -59,7 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index adcbaf2b8e08bf..5be981005725bf 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -233,4 +233,4 @@ def main(opcode_py, outfile='Include/opcode.h', if __name__ == '__main__': - main(sys.argv[1], sys.argv[2], sys.argv[3]) + main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) From 7d35c3121ade679dd6e8b4a0bac7b3702aee6921 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 4 May 2023 15:07:42 -0700 Subject: [PATCH 52/65] GH-103899: Provide a hint when accidentally calling a module (GH-103900) --- Lib/test/test_call.py | 32 ++++++++++++++ ...-04-26-15-14-23.gh-issue-103899.1pqKPF.rst | 3 ++ Objects/call.c | 44 ++++++++++++++++--- 3 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index aab7b1580eaf35..12759c53bb662c 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -10,6 +10,7 @@ import gc import contextlib import sys +import types class BadStr(str): @@ -202,6 +203,37 @@ def test_oldargs1_2_kw(self): msg = r"count\(\) takes no keyword arguments" self.assertRaisesRegex(TypeError, msg, [].count, x=2, y=2) + def test_object_not_callable(self): + msg = r"^'object' object is not callable$" + self.assertRaisesRegex(TypeError, msg, object()) + + def test_module_not_callable_no_suggestion_0(self): + msg = r"^'module' object is not callable$" + self.assertRaisesRegex(TypeError, msg, types.ModuleType("mod")) + + def test_module_not_callable_no_suggestion_1(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + mod.mod = 42 + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_no_suggestion_2(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + del mod.__name__ + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_no_suggestion_3(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + mod.__name__ = 42 + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_suggestion(self): + msg = r"^'module' object is not callable\. Did you mean: 'mod\.mod\(\.\.\.\)'\?$" + mod = types.ModuleType("mod") + mod.mod = lambda: ... + self.assertRaisesRegex(TypeError, msg, mod) class TestCallingConventions(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst new file mode 100644 index 00000000000000..c12a6b9cb841f2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-26-15-14-23.gh-issue-103899.1pqKPF.rst @@ -0,0 +1,3 @@ +Provide a helpful hint in the :exc:`TypeError` message when accidentally +calling a :term:`module` object that has a callable attribute of the same +name (such as :func:`dis.dis` or :class:`datetime.datetime`). diff --git a/Objects/call.c b/Objects/call.c index cf6e357a990441..0d548dcd5e1aed 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -157,6 +157,42 @@ PyObject_VectorcallDict(PyObject *callable, PyObject *const *args, return _PyObject_FastCallDictTstate(tstate, callable, args, nargsf, kwargs); } +static void +object_is_not_callable(PyThreadState *tstate, PyObject *callable) +{ + if (Py_IS_TYPE(callable, &PyModule_Type)) { + // >>> import pprint + // >>> pprint(thing) + // Traceback (most recent call last): + // File "", line 1, in + // TypeError: 'module' object is not callable. Did you mean: 'pprint.pprint(...)'? + PyObject *name = PyModule_GetNameObject(callable); + if (name == NULL) { + _PyErr_Clear(tstate); + goto basic_type_error; + } + PyObject *attr; + int res = _PyObject_LookupAttr(callable, name, &attr); + if (res < 0) { + _PyErr_Clear(tstate); + } + else if (res > 0 && PyCallable_Check(attr)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not callable. " + "Did you mean: '%U.%U(...)'?", + Py_TYPE(callable)->tp_name, name, name); + Py_DECREF(attr); + Py_DECREF(name); + return; + } + Py_XDECREF(attr); + Py_DECREF(name); + } +basic_type_error: + _PyErr_Format(tstate, PyExc_TypeError, "'%.200s' object is not callable", + Py_TYPE(callable)->tp_name); +} + PyObject * _PyObject_MakeTpCall(PyThreadState *tstate, PyObject *callable, @@ -171,9 +207,7 @@ _PyObject_MakeTpCall(PyThreadState *tstate, PyObject *callable, * temporary dictionary for keyword arguments (if any) */ ternaryfunc call = Py_TYPE(callable)->tp_call; if (call == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not callable", - Py_TYPE(callable)->tp_name); + object_is_not_callable(tstate, callable); return NULL; } @@ -322,9 +356,7 @@ _PyObject_Call(PyThreadState *tstate, PyObject *callable, else { call = Py_TYPE(callable)->tp_call; if (call == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not callable", - Py_TYPE(callable)->tp_name); + object_is_not_callable(tstate, callable); return NULL; } From 46361bb84332800bc3632688e6ef3b4dd4a48723 Mon Sep 17 00:00:00 2001 From: Michael Blahay Date: Thu, 4 May 2023 18:37:17 -0400 Subject: [PATCH 53/65] gh-68968: Correcting message display issue with assertEqual (#103937) --- Lib/test/test_unittest/test_assertions.py | 6 +- Lib/test/test_unittest/test_case.py | 60 +++++++++++++++++++ Lib/unittest/case.py | 31 +++++++--- ...3-04-27-18-46-31.gh-issue-68968.E3tnhy.rst | 1 + 4 files changed, 87 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst diff --git a/Lib/test/test_unittest/test_assertions.py b/Lib/test/test_unittest/test_assertions.py index 6557104b81fc0f..5c1a28ecda5b49 100644 --- a/Lib/test/test_unittest/test_assertions.py +++ b/Lib/test/test_unittest/test_assertions.py @@ -273,9 +273,9 @@ def testAssertDictEqual(self): def testAssertMultiLineEqual(self): self.assertMessages('assertMultiLineEqual', ("", "foo"), - [r"\+ foo$", "^oops$", - r"\+ foo$", - r"\+ foo : oops$"]) + [r"\+ foo\n$", "^oops$", + r"\+ foo\n$", + r"\+ foo\n : oops$"]) def testAssertLess(self): self.assertMessages('assertLess', (2, 1), diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index dd5ff6d553e61d..ed5eb5609a5dd1 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -1149,6 +1149,66 @@ def testAssertEqualSingleLine(self): error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + def testAssertEqualwithEmptyString(self): + '''Verify when there is an empty string involved, the diff output + does not treat the empty string as a single empty line. It should + instead be handled as a non-line. + ''' + sample_text = '' + revised_sample_text = 'unladen swallows fly quickly' + sample_text_error = '''\ ++ unladen swallows fly quickly +''' + try: + self.assertEqual(sample_text, revised_sample_text) + except self.failureException as e: + # need to remove the first line of the error message + error = str(e).split('\n', 1)[1] + self.assertEqual(sample_text_error, error) + + def testAssertEqualMultipleLinesMissingNewlineTerminator(self): + '''Verifying format of diff output from assertEqual involving strings + with multiple lines, but missing the terminating newline on both. + ''' + sample_text = 'laden swallows\nfly sloely' + revised_sample_text = 'laden swallows\nfly slowly' + sample_text_error = '''\ + laden swallows +- fly sloely +? ^ ++ fly slowly +? ^ +''' + try: + self.assertEqual(sample_text, revised_sample_text) + except self.failureException as e: + # need to remove the first line of the error message + error = str(e).split('\n', 1)[1] + self.assertEqual(sample_text_error, error) + + def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): + '''Verifying format of diff output from assertEqual involving strings + with multiple lines and mismatched newlines. The output should + include a - on it's own line to indicate the newline difference + between the two strings + ''' + sample_text = 'laden swallows\nfly sloely\n' + revised_sample_text = 'laden swallows\nfly slowly' + sample_text_error = '''\ + laden swallows +- fly sloely +? ^ ++ fly slowly +? ^ +-\x20 +''' + try: + self.assertEqual(sample_text, revised_sample_text) + except self.failureException as e: + # need to remove the first line of the error message + error = str(e).split('\n', 1)[1] + self.assertEqual(sample_text_error, error) + def testEqualityBytesWarning(self): if sys.flags.bytes_warning: def bytes_warning(): diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 018f22e7ce0c73..001b640dc43ad6 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -1217,19 +1217,34 @@ def assertCountEqual(self, first, second, msg=None): def assertMultiLineEqual(self, first, second, msg=None): """Assert that two multi-line strings are equal.""" - self.assertIsInstance(first, str, 'First argument is not a string') - self.assertIsInstance(second, str, 'Second argument is not a string') + self.assertIsInstance(first, str, "First argument is not a string") + self.assertIsInstance(second, str, "Second argument is not a string") if first != second: - # don't use difflib if the strings are too long + # Don't use difflib if the strings are too long if (len(first) > self._diffThreshold or len(second) > self._diffThreshold): self._baseAssertEqual(first, second, msg) - firstlines = first.splitlines(keepends=True) - secondlines = second.splitlines(keepends=True) - if len(firstlines) == 1 and first.strip('\r\n') == first: - firstlines = [first + '\n'] - secondlines = [second + '\n'] + + # Append \n to both strings if either is missing the \n. + # This allows the final ndiff to show the \n difference. The + # exception here is if the string is empty, in which case no + # \n should be added + first_presplit = first + second_presplit = second + if first and second: + if first[-1] != '\n' or second[-1] != '\n': + first_presplit += '\n' + second_presplit += '\n' + elif second and second[-1] != '\n': + second_presplit += '\n' + elif first and first[-1] != '\n': + first_presplit += '\n' + + firstlines = first_presplit.splitlines(keepends=True) + secondlines = second_presplit.splitlines(keepends=True) + + # Generate the message and diff, then raise the exception standardMsg = '%s != %s' % _common_shorten_repr(first, second) diff = '\n' + ''.join(difflib.ndiff(firstlines, secondlines)) standardMsg = self._truncateMessage(standardMsg, diff) diff --git a/Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst b/Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst new file mode 100644 index 00000000000000..bf29b64793b933 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-27-18-46-31.gh-issue-68968.E3tnhy.rst @@ -0,0 +1 @@ +Fixed garbled output of :meth:`~unittest.TestCase.assertEqual` when an input lacks final newline. From fa86a77589a06661fcebb806d36f3a7450e2aecf Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Thu, 4 May 2023 16:38:53 -0600 Subject: [PATCH 54/65] gh-104112: link from cached_property docs to method-caching FAQ (#104113) Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/library/functools.rst | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 29cbc87bf66d12..40f43f8b3519cd 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -110,18 +110,10 @@ The :mod:`functools` module defines the following functions: ``__slots__`` without including ``__dict__`` as one of the defined slots (as such classes don't provide a ``__dict__`` attribute at all). - If a mutable mapping is not available or if space-efficient key sharing - is desired, an effect similar to :func:`cached_property` can be achieved - by a stacking :func:`property` on top of :func:`cache`:: - - class DataSet: - def __init__(self, sequence_of_numbers): - self._data = sequence_of_numbers - - @property - @cache - def stdev(self): - return statistics.stdev(self._data) + If a mutable mapping is not available or if space-efficient key sharing is + desired, an effect similar to :func:`cached_property` can also be achieved by + stacking :func:`property` on top of :func:`lru_cache`. See + :ref:`faq-cache-method-calls` for more details on how this differs from :func:`cached_property`. .. versionadded:: 3.8 From ce871fdc3a02e8441ad73b13f9fced308a9d9ad1 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 4 May 2023 17:00:07 -0700 Subject: [PATCH 55/65] GH-104142: Fix _Py_RefcntAdd to respect immortality (GH-104143) --- Include/internal/pycore_object.h | 3 ++ Lib/test/test_builtin.py | 43 +++++++++++-------- ...-05-02-18-29-49.gh-issue-104142._5Et6I.rst | 2 + 3 files changed, 30 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b9e700ea280cbf..500b3eece68055 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -58,6 +58,9 @@ extern void _Py_DecRefTotal(PyInterpreterState *); // Increment reference count by n static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) { + if (_Py_IsImmortal(op)) { + return; + } #ifdef Py_REF_DEBUG _Py_AddRefTotal(_PyInterpreterState_GET(), n); #endif diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 04dd8ff3070c99..821710a7fa3286 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2372,24 +2372,31 @@ def __del__(self): @cpython_only class ImmortalTests(unittest.TestCase): - def test_immortal(self): - none_refcount = sys.getrefcount(None) - true_refcount = sys.getrefcount(True) - false_refcount = sys.getrefcount(False) - smallint_refcount = sys.getrefcount(100) - - # Assert that all of these immortal instances have large ref counts. - self.assertGreater(none_refcount, 2 ** 15) - self.assertGreater(true_refcount, 2 ** 15) - self.assertGreater(false_refcount, 2 ** 15) - self.assertGreater(smallint_refcount, 2 ** 15) - - # Confirm that the refcount doesn't change even with a new ref to them. - l = [None, True, False, 100] - self.assertEqual(sys.getrefcount(None), none_refcount) - self.assertEqual(sys.getrefcount(True), true_refcount) - self.assertEqual(sys.getrefcount(False), false_refcount) - self.assertEqual(sys.getrefcount(100), smallint_refcount) + + if sys.maxsize < (1 << 32): + IMMORTAL_REFCOUNT = (1 << 30) - 1 + else: + IMMORTAL_REFCOUNT = (1 << 32) - 1 + + IMMORTALS = (None, True, False, Ellipsis, NotImplemented, *range(-5, 257)) + + def assert_immortal(self, immortal): + with self.subTest(immortal): + self.assertEqual(sys.getrefcount(immortal), self.IMMORTAL_REFCOUNT) + + def test_immortals(self): + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + def test_list_repeat_respect_immortality(self): + refs = list(self.IMMORTALS) * 42 + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) + + def test_tuple_repeat_respect_immortality(self): + refs = tuple(self.IMMORTALS) * 42 + for immortal in self.IMMORTALS: + self.assert_immortal(immortal) class TestType(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst b/Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst new file mode 100644 index 00000000000000..6a19ae84057f4c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-05-02-18-29-49.gh-issue-104142._5Et6I.rst @@ -0,0 +1,2 @@ +Fix an issue where :class:`list` or :class:`tuple` repetition could fail to +respect :pep:`683`. From e5b8b19d99861c73ab76ee0175a685acf6082d7e Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Fri, 5 May 2023 12:27:25 +0900 Subject: [PATCH 56/65] gh-104106: Add gcc fallback of mkfifoat/mknodat for macOS (gh-104129) --- ...023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst | 1 + Modules/posixmodule.c | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst diff --git a/Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst b/Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst new file mode 100644 index 00000000000000..900e5bd61d6033 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2023-05-04-10-56-14.gh-issue-104106.-W9BJS.rst @@ -0,0 +1 @@ +Add gcc fallback of mkfifoat/mknodat for macOS. Patch by Dong-hee Na. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index dcb5e7a0e0408c..b395c265c72d0c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -175,6 +175,14 @@ # define HAVE_PWRITEV_RUNTIME (pwritev != NULL) # endif +# ifdef HAVE_MKFIFOAT +# define HAVE_MKFIFOAT_RUNTIME (mkfifoat != NULL) +# endif + +# ifdef HAVE_MKNODAT +# define HAVE_MKNODAT_RUNTIME (mknodat != NULL) +# endif + #endif #ifdef HAVE_FUTIMESAT @@ -4802,7 +4810,7 @@ os__path_isdir_impl(PyObject *module, PyObject *path) } Py_BEGIN_ALLOW_THREADS - if (_path.wide) { + if (_path.wide) { if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, &statInfo, sizeof(statInfo))) { if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { @@ -4899,7 +4907,7 @@ os__path_isfile_impl(PyObject *module, PyObject *path) } Py_BEGIN_ALLOW_THREADS - if (_path.wide) { + if (_path.wide) { if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, &statInfo, sizeof(statInfo))) { if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { @@ -4995,7 +5003,7 @@ os__path_exists_impl(PyObject *module, PyObject *path) } Py_BEGIN_ALLOW_THREADS - if (_path.wide) { + if (_path.wide) { if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, &statInfo, sizeof(statInfo))) { if (!(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) { @@ -5082,7 +5090,7 @@ os__path_islink_impl(PyObject *module, PyObject *path) } Py_BEGIN_ALLOW_THREADS - if (_path.wide) { + if (_path.wide) { if (_Py_GetFileInformationByName(_path.wide, FileStatBasicByNameInfo, &statInfo, sizeof(statInfo))) { slow_path = FALSE; From 163034515a81f137d1dd7d289dc048eb0f1cd424 Mon Sep 17 00:00:00 2001 From: sunmy2019 <59365878+sunmy2019@users.noreply.github.com> Date: Fri, 5 May 2023 17:54:03 +0800 Subject: [PATCH 57/65] gh-104190: fix ubsan crash (#104191) --- Modules/_testcapimodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index ae70a33eb4d1ed..376f04f23e324a 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4221,7 +4221,7 @@ PyInit__testcapi(void) return NULL; } int ret = PyModule_AddType(m, (PyTypeObject*)ObjExtraData_Type); - Py_DECREF(&ObjExtraData_Type); + Py_DECREF(ObjExtraData_Type); if (ret < 0) { return NULL; } From 81fc135f263e9ec5df3d967290b9665d7e100c7f Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Fri, 5 May 2023 11:34:13 +0100 Subject: [PATCH 58/65] gh-104051: fix crash in test_xxtestfuzz with -We (#104052) --- Modules/_xxtestfuzz/fuzzer.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Modules/_xxtestfuzz/fuzzer.c b/Modules/_xxtestfuzz/fuzzer.c index fb0c191d2c494d..37d402824853f0 100644 --- a/Modules/_xxtestfuzz/fuzzer.c +++ b/Modules/_xxtestfuzz/fuzzer.c @@ -526,13 +526,20 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { #if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_sre_compile) static int SRE_COMPILE_INITIALIZED = 0; if (!SRE_COMPILE_INITIALIZED && !init_sre_compile()) { - PyErr_Print(); - abort(); + if (!PyErr_ExceptionMatches(PyExc_DeprecationWarning)) { + PyErr_Print(); + abort(); + } + else { + PyErr_Clear(); + } } else { SRE_COMPILE_INITIALIZED = 1; } - rv |= _run_fuzz(data, size, fuzz_sre_compile); + if (SRE_COMPILE_INITIALIZED) { + rv |= _run_fuzz(data, size, fuzz_sre_compile); + } #endif #if !defined(_Py_FUZZ_ONE) || defined(_Py_FUZZ_fuzz_sre_match) static int SRE_MATCH_INITIALIZED = 0; From 2318bedb3645d2dfb56944553f01d6c434904c4b Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Fri, 5 May 2023 16:42:58 +0530 Subject: [PATCH 59/65] GH-103092: port `_asyncio` freelist to module state (#104196) --- Modules/_asynciomodule.c | 52 +++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 8b1a29b6d33e8b..fef34d65523870 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -18,6 +18,10 @@ module _asyncio /*[clinic end generated code: output=da39a3ee5e6b4b0d input=8fd17862aa989c69]*/ +#define FI_FREELIST_MAXLEN 255 + +typedef struct futureiterobject futureiterobject; + /* State of the _asyncio module */ typedef struct { PyTypeObject *FutureIterType; @@ -67,6 +71,9 @@ typedef struct { /* Counter for autogenerated Task names */ uint64_t task_name_counter; + + futureiterobject *fi_freelist; + Py_ssize_t fi_freelist_len; } asyncio_state; static inline asyncio_state * @@ -1574,28 +1581,24 @@ FutureObj_dealloc(PyObject *self) /*********************** Future Iterator **************************/ -typedef struct { +typedef struct futureiterobject { PyObject_HEAD FutureObj *future; } futureiterobject; -#define FI_FREELIST_MAXLEN 255 -static futureiterobject *fi_freelist = NULL; -static Py_ssize_t fi_freelist_len = 0; - - static void FutureIter_dealloc(futureiterobject *it) { PyTypeObject *tp = Py_TYPE(it); + asyncio_state *state = get_asyncio_state_by_def((PyObject *)it); PyObject_GC_UnTrack(it); tp->tp_clear((PyObject *)it); - if (fi_freelist_len < FI_FREELIST_MAXLEN) { - fi_freelist_len++; - it->future = (FutureObj*) fi_freelist; - fi_freelist = it; + if (state->fi_freelist_len < FI_FREELIST_MAXLEN) { + state->fi_freelist_len++; + it->future = (FutureObj*) state->fi_freelist; + state->fi_freelist = it; } else { PyObject_GC_Del(it); @@ -1799,17 +1802,12 @@ future_new_iter(PyObject *fut) futureiterobject *it; asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut); - if (!Future_Check(state, fut)) { - PyErr_BadInternalCall(); - return NULL; - } - ENSURE_FUTURE_ALIVE(state, fut) - if (fi_freelist_len) { - fi_freelist_len--; - it = fi_freelist; - fi_freelist = (futureiterobject*) it->future; + if (state->fi_freelist_len) { + state->fi_freelist_len--; + it = state->fi_freelist; + state->fi_freelist = (futureiterobject*) it->future; it->future = NULL; _Py_NewReference((PyObject*) it); } @@ -3556,22 +3554,22 @@ _asyncio_current_task_impl(PyObject *module, PyObject *loop) static void -module_free_freelists(void) +module_free_freelists(asyncio_state *state) { PyObject *next; PyObject *current; - next = (PyObject*) fi_freelist; + next = (PyObject*) state->fi_freelist; while (next != NULL) { - assert(fi_freelist_len > 0); - fi_freelist_len--; + assert(state->fi_freelist_len > 0); + state->fi_freelist_len--; current = next; next = (PyObject*) ((futureiterobject*) current)->future; PyObject_GC_Del(current); } - assert(fi_freelist_len == 0); - fi_freelist = NULL; + assert(state->fi_freelist_len == 0); + state->fi_freelist = NULL; } static int @@ -3603,7 +3601,7 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) Py_VISIT(state->context_kwname); // Visit freelist. - PyObject *next = (PyObject*) fi_freelist; + PyObject *next = (PyObject*) state->fi_freelist; while (next != NULL) { PyObject *current = next; Py_VISIT(current); @@ -3640,7 +3638,7 @@ module_clear(PyObject *mod) Py_CLEAR(state->context_kwname); - module_free_freelists(); + module_free_freelists(state); return 0; } From 721a78395d07c68351625aedb827a47dd4f48525 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 May 2023 13:32:00 +0200 Subject: [PATCH 60/65] gh-64658: Expand Argument Clinic return converter docs (#104175) --- Doc/howto/clinic.rst | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index 8a10fe327358c0..6ebc2d9b0a71a9 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -1033,19 +1033,36 @@ you're not permitted to use: Using a return converter ------------------------ -By default the impl function Argument Clinic generates for you returns ``PyObject *``. -But your C function often computes some C type, then converts it into the ``PyObject *`` +By default, the impl function Argument Clinic generates for you returns +:c:type:`PyObject * `. +But your C function often computes some C type, +then converts it into the :c:type:`!PyObject *` at the last moment. Argument Clinic handles converting your inputs from Python types into native C types—why not have it convert your return value from a native C type into a Python type too? That's what a "return converter" does. It changes your impl function to return some C type, then adds code to the generated (non-impl) function to handle converting -that value into the appropriate ``PyObject *``. +that value into the appropriate :c:type:`!PyObject *`. The syntax for return converters is similar to that of parameter converters. You specify the return converter like it was a return annotation on the -function itself. Return converters behave much the same as parameter converters; +function itself, using ``->`` notation. + +For example: + +.. code-block:: c + + /*[clinic input] + add -> int + + a: int + b: int + / + + [clinic start generated code]*/ + +Return converters behave much the same as parameter converters; they take arguments, the arguments are all keyword-only, and if you're not changing any of the default arguments you can omit the parentheses. @@ -1066,19 +1083,17 @@ Currently Argument Clinic supports only a few return converters: .. code-block:: none bool + double + float int - unsigned int long - unsigned int - size_t Py_ssize_t - float - double - DecodeFSDefault + size_t + unsigned int + unsigned long -None of these take parameters. For the first three, return -1 to indicate -error. For ``DecodeFSDefault``, the return type is ``const char *``; return a ``NULL`` -pointer to indicate an error. +None of these take parameters. +For all of these, return ``-1`` to indicate error. To see all the return converters Argument Clinic supports, along with their parameters (if any), From 5245cb64d9a898804edf683427ef4d4fd20ccaec Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 May 2023 13:40:18 +0200 Subject: [PATCH 61/65] gh-68395: Avoid naming conflicts by mangling variable names in Argument Clinic (#104065) Add all internally used variable names to CLINIC_PREFIXED_ARGS. --- Lib/test/clinic.test | 169 +++++++++++++++++++++++++++++++++++++++++ Tools/clinic/clinic.py | 13 +++- 2 files changed, 181 insertions(+), 1 deletion(-) diff --git a/Lib/test/clinic.test b/Lib/test/clinic.test index 53e5df5ba872ed..564205274edd73 100644 --- a/Lib/test/clinic.test +++ b/Lib/test/clinic.test @@ -4102,3 +4102,172 @@ exit: static PyObject * test_paramname_module_impl(PyObject *module, PyObject *mod) /*[clinic end generated code: output=4a2a849ecbcc8b53 input=afefe259667f13ba]*/ + +/*[clinic input] +mangle1 + + args: object + kwnames: object + return_value: object + _keywords: object + _parser: object + argsbuf: object + fastargs: object + nargs: object + noptargs: object + +[clinic start generated code]*/ + +PyDoc_STRVAR(mangle1__doc__, +"mangle1($module, /, args, kwnames, return_value, _keywords, _parser,\n" +" argsbuf, fastargs, nargs, noptargs)\n" +"--\n" +"\n"); + +#define MANGLE1_METHODDEF \ + {"mangle1", _PyCFunction_CAST(mangle1), METH_FASTCALL|METH_KEYWORDS, mangle1__doc__}, + +static PyObject * +mangle1_impl(PyObject *module, PyObject *args, PyObject *kwnames, + PyObject *return_value, PyObject *_keywords, PyObject *_parser, + PyObject *argsbuf, PyObject *fastargs, PyObject *nargs, + PyObject *noptargs); + +static PyObject * +mangle1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 9 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(args), &_Py_ID(kwnames), &_Py_ID(return_value), &_Py_ID(_keywords), &_Py_ID(_parser), &_Py_ID(argsbuf), &_Py_ID(fastargs), &_Py_ID(nargs), &_Py_ID(noptargs), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"args", "kwnames", "return_value", "_keywords", "_parser", "argsbuf", "fastargs", "nargs", "noptargs", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "mangle1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[9]; + PyObject *__clinic_args; + PyObject *__clinic_kwnames; + PyObject *__clinic_return_value; + PyObject *__clinic__keywords; + PyObject *__clinic__parser; + PyObject *__clinic_argsbuf; + PyObject *__clinic_fastargs; + PyObject *__clinic_nargs; + PyObject *__clinic_noptargs; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 9, 9, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + __clinic_kwnames = args[1]; + __clinic_return_value = args[2]; + __clinic__keywords = args[3]; + __clinic__parser = args[4]; + __clinic_argsbuf = args[5]; + __clinic_fastargs = args[6]; + __clinic_nargs = args[7]; + __clinic_noptargs = args[8]; + return_value = mangle1_impl(module, __clinic_args, __clinic_kwnames, __clinic_return_value, __clinic__keywords, __clinic__parser, __clinic_argsbuf, __clinic_fastargs, __clinic_nargs, __clinic_noptargs); + +exit: + return return_value; +} + +static PyObject * +mangle1_impl(PyObject *module, PyObject *args, PyObject *kwnames, + PyObject *return_value, PyObject *_keywords, PyObject *_parser, + PyObject *argsbuf, PyObject *fastargs, PyObject *nargs, + PyObject *noptargs) +/*[clinic end generated code: output=083e5076be9987c3 input=a3ed51bdedf8a3c7]*/ + +/*[clinic input] +mangle2 + + args: object + kwargs: object + return_value: object + +[clinic start generated code]*/ + +PyDoc_STRVAR(mangle2__doc__, +"mangle2($module, /, args, kwargs, return_value)\n" +"--\n" +"\n"); + +#define MANGLE2_METHODDEF \ + {"mangle2", _PyCFunction_CAST(mangle2), METH_FASTCALL|METH_KEYWORDS, mangle2__doc__}, + +static PyObject * +mangle2_impl(PyObject *module, PyObject *args, PyObject *kwargs, + PyObject *return_value); + +static PyObject * +mangle2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(args), &_Py_ID(kwargs), &_Py_ID(return_value), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"args", "kwargs", "return_value", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "mangle2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *__clinic_args; + PyObject *__clinic_kwargs; + PyObject *__clinic_return_value; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + __clinic_args = args[0]; + __clinic_kwargs = args[1]; + __clinic_return_value = args[2]; + return_value = mangle2_impl(module, __clinic_args, __clinic_kwargs, __clinic_return_value); + +exit: + return return_value; +} + +static PyObject * +mangle2_impl(PyObject *module, PyObject *args, PyObject *kwargs, + PyObject *return_value) +/*[clinic end generated code: output=2ebb62aaefe7590a input=391766fee51bad7a]*/ diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index d3e120c0e7a8ee..2746b24c333e66 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -43,7 +43,18 @@ NO_VARARG = "PY_SSIZE_T_MAX" CLINIC_PREFIX = "__clinic_" -CLINIC_PREFIXED_ARGS = {"args"} +CLINIC_PREFIXED_ARGS = { + "_keywords", + "_parser", + "args", + "argsbuf", + "fastargs", + "kwargs", + "kwnames", + "nargs", + "noptargs", + "return_value", +} class Unspecified: def __repr__(self): From d0b4abedfb8b0322df835065f85551d097cfecb8 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 May 2023 14:08:24 +0200 Subject: [PATCH 62/65] gh-64631: Test exception messages in cloned Argument Clinic funcs (#104167) --- Lib/test/test_clinic.py | 13 ++ Modules/_testclinic.c | 81 ++++++++++ Modules/clinic/_testclinic.c.h | 260 ++++++++++++++++++++++++++++++++- 3 files changed, 353 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 4abf739cf52ca3..660f7a1e296d8f 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1284,6 +1284,19 @@ def test_gh_99240_double_free(self): with self.assertRaisesRegex(TypeError, expected_error): ac_tester.gh_99240_double_free('a', '\0b') + def test_cloned_func_exception_message(self): + incorrect_arg = -1 # f1() and f2() accept a single str + with self.assertRaisesRegex(TypeError, "clone_f1"): + ac_tester.clone_f1(incorrect_arg) + with self.assertRaisesRegex(TypeError, "clone_f2"): + ac_tester.clone_f2(incorrect_arg) + + def test_cloned_func_with_converter_exception_message(self): + for name in "clone_with_conv_f1", "clone_with_conv_f2": + with self.subTest(name=name): + func = getattr(ac_tester, name) + self.assertEqual(func(), name) + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 91fdee24d328d9..6ff55a2755cf5a 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -9,6 +9,19 @@ #include "Python.h" + +// Used for clone_with_conv_f1 and clone_with_conv_v2 +typedef struct { + const char *name; +} custom_t; + +static int +custom_converter(PyObject *obj, custom_t *val) +{ + return 1; +} + + #include "clinic/_testclinic.c.h" @@ -1117,6 +1130,70 @@ gh_99240_double_free_impl(PyObject *module, char *a, char *b) } +/*[clinic input] +_testclinic.clone_f1 as clone_f1 + path: str +[clinic start generated code]*/ + +static PyObject * +clone_f1_impl(PyObject *module, const char *path) +/*[clinic end generated code: output=8c30b5620ba86715 input=9c614b7f025ebf70]*/ +{ + Py_RETURN_NONE; +} + + +/*[clinic input] +_testclinic.clone_f2 as clone_f2 = _testclinic.clone_f1 +[clinic start generated code]*/ + +static PyObject * +clone_f2_impl(PyObject *module, const char *path) +/*[clinic end generated code: output=6aa1c39bec3f5d9b input=1aaaf47d6ed2324a]*/ +{ + Py_RETURN_NONE; +} + + +/*[python input] +class custom_t_converter(CConverter): + type = 'custom_t' + converter = 'custom_converter' + + def pre_render(self): + self.c_default = f'''{{ + .name = "{self.function.name}", + }}''' + +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=b2fb801e99a06bf6]*/ + + +/*[clinic input] +_testclinic.clone_with_conv_f1 as clone_with_conv_f1 + path: custom_t = None +[clinic start generated code]*/ + +static PyObject * +clone_with_conv_f1_impl(PyObject *module, custom_t path) +/*[clinic end generated code: output=f7e030ffd5439cb0 input=bc77bc80dec3f46d]*/ +{ + return PyUnicode_FromString(path.name); +} + + +/*[clinic input] +_testclinic.clone_with_conv_f2 as clone_with_conv_f2 = _testclinic.clone_with_conv_f1 +[clinic start generated code]*/ + +static PyObject * +clone_with_conv_f2_impl(PyObject *module, custom_t path) +/*[clinic end generated code: output=9d7fdd6a75eecee4 input=cff459a205fa83bb]*/ +{ + return PyUnicode_FromString(path.name); +} + + static PyMethodDef tester_methods[] = { TEST_EMPTY_FUNCTION_METHODDEF OBJECTS_CONVERTER_METHODDEF @@ -1168,6 +1245,10 @@ static PyMethodDef tester_methods[] = { GH_32092_KW_PASS_METHODDEF GH_99233_REFCOUNT_METHODDEF GH_99240_DOUBLE_FREE_METHODDEF + CLONE_F1_METHODDEF + CLONE_F2_METHODDEF + CLONE_WITH_CONV_F1_METHODDEF + CLONE_WITH_CONV_F2_METHODDEF {NULL, NULL} }; diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 831f58ca650aab..cc69f5c3d2fe9f 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -2817,4 +2817,262 @@ gh_99240_double_free(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=e8211606b03d733a input=a9049054013a1b77]*/ + +PyDoc_STRVAR(clone_f1__doc__, +"clone_f1($module, /, path)\n" +"--\n" +"\n"); + +#define CLONE_F1_METHODDEF \ + {"clone_f1", _PyCFunction_CAST(clone_f1), METH_FASTCALL|METH_KEYWORDS, clone_f1__doc__}, + +static PyObject * +clone_f1_impl(PyObject *module, const char *path); + +static PyObject * +clone_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_f1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + const char *path; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("clone_f1", "argument 'path'", "str", args[0]); + goto exit; + } + Py_ssize_t path_length; + path = PyUnicode_AsUTF8AndSize(args[0], &path_length); + if (path == NULL) { + goto exit; + } + if (strlen(path) != (size_t)path_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = clone_f1_impl(module, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(clone_f2__doc__, +"clone_f2($module, /, path)\n" +"--\n" +"\n"); + +#define CLONE_F2_METHODDEF \ + {"clone_f2", _PyCFunction_CAST(clone_f2), METH_FASTCALL|METH_KEYWORDS, clone_f2__doc__}, + +static PyObject * +clone_f2_impl(PyObject *module, const char *path); + +static PyObject * +clone_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_f2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + const char *path; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("clone_f2", "argument 'path'", "str", args[0]); + goto exit; + } + Py_ssize_t path_length; + path = PyUnicode_AsUTF8AndSize(args[0], &path_length); + if (path == NULL) { + goto exit; + } + if (strlen(path) != (size_t)path_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = clone_f2_impl(module, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(clone_with_conv_f1__doc__, +"clone_with_conv_f1($module, /, path=None)\n" +"--\n" +"\n"); + +#define CLONE_WITH_CONV_F1_METHODDEF \ + {"clone_with_conv_f1", _PyCFunction_CAST(clone_with_conv_f1), METH_FASTCALL|METH_KEYWORDS, clone_with_conv_f1__doc__}, + +static PyObject * +clone_with_conv_f1_impl(PyObject *module, custom_t path); + +static PyObject * +clone_with_conv_f1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_with_conv_f1", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + custom_t path = { + .name = "clone_with_conv_f1", + }; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (!custom_converter(args[0], &path)) { + goto exit; + } +skip_optional_pos: + return_value = clone_with_conv_f1_impl(module, path); + +exit: + return return_value; +} + +PyDoc_STRVAR(clone_with_conv_f2__doc__, +"clone_with_conv_f2($module, /, path=None)\n" +"--\n" +"\n"); + +#define CLONE_WITH_CONV_F2_METHODDEF \ + {"clone_with_conv_f2", _PyCFunction_CAST(clone_with_conv_f2), METH_FASTCALL|METH_KEYWORDS, clone_with_conv_f2__doc__}, + +static PyObject * +clone_with_conv_f2_impl(PyObject *module, custom_t path); + +static PyObject * +clone_with_conv_f2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "clone_with_conv_f2", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + custom_t path = { + .name = "clone_with_conv_f2", + }; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (!custom_converter(args[0], &path)) { + goto exit; + } +skip_optional_pos: + return_value = clone_with_conv_f2_impl(module, path); + +exit: + return return_value; +} +/*[clinic end generated code: output=f58202a6e5df2d16 input=a9049054013a1b77]*/ From 45a9e3834a6ed20ee250e2e5a8583dffcef0eb73 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 5 May 2023 14:38:38 +0200 Subject: [PATCH 63/65] gh-64595: Argument Clinic: Touch source file if any output file changed (#104152) --- Lib/test/test_clinic.py | 7 ++++--- Tools/clinic/clinic.py | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 660f7a1e296d8f..6aaf4d1ed8d560 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -99,8 +99,9 @@ def test_eol(self): # the last line of the block got corrupted. c = clinic.Clinic(clinic.CLanguage(None), filename="file") raw = "/*[clinic]\nfoo\n[clinic]*/" - cooked = c.parse(raw).splitlines() - end_line = cooked[2].rstrip() + cooked, _ = c.parse(raw) + lines = cooked.splitlines() + end_line = lines[2].rstrip() # this test is redundant, it's just here explicitly to catch # the regression test so we don't forget what it looked like self.assertNotEqual(end_line, "[clinic]*/[clinic]*/") @@ -259,7 +260,7 @@ def _test_clinic(self, input, output): c = clinic.Clinic(language, filename="file") c.parsers['inert'] = InertParser(c) c.parsers['copy'] = CopyParser(c) - computed = c.parse(input) + computed, _ = c.parse(input) self.assertEqual(output, computed) def test_clinic_1(self): diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py index 2746b24c333e66..5f6e67e7d65a1d 100755 --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -1954,12 +1954,12 @@ def dump(self): return_converters = {} -def write_file(filename, new_contents): +def write_file(filename, new_contents, force=False): try: with open(filename, 'r', encoding="utf-8") as fp: old_contents = fp.read() - if old_contents == new_contents: + if old_contents == new_contents and not force: # no change: avoid modifying the file modification time return except FileNotFoundError: @@ -2123,6 +2123,8 @@ def parse(self, input): traceback.format_exc().rstrip()) printer.print_block(block) + clinic_out = [] + # these are destinations not buffers for name, destination in self.destinations.items(): if destination.type == 'suppress': @@ -2162,10 +2164,11 @@ def parse(self, input): block.input = 'preserve\n' printer_2 = BlockPrinter(self.language) printer_2.print_block(block, core_includes=True) - write_file(destination.filename, printer_2.f.getvalue()) + pair = destination.filename, printer_2.f.getvalue() + clinic_out.append(pair) continue - return printer.f.getvalue() + return printer.f.getvalue(), clinic_out def _module_and_class(self, fields): @@ -2221,9 +2224,13 @@ def parse_file(filename, *, verify=True, output=None): return clinic = Clinic(language, verify=verify, filename=filename) - cooked = clinic.parse(raw) + src_out, clinic_out = clinic.parse(raw) - write_file(output, cooked) + # If clinic output changed, force updating the source file as well. + force = bool(clinic_out) + write_file(output, src_out, force=force) + for fn, data in clinic_out: + write_file(fn, data) def compute_checksum(input, length=None): From a0df9ee8fc77443510ab7e9ba8fd830f255a8fec Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Fri, 5 May 2023 17:53:07 +0100 Subject: [PATCH 64/65] GH-96803: Add three C-API functions to make _PyInterpreterFrame less opaque for users of PEP 523. (GH-96849) --- Include/cpython/frameobject.h | 17 +++++++++++++++++ Include/internal/pycore_frame.h | 2 -- ...22-09-15-15-21-34.gh-issue-96803.ynBKIS.rst | 6 ++++++ Modules/_tracemalloc.c | 3 ++- Objects/frameobject.c | 2 +- Objects/genobject.c | 3 ++- Python/ceval.c | 3 ++- Python/frame.c | 18 +++++++++++++++++- Python/traceback.c | 2 +- 9 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 4e19535c656f2c..6f3efe36ede5d8 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -4,6 +4,8 @@ # error "this header file must not be included directly" #endif +struct _PyInterpreterFrame; + /* Standard object interface */ PyAPI_FUNC(PyFrameObject *) PyFrame_New(PyThreadState *, PyCodeObject *, @@ -27,3 +29,18 @@ PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + +/* The following functions are for use by debuggers and other tools + * implementing custom frame evaluators with PEP 523. */ + +/* Returns the code object of the frame (strong reference). + * Does not raise an exception. */ +PyAPI_FUNC(PyCodeObject *) PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame); + +/* Returns a byte ofsset into the last executed instruction. + * Does not raise an exception. */ +PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame); + +/* Returns the currently executing line number, or -1 if there is no line number. + * Does not raise an exception. */ +PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame); diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index d8d7fe9ef2ebde..3d3cbbff7aae81 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -265,8 +265,6 @@ _PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_l return new_frame; } -int _PyInterpreterFrame_GetLine(_PyInterpreterFrame *frame); - static inline PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) { diff --git a/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst b/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst new file mode 100644 index 00000000000000..6fc56d2249f581 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2022-09-15-15-21-34.gh-issue-96803.ynBKIS.rst @@ -0,0 +1,6 @@ +Add unstable C-API functions to get the code object, lasti and line number from +the internal ``_PyInterpreterFrame`` in the limited API. The functions are: + +* ``PyCodeObject * PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame)`` +* ``int PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame)`` +* ``int PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame)`` diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index d69c5636486da9..c5714d5e7d5a0f 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -7,6 +7,7 @@ #include "pycore_runtime.h" // _Py_ID() #include "pycore_traceback.h" #include +#include "frameobject.h" // _PyInterpreterFrame_GetLine #include // malloc() @@ -257,7 +258,7 @@ static void tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) { frame->filename = &_Py_STR(anon_unknown); - int lineno = _PyInterpreterFrame_GetLine(pyframe); + int lineno = PyUnstable_InterpreterFrame_GetLine(pyframe); if (lineno < 0) { lineno = 0; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index ef0070199ab2c0..d0eca447c012ea 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -38,7 +38,7 @@ PyFrame_GetLineNumber(PyFrameObject *f) return f->f_lineno; } else { - return _PyInterpreterFrame_GetLine(f->f_frame); + return PyUnstable_InterpreterFrame_GetLine(f->f_frame); } } diff --git a/Objects/genobject.c b/Objects/genobject.c index 6316fa9865fe65..937d497753e970 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -12,6 +12,7 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include "structmember.h" // PyMemberDef #include "opcode.h" // SEND +#include "frameobject.h" // _PyInterpreterFrame_GetLine #include "pystats.h" static PyObject *gen_close(PyGenObject *, PyObject *); @@ -1322,7 +1323,7 @@ compute_cr_origin(int origin_depth, _PyInterpreterFrame *current_frame) frame = current_frame; for (int i = 0; i < frame_count; ++i) { PyCodeObject *code = frame->f_code; - int line = _PyInterpreterFrame_GetLine(frame); + int line = PyUnstable_InterpreterFrame_GetLine(frame); PyObject *frameinfo = Py_BuildValue("OiO", code->co_filename, line, code->co_name); if (!frameinfo) { diff --git a/Python/ceval.c b/Python/ceval.c index 958689debc87f8..56a3b123f46331 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -27,6 +27,7 @@ #include "pycore_dict.h" #include "dictobject.h" #include "pycore_frame.h" +#include "frameobject.h" // _PyInterpreterFrame_GetLine #include "opcode.h" #include "pydtrace.h" #include "setobject.h" @@ -785,7 +786,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _PyErr_Format(tstate, PyExc_SystemError, "%U:%d: unknown opcode %d", frame->f_code->co_filename, - _PyInterpreterFrame_GetLine(frame), + PyUnstable_InterpreterFrame_GetLine(frame), opcode); goto error; diff --git a/Python/frame.c b/Python/frame.c index c2c0be30113912..d792b92fa57560 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -144,8 +144,24 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) Py_DECREF(frame->f_funcobj); } +/* Unstable API functions */ + +PyCodeObject * +PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame) +{ + PyCodeObject *code = frame->f_code; + Py_INCREF(code); + return code; +} + +int +PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame) +{ + return _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); +} + int -_PyInterpreterFrame_GetLine(_PyInterpreterFrame *frame) +PyUnstable_InterpreterFrame_GetLine(_PyInterpreterFrame *frame) { int addr = _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); return PyCode_Addr2Line(frame->f_code, addr); diff --git a/Python/traceback.c b/Python/traceback.c index 097f69c76abfe1..b2479542047308 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -1180,7 +1180,7 @@ dump_frame(int fd, _PyInterpreterFrame *frame) PUTS(fd, "???"); } - int lineno = _PyInterpreterFrame_GetLine(frame); + int lineno = PyUnstable_InterpreterFrame_GetLine(frame); PUTS(fd, ", line "); if (lineno >= 0) { _Py_DumpDecimal(fd, (size_t)lineno); From b9797417315cc2d1700cb2d427685016d3380711 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 5 May 2023 10:38:47 -0700 Subject: [PATCH 65/65] gh-103533: Use PEP 669 APIs for cprofile (GH-103534) --- Lib/test/test_cprofile.py | 19 +- ...-04-14-06-32-54.gh-issue-103533.n_AfcS.rst | 1 + Modules/_lsprof.c | 252 +++++++++++++----- Tools/c-analyzer/cpython/ignored.tsv | 1 + 4 files changed, 200 insertions(+), 73 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 98648528bc81f2..484b8f8e3a365c 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -25,7 +25,6 @@ def test_bad_counter_during_dealloc(self): with support.catch_unraisable_exception() as cm: obj = _lsprof.Profiler(lambda: int) obj.enable() - obj = _lsprof.Profiler(1) obj.disable() obj.clear() @@ -37,10 +36,11 @@ def test_profile_enable_disable(self): self.addCleanup(prof.disable) prof.enable() - self.assertIs(sys.getprofile(), prof) + self.assertEqual( + sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), "cProfile") prof.disable() - self.assertIs(sys.getprofile(), None) + self.assertIs(sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), None) def test_profile_as_context_manager(self): prof = self.profilerclass() @@ -53,10 +53,19 @@ def test_profile_as_context_manager(self): # profile should be set as the global profiler inside the # with-block - self.assertIs(sys.getprofile(), prof) + self.assertEqual( + sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), "cProfile") # profile shouldn't be set once we leave the with-block. - self.assertIs(sys.getprofile(), None) + self.assertIs(sys.monitoring.get_tool(sys.monitoring.PROFILER_ID), None) + + def test_second_profiler(self): + pr = self.profilerclass() + pr2 = self.profilerclass() + pr.enable() + self.assertRaises(ValueError, pr2.enable) + pr.disable() + class TestCommandLine(unittest.TestCase): def test_sort(self): diff --git a/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst b/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst new file mode 100644 index 00000000000000..1008ea076c71a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-14-06-32-54.gh-issue-103533.n_AfcS.rst @@ -0,0 +1 @@ +Update :mod:`cProfile` to use PEP 669 API diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 83d034ae7eed78..a7ce328cb5307a 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -49,6 +49,8 @@ typedef struct { int flags; PyObject *externalTimer; double externalTimerUnit; + int tool_id; + PyObject* missing; } ProfilerObject; #define POF_ENABLED 0x001 @@ -399,64 +401,6 @@ ptrace_leave_call(PyObject *self, void *key) pObj->freelistProfilerContext = pContext; } -static int -profiler_callback(PyObject *self, PyFrameObject *frame, int what, - PyObject *arg) -{ - switch (what) { - - /* the 'frame' of a called function is about to start its execution */ - case PyTrace_CALL: - { - PyCodeObject *code = PyFrame_GetCode(frame); - ptrace_enter_call(self, (void *)code, (PyObject *)code); - Py_DECREF(code); - break; - } - - /* the 'frame' of a called function is about to finish - (either normally or with an exception) */ - case PyTrace_RETURN: - { - PyCodeObject *code = PyFrame_GetCode(frame); - ptrace_leave_call(self, (void *)code); - Py_DECREF(code); - break; - } - - /* case PyTrace_EXCEPTION: - If the exception results in the function exiting, a - PyTrace_RETURN event will be generated, so we don't need to - handle it. */ - - /* the Python function 'frame' is issuing a call to the built-in - function 'arg' */ - case PyTrace_C_CALL: - if ((((ProfilerObject *)self)->flags & POF_BUILTINS) - && PyCFunction_Check(arg)) { - ptrace_enter_call(self, - ((PyCFunctionObject *)arg)->m_ml, - arg); - } - break; - - /* the call to the built-in function 'arg' is returning into its - caller 'frame' */ - case PyTrace_C_RETURN: /* ...normally */ - case PyTrace_C_EXCEPTION: /* ...with an exception set */ - if ((((ProfilerObject *)self)->flags & POF_BUILTINS) - && PyCFunction_Check(arg)) { - ptrace_leave_call(self, - ((PyCFunctionObject *)arg)->m_ml); - } - break; - - default: - break; - } - return 0; -} - static int pending_exception(ProfilerObject *pObj) { @@ -650,6 +594,99 @@ setBuiltins(ProfilerObject *pObj, int nvalue) return 0; } +PyObject* pystart_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + PyObject* code = args[0]; + ptrace_enter_call((PyObject*)self, (void *)code, (PyObject *)code); + + Py_RETURN_NONE; +} + +PyObject* pyreturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + PyObject* code = args[0]; + ptrace_leave_call((PyObject*)self, (void *)code); + + Py_RETURN_NONE; +} + +PyObject* get_cfunc_from_callable(PyObject* callable, PyObject* self_arg, PyObject* missing) +{ + // return a new reference + if (PyCFunction_Check(callable)) { + Py_INCREF(callable); + return (PyObject*)((PyCFunctionObject *)callable); + } + if (Py_TYPE(callable) == &PyMethodDescr_Type) { + /* For backwards compatibility need to + * convert to builtin method */ + + /* If no arg, skip */ + if (self_arg == missing) { + return NULL; + } + PyObject *meth = Py_TYPE(callable)->tp_descr_get( + callable, self_arg, (PyObject*)Py_TYPE(self_arg)); + if (meth == NULL) { + return NULL; + } + if (PyCFunction_Check(meth)) { + return (PyObject*)((PyCFunctionObject *)meth); + } + } + return NULL; +} + +PyObject* ccall_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + if (self->flags & POF_BUILTINS) { + PyObject* callable = args[2]; + PyObject* self_arg = args[3]; + + PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); + + if (cfunc) { + ptrace_enter_call((PyObject*)self, + ((PyCFunctionObject *)cfunc)->m_ml, + cfunc); + Py_DECREF(cfunc); + } + } + Py_RETURN_NONE; +} + +PyObject* creturn_callback(ProfilerObject* self, PyObject *const *args, Py_ssize_t size) +{ + if (self->flags & POF_BUILTINS) { + PyObject* callable = args[2]; + PyObject* self_arg = args[3]; + + PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); + + if (cfunc) { + ptrace_leave_call((PyObject*)self, + ((PyCFunctionObject *)cfunc)->m_ml); + Py_DECREF(cfunc); + } + } + Py_RETURN_NONE; +} + +static const struct { + int event; + const char* callback_method; +} callback_table[] = { + {PY_MONITORING_EVENT_PY_START, "_pystart_callback"}, + {PY_MONITORING_EVENT_PY_RESUME, "_pystart_callback"}, + {PY_MONITORING_EVENT_PY_RETURN, "_pyreturn_callback"}, + {PY_MONITORING_EVENT_PY_YIELD, "_pyreturn_callback"}, + {PY_MONITORING_EVENT_PY_UNWIND, "_pyreturn_callback"}, + {PY_MONITORING_EVENT_CALL, "_ccall_callback"}, + {PY_MONITORING_EVENT_C_RETURN, "_creturn_callback"}, + {PY_MONITORING_EVENT_C_RAISE, "_creturn_callback"}, + {0, NULL} +}; + PyDoc_STRVAR(enable_doc, "\ enable(subcalls=True, builtins=True)\n\ \n\ @@ -666,6 +703,8 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) int subcalls = -1; int builtins = -1; static char *kwlist[] = {"subcalls", "builtins", 0}; + int all_events = 0; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|pp:enable", kwlist, &subcalls, &builtins)) return NULL; @@ -673,11 +712,37 @@ profiler_enable(ProfilerObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, profiler_callback, (PyObject*)self) < 0) { + PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); + if (!monitoring) { + return NULL; + } + + if (PyObject_CallMethod(monitoring, "use_tool_id", "is", self->tool_id, "cProfile") == NULL) { + PyErr_Format(PyExc_ValueError, "Another profiling tool is already active"); + Py_DECREF(monitoring); + return NULL; + } + + for (int i = 0; callback_table[i].callback_method; i++) { + PyObject* callback = PyObject_GetAttrString((PyObject*)self, callback_table[i].callback_method); + if (!callback) { + Py_DECREF(monitoring); + return NULL; + } + Py_XDECREF(PyObject_CallMethod(monitoring, "register_callback", "iiO", self->tool_id, + (1 << callback_table[i].event), + callback)); + Py_DECREF(callback); + all_events |= (1 << callback_table[i].event); + } + + if (!PyObject_CallMethod(monitoring, "set_events", "ii", self->tool_id, all_events)) { + Py_DECREF(monitoring); return NULL; } + Py_DECREF(monitoring); + self->flags |= POF_ENABLED; Py_RETURN_NONE; } @@ -707,13 +772,44 @@ Stop collecting profiling information.\n\ static PyObject* profiler_disable(ProfilerObject *self, PyObject* noarg) { - PyThreadState *tstate = _PyThreadState_GET(); - if (_PyEval_SetProfile(tstate, NULL, NULL) < 0) { - return NULL; + if (self->flags & POF_ENABLED) { + PyObject* result = NULL; + PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); + + if (!monitoring) { + return NULL; + } + + for (int i = 0; callback_table[i].callback_method; i++) { + result = PyObject_CallMethod(monitoring, "register_callback", "iiO", self->tool_id, + (1 << callback_table[i].event), Py_None); + if (!result) { + Py_DECREF(monitoring); + return NULL; + } + Py_DECREF(result); + } + + result = PyObject_CallMethod(monitoring, "set_events", "ii", self->tool_id, 0); + if (!result) { + Py_DECREF(monitoring); + return NULL; + } + Py_DECREF(result); + + result = PyObject_CallMethod(monitoring, "free_tool_id", "i", self->tool_id); + if (!result) { + Py_DECREF(monitoring); + return NULL; + } + Py_DECREF(result); + + Py_DECREF(monitoring); + + self->flags &= ~POF_ENABLED; + flush_unmatched(self); } - self->flags &= ~POF_ENABLED; - flush_unmatched(self); if (pending_exception(self)) { return NULL; } @@ -778,17 +874,37 @@ profiler_init(ProfilerObject *pObj, PyObject *args, PyObject *kw) return -1; pObj->externalTimerUnit = timeunit; Py_XSETREF(pObj->externalTimer, Py_XNewRef(timer)); + pObj->tool_id = PY_MONITORING_PROFILER_ID; + + PyObject* monitoring = _PyImport_GetModuleAttrString("sys", "monitoring"); + if (!monitoring) { + return -1; + } + pObj->missing = PyObject_GetAttrString(monitoring, "MISSING"); + if (!pObj->missing) { + Py_DECREF(monitoring); + return -1; + } + Py_DECREF(monitoring); return 0; } static PyMethodDef profiler_methods[] = { _LSPROF_PROFILER_GETSTATS_METHODDEF - {"enable", _PyCFunction_CAST(profiler_enable), + {"enable", _PyCFunction_CAST(profiler_enable), METH_VARARGS | METH_KEYWORDS, enable_doc}, - {"disable", (PyCFunction)profiler_disable, + {"disable", (PyCFunction)profiler_disable, METH_NOARGS, disable_doc}, - {"clear", (PyCFunction)profiler_clear, + {"clear", (PyCFunction)profiler_clear, METH_NOARGS, clear_doc}, + {"_pystart_callback", _PyCFunction_CAST(pystart_callback), + METH_FASTCALL, NULL}, + {"_pyreturn_callback", _PyCFunction_CAST(pyreturn_callback), + METH_FASTCALL, NULL}, + {"_ccall_callback", _PyCFunction_CAST(ccall_callback), + METH_FASTCALL, NULL}, + {"_creturn_callback", _PyCFunction_CAST(creturn_callback), + METH_FASTCALL, NULL}, {NULL, NULL} }; diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index fee493ff7f1666..7ba116dcb171cf 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -216,6 +216,7 @@ Modules/_io/_iomodule.c - static_types - Modules/_io/textio.c - encodefuncs - Modules/_io/winconsoleio.c - _PyWindowsConsoleIO_Type - Modules/_localemodule.c - langinfo_constants - +Modules/_lsprof.c - callback_table - Modules/_pickle.c - READ_WHOLE_LINE - Modules/_sqlite/module.c - error_codes - Modules/_sre/sre.c pattern_repr flag_names -