Update to greenlet 2.0

.github/workflows/ci.yml

@@ -198,7 +198,7 @@ jobs:
           pip install -q -U 'faulthandler; python_version == "2.7" and platform_python_implementation == "CPython"'
           pip install -q -U 'cffi;platform_python_implementation=="CPython"'
           pip install -q -U 'cython>=3.0a9'
-          pip install 'greenlet>=1.0a1,<2;platform_python_implementation=="CPython"'
+          pip install 'greenlet>=2.0rc4 ;platform_python_implementation=="CPython"'
 
       - name: Build gevent
         run: |
@@ -393,7 +393,7 @@ jobs:
           pip install -q -U 'faulthandler; python_version == "2.7" and platform_python_implementation == "CPython"'
           pip install -q -U 'cffi;platform_python_implementation=="CPython"'
           pip install -q -U 'cython>=3.0a5'
-          pip install 'greenlet>=1.0a1,<2;platform_python_implementation=="CPython"'
+          pip install 'greenlet>=2.0rc4;platform_python_implementation=="CPython"'
 
       - name: build libs and gevent
         env:

appveyor.yml

@@ -46,6 +46,12 @@ environment:
     # a later point release.
 
     # 64-bit
+    - PYTHON: "C:\\Python311-x64"
+      PYTHON_VERSION: "3.11.0"
+      PYTHON_ARCH: "64"
+      PYTHON_EXE: python
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
+
     - PYTHON: "C:\\pypy3.7-v7.3.7-win64"
       PYTHON_ID: "pypy3"
       PYTHON_EXE: pypy3w
@@ -81,10 +87,6 @@ environment:
       PYTHON_EXE: python
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
 
-    - PYTHON: "C:\\Python27-x64"
-      PYTHON_VERSION: "2.7.x" # currently 2.7.13
-      PYTHON_ARCH: "64"
-      PYTHON_EXE: python
 
     - PYTHON: "C:\\Python38-x64"
       PYTHON_VERSION: "3.8.x"
@@ -138,6 +140,20 @@ environment:
       PYTHON_EXE: python
       GWHEEL_ONLY: true
 
+    - PYTHON: "C:\\Python27-x64"
+      PYTHON_VERSION: "2.7.x" # currently 2.7.13
+      PYTHON_ARCH: "64"
+      PYTHON_EXE: python
+      # greenlet 2.0 is evincing a warning (probably?)
+      # on shutdown, leading to the dreaded error:
+      #   Fatal Python error: PyImport_GetModuleDict: no module
+      #   dictionary!
+      # in some tests. This is hard to debug remotely, and as support
+      # for 2.7 is winding down quickly (hey, we're only two years
+      # late to the party) I'm not specifically going to try to debug
+      # it. We'll just provide a binary wheel still.
+      GWHEEL_ONLY: true
+
     # Also test a Python version not pre-installed
     # See: https://github.com/ogrisel/python-appveyor-demo/issues/10
 

deps/greenlet/greenlet.h

@@ -5,6 +5,7 @@
 #ifndef Py_GREENLETOBJECT_H
 #define Py_GREENLETOBJECT_H
 
+
 #include <Python.h>
 
 #ifdef __cplusplus
@@ -14,60 +15,24 @@ extern "C" {
 /* This is deprecated and undocumented. It does not change. */
 #define GREENLET_VERSION "1.0.0"
 
-#if PY_VERSION_HEX >= 0x30B00A6
-#  define GREENLET_PY311 1
-   /* _PyInterpreterFrame moved to the internal C API in Python 3.11 */
-#  include <internal/pycore_frame.h>
-#else
-#  define GREENLET_PY311 0
-#  define _PyCFrame CFrame
+#ifndef GREENLET_MODULE
+#define implementation_ptr_t void*
 #endif
 
 typedef struct _greenlet {
     PyObject_HEAD
-    char* stack_start;
-    char* stack_stop;
-    char* stack_copy;
-    intptr_t stack_saved;
-    struct _greenlet* stack_prev;
-    struct _greenlet* parent;
-    PyObject* run_info;
-    struct _frame* top_frame;
-    int recursion_depth;
-#if GREENLET_PY311
-    _PyInterpreterFrame *current_frame;
-    _PyStackChunk *datastack_chunk;
-    PyObject **datastack_top;
-    PyObject **datastack_limit;
-#endif
     PyObject* weakreflist;
-#if PY_VERSION_HEX >= 0x030700A3
-    _PyErr_StackItem* exc_info;
-    _PyErr_StackItem exc_state;
-#else
-    PyObject* exc_type;
-    PyObject* exc_value;
-    PyObject* exc_traceback;
-#endif
     PyObject* dict;
-#if PY_VERSION_HEX >= 0x030700A3
-    PyObject* context;
-#endif
-#if PY_VERSION_HEX >= 0x30A00B1
-    _PyCFrame* cframe;
-#endif
+    implementation_ptr_t pimpl;
 } PyGreenlet;
 
-#define PyGreenlet_Check(op) PyObject_TypeCheck(op, &PyGreenlet_Type)
-#define PyGreenlet_MAIN(op) (((PyGreenlet*)(op))->stack_stop == (char*)-1)
-#define PyGreenlet_STARTED(op) (((PyGreenlet*)(op))->stack_stop != NULL)
-#define PyGreenlet_ACTIVE(op) (((PyGreenlet*)(op))->stack_start != NULL)
-#define PyGreenlet_GET_PARENT(op) (((PyGreenlet*)(op))->parent)
+#define PyGreenlet_Check(op) (op && PyObject_TypeCheck(op, &PyGreenlet_Type))
+
 
 /* C API functions */
 
 /* Total number of symbols that are exported */
-#define PyGreenlet_API_pointers 8
+#define PyGreenlet_API_pointers 12
 
 #define PyGreenlet_Type_NUM 0
 #define PyExc_GreenletError_NUM 1
@@ -79,6 +44,11 @@ typedef struct _greenlet {
 #define PyGreenlet_Switch_NUM 6
 #define PyGreenlet_SetParent_NUM 7
 
+#define PyGreenlet_MAIN_NUM 8
+#define PyGreenlet_STARTED_NUM 9
+#define PyGreenlet_ACTIVE_NUM 10
+#define PyGreenlet_GET_PARENT_NUM 11
+
 #ifndef GREENLET_MODULE
 /* This section is used by modules that uses the greenlet C API */
 static void** _PyGreenlet_API = NULL;
@@ -144,6 +114,39 @@ static void** _PyGreenlet_API = NULL;
         (*(int (*)(PyGreenlet * greenlet, PyGreenlet * nparent)) \
              _PyGreenlet_API[PyGreenlet_SetParent_NUM])
 
+/*
+ * PyGreenlet_GetParent(PyObject* greenlet)
+ *
+ * return greenlet.parent;
+ *
+ * This could return NULL even if there is no exception active.
+ * If it does not return NULL, you are responsible for decrementing the
+ * reference count.
+ */
+#     define PyGreenlet_GetParent                                    \
+    (*(PyGreenlet* (*)(PyGreenlet*))                                 \
+     _PyGreenlet_API[PyGreenlet_GET_PARENT_NUM])
+
+/*
+ * deprecated, undocumented alias.
+ */
+#     define PyGreenlet_GET_PARENT PyGreenlet_GetParent
+
+#     define PyGreenlet_MAIN                                         \
+    (*(int (*)(PyGreenlet*))                                         \
+     _PyGreenlet_API[PyGreenlet_MAIN_NUM])
+
+#     define PyGreenlet_STARTED                                      \
+    (*(int (*)(PyGreenlet*))                                         \
+     _PyGreenlet_API[PyGreenlet_STARTED_NUM])
+
+#     define PyGreenlet_ACTIVE                                       \
+    (*(int (*)(PyGreenlet*))                                         \
+     _PyGreenlet_API[PyGreenlet_ACTIVE_NUM])
+
+
+
+
 /* Macro that imports greenlet and initializes C API */
 /* NOTE: This has actually moved to ``greenlet._greenlet._C_API``, but we
    keep the older definition to be sure older code that might have a copy of

docs/changes/1909.bugfix

@@ -0,0 +1,9 @@
+Update to greenlet 2.0. This fixes a deallocation issue that required
+a change in greenlet's ABI. The design of greenlet 2.0 is intended to
+prevent future fixes and enhancements from requiring an ABI change,
+making it easier to update gevent and greenlet independently.
+
+.. caution::
+
+   greenlet 2.0 requires a modern-ish C++ compiler. This may mean
+   certain older platforms are no longer supported.

pyproject.toml

@@ -23,8 +23,10 @@ requires = [
      # Python 3.7 requires at least 0.4.14, which is ABI incompatible with earlier
      # releases. Python 3.9 and 3.10 require 0.4.16;
      # 0.4.17 is ABI incompatible with earlier releases, but compatible with 1.0
-     # 1.1.3 is needed for CPython 3.11
-     "greenlet >= 1.1.3, < 2.0 ; platform_python_implementation == 'CPython'",
+     # 1.1.3 is needed for CPython 3.11.
+     # 2.0 is not ABI compatible with earlier releases, but with luck it won't
+     # have to break the ABI again.
+     "greenlet >= 2.0.0rc4 ; platform_python_implementation == 'CPython'",
 ]
 
 [tool.towncrier]

scripts/releases/make-manylinux

@@ -125,7 +125,7 @@ if [ -d /gevent -a -d /opt/python ]; then
         # The downside is that we must install dependencies manually.
         # NOTE: We can't upgrade ``wheel`` because ``auditwheel`` depends on
         # it, and auditwheel is installed in one of these environments.
-        python -mpip install -U "cython >= 3.0a6" cffi 'greenlet >= 1.0' setuptools
+        python -mpip install -U "cython >= 3.0a6" cffi 'greenlet >= 2.0rc4' setuptools
         time (python setup.py bdist_wheel)
         PATH="$OPATH" auditwheel repair dist/gevent*.whl
         cp wheelhouse/gevent*.whl /gevent/wheelhouse

setup.py

@@ -213,7 +213,9 @@
     # so we can add an upper bound).
     # 1.1.0 is required for 3.10; it has a new ABI, but only on 1.1.0.
     # 1.1.3 is needed for 3.11, and supports everything 1.1.0 did.
-    'greenlet >= 1.1.3, < 2.0; platform_python_implementation=="CPython"',
+    # 2.0.0 supports everything 1.1.3 did, but breaks the ABI in a way that hopefully
+    # won't break again.
+    'greenlet >= 2.0.0rc4 ; platform_python_implementation=="CPython"',
 ]
 
 # Note that we don't add cffi to install_requires, it's

src/gevent/_gevent_cgreenlet.pxd

@@ -1,6 +1,8 @@
 # cython: auto_pickle=False
 
 cimport cython
+from cpython.ref cimport Py_DECREF
+
 from gevent._gevent_c_ident cimport IdentRegistry
 from gevent._gevent_c_hub_local cimport get_hub_noargs as get_hub
 from gevent._gevent_c_waiter cimport Waiter
@@ -21,12 +23,14 @@ cdef extern from "greenlet/greenlet.h":
         # properly handle the case that it can be NULL. So instead we inline a getparent
         # function that does the same thing as the green_getparent accessor but without
         # going through the overhead of generic attribute lookup.
-        cdef void* parent
+        #cdef void* parent
+        pass
 
     # These are actually macros and so must be included
     # (defined) in each .pxd, as are the two functions
     # that call them.
     greenlet PyGreenlet_GetCurrent()
+    void* PyGreenlet_GetParent(greenlet)
     void PyGreenlet_Import()
 
 @cython.final
@@ -36,13 +40,26 @@ cdef inline greenlet getcurrent():
 cdef inline object get_generic_parent(greenlet s):
     # We don't use any typed functions on the return of this,
     # so save the type check by making it just an object.
-    if s.parent != NULL:
-        return <object>s.parent
+    cdef object result
+    cdef void* parent = PyGreenlet_GetParent(s)
+    if parent != NULL:
+        # The cast will perform an incref; but the GetParent
+        # function already did an incref if we got it (and not NULL).
+        # Therefore, we must DECREF immediately.
+        result = <object>parent
+        Py_DECREF(result)
+        return result
 
 cdef inline SwitchOutGreenletWithLoop get_my_hub(greenlet s):
+    # This one we do want type checked on the return value.
     # Must not be called with s = None
-    if s.parent != NULL:
-        return <object>s.parent
+    cdef object result
+    cdef void* parent = PyGreenlet_GetParent(s)
+    if parent != NULL:
+        result = <object>parent
+        # See above
+        Py_DECREF(result)
+        return result
 
 cdef bint _greenlet_imported
 

src/gevent/subprocess.py

@@ -224,6 +224,7 @@ def _use_posix_spawn():
     _fork_exec = None
     __implements__.extend([
         '_fork_exec',
+    ] if sys.platform != 'win32' else [
     ])
 
 actually_imported = copy_globals(__subprocess__, globals(),

src/gevent/tests/test__memleak.py

@@ -26,6 +26,16 @@ def test(self):
             refcounts.append(sys.gettotalrefcount())
 
         # Refcounts may go down, but not up
+        # XXX: JAM: I think this may just be broken. Each time we add
+        # a new integer to our list of refcounts, we'll be
+        # creating a new reference. This makes sense when we see the list
+        # go up by one each iteration:
+        #
+        #   AssertionError: 530631 not less than or equal to 530630
+        #     : total refcount mismatch:
+        #      [530381, 530618, 530619, 530620, 530621,
+        #       530622, 530623, 530624, 530625, 530626,
+        #       530627, 530628, 530629, 530630, 530631]
         final = refcounts[-1]
         previous = refcounts[-2]
         self.assertLessEqual(