greenlet_greenlet.hpp

#ifndef GREENLET_GREENLET_HPP
#define GREENLET_GREENLET_HPP
/*
 * Declarations of the core data structures.
*/

#define PY_SSIZE_T_CLEAN
#include <Python.h>

#include "greenlet_compiler_compat.hpp"
#include "greenlet_refs.hpp"
#include "greenlet_cpython_compat.hpp"
#include "greenlet_allocator.hpp"

using greenlet::refs::OwnedObject;
using greenlet::refs::OwnedGreenlet;
using greenlet::refs::OwnedMainGreenlet;
using greenlet::refs::BorrowedGreenlet;

#if PY_VERSION_HEX < 0x30B00A6
#  define _PyCFrame CFrame
#  define _PyInterpreterFrame _interpreter_frame
#endif

// XXX: TODO: Work to remove all virtual functions
// for speed of calling and size of objects (no vtable).
// One pattern is the Curiously Recurring Template
namespace greenlet
{
    class ExceptionState
    {
    private:
        G_NO_COPIES_OF_CLS(ExceptionState);

#if PY_VERSION_HEX >= 0x030700A3
        // Even though these are borrowed objects, we actually own
        // them, when they're not null.
        // XXX: Express that in the API.
    private:
        _PyErr_StackItem* exc_info;
        _PyErr_StackItem exc_state;
#else
        OwnedObject exc_value;
#if !GREENLET_PY311
        OwnedObject exc_type;
        OwnedObject exc_traceback;
#endif
#endif
    public:
        ExceptionState();
        void operator<<(const PyThreadState *const tstate) G_NOEXCEPT;
        void operator>>(PyThreadState* tstate) G_NOEXCEPT;
        void clear() G_NOEXCEPT;

        int tp_traverse(visitproc visit, void* arg) G_NOEXCEPT;
        void tp_clear() G_NOEXCEPT;
    };

    template<typename T>
    void operator<<(const PyThreadState *const tstate, T& exc);

    template<typename IsPy37>
    class PythonStateContext
    {};

    template<>
    class PythonStateContext<GREENLET_WHEN_PY37>
    {
    protected:
        greenlet::refs::OwnedContext _context;
    public:
        inline const greenlet::refs::OwnedContext& context() const
        {
            return this->_context;
        }
        inline greenlet::refs::OwnedContext& context()
        {
            return this->_context;
        }

        inline void tp_clear()
        {
            this->_context.CLEAR();
        }

        template<typename T>
        inline static PyObject* context(T* tstate)
        {
            return tstate->context;
        }

        template<typename T>
        inline static void context(T* tstate, PyObject* new_context)
        {
            tstate->context = new_context;
            tstate->context_ver++;
        }
    };


    template<>
    class PythonStateContext<GREENLET_WHEN_NOT_PY37>
    {
    public:
        inline const greenlet::refs::OwnedContext& context() const
        {
            throw AttributeError("no context");
        }

        inline greenlet::refs::OwnedContext& context()
        {
            throw AttributeError("no context");
        }

        inline void tp_clear(){};

        template<typename T>
        inline static PyObject* context(T* UNUSED(tstate))
        {
            throw PyFatalError("This should never be called.");
        }

        template<typename T>
        inline static void context(T* UNUSED(tstate), PyObject* UNUSED(new_context))
        {
            throw PyFatalError("This should never be called.");
        }
    };

    class PythonState : public PythonStateContext<G_IS_PY37>
    {
    public:
        typedef greenlet::refs::OwnedReference<struct _frame> OwnedFrame;
    private:
        G_NO_COPIES_OF_CLS(PythonState);
        // We own this if we're suspended (although currently we don't
        // tp_traverse into it; that's a TODO). If we're running, it's
        // empty. If we get deallocated and *still* have a frame, it
        // won't be reachable from the place that normally decref's
        // it, so we need to do it (hence owning it).
        OwnedFrame _top_frame;
#if GREENLET_USE_CFRAME
        _PyCFrame* cframe;
        int use_tracing;
#endif
        int recursion_depth;
        int trash_delete_nesting;
#if GREENLET_PY311
        _PyInterpreterFrame* current_frame;
        _PyStackChunk* datastack_chunk;
        PyObject** datastack_top;
        PyObject** datastack_limit;
#endif

    public:
        PythonState();
        // You can use this for testing whether we have a frame
        // or not. It returns const so they can't modify it.
        const OwnedFrame& top_frame() const G_NOEXCEPT;


        void operator<<(const PyThreadState *const tstate) G_NOEXCEPT;
        void operator>>(PyThreadState* tstate) G_NOEXCEPT;
        void clear() G_NOEXCEPT;

        int tp_traverse(visitproc visit, void* arg, bool visit_top_frame) G_NOEXCEPT;
        void tp_clear(bool own_top_frame) G_NOEXCEPT;
        void set_initial_state(const PyThreadState* const tstate) G_NOEXCEPT;
#if GREENLET_USE_CFRAME
        void set_new_cframe(_PyCFrame& frame) G_NOEXCEPT;
#endif
        void will_switch_from(PyThreadState *const origin_tstate) G_NOEXCEPT;
        void did_finish(PyThreadState* tstate) G_NOEXCEPT;
    };

    class StackState
    {
        // By having only plain C (POD) members, no virtual functions
        // or bases, we get a trivial assignment operator generated
        // for us. However, that's not safe since we do manage memory.
        // So we declare an assignment operator that only works if we
        // don't have any memory allocated. (We don't use
        // std::shared_ptr for reference counting just to keep this
        // object small)
    private:
        char* _stack_start;
        char* stack_stop;
        char* stack_copy;
        intptr_t _stack_saved;
        StackState* stack_prev;
        inline int copy_stack_to_heap_up_to(const char* const stop) G_NOEXCEPT;
        inline void free_stack_copy() G_NOEXCEPT;

    public:
        /**
         * Creates a started, but inactive, state, using *current*
         * as the previous.
         */
        StackState(void* mark, StackState& current);
        /**
         * Creates an inactive, unstarted, state.
         */
        StackState();
        ~StackState();
        StackState(const StackState& other);
        StackState& operator=(const StackState& other);
        inline void copy_heap_to_stack(const StackState& current) G_NOEXCEPT;
        inline int copy_stack_to_heap(char* const stackref, const StackState& current) G_NOEXCEPT;
        inline bool started() const G_NOEXCEPT;
        inline bool main() const G_NOEXCEPT;
        inline bool active() const G_NOEXCEPT;
        inline void set_active() G_NOEXCEPT;
        inline void set_inactive() G_NOEXCEPT;
        inline intptr_t stack_saved() const G_NOEXCEPT;
        inline char* stack_start() const G_NOEXCEPT;
        static inline StackState make_main() G_NOEXCEPT;
#ifdef GREENLET_USE_STDIO
        friend std::ostream& operator<<(std::ostream& os, const StackState& s);
#endif
    };
#ifdef GREENLET_USE_STDIO
    std::ostream& operator<<(std::ostream& os, const StackState& s);
#endif

    class SwitchingArgs
    {
    private:
        G_NO_ASSIGNMENT_OF_CLS(SwitchingArgs);
        // If args and kwargs are both false (NULL), this is a *throw*, not a
        // switch. PyErr_... must have been called already.
        OwnedObject _args;
        OwnedObject _kwargs;
    public:

        SwitchingArgs()
        {}

        SwitchingArgs(const OwnedObject& args, const OwnedObject& kwargs)
            : _args(args),
              _kwargs(kwargs)
        {}

        SwitchingArgs(const SwitchingArgs& other)
            : _args(other._args),
              _kwargs(other._kwargs)
        {}

        OwnedObject& args()
        {
            return this->_args;
        }

        OwnedObject& kwargs()
        {
            return this->_kwargs;
        }

        /**
         * Moves ownership from the argument to this object.
         */
        SwitchingArgs& operator<<=(SwitchingArgs& other)
        {
            if (this != &other) {
                this->_args = other._args;
                this->_kwargs = other._kwargs;
                other.CLEAR();
            }
            return *this;
        }

        /**
         * Acquires ownership of the argument (consumes the reference).
         */
        SwitchingArgs& operator<<=(PyObject* args)
        {
            this->_args = OwnedObject::consuming(args);
            this->_kwargs.CLEAR();
            return *this;
        }

        /**
         * Acquires ownership of the argument.
         *
         * Sets the args to be the given value; clears the kwargs.
         */
        SwitchingArgs& operator<<=(OwnedObject& args)
        {
            assert(&args != &this->_args);
            this->_args = args;
            this->_kwargs.CLEAR();
            args.CLEAR();

            return *this;
        }

        G_EXPLICIT_OP operator bool() const G_NOEXCEPT
        {
            return this->_args || this->_kwargs;
        }

        inline void CLEAR()
        {
            this->_args.CLEAR();
            this->_kwargs.CLEAR();
        }
    };

    class ThreadState;

    class UserGreenlet;
    class MainGreenlet;

    class Greenlet
    {
    private:
        G_NO_COPIES_OF_CLS(Greenlet);
    private:
        // XXX: Work to remove these.
        friend class ThreadState;
        friend class UserGreenlet;
        friend class MainGreenlet;
    protected:
        ExceptionState exception_state;
        SwitchingArgs switch_args;
        StackState stack_state;
        PythonState python_state;
        Greenlet(PyGreenlet* p, const StackState& initial_state);
    public:
        Greenlet(PyGreenlet* p);
        virtual ~Greenlet();

        template <typename IsPy37> // maybe we can use a value here?
        const OwnedObject context(const typename IsPy37::IsIt=nullptr) const;

        template <typename IsPy37>
        inline void context(refs::BorrowedObject new_context, typename IsPy37::IsIt=nullptr);

        inline SwitchingArgs& args()
        {
            return this->switch_args;
        }

        virtual const refs::BorrowedMainGreenlet main_greenlet() const = 0;

        inline intptr_t stack_saved() const G_NOEXCEPT
        {
            return this->stack_state.stack_saved();
        }

        // This is used by the macro SLP_SAVE_STATE to compute the
        // difference in stack sizes. It might be nice to handle the
        // computation ourself, but the type of the result
        // varies by platform, so doing it in the macro is the
        // simplest way.
        inline const char* stack_start() const G_NOEXCEPT
        {
            return this->stack_state.stack_start();
        }

        virtual OwnedObject throw_GreenletExit_during_dealloc(const ThreadState& current_thread_state);
        virtual OwnedObject g_switch() = 0;
        /**
         * Force the greenlet to appear dead. Used when it's not
         * possible to throw an exception into a greenlet anymore.
         *
         * This losses access to the thread state and the main greenlet.
         */
        virtual void murder_in_place();

        /**
         * Called when somebody notices we were running in a dead
         * thread to allow cleaning up resources (because we can't
         * raise GreenletExit into it anymore).
         * This is very similar to ``murder_in_place()``, except that
         * it DOES NOT lose the main greenlet or thread state.
         */
        inline void deactivate_and_free();


        // Called when some thread wants to deallocate a greenlet
        // object.
        // The thread may or may not be the same thread the greenlet
        // was running in.
        // The thread state will be null if the thread the greenlet
        // was running in was known to have exited.
        void deallocing_greenlet_in_thread(const ThreadState* current_state);

        // TODO: Figure out how to make these non-public.
        inline void slp_restore_state() G_NOEXCEPT;
        inline int slp_save_state(char *const stackref) G_NOEXCEPT;

        inline bool is_currently_running_in_some_thread() const;
        virtual bool belongs_to_thread(const ThreadState* state) const;

        inline bool started() const
        {
            return this->stack_state.started();
        }
        inline bool active() const
        {
            return this->stack_state.active();
        }
        inline bool main() const
        {
            return this->stack_state.main();
        }
        virtual refs::BorrowedMainGreenlet find_main_greenlet_in_lineage() const = 0;

        virtual const OwnedGreenlet parent() const = 0;
        virtual void parent(const refs::BorrowedObject new_parent) = 0;

        inline const PythonState::OwnedFrame& top_frame()
        {
            return this->python_state.top_frame();
        }

        virtual const OwnedObject& run() const = 0;
        virtual void run(const refs::BorrowedObject nrun) = 0;


        virtual int tp_traverse(visitproc visit, void* arg);
        virtual int tp_clear();


        // Return the thread state that the greenlet is running in, or
        // null if the greenlet is not running or the thread is known
        // to have exited.
        virtual ThreadState* thread_state() const G_NOEXCEPT = 0;

        // Return true if the greenlet is known to have been running
        // (active) in a thread that has now exited.
        virtual bool was_running_in_dead_thread() const G_NOEXCEPT = 0;

        // Return a borrowed greenlet that is the Python object
        // this object represents.
        virtual BorrowedGreenlet self() const G_NOEXCEPT = 0;

    protected:
        inline void release_args();

        // The functions that must not be inlined are declared virtual.
        // We also mark them as protected, not private, so that the
        // compiler is forced to call them through a function pointer.
        // (A sufficiently smart compiler could directly call a private
        // virtual function since it can never be overridden in a
        // subclass).

        // Also TODO: Switch away from integer error codes and to enums,
        // or throw exceptions when possible.
        struct switchstack_result_t
        {
            int status;
            Greenlet* the_state_that_switched;
            OwnedGreenlet origin_greenlet;

            switchstack_result_t()
                : status(0),
                  the_state_that_switched(nullptr)
            {}

            switchstack_result_t(int err)
                : status(err),
                  the_state_that_switched(nullptr)
            {}

            switchstack_result_t(int err, Greenlet* state, OwnedGreenlet& origin)
                : status(err),
                  the_state_that_switched(state),
                  origin_greenlet(origin)
            {
            }

            switchstack_result_t(int err, Greenlet* state, const BorrowedGreenlet& origin)
                : status(err),
                  the_state_that_switched(state),
                  origin_greenlet(origin)
            {
            }

            switchstack_result_t& operator=(const switchstack_result_t& other)
            {
                this->status = other.status;
                this->the_state_that_switched = other.the_state_that_switched;
                this->origin_greenlet = other.origin_greenlet;
                return *this;
            }
        };

        // Returns the previous greenlet we just switched away from.
        virtual OwnedGreenlet g_switchstack_success() G_NOEXCEPT;


        // Check the preconditions for switching to this greenlet; if they
        // aren't met, throws PyErrOccurred. Most callers will want to
        // catch this and clear the arguments
        inline void check_switch_allowed() const;
        class GreenletStartedWhileInPython : public std::runtime_error
        {
        public:
            GreenletStartedWhileInPython() : std::runtime_error("")
            {}
        };

    protected:


        /**
           Perform a stack switch into this greenlet.

           This temporarily sets the global variable
           ``switching_thread_state`` to this greenlet; as soon as the
           call to ``slp_switch`` completes, this is reset to NULL.
           Consequently, this depends on the GIL.

           TODO: Adopt the stackman model and pass ``slp_switch`` a
           callback function and context pointer; this eliminates the
           need for global variables altogether.

           Because the stack switch happens in this function, this
           function can't use its own stack (local) variables, set
           before the switch, and then accessed after the switch.

           Further, you con't even access ``g_thread_state_global``
           before and after the switch from the global variable.
           Because it is thread local some compilers cache it in a
           register/on the stack, notably new versions of MSVC; this
           breaks with strange crashes sometime later, because writing
           to anything in ``g_thread_state_global`` after the switch
           is actually writing to random memory. For this reason, we
           call a non-inlined function to finish the operation. (XXX:
           The ``/GT`` MSVC compiler argument probably fixes that.)

           It is very important that stack switch is 'atomic', i.e. no
           calls into other Python code allowed (except very few that
           are safe), because global variables are very fragile. (This
           should no longer be the case with thread-local variables.)

        */
        switchstack_result_t g_switchstack(void);
    private:
        OwnedObject g_switch_finish(const switchstack_result_t& err);

    };

    class UserGreenlet : public Greenlet
    {
    private:
        static greenlet::PythonAllocator<UserGreenlet> allocator;
        BorrowedGreenlet _self;
        OwnedMainGreenlet _main_greenlet;
        OwnedObject _run_callable;
        OwnedGreenlet _parent;
    public:
        static void* operator new(size_t UNUSED(count));
        static void operator delete(void* ptr);

        UserGreenlet(PyGreenlet* p, BorrowedGreenlet the_parent);
        virtual ~UserGreenlet();

        virtual refs::BorrowedMainGreenlet find_main_greenlet_in_lineage() const;
        virtual bool was_running_in_dead_thread() const G_NOEXCEPT;
        virtual ThreadState* thread_state() const G_NOEXCEPT;
        virtual OwnedObject g_switch();
        virtual const OwnedObject& run() const
        {
            if (this->started() || !this->_run_callable) {
                throw AttributeError("run");
            }
            return this->_run_callable;
        }
        virtual void run(const refs::BorrowedObject nrun);

        virtual const OwnedGreenlet parent() const;
        virtual void parent(const refs::BorrowedObject new_parent);

        virtual const refs::BorrowedMainGreenlet main_greenlet() const;

        virtual BorrowedGreenlet self() const G_NOEXCEPT;
        virtual void murder_in_place();
        virtual bool belongs_to_thread(const ThreadState* state) const;
        virtual int tp_traverse(visitproc visit, void* arg);
        virtual int tp_clear();
        class ParentIsCurrentGuard
        {
        private:
            OwnedGreenlet oldparent;
            UserGreenlet* greenlet;
            G_NO_COPIES_OF_CLS(ParentIsCurrentGuard);
        public:
            ParentIsCurrentGuard(UserGreenlet* p, const ThreadState& thread_state);
            ~ParentIsCurrentGuard();
        };
        virtual OwnedObject throw_GreenletExit_during_dealloc(const ThreadState& current_thread_state);
    protected:
        virtual switchstack_result_t g_initialstub(void* mark);
    private:
        void inner_bootstrap(OwnedGreenlet& origin_greenlet, OwnedObject& run) G_NOEXCEPT_WIN32;
    };

    class MainGreenlet : public Greenlet
    {
    private:
        static greenlet::PythonAllocator<MainGreenlet> allocator;
        refs::BorrowedMainGreenlet _self;
        ThreadState* _thread_state;
        G_NO_COPIES_OF_CLS(MainGreenlet);
    public:
        static void* operator new(size_t UNUSED(count));
        static void operator delete(void* ptr);

        MainGreenlet(refs::BorrowedMainGreenlet::PyType*, ThreadState*);
        virtual ~MainGreenlet();


        virtual const OwnedObject& run() const;
        virtual void run(const refs::BorrowedObject nrun);

        virtual const OwnedGreenlet parent() const;
        virtual void parent(const refs::BorrowedObject new_parent);

        virtual const refs::BorrowedMainGreenlet main_greenlet() const;

        virtual refs::BorrowedMainGreenlet find_main_greenlet_in_lineage() const;
        virtual bool was_running_in_dead_thread() const G_NOEXCEPT;
        virtual ThreadState* thread_state() const G_NOEXCEPT;
        void thread_state(ThreadState*) G_NOEXCEPT;
        virtual OwnedObject g_switch();
        virtual BorrowedGreenlet self() const G_NOEXCEPT;
        virtual int tp_traverse(visitproc visit, void* arg);
    };

};

template<typename T>
void greenlet::operator<<(const PyThreadState *const lhs, T& rhs)
{
    rhs.operator<<(lhs);
}

using greenlet::ExceptionState;

ExceptionState::ExceptionState()
{
    this->clear();
}

#if PY_VERSION_HEX >= 0x030700A3
// ******** Python 3.7 and above *********
void ExceptionState::operator<<(const PyThreadState *const tstate) G_NOEXCEPT
{
    this->exc_info = tstate->exc_info;
    this->exc_state = tstate->exc_state;
}

void ExceptionState::operator>>(PyThreadState *const tstate) G_NOEXCEPT
{
    tstate->exc_state = this->exc_state;
    tstate->exc_info =
        this->exc_info ? this->exc_info : &tstate->exc_state;
    this->clear();
}

void ExceptionState::clear() G_NOEXCEPT
{
    this->exc_info = nullptr;
    this->exc_state.exc_value = nullptr;
#if !GREENLET_PY311
    this->exc_state.exc_type = nullptr;
    this->exc_state.exc_traceback = nullptr;
#endif
    this->exc_state.previous_item = nullptr;
}

int ExceptionState::tp_traverse(visitproc visit, void* arg) G_NOEXCEPT
{
    Py_VISIT(this->exc_state.exc_value);
#if !GREENLET_PY311
    Py_VISIT(this->exc_state.exc_type);
    Py_VISIT(this->exc_state.exc_traceback);
#endif
    return 0;
}

void ExceptionState::tp_clear() G_NOEXCEPT
{
    Py_CLEAR(this->exc_state.exc_value);
#if !GREENLET_PY311
    Py_CLEAR(this->exc_state.exc_type);
    Py_CLEAR(this->exc_state.exc_traceback);
#endif
}
#else
// ********** Python 3.6 and below ********
void ExceptionState::operator<<(const PyThreadState *const tstate) G_NOEXCEPT
{
    this->exc_value.steal(tstate->exc_value);
#if !GREENLET_PY311
    this->exc_type.steal(tstate->exc_type);
    this->exc_traceback.steal(tstate->exc_traceback);
#endif
}

void ExceptionState::operator>>(PyThreadState *const tstate) G_NOEXCEPT
{
    tstate->exc_value <<= this->exc_value;
#if !GREENLET_PY311
    tstate->exc_type <<= this->exc_type;
    tstate->exc_traceback <<= this->exc_traceback;
#endif
    this->clear();
}

void ExceptionState::clear() G_NOEXCEPT
{
    this->exc_value = nullptr;
#if !GREENLET_PY311
    this->exc_type = nullptr;
    this->exc_traceback = nullptr;
#endif
}

int ExceptionState::tp_traverse(visitproc visit, void* arg) G_NOEXCEPT
{
    Py_VISIT(this->exc_value.borrow());
#if !GREENLET_PY311
    Py_VISIT(this->exc_type.borrow());
    Py_VISIT(this->exc_traceback.borrow());
#endif
    return 0;
}

void ExceptionState::tp_clear() G_NOEXCEPT
{
    this->exc_value.CLEAR();
#if !GREENLET_PY311
    this->exc_type.CLEAR();
    this->exc_traceback.CLEAR();
#endif
}
#endif


using greenlet::PythonState;

PythonState::PythonState()
    : _top_frame()
#if GREENLET_USE_CFRAME
    ,cframe(nullptr)
    ,use_tracing(0)
#endif
    ,recursion_depth(0)
    ,trash_delete_nesting(0)
#if GREENLET_PY311
    ,current_frame(nullptr)
    ,datastack_chunk(nullptr)
    ,datastack_top(nullptr)
    ,datastack_limit(nullptr)
#endif
{
#if GREENLET_USE_CFRAME
    /*
      The PyThreadState->cframe pointer usually points to memory on
      the stack, alloceted in a call into PyEval_EvalFrameDefault.

      Initially, before any evaluation begins, it points to the
      initial PyThreadState object's ``root_cframe`` object, which is
      statically allocated for the lifetime of the thread.

      A greenlet can last for longer than a call to
      PyEval_EvalFrameDefault, so we can't set its ``cframe`` pointer
      to be the current ``PyThreadState->cframe``; nor could we use
      one from the greenlet parent for the same reason. Yet a further
      no: we can't allocate one scoped to the greenlet and then
      destroy it when the greenlet is deallocated, because inside the
      interpreter the _PyCFrame objects form a linked list, and that too
      can result in accessing memory beyond its dynamic lifetime (if
      the greenlet doesn't actually finish before it dies, its entry
      could still be in the list).

      Using the ``root_cframe`` is problematic, though, because its
      members are never modified by the interpreter and are set to 0,
      meaning that its ``use_tracing`` flag is never updated. We don't
      want to modify that value in the ``root_cframe`` ourself: it
      *shouldn't* matter much because we should probably never get
      back to the point where that's the only cframe on the stack;
      even if it did matter, the major consequence of an incorrect
      value for ``use_tracing`` is that if its true the interpreter
      does some extra work --- however, it's just good code hygiene.

      Our solution: before a greenlet runs, after its initial
      creation, it uses the ``root_cframe`` just to have something to
      put there. However, once the greenlet is actually switched to
      for the first time, ``g_initialstub`` (which doesn't actually
      "return" while the greenlet is running) stores a new _PyCFrame on
      its local stack, and copies the appropriate values from the
      currently running _PyCFrame; this is then made the _PyCFrame for the
      newly-minted greenlet. ``g_initialstub`` then proceeds to call
      ``glet.run()``, which results in ``PyEval_...`` adding the
      _PyCFrame to the list. Switches continue as normal. Finally, when
      the greenlet finishes, the call to ``glet.run()`` returns and
      the _PyCFrame is taken out of the linked list and the stack value
      is now unused and free to expire.

      XXX: I think we can do better. If we're deallocing in the same
      thread, can't we traverse the list and unlink our frame?
      Can we just keep a reference to the thread state in case we
      dealloc in another thread? (Is that even possible if we're still
      running and haven't returned from g_initialstub?)
    */
    this->cframe = &PyThreadState_GET()->root_cframe;
#endif
}

void PythonState::operator<<(const PyThreadState *const tstate) G_NOEXCEPT
{
#if GREENLET_PY37
    this->_context.steal(tstate->context);
#endif
#if GREENLET_USE_CFRAME
    /*
      IMPORTANT: ``cframe`` is a pointer into the STACK. Thus, because
      the call to ``slp_switch()`` changes the contents of the stack,
      you cannot read from ``ts_current->cframe`` after that call and
      necessarily get the same values you get from reading it here.
      Anything you need to restore from now to then must be saved in a
      global/threadlocal variable (because we can't use stack
      variables here either). For things that need to persist across
      the switch, use `will_switch_from`.
    */
    this->cframe = tstate->cframe;
    this->use_tracing = tstate->cframe->use_tracing;
#endif
#if GREENLET_PY311
    this->recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
    this->current_frame = tstate->cframe->current_frame;
    this->datastack_chunk = tstate->datastack_chunk;
    this->datastack_top = tstate->datastack_top;
    this->datastack_limit = tstate->datastack_limit;
    PyFrameObject *frame = PyThreadState_GetFrame((PyThreadState *)tstate);
    Py_XDECREF(frame);  // PyThreadState_GetFrame gives us a new reference.
    this->_top_frame.steal(frame);
#else
    this->recursion_depth = tstate->recursion_depth;
    this->_top_frame.steal(tstate->frame);
#endif

    // All versions of Python.
    this->trash_delete_nesting = tstate->trash_delete_nesting;
}

void PythonState::operator>>(PyThreadState *const tstate) G_NOEXCEPT
{
#if GREENLET_PY37
    tstate->context = this->_context.relinquish_ownership();
    /* Incrementing this value invalidates the contextvars cache,
       which would otherwise remain valid across switches */
    tstate->context_ver++;
#endif
#if GREENLET_USE_CFRAME
    tstate->cframe = this->cframe;
    /*
      If we were tracing, we need to keep tracing.
      There should never be the possibility of hitting the
      root_cframe here. See note above about why we can't
      just copy this from ``origin->cframe->use_tracing``.
    */
    tstate->cframe->use_tracing = this->use_tracing;
#endif
#if GREENLET_PY311
    tstate->recursion_remaining = tstate->recursion_limit - this->recursion_depth;
    tstate->cframe->current_frame = this->current_frame;
    tstate->datastack_chunk = this->datastack_chunk;
    tstate->datastack_top = this->datastack_top;
    tstate->datastack_limit = this->datastack_limit;
    this->_top_frame.relinquish_ownership();
#else
    tstate->frame = this->_top_frame.relinquish_ownership();
    tstate->recursion_depth = this->recursion_depth;
#endif
    // All versions of Python.
    tstate->trash_delete_nesting = this->trash_delete_nesting;
}

void PythonState::will_switch_from(PyThreadState *const origin_tstate) G_NOEXCEPT
{
#if GREENLET_USE_CFRAME
    // The weird thing is, we don't actually save this for an
    // effect on the current greenlet, it's saved for an
    // effect on the target greenlet. That is, we want
    // continuity of this setting across the greenlet switch.
    this->use_tracing = origin_tstate->cframe->use_tracing;
#endif
}

void PythonState::set_initial_state(const PyThreadState* const tstate) G_NOEXCEPT
{
    this->_top_frame = nullptr;
#if GREENLET_PY311
    this->recursion_depth = tstate->recursion_limit - tstate->recursion_remaining;
#else
    this->recursion_depth = tstate->recursion_depth;
#endif
}
// TODO: Better state management about when we own the top frame.
int PythonState::tp_traverse(visitproc visit, void* arg, bool own_top_frame) G_NOEXCEPT
{
#if GREENLET_PY37
    Py_VISIT(this->_context.borrow());
#endif
    if (own_top_frame) {
        Py_VISIT(this->_top_frame.borrow());
    }
    return 0;
}

void PythonState::tp_clear(bool own_top_frame) G_NOEXCEPT
{
    PythonStateContext::tp_clear();
    // If we get here owning a frame,
    // we got dealloc'd without being finished. We may or may not be
    // in the same thread.
    if (own_top_frame) {
        this->_top_frame.CLEAR();
    }
}

#if GREENLET_USE_CFRAME
void PythonState::set_new_cframe(_PyCFrame& frame) G_NOEXCEPT
{
    frame = *PyThreadState_GET()->cframe;
    /* Make the target greenlet refer to the stack value. */
    this->cframe = &frame;
    /*
      And restore the link to the previous frame so this one gets
      unliked appropriately.
    */
    this->cframe->previous = &PyThreadState_GET()->root_cframe;
}
#endif

const PythonState::OwnedFrame& PythonState::top_frame() const G_NOEXCEPT
{
    return this->_top_frame;
}

void PythonState::did_finish(PyThreadState* tstate) G_NOEXCEPT
{
#if GREENLET_PY311
    // See https://github.com/gevent/gevent/issues/1924 and
    // https://github.com/python-greenlet/greenlet/issues/328. In
    // short, Python 3.11 allocates memory for frames as a sort of
    // linked list that's kept as part of PyThreadState in the
    // ``datastack_chunk`` member and friends. These are saved and
    // restored as part of switching greenlets.
    //
    // When we initially switch to a greenlet, we set those to NULL.
    // That causes the frame management code to treat this like a
    // brand new thread and start a fresh list of chunks, beginning
    // with a new "root" chunk. As we make calls in this greenlet,
    // those chunks get added, and as calls return, they get popped.
    // But the frame code (pystate.c) is careful to make sure that the
    // root chunk never gets popped.
    //
    // Thus, when a greenlet exits for the last time, there will be at
    // least a single root chunk that we must be responsible for
    // deallocating.
    //
    // The complex part is that these chunks are allocated and freed
    // using ``_PyObject_VirtualAlloc``/``Free``. Those aren't public
    // functions, and they aren't exported for linking. It so happens
    // that we know they are just thin wrappers around the Arena
    // allocator, so we can use that directly to deallocate in a
    // compatible way.
    //
    // CAUTION: Check this implementation detail on every major version.
    //
    // It might be nice to be able to do this in our destructor, but
    // can we be sure that no one else is using that memory? Plus, as
    // described below, our pointers may not even be valid anymore. As
    // a special case, there is one time that we know we can do this,
    // and that's from the destructor of the associated UserGreenlet
    // (NOT main greenlet)
    PyObjectArenaAllocator alloc;
    _PyStackChunk* chunk = nullptr;
    if (tstate) {
        // We really did finish, we can never be switched to again.
        chunk = tstate->datastack_chunk;
        // Unfortunately, we can't do much sanity checking. Our
        // this->datastack_chunk pointer is out of date (evaluation may
        // have popped down through it already) so we can't verify that
        // we deallocate it. I don't think we can even check datastack_top
        // for the same reason.

        PyObject_GetArenaAllocator(&alloc);
        tstate->datastack_chunk = nullptr;
        tstate->datastack_limit = nullptr;
        tstate->datastack_top = nullptr;

    }
    else if (this->datastack_chunk) {
        // The UserGreenlet (NOT the main greenlet!) is being deallocated. If we're
        // still holding a stack chunk, it's garbage because we know
        // we can never switch back to let cPython clean it up.
        // Because the last time we got switched away from, and we
        // haven't run since then, we know our chain is valid and can
        // be dealloced.
        chunk = this->datastack_chunk;
        PyObject_GetArenaAllocator(&alloc);
    }

    if (alloc.free && chunk) {
        // In case the arena mechanism has been torn down already.
        while (chunk) {
            _PyStackChunk *prev = chunk->previous;
            chunk->previous = nullptr;
            alloc.free(alloc.ctx, chunk, chunk->size);
            chunk = prev;
        }
    }

    this->datastack_chunk = nullptr;
    this->datastack_limit = nullptr;
    this->datastack_top = nullptr;
#endif
}




using greenlet::StackState;

#ifdef GREENLET_USE_STDIO
#include <iostream>
using std::cerr;
using std::endl;

std::ostream& greenlet::operator<<(std::ostream& os, const StackState& s)
{
    os << "StackState(stack_start=" << (void*)s._stack_start
       << ", stack_stop=" << (void*)s.stack_stop
       << ", stack_copy=" << (void*)s.stack_copy
       << ", stack_saved=" << s._stack_saved
       << ", stack_prev=" << s.stack_prev
       << ", addr=" << &s
       << ")";
    return os;
}
#endif

StackState::StackState(void* mark, StackState& current)
    : _stack_start(nullptr),
      stack_stop((char*)mark),
      stack_copy(nullptr),
      _stack_saved(0),
      /* Skip a dying greenlet */
      stack_prev(current._stack_start
                 ? &current
                 : current.stack_prev)
{
}

StackState::StackState()
    : _stack_start(nullptr),
      stack_stop(nullptr),
      stack_copy(nullptr),
      _stack_saved(0),
      stack_prev(nullptr)
{
}

StackState::StackState(const StackState& other)
// can't use a delegating constructor because of
// MSVC for Python 2.7
    : _stack_start(nullptr),
      stack_stop(nullptr),
      stack_copy(nullptr),
      _stack_saved(0),
      stack_prev(nullptr)
{
    this->operator=(other);
}

StackState& StackState::operator=(const StackState& other)
{
    if (&other == this) {
        return *this;
    }
    if (other._stack_saved) {
        throw std::runtime_error("Refusing to steal memory.");
    }

    //If we have memory allocated, dispose of it
    this->free_stack_copy();

    this->_stack_start = other._stack_start;
    this->stack_stop = other.stack_stop;
    this->stack_copy = other.stack_copy;
    this->_stack_saved = other._stack_saved;
    this->stack_prev = other.stack_prev;
    return *this;
}

inline void StackState::free_stack_copy() G_NOEXCEPT
{
    PyMem_Free(this->stack_copy);
    this->stack_copy = nullptr;
    this->_stack_saved = 0;
}

inline void StackState::copy_heap_to_stack(const StackState& current) G_NOEXCEPT
{
    // cerr << "copy_heap_to_stack" << endl
    //      << "\tFrom    : " << *this << endl
    //      << "\tCurrent:" << current
    //      << endl;
    /* Restore the heap copy back into the C stack */
    if (this->_stack_saved != 0) {
        memcpy(this->_stack_start, this->stack_copy, this->_stack_saved);
        this->free_stack_copy();
    }
    StackState* owner = const_cast<StackState*>(&current);
    if (!owner->_stack_start) {
        owner = owner->stack_prev; /* greenlet is dying, skip it */
    }
    while (owner && owner->stack_stop <= this->stack_stop) {
        // cerr << "\tOwner: " << owner << endl;
        owner = owner->stack_prev; /* find greenlet with more stack */
    }
    this->stack_prev = owner;
    // cerr << "\tFinished with: " << *this << endl;
}

inline int StackState::copy_stack_to_heap_up_to(const char* const stop) G_NOEXCEPT
{
    /* Save more of g's stack into the heap -- at least up to 'stop'
       g->stack_stop |________|
                     |        |
                     |    __ stop       . . . . .
                     |        |    ==>  .       .
                     |________|          _______
                     |        |         |       |
                     |        |         |       |
      g->stack_start |        |         |_______| g->stack_copy
     */
    intptr_t sz1 = this->_stack_saved;
    intptr_t sz2 = stop - this->_stack_start;
    assert(this->_stack_start);
    if (sz2 > sz1) {
        char* c = (char*)PyMem_Realloc(this->stack_copy, sz2);
        if (!c) {
            PyErr_NoMemory();
            return -1;
        }
        memcpy(c + sz1, this->_stack_start + sz1, sz2 - sz1);
        this->stack_copy = c;
        this->_stack_saved = sz2;
    }
    return 0;
}

inline int StackState::copy_stack_to_heap(char* const stackref,
                                          const StackState& current) G_NOEXCEPT
{
    // cerr << "copy_stack_to_heap: " << endl
    //      << "\tstackref: " << (void*)stackref << endl
    //      << "\tthis: " << *this << endl
    //      << "\tcurrent: " << current
    //      << endl;
    /* must free all the C stack up to target_stop */
    const char* const target_stop = this->stack_stop;

    StackState* owner = const_cast<StackState*>(&current);
    assert(owner->_stack_saved == 0); // everything is present on the stack
    if (!owner->_stack_start) {
        // cerr << "\tcurrent is dead; using: " << owner->stack_prev << endl;
        owner = owner->stack_prev; /* not saved if dying */
    }
    else {
        owner->_stack_start = stackref;
    }

    while (owner->stack_stop < target_stop) {
        // cerr << "\tCopying from " << *owner << endl;
        /* ts_current is entierely within the area to free */
        if (owner->copy_stack_to_heap_up_to(owner->stack_stop)) {
            return -1; /* XXX */
        }
        owner = owner->stack_prev;
    }
    if (owner != this) {
        if (owner->copy_stack_to_heap_up_to(target_stop)) {
            return -1; /* XXX */
        }
    }
    return 0;
}

inline bool StackState::started() const G_NOEXCEPT
{
    return this->stack_stop != nullptr;
}

inline bool StackState::main() const G_NOEXCEPT
{
    return this->stack_stop == (char*)-1;
}

inline bool StackState::active() const G_NOEXCEPT
{
    return this->_stack_start != nullptr;
}

inline void StackState::set_active() G_NOEXCEPT
{
    assert(this->_stack_start == nullptr);
    this->_stack_start = (char*)1;
}

inline void StackState::set_inactive() G_NOEXCEPT
{
    this->_stack_start = nullptr;
    // XXX: What if we still have memory out there?
    // That case is actually triggered by
    // test_issue251_issue252_explicit_reference_not_collectable (greenlet.tests.test_leaks.TestLeaks)
    // and
    // test_issue251_issue252_need_to_collect_in_background
    // (greenlet.tests.test_leaks.TestLeaks)
    //
    // Those objects never get deallocated, so the destructor never
    // runs.
    // It *seems* safe to clean up the memory here?
    if (this->_stack_saved) {
        this->free_stack_copy();
    }
}

inline intptr_t StackState::stack_saved() const G_NOEXCEPT
{
    return this->_stack_saved;
}

inline char* StackState::stack_start() const G_NOEXCEPT
{
    return this->_stack_start;
}


inline StackState StackState::make_main() G_NOEXCEPT
{
    StackState s;
    s._stack_start = (char*)1;
    s.stack_stop = (char*)-1;
    return s;
}

StackState::~StackState()
{
    if (this->_stack_saved != 0) {
        this->free_stack_copy();
    }
}

using greenlet::Greenlet;

bool Greenlet::is_currently_running_in_some_thread() const
{
    return this->stack_state.active() && !this->python_state.top_frame();
}



#endif