enable extending native types on abi3 for 3.12+

guide/src/class.md

@@ -528,7 +528,7 @@ To convert between the Rust type and its native base class, you can take `slf` a
 To access the Rust fields use `slf.borrow()` or `slf.borrow_mut()`, and to access the base class use `slf.cast::<BaseClass>()`.
 
 ```rust
-# #[cfg(not(Py_LIMITED_API))] {
+# #[cfg(any(not(Py_LIMITED_API), Py_3_12))] {
 # use pyo3::prelude::*;
 use pyo3::types::PyDict;
 use std::collections::HashMap;
@@ -588,7 +588,7 @@ Be sure to accept arguments in the `#[new]` method that you want the base class
 
 ```rust
 # #[allow(dead_code)]
-# #[cfg(not(Py_LIMITED_API))] {
+# #[cfg(any(not(Py_LIMITED_API), Py_3_12))] {
 # use pyo3::prelude::*;
 use pyo3::types::PyDict;
 

guide/src/exception.md

@@ -133,7 +133,7 @@ If you need to create an exception with more complex behavior, you can also manu
 
 ```rust
 #![allow(dead_code)]
-# #[cfg(any(not(feature = "abi3")))] {
+# #[cfg(any(not(Py_LIMITED_API), Py_3_12))] {
 use pyo3::prelude::*;
 use pyo3::types::IntoPyDict;
 use pyo3::exceptions::PyException;

newsfragments/5733.added.md

@@ -0,0 +1 @@
+added support for subclassing native types (`PyDict`, exceptions, ...) when building for abi3 on Python 3.12+

src/impl_/pyclass_init.rs

@@ -1,11 +1,11 @@
 //! Contains initialization utilities for `#[pyclass]`.
+use crate::exceptions::PyTypeError;
 use crate::ffi_ptr_ext::FfiPtrExt;
-use crate::internal::get_slot::TP_ALLOC;
-use crate::types::PyType;
-use crate::{ffi, Borrowed, PyErr, PyResult, Python};
+use crate::internal::get_slot::TP_NEW;
+use crate::types::{PyTuple, PyType};
+use crate::{ffi, PyErr, PyResult, Python};
 use crate::{ffi::PyTypeObject, sealed::Sealed, type_object::PyTypeInfo};
 use std::marker::PhantomData;
-use std::ptr;
 
 /// Initializer for Python types.
 ///
@@ -35,55 +35,27 @@ impl<T: PyTypeInfo> PyObjectInit<T> for PyNativeTypeInitializer<T> {
     ) -> PyResult<*mut ffi::PyObject> {
         unsafe fn inner(
             py: Python<'_>,
-            type_object: *mut PyTypeObject,
+            type_ptr: *mut PyTypeObject,
             subtype: *mut PyTypeObject,
         ) -> PyResult<*mut ffi::PyObject> {
-            // HACK (due to FIXME below): PyBaseObject_Type's tp_new isn't happy with NULL arguments
-            let is_base_object = ptr::eq(type_object, ptr::addr_of!(ffi::PyBaseObject_Type));
-            let subtype_borrowed: Borrowed<'_, '_, PyType> = unsafe {
-                subtype
+            let tp_new = unsafe {
+                type_ptr
                     .cast::<ffi::PyObject>()
                     .assume_borrowed_unchecked(py)
-                    .cast_unchecked()
+                    .cast_unchecked::<PyType>()
+                    .get_slot(TP_NEW)
+                    .ok_or_else(|| PyTypeError::new_err("base type without tp_new"))?
             };
 
-            if is_base_object {
-                let alloc = subtype_borrowed
-                    .get_slot(TP_ALLOC)
-                    .unwrap_or(ffi::PyType_GenericAlloc);
-
-                let obj = unsafe { alloc(subtype, 0) };
-                return if obj.is_null() {
-                    Err(PyErr::fetch(py))
-                } else {
-                    Ok(obj)
-                };
-            }
-
-            #[cfg(Py_LIMITED_API)]
-            unreachable!("subclassing native types is not possible with the `abi3` feature");
-
-            #[cfg(not(Py_LIMITED_API))]
-            {
-                match unsafe { (*type_object).tp_new } {
-                    // FIXME: Call __new__ with actual arguments
-                    Some(newfunc) => {
-                        let obj =
-                            unsafe { newfunc(subtype, std::ptr::null_mut(), std::ptr::null_mut()) };
-                        if obj.is_null() {
-                            Err(PyErr::fetch(py))
-                        } else {
-                            Ok(obj)
-                        }
-                    }
-                    None => Err(crate::exceptions::PyTypeError::new_err(
-                        "base type without tp_new",
-                    )),
-                }
+            // TODO: make it possible to provide real arguments to the base tp_new
+            let obj = unsafe { tp_new(subtype, PyTuple::empty(py).as_ptr(), std::ptr::null_mut()) };
+            if obj.is_null() {
+                Err(PyErr::fetch(py))
+            } else {
+                Ok(obj)
             }
         }
-        let type_object = T::type_object_raw(py);
-        unsafe { inner(py, type_object, subtype) }
+        unsafe { inner(py, T::type_object_raw(py), subtype) }
     }
 
     #[inline]

src/internal/get_slot.rs

@@ -124,7 +124,8 @@ macro_rules! impl_slots {
 
 // Slots are implemented on-demand as needed.)
 impl_slots! {
-    TP_ALLOC: (Py_tp_alloc, tp_alloc) -> Option<ffi::allocfunc>,
+    TP_NEW: (Py_tp_new, tp_new) -> Option<ffi::newfunc>,
+    TP_DEALLOC: (Py_tp_dealloc, tp_dealloc) -> Option<ffi::destructor>,
     TP_BASE: (Py_tp_base, tp_base) -> *mut ffi::PyTypeObject,
     TP_CLEAR: (Py_tp_clear, tp_clear) -> Option<ffi::inquiry>,
     TP_DESCR_GET: (Py_tp_descr_get, tp_descr_get) -> Option<ffi::descrgetfunc>,

src/pycell/impl_.rs

@@ -10,7 +10,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use crate::impl_::pyclass::{
     PyClassBaseType, PyClassDict, PyClassImpl, PyClassThreadChecker, PyClassWeakRef, PyObjectOffset,
 };
-use crate::internal::get_slot::TP_FREE;
+use crate::internal::get_slot::{TP_DEALLOC, TP_FREE};
 use crate::type_object::{PyLayout, PySizedLayout};
 use crate::types::PyType;
 use crate::{ffi, PyClass, PyTypeInfo, Python};
@@ -273,27 +273,19 @@ unsafe fn tp_dealloc(slf: *mut ffi::PyObject, type_obj: &crate::Bound<'_, PyType
         }
 
         // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc.
-        #[cfg(not(Py_LIMITED_API))]
-        {
-            // FIXME: should this be using actual_type.tp_dealloc?
-            if let Some(dealloc) = (*type_ptr).tp_dealloc {
-                // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which
-                // assumes the exception is currently GC tracked, so we have to re-track
-                // before calling the dealloc so that it can safely call Py_GC_UNTRACK.
-                #[cfg(not(any(Py_3_11, PyPy)))]
-                if ffi::PyType_FastSubclass(type_ptr, ffi::Py_TPFLAGS_BASE_EXC_SUBCLASS) == 1 {
-                    ffi::PyObject_GC_Track(slf.cast());
-                }
-                dealloc(slf);
-            } else {
-                (*actual_type.as_type_ptr())
-                    .tp_free
-                    .expect("type missing tp_free")(slf.cast());
+        // FIXME: should this be using actual_type.tp_dealloc?
+        if let Some(dealloc) = type_obj.get_slot(TP_DEALLOC) {
+            // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which
+            // assumes the exception is currently GC tracked, so we have to re-track
+            // before calling the dealloc so that it can safely call Py_GC_UNTRACK.
+            #[cfg(not(any(Py_3_11, PyPy)))]
+            if ffi::PyType_FastSubclass(type_ptr, ffi::Py_TPFLAGS_BASE_EXC_SUBCLASS) == 1 {
+                ffi::PyObject_GC_Track(slf.cast());
             }
+            dealloc(slf);
+        } else {
+            type_obj.get_slot(TP_FREE).expect("type missing tp_free")(slf.cast());
         }
-
-        #[cfg(Py_LIMITED_API)]
-        unreachable!("subclassing native types is not possible with the `abi3` feature");
     }
 }
 

src/tests/hygiene/pymethods.rs

@@ -458,11 +458,11 @@ struct WarningDummy {
     value: i32,
 }
 
-#[cfg(not(Py_LIMITED_API))]
+#[cfg(any(not(Py_LIMITED_API), Py_3_12))]
 #[crate::pyclass(crate = "crate", extends=crate::exceptions::PyWarning)]
 pub struct UserDefinedWarning {}
 
-#[cfg(not(Py_LIMITED_API))]
+#[cfg(any(not(Py_LIMITED_API), Py_3_12))]
 #[crate::pymethods(crate = "crate")]
 impl UserDefinedWarning {
     #[new]
@@ -489,7 +489,7 @@ impl WarningDummy {
     #[pyo3(warn(message = "this method raises warning", category = crate::exceptions::PyFutureWarning))]
     fn method_with_warning_and_custom_category(_slf: crate::PyRef<'_, Self>) {}
 
-    #[cfg(not(Py_LIMITED_API))]
+    #[cfg(any(not(Py_LIMITED_API), Py_3_12))]
     #[pyo3(warn(message = "this method raises user-defined warning", category = UserDefinedWarning))]
     fn method_with_warning_and_user_defined_category(&self) {}
 
@@ -561,7 +561,7 @@ impl WarningDummy2 {
     #[pyo3(warn(message = "this class-method raises future warning", category = crate::exceptions::PyFutureWarning))]
     fn multiple_warnings_fn(&self) {}
 
-    #[cfg(not(Py_LIMITED_API))]
+    #[cfg(any(not(Py_LIMITED_API), Py_3_12))]
     #[pyo3(warn(message = "this class-method raises future warning", category = crate::exceptions::PyFutureWarning))]
     #[pyo3(warn(message = "this class-method raises user-defined warning", category = UserDefinedWarning))]
     fn multiple_warnings_fn_with_custom_category(&self) {}

src/types/mod.rs

@@ -228,6 +228,15 @@ macro_rules! pyobject_subclassable_native_type {
             type PyClassMutability = $crate::pycell::impl_::ImmutableClass;
             type Layout<T: $crate::impl_::pyclass::PyClassImpl> = $crate::impl_::pycell::PyStaticClassObject<T>;
         }
+
+        #[cfg(all(Py_3_12, Py_LIMITED_API))]
+        impl<$($generics,)*> $crate::impl_::pyclass::PyClassBaseType for $name {
+            type LayoutAsBase = $crate::impl_::pycell::PyVariableClassObjectBase;
+            type BaseNativeType = Self;
+            type Initializer = $crate::impl_::pyclass_init::PyNativeTypeInitializer<Self>;
+            type PyClassMutability = $crate::pycell::impl_::ImmutableClass;
+            type Layout<T: $crate::impl_::pyclass::PyClassImpl> = $crate::impl_::pycell::PyVariableClassObject<T>;
+        }
     }
 }
 

tests/test_compile_error.rs

@@ -49,7 +49,7 @@ fn test_compile_errors() {
     t.compile_fail("tests/ui/invalid_pyfunctions.rs");
     t.compile_fail("tests/ui/invalid_pymethods.rs");
     // output changes with async feature
-    #[cfg(all(Py_LIMITED_API, feature = "experimental-async"))]
+    #[cfg(all(not(Py_3_12), Py_LIMITED_API, feature = "experimental-async"))]
     t.compile_fail("tests/ui/abi3_nativetype_inheritance.rs");
     #[cfg(not(feature = "experimental-async"))]
     t.compile_fail("tests/ui/invalid_async.rs");

tests/test_inheritance.rs

@@ -172,8 +172,8 @@ except Exception as e:
     });
 }
 
-// Subclassing builtin types is not allowed in the LIMITED API.
-#[cfg(not(Py_LIMITED_API))]
+// Subclassing builtin types is not possible in the LIMITED API before 3.12
+#[cfg(any(not(Py_LIMITED_API), Py_3_12))]
 mod inheriting_native_type {
     use super::*;
     use pyo3::exceptions::PyException;

tests/test_methods.rs

@@ -1,6 +1,6 @@
 #![cfg(feature = "macros")]
 
-#[cfg(not(Py_LIMITED_API))]
+#[cfg(any(not(Py_LIMITED_API), Py_3_12))]
 use pyo3::exceptions::PyWarning;
 use pyo3::exceptions::{PyFutureWarning, PyUserWarning};
 use pyo3::prelude::*;
@@ -1220,11 +1220,11 @@ fn test_issue_2988() {
     }
 }
 
-#[cfg(not(Py_LIMITED_API))]
+#[cfg(any(not(Py_LIMITED_API), Py_3_12))]
 #[pyclass(extends=PyWarning)]
 pub struct UserDefinedWarning {}
 
-#[cfg(not(Py_LIMITED_API))]
+#[cfg(any(not(Py_LIMITED_API), Py_3_12))]
 #[pymethods]
 impl UserDefinedWarning {
     #[new]
@@ -1258,7 +1258,7 @@ fn test_pymethods_warn() {
         #[pyo3(warn(message = "this method raises warning", category = PyFutureWarning))]
         fn method_with_warning_and_custom_category(_slf: PyRef<'_, Self>) {}
 
-        #[cfg(not(Py_LIMITED_API))]
+        #[cfg(any(not(Py_LIMITED_API), Py_3_12))]
         #[pyo3(warn(message = "this method raises user-defined warning", category = UserDefinedWarning))]
         fn method_with_warning_and_user_defined_category(&self) {}
 
@@ -1325,7 +1325,7 @@ fn test_pymethods_warn() {
         );
 
         // FnType::Fn, user-defined warning
-        #[cfg(not(Py_LIMITED_API))]
+        #[cfg(any(not(Py_LIMITED_API), Py_3_12))]
         py_expect_warning!(
             py,
             obj,

tests/test_pyfunction.rs

@@ -5,7 +5,7 @@ use std::collections::HashMap;
 
 #[cfg(not(Py_LIMITED_API))]
 use pyo3::buffer::PyBuffer;
-#[cfg(not(Py_LIMITED_API))]
+#[cfg(any(not(Py_LIMITED_API), Py_3_12))]
 use pyo3::exceptions::PyWarning;
 use pyo3::exceptions::{PyFutureWarning, PyUserWarning};
 use pyo3::prelude::*;
@@ -674,11 +674,11 @@ fn test_pyfunction_raw_ident() {
     })
 }
 
-#[cfg(not(Py_LIMITED_API))]
+#[cfg(any(not(Py_LIMITED_API), Py_3_12))]
 #[pyclass(extends=PyWarning)]
 pub struct UserDefinedWarning {}
 
-#[cfg(not(Py_LIMITED_API))]
+#[cfg(any(not(Py_LIMITED_API), Py_3_12))]
 #[pymethods]
 impl UserDefinedWarning {
     #[new]
@@ -726,12 +726,12 @@ fn test_pyfunction_warn() {
         )]
     );
 
-    #[cfg(not(Py_LIMITED_API))]
+    #[cfg(any(not(Py_LIMITED_API), Py_3_12))]
     #[pyfunction]
     #[pyo3(warn(message = "TPW: this function raises user-defined warning", category = UserDefinedWarning))]
     fn function_with_warning_and_user_defined_category() {}
 
-    #[cfg(not(Py_LIMITED_API))]
+    #[cfg(any(not(Py_LIMITED_API), Py_3_12))]
     py_expect_warning_for_fn!(
         function_with_warning_and_user_defined_category,
         f,