CPython 3.10 support

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
       # Duplicate changes to this matrix to 'poc_tests'
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.9']
+        python-version: ['3.9', '3.10']
         # Possible values: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10.0-rc.2']
         include:
           - os: ubuntu-latest
@@ -49,7 +49,7 @@ jobs:
       - name: Run tests
         run: |
           python -m pip install pytest pytest-xdist
-          python -m pytest -n auto test/
+          python -m pytest --durations=16 -n auto test/
 
   main_tests_debug:
     name: Main tests on CPython debug builds
@@ -58,7 +58,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python-version: ['3.9', '3.8', '3.7']
+        python-version: ['3.10', '3.9', '3.8', '3.7']
 
     env:
       SETUPTOOLS_USE_DISTUTILS: stdlib
@@ -84,7 +84,7 @@ jobs:
       - name: Run tests
         run: |
           python -m pip install pytest pytest-xdist
-          python -m pytest -n auto test/
+          python -m pytest --durations=16 -n auto test/
 
   poc_tests:
     name: Proof of concept tests
@@ -101,6 +101,8 @@ jobs:
             python-version: '3.7'
           - os: ubuntu-latest
             python-version: '3.8'
+          - os: ubuntu-latest
+            python-version: '3.10'
           ## Expected failure
           # - os: ubuntu-latest
           #   python-version: '3.10.0-rc.2'

hpy/devel/include/hpy/hpytype.h

@@ -13,6 +13,30 @@ typedef struct {
     int basicsize;
     int itemsize;
     unsigned long flags;
+    /*
+       A type whose struct starts with PyObject_HEAD is a legacy type. A
+       legacy type must set .legacy = true in its HPyType_Spec.
+       A type is a non-legacy type, also called HPy pure type, if its struct
+       does not include PyObject_HEAD. Using pure types should be preferred.
+       Legacy types are available to allow gradual porting of existing CPython
+       extensions.
+
+       A type with .legacy_slots not NULL is required to have .legacy = true and to
+       include PyObject_HEAD at the start of its struct. It would be easy to
+       relax this requirement on CPython (where the PyObject_HEAD fields are
+       always present) but a large burden on other implementations (e.g. PyPy,
+       GraalPython) where a struct starting with PyObject_HEAD might not exist.
+
+       Types that do not define a struct of their own, should set the value of
+       .legacy to the same value as the type they inherit from. If they inherit
+       from a built-in type, they may set .legacy to either true or false, depending
+       on whether they still use .legacy_slots or not.
+
+       Pure HPy types that inherit a builtin type and define their own struct are
+       not supported at the moment. One can use legacy types in the meanwhile.
+
+       Types created via the old Python C API are automatically legacy types.
+     */
     int legacy;
     void *legacy_slots; // PyType_Slot *
     HPyDef **defines;   /* points to an array of 'HPyDef *' */
@@ -36,37 +60,6 @@ typedef struct {
 #define _Py_TPFLAGS_HAVE_VERSION_TAG (1UL << 18)
 #define HPy_TPFLAGS_DEFAULT (_Py_TPFLAGS_HEAPTYPE | _Py_TPFLAGS_HAVE_VERSION_TAG)
 
-/* HPy_TPFLAGS_INTERNAL_PURE is set automatically on pure types created with
-   HPyType_FromSpec. This flag should not be used directly. Set
-   `.legacy = false` or `.legacy = true` instead.
-
-   A custom type is a pure type if its struct does not include PyObject_HEAD.
-   A type whose struct does start with PyObject_HEAD is a legacy type. A
-   legacy type must set .legacy = true in its HPyType_Spec.
-
-   A type with .legacy_slots not NULL is required to have .legacy = true and to
-   include PyObject_HEAD at the start of its struct. It would be easy to
-   relax this requirement on CPython (where the PyObject_HEAD fields are
-   always present) but a large burden on other implementations (e.g. PyPy,
-   GraalPython) where a struct starting with PyObject_HEAD might not exist.
-
-   Types that do not define a struct of their own, should set the value of
-   .legacy to the same value as the type they inherit from. If they inherit
-   from a built-in type, they may .legacy to either true or false, depending on
-   whether they still use .legacy_slots or not.
-
-   Types created via the old Python C API are automatically legacy types.
-
-   Note on the choice of bit 8: Bit 8 looks likely to be the last free TPFLAG
-   bit that C Python will allocate. Bits 0 to 8 were dropped in Python 3.0 and
-   are being re-allocated slowly from 0 towards 8. As of 3.10, only bit 0 has
-   been re-allocated.
-*/
-#define HPy_TPFLAGS_INTERNAL_PURE (1UL << 8)
-
-#define HPy_TPFLAGS_INTERNAL_IS_HPY_TYPE (1UL << 7)
-
-
 /* Set if the type allows subclassing */
 #define HPy_TPFLAGS_BASETYPE (1UL << 10)
 

hpy/devel/src/runtime/ctx_type.c

@@ -9,6 +9,20 @@
 #  include "handles.h"
 #endif
 
+/* HPy_TPFLAGS_INTERNAL_IS_HPY_TYPE is set automatically on HPy types created
+   with HPyType_FromSpec. This is used internally within HPy to distinguish
+   HPy types.
+
+   Note on the choice of bit 5: CPython uses bit 0 and all bits from 5 up.
+   Using a random currently unused bit in type flags is a temporary solution.
+   Going forward, HPy may ask CPython to reserve one bit for HPy or find
+   another more reliable solution.
+*/
+#define HPy_TPFLAGS_INTERNAL_IS_HPY_TYPE (1UL << 4)
+
+#define HPy_TYPE_MAGIC 0xba5f
+
+
 static bool has_tp_traverse(HPyType_Spec *hpyspec);
 static bool needs_hpytype_dealloc(HPyType_Spec *hpyspec);
 
@@ -18,17 +32,29 @@ static bool needs_hpytype_dealloc(HPyType_Spec *hpyspec);
    of type HPyType_Extra_t, which we never free for now.  We can access
    it because tp->tp_name points to the "name" field at the end... */
 typedef struct {
+    uint16_t magic;
     HPyFunc_traverseproc tp_traverse_impl;
     HPyFunc_destroyfunc tp_destroy_impl;
+    bool is_pure;
     char name[];
 } HPyType_Extra_t;
 
+static inline bool _is_HPyType(PyTypeObject *tp) {
+    return tp->tp_flags & HPy_TPFLAGS_INTERNAL_IS_HPY_TYPE;
+}
+
 static inline HPyType_Extra_t *_HPyType_EXTRA(PyTypeObject *tp) {
-    assert(tp->tp_flags & HPy_TPFLAGS_INTERNAL_IS_HPY_TYPE);
-    return (HPyType_Extra_t *)(tp->tp_name - offsetof(HPyType_Extra_t, name));
+    assert(_is_HPyType(tp));
+    HPyType_Extra_t *result = (HPyType_Extra_t *)(tp->tp_name - offsetof(HPyType_Extra_t, name));
+    assert(result->magic == HPy_TYPE_MAGIC);
+    return result;
+}
+
+static inline bool _is_pure_HPyType(PyTypeObject *tp) {
+    return _is_HPyType(tp) && _HPyType_EXTRA(tp)->is_pure;
 }
 
-static HPyType_Extra_t *_HPyType_Extra_Alloc(const char *name)
+static HPyType_Extra_t *_HPyType_Extra_Alloc(const char *name, bool is_pure)
 {
     size_t name_size = strlen(name) + 1;
     size_t size = offsetof(HPyType_Extra_t, name) + name_size;
@@ -38,13 +64,15 @@ static HPyType_Extra_t *_HPyType_Extra_Alloc(const char *name)
         return NULL;
     }
     memcpy(result->name, name, name_size);
+    result->is_pure = is_pure;
+    result->magic = HPy_TYPE_MAGIC;
     /* XXX the returned struct is never freed */
     return result;
 }
 
 static void *_pyobj_as_struct(PyObject *obj)
 {
-    if (Py_TYPE(obj)->tp_flags & HPy_TPFLAGS_INTERNAL_PURE) {
+    if (_is_pure_HPyType(Py_TYPE(obj))) {
         return _HPy_PyObject_Payload(obj);
     }
     else {
@@ -66,7 +94,7 @@ static void hpytype_clear(PyObject *self)
     PyTypeObject *tp = Py_TYPE(self);
     PyTypeObject *base = tp;
     while(base) {
-        if (base->tp_flags & HPy_TPFLAGS_INTERNAL_IS_HPY_TYPE) {
+        if (_is_HPyType(base)) {
             HPyType_Extra_t *extra = _HPyType_EXTRA(base);
             assert(extra != NULL);
             if (extra->tp_traverse_impl != NULL) {
@@ -98,7 +126,7 @@ static void hpytype_dealloc(PyObject *self)
     // call tp_destroy on all the HPy types of the hierarchy
     PyTypeObject *base = tp;
     while(base) {
-        if (base->tp_flags & HPy_TPFLAGS_INTERNAL_IS_HPY_TYPE) {
+        if (_is_HPyType(base)) {
             HPyType_Extra_t *extra = _HPyType_EXTRA(base);
             assert(extra != NULL);
             if (extra->tp_destroy_impl != NULL) {
@@ -546,12 +574,6 @@ static int check_legacy_consistent(HPyType_Spec *hpyspec)
             "cannot specify .legacy_slots without setting .legacy=true");
         return -1;
     }
-    if (hpyspec->flags & HPy_TPFLAGS_INTERNAL_PURE) {
-        PyErr_SetString(PyExc_TypeError,
-            "HPy_TPFLAGS_INTERNAL_PURE should not be used directly,"
-            " set .legacy=true instead");
-        return -1;
-    }
     if (hpyspec->legacy_slots && needs_hpytype_dealloc(hpyspec)) {
         PyType_Slot *legacy_slots = (PyType_Slot *)hpyspec->legacy_slots;
         for (int i = 0; legacy_slots[i].slot != 0; i++) {
@@ -603,8 +625,8 @@ static int check_have_gc_and_tp_traverse(HPyContext *ctx, HPyType_Spec *hpyspec)
 
 static int check_inheritance_constraints(PyTypeObject *tp)
 {
-    int tp_pure = tp->tp_flags & HPy_TPFLAGS_INTERNAL_PURE;
-    int tp_base_pure = tp->tp_base->tp_flags & HPy_TPFLAGS_INTERNAL_PURE;
+    int tp_pure = _is_pure_HPyType(tp);
+    int tp_base_pure = _is_pure_HPyType(tp->tp_base);
     if (tp_pure) {
         // Pure types may inherit from:
         //
@@ -693,10 +715,11 @@ ctx_Type_FromSpec(HPyContext *ctx, HPyType_Spec *hpyspec,
     HPy_ssize_t base_member_offset;
     unsigned long flags = hpyspec->flags;
 
+    bool is_pure;
     if (hpyspec->legacy != 0) {
         basicsize = hpyspec->basicsize;
         base_member_offset = 0;
-        flags &= ~HPy_TPFLAGS_INTERNAL_PURE;
+        is_pure = false;
     }
     else {
         // _HPy_PyObject_HEAD_SIZE ensures that the custom struct is
@@ -712,9 +735,9 @@ ctx_Type_FromSpec(HPyContext *ctx, HPyType_Spec *hpyspec,
             basicsize = 0;
             base_member_offset = 0;
         }
-        flags |= HPy_TPFLAGS_INTERNAL_PURE;
+        is_pure = true;
     }
-    HPyType_Extra_t *extra = _HPyType_Extra_Alloc(hpyspec->name);
+    HPyType_Extra_t *extra = _HPyType_Extra_Alloc(hpyspec->name, is_pure);
     if (extra == NULL) {
         PyMem_Free(spec);
         return HPy_NULL;
@@ -762,6 +785,7 @@ ctx_Type_FromSpec(HPyContext *ctx, HPyType_Spec *hpyspec,
         Py_DECREF(result);
         return HPy_NULL;
     }
+    assert(_is_HPyType((PyTypeObject*) result));
     return _py2h(result);
 }
 
@@ -800,7 +824,7 @@ ctx_New(HPyContext *ctx, HPy h_type, void **data)
     Py_INCREF(tp);
 #endif
 
-    if (tp->tp_flags & HPy_TPFLAGS_INTERNAL_PURE) {
+    if (_is_pure_HPyType(tp)) {
         // For pure HPy custom types, we return a pointer to only the custom
         // struct data, without the hidden PyObject header.
         *data = _HPy_PyObject_Payload(result);

test/test_hpylong.py

@@ -34,6 +34,16 @@ def python_supports_magic_index(self):
         vi = sys.version_info
         return (vi.major > 3 or (vi.major == 3 and vi.minor >= 8))
 
+    def python_supports_magic_int(self):
+        """ Return True if the Python version is 3.9 or earlier and thus
+            should support calling __int__ on non-int based types in some
+            HPyLong_As... methods.
+        """
+        import sys
+        vi = sys.version_info
+        assert vi.major >= 3
+        return (vi.major == 3 and vi.minor <= 9)
+
     def test_Long_FromLong(self):
         mod = self.make_module("""
             HPyDef_METH(f, "f", f_impl, HPyFunc_NOARGS)
@@ -64,7 +74,8 @@ def test_Long_AsLong(self):
         assert mod.f(45) == 90
         with pytest.raises(TypeError):
             mod.f("this is not a number")
-        assert mod.f(self.magic_int(2)) == 4
+        if self.python_supports_magic_int():
+            assert mod.f(self.magic_int(2)) == 4
         if self.python_supports_magic_index():
             assert mod.f(self.magic_index(2)) == 4
 
@@ -123,7 +134,8 @@ def test_Long_AsUnsignedLongMask(self):
         assert mod.f(-1) == 2**self.unsigned_long_bits() - 1
         with pytest.raises(TypeError):
             mod.f("this is not a number")
-        assert mod.f(self.magic_int(2)) == 2
+        if self.python_supports_magic_int():
+            assert mod.f(self.magic_int(2)) == 2
         if self.python_supports_magic_index():
             assert mod.f(self.magic_index(2)) == 2
 
@@ -159,7 +171,8 @@ def test_Long_AsLongLong(self):
         assert mod.f(-2147483648) == -2147483648
         with pytest.raises(TypeError):
             mod.f("this is not a number")
-        assert mod.f(self.magic_int(2)) == 2
+        if self.python_supports_magic_int():
+            assert mod.f(self.magic_int(2)) == 2
         if self.python_supports_magic_index():
             assert mod.f(self.magic_index(2)) == 2
 
@@ -219,7 +232,8 @@ def test_Long_AsUnsignedLongLongMask(self):
         assert mod.f(-1) == 2**64 - 1
         with pytest.raises(TypeError):
             mod.f("this is not a number")
-        assert mod.f(self.magic_int(2)) == 2
+        if self.python_supports_magic_int():
+            assert mod.f(self.magic_int(2)) == 2
         if self.python_supports_magic_index():
             assert mod.f(self.magic_index(2)) == 2
 

test/test_hpytype.py

@@ -796,22 +796,3 @@ def test_specparam_basestuple(self):
         class Sub(mod.Dummy):
             pass
         assert isinstance(Sub(), mod.Dummy)
-
-    def test_directly_setting_hpy_tpflags_internal_pure_raises(self):
-        import pytest
-        mod_src = """
-            static HPyType_Spec Dummy_spec = {
-                .name = "mytest.Dummy",
-                .itemsize = 0,
-                .flags = HPy_TPFLAGS_DEFAULT | HPy_TPFLAGS_INTERNAL_PURE,
-                @IS_LEGACY
-            };
-
-            @EXPORT_TYPE("Dummy", Dummy_spec)
-            @INIT
-        """
-        with pytest.raises(TypeError) as err:
-            self.make_module(mod_src)
-        assert str(err.value) == (
-            "HPy_TPFLAGS_INTERNAL_PURE should not be used directly,"
-            " set .legacy=true instead")