PYCBC-1854: Migrate _core to the stable ABI (abi3)

Changes
--------
* Convert every C-extension type (logger, exceptions, result/streamed
  result/scan iterator, connection, transactions, hdr_histogram,
  kv_request) from static PyTypeObject structs to PyType_Spec/
  PyType_FromSpec heap types, using PyType_GenericAlloc for allocation
  and a shared free_heap_type_instance() helper for teardown: heap-type
  instances hold a reference to their type, so every tp_dealloc must
  free through the type's own Py_tp_free slot (a Python subclass is
  GC-allocated and needs PyObject_GC_Del) and then release that
  reference
* Regenerate pycbc_kv_request.cxx/.hxx via autogen from updated
  jinja2 templates carrying the same heap-type conversion
* Modernize module init: PyModule_AddType/PyModule_AddObjectRef
  replace the now-dead register_pytype/register_heap_type helpers
  (deleted) at all call sites; spot-sweep Py_INCREF+assign pairs to
  Py_NewRef
* Add limited-API compatibility shims in pytype_utils.hxx (Py_T_*
  PyMemberDef macros, a PyUnicode_AsUTF8 shim pre-3.13) and fix a
  missing include in pytocbpp_defs.hxx that needed it
* Add PYCBC_PY_LIMITED_API build option (default on, floor 3.10):
  CMakeLists.txt/setup.py/pycbc_build_setup.py wire it through to
  abi3 wheel tagging and bdist_wheel's --py-limited-api;
  python_requires corrected to >=3.10
* Build pycbc_core with hidden default visibility (no-op under MSVC),
  leaving PyInit__core -- which carries its own visibility("default")
  via PyMODINIT_FUNC -- as the module's only exported symbol
* Link python3.lib rather than the version-specific python3XY.lib on
  Windows for a limited-API build; explicitly-listed libraries beat
  the /DEFAULTLIB that pyconfig.h's auto-link pragma injects, so the
  old link would have bound an abi3-tagged .pyd to one interpreter
* Fix get_ext_filename/CMake filename agreement: a limited-API build
  emits <name>.abi3.<so|pyd>, and a non-limited build keeps the
  interpreter-specific suffix instead of an untagged .so that any
  interpreter's import machinery would load with no version check
* pycbc_build_setup.py: pass the package directory once, on every
  platform, as -DPYCBC_MODULE_OUTPUT_DIRECTORY, replacing the global
  -DCMAKE_LIBRARY_OUTPUT_DIRECTORY[_<CONFIG>] and the Windows-only
  -DCMAKE_RUNTIME_OUTPUT_DIRECTORY_<CONFIG>, which retargeted every
  library, executable and DLL in the build tree rather than just the
  extension module (on Windows that pulled BoringSSL's bssl.exe into
  the wheel)
* CMakeLists.txt: apply that path as a per-target property on
  pycbc_core only, selecting the category by platform (RUNTIME for the
  Windows DLL, LIBRARY everywhere else) and pointing
  PDB_OUTPUT_DIRECTORY at the build tree, including the per-config
  variants that multi-config generators require
* CMakeLists.txt: warn when PYCBC_MODULE_OUTPUT_DIRECTORY is unset;
  nothing else places the module, so the wheel would otherwise be
  packed without an extension and fail at import instead of at build
  time
* Modernize license metadata to SPDX (license="Apache-2.0",
  license_files), drop the now-deprecated OSI classifier, and raise
  the setuptools floor to >=70.1

Change-Id: I0f80fa57dd775df007600bf51f10007529a91c9b
Reviewed-on: https://review.couchbase.org/c/couchbase-python-client/+/250742
Tested-by: Build Bot <build@couchbase.com>
Reviewed-by: Dimitris Christodoulou <dimitris.christodoulou@couchbase.com>

Author: thejcfactor

CMakeLists.txt

@@ -67,10 +67,15 @@ endif()
 
 
 if(WIN32)
-  set(PYCBC_C_MOD_SUFFIX ".pyd")
+  set(PYCBC_C_MOD_SUFFIX_DEFAULT ".pyd")
 else()
-  set(PYCBC_C_MOD_SUFFIX ".so")
+  set(PYCBC_C_MOD_SUFFIX_DEFAULT ".so")
 endif()
+# Normally set by pycbc_build_setup.py to the building interpreter's own
+# sysconfig EXT_SUFFIX (e.g. .cpython-310-darwin.so). This is what makes a
+# version-specific (non-abi3) build loadable only by a matching interpreter;
+# the plain fallback below is for a raw `cmake` invocation outside setup.py.
+set(PYCBC_C_MOD_SUFFIX "${PYCBC_C_MOD_SUFFIX_DEFAULT}" CACHE STRING "Compiled extension module suffix, including any interpreter tag")
 
 file(READ "${PROJECT_SOURCE_DIR}/couchbase/_version.py" PYCBC_VERSION_CONTENTS)
 string(REGEX MATCH "__version__.*([0-9]+\\.[0-9]+\\.[0-9]+)(\\.?[+a-z0-9]*)" PYCBC_FOUND_VERSION ${PYCBC_VERSION_CONTENTS})
@@ -254,6 +259,36 @@ add_library(pycbc_core SHARED ${SOURCE_FILES})
 
 target_compile_definitions(pycbc_core PRIVATE COUCHBASE_CXX_CLIENT_IGNORE_CORE_DEPRECATIONS)
 
+# PYCBC-1854: Build against the Python Stable ABI (abi3) by default so a single
+# wheel can target every supported CPython. Power users can opt out via the
+# PYCBC_PY_LIMITED_API env var (handled in pycbc_build_setup.py) which passes
+# -DPYCBC_PY_LIMITED_API:BOOL=OFF here, producing a version-specific build.
+option(PYCBC_PY_LIMITED_API "Build the C extension against Py_LIMITED_API (abi3)" ON)
+# PYCBC_PY_LIMITED_API_HEX is the lowest CPython release this binary will load on.
+# Driven by PYCBC_PY_LIMITED_API_VERSION in pycbc_build_setup.py so it stays in
+# lockstep with the bdist_wheel --py-limited-api tag. 0x030A0000 == CPython 3.10,
+# the floor required by the limited-API symbols pycbc uses.
+set(PYCBC_PY_LIMITED_API_HEX "0x030A0000" CACHE STRING "Py_LIMITED_API hex (e.g. 0x030A0000 for CPython 3.10)")
+message(STATUS "PYCBC_PY_LIMITED_API=${PYCBC_PY_LIMITED_API}")
+set(PYCBC_C_MOD_OUTPUT_NAME "_core")
+if(PYCBC_PY_LIMITED_API)
+  message(STATUS "PYCBC_PY_LIMITED_API_HEX=${PYCBC_PY_LIMITED_API_HEX}")
+  target_compile_definitions(pycbc_core PRIVATE Py_LIMITED_API=${PYCBC_PY_LIMITED_API_HEX})
+  # abi3 tag goes on the suffix (_core.abi3.so), not the output name. The
+  # interpreter tag in PYCBC_C_MOD_SUFFIX is dropped here so the filename matches
+  # CMakeBuildExt.get_ext_filename(), which pairs ".abi3" with the bare .so.
+  string(REGEX REPLACE "^.*(\\.[^.]+)$" "\\1" PYCBC_C_MOD_SHARED_LIB_EXT "${PYCBC_C_MOD_SUFFIX}")
+  if(WIN32)
+    # Windows has no ".abi3" import suffix: importlib only recognizes
+    # ".cp3XY-win_amd64.pyd" and a bare ".pyd", so the stable-ABI module is _core.pyd.
+    set(PYCBC_C_MOD_FULL_SUFFIX "${PYCBC_C_MOD_SHARED_LIB_EXT}")
+  else()
+    set(PYCBC_C_MOD_FULL_SUFFIX ".abi3${PYCBC_C_MOD_SHARED_LIB_EXT}")
+  endif()
+else()
+  set(PYCBC_C_MOD_FULL_SUFFIX "${PYCBC_C_MOD_SUFFIX}")
+endif()
+
 target_include_directories(
   pycbc_core PRIVATE SYSTEM
   "${COUCHBASE_CXX_BINARY_DIR}/generated"
@@ -264,10 +299,28 @@ target_include_directories(
 set(COUCHBASE_CXX_CLIENT_TARGET couchbase_cxx_client_static_intermediate)
 
 if(WIN32)
+  # Python3_LIBRARIES is the version-specific python3XY.lib. Linking it in a stable-ABI
+  # build takes precedence over the python3.lib that pyconfig.h's MSVC auto-link pragma
+  # selects under Py_LIMITED_API, which would bind an abi3-tagged .pyd to one interpreter
+  # version. Prefer python3.lib explicitly, and fall back to the pragma if it isn't found.
+  if(PYCBC_PY_LIMITED_API)
+    find_library(
+      PYCBC_PYTHON3_SABI_LIBRARY
+      NAMES python3
+      HINTS ${Python3_LIBRARY_DIRS}
+      NO_DEFAULT_PATH)
+    set(PYCBC_PYTHON_LINK_LIBRARIES "")
+    if(PYCBC_PYTHON3_SABI_LIBRARY)
+      set(PYCBC_PYTHON_LINK_LIBRARIES ${PYCBC_PYTHON3_SABI_LIBRARY})
+    endif()
+    message(STATUS "PYCBC_PYTHON3_SABI_LIBRARY=${PYCBC_PYTHON3_SABI_LIBRARY}")
+  else()
+    set(PYCBC_PYTHON_LINK_LIBRARIES ${Python3_LIBRARIES})
+  endif()
   target_link_libraries(
     pycbc_core PRIVATE
     ${COUCHBASE_CXX_CLIENT_TARGET}
-    ${Python3_LIBRARIES}
+    ${PYCBC_PYTHON_LINK_LIBRARIES}
     asio
     Microsoft.GSL::GSL
     taocpp::json
@@ -299,8 +352,55 @@ if(NOT USE_STATIC_BORINGSSL)
   target_link_libraries(pycbc_core PUBLIC ${OPENSSL_LIBRARIES})
 endif()
 
+# PyInit__core carries its own visibility("default") via PyMODINIT_FUNC, so hiding
+# everything else keeps the module's exported surface to the one symbol CPython needs.
+# No-op on MSVC, which exports nothing without __declspec(dllexport).
 set_target_properties(
   pycbc_core
   PROPERTIES PREFIX ""
-             OUTPUT_NAME _core
-             SUFFIX ${PYCBC_C_MOD_SUFFIX})
+             OUTPUT_NAME ${PYCBC_C_MOD_OUTPUT_NAME}
+             SUFFIX ${PYCBC_C_MOD_FULL_SUFFIX}
+             C_VISIBILITY_PRESET hidden
+             CXX_VISIBILITY_PRESET hidden
+             VISIBILITY_INLINES_HIDDEN ON)
+
+# The extension module is the only build artifact that belongs in the Python package
+# directory, so the directory is set on the target rather than through a global
+# CMAKE_<category>_OUTPUT_DIRECTORY, which applies to every target in the build tree
+# (PYCBC-1878).  The category differs by platform: on Windows a DLL is RUNTIME, which also
+# covers executables, and everywhere else the module is LIBRARY.  PDB_OUTPUT_DIRECTORY
+# otherwise follows the target's own output directory, which would put the symbols back into
+# the package directory and therefore into the wheel.
+#
+# The path is used as passed.  These properties accept native separators, and
+# file(TO_CMAKE_PATH) splits on the host's path-list separator, so it is not a safe
+# normalizer for a single path.
+if(PYCBC_MODULE_OUTPUT_DIRECTORY)
+  message(STATUS "PYCBC_MODULE_OUTPUT_DIRECTORY=${PYCBC_MODULE_OUTPUT_DIRECTORY}")
+  if(WIN32)
+    set(PYCBC_MOD_DIR_CATEGORY RUNTIME)
+  else()
+    set(PYCBC_MOD_DIR_CATEGORY LIBRARY)
+  endif()
+  set_target_properties(
+    pycbc_core
+    PROPERTIES ${PYCBC_MOD_DIR_CATEGORY}_OUTPUT_DIRECTORY "${PYCBC_MODULE_OUTPUT_DIRECTORY}"
+               PDB_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
+  # Multi-config generators (Visual Studio, Xcode) append the configuration name unless the
+  # per-config property is set.  Single-config generators leave CMAKE_CONFIGURATION_TYPES
+  # empty, so this loop is a no-op there and the plain property above applies.
+  foreach(pycbc_config IN LISTS CMAKE_CONFIGURATION_TYPES)
+    string(TOUPPER "${pycbc_config}" pycbc_config_uc)
+    set_target_properties(
+      pycbc_core
+      PROPERTIES ${PYCBC_MOD_DIR_CATEGORY}_OUTPUT_DIRECTORY_${pycbc_config_uc} "${PYCBC_MODULE_OUTPUT_DIRECTORY}"
+                 PDB_OUTPUT_DIRECTORY_${pycbc_config_uc} "${CMAKE_BINARY_DIR}")
+  endforeach()
+else()
+  # Nothing else places the module, and the CMake-driven build_ext does not check, so the
+  # wheel would be packed without an extension and fail at import instead of here.
+  message(
+    WARNING
+      "PYCBC_MODULE_OUTPUT_DIRECTORY is not set: ${PYCBC_C_MOD_OUTPUT_NAME}${PYCBC_C_MOD_FULL_SUFFIX} will be left in the build tree, not the package directory"
+  )
+endif()

pycbc_build_setup.py

@@ -43,6 +43,57 @@
 ENV_TRUE = ['true', '1', 'y', 'yes', 'on']
 
 
+def use_py_limited_api() -> bool:
+    """Return True if the extension should be built against Py_LIMITED_API (abi3).
+
+    Defaults to True so we publish a single stable-ABI wheel per platform.
+    Power users can disable this by exporting ``PYCBC_PY_LIMITED_API=false``
+    (or 0/no/off) to build a CPython-version-specific binary.
+    """
+    return os.getenv('PYCBC_PY_LIMITED_API', 'true').lower() in ENV_TRUE
+
+
+# Lowest CPython version pycbc's C extension supports under the stable ABI.
+# Raising this floor frees up symbols (e.g. Py_T_OBJECT_EX is limited-API only
+# from 3.12), but invalidates wheels for older CPython releases.
+DEFAULT_PY_LIMITED_API_VERSION = '3.10'
+
+
+def _parse_py_limited_api_version(value: str) -> tuple:
+    try:
+        major_str, minor_str = value.split('.', 1)
+        major, minor = int(major_str), int(minor_str)
+    except (ValueError, AttributeError) as e:
+        raise OptionError(
+            f'PYCBC_PY_LIMITED_API_VERSION must be in MAJOR.MINOR form (e.g. "3.10"); got {value!r}'
+        ) from e
+    if major != 3 or minor < 10:
+        raise OptionError(
+            f'PYCBC_PY_LIMITED_API_VERSION must be 3.10 or newer; got {value!r}. '
+            'pycbc relies on stable-ABI symbols (PyType_FromSpec, PyUnicode_AsUTF8, ...) '
+            'first exposed in 3.10.'
+        )
+    return major, minor
+
+
+def py_limited_api_version() -> tuple:
+    """Return the (major, minor) CPython version that bounds the stable-ABI build."""
+    return _parse_py_limited_api_version(
+        os.getenv('PYCBC_PY_LIMITED_API_VERSION', DEFAULT_PY_LIMITED_API_VERSION))
+
+
+def py_limited_api_hex() -> str:
+    """Return the Py_LIMITED_API hex literal (e.g. '0x030A0000') passed to the C compiler."""
+    major, minor = py_limited_api_version()
+    return f'0x{major:02X}{minor:02X}0000'
+
+
+def py_limited_api_wheel_tag() -> str:
+    """Return the bdist_wheel --py-limited-api tag (e.g. 'cp310')."""
+    major, minor = py_limited_api_version()
+    return f'cp{major}{minor}'
+
+
 def check_for_cmake():
     if not CMAKE_EXE:
         print('cmake executable not found. '
@@ -120,6 +171,27 @@ def process_build_env_vars():  # noqa: C901
     if pycbc_tls_key_log_file is not None:
         cmake_extra_args += [f'-DCOUCHBASE_CXX_CLIENT_TLS_KEY_LOG_FILE={pycbc_tls_key_log_file}']
 
+    # Hand CMake this interpreter's own module suffix (e.g. .cpython-310-darwin.so)
+    # so it stays the single source of truth for both build modes: a version-specific
+    # build uses it as-is, while a stable-ABI build keeps only its trailing .so/.pyd
+    # (pairing the .so with ".abi3"), matching CMakeBuildExt.get_ext_filename() below.
+    # Without the interpreter tag, a version-specific build would produce a bare
+    # "_core.so" that any interpreter's import machinery will load, defeating the
+    # point of opting out of the stable ABI.
+    cmake_extra_args += [f'-DPYCBC_C_MOD_SUFFIX={get_config_var("EXT_SUFFIX")}']
+
+    # PYCBC_PY_LIMITED_API: build against the Python stable ABI (abi3).
+    # Defaults to ON; opt out for a Python-version-specific binary.
+    # The hex value (e.g. 0x030A0000 == 3.10) is the lowest CPython the resulting
+    # binary will load against, and must match the wheel tag set in BdistWheelCommand.
+    if use_py_limited_api():
+        cmake_extra_args += [
+            '-DPYCBC_PY_LIMITED_API:BOOL=ON',
+            f'-DPYCBC_PY_LIMITED_API_HEX={py_limited_api_hex()}',
+        ]
+    else:
+        cmake_extra_args += ['-DPYCBC_PY_LIMITED_API:BOOL=OFF']
+
     # now pop these in CMAKE_COMMON_VARIABLES, and they will be used by cmake...
     os.environ['CMAKE_COMMON_VARIABLES'] = ' '.join(cmake_extra_args)
 
@@ -143,11 +215,12 @@ def create_cmake_config(cls,  # noqa: C901
         build_type = env.pop('PYCBC_BUILD_TYPE')
         cmake_generator = env.pop('PYCBC_CMAKE_SET_GENERATOR', None)
         cmake_arch = env.pop('PYCBC_CMAKE_SET_ARCH', None)
+        # PYCBC_MODULE_OUTPUT_DIRECTORY is consumed by a per-target property, so only the
+        # extension module lands in the package directory (PYCBC-1878).
         cmake_config_args = [CMAKE_EXE,
                              source_dir,
                              f'-DCMAKE_BUILD_TYPE={build_type}',
-                             f'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={output_dir}',
-                             f'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{build_type.upper()}={output_dir}']
+                             f'-DPYCBC_MODULE_OUTPUT_DIRECTORY={output_dir}']
 
         cmake_config_args.extend(
             [x for x in
@@ -207,8 +280,6 @@ def create_cmake_config(cls,  # noqa: C901
                 cmake_config_args.append(f'-DCOUCHBASE_CXX_CLIENT_EMBED_MOZILLA_CA_BUNDLE_ROOT={CXXCBC_CACHE_DIR}')
 
         if platform.system() == "Windows":
-            cmake_config_args += [f'-DCMAKE_RUNTIME_OUTPUT_DIRECTORY_{build_type.upper()}={output_dir}']
-
             if cmake_generator:
                 if cmake_generator.upper() == 'TRUE':
                     cmake_config_args += ['-G', 'Visual Studio 16 2019']
@@ -232,8 +303,10 @@ def create_cmake_config(cls,  # noqa: C901
 
 
 class CMakeExtension(Extension):
-    def __init__(self, name, sourcedir=''):
-        Extension.__init__(self, name, sources=[])
+    def __init__(self, name, sourcedir='', py_limited_api=False):
+        # py_limited_api is a documented setuptools Extension kwarg; passing it
+        # via Extension.__init__ lets bdist_wheel auto-tag the wheel as abi3.
+        Extension.__init__(self, name, sources=[], py_limited_api=py_limited_api)
         self.sourcedir = os.path.abspath(sourcedir)
 
 
@@ -288,8 +361,25 @@ class CMakeBuildExt(build_ext):
 
     def get_ext_filename(self, ext_name):
         ext_path = ext_name.split('.')
-        ext_suffix = get_config_var('EXT_SUFFIX')
-        ext_suffix = "." + ext_suffix.split('.')[-1]
+        ext = next(
+            (e for e in self.distribution.ext_modules if e.name == ext_name),
+            None,
+        )
+        if ext is not None and getattr(ext, 'py_limited_api', False):
+            # Stable-ABI build: emit `<name>.abi3.so` so the file CMake produces
+            # (see CMakeLists.txt) matches what setuptools expects to find on disk,
+            # regardless of which interpreter compiled it. Windows is the exception:
+            # importlib knows only `.cp3XY-win_amd64.pyd` and `.pyd` there, so an
+            # `.abi3.pyd` is unimportable and the module must stay `<name>.pyd`.
+            so_ext = "." + get_config_var('EXT_SUFFIX').split('.')[-1]
+            ext_suffix = so_ext if platform.system() == 'Windows' else '.abi3' + so_ext
+        else:
+            # Full C-API build: keep the interpreter-specific suffix (e.g.
+            # .cpython-310-darwin.so) so this extension can't be picked up
+            # by a mismatched interpreter's import machinery. A bare .so
+            # would be loadable by any version, defeating the point of a
+            # non-limited build.
+            ext_suffix = get_config_var('EXT_SUFFIX')
         return os.path.join(*ext_path) + ext_suffix
 
     def build_extension(self, ext):  # noqa: C901
@@ -340,6 +430,33 @@ def _clean_cache_cpm_dependencies(self):
                     print(line.rstrip())
 
 
+# bdist_wheel moved from the `wheel` package into setuptools in setuptools 70.1.
+# Prefer the setuptools-vendored location so we work without an explicit `wheel`
+# build dependency, but fall back for older setuptools.
+try:
+    from setuptools.command.bdist_wheel import bdist_wheel as _bdist_wheel
+except ImportError:  # pragma: no cover - older setuptools
+    from wheel.bdist_wheel import bdist_wheel as _bdist_wheel  # type: ignore[no-redef]
+
+
+class BdistWheelCommand(_bdist_wheel):
+    """bdist_wheel that tags the wheel as abi3 when building against Py_LIMITED_API.
+
+    `pip wheel .` invokes bdist_wheel without flags, so even if Extension
+    declares py_limited_api=True the wheel still gets a CPython-specific tag.
+    Default --py-limited-api here so the produced wheel is named
+    `<dist>-<ver>-cp310-abi3-<platform>.whl` and installs onto any CPython >=3.10.
+
+    Both the wheel tag and the C-level Py_LIMITED_API hex come from
+    py_limited_api_version() so they cannot drift apart.
+    """
+
+    def finalize_options(self):
+        if use_py_limited_api() and not self.py_limited_api:
+            self.py_limited_api = py_limited_api_wheel_tag()
+        super().finalize_options()
+
+
 class BuildCommand(build):
     def finalize_options(self):
         # Setting the build_base to an absolute path will make sure that build (i.e. temp) and lib dirs are in sync

pyproject.toml

@@ -1,6 +1,9 @@
 [build-system]
 requires = [
-  "setuptools>=42",
+  # 70.1 is when bdist_wheel moved into setuptools proper; see the fallback
+  # import in pycbc_build_setup.py. Also comfortably new enough to treat
+  # setup.py's SPDX `license="Apache-2.0"` as a real license expression.
+  "setuptools>=70.1",
   "wheel",
 ]
 build-backend = "setuptools.build_meta"