Refactor dpnp includes and add extension example (#2941)

This PR proposes a refactor of dpnp's includes and the introduction of
an example extension, to be tested to make sure examples using dpnp and
dpnp.tensor can be written.

The `usm_ndarray_constants.h` header and `dpnp4pybind11.hpp` are moved
to `dpnp/include`, removing a relative include and aligning with
previous dpctl behavior.

Also introduces dpnp-config.cmake, which enables `find_package(Dpnp)`
out of the box, which is installed with dpnp, and `--cmakedir` command
line option, which leads to the CMake config.

Author: ndgrigorian

.github/workflows/conda-package.yml

@@ -269,6 +269,113 @@ jobs:
 
             python -m pytest -n auto -ra --pyargs dpnp.tests.tensor
 
+  test_examples_linux:
+    name: Test examples
+
+    needs: build
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    strategy:
+      matrix:
+        python: ['3.14']
+
+    env:
+      dpnp-repo-path: '${{ github.workspace }}/source/'
+      examples-conda-pkg-env: 'source/environments/examples.yml'
+      channel-path: '${{ github.workspace }}/channel/'
+      pkg-path-in-channel: '${{ github.workspace }}/channel/linux-64/'
+      ver-json-path: '${{ github.workspace }}/version.json'
+      examples-env-name: 'examples_test'
+
+    steps:
+      - name: Checkout DPNP repo
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          fetch-depth: ${{ env.fetch-depth }}
+          path: ${{ env.dpnp-repo-path }}
+
+      - name: Download artifact
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: ${{ env.package-name }} ${{ runner.os }} Python ${{ matrix.python }}
+          path: ${{ env.pkg-path-in-channel }}
+
+      - name: Setup miniconda
+        id: setup_miniconda
+        continue-on-error: true
+        uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1
+        with:
+          miniforge-version: latest
+          use-mamba: 'true'
+          conda-remove-defaults: 'true'
+          environment-file: ${{ env.examples-conda-pkg-env }}
+          activate-environment: ${{ env.examples-env-name }}
+
+      - name: ReSetup miniconda
+        if: steps.setup_miniconda.outcome == 'failure'
+        uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1
+        with:
+          miniforge-version: latest
+          use-mamba: 'true'
+          conda-remove-defaults: 'true'
+          environment-file: ${{ env.examples-conda-pkg-env }}
+          activate-environment: ${{ env.examples-env-name }}
+
+      - name: Create conda channel
+        run: |
+          python -m conda_index ${{ env.channel-path }}
+
+      - name: Test conda channel
+        run: |
+          conda search ${{ env.package-name }} -c ${{ env.channel-path }} --override-channels --info --json > ${{ env.ver-json-path }}
+          cat ${{ env.ver-json-path }}
+
+      - name: Get package version
+        run: |
+          PACKAGE_VERSION=$(python -c "${{ env.ver-script-part1 }} ${{ env.ver-script-part2 }}")
+          echo "PACKAGE_VERSION=${PACKAGE_VERSION}" >> "$GITHUB_ENV"
+
+      - name: Install dpnp
+        id: install_dpnp
+        continue-on-error: true
+        run: |
+          mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} python=${{ matrix.python }} ${{ env.TEST_CHANNELS }}
+        env:
+          TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}'
+
+      - name: ReInstall dpnp
+        if: steps.install_dpnp.outcome == 'failure'
+        run: |
+          mamba install ${{ env.package-name }}=${{ env.PACKAGE_VERSION }} python=${{ matrix.python }} ${{ env.TEST_CHANNELS }}
+        env:
+          TEST_CHANNELS: '-c ${{ env.channel-path }} ${{ env.channels-list }}'
+
+      - name: List installed packages
+        run: mamba list
+
+      - name: Smoke test
+        run: |
+          python -c "import dpctl; dpctl.lsplatform()"
+          python -c "import dpnp; print(dpnp.__version__)"
+
+      - name: Build and run pybind11 examples
+        run: |
+          cd ${{ env.dpnp-repo-path }}/examples/pybind11
+          for d in */; do
+            pushd "$d" > /dev/null
+            CC=icx CXX=icpx python setup.py build_ext --inplace -G Ninja || exit 1
+            if [ -d tests ]; then
+              python -m pytest tests || exit 1
+            fi
+            popd > /dev/null
+          done
+
   test_windows:
     name: Test
 
@@ -451,7 +558,7 @@ jobs:
   upload:
     name: Upload
 
-    needs: [test_linux, test_windows]
+    needs: [test_linux, test_windows, test_examples_linux]
 
     strategy:
       fail-fast: false

.gitignore

@@ -2,7 +2,7 @@
 _skbuild
 build_cython
 cython_debug
-dpnp.egg-info
+*.egg-info
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
@@ -27,9 +27,12 @@ dpnp_pytest.*
 # Build examples
 example3
 
+# moved cmake scripts
+dpnp/resources/cmake
+
 *dpnp_backend*
 dpnp/include/dpnp/tensor/*.h
-dpnp/**/*.cpython*.so
 dpnp/**/*.pyd
 *~
 core
+*.so

CHANGELOG.md

@@ -13,6 +13,7 @@ This release is compatible with NumPy 2.5.
 * Added C API functions for `dpnp.tensor.usm_ndarray` setters and getters to avoid ABI breakage if `dpnp.tensor.usm_ndarray` is modified [#2866](https://github.com/IntelPython/dpnp/pull/2866)
 * Added support for buffer protocol objects as advanced index keys in `dpnp.ndarray` [#2889](https://github.com/IntelPython/dpnp/pull/2889)
 * Added `--includes` and `--include-dir` options to the `dpnp` CLI [#2916](https://github.com/IntelPython/dpnp/pull/2916)
+* Added `dpnp-config.cmake` to make `find_package(Dpnp)` work out of the box, and an example which uses it [#2941](https://github.com/IntelPython/dpnp/pull/2941)
 
 ### Changed
 

CMakeLists.txt

@@ -327,6 +327,19 @@ else()
     message(FATAL_ERROR "Unsupported system.")
 endif()
 
+# install dpnp-config.cmake for find_package() support
+foreach(_dest IN ITEMS dpnp/resources/cmake lib/cmake/dpnp)
+    install(FILES ${CMAKE_SOURCE_DIR}/cmake/dpnp-config.cmake DESTINATION ${_dest})
+endforeach()
+
+# install dpnp4pybind11.hpp and usm_ndarray_constants.h into include folder
+install(
+    FILES
+        ${CMAKE_SOURCE_DIR}/dpnp/include/dpnp4pybind11.hpp
+        ${CMAKE_SOURCE_DIR}/dpnp/include/usm_ndarray_constants.h
+    DESTINATION dpnp/include
+)
+
 # Define flags for CMAKE_BUILD_TYPE=Coverage
 set(CMAKE_C_FLAGS_COVERAGE "${CMAKE_C_FLAGS_DEBUG} -O1 -g1 -DDEBUG")
 set(CMAKE_CXX_FLAGS_COVERAGE "${CMAKE_CXX_FLAGS_DEBUG} -O1 -g1 -DDEBUG")

cmake/dpnp-config.cmake

@@ -0,0 +1,69 @@
+#.rst:
+#
+# Find the include directory for ``dpnp4pybind11.hpp`` and dpnp tensor kernels.
+#
+# This module sets the following variables:
+#
+# ``Dpnp_FOUND``
+#   True if DPNP was found.
+# ``Dpnp_INCLUDE_DIR``
+#   The include directory needed to use dpnp.
+# ``Dpnp_TENSOR_INCLUDE_DIR``
+#   The include directory for tensor kernels implementation.
+# ``Dpnp_VERSION``
+#   The version of dpnp found.
+#
+# The module will also explicitly define two cache variables:
+#
+# ``Dpnp_INCLUDE_DIR``
+# ``Dpnp_TENSOR_INCLUDE_DIR``
+#
+
+if(NOT Dpnp_FOUND)
+    find_package(Python 3.10 REQUIRED COMPONENTS Interpreter Development.Module)
+
+    if(Python_EXECUTABLE)
+        execute_process(
+            COMMAND "${Python_EXECUTABLE}" -m dpnp --include-dir
+            OUTPUT_VARIABLE _dpnp_include_dir
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            ERROR_QUIET
+        )
+        execute_process(
+            COMMAND "${Python_EXECUTABLE}" -c "import dpnp; print(dpnp.__version__)"
+            OUTPUT_VARIABLE Dpnp_VERSION
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            ERROR_QUIET
+        )
+    endif()
+endif()
+
+find_path(
+    Dpnp_INCLUDE_DIR
+    dpnp4pybind11.hpp
+    PATHS "${_dpnp_include_dir}" "${Python_INCLUDE_DIRS}"
+    PATH_SUFFIXES dpnp/include
+    NO_DEFAULT_PATH
+)
+get_filename_component(_dpnp_dir "${Dpnp_INCLUDE_DIR}" DIRECTORY)
+
+find_path(
+    Dpnp_TENSOR_INCLUDE_DIR
+    kernels
+    PATHS "${_dpnp_dir}/tensor/libtensor/include"
+    NO_DEFAULT_PATH
+)
+
+set(Dpnp_INCLUDE_DIRS ${Dpnp_INCLUDE_DIR})
+
+# handle the QUIETLY and REQUIRED arguments and set Dpnp_FOUND to TRUE if
+# all listed variables are TRUE
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(
+    Dpnp
+    REQUIRED_VARS Dpnp_INCLUDE_DIR Dpnp_TENSOR_INCLUDE_DIR
+    VERSION_VAR Dpnp_VERSION
+)
+
+mark_as_advanced(Dpnp_INCLUDE_DIR)
+mark_as_advanced(Dpnp_TENSOR_INCLUDE_DIR)

dpnp/__main__.py

@@ -41,7 +41,7 @@ def _dpnp_dir() -> str:
 
 def get_include_dir() -> str:
     """Returns path to dpnp include directory containing dpnp4pybind11.hpp"""
-    return os.path.join(_dpnp_dir(), "backend", "include")
+    return os.path.join(_dpnp_dir(), "include")
 
 
 def print_include_flags() -> None:
@@ -61,6 +61,13 @@ def print_tensor_include_flags() -> None:
     print("-I " + libtensor_dir)
 
 
+def print_cmake_dir() -> None:
+    """Prints directory with dpnp-config.cmake"""
+    dpnp_dir = _dpnp_dir()
+    cmake_dir = os.path.join(dpnp_dir, "resources", "cmake")
+    print(cmake_dir)
+
+
 def main() -> None:
     """Main entry-point."""
     parser = argparse.ArgumentParser()
@@ -84,6 +91,11 @@ def main() -> None:
         action="store_true",
         help="Path to dpnp libtensor include directory.",
     )
+    parser.add_argument(
+        "--cmakedir",
+        action="store_true",
+        help="CMake module directory, ideal for setting -DDpnp_ROOT in CMake.",
+    )
     args = parser.parse_args()
     if not sys.argv[1:]:
         parser.print_help()
@@ -95,6 +107,8 @@ def main() -> None:
         print_tensor_include_flags()
     if args.tensor_include_dir:
         print(get_tensor_include_dir())
+    if args.cmakedir:
+        print_cmake_dir()
 
 
 if __name__ == "__main__":

dpnp/backend/extensions/blas/CMakeLists.txt

@@ -70,7 +70,7 @@ target_include_directories(
     ${python_module_name}
     PRIVATE
         ${CMAKE_CURRENT_SOURCE_DIR}/../common
-        ${CMAKE_SOURCE_DIR}/dpnp/backend/include
+        ${CMAKE_SOURCE_DIR}/dpnp/include
         ${CMAKE_SOURCE_DIR}/dpnp/tensor/libtensor/include
 )
 

dpnp/backend/extensions/blas/dot_common.hpp

@@ -32,6 +32,8 @@
 
 #include <pybind11/pybind11.h>
 
+#include "dpnp4pybind11.hpp"
+
 // dpnp tensor headers
 #include "utils/memory_overlap.hpp"
 #include "utils/output_validation.hpp"

dpnp/backend/extensions/fft/CMakeLists.txt

@@ -63,7 +63,7 @@ set_target_properties(
 target_include_directories(
     ${python_module_name}
     PRIVATE
-        ${CMAKE_SOURCE_DIR}/dpnp/backend/include
+        ${CMAKE_SOURCE_DIR}/dpnp/include
         ${CMAKE_SOURCE_DIR}/dpnp/tensor/libtensor/include
 )
 

dpnp/backend/extensions/indexing/CMakeLists.txt

@@ -68,7 +68,7 @@ target_include_directories(
     PRIVATE
         ${CMAKE_CURRENT_SOURCE_DIR}/../../
         ${CMAKE_CURRENT_SOURCE_DIR}/../common
-        ${CMAKE_SOURCE_DIR}/dpnp/backend/include
+        ${CMAKE_SOURCE_DIR}/dpnp/include
         ${CMAKE_SOURCE_DIR}/dpnp/tensor/libtensor/include
 )