MT#55283 kernel-module: detect nft_expr_ops.validate signature via compile test

The current LINUX_VERSION_CODE check for the nft_expr_ops.validate
callback signature breaks on distribution kernels that backport the
API change (mainline commit eaf9b2c875ec, merged in 6.12) without
updating LINUX_VERSION_CODE.

For example, Ubuntu 24.04's 6.8.0-103+ kernel (stable patchset
2026-01-27, LP: #2139158) includes this backport, causing DKMS
builds to fail with -Werror=incompatible-pointer-types.

Replace the version-based #if with a compile test in the existing
gen-rtpengine-kmod-flags configure script. The test tries to assign
a 3-param function to .validate -- if it compiles, the old API is
present and NFT_EXPR_OPS_VALIDATE_HAS_DATA is set. If it fails, the
kernel has the new 2-param version.

Also use kbuild's KERNELRELEASE variable (instead of uname -r) to
resolve the kernel build directory, so that compile tests and KSRC
target the correct kernel during cross-version DKMS builds.

Tested against Ubuntu 6.8.0-90 (3-param) and 6.8.0-106 (2-param),
including cross-kernel builds where the running kernel differs from
the DKMS target.

Closes #2085

Change-Id: I4c8e55b94fb98c3fcda9dccb091d0d1c0c67f9aa
(cherry picked from commit b1aec46)
(cherry picked from commit 4ea947f)

Author: andrewyager

kernel-module/Makefile

@@ -1,4 +1,4 @@
-KSRC   ?= /lib/modules/$(shell uname -r)/build
+KSRC   ?= /lib/modules/$(or $(KERNELRELEASE),$(shell uname -r))/build
 KBUILD := $(KSRC)
 M      ?= $(CURDIR)
 

kernel-module/gen-rtpengine-kmod-flags

@@ -44,3 +44,39 @@ fi
 if [ -n "${RTPENGINE_VERSION}" ]; then
   echo "RTPENGINE_VERSION := ${RTPENGINE_VERSION}"
 fi
+
+# Compile test: detect nft_expr_ops.validate callback signature.
+# Distribution kernels (e.g. Ubuntu) may backport the 2-param validate
+# callback (mainline commit eaf9b2c875ec, merged in 6.12) to earlier
+# kernel versions without changing LINUX_VERSION_CODE. A compile test
+# is the only reliable way to detect the actual API.
+#
+# The test tries to assign a 3-param function to .validate. If it
+# compiles, the old API is present and we set the flag. If it fails
+# (incompatible pointer types), the kernel has the new 2-param version.
+
+# KERNELRELEASE is set by kbuild to the target kernel version, which
+# may differ from the running kernel during DKMS cross-kernel builds.
+KSRC="${KSRC:-/lib/modules/${KERNELRELEASE:-$(uname -r)}/build}"
+
+if [ -d "${KSRC}" ]; then
+  nft_test_dir=$(mktemp -d)
+  cat > "${nft_test_dir}/nft_test.c" <<'TESTEOF'
+#include <linux/module.h>
+#include <net/netfilter/nf_tables.h>
+static int test_validate(const struct nft_ctx *ctx, const struct nft_expr *expr,
+    const struct nft_data **data) { return 0; }
+static struct nft_expr_ops __attribute__((unused)) test_ops = {
+    .validate = test_validate };
+MODULE_LICENSE("GPL");
+TESTEOF
+  echo "obj-m := nft_test.o" > "${nft_test_dir}/Makefile"
+
+  # Use env -i to prevent kbuild state (KBUILD_EXTMOD, M, MAKEFLAGS, etc.)
+  # leaking from an outer kbuild invocation into the compile test.
+  if env -i PATH="$PATH" make -j1 -C "${KSRC}" M="${nft_test_dir}" modules >/dev/null 2>&1; then
+    echo "ccflags-y += -DNFT_EXPR_OPS_VALIDATE_HAS_DATA"
+  fi
+
+  rm -rf "${nft_test_dir}"
+fi

kernel-module/nft_rtpengine.c

@@ -6870,7 +6870,7 @@ static int rtpengine_expr_dump(struct sk_buff *skb, const struct nft_expr *expr
 }
 
 static int rtpengine_expr_validate(const struct nft_ctx *ctx, const struct nft_expr *expr
-#if LINUX_VERSION_CODE < KERNEL_VERSION(6,12,0)
+#if defined(NFT_EXPR_OPS_VALIDATE_HAS_DATA)
 		, const struct nft_data **data
 #endif
 )