Feature: introduce a high-performance hybrid row-columnar storage engine (4/4)

.asf.yaml

@@ -90,6 +90,10 @@ github:
           - RPM Install Test Apache Cloudberry
           - ic-good-opt-off
           - ic-good-opt-on
+          - pax-ic-good-opt-off
+          - pax-ic-good-opt-on
+          - pax-ic-isolation2-opt-off
+          - pax-ic-isolation2-opt-on
           - ic-expandshrink
           - ic-singlenode
           - ic-resgroup-v2

.github/workflows/build-cloudberry.yml

@@ -230,6 +230,42 @@ jobs:
                "make_configs":["src/test/regress:installcheck-good"],
                "pg_settings":{"optimizer":"on"}
               },
+              {"test":"pax-ic-good-opt-off",
+               "make_configs":[
+                "contrib/pax_storage/:pax-test",
+                "contrib/pax_storage/:regress_test"
+              ],
+               "pg_settings":{
+                 "optimizer":"off",
+                 "default_table_access_method":"pax"
+                }
+              },
+              {"test":"pax-ic-good-opt-on",
+               "make_configs":[
+                "contrib/pax_storage/:pax-test",
+                "contrib/pax_storage/:regress_test"
+              ],
+               "pg_settings":{
+                 "optimizer":"on",
+                 "default_table_access_method":"pax"
+                }
+              },
+              {"test":"pax-ic-isolation2-opt-off",
+               "make_configs":["contrib/pax_storage/:isolation2_test"],
+               "pg_settings":{
+                 "optimizer":"off",
+                 "default_table_access_method":"pax"
+                },
+                "enable_core_check":false
+              },
+              {"test":"pax-ic-isolation2-opt-on",
+               "make_configs":["contrib/pax_storage/:isolation2_test"],
+               "pg_settings":{
+                 "optimizer":"on",
+                 "default_table_access_method":"pax"
+                },
+                "enable_core_check":false
+              },
               {"test":"ic-expandshrink",
                "make_configs":["src/test/isolation2:installcheck-expandshrink"]
               },
@@ -482,6 +518,11 @@ jobs:
         run: |
           set -eo pipefail
 
+          if ! time su - gpadmin -c "cd ${SRC_DIR} && git submodule update --init"; then
+            echo "::error::Fail to pull submodule."
+            exit 1
+          fi
+
           chmod +x "${SRC_DIR}"/../cloudberry-devops-release/build_automation/cloudberry/scripts/build-cloudberry.sh
           if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/../cloudberry-devops-release/build_automation/cloudberry/scripts/build-cloudberry.sh"; then
             echo "::error::Build script failed"
@@ -1267,6 +1308,10 @@ jobs:
             PG_OPTS="$PG_OPTS -c optimizer=${{ matrix.pg_settings.optimizer }}"
           fi
 
+          if [[ "${{ matrix.pg_settings.default_table_access_method != '' }}" == "true" ]]; then
+            PG_OPTS="$PG_OPTS -c default_table_access_method=${{ matrix.pg_settings.default_table_access_method }}"
+          fi
+
           # Read configs into array
           IFS=' ' read -r -a configs <<< "${{ join(matrix.make_configs, ' ') }}"
 

.github/workflows/build-dbg-cloudberry.yml

@@ -427,6 +427,11 @@ jobs:
         run: |
           set -eo pipefail
 
+          if ! time su - gpadmin -c "cd ${SRC_DIR} && git submodule update --init"; then
+            echo "::error::Fail to pull submodule."
+            exit 1
+          fi
+
           chmod +x "${SRC_DIR}"/../cloudberry-devops-release/build_automation/cloudberry/scripts/build-cloudberry.sh
           if ! time su - gpadmin -c "cd ${SRC_DIR} && SRC_DIR=${SRC_DIR} ${SRC_DIR}/../cloudberry-devops-release/build_automation/cloudberry/scripts/build-cloudberry.sh"; then
             echo "::error::Build script failed"

.gitmodules

@@ -3,14 +3,18 @@
 	url = https://github.com/google/googletest.git
 [submodule "contrib/pax_storage/src/cpp/contrib/googletest"]
 	path = contrib/pax_storage/src/cpp/contrib/googletest
-	url = https://code.hashdata.xyz/cloudberry/googletest
+	url = https://github.com/google/googletest.git
+	branch = v1.15.x
 [submodule "contrib/pax_storage/src/cpp/contrib/tabulate"]
 	path = contrib/pax_storage/src/cpp/contrib/tabulate
-	url = https://code.hashdata.xyz/cloudberry/tabulate
+	url = https://github.com/p-ranav/tabulate.git
 [submodule "contrib/pax_storage/src/cpp/contrib/googlebench"]
 	path = contrib/pax_storage/src/cpp/contrib/googlebench
-	url = https://code.hashdata.xyz/cloudberry/benchmark.git
+	url = https://github.com/google/benchmark.git
 [submodule "contrib/pax_storage/src/cpp/contrib/cpp-stub"]
 	path = contrib/pax_storage/src/cpp/contrib/cpp-stub
-	url = https://code.hashdata.xyz/cloudberry/cpp-stub.git
-	branch = dev
+	url = https://github.com/coolxv/cpp-stub.git
+[submodule "dependency/yyjson"]
+	path = dependency/yyjson
+	url = https://github.com/ibireme/yyjson.git
+

configure

@@ -9124,11 +9124,11 @@ $as_echo "#define USE_PAX_STORAGE 1" >>confdefs.h
   esac
 
 else
-  enable_pax=no
-
+  enable_pax=yes
+
+  $as_echo "#define USE_PAX_STORAGE 1" >>confdefs.h
 fi
 
-
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: checking whether to build with pax support ... $enable_pax" >&5
 $as_echo "checking whether to build with pax support ... $enable_pax" >&6; }
 

configure.ac

@@ -928,11 +928,11 @@ AC_SUBST(enable_preload_ic_module)
 #
 # pax support
 #
-PGAC_ARG_BOOL(enable, pax, no,
-              [enable pax support],
+PGAC_ARG_BOOL(enable, pax, yes,
+              [disable PAX support],
               [AC_DEFINE(USE_PAX_STORAGE, 1,
-                         [Define to 1 to support pax])])
-AC_MSG_RESULT([checking whether to build with pax support ... $enable_pax])
+                         [Define to 1 to support PAX])])
+AC_MSG_RESULT([checking whether to build with PAX support ... $enable_pax])
 AC_SUBST(enable_pax)
 
 #

contrib/pax_storage/.gitignore

@@ -16,6 +16,8 @@ Thumbs.db
 build*/**
 results/**
 clang-tidy.result
+src/test/regress/hooktest/**
+src/test/regress/query_info_hook_test/**
 # pb
 **/*.pb.h
 **/*.pb.cc

contrib/pax_storage/CMakeLists.txt

@@ -1,18 +1,35 @@
 project(Pax)
 cmake_minimum_required (VERSION 3.11.0)
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
 
 set(TOP_DIR ${PROJECT_SOURCE_DIR}/../..)
 set(CBDB_INCLUDE_DIR ${TOP_DIR}/src/include)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror -Wno-error=ignored-qualifiers -Wuninitialized -Winit-self -Wstrict-aliasing -Wno-missing-field-initializers -Wno-unused-parameter -Wno-clobbered -Wno-sized-deallocation -g")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror -Wno-unused-function -Wno-redundant-move -Wno-error=redundant-move -Wno-error=ignored-qualifiers -Wuninitialized -Winit-self -Wstrict-aliasing -Wno-missing-field-initializers -Wno-unused-parameter -Wno-clobbered -Wno-sized-deallocation -g")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wno-unused-parameter -Wno-parameter-name")
+
+option(USE_MANIFEST_API "Use manifest API" OFF)
+option(USE_PAX_CATALOG "Use manifest API, by pax impl" ON)
 
 # Build gtest options
 option(BUILD_GTEST "Build with google test" ON)
-option(BUILD_GBENCH "Build with google benchmark" OFF)
+option(BUILD_GBENCH "Build with google benchmark" ON)
 
 # Build pax tools
-option(BUILD_TOOLS "Build with pax tools" ON)
+option(BUILD_TOOLS "Build with pax tools" OFF)
+
+# Build with return dictionary in record batch
+option(BUILD_RB_RET_DICT "Build with pax direct return dictionary in record batch." OFF)
+
+if (USE_MANIFEST_API)
+  ADD_DEFINITIONS(-DUSE_MANIFEST_API)
+else()
+  set(USE_PAX_CATALOG ON)
+endif(USE_MANIFEST_API)
+
+if (USE_PAX_CATALOG)
+  ADD_DEFINITIONS(-DUSE_PAX_CATALOG)
+endif()
 
 include(CheckSymbolExists)
 SET(PG_CONFIG_HEADER_FILE "${CBDB_INCLUDE_DIR}/pg_config.h")
@@ -25,6 +42,7 @@ if (PAX_USE_DEBUG)
   # Use to build compile_commands.json
   SET(CMAKE_BUILD_TYPE "Debug")
   SET(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -ggdb")
+  SET(INSTALL_GTEST OFF) # disable gtest install() target
 else()
   SET(CMAKE_BUILD_TYPE "Release")
   SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O3")
@@ -35,6 +53,7 @@ message(STATUS "PAX BUILD TYPE: ${CMAKE_BUILD_TYPE}")
 
 if(BUILD_GBENCH)
   SET(BUILD_GTEST ON)
+  SET(BENCHMARK_ENABLE_INSTALL OFF) # disable benchmark install() target
 endif(BUILD_GBENCH)
 
 if (BUILD_GTEST)
@@ -47,8 +66,13 @@ CHECK_SYMBOL_EXISTS(USE_VECTORIZATION "${PG_CONFIG_HEADER_FILE}" VEC_BUILD)
 if (VEC_BUILD)
   set(VEC_HOME "${PROJECT_SOURCE_DIR}/../vectorization")
   set(INSTALL_HOME "${CMAKE_INSTALL_PREFIX}")
+
+  if (BUILD_RB_RET_DICT)
+    ADD_DEFINITIONS(-DBUILD_RB_RET_DICT)
+  endif(BUILD_RB_RET_DICT)
+
   ADD_DEFINITIONS(-DVEC_BUILD)
-  message(STATUS "Build pax with vectorization support, VEC_HOME=${VEC_HOME} INSTALL_HOME=${INSTALL_HOME}")
+  message(STATUS "Build pax with vectorization support, VEC_HOME=${VEC_HOME} INSTALL_HOME=${INSTALL_HOME}, BUILD_RB_RET_DICT=${BUILD_RB_RET_DICT}")
 else()
   message(STATUS "Build pax without vectorization support")
 endif(VEC_BUILD)
@@ -58,3 +82,15 @@ endif(VEC_BUILD)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/")
 include(FindDependencies)
 add_subdirectory(src/cpp)
+
+if(USE_MANIFEST_API AND NOT USE_PAX_CATALOG)
+    set(SAVED_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
+    set(SAVED_CMAKE_INSTALL_SO_NO_EXE ${CMAKE_INSTALL_SO_NO_EXE})
+
+    set(BUILD_SHARED_LIBS ON)
+    set(CMAKE_INSTALL_SO_NO_EXE OFF)
+    add_subdirectory(${CMAKE_SOURCE_DIR}/../../dependency/yyjson ${CMAKE_BINARY_DIR}/yyjson_build)
+
+    set(BUILD_SHARED_LIBS ${SAVED_BUILD_SHARED_LIBS})
+    set(CMAKE_INSTALL_SO_NO_EXE ${SAVED_CMAKE_INSTALL_SO_NO_EXE})
+endif()

contrib/pax_storage/FindDependencies.cmake

@@ -1,5 +1,3 @@
-find_package(BISON REQUIRED)
-
 ## protobuf
 include(FindProtobuf)
 find_package(Protobuf 3.6.1 REQUIRED)

contrib/pax_storage/Makefile

@@ -4,13 +4,13 @@ MODULE_big = pax
 OBJS = \
 	$(WIN32RES) 
 PG_CPPFLAGS = -I/usr/local/include
-PG_CXXFLAGS = -std=c++14
+PG_CXXFLAGS = -std=c++17
 
 PGFILEDESC = "pax - PAX table access method"
 SHLIB_LINK += -luuid
 
 PAX_REGRESS_DIR = src/test/regress
-PAX_VEC_REGRESS_DIR = src/test/vec
+PAX_ISOLATION2_DIR = src/test/isolation2
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -31,46 +31,73 @@ all: build
 
 DIRS = $(shell find src/cpp -maxdepth 2 -type d)
 SRC = $(foreach dir, $(DIRS), $(wildcard $(dir)/*.cc))
+CSRC = $(foreach dir, $(DIRS), $(wildcard $(dir)/*.c))
 
-.PHONY: install-data
-build: $(SRC)
-	@echo "build pax"
-	mkdir -p build
-	cd build && cmake .. -DCMAKE_INSTALL_PREFIX=$(DESTDIR)$(prefix) && make -j8 && cp src/cpp/libpax.so ../pax.so
+# read options from ENV
+USE_MANIFEST_API := $(shell echo $$USE_MANIFEST_API)
+USE_PAX_CATALOG := $(shell echo $$USE_PAX_CATALOG)
+
+# if the ENVs are not set, set default value
+ifeq ($(USE_MANIFEST_API),)
+    USE_MANIFEST_API := OFF
+endif
+
+ifeq ($(USE_PAX_CATALOG),)
+    USE_PAX_CATALOG := ON
+endif
+
+.PHONY: install-data build
+build: $(SRC) $(CSRC)
+	@echo "build pax, USE_MANIFEST_API=$(USE_MANIFEST_API) USE_PAX_CATALOG=$(USE_PAX_CATALOG)"
+	@if [ ! -f build/Makefile ]; then \
+		mkdir -p build && \
+		cd build && \
+		cmake -DCMAKE_INSTALL_PREFIX=$(DESTDIR)$(prefix) \
+			-DUSE_MANIFEST_API=$(USE_MANIFEST_API) \
+			-DUSE_PAX_CATALOG=$(USE_PAX_CATALOG) .. ; \
+	fi
+	cd build && make -j8
+	@cp -f build/src/cpp/libpax.so pax.so
 
 pax-unit-test: 
 ifeq ($(enable_debug), yes)
 	cd build && ./src/cpp/test_main
 endif
 
 install-data: build
+	@echo "install data"
 	$(INSTALL_DATA) pax-cdbinit--1.0.sql '$(DESTDIR)$(datadir)/cdb_init.d/pax-cdbinit--1.0.sql'
 
 .PHONY: install
 install: install-data
-	@echo "install data"
+	$(INSTALL_SHLIB) pax.so '$(DESTDIR)$(pkglibdir)/pax.so'
 	@if [ -f build/src/cpp/pax_dump ]; then $(INSTALL_PROGRAM) build/src/cpp/pax_dump '$(DESTDIR)$(bindir)/pax_dump'; fi;
-	make -C build install
+	make -j10 -C build install
 
 .PHONY: pax-test
 pax-test: 
 	@echo "Begin running pax tests"
+	PGOPTIONS="$(PGOPTIONS)" \
 	$(PG_REGRESS) \
 		--init-file=init_file --schedule=pax_schedule \
 		--load-extension=gp_inject_fault --dbname=pax_test
 	$(bindir)/dropdb --if-exists pax_test
 
 .PHONY: icw_test
-icw_test: pax-test
+icw_test: pax-test regress_test isolation2_test
+
+.PHONY: regress_test isolation2_test
+regress_test:
 	@echo "Begin running pax regression tests"
+	make -C $(PAX_REGRESS_DIR) regress_link
 	@echo "Current PGOPTIONS=$(PGOPTIONS)"
 	make -C $(PAX_REGRESS_DIR) icw_test
-	@echo "Begin running pax+vector regression tests"
-	@echo "Current PGOPTIONS=$(PGOPTIONS) -c vector.enable_vectorization=on"
-	PGOPTIONS="$(PGOPTIONS) -c vector.enable_vectorization=on" make -C $(PAX_VEC_REGRESS_DIR) icw_test
-	@echo "Begin running pax+vec format+vector regression tests"
-	@echo "Current PGOPTIONS=$(PGOPTIONS) -c vector.enable_vectorization=on -c pax_default_storage_format=porc_vec"
-	PGOPTIONS="$(PGOPTIONS) -c vector.enable_vectorization=on -c pax_default_storage_format=porc_vec" make -C $(PAX_VEC_REGRESS_DIR) icw_test
+
+isolation2_test:
+	@echo "Begin running pax isolation2 tests"
+	make -C $(PAX_REGRESS_DIR) regress_link
+	@echo "Current PGOPTIONS=$(PGOPTIONS)"
+	make -C $(PAX_ISOLATION2_DIR) installcheck
 
 .PHONY: uninstall-data
 uninstall-data:
@@ -81,4 +108,7 @@ clean-data:
 	$(RM) pax-cdbinit--1.0.sql
 	$(RM) -r build
 
-clean: clean-data
+clean_isolation2_test:
+	make -C $(PAX_ISOLATION2_DIR) clean
+
+clean: clean-data clean_isolation2_test