Feature: introduce a high-performance hybrid row-columnar storage engine (3/4)

contrib/pax_storage/.gitignore

@@ -14,11 +14,13 @@ Thumbs.db
 # Temp files dir
 .tmp/**
 build*/**
+results/**
 clang-tidy.result
 # pb
 **/*.pb.h
 **/*.pb.cc
-
+tools/**/*.data
+tools/**/*.txt
 # Executables
 /*.out
 !src/data/expected/*.out

contrib/pax_storage/CMakeLists.txt

@@ -5,38 +5,33 @@ set(CMAKE_CXX_STANDARD 14)
 set(TOP_DIR ${PROJECT_SOURCE_DIR}/../..)
 set(CBDB_INCLUDE_DIR ${TOP_DIR}/src/include)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -g")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror -Wno-error=ignored-qualifiers -Wuninitialized -Winit-self -Wstrict-aliasing -Wno-missing-field-initializers -Wno-unused-parameter -Wno-clobbered -Wno-sized-deallocation -g")
 
 # Build gtest options
 option(BUILD_GTEST "Build with google test" ON)
-
-
 option(BUILD_GBENCH "Build with google benchmark" OFF)
 
 # Build pax tools
 option(BUILD_TOOLS "Build with pax tools" ON)
 
-# env CBDB_BUILD_TYPE is set by Lighting pipeline
-if (NOT DEFINED ENV{CBDB_BUILD_TYPE})
-    set(ENV{CBDB_BUILD_TYPE} "debug")
-endif()
-
-set(CBDB_BUILD_TYPE $ENV{CBDB_BUILD_TYPE})
-message(STATUS "env CBDB_BUILD_TYPE=$ENV{CBDB_BUILD_TYPE} => ${CBDB_BUILD_TYPE}")
+include(CheckSymbolExists)
+SET(PG_CONFIG_HEADER_FILE "${CBDB_INCLUDE_DIR}/pg_config.h")
+message(STATUS "pg_config.h => ${PG_CONFIG_HEADER_FILE}")
 
-if (${CBDB_BUILD_TYPE} STREQUAL "release")
-  SET(CMAKE_BUILD_TYPE "Release")
-  SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O3")
-  # no need build gtest in release mode
-  SET(BUILD_GTEST OFF)
-elseif(${CBDB_BUILD_TYPE} STREQUAL "debug")
+CHECK_SYMBOL_EXISTS(USE_ASSERT_CHECKING "${PG_CONFIG_HEADER_FILE}" PAX_USE_DEBUG)
+CHECK_SYMBOL_EXISTS(USE_LZ4 "${PG_CONFIG_HEADER_FILE}" PAX_USE_LZ4)
+if (PAX_USE_DEBUG)
   ADD_DEFINITIONS(-DENABLE_DEBUG)
   # Use to build compile_commands.json
   SET(CMAKE_BUILD_TYPE "Debug")
   SET(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -ggdb")
 else()
-  message(FATAL_ERROR "unknown CBDB_BUILD_TYPE: ${CBDB_BUILD_TYPE}")
+  SET(CMAKE_BUILD_TYPE "Release")
+  SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O3")
+  # no need build gtest in release mode
+  SET(BUILD_GTEST OFF)
 endif()
+message(STATUS "PAX BUILD TYPE: ${CMAKE_BUILD_TYPE}")
 
 if(BUILD_GBENCH)
   SET(BUILD_GTEST ON)
@@ -47,23 +42,13 @@ if (BUILD_GTEST)
   SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-access-control -fno-inline -Wno-pmf-conversions -Wl,--allow-multiple-definition -no-pie -fno-stack-protector")
 endif(BUILD_GTEST)
 
-# Plasma options
-option (ENABLE_PLASMA "Enable plasma cache" OFF)
-if (ENABLE_PLASMA)
-ADD_DEFINITIONS(-DENABLE_PLASMA)
-# plasma need use CXX_STANDARD 17
-set(CMAKE_CXX_STANDARD 17)
-endif()
-
 # Vec options
-include(CheckSymbolExists)
-SET(PG_CONFIG_HEADER_FILE "${CBDB_INCLUDE_DIR}/pg_config.h")
 CHECK_SYMBOL_EXISTS(USE_VECTORIZATION "${PG_CONFIG_HEADER_FILE}" VEC_BUILD)
-message(STATUS "pg_config.h => ${PG_CONFIG_HEADER_FILE}")
 if (VEC_BUILD)
   set(VEC_HOME "${PROJECT_SOURCE_DIR}/../vectorization")
+  set(INSTALL_HOME "${CMAKE_INSTALL_PREFIX}")
   ADD_DEFINITIONS(-DVEC_BUILD)
-  message(STATUS "Build pax with vectorization support, VEC_HOME=${VEC_HOME}")
+  message(STATUS "Build pax with vectorization support, VEC_HOME=${VEC_HOME} INSTALL_HOME=${INSTALL_HOME}")
 else()
   message(STATUS "Build pax without vectorization support")
 endif(VEC_BUILD)

contrib/pax_storage/FindDependencies.cmake

@@ -21,17 +21,4 @@ endif()
 if (VEC_BUILD)
     find_package(PkgConfig REQUIRED)
     pkg_check_modules(GLIB REQUIRED glib-2.0)
-
-    # resolve vectorization dependency, or the header files will miss
-    message(STATUS "Resolve vectorization dependency ...")
-    execute_process(
-        COMMAND make download_arrow
-        WORKING_DIRECTORY ${VEC_HOME}
-        RESULT_VARIABLE CMD_RESULT
-    )
-    if(CMD_RESULT EQUAL 0)
-        message(STATUS "Resolve vectorization dependency succeeded.")
-    else()
-        message(FATAL_ERROR "Resolve vectorization dependency failed with result: ${CMD_RESULT}")
-    endif()
-endif(VEC_BUILD)
+endif(VEC_BUILD)

contrib/pax_storage/Makefile

@@ -9,49 +9,74 @@ PG_CXXFLAGS = -std=c++14
 PGFILEDESC = "pax - PAX table access method"
 SHLIB_LINK += -luuid
 
-REGRESS = setup
-REGRESS += detoast ddl types update
-# FIXME: several plans are bad in update_gp when use orca
-# REGRESS += update_gp
-REGRESS +=  teardown
+PAX_REGRESS_DIR = src/test/regress
+PAX_VEC_REGRESS_DIR = src/test/vec
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
+PGXS_DIR := $(dir $(PGXS))
+PG_REGRESS = $(PGXS_DIR)/../../src/test/regress/pg_regress
 else
 subdir = contrib/pax_storage
 top_builddir = ../../
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
+PG_REGRESS = $(top_builddir)/src/test/regress/pg_regress
 endif
 
-REGRESS_OPTS += --init-file=$(top_builddir)/src/test/regress/init_file
-
 .PHONY: all
-all: build-pax
+all: build
+
+DIRS = $(shell find src/cpp -maxdepth 2 -type d)
+SRC = $(foreach dir, $(DIRS), $(wildcard $(dir)/*.cc))
 
-.PHONY: install-data build-pax
-build-pax:
+.PHONY: install-data
+build: $(SRC)
 	@echo "build pax"
 	mkdir -p build
 	cd build && cmake .. -DCMAKE_INSTALL_PREFIX=$(DESTDIR)$(prefix) && make -j8 && cp src/cpp/libpax.so ../pax.so
 
-install-data: build-pax
+pax-unit-test: 
+ifeq ($(enable_debug), yes)
+	cd build && ./src/cpp/test_main
+endif
+
+install-data: build
 	$(INSTALL_DATA) pax-cdbinit--1.0.sql '$(DESTDIR)$(datadir)/cdb_init.d/pax-cdbinit--1.0.sql'
 
 .PHONY: install
 install: install-data
 	@echo "install data"
+	@if [ -f build/src/cpp/pax_dump ]; then $(INSTALL_PROGRAM) build/src/cpp/pax_dump '$(DESTDIR)$(bindir)/pax_dump'; fi;
 	make -C build install
 
-.PHONY: uninstall-data
+.PHONY: pax-test
+pax-test: 
+	@echo "Begin running pax tests"
+	$(PG_REGRESS) \
+		--init-file=init_file --schedule=pax_schedule \
+		--load-extension=gp_inject_fault --dbname=pax_test
+	$(bindir)/dropdb --if-exists pax_test
+
+.PHONY: icw_test
+icw_test: pax-test
+	@echo "Begin running pax regression tests"
+	@echo "Current PGOPTIONS=$(PGOPTIONS)"
+	make -C $(PAX_REGRESS_DIR) icw_test
+	@echo "Begin running pax+vector regression tests"
+	@echo "Current PGOPTIONS=$(PGOPTIONS) -c vector.enable_vectorization=on"
+	PGOPTIONS="$(PGOPTIONS) -c vector.enable_vectorization=on" make -C $(PAX_VEC_REGRESS_DIR) icw_test
+	@echo "Begin running pax+vec format+vector regression tests"
+	@echo "Current PGOPTIONS=$(PGOPTIONS) -c vector.enable_vectorization=on -c pax_default_storage_format=porc_vec"
+	PGOPTIONS="$(PGOPTIONS) -c vector.enable_vectorization=on -c pax_default_storage_format=porc_vec" make -C $(PAX_VEC_REGRESS_DIR) icw_test
 
+.PHONY: uninstall-data
 uninstall-data:
 	$(RM) '$(DESTDIR)$(datadir)/cdb_init.d/pax-cdbinit--1.0.sql'
 
 uninstall: uninstall-data
-
 clean-data:
 	$(RM) pax-cdbinit--1.0.sql
 	$(RM) -r build