performance: length_stream changes from int64 type to int32 type

In the current implementation, the length of a single tuple field will not exceed 2GB,
so a single element of the lengths stream can use int32 to represent the length, so that
each variable-length field can save 4 bytes. In the large-wide table test with 500
variable-length columns and 1 million tuples, 2GB of storage space was saved.

Author: gongxun0928

contrib/pax_storage/src/cpp/storage/columns/pax_column.cc

@@ -178,7 +178,7 @@ template class PaxCommColumn<double>;
 
 PaxNonFixedColumn::PaxNonFixedColumn(uint32 capacity) : estimated_size_(0) {
   data_ = PAX_NEW<DataBuffer<char>>(capacity * sizeof(char));
-  lengths_ = PAX_NEW<DataBuffer<int64>>(capacity * sizeof(char));
+  lengths_ = PAX_NEW<DataBuffer<int32>>(capacity * sizeof(char));
 }
 
 PaxNonFixedColumn::PaxNonFixedColumn() : PaxNonFixedColumn(DEFAULT_CAPACITY) {}
@@ -188,7 +188,7 @@ PaxNonFixedColumn::~PaxNonFixedColumn() {
   PAX_DELETE(lengths_);
 }
 
-void PaxNonFixedColumn::Set(DataBuffer<char> *data, DataBuffer<int64> *lengths,
+void PaxNonFixedColumn::Set(DataBuffer<char> *data, DataBuffer<int32> *lengths,
                             size_t total_size) {
   PAX_DELETE(data_);
   PAX_DELETE(lengths_);
@@ -222,15 +222,15 @@ void PaxNonFixedColumn::Append(char *buffer, size_t size) {
   }
 
   if (lengths_->Available() == 0) {
-    lengths_->ReSize(lengths_->Used() + sizeof(int64), 2);
+    lengths_->ReSize(lengths_->Used() + sizeof(int32), 2);
   }
 
   estimated_size_ += size;
   data_->Write(buffer, origin_size);
   data_->Brush(size);
 
-  lengths_->Write(reinterpret_cast<int64 *>(&size), sizeof(int64));
-  lengths_->Brush(sizeof(int64));
+  lengths_->Write(reinterpret_cast<int32 *>(&size), sizeof(int32));
+  lengths_->Brush(sizeof(int32));
 
   offsets_.emplace_back(offsets_.empty()
                             ? 0
@@ -239,7 +239,7 @@ void PaxNonFixedColumn::Append(char *buffer, size_t size) {
   Assert(offsets_.size() == lengths_->GetSize());
 }
 
-DataBuffer<int64> *PaxNonFixedColumn::GetLengthBuffer() const {
+DataBuffer<int32> *PaxNonFixedColumn::GetLengthBuffer() const {
   return lengths_;
 }
 

contrib/pax_storage/src/cpp/storage/columns/pax_column.h

@@ -286,7 +286,7 @@ class PaxNonFixedColumn : public PaxColumn {
 
   ~PaxNonFixedColumn() override;
 
-  virtual void Set(DataBuffer<char> *data, DataBuffer<int64> *lengths,
+  virtual void Set(DataBuffer<char> *data, DataBuffer<int32> *lengths,
                    size_t total_size);
 
   void Append(char *buffer, size_t size) override;
@@ -310,7 +310,7 @@ class PaxNonFixedColumn : public PaxColumn {
 
   size_t GetNonNullRows() const override;
 
-  DataBuffer<int64> *GetLengthBuffer() const;
+  DataBuffer<int32> *GetLengthBuffer() const;
 
   DataBuffer<int32> *GetOffsetBuffer(bool append_last = false);
 
@@ -321,8 +321,11 @@ class PaxNonFixedColumn : public PaxColumn {
   size_t estimated_size_;
   DataBuffer<char> *data_;
 
-  // orc needs to serialize int64 array
-  DataBuffer<int64> *lengths_;
+  // orc needs to serialize int32 array
+  // the length of a single tuple field will not exceed 2GB,
+  // so a variable-length element of the lengths stream can use int32 
+  // to represent the length
+  DataBuffer<int32> *lengths_;
   std::vector<uint64> offsets_;
 };
 

contrib/pax_storage/src/cpp/storage/columns/pax_column_test.cc

@@ -732,7 +732,7 @@ TEST_P(PaxNonFixedColumnCompressTest,
   auto data_buffer_for_read =
       new DataBuffer<char>(encoded_buff, encoded_len, false, false);
   data_buffer_for_read->Brush(encoded_len);
-  auto length_buffer_cpy = new DataBuffer<int64>(*length_buffer);
+  auto length_buffer_cpy = new DataBuffer<int32>(*length_buffer);
   non_fixed_column_for_read->Set(data_buffer_for_read, length_buffer_cpy,
                                  origin_len);
   ASSERT_EQ(non_fixed_column_for_read->GetCompressLevel(), 5);

contrib/pax_storage/src/cpp/storage/columns/pax_columns.cc

@@ -335,7 +335,7 @@ size_t PaxColumns::MeasureOrcDataBuffer(
 
     switch (column->GetPaxColumnTypeInMem()) {
       case kTypeNonFixed: {
-        size_t lengths_size = column_size * sizeof(int64);
+        size_t lengths_size = column_size * sizeof(int32);
 
         if ((buffer_len + lengths_size) % column->GetAlignSize() != 0) {
           auto align_buffer_len =

contrib/pax_storage/src/cpp/storage/columns/pax_encoding_non_fixed_column.cc

@@ -49,7 +49,7 @@ PaxNonFixedEncodingColumn::~PaxNonFixedEncodingColumn() {
 }
 
 void PaxNonFixedEncodingColumn::Set(DataBuffer<char> *data,
-                                    DataBuffer<int64> *lengths,
+                                    DataBuffer<int32> *lengths,
                                     size_t total_size) {
   if (compressor_) {
     Assert(!compress_route_);

contrib/pax_storage/src/cpp/storage/columns/pax_encoding_non_fixed_column.h

@@ -15,7 +15,7 @@ class PaxNonFixedEncodingColumn final : public PaxNonFixedColumn {
 
   ~PaxNonFixedEncodingColumn() override;
 
-  void Set(DataBuffer<char> *data, DataBuffer<int64> *lengths,
+  void Set(DataBuffer<char> *data, DataBuffer<int32> *lengths,
            size_t total_size) override;
 
   std::pair<char *, size_t> GetBuffer() override;

contrib/pax_storage/src/cpp/storage/orc/orc_format_reader.cc

@@ -425,19 +425,19 @@ static PaxColumn *BuildEncodingNonFixedColumn(
   uint32 column_lens_size = 0;
   uint64 column_lens_len = 0;
   uint64 column_data_len = 0;
-  DataBuffer<int64> *column_len_buffer = nullptr;
+  DataBuffer<int32> *column_len_buffer = nullptr;
   DataBuffer<char> *column_data_buffer = nullptr;
   PaxNonFixedColumn *pax_column = nullptr;
 
   column_lens_size = static_cast<uint32>(len_stream.column());
   column_lens_len = static_cast<uint64>(len_stream.length());
 
-  column_len_buffer = PAX_NEW<DataBuffer<int64>>(
-      reinterpret_cast<int64 *>(data_buffer->GetAvailableBuffer()),
+  column_len_buffer = PAX_NEW<DataBuffer<int32>>(
+      reinterpret_cast<int32 *>(data_buffer->GetAvailableBuffer()),
       column_lens_len, false, false);
 
-  Assert(column_lens_len >= column_lens_size * sizeof(int64));
-  column_len_buffer->Brush(column_lens_size * sizeof(int64));
+  Assert(column_lens_len >= column_lens_size * sizeof(int32));
+  column_len_buffer->Brush(column_lens_size * sizeof(int32));
   data_buffer->Brush(column_lens_len);
 
   column_data_len = data_stream.length();