Skip to content

Commit 9c2b954

Browse files
wesmwesm
committed
ARROW-23: Add a logical Column data structure
I also added global const instances of common primitive types Author: Wes McKinney <wesm@apache.org> Closes #15 from wesm/ARROW-23 and squashes the following commits: 1835d33 [Wes McKinney] Don't use auto 988135c [Wes McKinney] Add Column chunk type validation function 8a2e40e [Wes McKinney] Remove unneeded operator()/shared_from_this experiment de9ec70 [Wes McKinney] Aggregate null counts too 7049314 [Wes McKinney] cpplint a565d26 [Wes McKinney] Add ChunkedArray / Column ctors, test passes 0648ed2 [Wes McKinney] Prototyping
1 parent 3b777c7 commit 9c2b954

12 files changed

Lines changed: 347 additions & 15 deletions

File tree

‎cpp/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -458,10 +458,12 @@ endif()
458458
459459
add_subdirectory(src/arrow)
460460
add_subdirectory(src/arrow/util)
461+
add_subdirectory(src/arrow/table)
461462
add_subdirectory(src/arrow/types)
462463
463464
set(LINK_LIBS
464465
arrow_util
466+
arrow_table
465467
arrow_types)
466468
467469
set(ARROW_SRCS

‎cpp/src/arrow/array.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,6 @@ class Array {
8181
DISALLOW_COPY_AND_ASSIGN(Array);
8282
};
8383

84-
8584
typedef std::shared_ptr<Array> ArrayPtr;
8685

8786
} // namespace arrow

‎cpp/src/arrow/schema-test.cc‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ using std::vector;
3131
namespace arrow {
3232

3333
TEST(TestField, Basics) {
34-
shared_ptr<DataType> ftype = std::make_shared<Int32Type>();
34+
shared_ptr<DataType> ftype = INT32;
3535
shared_ptr<DataType> ftype_nn = std::make_shared<Int32Type>(false);
3636
Field f0("f0", ftype);
3737
Field f0_nn("f0", ftype_nn);
@@ -44,7 +44,7 @@ TEST(TestField, Basics) {
4444
}
4545

4646
TEST(TestField, Equals) {
47-
shared_ptr<DataType> ftype = std::make_shared<Int32Type>();
47+
shared_ptr<DataType> ftype = INT32;
4848
shared_ptr<DataType> ftype_nn = std::make_shared<Int32Type>(false);
4949

5050
Field f0("f0", ftype);
@@ -61,8 +61,7 @@ class TestSchema : public ::testing::Test {
6161
};
6262

6363
TEST_F(TestSchema, Basics) {
64-
auto f0 = std::make_shared<Field>("f0", std::make_shared<Int32Type>());
65-
64+
auto f0 = std::make_shared<Field>("f0", INT32);
6665
auto f1 = std::make_shared<Field>("f1", std::make_shared<UInt8Type>(false));
6766
auto f1_optional = std::make_shared<Field>("f1", std::make_shared<UInt8Type>());
6867

‎cpp/src/arrow/table/CMakeLists.txt‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
#######################################
19+
# arrow_table
20+
#######################################
21+
22+
set(TABLE_SRCS
23+
column.cc
24+
)
25+
26+
set(TABLE_LIBS
27+
)
28+
29+
add_library(arrow_table STATIC
30+
${TABLE_SRCS}
31+
)
32+
target_link_libraries(arrow_table ${TABLE_LIBS})
33+
SET_TARGET_PROPERTIES(arrow_table PROPERTIES LINKER_LANGUAGE CXX)
34+
35+
# Headers: top level
36+
install(FILES
37+
DESTINATION include/arrow/table)
38+
39+
ADD_ARROW_TEST(column-test)

‎cpp/src/arrow/table/column-test.cc‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include <gtest/gtest.h>
19+
#include <cstdint>
20+
#include <memory>
21+
#include <string>
22+
#include <vector>
23+
24+
#include "arrow/field.h"
25+
#include "arrow/schema.h"
26+
#include "arrow/table/column.h"
27+
#include "arrow/test-util.h"
28+
#include "arrow/type.h"
29+
#include "arrow/types/integer.h"
30+
#include "arrow/util/bit-util.h"
31+
#include "arrow/util/buffer.h"
32+
#include "arrow/util/memory-pool.h"
33+
#include "arrow/util/status.h"
34+
35+
using std::shared_ptr;
36+
using std::vector;
37+
38+
namespace arrow {
39+
40+
class TestColumn : public ::testing::Test {
41+
public:
42+
void SetUp() {
43+
pool_ = GetDefaultMemoryPool();
44+
}
45+
46+
template <typename ArrayType>
47+
std::shared_ptr<Array> MakeArray(int32_t length, int32_t null_count = 0) {
48+
auto data = std::make_shared<PoolBuffer>(pool_);
49+
auto nulls = std::make_shared<PoolBuffer>(pool_);
50+
data->Resize(length * sizeof(typename ArrayType::value_type));
51+
nulls->Resize(util::bytes_for_bits(length));
52+
return std::make_shared<ArrayType>(length, data, 10, nulls);
53+
}
54+
55+
protected:
56+
MemoryPool* pool_;
57+
58+
std::shared_ptr<ChunkedArray> data_;
59+
std::unique_ptr<Column> column_;
60+
};
61+
62+
TEST_F(TestColumn, BasicAPI) {
63+
ArrayVector arrays;
64+
arrays.push_back(MakeArray<Int32Array>(100));
65+
arrays.push_back(MakeArray<Int32Array>(100, 10));
66+
arrays.push_back(MakeArray<Int32Array>(100, 20));
67+
68+
auto field = std::make_shared<Field>("c0", INT32);
69+
column_.reset(new Column(field, arrays));
70+
71+
ASSERT_EQ("c0", column_->name());
72+
ASSERT_TRUE(column_->type()->Equals(INT32));
73+
ASSERT_EQ(300, column_->length());
74+
ASSERT_EQ(30, column_->null_count());
75+
ASSERT_EQ(3, column_->data()->num_chunks());
76+
}
77+
78+
TEST_F(TestColumn, ChunksInhomogeneous) {
79+
ArrayVector arrays;
80+
arrays.push_back(MakeArray<Int32Array>(100));
81+
arrays.push_back(MakeArray<Int32Array>(100, 10));
82+
83+
auto field = std::make_shared<Field>("c0", INT32);
84+
column_.reset(new Column(field, arrays));
85+
86+
ASSERT_OK(column_->ValidateData());
87+
88+
arrays.push_back(MakeArray<Int16Array>(100, 10));
89+
column_.reset(new Column(field, arrays));
90+
ASSERT_RAISES(Invalid, column_->ValidateData());
91+
}
92+
93+
} // namespace arrow

‎cpp/src/arrow/table/column.cc‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include "arrow/table/column.h"
19+
20+
#include <memory>
21+
#include <sstream>
22+
23+
#include "arrow/field.h"
24+
#include "arrow/util/status.h"
25+
26+
namespace arrow {
27+
28+
ChunkedArray::ChunkedArray(const ArrayVector& chunks) :
29+
chunks_(chunks) {
30+
length_ = 0;
31+
for (const std::shared_ptr<Array>& chunk : chunks) {
32+
length_ += chunk->length();
33+
null_count_ += chunk->null_count();
34+
}
35+
}
36+
37+
Column::Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks) :
38+
field_(field) {
39+
data_ = std::make_shared<ChunkedArray>(chunks);
40+
}
41+
42+
Column::Column(const std::shared_ptr<Field>& field,
43+
const std::shared_ptr<ChunkedArray>& data) :
44+
field_(field),
45+
data_(data) {}
46+
47+
Status Column::ValidateData() {
48+
for (int i = 0; i < data_->num_chunks(); ++i) {
49+
const std::shared_ptr<DataType>& type = data_->chunk(i)->type();
50+
if (!this->type()->Equals(type)) {
51+
std::stringstream ss;
52+
ss << "In chunk " << i << " expected type "
53+
<< this->type()->ToString()
54+
<< " but saw "
55+
<< type->ToString();
56+
return Status::Invalid(ss.str());
57+
}
58+
}
59+
return Status::OK();
60+
}
61+
62+
} // namespace arrow

‎cpp/src/arrow/table/column.h‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#ifndef ARROW_TABLE_COLUMN_H
19+
#define ARROW_TABLE_COLUMN_H
20+
21+
#include <memory>
22+
#include <string>
23+
#include <vector>
24+
25+
#include "arrow/array.h"
26+
#include "arrow/field.h"
27+
28+
namespace arrow {
29+
30+
typedef std::vector<std::shared_ptr<Array> > ArrayVector;
31+
32+
// A data structure managing a list of primitive Arrow arrays logically as one
33+
// large array
34+
class ChunkedArray {
35+
public:
36+
explicit ChunkedArray(const ArrayVector& chunks);
37+
38+
// @returns: the total length of the chunked array; computed on construction
39+
int64_t length() const {
40+
return length_;
41+
}
42+
43+
int64_t null_count() const {
44+
return null_count_;
45+
}
46+
47+
int num_chunks() const {
48+
return chunks_.size();
49+
}
50+
51+
const std::shared_ptr<Array>& chunk(int i) const {
52+
return chunks_[i];
53+
}
54+
55+
protected:
56+
ArrayVector chunks_;
57+
int64_t length_;
58+
int64_t null_count_;
59+
};
60+
61+
// An immutable column data structure consisting of a field (type metadata) and
62+
// a logical chunked data array (which can be validated as all being the same
63+
// type).
64+
class Column {
65+
public:
66+
Column(const std::shared_ptr<Field>& field, const ArrayVector& chunks);
67+
Column(const std::shared_ptr<Field>& field,
68+
const std::shared_ptr<ChunkedArray>& data);
69+
70+
int64_t length() const {
71+
return data_->length();
72+
}
73+
74+
int64_t null_count() const {
75+
return data_->null_count();
76+
}
77+
78+
// @returns: the column's name in the passed metadata
79+
const std::string& name() const {
80+
return field_->name;
81+
}
82+
83+
// @returns: the column's type according to the metadata
84+
const std::shared_ptr<DataType>& type() const {
85+
return field_->type;
86+
}
87+
88+
// @returns: the column's data as a chunked logical array
89+
const std::shared_ptr<ChunkedArray>& data() const {
90+
return data_;
91+
}
92+
// Verify that the column's array data is consistent with the passed field's
93+
// metadata
94+
Status ValidateData();
95+
96+
protected:
97+
std::shared_ptr<Field> field_;
98+
std::shared_ptr<ChunkedArray> data_;
99+
};
100+
101+
} // namespace arrow
102+
103+
#endif // ARROW_TABLE_COLUMN_H

‎cpp/src/arrow/type.cc‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,16 @@
1919

2020
namespace arrow {
2121

22+
const std::shared_ptr<BooleanType> BOOL = std::make_shared<BooleanType>();
23+
const std::shared_ptr<UInt8Type> UINT8 = std::make_shared<UInt8Type>();
24+
const std::shared_ptr<UInt16Type> UINT16 = std::make_shared<UInt16Type>();
25+
const std::shared_ptr<UInt32Type> UINT32 = std::make_shared<UInt32Type>();
26+
const std::shared_ptr<UInt64Type> UINT64 = std::make_shared<UInt64Type>();
27+
const std::shared_ptr<Int8Type> INT8 = std::make_shared<Int8Type>();
28+
const std::shared_ptr<Int16Type> INT16 = std::make_shared<Int16Type>();
29+
const std::shared_ptr<Int32Type> INT32 = std::make_shared<Int32Type>();
30+
const std::shared_ptr<Int64Type> INT64 = std::make_shared<Int64Type>();
31+
const std::shared_ptr<FloatType> FLOAT = std::make_shared<FloatType>();
32+
const std::shared_ptr<DoubleType> DOUBLE = std::make_shared<DoubleType>();
33+
2234
} // namespace arrow

0 commit comments

Comments
 (0)