Skip to content

Commit 2a0c1f7

Browse files
cbb330claude
cbb330
and
claude
committed
Task #2: Add unit tests for ORC column statistics APIs (apache#134)
- Added GetColumnStatisticsInteger: Tests file-level statistics for int columns - Added GetStripeColumnStatistics: Tests stripe-level statistics - Added GetColumnStatisticsString: Tests string column statistics with StringScalar min/max - Added GetColumnStatisticsOutOfRange: Tests error handling for invalid indices - Added GetColumnStatisticsWithNulls: Tests has_null flag when nulls are present All 5 new tests pass. Total: 45 tests. Verified: Build succeeds, all tests pass Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent 319818b commit 2a0c1f7

1 file changed

Lines changed: 205 additions & 0 deletions

File tree

‎cpp/src/arrow/adapters/orc/adapter_test.cc‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1179,4 +1179,209 @@ TEST_F(TestORCWriterMultipleWrite, MultipleWritesIntFieldRecordBatch) {
11791179
AssertBatchWriteReadEqual(input_batches, expected_output_table,
11801180
kDefaultSmallMemStreamSize * 100);
11811181
}
1182+
1183+
TEST(TestAdapterRead, GetColumnStatisticsInteger) {
1184+
MemoryOutputStream mem_stream(kDefaultMemStreamSize);
1185+
std::unique_ptr<liborc::Type> type(
1186+
liborc::Type::buildTypeFromString("struct<col1:int,col2:bigint>"));
1187+
1188+
constexpr uint64_t stripe_size = 1024;
1189+
constexpr uint64_t row_count = 1000;
1190+
1191+
auto writer = CreateWriter(stripe_size, *type, &mem_stream);
1192+
auto batch = writer->createRowBatch(row_count);
1193+
auto struct_batch = internal::checked_cast<liborc::StructVectorBatch*>(batch.get());
1194+
auto int_batch =
1195+
internal::checked_cast<liborc::LongVectorBatch*>(struct_batch->fields[0]);
1196+
auto bigint_batch =
1197+
internal::checked_cast<liborc::LongVectorBatch*>(struct_batch->fields[1]);
1198+
1199+
for (uint64_t i = 0; i < row_count; ++i) {
1200+
int_batch->data[i] = static_cast<int64_t>(i);
1201+
bigint_batch->data[i] = static_cast<int64_t>(i + 1000);
1202+
}
1203+
struct_batch->numElements = row_count;
1204+
int_batch->numElements = row_count;
1205+
bigint_batch->numElements = row_count;
1206+
writer->add(*batch);
1207+
writer->close();
1208+
1209+
std::shared_ptr<io::RandomAccessFile> in_stream(new io::BufferReader(
1210+
std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(mem_stream.getData()),
1211+
static_cast<int64_t>(mem_stream.getLength()))));
1212+
1213+
ASSERT_OK_AND_ASSIGN(auto reader,
1214+
adapters::orc::ORCFileReader::Open(in_stream, default_memory_pool()));
1215+
1216+
ASSERT_OK_AND_ASSIGN(auto col1_stats, reader->GetColumnStatistics(1));
1217+
EXPECT_EQ(col1_stats.num_values, row_count);
1218+
EXPECT_TRUE(col1_stats.has_min_max);
1219+
ASSERT_NE(col1_stats.min, nullptr);
1220+
ASSERT_NE(col1_stats.max, nullptr);
1221+
EXPECT_EQ(checked_pointer_cast<Int64Scalar>(col1_stats.min)->value, 0);
1222+
EXPECT_EQ(checked_pointer_cast<Int64Scalar>(col1_stats.max)->value, 999);
1223+
1224+
ASSERT_OK_AND_ASSIGN(auto col2_stats, reader->GetColumnStatistics(2));
1225+
EXPECT_EQ(col2_stats.num_values, row_count);
1226+
EXPECT_TRUE(col2_stats.has_min_max);
1227+
ASSERT_NE(col2_stats.min, nullptr);
1228+
ASSERT_NE(col2_stats.max, nullptr);
1229+
EXPECT_EQ(checked_pointer_cast<Int64Scalar>(col2_stats.min)->value, 1000);
1230+
EXPECT_EQ(checked_pointer_cast<Int64Scalar>(col2_stats.max)->value, 1999);
1231+
}
1232+
1233+
TEST(TestAdapterRead, GetStripeColumnStatistics) {
1234+
MemoryOutputStream mem_stream(kDefaultMemStreamSize);
1235+
std::unique_ptr<liborc::Type> type(
1236+
liborc::Type::buildTypeFromString("struct<col1:int>"));
1237+
1238+
constexpr uint64_t stripe_size = 1024;
1239+
constexpr uint64_t row_count = 500;
1240+
1241+
auto writer = CreateWriter(stripe_size, *type, &mem_stream);
1242+
auto batch = writer->createRowBatch(row_count);
1243+
auto struct_batch = internal::checked_cast<liborc::StructVectorBatch*>(batch.get());
1244+
auto int_batch =
1245+
internal::checked_cast<liborc::LongVectorBatch*>(struct_batch->fields[0]);
1246+
1247+
for (uint64_t i = 0; i < row_count; ++i) {
1248+
int_batch->data[i] = static_cast<int64_t>(i + 100);
1249+
}
1250+
struct_batch->numElements = row_count;
1251+
int_batch->numElements = row_count;
1252+
writer->add(*batch);
1253+
writer->close();
1254+
1255+
std::shared_ptr<io::RandomAccessFile> in_stream(new io::BufferReader(
1256+
std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(mem_stream.getData()),
1257+
static_cast<int64_t>(mem_stream.getLength()))));
1258+
1259+
ASSERT_OK_AND_ASSIGN(auto reader,
1260+
adapters::orc::ORCFileReader::Open(in_stream, default_memory_pool()));
1261+
1262+
ASSERT_OK_AND_ASSIGN(auto stripe0_stats, reader->GetStripeColumnStatistics(0, 1));
1263+
EXPECT_TRUE(stripe0_stats.has_min_max);
1264+
EXPECT_EQ(checked_pointer_cast<Int64Scalar>(stripe0_stats.min)->value, 100);
1265+
EXPECT_EQ(checked_pointer_cast<Int64Scalar>(stripe0_stats.max)->value, 599);
1266+
}
1267+
1268+
TEST(TestAdapterRead, GetColumnStatisticsString) {
1269+
MemoryOutputStream mem_stream(kDefaultMemStreamSize);
1270+
std::unique_ptr<liborc::Type> type(
1271+
liborc::Type::buildTypeFromString("struct<col1:string>"));
1272+
1273+
constexpr uint64_t stripe_size = 1024;
1274+
constexpr uint64_t row_count = 5;
1275+
1276+
auto writer = CreateWriter(stripe_size, *type, &mem_stream);
1277+
auto batch = writer->createRowBatch(row_count);
1278+
auto struct_batch = internal::checked_cast<liborc::StructVectorBatch*>(batch.get());
1279+
auto str_batch =
1280+
internal::checked_cast<liborc::StringVectorBatch*>(struct_batch->fields[0]);
1281+
1282+
std::vector<std::string> strings = {"apple", "banana", "cherry", "date", "elderberry"};
1283+
std::string data_buffer;
1284+
for (const auto& s : strings) {
1285+
data_buffer += s;
1286+
}
1287+
1288+
size_t offset = 0;
1289+
for (size_t i = 0; i < strings.size(); ++i) {
1290+
str_batch->data[i] = const_cast<char*>(&data_buffer[offset]);
1291+
str_batch->length[i] = static_cast<int64_t>(strings[i].size());
1292+
offset += strings[i].size();
1293+
}
1294+
struct_batch->numElements = row_count;
1295+
str_batch->numElements = row_count;
1296+
writer->add(*batch);
1297+
writer->close();
1298+
1299+
std::shared_ptr<io::RandomAccessFile> in_stream(new io::BufferReader(
1300+
std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(mem_stream.getData()),
1301+
static_cast<int64_t>(mem_stream.getLength()))));
1302+
1303+
ASSERT_OK_AND_ASSIGN(auto reader,
1304+
adapters::orc::ORCFileReader::Open(in_stream, default_memory_pool()));
1305+
1306+
ASSERT_OK_AND_ASSIGN(auto col_stats, reader->GetColumnStatistics(1));
1307+
EXPECT_EQ(col_stats.num_values, row_count);
1308+
EXPECT_TRUE(col_stats.has_min_max);
1309+
ASSERT_NE(col_stats.min, nullptr);
1310+
ASSERT_NE(col_stats.max, nullptr);
1311+
EXPECT_EQ(checked_pointer_cast<StringScalar>(col_stats.min)->ToString(), "apple");
1312+
EXPECT_EQ(checked_pointer_cast<StringScalar>(col_stats.max)->ToString(), "elderberry");
1313+
}
1314+
1315+
TEST(TestAdapterRead, GetColumnStatisticsOutOfRange) {
1316+
MemoryOutputStream mem_stream(kDefaultMemStreamSize);
1317+
std::unique_ptr<liborc::Type> type(
1318+
liborc::Type::buildTypeFromString("struct<col1:int>"));
1319+
1320+
constexpr uint64_t stripe_size = 1024;
1321+
constexpr uint64_t row_count = 10;
1322+
1323+
auto writer = CreateWriter(stripe_size, *type, &mem_stream);
1324+
auto batch = writer->createRowBatch(row_count);
1325+
auto struct_batch = internal::checked_cast<liborc::StructVectorBatch*>(batch.get());
1326+
auto int_batch =
1327+
internal::checked_cast<liborc::LongVectorBatch*>(struct_batch->fields[0]);
1328+
1329+
for (uint64_t i = 0; i < row_count; ++i) {
1330+
int_batch->data[i] = static_cast<int64_t>(i);
1331+
}
1332+
struct_batch->numElements = row_count;
1333+
int_batch->numElements = row_count;
1334+
writer->add(*batch);
1335+
writer->close();
1336+
1337+
std::shared_ptr<io::RandomAccessFile> in_stream(new io::BufferReader(
1338+
std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(mem_stream.getData()),
1339+
static_cast<int64_t>(mem_stream.getLength()))));
1340+
1341+
ASSERT_OK_AND_ASSIGN(auto reader,
1342+
adapters::orc::ORCFileReader::Open(in_stream, default_memory_pool()));
1343+
1344+
EXPECT_THAT(reader->GetColumnStatistics(999),
1345+
Raises(StatusCode::Invalid, testing::HasSubstr("out of range")));
1346+
1347+
EXPECT_THAT(reader->GetStripeColumnStatistics(999, 1),
1348+
Raises(StatusCode::Invalid, testing::HasSubstr("out of range")));
1349+
}
1350+
1351+
TEST(TestAdapterRead, GetColumnStatisticsWithNulls) {
1352+
MemoryOutputStream mem_stream(kDefaultMemStreamSize);
1353+
std::unique_ptr<liborc::Type> type(
1354+
liborc::Type::buildTypeFromString("struct<col1:int>"));
1355+
1356+
constexpr uint64_t stripe_size = 1024;
1357+
constexpr uint64_t row_count = 10;
1358+
1359+
auto writer = CreateWriter(stripe_size, *type, &mem_stream);
1360+
auto batch = writer->createRowBatch(row_count);
1361+
auto struct_batch = internal::checked_cast<liborc::StructVectorBatch*>(batch.get());
1362+
auto int_batch =
1363+
internal::checked_cast<liborc::LongVectorBatch*>(struct_batch->fields[0]);
1364+
1365+
for (uint64_t i = 0; i < row_count; ++i) {
1366+
int_batch->data[i] = static_cast<int64_t>(i);
1367+
int_batch->notNull[i] = (i % 2 == 0) ? 1 : 0;
1368+
}
1369+
int_batch->hasNulls = true;
1370+
struct_batch->numElements = row_count;
1371+
int_batch->numElements = row_count;
1372+
writer->add(*batch);
1373+
writer->close();
1374+
1375+
std::shared_ptr<io::RandomAccessFile> in_stream(new io::BufferReader(
1376+
std::make_shared<Buffer>(reinterpret_cast<const uint8_t*>(mem_stream.getData()),
1377+
static_cast<int64_t>(mem_stream.getLength()))));
1378+
1379+
ASSERT_OK_AND_ASSIGN(auto reader,
1380+
adapters::orc::ORCFileReader::Open(in_stream, default_memory_pool()));
1381+
1382+
ASSERT_OK_AND_ASSIGN(auto col_stats, reader->GetColumnStatistics(1));
1383+
EXPECT_TRUE(col_stats.has_null);
1384+
EXPECT_TRUE(col_stats.has_min_max);
1385+
}
1386+
11821387
} // namespace arrow

0 commit comments

Comments
 (0)