PARQUET-793: Do not return incorrect statistics

Author: Deepak Majeti <deepak.majeti@hpe.com>

Closes apache#239 from majetideepak/FixStats and squashes the following commits:

948c9b9 [Deepak Majeti] fix failing test
ea0693f [Deepak Majeti] review comments
dbb8569 [Deepak Majeti] minor fix
75d392a [Deepak Majeti] format
e89f85f [Deepak Majeti] stats test and comments
800e239 [Deepak Majeti] added tests
92f31cd [Deepak Majeti] Return only Signed stats
feeeb88 [Deepak Majeti] Update Version and add checkCorrectStatistics

Change-Id: Id8e86893e0987849cf08e20793daf9fbefcb9f37

Author: Deepak Majeti

cpp/src/parquet/column/statistics-test.cc

@@ -32,6 +32,7 @@
 #include "parquet/file/reader.h"
 #include "parquet/file/writer.h"
 #include "parquet/schema.h"
+#include "parquet/thrift.h"
 #include "parquet/types.h"
 #include "parquet/util/memory.h"
 
@@ -167,6 +168,7 @@ class TestRowGroupStatistics : public PrimitiveTypedTest<TestType> {
     auto file_reader = ParquetFileReader::Open(source);
     auto rg_reader = file_reader->RowGroup(0);
     auto column_chunk = rg_reader->metadata()->ColumnChunk(0);
+    if (!column_chunk->is_stats_set()) return;
     std::shared_ptr<RowGroupStatistics> stats = column_chunk->statistics();
     // check values after serialization + deserialization
     ASSERT_EQ(null_count, stats->null_count());
@@ -308,5 +310,49 @@ TYPED_TEST(TestNumericRowGroupStatistics, Merge) {
   this->TestMerge();
 }
 
+TEST(CorruptStatistics, Basics) {
+  ApplicationVersion version("parquet-mr version 1.8.0");
+  SchemaDescriptor schema;
+  schema::NodePtr node;
+  std::vector<schema::NodePtr> fields;
+  // Test Physical Types
+  fields.push_back(schema::PrimitiveNode::Make(
+      "col1", Repetition::OPTIONAL, Type::INT32, LogicalType::NONE));
+  fields.push_back(schema::PrimitiveNode::Make(
+      "col2", Repetition::OPTIONAL, Type::BYTE_ARRAY, LogicalType::NONE));
+  // Test Logical Types
+  fields.push_back(schema::PrimitiveNode::Make(
+      "col3", Repetition::OPTIONAL, Type::INT32, LogicalType::DATE));
+  fields.push_back(schema::PrimitiveNode::Make(
+      "col4", Repetition::OPTIONAL, Type::INT32, LogicalType::UINT_32));
+  fields.push_back(schema::PrimitiveNode::Make("col5", Repetition::OPTIONAL,
+      Type::FIXED_LEN_BYTE_ARRAY, LogicalType::INTERVAL, 12));
+  fields.push_back(schema::PrimitiveNode::Make(
+      "col6", Repetition::OPTIONAL, Type::BYTE_ARRAY, LogicalType::UTF8));
+  node = schema::GroupNode::Make("schema", Repetition::REQUIRED, fields);
+  schema.Init(node);
+
+  format::ColumnChunk col_chunk;
+  col_chunk.meta_data.__isset.statistics = true;
+  auto column_chunk1 = ColumnChunkMetaData::Make(
+      reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(0), &version);
+  ASSERT_TRUE(column_chunk1->is_stats_set());
+  auto column_chunk2 = ColumnChunkMetaData::Make(
+      reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(1), &version);
+  ASSERT_FALSE(column_chunk2->is_stats_set());
+  auto column_chunk3 = ColumnChunkMetaData::Make(
+      reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(2), &version);
+  ASSERT_TRUE(column_chunk3->is_stats_set());
+  auto column_chunk4 = ColumnChunkMetaData::Make(
+      reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(3), &version);
+  ASSERT_FALSE(column_chunk4->is_stats_set());
+  auto column_chunk5 = ColumnChunkMetaData::Make(
+      reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(4), &version);
+  ASSERT_FALSE(column_chunk5->is_stats_set());
+  auto column_chunk6 = ColumnChunkMetaData::Make(
+      reinterpret_cast<const uint8_t*>(&col_chunk), schema.Column(5), &version);
+  ASSERT_FALSE(column_chunk6->is_stats_set());
+}
+
 }  // namespace test
 }  // namespace parquet

cpp/src/parquet/file/file-metadata-test.cc

@@ -181,13 +181,29 @@ TEST(Metadata, TestV1Version) {
   ASSERT_EQ(ParquetVersion::PARQUET_1_0, f_accessor->version());
 }
 
-TEST(FileVersion, Basics) {
-  FileMetaData::Version version("parquet-mr version 1.2.8");
+TEST(ApplicationVersion, Basics) {
+  ApplicationVersion version("parquet-mr version 1.7.9");
+  ApplicationVersion version1("parquet-mr version 1.8.0");
+  ApplicationVersion version2("parquet-cpp version 1.0.0");
+  ApplicationVersion version3("");
 
   ASSERT_EQ("parquet-mr", version.application);
   ASSERT_EQ(1, version.version.major);
-  ASSERT_EQ(2, version.version.minor);
-  ASSERT_EQ(8, version.version.patch);
+  ASSERT_EQ(7, version.version.minor);
+  ASSERT_EQ(9, version.version.patch);
+
+  ASSERT_EQ("parquet-cpp", version2.application);
+  ASSERT_EQ(1, version2.version.major);
+  ASSERT_EQ(0, version2.version.minor);
+  ASSERT_EQ(0, version2.version.patch);
+
+  ASSERT_EQ(true, version.VersionLt(version1));
+
+  ASSERT_FALSE(version1.HasCorrectStatistics(Type::INT96));
+  ASSERT_TRUE(version.HasCorrectStatistics(Type::INT32));
+  ASSERT_FALSE(version.HasCorrectStatistics(Type::BYTE_ARRAY));
+  ASSERT_TRUE(version1.HasCorrectStatistics(Type::BYTE_ARRAY));
+  ASSERT_TRUE(version3.HasCorrectStatistics(Type::FIXED_LEN_BYTE_ARRAY));
 }
 
 }  // namespace metadata