Update datafusion-proto

Author: alamb

datafusion/expr/src/utils.rs

@@ -814,6 +814,8 @@ pub fn can_hash(data_type: &DataType) -> bool {
         DataType::Float16 => true,
         DataType::Float32 => true,
         DataType::Float64 => true,
+        DataType::Decimal32(_, _) => true,
+        DataType::Decimal64(_, _) => true,
         DataType::Decimal128(_, _) => true,
         DataType::Decimal256(_, _) => true,
         DataType::Timestamp(_, _) => true,

datafusion/proto-common/proto/datafusion_common.proto

@@ -134,7 +134,19 @@ enum IntervalUnit{
   MonthDayNano = 2;
 }
 
-message Decimal{
+message Decimal32Type {
+  reserved 1, 2;
+  uint32 precision = 3;
+  int32 scale = 4;
+}
+
+message Decimal64Type {
+  reserved 1, 2;
+  uint32 precision = 3;
+  int32 scale = 4;
+}
+
+message Decimal128Type {
   reserved 1, 2;
   uint32 precision = 3;
   int32 scale = 4;
@@ -284,6 +296,8 @@ message ScalarValue{
     ScalarNestedValue struct_value = 32;
     ScalarNestedValue map_value = 41;
 
+    Decimal32 decimal32_value = 43;
+    Decimal64 decimal64_value = 44;
     Decimal128 decimal128_value = 20;
     Decimal256 decimal256_value = 39;
 
@@ -308,6 +322,18 @@ message ScalarValue{
   }
 }
 
+message Decimal32{
+  bytes value = 1;
+  int64 p = 2;
+  int64 s = 3;
+}
+
+message Decimal64{
+  bytes value = 1;
+  int64 p = 2;
+  int64 s = 3;
+}
+
 message Decimal128{
   bytes value = 1;
   int64 p = 2;
@@ -350,7 +376,9 @@ message ArrowType{
     TimeUnit TIME32 = 21 ;
     TimeUnit TIME64 = 22 ;
     IntervalUnit INTERVAL = 23 ;
-    Decimal DECIMAL = 24 ;
+    Decimal32Type DECIMAL32 = 40;
+    Decimal64Type DECIMAL64 = 41;
+    Decimal128Type DECIMAL128 = 24;
     Decimal256Type DECIMAL256 = 36;
     List LIST = 25;
     List LARGE_LIST = 26;
@@ -478,9 +506,7 @@ message ParquetColumnOptions {
     uint64 bloom_filter_ndv = 7;
   }
 
-  oneof max_statistics_size_opt {
-    uint32 max_statistics_size = 8;
-  }
+  reserved 8; // used to be uint32 max_statistics_size = 8;
 }
 
 message ParquetOptions {
@@ -519,9 +545,7 @@ message ParquetOptions {
     string statistics_enabled = 13;
   }
 
-  oneof max_statistics_size_opt {
-    uint64 max_statistics_size = 14;
-  }
+  reserved 14; // used to be uint32 max_statistics_size = 20;
 
   oneof column_index_truncate_length_opt {
     uint64 column_index_truncate_length = 17;

datafusion/proto-common/src/from_proto/mod.rs

@@ -37,6 +37,7 @@ use datafusion_common::{
         TableParquetOptions,
     },
     file_options::{csv_writer::CsvWriterOptions, json_writer::JsonWriterOptions},
+    not_impl_err,
     parsers::CompressionTypeVariant,
     plan_datafusion_err,
     stats::Precision,
@@ -257,7 +258,15 @@ impl TryFrom<&protobuf::arrow_type::ArrowTypeEnum> for DataType {
             arrow_type::ArrowTypeEnum::Interval(interval_unit) => {
                 DataType::Interval(parse_i32_to_interval_unit(interval_unit)?)
             }
-            arrow_type::ArrowTypeEnum::Decimal(protobuf::Decimal {
+            arrow_type::ArrowTypeEnum::Decimal32(protobuf::Decimal32Type {
+                precision,
+                scale,
+            }) => DataType::Decimal32(*precision as u8, *scale as i8),
+            arrow_type::ArrowTypeEnum::Decimal64(protobuf::Decimal64Type {
+                precision,
+                scale,
+            }) => DataType::Decimal64(*precision as u8, *scale as i8),
+            arrow_type::ArrowTypeEnum::Decimal128(protobuf::Decimal128Type {
                 precision,
                 scale,
             }) => DataType::Decimal128(*precision as u8, *scale as i8),
@@ -469,6 +478,14 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue {
                 let null_type: DataType = v.try_into()?;
                 null_type.try_into().map_err(Error::DataFusionError)?
             }
+            Value::Decimal32Value(_val) => {
+                return not_impl_err!("Decimal32 protobuf deserialization")
+                    .map_err(Error::DataFusionError)
+            }
+            Value::Decimal64Value(_val) => {
+                return not_impl_err!("Decimal64 protobuf deserialization")
+                    .map_err(Error::DataFusionError)
+            }
             Value::Decimal128Value(val) => {
                 let array = vec_to_array(val.value.clone());
                 Self::Decimal128(
@@ -938,12 +955,6 @@ impl TryFrom<&protobuf::ParquetOptions> for ParquetOptions {
                     protobuf::parquet_options::StatisticsEnabledOpt::StatisticsEnabled(v) => Some(v),
                 })
                 .unwrap_or(None),
-            max_statistics_size: value
-                .max_statistics_size_opt.as_ref()
-                .map(|opt| match opt {
-                    protobuf::parquet_options::MaxStatisticsSizeOpt::MaxStatisticsSize(v) => Some(*v as usize),
-                })
-                .unwrap_or(None),
             max_row_group_size: value.max_row_group_size as usize,
             created_by: value.created_by.clone(),
             column_index_truncate_length: value
@@ -1009,12 +1020,6 @@ impl TryFrom<&protobuf::ParquetColumnOptions> for ParquetColumnOptions {
                     protobuf::parquet_column_options::StatisticsEnabledOpt::StatisticsEnabled(v) => Some(v),
                 })
                 .unwrap_or(None),
-            max_statistics_size: value
-                .max_statistics_size_opt
-                .map(|opt| match opt {
-                    protobuf::parquet_column_options::MaxStatisticsSizeOpt::MaxStatisticsSize(v) => Some(v as usize),
-                })
-                .unwrap_or(None),
             encoding: value
                 .encoding_opt.clone()
                 .map(|opt| match opt {