output to_timestamp signature as ns

Author: comphead

datafusion-examples/examples/expr_api.rs

@@ -82,10 +82,10 @@ fn simplify_demo() -> Result<()> {
     let expr = simplifier.simplify(expr)?;
 
     // DataFusion has simplified the expression to a comparison with a constant
-    // ts = 1599566400i64; Tada!
+    // ts = 1599566400000000000; Tada!
     assert_eq!(
         expr,
-        col("ts").eq(lit(ScalarValue::TimestampSecond(Some(1599566400i64), None)))
+        col("ts").eq(lit_timestamp_nano(1599566400000000000i64))
     );
 
     // here are some other examples of what DataFusion is capable of

datafusion/core/tests/sql/expr.rs

@@ -639,7 +639,7 @@ async fn test_uuid_expression() -> Result<()> {
 async fn test_extract_date_part() -> Result<()> {
     test_expression!("date_part('YEAR', CAST('2000-01-01' AS DATE))", "2000.0");
     test_expression!(
-        "EXTRACT(year FROM to_timestamp('2020-09-08T12:00:00+00:00'))",
+        "EXTRACT(year FROM  timestamp '2020-09-08T12:00:00+00:00')",
         "2020.0"
     );
     test_expression!("date_part('QUARTER', CAST('2000-01-01' AS DATE))", "1.0");

datafusion/expr/src/built_in_function.rs

@@ -748,7 +748,7 @@ impl BuiltinScalarFunction {
                     return plan_err!("The to_hex function can only accept integers.");
                 }
             }),
-            BuiltinScalarFunction::ToTimestamp => Ok(Timestamp(Second, None)),
+            BuiltinScalarFunction::ToTimestamp => Ok(Timestamp(Nanosecond, None)),
             BuiltinScalarFunction::ToTimestampMillis => Ok(Timestamp(Millisecond, None)),
             BuiltinScalarFunction::ToTimestampMicros => Ok(Timestamp(Microsecond, None)),
             BuiltinScalarFunction::ToTimestampNanos => Ok(Timestamp(Nanosecond, None)),

datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs

@@ -1504,10 +1504,7 @@ mod tests {
         // to_timestamp("2020-09-08T12:00:00+00:00") --> timestamp(1599566400i64)
         let expr =
             call_fn("to_timestamp", vec![lit("2020-09-08T12:00:00+00:00")]).unwrap();
-        test_evaluate(
-            expr,
-            lit(ScalarValue::TimestampSecond(Some(1599566400i64), None)),
-        );
+        test_evaluate(expr, lit_timestamp_nano(1599566400000000000i64));
 
         // check that non foldable arguments are folded
         // to_timestamp(a) --> to_timestamp(a) [no rewrite possible]
@@ -1543,10 +1540,10 @@ mod tests {
         let expr = cast_to_int64_expr(now_expr()) + lit(100_i64);
         test_evaluate_with_start_time(expr, lit(ts_nanos + 100), &time);
 
-        //  CAST(now() as int64) < cast(to_timestamp(...) as int64) + 50000_i64 ---> false
+        //  CAST(now() as int64) < cast(to_timestamp(...) as int64) + 50000_i64 ---> true
         let expr = cast_to_int64_expr(now_expr())
             .lt(cast_to_int64_expr(to_timestamp_expr(ts_string)) + lit(50000i64));
-        test_evaluate_with_start_time(expr, lit(false), &time);
+        test_evaluate_with_start_time(expr, lit(true), &time);
     }
 
     #[test]

datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs

@@ -460,7 +460,7 @@ mod tests {
             .project(proj)?
             .build()?;
 
-        let expected = "Projection: TimestampSecond(1599566400, None) AS to_timestamp(Utf8(\"2020-09-08T12:00:00+00:00\"))\
+        let expected = "Projection: TimestampNanosecond(1599566400000000000, None) AS to_timestamp(Utf8(\"2020-09-08T12:00:00+00:00\"))\
             \n  TableScan: test"
             .to_string();
         let actual = get_optimized_plan_formatted(&plan, &Utc::now());
@@ -571,8 +571,8 @@ mod tests {
                 .build()?;
 
         // Note that constant folder runs and folds the entire
-        // expression down to a single constant (false)
-        let expected = "Filter: Boolean(false)\
+        // expression down to a single constant (true)
+        let expected = "Filter: Boolean(true)\
                         \n  TableScan: test";
         let actual = get_optimized_plan_formatted(&plan, &time);
 

datafusion/physical-expr/src/datetime_expressions.rs

@@ -129,9 +129,9 @@ fn string_to_timestamp_nanos_shim(s: &str) -> Result<i64> {
 
 /// to_timestamp SQL function
 pub fn to_timestamp(args: &[ColumnarValue]) -> Result<ColumnarValue> {
-    handle::<TimestampSecondType, _, TimestampSecondType>(
+    handle::<TimestampNanosecondType, _, TimestampNanosecondType>(
         args,
-        |s| string_to_timestamp_nanos_shim(s).map(|n| n / 1_000_000_000),
+        string_to_timestamp_nanos_shim,
         "to_timestamp",
     )
 }
@@ -969,10 +969,10 @@ mod tests {
         // ensure that arrow array implementation is wired up and handles nulls correctly
 
         let mut string_builder = StringBuilder::with_capacity(2, 1024);
-        let mut ts_builder = TimestampSecondArray::builder(2);
+        let mut ts_builder = TimestampNanosecondArray::builder(2);
 
-        string_builder.append_value("2020-09-08T13:42:29");
-        ts_builder.append_value(1599572549);
+        string_builder.append_value("2020-09-08T13:42:29.190855");
+        ts_builder.append_value(1599572549190855000);
 
         string_builder.append_null();
         ts_builder.append_null();

datafusion/physical-expr/src/functions.rs

@@ -74,15 +74,20 @@ pub fn create_physical_expr(
         // so we don't have to pay a per-array/batch cost.
         BuiltinScalarFunction::ToTimestamp => {
             Arc::new(match input_phy_exprs[0].data_type(input_schema) {
-                Ok(DataType::Int64) | Ok(DataType::Timestamp(_, None)) => {
-                    |col_values: &[ColumnarValue]| {
-                        cast_column(
-                            &col_values[0],
-                            &DataType::Timestamp(TimeUnit::Second, None),
-                            None,
-                        )
-                    }
-                }
+                Ok(DataType::Int64) => |col_values: &[ColumnarValue]| {
+                    cast_column(
+                        &col_values[0],
+                        &DataType::Timestamp(TimeUnit::Second, None),
+                        None,
+                    )
+                },
+                Ok(DataType::Timestamp(_, None)) => |col_values: &[ColumnarValue]| {
+                    cast_column(
+                        &col_values[0],
+                        &DataType::Timestamp(TimeUnit::Nanosecond, None),
+                        None,
+                    )
+                },
                 Ok(DataType::Utf8) => datetime_expressions::to_timestamp,
                 other => {
                     return internal_err!(

datafusion/sql/tests/sql_integration.rs

@@ -444,7 +444,7 @@ Dml: op=[Insert Into] table=[test_decimal]
 )]
 #[case::type_mismatch(
     "INSERT INTO test_decimal SELECT '2022-01-01', to_timestamp('2022-01-01T12:00:00')",
-    "Error during planning: Cannot automatically convert Timestamp(Second, None) to Decimal128(10, 2)"
+    "Error during planning: Cannot automatically convert Timestamp(Nanosecond, None) to Decimal128(10, 2)"
 )]
 #[case::target_column_count_mismatch(
     "INSERT INTO person (id, first_name, last_name) VALUES ($1, $2)",

datafusion/sqllogictest/test_files/timestamps.slt

@@ -249,9 +249,9 @@ SELECT to_timestamp_seconds(ts) FROM ts_data_micros LIMIT 3
 query P
 SELECT to_timestamp(ts) FROM ts_data_micros LIMIT 3
 ----
-2020-09-08T13:42:29
-2020-09-08T12:42:29
-2020-09-08T11:42:29
+2020-09-08T13:42:29.190855
+2020-09-08T12:42:29.190855
+2020-09-08T11:42:29.190855
 
 # query_cast_timestamp_from_unixtime
 
@@ -266,7 +266,7 @@ SELECT from_unixtime(ts / 1000000000) FROM ts_data LIMIT 3;
 # to_timestamp
 
 query I
-SELECT COUNT(*) FROM ts_data_nanos where ts > to_timestamp('2020-09-08T12:00:00+00:00')
+SELECT COUNT(*) FROM ts_data_nanos where ts > timestamp '2020-09-08T12:00:00+00:00'
 ----
 2
 

docs/source/user-guide/sql/scalar_functions.md

@@ -1391,9 +1391,9 @@ extract(field FROM source)
 
 ### `to_timestamp`
 
-Converts a value to RFC3339 second timestamp format (`YYYY-MM-DDT00:00:00Z`).
+Converts a value to RFC3339 nanosecond timestamp format (`YYYY-MM-DDT00:00:00Z`).
 Supports timestamp, integer, and unsigned integer types as input.
-Integers and unsigned integers are parsed as Unix nanosecond timestamps and
+Integers and unsigned integers are parsed as Unix second timestamps and
 return the corresponding RFC3339 timestamp.
 
 ```
@@ -1450,7 +1450,10 @@ to_timestamp_nanos(expression)
 
 ### `to_timestamp_seconds`
 
-Alias for `to_timestamp()` function.
+Converts a value to RFC3339 second timestamp format (`YYYY-MM-DDT00:00:00Z`).
+Supports timestamp, integer, and unsigned integer types as input.
+Integers and unsigned integers are parsed as Unix nanosecond timestamps and
+return the corresponding RFC3339 timestamp.
 
 ```
 to_timestamp_seconds(expression)