Minimize stack space required to plan deeply nested binary expressions

datafusion/core/tests/tpcds_planning.rs

@@ -343,7 +343,6 @@ async fn tpcds_logical_q63() -> Result<()> {
     create_logical_plan(63).await
 }
 
-#[ignore] // thread 'q64' has overflowed its stack]
 #[tokio::test]
 async fn tpcds_logical_q64() -> Result<()> {
     create_logical_plan(64).await
@@ -851,7 +850,6 @@ async fn tpcds_physical_q63() -> Result<()> {
     create_physical_plan(63).await
 }
 
-#[ignore] // thread 'q64' has overflowed its stack
 #[tokio::test]
 async fn tpcds_physical_q64() -> Result<()> {
     create_physical_plan(64).await

datafusion/sql/src/planner.rs

@@ -1839,6 +1839,37 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
         sql: SQLExpr,
         schema: &DFSchema,
         planner_context: &mut PlannerContext,
+    ) -> Result<Expr> {
+        // Workaround for https://github.com/apache/arrow-datafusion/issues/4065
+        //
+        // Minimize stack space required in debug builds to plan
+        // deeply nested binary operators by keeping the stack space
+        // needed for sql_expr_to_logical_expr minimal for BinaryOp
+        //
+        // The reason this reduces stack size in debug builds is
+        // explained in the "Technical Backstory" heading of
+        // https://github.com/apache/arrow-datafusion/pull/1047
+        //
+        // A likely better way to support deeply nested expressions
+        // would be to avoid recursion all together and use an
+        // iterative algorithm.
+        match sql {
+            SQLExpr::BinaryOp { left, op, right } => {
+                self.parse_sql_binary_op(*left, op, *right, schema, planner_context)
+            }
+            // since this function requires more space per frame
+            // avoid calling it for binary ops
+            _ => self.sql_expr_to_logical_expr_internal(sql, schema, planner_context),
+        }
+    }
+
+    /// Internal implementation. Use
+    /// [`Self::sql_expr_to_logical_expr`] to plan exprs.
+    fn sql_expr_to_logical_expr_internal(
+        &self,
+        sql: SQLExpr,
+        schema: &DFSchema,
+        planner_context: &mut PlannerContext,
     ) -> Result<Expr> {
         match sql {
             SQLExpr::Value(value) => {
@@ -1976,10 +2007,13 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
 
 
             SQLExpr::BinaryOp {
-                left,
-                op,
-                right,
-            } => self.parse_sql_binary_op(*left, op, *right, schema, planner_context),
+                ..
+            } => {
+                Err(DataFusionError::Internal(
+                    "binary_op should be handled by sql_expr_to_logical_expr.".to_string()
+                ))
+            }
+
 
             #[cfg(feature = "unicode_expressions")]
             SQLExpr::Substring {