initial

Author: myrrc

vortex-duckdb/cpp/table_function.cpp

@@ -277,6 +277,19 @@ void c_function(ClientContext &context, TableFunctionInput &input, DataChunk &ou
     }
 }
 
+/*
+ * Table filter pushdown is used twice in duckdb:
+ *
+ * 1. Planning time: duckdb uses file metadata (filename, hive_partitioning
+ * options in MultiFileReader) to prune files based on filename or hive
+ * partition data i.e. month, year, etc. This happens before any file IO.
+ * We don't use this because we have own file-level pruning in
+ * FileStatsLayoutReader.
+ *
+ * 2. Scan time. As we have filter_pushdown = true, filter expressions are
+ * converted to TableFilterSet and pushed down to Vortex. We convert them to
+ * vortex expressions and use as filter options while initializing the scan.
+ */
 void c_pushdown_complex_filter(ClientContext &,
                                LogicalGet &,
                                FunctionData *bind_data,
@@ -292,8 +305,6 @@ void c_pushdown_complex_filter(ClientContext &,
         if (error_out) {
             throw BinderException(IntoErrString(error_out));
         }
-
-        // If the pushdown complex filter returns true, we can remove the filter from the list.
         iter = pushed ? filters.erase(iter) : std::next(iter);
     }
 }
@@ -394,6 +405,10 @@ extern "C" duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const d
         return {{COLUMN_IDENTIFIER_EMPTY, TableColumn("", LogicalTypeId::BOOLEAN)}};
     };
 
+    tf.pushdown_expression = [](auto &, auto &, auto &) {
+        return true;
+    };
+
     tf.arguments.resize(vtab->parameter_count);
     for (size_t i = 0; i < vtab->parameter_count; i++) {
         tf.arguments[i] = *reinterpret_cast<LogicalType *>(vtab->parameters[i]);

vortex-duckdb/src/convert/expr.rs

@@ -46,34 +46,52 @@ fn like_pattern_str(value: &duckdb::ExpressionRef) -> VortexResult<Option<String
 
 pub fn try_from_bound_expression(
     value: &duckdb::ExpressionRef,
+) -> VortexResult<Option<Expression>> {
+    try_from_expression_inner(value, None)
+}
+
+pub(super) fn try_from_bound_expression_with_col_sub(
+    value: &duckdb::ExpressionRef,
+    col_sub: &Expression,
+) -> VortexResult<Option<Expression>> {
+    try_from_expression_inner(value, Some(col_sub))
+}
+
+fn try_from_expression_inner(
+    value: &duckdb::ExpressionRef,
+    col_sub: Option<&Expression>,
 ) -> VortexResult<Option<Expression>> {
     let Some(value) = value.as_class() else {
         tracing::debug!("no expression class id {:?}", value.as_class_id());
         return Ok(None);
     };
     Ok(Some(match value {
+        duckdb::ExpressionClass::BoundRef => {
+            let Some(col) = col_sub else { return Ok(None) };
+            col.clone()
+        }
         duckdb::ExpressionClass::BoundColumnRef(col_ref) => col(col_ref.name.as_ref()),
         duckdb::ExpressionClass::BoundConstant(const_) => lit(Scalar::try_from(const_.value)?),
         duckdb::ExpressionClass::BoundComparison(compare) => {
             let operator: Operator = compare.op.try_into()?;
 
-            let Some(left) = try_from_bound_expression(compare.left)? else {
+            let Some(left) = try_from_expression_inner(compare.left, col_sub)? else {
                 return Ok(None);
             };
-            let Some(right) = try_from_bound_expression(compare.right)? else {
+            let Some(right) = try_from_expression_inner(compare.right, col_sub)? else {
                 return Ok(None);
             };
 
             Binary.new_expr(operator, [left, right])
         }
         duckdb::ExpressionClass::BoundBetween(between) => {
-            let Some(array) = try_from_bound_expression(between.input)? else {
+            let Some(array) = try_from_expression_inner(between.input, col_sub)? else {
                 return Ok(None);
             };
-            let Some(lower) = try_from_bound_expression(between.lower)? else {
+            let Some(lower) = try_from_expression_inner(between.lower, col_sub)? else {
                 return Ok(None);
             };
-            let Some(upper) = try_from_bound_expression(between.upper)? else {
+            let Some(upper) = try_from_expression_inner(between.upper, col_sub)? else {
                 return Ok(None);
             };
             Between.new_expr(
@@ -98,7 +116,7 @@ pub fn try_from_bound_expression(
             | DUCKDB_VX_EXPR_TYPE::DUCKDB_VX_EXPR_TYPE_OPERATOR_IS_NOT_NULL => {
                 let children: Vec<_> = operator.children().collect();
                 assert_eq!(children.len(), 1);
-                let Some(child) = try_from_bound_expression(children[0])? else {
+                let Some(child) = try_from_expression_inner(children[0], col_sub)? else {
                     return Ok(None);
                 };
                 match operator.op {
@@ -114,15 +132,15 @@ pub fn try_from_bound_expression(
                 // First child is element, rest form the list.
                 let children: Vec<_> = operator.children().collect();
                 assert!(children.len() >= 2);
-                let Some(element) = try_from_bound_expression(children[0])? else {
+                let Some(element) = try_from_expression_inner(children[0], col_sub)? else {
                     return Ok(None);
                 };
 
                 let Some(list_elements) = children
                     .iter()
                     .skip(1)
                     .map(|c| {
-                        let Some(value) = try_from_bound_expression(c)? else {
+                        let Some(value) = try_from_expression_inner(c, col_sub)? else {
                             return Ok(None);
                         };
                         Ok(Some(
@@ -154,7 +172,7 @@ pub fn try_from_bound_expression(
             DUCKDB_FUNCTION_NAME_CONTAINS => {
                 let children: Vec<_> = func.children().collect();
                 assert_eq!(children.len(), 2);
-                let Some(value) = try_from_bound_expression(children[0])? else {
+                let Some(value) = try_from_expression_inner(children[0], col_sub)? else {
                     return Ok(None);
                 };
                 let Some(pattern_lit) = like_pattern_str(children[1])? else {
@@ -171,7 +189,7 @@ pub fn try_from_bound_expression(
         duckdb::ExpressionClass::BoundConjunction(conj) => {
             let Some(children) = conj
                 .children()
-                .map(try_from_bound_expression)
+                .map(|c| try_from_expression_inner(c, col_sub))
                 .collect::<VortexResult<Option<Vec<_>>>>()?
             else {
                 return Ok(None);

vortex-duckdb/src/convert/table_filter.rs

@@ -117,8 +117,10 @@ pub fn try_from_table_filter(
             )
         }
         TableFilterClass::ExpressionRef(expr) => {
-            // TODO(ngates): figure out which column ID DuckDB is using for the expression.
-            vortex_bail!("expression table filter is not supported: {}", expr);
+            match super::expr::try_from_bound_expression_with_col_sub(expr, col)? {
+                Some(expression) => expression,
+                None => vortex_bail!("No output expression"),
+            }
         }
         TableFilterClass::Bloom => {
             vortex_bail!("bloom filter table filter is not supported")

vortex-duckdb/src/duckdb/expr.rs

@@ -139,6 +139,9 @@ impl ExpressionRef {
                         bind_info: out.bind_info,
                     })
                 }
+                cpp::DUCKDB_VX_EXPR_CLASS::DUCKDB_VX_EXPR_CLASS_BOUND_REF => {
+                    ExpressionClass::BoundRef
+                }
                 _ => {
                     return None;
                 }
@@ -155,6 +158,8 @@ pub enum ExpressionClass<'a> {
     BoundBetween(BoundBetween<'a>),
     BoundOperator(BoundOperator<'a>),
     BoundFunction(BoundFunction<'a>),
+    /// Column inside ExpressionFilter for expression pushed down to Vortex.
+    BoundRef,
 }
 
 pub struct BoundColumnRef {