Ji/duckdb virtual column pushdown

vortex-duckdb/build.rs

@@ -243,6 +243,7 @@ fn main() {
         .file("cpp/expr.cpp")
         .file("cpp/logical_type.cpp")
         .file("cpp/object_cache.cpp")
+        .file("cpp/logical_expr.cpp")
         .file("cpp/scalar_function.cpp")
         .file("cpp/table_filter.cpp")
         .file("cpp/table_function.cpp")

vortex-duckdb/cpp/CMakeLists.txt

@@ -34,6 +34,7 @@ add_library(vortex
         error.cpp
         expr.cpp
         object_cache.cpp
+        logical_expr.cpp
         scalar_function.cpp
         table_filter.cpp
         table_function.cpp

vortex-duckdb/cpp/expr.cpp

@@ -23,9 +23,102 @@ extern "C" const char *duckdb_vx_expr_to_string(duckdb_vx_expr ffi_expr) {
     return result;
 }
 
+// Get detailed debug string representation of expression
+extern "C" duckdb_vx_string duckdb_vx_expr_to_debug_string(duckdb_vx_expr ffi_expr) {
+    try {
+        if (!ffi_expr) {
+            return nullptr;
+        }
+
+        auto expr = reinterpret_cast<Expression *>(ffi_expr);
+
+        // Create detailed debug string with class, type, and content information
+        std::string debug_str = "Expression Debug Info:\n";
+        debug_str += "  Class: " + ExpressionClassToString(expr->GetExpressionClass()) + "\n";
+        debug_str += "  Type: " + ExpressionTypeToString(expr->GetExpressionType()) + "\n";
+        debug_str += "  Return Type: " + expr->return_type.ToString() + "\n";
+        debug_str += "  ToString(): " + expr->ToString() + "\n";
+
+        // Add specific information based on expression class
+        switch (expr->GetExpressionClass()) {
+        case ExpressionClass::BOUND_COLUMN_REF: {
+            auto &col_ref = expr->Cast<BoundColumnRefExpression>();
+            debug_str += "  Column Binding: table=" + std::to_string(col_ref.binding.table_index) +
+                         ", column=" + std::to_string(col_ref.binding.column_index) + "\n";
+            debug_str += "  Depth: " + std::to_string(col_ref.depth) + "\n";
+            break;
+        }
+        case ExpressionClass::BOUND_FUNCTION: {
+            auto &func_expr = expr->Cast<BoundFunctionExpression>();
+            debug_str += "  Function: " + func_expr.function.name + "\n";
+            debug_str += "  Arguments: " + std::to_string(func_expr.children.size()) + "\n";
+            for (size_t i = 0; i < func_expr.children.size(); i++) {
+                debug_str += "    [" + std::to_string(i) + "] " + func_expr.children[i]->ToString() + "\n";
+            }
+            break;
+        }
+        case ExpressionClass::BOUND_CONSTANT: {
+            auto &const_expr = expr->Cast<BoundConstantExpression>();
+            debug_str += "  Value: " + const_expr.value.ToString() + "\n";
+            break;
+        }
+        case ExpressionClass::BOUND_COMPARISON: {
+            auto &comp_expr = expr->Cast<BoundComparisonExpression>();
+            debug_str += "  Left: " + comp_expr.left->ToString() + "\n";
+            debug_str += "  Right: " + comp_expr.right->ToString() + "\n";
+            break;
+        }
+        case ExpressionClass::BOUND_CONJUNCTION: {
+            auto &conj_expr = expr->Cast<BoundConjunctionExpression>();
+            debug_str += "  Children: " + std::to_string(conj_expr.children.size()) + "\n";
+            for (size_t i = 0; i < conj_expr.children.size(); i++) {
+                debug_str += "    [" + std::to_string(i) + "] " + conj_expr.children[i]->ToString() + "\n";
+            }
+            break;
+        }
+        case ExpressionClass::BOUND_OPERATOR: {
+            auto &op_expr = expr->Cast<BoundOperatorExpression>();
+            debug_str += "  Children: " + std::to_string(op_expr.children.size()) + "\n";
+            for (size_t i = 0; i < op_expr.children.size(); i++) {
+                debug_str += "    [" + std::to_string(i) + "] " + op_expr.children[i]->ToString() + "\n";
+            }
+            break;
+        }
+        default:
+            debug_str += "  (No additional debug info for this expression class)\n";
+            break;
+        }
+
+        // Create string wrapper
+        return new std::string(debug_str);
+    } catch (...) {
+        return nullptr;
+    }
+}
+
+// Legacy alias for backwards compatibility with optimizer_rule.h
+extern "C" duckdb_vx_string duckdb_vx_expression_to_string(duckdb_vx_expr ffi_expr) {
+    try {
+        if (!ffi_expr) {
+            return nullptr;
+        }
+
+        auto expr = reinterpret_cast<Expression *>(ffi_expr);
+        std::string str = expr->ToString();
+
+        // Create string wrapper
+        return new std::string(str);
+    } catch (...) {
+        return nullptr;
+    }
+}
+
 //! Create a DuckDB vortex error.
 extern "C" void duckdb_vx_destroy_expr(duckdb_vx_expr *ffi_expr) {
-    auto expr = reinterpret_cast<Expression *>(ffi_expr);
+    if (ffi_expr == nullptr) {
+        return;
+    }
+    auto expr = reinterpret_cast<Expression *>(*ffi_expr);
     delete expr;
     memset(ffi_expr, 0, sizeof(duckdb_vx_expr));
 }
@@ -49,6 +142,14 @@ extern "C" const char *duckdb_vx_expr_get_bound_column_ref_get_name(duckdb_vx_ex
     return result;
 }
 
+extern "C" uint64_t duckdb_vx_expr_get_bound_column_ref_depth(duckdb_vx_expr ffi_expr) {
+    if (!ffi_expr) {
+        return 0;
+    }
+    auto &expr = reinterpret_cast<Expression *>(ffi_expr)->Cast<BoundColumnRefExpression>();
+    return expr.depth;
+}
+
 extern "C" duckdb_value duckdb_vx_expr_bound_constant_get_value(duckdb_vx_expr ffi_expr) {
     if (!ffi_expr) {
         return nullptr;

vortex-duckdb/cpp/include/duckdb_vx.h

@@ -11,6 +11,7 @@
 #include "duckdb_vx/error.h"
 #include "duckdb_vx/expr.h"
 #include "duckdb_vx/logical_type.h"
+#include "duckdb_vx/logical_operator.h"
 #include "duckdb_vx/object_cache.h"
 #include "duckdb_vx/scalar_function.h"
 #include "duckdb_vx/table_filter.h"

vortex-duckdb/cpp/include/duckdb_vx/expr.h

@@ -10,10 +10,17 @@ extern "C" {
 #endif
 
 typedef struct duckdb_vx_expr_ *duckdb_vx_expr;
+typedef void* duckdb_vx_string;
 
 /// Return the string representation of the expression. Must be freed with `duckdb_vx_free`.
 const char *duckdb_vx_expr_to_string(duckdb_vx_expr expr);
 
+/// Return a detailed debug string representation of the expression
+duckdb_vx_string duckdb_vx_expr_to_debug_string(duckdb_vx_expr expr);
+
+/// Legacy alias for backwards compatibility with optimizer_rule.h
+duckdb_vx_string duckdb_vx_expression_to_string(duckdb_vx_expr expr);
+
 void duckdb_vx_destroy_expr(duckdb_vx_expr *expr);
 
 // See ExpressionClass in duckdb/include/duckdb/common/enums/expression_class.hpp
@@ -211,6 +218,8 @@ duckdb_vx_expr_class duckdb_vx_expr_get_class(duckdb_vx_expr expr);
 
 const char *duckdb_vx_expr_get_bound_column_ref_get_name(duckdb_vx_expr expr);
 
+uint64_t duckdb_vx_expr_get_bound_column_ref_depth(duckdb_vx_expr expr);
+
 duckdb_value duckdb_vx_expr_bound_constant_get_value(duckdb_vx_expr expr);
 
 typedef struct {
@@ -256,6 +265,22 @@ typedef struct {
 
 void duckdb_vx_expr_get_bound_function(duckdb_vx_expr expr, duckdb_vx_expr_bound_function *out);
 
+// ==============================================
+// String Wrapper Functions
+// ==============================================
+
+// Create a string wrapper from std::string
+duckdb_vx_string duckdb_vx_create_string(const char* str);
+
+// Get length of wrapped string
+uint64_t duckdb_vx_string_length(duckdb_vx_string str);
+
+// Get C string data from wrapped string
+const char* duckdb_vx_string_data(duckdb_vx_string str);
+
+// Free wrapped string
+void duckdb_vx_string_free(duckdb_vx_string str);
+
 #ifdef __cplusplus /* End C ABI */
 }
 #endif

vortex-duckdb/cpp/include/duckdb_vx/logical_operator.h

@@ -0,0 +1,177 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+#pragma once
+
+#include "duckdb.h"
+#include "expr.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// ==============================================
+// Forward Declarations
+// ==============================================
+
+// Forward declarations for DuckDB types - opaque pointers
+typedef void* duckdb_vx_logical_operator;
+typedef void* duckdb_vx_string;
+
+// ==============================================
+// Type Definitions
+// ==============================================
+
+// Logical operator types enum (subset of DuckDB's LogicalOperatorType)
+typedef enum {
+    DUCKDB_VX_LOGICAL_GET = 0,
+    DUCKDB_VX_LOGICAL_PROJECTION = 1,
+    DUCKDB_VX_LOGICAL_FILTER = 2,
+    DUCKDB_VX_LOGICAL_JOIN = 3,
+    DUCKDB_VX_LOGICAL_AGGREGATE = 4,
+    DUCKDB_VX_LOGICAL_UNKNOWN = 999
+} DUCKDB_VX_LOGICAL_OPERATOR_TYPE;
+
+// Expression types enum (subset of DuckDB's ExpressionType) for logical plan compatibility
+typedef enum {
+    DUCKDB_VX_BOUND_COLUMN_REF = 0,
+    DUCKDB_VX_BOUND_FUNCTION = 1,
+    DUCKDB_VX_CONSTANT = 2,
+    DUCKDB_VX_EXPRESSION_UNKNOWN = 999
+} DUCKDB_VX_EXPRESSION_TYPE;
+
+// Column binding structure for logical plans
+typedef struct {
+    uint64_t table_index;
+    uint64_t column_index;
+} duckdb_vx_column_binding;
+
+// Rust callback function type for visiting operators
+typedef void (*duckdb_vx_rust_visitor_callback)(duckdb_vx_logical_operator op, void* user_data);
+
+// ==============================================
+// Basic Logical Operator Inspection
+// ==============================================
+
+// Get operator type
+DUCKDB_VX_LOGICAL_OPERATOR_TYPE duckdb_vx_get_operator_type(duckdb_vx_logical_operator op);
+
+// Get string representation of operator
+duckdb_vx_string duckdb_vx_logical_operator_to_string(duckdb_vx_logical_operator op);
+
+// Get operator children count
+uint64_t duckdb_vx_get_children_count(duckdb_vx_logical_operator op);
+
+// Get operator child by index
+duckdb_vx_logical_operator duckdb_vx_get_child(duckdb_vx_logical_operator op, uint64_t index);
+
+// Get operator expressions count
+uint64_t duckdb_vx_get_expressions_count(duckdb_vx_logical_operator op);
+
+// Get operator expression by index
+duckdb_vx_expr duckdb_vx_get_expression(duckdb_vx_logical_operator op, uint64_t index);
+
+// Set operator expression by index
+void duckdb_vx_set_expression(duckdb_vx_logical_operator op, uint64_t index, duckdb_vx_expr expr);
+
+// ==============================================
+// LogicalGet (Table Scan) Functions
+// ==============================================
+
+// Get table function name from LogicalGet
+char* duckdb_vx_get_function_name(duckdb_vx_logical_operator get_op);
+
+// Get column names count from LogicalGet
+uint64_t duckdb_vx_get_column_names_count(duckdb_vx_logical_operator get_op);
+
+// Get individual column name by index from LogicalGet
+char* duckdb_vx_get_column_name(duckdb_vx_logical_operator get_op, uint64_t index);
+
+// Get projection IDs count from LogicalGet
+uint64_t duckdb_vx_get_projection_ids_count(duckdb_vx_logical_operator get_op);
+
+// Get individual projection ID by index from LogicalGet
+uint64_t duckdb_vx_get_projection_id(duckdb_vx_logical_operator get_op, uint64_t index);
+
+// Update projection IDs in LogicalGet
+void duckdb_vx_update_projection_ids(duckdb_vx_logical_operator get_op, 
+                                     uint64_t* new_projection_ids,
+                                     uint64_t count);
+
+// Add column ID to LogicalGet
+void duckdb_vx_add_column_id(duckdb_vx_logical_operator get_op, uint64_t column_id);
+
+// Clear column IDs in LogicalGet
+void duckdb_vx_clear_column_ids(duckdb_vx_logical_operator get_op);
+
+// Get detailed string representation of LogicalGet operator
+duckdb_vx_string duckdb_vx_logical_get_to_string(duckdb_vx_logical_operator get_op);
+
+// ==============================================
+// LogicalProjection Functions
+// ==============================================
+
+// Get detailed string representation of LogicalProjection operator
+duckdb_vx_string duckdb_vx_logical_projection_to_string(duckdb_vx_logical_operator proj_op);
+
+// ==============================================
+// Expression Functions
+// ==============================================
+
+// Logical plan expression functions - using simplified type enum
+DUCKDB_VX_EXPRESSION_TYPE duckdb_vx_get_expression_type(duckdb_vx_expr expr);
+char* duckdb_vx_get_function_name_from_expr(duckdb_vx_expr expr);
+uint64_t duckdb_vx_get_function_arg_count(duckdb_vx_expr expr);
+duckdb_vx_expr duckdb_vx_get_function_arg(duckdb_vx_expr expr, uint64_t index);
+char* duckdb_vx_get_column_alias(duckdb_vx_expr expr);
+duckdb_vx_column_binding duckdb_vx_get_column_binding(duckdb_vx_expr expr);
+duckdb_vx_expr duckdb_vx_create_column_ref(const char* name, duckdb_vx_column_binding binding, uint64_t depth);
+void duckdb_vx_update_column_binding(duckdb_vx_expr expr, duckdb_vx_column_binding binding);
+
+// ==============================================
+// Visitor Pattern
+// ==============================================
+
+// Visit all operators in plan tree with Rust callback
+void duckdb_vx_visit_operators(duckdb_vx_logical_operator plan,
+                              duckdb_vx_rust_visitor_callback callback,
+                              void* user_data);
+
+// ==============================================
+// Optimizer Registration
+// ==============================================
+
+// Register a Rust-based optimizer function
+void duckdb_vx_register_rust_optimizer(duckdb_database db_handle,
+                                       duckdb_vx_rust_visitor_callback optimizer_func,
+                                       void* user_data);
+
+// ==============================================
+// String Wrapper Functions
+// ==============================================
+
+// Create a string wrapper from std::string
+duckdb_vx_string duckdb_vx_create_string(const char* str);
+
+// Get length of wrapped string
+uint64_t duckdb_vx_string_length(duckdb_vx_string str);
+
+// Get C string data from wrapped string
+const char* duckdb_vx_string_data(duckdb_vx_string str);
+
+// Free wrapped string
+void duckdb_vx_string_free(duckdb_vx_string str);
+
+// ==============================================
+// Memory Management
+// ==============================================
+
+// Memory management functions
+void duckdb_vx_free_string(char* str);
+
+// String utility functions for C strings
+uint64_t duckdb_vx_c_string_length(const char* str);
+
+#ifdef __cplusplus
+}
+#endif

vortex-duckdb/cpp/include/duckdb_vx/vector.h

@@ -21,6 +21,10 @@ void duckdb_vx_set_dictionary_vector_length(duckdb_vector dict, unsigned int len
 // Add the buffer to the string vector (basically, keep it alive as long as the vector).
 void duckdb_vx_string_vector_add_buffer(duckdb_vector ffi_vector, duckdb_vx_data buffer);
 
+// Get string data and size for computing virtual columns
+uint32_t duckdb_vx_string_vector_get_string_size(duckdb_vector ffi_vector, idx_t index);
+const char *duckdb_vx_string_vector_get_string_data(duckdb_vector ffi_vector, idx_t index);
+
 // Converts a duckdb flat vector into a Sequence vector.
 void duckdb_vx_sequence_vector(duckdb_vector c_vector, int64_t start, int64_t step, idx_t capacity);