feat: wrap higher-order, lambda, and lambda-variable Expr variants

DataFusion 54 introduces Expr::HigherOrderFunction, Expr::Lambda, and
Expr::LambdaVariable. PyExpr::to_variant previously errored on each
with py_unsupported_variant_err. Add PyHigherOrderFunction, PyLambda,
and PyLambdaVariable wrappers, register them in the expr pymodule and
re-export from python/datafusion/expr.py, and dispatch to_variant to
the new wrappers.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: timsaucer

crates/core/src/expr.rs

@@ -85,9 +85,12 @@ pub mod explain;
 pub mod extension;
 pub mod filter;
 pub mod grouping_set;
+pub mod higher_order_function;
 pub mod in_list;
 pub mod in_subquery;
 pub mod join;
+pub mod lambda;
+pub mod lambda_variable;
 pub mod like;
 pub mod limit;
 pub mod literal;
@@ -220,15 +223,14 @@ impl PyExpr {
             Expr::SetComparison(value) => {
                 Ok(set_comparison::PySetComparison::from(value.clone()).into_bound_py_any(py)?)
             }
-            Expr::HigherOrderFunction(value) => Err(py_unsupported_variant_err(format!(
-                "Converting Expr::HigherOrderFunction to a Python object is not implemented: {value:?}"
-            ))),
-            Expr::Lambda(value) => Err(py_unsupported_variant_err(format!(
-                "Converting Expr::Lambda to a Python object is not implemented: {value:?}"
-            ))),
-            Expr::LambdaVariable(value) => Err(py_unsupported_variant_err(format!(
-                "Converting Expr::LambdaVariable to a Python object is not implemented: {value:?}"
-            ))),
+            Expr::HigherOrderFunction(value) => Ok(
+                higher_order_function::PyHigherOrderFunction::from(value.clone())
+                    .into_bound_py_any(py)?,
+            ),
+            Expr::Lambda(value) => Ok(lambda::PyLambda::from(value.clone()).into_bound_py_any(py)?),
+            Expr::LambdaVariable(value) => {
+                Ok(lambda_variable::PyLambdaVariable::from(value.clone()).into_bound_py_any(py)?)
+            }
         })
     }
 
@@ -860,6 +862,9 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<union::PyUnion>()?;
     m.add_class::<unnest::PyUnnest>()?;
     m.add_class::<unnest_expr::PyUnnestExpr>()?;
+    m.add_class::<higher_order_function::PyHigherOrderFunction>()?;
+    m.add_class::<lambda::PyLambda>()?;
+    m.add_class::<lambda_variable::PyLambdaVariable>()?;
     m.add_class::<extension::PyExtension>()?;
     m.add_class::<filter::PyFilter>()?;
     m.add_class::<projection::PyProjection>()?;

crates/core/src/expr/higher_order_function.rs

@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::{self, Display, Formatter};
+
+use datafusion::logical_expr::expr::HigherOrderFunction;
+use pyo3::prelude::*;
+
+use super::PyExpr;
+
+#[pyclass(
+    from_py_object,
+    frozen,
+    name = "HigherOrderFunction",
+    module = "datafusion.expr",
+    subclass
+)]
+#[derive(Clone)]
+pub struct PyHigherOrderFunction {
+    higher_order: HigherOrderFunction,
+}
+
+impl From<HigherOrderFunction> for PyHigherOrderFunction {
+    fn from(higher_order: HigherOrderFunction) -> PyHigherOrderFunction {
+        PyHigherOrderFunction { higher_order }
+    }
+}
+
+impl From<PyHigherOrderFunction> for HigherOrderFunction {
+    fn from(higher_order: PyHigherOrderFunction) -> Self {
+        higher_order.higher_order
+    }
+}
+
+impl Display for PyHigherOrderFunction {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(
+            f,
+            "HigherOrderFunction(name={}, args={:?})",
+            self.higher_order.name(),
+            &self.higher_order.args,
+        )
+    }
+}
+
+#[pymethods]
+impl PyHigherOrderFunction {
+    /// Name of the higher-order function being invoked.
+    fn name(&self) -> String {
+        self.higher_order.name().to_string()
+    }
+
+    /// Arguments passed to the higher-order function. Some entries may be
+    /// `Lambda` expressions; others are ordinary value expressions.
+    fn args(&self) -> Vec<PyExpr> {
+        self.higher_order
+            .args
+            .iter()
+            .map(|e| PyExpr::from(e.clone()))
+            .collect()
+    }
+
+    fn __repr__(&self) -> PyResult<String> {
+        Ok(format!("HigherOrderFunction({self})"))
+    }
+
+    fn __name__(&self) -> PyResult<String> {
+        Ok("HigherOrderFunction".to_string())
+    }
+}

crates/core/src/expr/lambda.rs

@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::{self, Display, Formatter};
+
+use datafusion::logical_expr::expr::Lambda;
+use pyo3::prelude::*;
+
+use super::PyExpr;
+
+#[pyclass(
+    from_py_object,
+    frozen,
+    name = "Lambda",
+    module = "datafusion.expr",
+    subclass
+)]
+#[derive(Clone)]
+pub struct PyLambda {
+    lambda: Lambda,
+}
+
+impl From<Lambda> for PyLambda {
+    fn from(lambda: Lambda) -> PyLambda {
+        PyLambda { lambda }
+    }
+}
+
+impl From<PyLambda> for Lambda {
+    fn from(lambda: PyLambda) -> Self {
+        lambda.lambda
+    }
+}
+
+impl Display for PyLambda {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(
+            f,
+            "Lambda(params={:?}, body={:?})",
+            &self.lambda.params, &self.lambda.body,
+        )
+    }
+}
+
+#[pymethods]
+impl PyLambda {
+    /// Parameter names of the lambda.
+    fn params(&self) -> Vec<String> {
+        self.lambda.params.clone()
+    }
+
+    /// Body expression of the lambda.
+    fn body(&self) -> PyExpr {
+        (*self.lambda.body).clone().into()
+    }
+
+    fn __repr__(&self) -> PyResult<String> {
+        Ok(format!("Lambda({self})"))
+    }
+
+    fn __name__(&self) -> PyResult<String> {
+        Ok("Lambda".to_string())
+    }
+}

crates/core/src/expr/lambda_variable.rs

@@ -0,0 +1,67 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::{self, Display, Formatter};
+
+use datafusion::logical_expr::expr::LambdaVariable;
+use pyo3::prelude::*;
+
+#[pyclass(
+    from_py_object,
+    frozen,
+    name = "LambdaVariable",
+    module = "datafusion.expr",
+    subclass
+)]
+#[derive(Clone)]
+pub struct PyLambdaVariable {
+    variable: LambdaVariable,
+}
+
+impl From<LambdaVariable> for PyLambdaVariable {
+    fn from(variable: LambdaVariable) -> PyLambdaVariable {
+        PyLambdaVariable { variable }
+    }
+}
+
+impl From<PyLambdaVariable> for LambdaVariable {
+    fn from(variable: PyLambdaVariable) -> Self {
+        variable.variable
+    }
+}
+
+impl Display for PyLambdaVariable {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "LambdaVariable({})", &self.variable.name)
+    }
+}
+
+#[pymethods]
+impl PyLambdaVariable {
+    /// Reference name of the lambda parameter.
+    fn name(&self) -> String {
+        self.variable.name.clone()
+    }
+
+    fn __repr__(&self) -> PyResult<String> {
+        Ok(format!("LambdaVariable({self})"))
+    }
+
+    fn __name__(&self) -> PyResult<String> {
+        Ok("LambdaVariable".to_string())
+    }
+}

python/datafusion/expr.py

@@ -149,6 +149,9 @@
 TransactionStart = expr_internal.TransactionStart
 TryCast = expr_internal.TryCast
 Union = expr_internal.Union
+HigherOrderFunction = expr_internal.HigherOrderFunction
+Lambda = expr_internal.Lambda
+LambdaVariable = expr_internal.LambdaVariable
 Unnest = expr_internal.Unnest
 UnnestExpr = expr_internal.UnnestExpr
 Values = expr_internal.Values
@@ -192,6 +195,7 @@
     "FileType",
     "Filter",
     "GroupingSet",
+    "HigherOrderFunction",
     "ILike",
     "InList",
     "InSubquery",
@@ -206,6 +210,8 @@
     "Join",
     "JoinConstraint",
     "JoinType",
+    "Lambda",
+    "LambdaVariable",
     "Like",
     "Limit",
     "Literal",