[CALCITE-6242] Enhance lambda closure

core/src/main/java/org/apache/calcite/plan/RelOptUtil.java

@@ -70,6 +70,7 @@
 import org.apache.calcite.rex.RexExecutorImpl;
 import org.apache.calcite.rex.RexFieldAccess;
 import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexLambda;
 import org.apache.calcite.rex.RexLiteral;
 import org.apache.calcite.rex.RexLocalRef;
 import org.apache.calcite.rex.RexNode;
@@ -3363,6 +3364,12 @@ private static RexShuttle pushShuttle(final Project project) {
       @Override public RexNode visitInputRef(RexInputRef ref) {
         return project.getProjects().get(ref.getIndex());
       }
+
+      @Override public RexNode visitLambda(RexLambda lambda) {
+        // Lambda body references are at a different scope level.
+        // Do not remap indices inside lambda body against this project.
+        return lambda;
+      }
     };
   }
 
@@ -3386,6 +3393,12 @@ private static RexShuttle pushShuttle(final Calc calc) {
       @Override public RexNode visitInputRef(RexInputRef ref) {
         return projects.get(ref.getIndex());
       }
+
+      @Override public RexNode visitLambda(RexLambda lambda) {
+        // Lambda body references are at a different scope level.
+        // Do not remap indices inside lambda body against this calc.
+        return lambda;
+      }
     };
   }
 

core/src/main/java/org/apache/calcite/rex/RexBiVisitorImpl.java

@@ -119,8 +119,18 @@ protected RexBiVisitorImpl(boolean deep) {
     return null;
   }
 
+  /**
+   * Visits a lambda expression. When {@code deep} is true, recurses into
+   * the lambda body so that analysis visitors (e.g. InputFinder) can discover
+   * field references inside the lambda. When {@code deep} is false, returns
+   * null without recursing — this is the shallow traversal mode used by
+   * visitors that only need top-level information.
+   */
   @Override public R visitLambda(RexLambda lambda, P arg) {
-    return null;
+    if (!deep) {
+      return null;
+    }
+    return lambda.getExpression().accept(this, arg);
   }
 
   @Override public R visitNodeAndFieldIndex(RexNodeAndFieldIndex nodeAndFieldIndex, P arg) {

core/src/main/java/org/apache/calcite/rex/RexProgramBuilder.java

@@ -910,8 +910,9 @@ private abstract class RegisterShuttle extends RexShuttle {
     }
 
     @Override public RexNode visitLambda(RexLambda lambda) {
-      super.visitLambda(lambda);
-      return registerInternal(lambda);
+      // Lambda body references are at a different scope level.
+      // Do not validate or register lambda body indices against this program's input.
+      return lambda;
     }
   }
 

core/src/main/java/org/apache/calcite/rex/RexVisitorImpl.java

@@ -118,8 +118,17 @@ protected RexVisitorImpl(boolean deep) {
     return null;
   }
 
+  /**
+   * Visits a lambda expression. When {@code deep} is true, recurses into
+   * the lambda body to analyze its sub-expressions (critical for InputFinder
+   * to detect field references inside lambda bodies during pushDownJoinConditions).
+   * When {@code deep} is false, returns null without recursing.
+   */
   @Override public R visitLambda(RexLambda lambda) {
-    return null;
+    if (!deep) {
+      return null;
+    }
+    return lambda.getExpression().accept(this);
   }
 
   @Override public R visitLambdaRef(RexLambdaRef lambdaRef) {

core/src/main/java/org/apache/calcite/runtime/CalciteResource.java

@@ -244,6 +244,9 @@ ExInst<SqlValidatorException> columnNotFoundInTableDidYouMean(String a0,
   ExInst<SqlValidatorException> paramNotFoundInFunctionDidYouMean(String a0,
       String a1, String a2);
 
+  @BaseMessage("Lambda closure is not allowed in this conformance: reference to ''{0}'' from enclosing scope")
+  ExInst<SqlValidatorException> lambdaClosureNotAllowed(String identifier);
+
   @BaseMessage("Param ''{0}'' not found in lambda expression ''{1}''")
   ExInst<SqlValidatorException> paramNotFoundInLambdaExpression(String a0, String a1);
 

core/src/main/java/org/apache/calcite/sql/validate/SqlAbstractConformance.java

@@ -145,6 +145,10 @@ public abstract class SqlAbstractConformance implements SqlConformance {
     return SqlConformanceEnum.DEFAULT.allowQualifyingCommonColumn();
   }
 
+  @Override public boolean allowLambdaClosure() {
+    return SqlConformanceEnum.DEFAULT.allowLambdaClosure();
+  }
+
   @Override public boolean allowAliasUnnestItems() {
     return SqlConformanceEnum.DEFAULT.allowAliasUnnestItems();
   }

core/src/main/java/org/apache/calcite/sql/validate/SqlConformance.java

@@ -598,6 +598,28 @@ enum SelectAliasLookup {
    */
   boolean allowQualifyingCommonColumn();
 
+  /**
+   * Whether to allow lambda expressions to access variables from enclosing
+   * scopes (closure semantics).
+   *
+   * <p>For example, in a higher-order function context like:
+   *
+   * <blockquote><pre>
+   * SELECT *
+   * FROM t1
+   * JOIN t2 ON EXISTS(t1.arr, x -&gt; x = t2.v)</pre></blockquote>
+   *
+   * <p>The {@code t2.v} from the enclosing scope would be accessible inside
+   * the lambda body if closures are allowed.
+   *
+   * <p>Among the built-in conformance levels, false in
+   * {@link SqlConformanceEnum#STRICT_92},
+   * {@link SqlConformanceEnum#STRICT_99},
+   * {@link SqlConformanceEnum#STRICT_2003};
+   * true otherwise.
+   */
+  boolean allowLambdaClosure();
+
   /**
    * Whether {@code VALUE} is allowed as an alternative to {@code VALUES} in
    * the parser.

core/src/main/java/org/apache/calcite/sql/validate/SqlConformanceEnum.java

@@ -429,6 +429,17 @@ public enum SqlConformanceEnum implements SqlConformance {
     }
   }
 
+  @Override public boolean allowLambdaClosure() {
+    switch (this) {
+    case STRICT_92:
+    case STRICT_99:
+    case STRICT_2003:
+      return false;
+    default:
+      return true;
+    }
+  }
+
   @Override public boolean allowAliasUnnestItems() {
     switch (this) {
     case PRESTO:

core/src/main/java/org/apache/calcite/sql/validate/SqlDelegatingConformance.java

@@ -154,6 +154,10 @@ protected SqlDelegatingConformance(SqlConformance delegate) {
     return delegate.allowQualifyingCommonColumn();
   }
 
+  @Override public boolean allowLambdaClosure() {
+    return delegate.allowLambdaClosure();
+  }
+
   @Override public boolean isValueAllowed() {
     return delegate.isValueAllowed();
   }

core/src/main/java/org/apache/calcite/sql/validate/SqlLambdaScope.java

@@ -67,10 +67,12 @@ public boolean isParameter(SqlIdentifier id) {
         .anyMatch(param -> param.equalsDeep(identifier, Litmus.IGNORE));
     if (found) {
       return SqlQualified.create(this, 1, null, identifier);
-    } else {
+    }
+    if (!validator.config().conformance().allowLambdaClosure()) {
       throw validator.newValidationError(identifier,
-          RESOURCE.paramNotFoundInLambdaExpression(identifier.toString(), lambdaExpr.toString()));
+          RESOURCE.lambdaClosureNotAllowed(identifier.toString()));
     }
+    return parent.fullyQualify(identifier);
   }
 
   @Override public @Nullable RelDataType resolveColumn(String columnName, SqlNode ctx) {

core/src/main/java/org/apache/calcite/sql2rel/SqlToRelConverter.java

@@ -5563,11 +5563,16 @@ void setRoot(List<RelNode> inputs) {
         SqlQualified qualified) {
       if (nameToNodeMap != null && qualified.prefixLength == 1) {
         RexNode node = nameToNodeMap.get(qualified.identifier.names.get(0));
-        if (node == null) {
+        if (node != null) {
+          return Pair.of(node, null);
+        }
+        // If the identifier is not found in nameToNodeMap and the current scope
+        // is a lambda scope, fall through to standard scope resolution to allow
+        // external references (e.g., t2.v in a JOIN ON lambda expression).
+        if (!(scope instanceof SqlLambdaScope)) {
           throw new AssertionError("Unknown identifier '" + qualified.identifier
               + "' encountered while expanding expression");
         }
-        return Pair.of(node, null);
       }
       final SqlNameMatcher nameMatcher =
           scope.getValidator().getCatalogReader().nameMatcher();
@@ -5586,6 +5591,13 @@ void setRoot(List<RelNode> inputs) {
       // preserved.
       final SqlValidatorScope ancestorScope = resolve.scope;
       boolean isParent = ancestorScope != scope;
+      // When in a lambda scope, external references to tables that are part
+      // of the current blackboard's inputs should be resolved locally, not
+      // as correlation variables. The lambda blackboard inherits inputs from
+      // its parent blackboard.
+      if (isParent && scope instanceof SqlLambdaScope && inputs != null) {
+        isParent = false;
+      }
       if ((inputs != null) && !isParent) {
         final LookupContext rels =
             new LookupContext(this, inputs, systemFieldList.size());

core/src/main/resources/org/apache/calcite/runtime/CalciteResource.properties

@@ -387,4 +387,5 @@ CannotInferReturnType=Cannot infer return type for {0}; operand types: {1}
 SelectByCannotWithGroupBy=SELECT BY cannot be used with GROUP BY
 SelectByCannotWithOrderBy=SELECT BY cannot be used with ORDER BY
 DescriptorMustBeIdentifier=The argument of DESCRIPTOR must be an identifier
+LambdaClosureNotAllowed=Lambda closure is not allowed in this conformance: reference to ''{0}'' from enclosing scope
 # End CalciteResource.properties

core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java

@@ -8115,7 +8115,10 @@ void testGroupExpressionEquivalenceParams() {
 
   /** Test case for
    * <a href="https://issues.apache.org/jira/browse/CALCITE-3679">[CALCITE-3679]
-   * Allow lambda expressions in SQL queries</a>. */
+   * Allow lambda expressions in SQL queries</a>.
+   * <a href="https://issues.apache.org/jira/browse/CALCITE-6242">[CALCITE-6242]
+   * Enhance lambda closure parsing</a>.
+   * */
   @Test void testHigherOrderFunction() {
     final SqlValidatorFixture s = fixture()
         .withOperatorTable(MockSqlOperatorTable.standard().extend());
@@ -8129,6 +8132,10 @@ void testGroupExpressionEquivalenceParams() {
         .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL");
     s.withSql("select HIGHER_ORDER_FUNCTION2(1, () -> 0.1)")
         .type("RecordType(INTEGER NOT NULL EXPR$0) NOT NULL");
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp")
+        .ok();
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp")
+        .ok();
 
     // test for type check
     s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> ^x + 1^)")
@@ -8146,13 +8153,70 @@ void testGroupExpressionEquivalenceParams() {
         .fails("Cannot apply '(?s).*HIGHER_ORDER_FUNCTION' to arguments of type "
             + "'HIGHER_ORDER_FUNCTION\\(<INTEGER>, <FUNCTION\\(ANY, ANY, ANY\\) -> ANY>\\)'.*");
 
-    // test for illegal parameters
-    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp")
-        .fails("Param 'EMP\\.DEPTNO' not found in lambda expression "
-            + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `EMP`\\.`DEPTNO`'");
+  }
+
+  /** Test case for lambda closure conformance checking.
+   * Tests that lambda expressions can or cannot access variables from enclosing
+   * scopes based on the SQL conformance level.
+   * <a href="https://issues.apache.org/jira/browse/CALCITE-6242">[CALCITE-6242]
+   * Enhance lambda closure parsing</a>.
+   * */
+  @Test void testLambdaClosureConformance() {
+    final SqlValidatorFixture s = fixture()
+        .withOperatorTable(MockSqlOperatorTable.standard().extend());
+
+    // Lambda accessing outer scope variable (closure)
+    // In DEFAULT conformance, closure is allowed
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + deptno) from emp")
+        .withConformance(SqlConformanceEnum.DEFAULT)
+        .ok();
+
+    // In STRICT_92, closure is NOT allowed
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp")
+        .withConformance(SqlConformanceEnum.STRICT_92)
+        .fails("Lambda closure is not allowed in this conformance: "
+            + "reference to 'DEPTNO' from enclosing scope");
+
+    // In STRICT_99, closure is NOT allowed
     s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp")
-        .fails("Param 'DEPTNO' not found in lambda expression "
-            + "'\\(`X`, `Y`\\) -> `X` \\+ 1 \\+ `DEPTNO`'");
+        .withConformance(SqlConformanceEnum.STRICT_99)
+        .fails("Lambda closure is not allowed in this conformance: "
+            + "reference to 'DEPTNO' from enclosing scope");
+
+    // In STRICT_2003, closure is NOT allowed
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^deptno^) from emp")
+        .withConformance(SqlConformanceEnum.STRICT_2003)
+        .fails("Lambda closure is not allowed in this conformance: "
+            + "reference to 'DEPTNO' from enclosing scope");
+
+    // In BABEL conformance, closure is allowed
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + deptno) from emp")
+        .withConformance(SqlConformanceEnum.BABEL)
+        .ok();
+
+    // In LENIENT conformance, closure is allowed
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + deptno) from emp")
+        .withConformance(SqlConformanceEnum.LENIENT)
+        .ok();
+
+    // Lambda using only its own parameters (no closure) - should always work
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1) from emp")
+        .withConformance(SqlConformanceEnum.STRICT_92)
+        .ok();
+
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> y) from emp")
+        .withConformance(SqlConformanceEnum.STRICT_92)
+        .ok();
+
+    // Test with qualified column name in closure
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + emp.deptno) from emp")
+        .withConformance(SqlConformanceEnum.DEFAULT)
+        .ok();
+
+    s.withSql("select HIGHER_ORDER_FUNCTION(1, (x, y) -> x + 1 + ^emp.deptno^) from emp")
+        .withConformance(SqlConformanceEnum.STRICT_92)
+        .fails("Lambda closure is not allowed in this conformance: "
+            + "reference to 'EMP.DEPTNO' from enclosing scope");
   }
 
   /** Test case for <a href="https://issues.apache.org/jira/browse/CALCITE-7193">[CALCITE-7193]

core/src/test/resources/sql/lambda.iq

@@ -102,3 +102,16 @@ select "EXISTS"(array[array[1, 2], array[3, 4]], x -> x[1] = 1);
 (1 row)
 
 !ok
+
+# [CALCITE-6242] Enhance lambda closure parsing
+select *
+ from (select array(1, 2, 3) as arr) as t1 inner join
+     (select 1 as v) as t2 on "EXISTS"(arr, x -> x = t2.v);
++-----------+---+
+| ARR       | V |
++-----------+---+
+| [1, 2, 3] | 1 |
++-----------+---+
+(1 row)
+
+!ok