[fix](plan rewriter) remove rule EliminateNotNull

fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java

@@ -80,7 +80,6 @@
 import org.apache.doris.nereids.rules.rewrite.EliminateJoinByUnique;
 import org.apache.doris.nereids.rules.rewrite.EliminateJoinCondition;
 import org.apache.doris.nereids.rules.rewrite.EliminateLimit;
-import org.apache.doris.nereids.rules.rewrite.EliminateNotNull;
 import org.apache.doris.nereids.rules.rewrite.EliminateNullAwareLeftAntiJoin;
 import org.apache.doris.nereids.rules.rewrite.EliminateOrderByConstant;
 import org.apache.doris.nereids.rules.rewrite.EliminateOrderByKey;
@@ -340,9 +339,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
                                     topDown(
                                             new EliminateDedupJoinCondition()
                                     ),
-                                    // eliminate useless not null or inferred not null
                                     // TODO: wait InferPredicates to infer more not null.
-                                    bottomUp(new EliminateNotNull()),
                                     topDown(new ConvertInnerOrCrossJoin())
                             ),
                             topic("Set operation optimization",
@@ -577,9 +574,7 @@ public class Rewriter extends AbstractBatchJobExecutor {
                                 )
                         ),
 
-                        // eliminate useless not null or inferred not null
                         // TODO: wait InferPredicates to infer more not null.
-                        bottomUp(new EliminateNotNull()),
                         topDown(new ConvertInnerOrCrossJoin())
                 ),
                 topic("Set operation optimization",

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java

@@ -291,7 +291,6 @@ public enum RuleType {
     ELIMINATE_JOIN_CONDITION(RuleTypeClass.REWRITE),
     ELIMINATE_FILTER_ON_ONE_RELATION(RuleTypeClass.REWRITE),
     ELIMINATE_SEMI_JOIN(RuleTypeClass.REWRITE),
-    ELIMINATE_NOT_NULL(RuleTypeClass.REWRITE),
     ELIMINATE_UNNECESSARY_PROJECT(RuleTypeClass.REWRITE),
     RECORD_PLAN_FOR_MV_PRE_REWRITE(RuleTypeClass.REWRITE),
     ELIMINATE_OUTER_JOIN(RuleTypeClass.REWRITE),

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java

@@ -44,7 +44,6 @@
 import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
 import org.apache.doris.nereids.trees.expressions.Expression;
 import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Not;
 import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc;
@@ -866,14 +865,6 @@ private boolean containsNullRejectSlot(Set<Set<Slot>> requireNoNullableViewSlot,
             CascadesContext cascadesContext) {
         Set<Expression> queryPulledUpPredicates = queryPredicates.stream()
                 .flatMap(expr -> ExpressionUtils.extractConjunction(expr).stream())
-                .map(expr -> {
-                    // NOTICE inferNotNull generate Not with isGeneratedIsNotNull = false,
-                    //  so, we need set this flag to false before comparison.
-                    if (expr instanceof Not) {
-                        return ((Not) expr).withGeneratedIsNotNull(false);
-                    }
-                    return expr;
-                })
                 .collect(Collectors.toSet());
         Set<Expression> queryNullRejectPredicates =
                 ExpressionUtils.inferNotNull(queryPulledUpPredicates, cascadesContext);

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangeInference.java

@@ -164,7 +164,7 @@ public ValueDesc visitNot(Not not, ExpressionRewriteContext context) {
         if (childValue instanceof DiscreteValue) {
             return new NotDiscreteValue(context, childValue.getReference(), ((DiscreteValue) childValue).values);
         } else if (childValue instanceof IsNullValue) {
-            return new IsNotNullValue(context, childValue.getReference(), not);
+            return new IsNotNullValue(context, childValue.getReference());
         } else {
             return new UnknownValue(context, not);
         }
@@ -190,8 +190,7 @@ private ValueDesc processCompound(ExpressionRewriteContext context, List<Express
         boolean hasIsNullExpression = false;
         boolean hasNotIsNullExpression = false;
         Predicate<Expression> isNotNull = expression -> expression instanceof Not
-                && expression.child(0) instanceof IsNull
-                && !((Not) expression).isGeneratedIsNotNull();
+                && expression.child(0) instanceof IsNull;
         for (Expression predicate : predicates) {
             hasNullExpression = hasNullExpression || predicate.isNullLiteral();
             hasIsNullExpression = hasIsNullExpression || predicate instanceof IsNull;
@@ -278,14 +277,7 @@ private ValueDesc intersect(ExpressionRewriteContext context, Expression referen
                 // = (TA is not null or null) and (TA is not null)
                 // = TA is not null
                 // = IsNotNull(TA)
-                if (rangeValue.isRangeAll() && collector.isNotNullValueOpt.isPresent()) {
-                    // Notice that if collector has only isGenerateNotNullValueOpt, we should not keep the rangeAll here
-                    // for expression:  (Not(IsNull(TA)) OR NULL) AND GeneratedNot(IsNull(TA))
-                    // will be converted to RangeAll(TA) AND IsNotNullValue(TA, generated=true)
-                    // if we skip this RangeAll, the final result will be IsNotNullValue(TA, generated=true)
-                    // then convert back to expression: GeneratedNot(IsNull(TA)),
-                    // but later EliminateNotNull rule will remove this generated Not expression,
-                    // then the final result will be TRUE, which is wrong.
+                if (rangeValue.isRangeAll() && collector.hasIsNotNullValue) {
                     continue;
                 }
                 if (mergeRangeValueDesc == null) {
@@ -366,20 +358,19 @@ private ValueDesc intersect(ExpressionRewriteContext context, Expression referen
             resultValues.add(new EmptyValue(context, reference));
         }
         if (collector.hasIsNullValue) {
-            if (collector.hasIsNotNullValue()) {
+            if (collector.hasIsNotNullValue) {
                 return new UnknownValue(context, BooleanLiteral.FALSE);
             }
             // nullable's EmptyValue have contains IsNull, no need to add
             if (!collector.hasEmptyValue) {
                 resultValues.add(new IsNullValue(context, reference));
             }
         }
-        if (collector.hasIsNotNullValue()) {
+        if (collector.hasIsNotNullValue) {
             if (collector.hasEmptyValue) {
                 return new UnknownValue(context, BooleanLiteral.FALSE);
             }
-            collector.isNotNullValueOpt.ifPresent(resultValues::add);
-            collector.isGenerateNotNullValueOpt.ifPresent(resultValues::add);
+            resultValues.add(new IsNotNullValue(context, reference));
         }
         Optional<ValueDesc> shortCutResult = mergeCompoundValues(context, reference, resultValues, collector, true);
         if (shortCutResult.isPresent()) {
@@ -397,7 +388,7 @@ private ValueDesc intersect(ExpressionRewriteContext context, Expression referen
     }
 
     private ValueDesc union(ExpressionRewriteContext context, Expression reference, ValueDescCollector collector) {
-        if (collector.hasIsNotNullValue()) {
+        if (collector.hasIsNotNullValue) {
             if (!collector.rangeValues.isEmpty()
                     || !collector.discreteValues.isEmpty()
                     || !collector.notDiscreteValues.isEmpty()) {
@@ -471,21 +462,20 @@ private ValueDesc union(ExpressionRewriteContext context, Expression reference,
         }
 
         if (collector.hasIsNullValue) {
-            if (collector.hasIsNotNullValue() || hasRangeAll) {
+            if (collector.hasIsNotNullValue || hasRangeAll) {
                 return new UnknownValue(context, BooleanLiteral.TRUE);
             }
             resultValues.add(new IsNullValue(context, reference));
         }
-        if (collector.hasIsNotNullValue()) {
+        if (collector.hasIsNotNullValue) {
             if (collector.hasEmptyValue) {
                 // EmptyValue(TA) or TA is not null
                 // = TA is null and null or TA is not null
                 // = TA is not null or null
                 // = RangeAll(TA)
                 resultValues.add(new RangeValue(context, reference, Range.all()));
             } else {
-                collector.isNotNullValueOpt.ifPresent(resultValues::add);
-                collector.isGenerateNotNullValueOpt.ifPresent(resultValues::add);
+                resultValues.add(new IsNotNullValue(context, reference));
             }
         }
 
@@ -615,11 +605,8 @@ public interface ValueDescVisitor<R, C> {
     }
 
     private static class ValueDescCollector implements ValueDescVisitor<Void, Void> {
-        // generated not is null != not is null
-        Optional<IsNotNullValue> isNotNullValueOpt = Optional.empty();
-        Optional<IsNotNullValue> isGenerateNotNullValueOpt = Optional.empty();
-
         boolean hasIsNullValue = false;
+        boolean hasIsNotNullValue = false;
         boolean hasEmptyValue = false;
         List<RangeValue> rangeValues = Lists.newArrayList();
         List<DiscreteValue> discreteValues = Lists.newArrayList();
@@ -635,10 +622,6 @@ int size() {
             return rangeValues.size() + discreteValues.size() + compoundValues.size() + unknownValues.size();
         }
 
-        boolean hasIsNotNullValue() {
-            return isNotNullValueOpt.isPresent() || isGenerateNotNullValueOpt.isPresent();
-        }
-
         @Override
         public Void visitEmptyValue(EmptyValue emptyValue, Void context) {
             hasEmptyValue = true;
@@ -671,11 +654,7 @@ public Void visitIsNullValue(IsNullValue isNullValue, Void context) {
 
         @Override
         public Void visitIsNotNullValue(IsNotNullValue isNotNullValue, Void context) {
-            if (isNotNullValue.not.isGeneratedIsNotNull()) {
-                isGenerateNotNullValueOpt = Optional.of(isNotNullValue);
-            } else {
-                isNotNullValueOpt = Optional.of(isNotNullValue);
-            }
+            hasIsNotNullValue = true;
             return null;
         }
 
@@ -1214,15 +1193,8 @@ protected UnionType getUnionType(ValueDesc other, int depth) {
      * a is not null
      */
     public static class IsNotNullValue extends ValueDesc {
-        final Not not;
-
-        public IsNotNullValue(ExpressionRewriteContext context, Expression reference, Not not) {
+        public IsNotNullValue(ExpressionRewriteContext context, Expression reference) {
             super(context, reference);
-            this.not = not;
-        }
-
-        public Not getNotExpression() {
-            return this.not;
         }
 
         @Override
@@ -1238,7 +1210,7 @@ protected boolean nullable() {
         @Override
         protected boolean containsAll(ValueDesc other, int depth) {
             if (other instanceof IsNotNullValue) {
-                return not.isGeneratedIsNotNull() == ((IsNotNullValue) other).not.isGeneratedIsNotNull();
+                return true;
             } else if (other instanceof CompoundValue) {
                 return ((CompoundValue) other).isContainedAllBy(this, depth);
             } else {

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/SimplifyRange.java

@@ -177,7 +177,7 @@ public Expression visitIsNullValue(IsNullValue value, Void context) {
 
     @Override
     public Expression visitIsNotNullValue(IsNotNullValue value, Void context) {
-        return value.getNotExpression();
+        return new Not(new IsNull(value.getReference()));
     }
 
     @Override

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ConstantPropagation.java

@@ -481,11 +481,6 @@ private boolean needReplaceWithConstant(Expression expression, Map<Slot, Literal
     }
 
     private boolean canReplaceExpression(Expression expression) {
-        // 'a is not null', EliminateOuterJoin will call TypeUtils.isNotNull
-        if (ExpressionUtils.isGeneratedNotNull(expression)) {
-            return false;
-        }
-
         // "https://doris.apache.org/docs/sql-manual/basic-element/operators/conditional-operators
         // /full-text-search-operators", the match function require left is a slot, not a literal.
         if (expression instanceof Match) {

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateOuterJoin.java

@@ -75,7 +75,7 @@ public Rule build() {
             boolean conjunctsChanged = false;
             if (!notNullSlots.isEmpty()) {
                 for (Slot slot : notNullSlots) {
-                    Not isNotNull = new Not(new IsNull(slot), true);
+                    Not isNotNull = new Not(new IsNull(slot));
                     conjunctsChanged |= conjuncts.add(isNotNull);
                 }
             }
@@ -134,11 +134,11 @@ private JoinType tryEliminateOuterJoin(JoinType joinType, boolean canFilterLeftN
     private boolean createIsNotNullIfNecessary(EqualPredicate swapedEqualTo, Collection<Expression> container) {
         boolean containerChanged = false;
         if (swapedEqualTo.left().nullable()) {
-            Not not = new Not(new IsNull(swapedEqualTo.left()), true);
+            Not not = new Not(new IsNull(swapedEqualTo.left()));
             containerChanged |= container.add(not);
         }
         if (swapedEqualTo.right().nullable()) {
-            Not not = new Not(new IsNull(swapedEqualTo.right()), true);
+            Not not = new Not(new IsNull(swapedEqualTo.right()));
             containerChanged |= container.add(not);
         }
         return containerChanged;