Skip to content

[CALCITE-7529] RexExecutor constant reduction loses sub-millisecond precision for TIME/TIMESTAMP literals #4942

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zzwqqq wants to merge 1 commit into
base: main
Choose a base branch
from
+279 −3
Open

[CALCITE-7529] RexExecutor constant reduction loses sub-millisecond precision for TIME/TIMESTAMP literals #4942

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 142 additions & 0 deletions core/src/main/java/org/apache/calcite/rex/RexBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,17 +29,21 @@
import org.apache.calcite.rel.type.RelDataTypeSystemImpl;
import org.apache.calcite.runtime.FlatLists;
import org.apache.calcite.runtime.SqlFunctions;
import org.apache.calcite.sql.SqlAbstractDateTimeLiteral;
import org.apache.calcite.sql.SqlAggFunction;
import org.apache.calcite.sql.SqlCollation;
import org.apache.calcite.sql.SqlIntervalQualifier;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlSpecialOperator;
import org.apache.calcite.sql.SqlTimeLiteral;
import org.apache.calcite.sql.SqlTimestampLiteral;
import org.apache.calcite.sql.SqlUtil;
import org.apache.calcite.sql.fun.SqlCountAggFunction;
import org.apache.calcite.sql.fun.SqlLibraryOperators;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.parser.SqlParserPos;
import org.apache.calcite.sql.parser.SqlParserUtil;
import org.apache.calcite.sql.type.ArraySqlType;
import org.apache.calcite.sql.type.IntervalSqlType;
import org.apache.calcite.sql.type.MapSqlType;
Expand Down Expand Up @@ -805,6 +809,12 @@ public RexNode makeCast(
&& SqlTypeUtil.isExactNumeric(type)) {
return makeCastBooleanToExact(type, exp);
}
final RexNode literalCast =
makeCastForTemporalLiteral(
pos, type, literal, matchNullability, safe, format);
if (literalCast != null) {
return literalCast;
}
if (canRemoveCastFromLiteral(type, value, typeName)) {
switch (typeName) {
case INTERVAL_YEAR:
Expand Down Expand Up @@ -877,6 +887,138 @@ public RexNode makeCast(
return makeAbstractCast(pos, type, exp, safe, format);
}

private @Nullable RexNode makeCastForTemporalLiteral(
SqlParserPos pos,
RelDataType type,
RexLiteral literal,
boolean matchNullability,
boolean safe,
RexLiteral format) {
if (!format.isNull()) {
return null;
}
if (SqlTypeUtil.isCharacter(literal.getType())) {
return makeCastFromCharacterLiteralToTemporal(
pos, type, literal, matchNullability, safe, format);
}
if (SqlTypeUtil.isCharacter(type)) {
final @Nullable String value = formatTemporalLiteral(literal);
return value == null || !fitsInCharacterType(type, value)
? null
: makeLiteral(value, type, true);
}
return null;
}

private @Nullable RexNode makeCastFromCharacterLiteralToTemporal(
SqlParserPos pos,
RelDataType type,
RexLiteral literal,
boolean matchNullability,
boolean safe,
RexLiteral format) {
final NlsString nlsString = literal.getValueAs(NlsString.class);
if (nlsString == null) {
return null;
}
final String value = nlsString.getValue().trim();
final RexNode temporalLiteral;
try {
switch (type.getSqlTypeName()) {
case TIME:
final SqlTimeLiteral timeLiteral =
SqlParserUtil.parseTimeLiteral(value, pos);
if (!isSubMillisecondLiteral(timeLiteral, value)) {
return null;
}
final TimeString time =
requireNonNull(timeLiteral.getValueAs(TimeString.class),
"timeLiteral.getValueAs(TimeString.class)")
.round(precision(type));
if (!hasSubMillisecondPrecision(time)) {
return null;
}
temporalLiteral = makeTimeLiteral(time, precision(type));
break;
case TIMESTAMP:
final SqlTimestampLiteral timestampLiteral =
SqlParserUtil.parseTimestampLiteral(value, pos);
if (!isSubMillisecondLiteral(timestampLiteral, value)) {
return null;
}
final TimestampString timestamp =
requireNonNull(timestampLiteral.getValueAs(TimestampString.class),
"timestampLiteral.getValueAs(TimestampString.class)")
.round(precision(type));
if (!hasSubMillisecondPrecision(timestamp)) {
return null;
}
temporalLiteral = makeTimestampLiteral(timestamp, precision(type));
break;
default:
return null;
}
} catch (RuntimeException e) {
return safe ? makeNullLiteral(type) : null;
}
if (type.isNullable()
&& !temporalLiteral.getType().isNullable()
&& matchNullability) {
return makeAbstractCast(pos, type, temporalLiteral, safe, format);
}
return temporalLiteral;
}

private static @Nullable String formatTemporalLiteral(RexLiteral literal) {
switch (literal.getType().getSqlTypeName()) {
case TIME:
final TimeString time = literal.getValueAs(TimeString.class);
return time == null || !hasSubMillisecondPrecision(time)
? null
: time.toString(precision(literal.getType()));
case TIMESTAMP:
final TimestampString timestamp = literal.getValueAs(TimestampString.class);
return timestamp == null || !hasSubMillisecondPrecision(timestamp)
? null
: timestamp.toString(precision(literal.getType()));
default:
return null;
}
}

private static boolean hasSubMillisecondPrecision(TimeString time) {
return hasFractionPrecisionBeyond(time.toString(), 3);
}

private static boolean hasSubMillisecondPrecision(TimestampString timestamp) {
return hasFractionPrecisionBeyond(timestamp.toString(), 3);
}

private static boolean hasFractionPrecisionBeyond(String value, int precision) {
final int dot = value.indexOf('.');
return dot >= 0 && value.length() - dot - 1 > precision;
}

private static boolean isSubMillisecondLiteral(
SqlAbstractDateTimeLiteral literal,
String value) {
return literal.getPrec() > 3 && literal.toFormattedString().equals(value);
}

private static int precision(RelDataType type) {
return type.getPrecision() < 0 ? 0 : type.getPrecision();
}

private static boolean fitsInCharacterType(RelDataType type, String value) {
switch (type.getSqlTypeName()) {
case CHAR:
case VARCHAR:
return SqlTypeUtil.comparePrecision(type.getPrecision(), value.length()) >= 0;
default:
return false;
}
}

/** Returns the lowest granularity unit for the given unit.
* YEAR and MONTH intervals are stored as months;
* HOUR, MINUTE, SECOND intervals are stored as milliseconds. */
Expand Down
25 changes: 22 additions & 3 deletions core/src/main/java/org/apache/calcite/rex/RexExecutorImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@

import java.lang.reflect.Modifier;
import java.lang.reflect.Type;
import java.util.ArrayList;
import java.util.List;

/**
Expand Down Expand Up @@ -136,14 +137,32 @@ public static RexExecutable getExecutable(RexBuilder rexBuilder, List<RexNode> e
@Override public void reduce(RexBuilder rexBuilder, List<RexNode> constExps,
List<RexNode> reducedValues) {
assert reducedValues.isEmpty();
final List<RexNode> exps = new ArrayList<>();
Copy link
Copy Markdown
Contributor

@mihaibudiu mihaibudiu May 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what happens here? you are only reducing expressions which are already literals?
This seems to be some kind of regression - other constant expressions are not reduced?

Copy link
Copy Markdown
Contributor Author

@zzwqqq zzwqqq May 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm copying literals through unchanged, and compiling/reducing only non-literal constant expressions.

final List<Integer> ordinals = new ArrayList<>();
for (int i = 0; i < constExps.size(); i++) {
final RexNode constExp = constExps.get(i);
if (!(constExp instanceof RexLiteral)) {
ordinals.add(i);
exps.add(constExp);
}
}
if (exps.isEmpty()) {
reducedValues.addAll(constExps);
return;
}
try {
String code = compile(rexBuilder, constExps, (list, index, storageType) -> {
String code = compile(rexBuilder, exps, (list, index, storageType) -> {
throw new UnsupportedOperationException();
});

final RexExecutable executable = new RexExecutable(code, constExps);
final RexExecutable executable = new RexExecutable(code, exps);
executable.setDataContext(dataContext);
executable.reduce(rexBuilder, constExps, reducedValues);
final List<RexNode> values = new ArrayList<>(exps.size());
executable.reduce(rexBuilder, exps, values);
reducedValues.addAll(constExps);
for (int i = 0; i < ordinals.size(); i++) {
reducedValues.set(ordinals.get(i), values.get(i));
}
} catch (RuntimeException ex) {
// Something went wrong during constant reduction (for example,
// we may have attempted a division by zero).
Expand Down
115 changes: 115 additions & 0 deletions core/src/test/java/org/apache/calcite/rex/RexExecutorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,14 @@
* limitations under the License.
*/
package org.apache.calcite.rex;

import org.apache.calcite.DataContext;
import org.apache.calcite.DataContexts;
import org.apache.calcite.avatica.util.ByteString;
import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rel.type.RelDataTypeSystemImpl;
import org.apache.calcite.sql.SqlBinaryOperator;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
Expand All @@ -35,6 +38,7 @@
import org.apache.calcite.util.DateString;
import org.apache.calcite.util.NlsString;
import org.apache.calcite.util.TestUtil;
import org.apache.calcite.util.TimeString;
import org.apache.calcite.util.TimestampString;
import org.apache.calcite.util.Util;

Expand Down Expand Up @@ -160,6 +164,117 @@ protected void check(final Action action) {
});
}

@Test void testReduceTimeCastWithMicros() {
checkHighPrecision((rexBuilder, executor) -> {
final RexNode cast =
rexBuilder.makeCast(
rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TIME, 6),
rexBuilder.makeLiteral("12:34:56.123456"));

final RexNode reduced = reduce(rexBuilder, executor, cast).get(0);

assertThat(reduced, instanceOf(RexLiteral.class));
assertThat(
((RexLiteral) reduced).getValueAs(TimeString.class).toString(6),
equalTo("12:34:56.123456"));
});
}

@Test void testReduceTimestampCastWithMicros() {
checkHighPrecision((rexBuilder, executor) -> {
final RexNode cast =
rexBuilder.makeCast(
rexBuilder.getTypeFactory().createSqlType(SqlTypeName.TIMESTAMP, 6),
rexBuilder.makeLiteral("2026-05-13 12:34:56.123456"));

final RexNode reduced = reduce(rexBuilder, executor, cast).get(0);

assertThat(reduced, instanceOf(RexLiteral.class));
assertThat(
((RexLiteral) reduced).getValueAs(TimestampString.class).toString(6),
equalTo("2026-05-13 12:34:56.123456"));
});
}

@Test void testReduceLiteralWithMicros() {
checkHighPrecision((rexBuilder, executor) -> {
final RexLiteral timeLiteral =
rexBuilder.makeTimeLiteral(new TimeString("12:34:56.123456"), 6);
final RexLiteral timestampLiteral =
rexBuilder.makeTimestampLiteral(
new TimestampString("2026-05-13 12:34:56.123456"), 6);
final RexNode expression =
rexBuilder.makeCall(SqlStdOperatorTable.PLUS,
rexBuilder.makeExactLiteral(BigDecimal.TEN),
rexBuilder.makeExactLiteral(BigDecimal.ONE));

final List<RexNode> reducedValues =
reduce(rexBuilder, executor, timeLiteral, expression, timestampLiteral);

assertThat(reducedValues, hasSize(3));
assertThat(
((RexLiteral) reducedValues.get(0)).getValueAs(TimeString.class).toString(6),
equalTo("12:34:56.123456"));
assertThat(((RexLiteral) reducedValues.get(1)).getValue2(), equalTo(11L));
assertThat(
((RexLiteral) reducedValues.get(2)).getValueAs(TimestampString.class)
.toString(6),
equalTo("2026-05-13 12:34:56.123456"));
});
}

@Test void testReduceTimeToVarcharWithMicros() {
checkHighPrecision((rexBuilder, executor) -> {
final RelDataTypeFactory typeFactory = rexBuilder.getTypeFactory();
final RexNode castToTime =
rexBuilder.makeCast(typeFactory.createSqlType(SqlTypeName.TIME, 6),
rexBuilder.makeLiteral("12:34:56.123456"));
final RexNode castToVarchar =
rexBuilder.makeCast(typeFactory.createSqlType(SqlTypeName.VARCHAR, 30),
castToTime);

final RexNode reduced = reduce(rexBuilder, executor, castToVarchar).get(0);

assertThat(reduced, instanceOf(RexLiteral.class));
assertThat(((RexLiteral) reduced).getValueAs(String.class),
equalTo("12:34:56.123456"));
});
}

private static void checkHighPrecision(Action action) {
action.check(new RexBuilder(highPrecisionTemporalTypeFactory()), executor());
}

private static List<RexNode> reduce(RexBuilder rexBuilder,
RexExecutorImpl executor, RexNode... nodes) {
final List<RexNode> reducedValues = new ArrayList<>();
executor.reduce(rexBuilder, ImmutableList.copyOf(nodes), reducedValues);
return reducedValues;
}

private static RelDataTypeFactory highPrecisionTemporalTypeFactory() {
return new JavaTypeFactoryImpl(
new RelDataTypeSystemImpl() {
@Override public int getMaxPrecision(SqlTypeName typeName) {
switch (typeName) {
case TIME:
case TIMESTAMP:
return 6;
default:
return super.getMaxPrecision(typeName);
}
}
});
}

private static RexExecutorImpl executor() {
return new RexExecutorImpl(
DataContexts.of(
ImmutableMap.of(
DataContext.Variable.TIME_ZONE.camelName, TimeZone.getTimeZone("GMT"),
DataContext.Variable.LOCALE.camelName, Locale.US)));
}

private void checkConstant(final Object operand,
final Function<RexBuilder, RexNode> function) {
check((rexBuilder, executor) -> {
Expand Down
Loading