HIVE-29516 Fix NPE in StatsUtils.updateStats when column statistics a…

ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java

@@ -1619,7 +1619,8 @@ protected Statistics getMaterializedTableStats(Operator<?> sinkOp) {
     }
 
     final List<ColStatistics> colStatsList = tableStats.getColumnStats();
-    if (!mapping.keySet().equals(colStatsList.stream().map(ColStatistics::getColumnName).collect(Collectors.toSet()))) {
+    if (colStatsList == null ||
+        !mapping.keySet().equals(colStatsList.stream().map(ColStatistics::getColumnName).collect(Collectors.toSet()))) {
       LOG.warn("The column statistics are inconsistent with the expected column names. Actual = {}, Expected = {}",
           colStatsList, parentColumnNames);
       tableStats.setColumnStatsState(Statistics.State.NONE);

ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java

@@ -1977,8 +1977,9 @@ private void removeSemijoinOptimizationByBenefit(OptimizeTezProcContext procCtx)
           LOG.debug("Old stats for {}: {}", roi.filterOperator, roi.filterStats);
           LOG.debug("Number of rows reduction: {}/{}", newNumRows, roi.filterStats.getNumRows());
         }
+        boolean useColStats = roi.filterStats.getColumnStats() != null;
         StatsUtils.updateStats(roi.filterStats, newNumRows,
-            true, roi.filterOperator, roi.colNames);
+            useColStats, roi.filterOperator, roi.colNames);
         if (LOG.isDebugEnabled()) {
           LOG.debug("New stats for {}: {}", roi.filterOperator, roi.filterStats);
         }

ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java

@@ -1481,8 +1481,12 @@ public static List<ColStatistics> getColStatisticsFromExprMap(HiveConf conf,
    */
   public static List<ColStatistics> getColStatisticsUpdatingTableAlias(Statistics parentStats) {
     List<ColStatistics> cs = Lists.newArrayList();
+    List<ColStatistics> parentColStats = parentStats.getColumnStats();
+    if (parentColStats == null) {
+      return cs;
+    }
 
-    for (ColStatistics parentColStat : parentStats.getColumnStats()) {
+    for (ColStatistics parentColStat : parentColStats) {
       ColStatistics colStat;
       colStat = parentColStat.clone();
       if (colStat != null) {
@@ -2030,6 +2034,10 @@ public static void updateStats(Statistics stats, long newNumRows,
 
     if (useColStats) {
       List<ColStatistics> colStats = stats.getColumnStats();
+      if (colStats == null || colStats.isEmpty()) {
+        throw new IllegalArgumentException("useColStats is true but column statistics are unavailable for " +
+            op.toString() + ". Caller should pass useColStats=false when column stats are missing.");
+      }
       for (ColStatistics cs : colStats) {
         long oldDV = cs.getCountDistint();
         if (affectedColumns.contains(cs.getColumnName())) {

ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java

@@ -22,6 +22,7 @@
 import static org.junit.jupiter.api.Assertions.assertNotEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
 
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
@@ -499,4 +500,57 @@ void testScaleColStatisticsPreservesUnknownNumFalses() {
     assertEquals(-1, colStats.get(0).getNumFalses(), "Unknown numFalses (-1) should be preserved after scaling");
   }
 
+  @Test
+  void testUpdateStatsThrowsWhenUseColStatsButNoColumnStats() {
+    Statistics stats = new Statistics(1000, 8000, 0, 0);
+
+    assertThrows(IllegalArgumentException.class, () -> {
+      StatsUtils.updateStats(stats, 500, true, null);
+    }, "Should throw IllegalArgumentException when useColStats=true but column stats are null");
+  }
+
+  @Test
+  void testUpdateStatsThrowsWhenUseColStatsButEmptyColumnStats() {
+    Statistics stats = new Statistics(1000, 8000, 0, 0);
+    stats.setColumnStats(Collections.emptyList());
+
+    assertThrows(IllegalArgumentException.class, () -> {
+      StatsUtils.updateStats(stats, 500, true, null);
+    }, "Should throw IllegalArgumentException when useColStats=true but column stats are empty");
+  }
+
+  @Test
+  void testUpdateStatsWithUseColStatsFalse() {
+    Statistics stats = new Statistics(1000, 8000, 0, 0);
+    long originalDataSize = stats.getDataSize();
+
+    StatsUtils.updateStats(stats, 500, false, null);
+
+    assertEquals(500, stats.getNumRows(), "Row count should be updated");
+    assertNotEquals(originalDataSize, stats.getDataSize(), "Data size should be recalculated via ratio");
+  }
+
+  @Test
+  void testGetColStatisticsUpdatingTableAliasWithNullColumnStats() {
+    Statistics stats = new Statistics(1000, 8000, 0, 0);
+
+    List<ColStatistics> result = StatsUtils.getColStatisticsUpdatingTableAlias(stats);
+
+    assertNotNull(result, "Result should not be null");
+    assertEquals(0, result.size(), "Result should be empty when column stats are null");
+  }
+
+  @Test
+  void testGetColStatisticsUpdatingTableAliasWithColumnStats() {
+    Statistics stats = new Statistics(1000, 8000, 0, 0);
+    ColStatistics cs = createColStats("col1", 100, 50);
+    stats.setColumnStats(Collections.singletonList(cs));
+
+    List<ColStatistics> result = StatsUtils.getColStatisticsUpdatingTableAlias(stats);
+
+    assertNotNull(result, "Result should not be null");
+    assertEquals(1, result.size(), "Result should contain one column stat");
+    assertEquals("col1", result.get(0).getColumnName(), "Column name should match");
+  }
+
 }

ql/src/test/queries/clientpositive/semijoin_stats_missing_colstats.q

@@ -0,0 +1,24 @@
+-- HIVE-29516: Test that semijoin optimization handles missing column statistics gracefully
+-- This test verifies that queries don't fail with NPE when basic table statistics exist
+-- but column-level statistics are unavailable during semijoin removal optimization.
+
+set hive.tez.dynamic.semijoin.reduction=true;
+set hive.tez.dynamic.semijoin.reduction.threshold=0.1;
+set hive.auto.convert.join=true;
+set hive.auto.convert.join.noconditionaltask=true;
+set hive.auto.convert.join.noconditionaltask.size=10000000000;
+
+create table t1_semijoin_nocolstats (id int, val string);
+create table t2_semijoin_nocolstats (id int, val string);
+
+-- Set only basic table statistics (numRows, rawDataSize) WITHOUT column statistics
+-- This simulates the scenario where column stats are unavailable (e.g., large TPC-DS datasets)
+alter table t1_semijoin_nocolstats update statistics set('numRows'='100000000', 'rawDataSize'='2000000000');
+alter table t2_semijoin_nocolstats update statistics set('numRows'='1000', 'rawDataSize'='20000');
+
+-- This join query should trigger semijoin optimization evaluation
+-- Previously this would cause NPE in StatsUtils.updateStats when column stats were null
+-- With the fix, it compiles successfully by skipping semijoin optimization when column stats are unavailable
+explain
+select t1.id, t1.val, t2.val
+from t1_semijoin_nocolstats t1 join t2_semijoin_nocolstats t2 on t1.id = t2.id;

ql/src/test/results/clientpositive/llap/semijoin_stats_missing_colstats.q.out

@@ -0,0 +1,123 @@
+PREHOOK: query: create table t1_semijoin_nocolstats (id int, val string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1_semijoin_nocolstats
+POSTHOOK: query: create table t1_semijoin_nocolstats (id int, val string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1_semijoin_nocolstats
+PREHOOK: query: create table t2_semijoin_nocolstats (id int, val string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t2_semijoin_nocolstats
+POSTHOOK: query: create table t2_semijoin_nocolstats (id int, val string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t2_semijoin_nocolstats
+PREHOOK: query: alter table t1_semijoin_nocolstats update statistics set('numRows'='100000000', 'rawDataSize'='2000000000')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t1_semijoin_nocolstats
+PREHOOK: Output: default@t1_semijoin_nocolstats
+POSTHOOK: query: alter table t1_semijoin_nocolstats update statistics set('numRows'='100000000', 'rawDataSize'='2000000000')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t1_semijoin_nocolstats
+POSTHOOK: Output: default@t1_semijoin_nocolstats
+PREHOOK: query: alter table t2_semijoin_nocolstats update statistics set('numRows'='1000', 'rawDataSize'='20000')
+PREHOOK: type: ALTERTABLE_UPDATETABLESTATS
+PREHOOK: Input: default@t2_semijoin_nocolstats
+PREHOOK: Output: default@t2_semijoin_nocolstats
+POSTHOOK: query: alter table t2_semijoin_nocolstats update statistics set('numRows'='1000', 'rawDataSize'='20000')
+POSTHOOK: type: ALTERTABLE_UPDATETABLESTATS
+POSTHOOK: Input: default@t2_semijoin_nocolstats
+POSTHOOK: Output: default@t2_semijoin_nocolstats
+PREHOOK: query: explain
+select t1.id, t1.val, t2.val
+from t1_semijoin_nocolstats t1 join t2_semijoin_nocolstats t2 on t1.id = t2.id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1_semijoin_nocolstats
+PREHOOK: Input: default@t2_semijoin_nocolstats
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select t1.id, t1.val, t2.val
+from t1_semijoin_nocolstats t1 join t2_semijoin_nocolstats t2 on t1.id = t2.id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1_semijoin_nocolstats
+POSTHOOK: Input: default@t2_semijoin_nocolstats
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+  Stage-1 is a root stage
+  Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+  Stage: Stage-1
+    Tez
+#### A masked pattern was here ####
+      Edges:
+        Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+      Vertices:
+        Map 1
+            Map Operator Tree:
+                TableScan
+                  alias: t1
+                  filterExpr: id is not null (type: boolean)
+                  probeDecodeDetails: cacheKey:HASH_MAP_MAPJOIN_25_container, bigKeyColName:id, smallTablePos:1, keyRatio:1.04500002
+                  Statistics: Num rows: 100000000 Data size: 17860000188 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: id is not null (type: boolean)
+                    Statistics: Num rows: 95000000 Data size: 16967000178 Basic stats: COMPLETE Column stats: NONE
+                    Select Operator
+                      expressions: id (type: int), val (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 95000000 Data size: 16967000178 Basic stats: COMPLETE Column stats: NONE
+                      Map Join Operator
+                        condition map:
+                             Inner Join 0 to 1
+                        keys:
+                          0 _col0 (type: int)
+                          1 _col0 (type: int)
+                        outputColumnNames: _col0, _col1, _col3
+                        input vertices:
+                          1 Map 2
+                        Statistics: Num rows: 104500002 Data size: 18663700600 Basic stats: COMPLETE Column stats: NONE
+                        Select Operator
+                          expressions: _col0 (type: int), _col1 (type: string), _col3 (type: string)
+                          outputColumnNames: _col0, _col1, _col2
+                          Statistics: Num rows: 104500002 Data size: 18663700600 Basic stats: COMPLETE Column stats: NONE
+                          File Output Operator
+                            compressed: false
+                            Statistics: Num rows: 104500002 Data size: 18663700600 Basic stats: COMPLETE Column stats: NONE
+                            table:
+                                input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+                                output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+                                serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+            Execution mode: vectorized, llap
+            LLAP IO: all inputs
+        Map 2
+            Map Operator Tree:
+                TableScan
+                  alias: t2
+                  filterExpr: id is not null (type: boolean)
+                  Statistics: Num rows: 1000 Data size: 178788 Basic stats: COMPLETE Column stats: NONE
+                  Filter Operator
+                    predicate: id is not null (type: boolean)
+                    Statistics: Num rows: 950 Data size: 169848 Basic stats: COMPLETE Column stats: NONE
+                    Select Operator
+                      expressions: id (type: int), val (type: string)
+                      outputColumnNames: _col0, _col1
+                      Statistics: Num rows: 950 Data size: 169848 Basic stats: COMPLETE Column stats: NONE
+                      Reduce Output Operator
+                        key expressions: _col0 (type: int)
+                        null sort order: z
+                        sort order: +
+                        Map-reduce partition columns: _col0 (type: int)
+                        Statistics: Num rows: 950 Data size: 169848 Basic stats: COMPLETE Column stats: NONE
+                        value expressions: _col1 (type: string)
+            Execution mode: vectorized, llap
+            LLAP IO: all inputs
+
+  Stage: Stage-0
+    Fetch Operator
+      limit: -1
+      Processor Tree:
+        ListSink