HIVE-28170: Implement drop stats

parser/src/java/org/apache/hadoop/hive/ql/parse/AlterClauseParser.g

@@ -86,6 +86,7 @@ alterTableStatementSuffix
     | alterStatementSuffixReplaceBranch
     | alterStatementSuffixReplaceTag
     | alterStatementSuffixSetWriteOrder
+    | alterStatementSuffixDropStatsForColumns
     ;
 
 alterTblPartitionStatementSuffix[boolean partition]
@@ -285,6 +286,13 @@ alterStatementSuffixUpdateStats[boolean partition]
     ->              ^(TOK_ALTERTABLE_UPDATESTATS tableProperties)
     ;
 
+alterStatementSuffixDropStatsForColumns
+@init { gParent.pushMsg("drop statistics for columns", state); }
+@after { gParent.popMsg(state); }
+    : KW_DROP KW_STATISTICS KW_FOR KW_COLUMNS
+    -> ^(TOK_ALTERTABLE_DROPCOLSTATS)
+    ;
+
 alterStatementChangeColPosition
     : first=KW_FIRST|KW_AFTER afterCol=identifier
     ->{$first != null}? ^(TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION )
@@ -790,3 +798,4 @@ alterForeignKeyWithName
     : KW_CONSTRAINT constraintName=identifier KW_FOREIGN KW_KEY fkCols=columnParenthesesList  KW_REFERENCES tabName=tableName parCols=columnParenthesesList constraintOptsAlter?
     -> ^(TOK_FOREIGN_KEY ^(TOK_CONSTRAINT_NAME $constraintName) $fkCols $tabName $parCols constraintOptsAlter?)
     ;
+

parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g

@@ -199,6 +199,7 @@ TOK_ALTERTABLE_UPDATECOLSTATS;
 TOK_ALTERPARTITION_UPDATECOLSTATS;
 TOK_ALTERTABLE_UPDATESTATS;
 TOK_ALTERPARTITION_UPDATESTATS;
+TOK_ALTERTABLE_DROPCOLSTATS;
 TOK_TABLE_PARTITION;
 TOK_ALTERTABLE_FILEFORMAT;
 TOK_ALTERPARTITION_FILEFORMAT;
@@ -2938,4 +2939,4 @@ compactionStatus
   ;
 /*
 END SHOW COMPACTIONS statement
-*/
+*/

ql/src/java/org/apache/hadoop/hive/ql/ddl/table/AlterTableType.java

@@ -73,7 +73,8 @@ public enum AlterTableType {
   COMPACT("compact"),
   TRUNCATE("truncate"),
   MERGEFILES("merge files"),
-  UPDATESTATS("update stats"); // Note: used in ColumnStatsUpdateWork, not here.
+  UPDATESTATS("update stats"), // Note: used in ColumnStatsUpdateWork, not here.
+  DROP_COL_STATS("drop column stats");
 
   private final String name;
 

ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/columnstats/AlterTableDropColumnStatisticsAnalyzer.java

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.ddl.table.misc.columnstats;
+
+import java.util.Map;
+
+import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.ql.QueryState;
+import org.apache.hadoop.hive.ql.ddl.DDLSemanticAnalyzerFactory.DDLType;
+import org.apache.hadoop.hive.ql.ddl.table.AbstractAlterTableAnalyzer;
+import org.apache.hadoop.hive.ql.ddl.table.AlterTableType;
+import org.apache.hadoop.hive.ql.exec.ColumnStatsDropTask;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ColumnStatsDropWork;
+
+/**
+ * Analyzer for drop column statistics commands.
+ */
+@DDLType(types = HiveParser.TOK_ALTERTABLE_DROPCOLSTATS)
+public class AlterTableDropColumnStatisticsAnalyzer extends AbstractAlterTableAnalyzer {
+  public AlterTableDropColumnStatisticsAnalyzer(QueryState queryState) throws SemanticException {
+    super(queryState);
+  }
+
+  @Override
+  protected void analyzeCommand(TableName tableName, Map<String, String> partitionSpec, ASTNode command)
+      throws SemanticException {
+    Table table = getTable(tableName);
+
+    if (table.isNonNative() && table.getStorageHandler().canSetColStatistics(table)) {
+      throw new SemanticException("DROP STATISTICS FOR COLUMNS is not supported for non-native table that " +
+          "doesn't store stats in metastore.");
+    }
+
+    ColumnStatsDropWork work = new ColumnStatsDropWork(tableName);
+    ColumnStatsDropTask task = (ColumnStatsDropTask) TaskFactory.get(work);
+
+    addInputsOutputsAlterTable(tableName, partitionSpec, null, AlterTableType.DROP_COL_STATS, false);
+
+    rootTasks.add(task);
+  }
+}

ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/columnstats/AlterTableUpdateColumnStatistictAnalyzer.java → ql/src/java/org/apache/hadoop/hive/ql/ddl/table/misc/columnstats/AlterTableUpdateColumnStatisticsAnalyzer.java

@@ -39,8 +39,8 @@
  * Analyzer for update column statistics commands.
  */
 @DDLType(types = {HiveParser.TOK_ALTERTABLE_UPDATECOLSTATS, HiveParser.TOK_ALTERPARTITION_UPDATECOLSTATS})
-public class AlterTableUpdateColumnStatistictAnalyzer extends AbstractAlterTableAnalyzer {
-  public AlterTableUpdateColumnStatistictAnalyzer(QueryState queryState) throws SemanticException {
+public class AlterTableUpdateColumnStatisticsAnalyzer extends AbstractAlterTableAnalyzer {
+  public AlterTableUpdateColumnStatisticsAnalyzer(QueryState queryState) throws SemanticException {
     super(queryState);
   }
 

ql/src/java/org/apache/hadoop/hive/ql/exec/ColumnStatsDropTask.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import java.io.Serial;
+import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.plan.ColumnStatsDropWork;
+import org.apache.hadoop.hive.ql.plan.api.StageType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * ColumnStatsDropTask implementation. Example:
+ * ALTER TABLE src_stat DROP STATISTICS for columns;
+ **/
+
+public class ColumnStatsDropTask extends Task<ColumnStatsDropWork> {
+  @Serial
+  private static final long serialVersionUID = 1L;
+  private static final Logger LOG = LoggerFactory.getLogger(ColumnStatsDropTask.class);
+
+  @Override
+  public int execute() {
+    try {
+      getHive().deleteColumnStatistics(work.tableName());
+      return 0;
+    } catch (Exception e) {
+      setException(e);
+      LOG.info("Failed to drop column stats in metastore", e);
+      return ErrorMsg.GENERIC_ERROR.getErrorCode();
+    }
+  }
+
+  @Override
+  public StageType getType() {
+    return StageType.COLUMNSTATS;
+  }
+
+  @Override
+  public String getName() {
+    return "COLUMN STATS DROP TASK";
+  }
+}

ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java

@@ -54,6 +54,7 @@
 import org.apache.hadoop.hive.ql.io.merge.MergeFileTask;
 import org.apache.hadoop.hive.ql.io.merge.MergeFileWork;
 import org.apache.hadoop.hive.ql.plan.ColumnStatsUpdateWork;
+import org.apache.hadoop.hive.ql.plan.ColumnStatsDropWork;
 import org.apache.hadoop.hive.ql.plan.ConditionalWork;
 import org.apache.hadoop.hive.ql.plan.CopyWork;
 import org.apache.hadoop.hive.ql.plan.DependencyCollectionWork;
@@ -114,6 +115,7 @@ public TaskTuple(Class<T> workClass, Class<? extends Task<T>> taskClass) {
         MapredLocalTask.class));
     taskvec.add(new TaskTuple<StatsWork>(StatsWork.class, StatsTask.class));
     taskvec.add(new TaskTuple<ColumnStatsUpdateWork>(ColumnStatsUpdateWork.class, ColumnStatsUpdateTask.class));
+    taskvec.add(new TaskTuple<>(ColumnStatsDropWork.class, ColumnStatsDropTask.class));
     taskvec.add(new TaskTuple<MergeFileWork>(MergeFileWork.class,
         MergeFileTask.class));
     taskvec.add(new TaskTuple<DependencyCollectionWork>(DependencyCollectionWork.class,

ql/src/java/org/apache/hadoop/hive/ql/hooks/HiveProtoLoggingHook.java

@@ -55,6 +55,7 @@
 import static org.apache.hadoop.hive.ql.plan.HiveOperation.ALTERTABLE_UNARCHIVE;
 import static org.apache.hadoop.hive.ql.plan.HiveOperation.ALTERTABLE_UPDATEPARTSTATS;
 import static org.apache.hadoop.hive.ql.plan.HiveOperation.ALTERTABLE_UPDATETABLESTATS;
+import static org.apache.hadoop.hive.ql.plan.HiveOperation.ALTERTABLE_DROP_COL_STATS;
 import static org.apache.hadoop.hive.ql.plan.HiveOperation.ALTERTBLPART_SKEWED_LOCATION;
 import static org.apache.hadoop.hive.ql.plan.HiveOperation.ALTERVIEW_AS;
 import static org.apache.hadoop.hive.ql.plan.HiveOperation.ALTERVIEW_PROPERTIES;
@@ -145,7 +146,8 @@ public class HiveProtoLoggingHook implements ExecuteWithHookContext {
     includedOperationSet = Arrays.stream(new HiveOperation[] { LOAD, EXPORT, IMPORT,
         CREATEDATABASE, DROPDATABASE, DROPTABLE, MSCK, ALTERTABLE_ADDCOLS, ALTERTABLE_REPLACECOLS,
         ALTERTABLE_RENAMECOL, ALTERTABLE_RENAMEPART, ALTERTABLE_UPDATEPARTSTATS,
-        ALTERTABLE_UPDATETABLESTATS, ALTERTABLE_RENAME, ALTERTABLE_DROPPARTS, ALTERTABLE_ADDPARTS,
+        ALTERTABLE_UPDATETABLESTATS, ALTERTABLE_DROP_COL_STATS,
+        ALTERTABLE_RENAME, ALTERTABLE_DROPPARTS, ALTERTABLE_ADDPARTS,
         ALTERTABLE_TOUCH, ALTERTABLE_ARCHIVE, ALTERTABLE_UNARCHIVE, ALTERTABLE_PROPERTIES,
         ALTERTABLE_SERIALIZER, ALTERPARTITION_SERIALIZER, ALTERTABLE_SERDEPROPERTIES,
         ALTERPARTITION_SERDEPROPERTIES, ALTERTABLE_CLUSTER_SORT, ANALYZE_TABLE, CACHE_METADATA,

ql/src/java/org/apache/hadoop/hive/ql/hooks/WriteEntity.java

@@ -276,6 +276,7 @@ public static WriteType determineAlterTableWriteType(AlterTableType op, Table ta
       case SET_SERDE_PROPS:
       case ADDPROPS:
       case UPDATESTATS:
+      case DROP_COL_STATS:
         return WriteType.DDL_SHARED;
 
       case COMPACT:

ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

@@ -86,6 +86,7 @@
 import org.apache.hadoop.hive.metastore.api.DataConnector;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest;
+import org.apache.hadoop.hive.metastore.api.DeleteColumnStatisticsRequest;
 import org.apache.hadoop.hive.metastore.api.DropDatabaseRequest;
 import org.apache.hadoop.hive.metastore.api.DropPartitionsExpr;
 import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
@@ -6330,27 +6331,18 @@ public AggrStats getAggrColStatsFor(Table tbl,
       perfLogger.perfLogEnd(CLASS_NAME, PerfLogger.HIVE_GET_AGGR_COL_STATS, "HS2-cache");
     }
   }
-
-  public boolean deleteTableColumnStatistics(String dbName, String tableName, String colName)
-    throws HiveException {
+
+  public void deleteColumnStatistics(TableName tableName) throws HiveException {
+    DeleteColumnStatisticsRequest request = 
+        new DeleteColumnStatisticsRequest(tableName.getDb(), tableName.getTable());
+    request.setEngine(Constants.HIVE_ENGINE);
     try {
-      return getMSC().deleteTableColumnStatistics(dbName, tableName, colName, Constants.HIVE_ENGINE);
-    } catch(Exception e) {
-      LOG.debug("Failed deleteTableColumnStatistics", e);
-      throw new HiveException(e);
+      getMSC().deleteColumnStatistics(request);
+    } catch (Exception e) {
+      throw new HiveException(e, ErrorMsg.GENERIC_ERROR);
     }
   }
 
-  public boolean deletePartitionColumnStatistics(String dbName, String tableName, String partName,
-    String colName) throws HiveException {
-      try {
-        return getMSC().deletePartitionColumnStatistics(dbName, tableName, partName, colName, Constants.HIVE_ENGINE);
-      } catch(Exception e) {
-        LOG.debug("Failed deletePartitionColumnStatistics", e);
-        throw new HiveException(e);
-      }
-    }
-
   public void updateTransactionalStatistics(UpdateTransactionalStatsRequest req) throws HiveException {
     try {
       getMSC().updateTransactionalStatistics(req);

ql/src/java/org/apache/hadoop/hive/ql/plan/ColumnStatsDropWork.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.plan;
+
+import java.io.Serial;
+import java.io.Serializable;
+
+import org.apache.hadoop.hive.common.TableName;
+import org.apache.hadoop.hive.ql.plan.Explain.Level;
+
+
+/**
+ * ColumnStatsDropWork implementation. ColumnStatsDropWork will drop the
+ * colStats from the metastore. Work corresponds to statements like:
+ * ALTER TABLE src_stat DROP STATISTICS for columns;
+ */
+@Explain(displayName = "Column Stats Drop Work", explainLevels = {Level.USER, Level.DEFAULT, Level.EXTENDED})
+public record ColumnStatsDropWork(TableName tableName) implements Serializable {
+  @Serial
+  private static final long serialVersionUID = 1L;
+}

ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java

@@ -62,6 +62,8 @@ public enum HiveOperation {
   ALTERTABLE_UPDATETABLESTATS("ALTERTABLE_UPDATETABLESTATS",
       new int[] {HiveParser.TOK_ALTERTABLE_UPDATESTATS, HiveParser.TOK_ALTERTABLE_UPDATECOLSTATS},
       new Privilege[]{Privilege.ALTER_METADATA}, null),
+  ALTERTABLE_DROP_COL_STATS("ALTERTABLE_DROP_COL_STATS", 
+      HiveParser.TOK_ALTERTABLE_DROPCOLSTATS, new Privilege[]{Privilege.ALTER_METADATA}, null),
   ALTERTABLE_RENAME("ALTERTABLE_RENAME", HiveParser.TOK_ALTERTABLE_RENAME, new Privilege[]{Privilege.ALTER_METADATA},
       null),
   ALTERTABLE_DROPPARTS("ALTERTABLE_DROPPARTS",

ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java

@@ -80,6 +80,7 @@ public enum HiveOperationType {
   ALTERPARTITION_BUCKETNUM,
   ALTERTABLE_UPDATETABLESTATS,
   ALTERTABLE_UPDATEPARTSTATS,
+  ALTERTABLE_DROP_COL_STATS,
   SHOWCATALOGS,
   SHOWDATABASES,
   SHOWDATACONNECTORS,

ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java

@@ -180,6 +180,8 @@ public HivePrivilegeObjectType getObjectType() {
         PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR));
     op2Priv.put(HiveOperationType.ALTERTABLE_UPDATEPARTSTATS,
         PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR));
+    op2Priv.put(HiveOperationType.ALTERTABLE_DROP_COL_STATS,
+        PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR));
     op2Priv.put(HiveOperationType.ALTERTABLE_TOUCH,
         PrivRequirement.newIOPrivRequirement(OWNER_PRIV_AR, OWNER_PRIV_AR));
     op2Priv.put(HiveOperationType.ALTERTABLE_ARCHIVE,

ql/src/test/queries/clientnegative/drop_stats_for_columns_iceberg.q

@@ -0,0 +1,3 @@
+create table t1(i int, j int, k int) stored by iceberg;
+
+alter table t1 drop statistics for columns;

ql/src/test/queries/clientpositive/drop_histogram_stats_for_columns.q

@@ -0,0 +1,21 @@
+set hive.stats.kll.enable=true;
+set metastore.stats.fetch.bitvector=true;
+set metastore.stats.fetch.kll=true;
+set hive.stats.autogather=true;
+set hive.stats.column.autogather=true;
+
+CREATE TABLE test_stats (a string, b int, c double) STORED AS ORC;
+
+insert into test_stats (a, b, c) values ("a", 2, 1.1);
+
+describe formatted test_stats;
+describe formatted test_stats a;
+describe formatted test_stats b;
+describe formatted test_stats c;
+
+alter table test_stats drop statistics for columns;
+
+describe formatted test_stats;
+describe formatted test_stats a;
+describe formatted test_stats b;
+describe formatted test_stats c;

ql/src/test/queries/clientpositive/drop_stats_for_columns.q

@@ -0,0 +1,19 @@
+create table t1(i int, j int, k int);
+
+alter table t1 update statistics set('numRows'='10000', 'rawDataSize'='18000');
+alter table t1 update statistics for column i set('numDVs'='2500','numNulls'='50','highValue'='1000','lowValue'='0');
+alter table t1 update statistics for column j set('numDVs'='3000','numNulls'='40','highValue'='1200','lowValue'='2');
+alter table t1 update statistics for column k set('numDVs'='4000','numNulls'='60','highValue'='1500','lowValue'='5');
+
+describe formatted t1;
+describe formatted t1 i;
+describe formatted t1 j;
+describe formatted t1 k;
+
+-- drop stats from all columns
+alter table t1 drop statistics for columns;
+
+describe formatted t1;
+describe formatted t1 i;
+describe formatted t1 j;
+describe formatted t1 k;

ql/src/test/results/clientnegative/drop_stats_for_columns_iceberg.q.out

@@ -0,0 +1,9 @@
+PREHOOK: query: create table t1(i int, j int, k int) stored by iceberg
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1
+POSTHOOK: query: create table t1(i int, j int, k int) stored by iceberg
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1
+FAILED: SemanticException DROP STATISTICS FOR COLUMNS is not supported for non-native table that doesn't store stats in metastore.