HIVE-29191: Gracefully Handle Removed Configuration Properties in Hive 4 to Avoid Job Failures

Author: vikramahuja1001

common/src/java/org/apache/hadoop/hive/conf/HiveConf.java

@@ -5607,11 +5607,16 @@ public static enum ConfVars {
     HIVE_MM_AVOID_GLOBSTATUS_ON_S3("hive.mm.avoid.s3.globstatus", true,
         "Whether to use listFiles (optimized on S3) instead of globStatus when on S3."),
 
+    HIVE_IGNORE_REMOVED_CONFIGS_LIST("hive.ignore.removed.configs.list",
+            "",
+            "Comma separated list of configuration options which are removed from hive code. Silently ignore if the user tries to set them"),
+
     // If a parameter is added to the restricted list, add a test in TestRestrictedList.Java
     HIVE_CONF_RESTRICTED_LIST("hive.conf.restricted.list",
         "hive.security.authenticator.manager,hive.security.authorization.manager," +
         "hive.security.metastore.authorization.manager,hive.security.metastore.authenticator.manager," +
         "hive.users.in.admin.role,hive.server2.xsrf.filter.enabled,hive.server2.csrf.filter.enabled,hive.security.authorization.enabled," +
+            "hive.ignore.removed.configs.list," +
             "hive.distcp.privileged.doAs," +
             "hive.server2.authentication.ldap.baseDN," +
             "hive.server2.authentication.ldap.url," +
@@ -7261,6 +7266,10 @@ public static String generateDeprecationWarning() {
         + "versions. Please adjust DDL towards the new semantics.";
   }
 
+  public static String generateRemovedWarning() {
+    return "This config does not exist in the current version of Hive. Consider removing this config.";
+  }
+
   private static final Object reverseMapLock = new Object();
   private static HashMap<String, ConfVars> reverseMap = null;
 

itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestRestrictedList.java

@@ -70,6 +70,7 @@ public static void startServices() throws Exception {
 
     // Add the parameter here if it cannot change at runtime
     addToExpectedRestrictedMap("hive.conf.restricted.list");
+    addToExpectedRestrictedMap("hive.ignore.removed.configs.list");
     addToExpectedRestrictedMap("hive.security.authenticator.manager");
     addToExpectedRestrictedMap("hive.security.authorization.manager");
     addToExpectedRestrictedMap("hive.security.metastore.authorization.manager");

ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java

@@ -24,6 +24,7 @@
 
 import static org.apache.hadoop.hive.conf.SystemVariables.*;
 
+import java.util.HashSet;
 import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
@@ -55,19 +56,26 @@ public class SetProcessor implements CommandProcessor {
   private static final SessionState.LogHelper console = SessionState.getConsole();
 
   private static final String prefix = "set: ";
-  private static final Set<String> removedConfigs =
+  private static final Set<String> removedHiveConfigs =
       Sets.newHashSet("hive.mapred.supports.subdirectories",
           "hive.enforce.sorting","hive.enforce.bucketing",
           "hive.outerjoin.supports.filters",
           "hive.llap.zk.sm.principal",
-          "hive.llap.zk.sm.keytab.file"
+          "hive.llap.zk.sm.keytab.file",
+          "hive.stats.fetch.partition.stats",
+          "hive.optimize.sort.dynamic.partition",
+          "hive.metastore.initial.metadata.count.enabled",
+          "hive.cli.pretty.output.num.cols",
+          "hive.debug.localtask",
+          "hive.timedout.txn.reaper.start"
           );
   // Allow the user to set the ORC properties without getting an error.
+  private static final Set<String> allowOrcConfigs = new HashSet<>();
   static {
     for(OrcConf var: OrcConf.values()) {
       String name = var.getHiveConfName();
       if (name != null && name.startsWith("hive.")) {
-        removedConfigs.add(name);
+        allowOrcConfigs.add(name);
       }
     }
   }
@@ -140,6 +148,10 @@ private boolean isHidden(String key) {
     return false;
   }
 
+  public Set<String> getRemovedHiveConfigs() {
+    return removedHiveConfigs;
+  }
+
   private void dumpOption(String s) {
     SessionState ss = SessionState.get();
 
@@ -228,6 +240,22 @@ static String setConf(SessionState ss, String varName, String key, String varVal
         throws IllegalArgumentException {
     String result = null;
     HiveConf conf = ss.getConf();
+
+    String removedHiveConfigsList = conf.getVar(HiveConf.ConfVars.HIVE_IGNORE_REMOVED_CONFIGS_LIST);
+    if (removedHiveConfigsList != null && !removedHiveConfigsList.isEmpty()) {
+      for (String entry : removedHiveConfigsList.split(",")) {
+        if (!removedHiveConfigs.contains(entry.trim())) {
+          removedHiveConfigs.add(entry.trim());
+        }
+      }
+    }
+    if (removedHiveConfigs.contains(key)) {
+      // do not do anything. do not throw any error, just silently return
+      result = HiveConf.generateRemovedWarning();
+      LOG.warn(result);
+      return result;
+    }
+
     String value = new VariableSubstitution(new HiveVariableSource() {
       @Override
       public Map<String, String> getHiveVariable() {
@@ -251,7 +279,7 @@ public Map<String, String> getHiveVariable() {
           message.append("' FAILED in validation : ").append(fail).append('.');
           throw new IllegalArgumentException(message.toString());
         }
-      } else if (!removedConfigs.contains(key) && key.startsWith("hive.")) {
+      } else if (!allowOrcConfigs.contains(key) && key.startsWith("hive.")) {
         throw new IllegalArgumentException("hive configuration " + key + " does not exists.");
       }
     }

ql/src/test/org/apache/hadoop/hive/ql/processors/TestSetProcessor.java

@@ -107,6 +107,22 @@ public void testSystemPropertyIndividual() throws Exception {
     Assert.assertTrue(output.contains("hidden"));
   }
 
+  @Test
+  public void testRemovedConfigs() throws Exception {
+    Assert.assertEquals(baos.size(),0);
+    for (String entry : processor.getRemovedHiveConfigs()) {
+      // trying to set value
+      runSetProcessor(entry + "=Something");
+      // trying to get the value based on the previous set
+      runSetProcessor(entry);
+      Assert.assertTrue(baos.size() > 0);
+      String output = baos.toString();
+      Assert.assertTrue(output.contains("undefined"));
+      Assert.assertFalse(output.contains("Something"));
+    }
+    Assert.assertTrue(baos.size() > 0);
+  }
+
   /*
    * Simulates the set <command>;
    */