diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index a9330289977b..4ba8042abe4a 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -7261,6 +7261,10 @@ public static String generateDeprecationWarning() { + "versions. Please adjust DDL towards the new semantics."; } + public static String generateRemovedWarning() { + return "This config does not exist in the current version of Hive. Consider removing this config."; + } + private static final Object reverseMapLock = new Object(); private static HashMap reverseMap = null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java index 87547116fed1..bebfbe4121e6 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/processors/SetProcessor.java @@ -24,6 +24,7 @@ import static org.apache.hadoop.hive.conf.SystemVariables.*; +import java.util.HashSet; import java.util.Map; import java.util.Properties; import java.util.Set; @@ -55,19 +56,30 @@ public class SetProcessor implements CommandProcessor { private static final SessionState.LogHelper console = SessionState.getConsole(); private static final String prefix = "set: "; - private static final Set removedConfigs = + private static final Set removedHiveConfigs = Sets.newHashSet("hive.mapred.supports.subdirectories", "hive.enforce.sorting","hive.enforce.bucketing", "hive.outerjoin.supports.filters", "hive.llap.zk.sm.principal", - "hive.llap.zk.sm.keytab.file" + "hive.llap.zk.sm.keytab.file", + "hive.stats.fetch.partition.stats", + "hive.optimize.sort.dynamic.partition", + "hive.metastore.initial.metadata.count.enabled", + "hive.cli.pretty.output.num.cols", + "hive.debug.localtask", + "hive.timedout.txn.reaper.start", + "hive.repl.dumpdir.ttl", + "hive.repl.dumpdir.clean.freq", + "hive.llap.io.vrb.queue.limit.base", + "hive.llap.external.splits.order.by.force.single.split" ); // Allow the user to set the ORC properties without getting an error. + private static final Set allowOrcConfigs = new HashSet<>(); static { for(OrcConf var: OrcConf.values()) { String name = var.getHiveConfName(); if (name != null && name.startsWith("hive.")) { - removedConfigs.add(name); + allowOrcConfigs.add(name); } } } @@ -140,6 +152,10 @@ private boolean isHidden(String key) { return false; } + public Set getRemovedHiveConfigs() { + return removedHiveConfigs; + } + private void dumpOption(String s) { SessionState ss = SessionState.get(); @@ -228,6 +244,14 @@ static String setConf(SessionState ss, String varName, String key, String varVal throws IllegalArgumentException { String result = null; HiveConf conf = ss.getConf(); + + if (removedHiveConfigs.contains(key)) { + // do not do anything. do not throw any error, just silently return + result = HiveConf.generateRemovedWarning(); + LOG.warn(result); + return result; + } + String value = new VariableSubstitution(new HiveVariableSource() { @Override public Map getHiveVariable() { @@ -251,7 +275,7 @@ public Map getHiveVariable() { message.append("' FAILED in validation : ").append(fail).append('.'); throw new IllegalArgumentException(message.toString()); } - } else if (!removedConfigs.contains(key) && key.startsWith("hive.")) { + } else if (!allowOrcConfigs.contains(key) && key.startsWith("hive.")) { throw new IllegalArgumentException("hive configuration " + key + " does not exists."); } } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/processors/TestSetProcessor.java b/ql/src/test/org/apache/hadoop/hive/ql/processors/TestSetProcessor.java index bb8cf5356551..30a73ff147a9 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/processors/TestSetProcessor.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/processors/TestSetProcessor.java @@ -107,6 +107,22 @@ public void testSystemPropertyIndividual() throws Exception { Assert.assertTrue(output.contains("hidden")); } + @Test + public void testRemovedConfigs() throws Exception { + Assert.assertEquals(baos.size(),0); + for (String entry : processor.getRemovedHiveConfigs()) { + // trying to set value + runSetProcessor(entry + "=Something"); + // trying to get the value based on the previous set + runSetProcessor(entry); + Assert.assertTrue(baos.size() > 0); + String output = baos.toString(); + Assert.assertTrue(output.contains("undefined")); + Assert.assertFalse(output.contains("Something")); + } + Assert.assertTrue(baos.size() > 0); + } + /* * Simulates the set ; */