HIVE-29073: Overlay modified session level metaconf to new hmsclient connection (#6042)

Author: Aggarwal-Raghav

ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java

@@ -6465,6 +6465,21 @@ public List<Function> getFunctionsInDb(String dbName, String pattern) throws Hiv
 
   public void setMetaConf(String propName, String propValue) throws HiveException {
     try {
+      /*
+       * Updates the 'conf' object with session-level metastore variables
+       * ('metaConfVars'). This object is used to initialize the
+       * Thrift client connection to the Hive Metastore, ensuring that any
+       * session-specific overrides are propagated to the underlying connection.
+       *
+       * For reference on how this 'conf' object is consumed, see the client
+       * instantiation logic in:
+       * org.apache.hadoop.hive.ql.metadata.Hive#createMetaStoreClient()
+       */
+      if (Arrays.stream(MetastoreConf.metaConfVars)
+          .anyMatch(s -> s.getVarname().equals(propName))) {
+        // Storing varname prevents conflicts with HiveServer2-level configurations
+        conf.set(propName, propValue);
+      }
       getMSC().setMetaConf(propName, propValue);
     } catch (TException te) {
       throw new HiveException(te);

standalone-metastore/metastore-client/src/main/java/org/apache/hadoop/hive/metastore/client/ThriftHiveMetaStoreClient.java

@@ -763,6 +763,8 @@ private void open() throws MetaException {
           LOG.error(errMsg, e);
         }
         if (isConnected) {
+          // Set the beeline session modified metaConfVars for new HMS connection
+          overlaySessionModifiedMetaConf();
           break;
         }
       }
@@ -792,6 +794,23 @@ private void open() throws MetaException {
     snapshotActiveConf();
   }
 
+  private void overlaySessionModifiedMetaConf() {
+    for (MetastoreConf.ConfVars confVar : MetastoreConf.metaConfVars) {
+      String confVal = conf.get(confVar.getVarname());
+      if (!org.apache.commons.lang3.StringUtils.isBlank(confVal)) {
+        try {
+          setMetaConf(confVar.getVarname(), confVal);
+        } catch (TException e) {
+          LOG.error(
+              "Failed to set metastore config for {} with value {}",
+              confVar.getVarname(),
+              confVal,
+              e);
+        }
+      }
+    }
+  }
+
   // wraps the underlyingTransport in the appropriate transport based on mode of authentication
   private TTransport createAuthBinaryTransport(URI store, TTransport underlyingTransport)
       throws MetaException {

standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java

@@ -225,7 +225,6 @@ public String toString() {
       ConfVars.HMS_HANDLER_ATTEMPTS,
       ConfVars.HMS_HANDLER_INTERVAL,
       ConfVars.HMS_HANDLER_FORCE_RELOAD_CONF,
-      ConfVars.PARTITION_NAME_WHITELIST_PATTERN,
       ConfVars.ORM_RETRIEVE_MAPNULLS_AS_EMPTY_STRINGS,
       ConfVars.USERS_IN_ADMIN_ROLE,
       ConfVars.HIVE_TXN_MANAGER,
@@ -244,18 +243,26 @@ public String toString() {
       ConfVars.AGGREGATE_STATS_CACHE_MAX_READER_WAIT,
       ConfVars.AGGREGATE_STATS_CACHE_MAX_FULL,
       ConfVars.AGGREGATE_STATS_CACHE_CLEAN_UNTIL,
-      ConfVars.DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES,
       ConfVars.FILE_METADATA_THREADS,
       ConfVars.METASTORE_CLIENT_FILTER_ENABLED,
       ConfVars.METASTORE_SERVER_FILTER_ENABLED,
       ConfVars.METASTORE_PARTITIONS_PARAMETERS_INCLUDE_PATTERN,
-      ConfVars.METASTORE_PARTITIONS_PARAMETERS_EXCLUDE_PATTERN
+      ConfVars.METASTORE_PARTITIONS_PARAMETERS_EXCLUDE_PATTERN,
+      // Add metaConfVars here as well
+      ConfVars.TRY_DIRECT_SQL,
+      ConfVars.TRY_DIRECT_SQL_DDL,
+      ConfVars.CLIENT_SOCKET_TIMEOUT,
+      ConfVars.PARTITION_NAME_WHITELIST_PATTERN,
+      ConfVars.PARTITION_ORDER_EXPR,
+      ConfVars.CAPABILITY_CHECK,
+      ConfVars.DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES,
+      ConfVars.EXPRESSION_PROXY_CLASS
   };
 
   /**
    * User configurable Metastore vars
    */
-  private static final MetastoreConf.ConfVars[] metaConfVars = {
+  public static final MetastoreConf.ConfVars[] metaConfVars = {
       ConfVars.TRY_DIRECT_SQL,
       ConfVars.TRY_DIRECT_SQL_DDL,
       ConfVars.CLIENT_SOCKET_TIMEOUT,

standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreHttpHeaders.java

@@ -96,7 +96,15 @@ public void testHttpHeaders() throws Exception {
   public void testIllegalHttpHeaders() throws Exception {
     MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METASTORE_CLIENT_ADDITIONAL_HEADERS,
         String.format("%s%s", testHeaderKey1, testHeaderVal1));
-    msc = new TestHiveMetaStoreClient(conf);
+    try {
+      msc = new TestHiveMetaStoreClient(conf);
+    } catch (Exception ignored) {
+      /*
+       * This try catch is added because of setMetaConf in
+       * org.apache.hadoop.hive.metastore.client.ThriftHiveMetaStoreClient.overlaySessionModifiedMetaConf
+       * Because of wrong header (Negative Test) the exception is thrown during Client creation itself
+       */
+    }
     boolean exceptionThrown = false;
     try {
       Database db = new DatabaseBuilder().setName("testHttpHeader").create(msc, conf);

standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java

@@ -262,6 +262,9 @@ public void testListener() throws Exception {
     // Test adding multiple partitions in a single partition-set, atomically.
     int currentTime = (int)System.currentTimeMillis();
     HiveMetaStoreClient hmsClient = new HiveMetaStoreClient(conf);
+    // A ConfigChangeEvent will be triggered on new Client creation because of
+    // org.apache.hadoop.hive.metastore.client.ThriftHiveMetaStoreClient#overlaySessionModifiedMetaConf()
+    ++listSize;
     table = hmsClient.getTable(dbName, "tmptbl");
     Partition partition1 = new Partition(Arrays.asList("20110101"), dbName, "tmptbl", currentTime,
                                         currentTime, table.getSd(), table.getParameters());