HIVE-29234: Iceberg: Validate HMS REST Catalog Client with OAuth2

itests/hive-iceberg/pom.xml

@@ -46,6 +46,12 @@
       <classifier>tests</classifier>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.keycloak</groupId>
+      <artifactId>keycloak-admin-client</artifactId>
+      <version>${keycloak.version}</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-standalone-metastore-common</artifactId>

itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientIT.java → itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java

@@ -23,9 +23,9 @@
 import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.ServletSecurity;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.GetTableRequest;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.api.PrincipalType;
 import org.apache.hadoop.hive.metastore.api.SerDeInfo;
@@ -46,59 +46,54 @@
 import org.apache.iceberg.hive.CatalogUtils;
 import org.apache.iceberg.hive.HiveSchemaUtil;
 import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
-import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestInstance;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.util.Collections;
-import java.util.Map;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 /* 
-  * This test is an integration test for the hive-iceberg REST Catalog client and HMS REST Catalog Server.
-  * It uses the HiveMetaStoreClient backed by hive-iceberg REST catalog adapter to connect to the HMS RESTCatalog Server.
+  * This is an integration test for the HiveMetaStoreClient and HMS REST Catalog Server. It creates and uses the 
+  * HMS IMetaStoreClient backed by HiveMetaStoreClient adapter to connect to the HMS RESTCatalog Server.
   * The flow is as follows:
   * Hive ql wrapper --> HiveMetaStoreClient --> HiveRESTCatalogClient --> HMS RESTCatalog Server --> HMS
  */
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public class TestHiveRESTCatalogClientIT {
-
-  private static final String DB_NAME = "ice_db";
-  private static final String TABLE_NAME = "ice_tbl";
-  private static final String CATALOG_NAME = "ice01";
-  private static final String HIVE_ICEBERG_STORAGE_HANDLER = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
+public abstract class TestHiveRESTCatalogClientITBase {
+
+  static final String DB_NAME = "ice_db";
+  static final String TABLE_NAME = "ice_tbl";
+  static final String CATALOG_NAME = "ice01";
+  static final String HIVE_ICEBERG_STORAGE_HANDLER = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
+  static final String REST_CATALOG_PREFIX = String.format("%s%s.", CatalogUtils.CATALOG_CONFIG_PREFIX, CATALOG_NAME);
+
+  HiveConf hiveConf;
+  Configuration conf;
+  Hive hive;
+  IMetaStoreClient msClient;
 
-  private Configuration conf;
-  private HiveConf hiveConf;
-  private Hive hive;
-
-  private IMetaStoreClient msClient;
-
-  @RegisterExtension
-  private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
-      HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.NONE)
-          .addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
-          .build();
+  abstract HiveRESTCatalogServerExtension getHiveRESTCatalogServerExtension();
 
-  @BeforeAll
-  public void setup() throws Exception {
-    // Starting msClient with Iceberg REST Catalog client underneath
-    String restCatalogPrefix = String.format("%s%s.", CatalogUtils.CATALOG_CONFIG_PREFIX, CATALOG_NAME);
+  public void setupConf() {
+    HiveRESTCatalogServerExtension restCatalogExtension = getHiveRESTCatalogServerExtension();
 
-    conf = REST_CATALOG_EXTENSION.getConf();
+    conf = restCatalogExtension.getConf();
 
     MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METASTORE_CLIENT_IMPL,
         "org.apache.iceberg.hive.client.HiveRESTCatalogClient");
     conf.set(MetastoreConf.ConfVars.CATALOG_DEFAULT.getVarname(), CATALOG_NAME);
-    conf.set(restCatalogPrefix + "uri", REST_CATALOG_EXTENSION.getRestEndpoint());
-    conf.set(restCatalogPrefix + "type", CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
+    conf.set(REST_CATALOG_PREFIX + "uri", restCatalogExtension.getRestEndpoint());
+    conf.set(REST_CATALOG_PREFIX + "type", CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
+  }
+
+  @BeforeEach
+  void setup() throws Exception {
+    setupConf();
 
     HiveMetaHookLoader hookLoader = tbl -> {
       HiveStorageHandler storageHandler;
@@ -109,18 +104,19 @@ public void setup() throws Exception {
       }
       return storageHandler == null ? null : storageHandler.getMetaHook();
     };
-    
+
     msClient = new HiveMetaStoreClient(conf, hookLoader);
     hiveConf = new HiveConf(conf, HiveConf.class);
     hive = Hive.get(hiveConf);
   }
 
-  @AfterAll public void tearDown() {
+  @AfterEach
+  public void tearDown() {
     if (msClient != null) {
       msClient.close();
     }
   }
-
+  
   @Test
   public void testIceberg() throws Exception {
 
@@ -142,7 +138,7 @@ public void testIceberg() throws Exception {
     // --- Get Databases ---
     List<String> dbs = msClient.getDatabases(CATALOG_NAME, "ice_*");
     Assertions.assertEquals(1, dbs.size());
-    Assertions.assertEquals(DB_NAME, dbs.get(0));
+    Assertions.assertEquals(DB_NAME, dbs.getFirst());
 
     // --- Get All Databases ---
     List<String> allDbs = msClient.getAllDatabases(CATALOG_NAME);
@@ -151,7 +147,7 @@ public void testIceberg() throws Exception {
     Assertions.assertTrue(allDbs.contains(DB_NAME));
 
     // --- Create Table ---
-    org.apache.hadoop.hive.metastore.api.Table tTable = createPartitionedTable(msClient,
+    Table tTable = createPartitionedTable(msClient,
         CATALOG_NAME, DB_NAME, TABLE_NAME, new java.util.HashMap<>());
     Assertions.assertNotNull(tTable);
     Assertions.assertEquals(HiveMetaHook.ICEBERG, tTable.getParameters().get(HiveMetaHook.TABLE_TYPE));
@@ -166,7 +162,12 @@ public void testIceberg() throws Exception {
     Assertions.assertTrue(msClient.tableExists(CATALOG_NAME, DB_NAME, TABLE_NAME));
 
     // --- Get Table ---
-    org.apache.hadoop.hive.metastore.api.Table table = msClient.getTable(CATALOG_NAME, DB_NAME, TABLE_NAME);
+    GetTableRequest getTableRequest = new GetTableRequest();
+    getTableRequest.setCatName(CATALOG_NAME);
+    getTableRequest.setDbName(DB_NAME);
+    getTableRequest.setTblName(TABLE_NAME);
+
+    Table table = msClient.getTable(getTableRequest);
     Assertions.assertEquals(DB_NAME, table.getDbName());
     Assertions.assertEquals(TABLE_NAME, table.getTableName());
     Assertions.assertEquals(HIVE_ICEBERG_STORAGE_HANDLER, table.getParameters().get("storage_handler"));
@@ -193,8 +194,8 @@ public void testIceberg() throws Exception {
     Assertions.assertFalse(msClient.getAllDatabases(CATALOG_NAME).contains(DB_NAME));
   }
 
-  private static Table createPartitionedTable(IMetaStoreClient db, String catName, String dbName, String tableName,
-    Map<String, String> tableParameters) throws Exception {
+  private static Table createPartitionedTable(IMetaStoreClient db, String catName, String dbName, String tableName, 
+      Map<String, String> tableParameters) throws Exception {
     db.dropTable(catName, dbName, tableName);
     Table table = new Table();
     table.setCatName(catName);
@@ -222,6 +223,12 @@ private static Table createPartitionedTable(IMetaStoreClient db, String catName,
     table.getParameters().put(TableProperties.DEFAULT_PARTITION_SPEC, specString);
 
     db.createTable(table);
-    return db.getTable(catName, dbName, tableName);
+
+    GetTableRequest getTableRequest = new GetTableRequest();
+    getTableRequest.setCatName(CATALOG_NAME);
+    getTableRequest.setDbName(DB_NAME);
+    getTableRequest.setTblName(TABLE_NAME);
+
+    return db.getTable(getTableRequest);
   }
 }

itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITNoAuth.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive;
+
+import org.apache.hadoop.hive.metastore.ServletSecurity;
+import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class TestHiveRESTCatalogClientITNoAuth extends TestHiveRESTCatalogClientITBase {
+
+  @RegisterExtension
+  private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
+      HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.NONE)
+          .addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
+          .build();
+
+  @Override
+  HiveRESTCatalogServerExtension getHiveRESTCatalogServerExtension() {
+    return REST_CATALOG_EXTENSION;
+  }
+}

itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITOauth2.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive;
+
+import org.apache.hadoop.hive.metastore.ServletSecurity;
+import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class TestHiveRESTCatalogClientITOauth2 extends TestHiveRESTCatalogClientITBase {
+
+  @RegisterExtension
+  private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
+      HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.OAUTH2)
+          .addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
+          .build();
+
+  @Override
+  public void setupConf() {
+    super.setupConf();
+
+    // Oauth2 properties
+    conf.set(REST_CATALOG_PREFIX + "rest.auth.type", "oauth2");
+    conf.set(REST_CATALOG_PREFIX + "oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint());
+    conf.set(REST_CATALOG_PREFIX + "credential", REST_CATALOG_EXTENSION.getOAuth2ClientCredential());
+  }
+
+  @Override
+  HiveRESTCatalogServerExtension getHiveRESTCatalogServerExtension() {
+    return REST_CATALOG_EXTENSION;
+  }
+}

itests/qtest-iceberg/pom.xml

@@ -480,6 +480,37 @@
       <artifactId>testcontainers</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.keycloak</groupId>
+      <artifactId>keycloak-admin-client</artifactId>
+      <version>${keycloak.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>jakarta.annotation</groupId>
+      <artifactId>jakarta.annotation-api</artifactId>
+      <version>2.1.1</version>
+    </dependency>
+    <dependency>
+      <groupId>com.nimbusds</groupId>
+      <artifactId>oauth2-oidc-sdk</artifactId>
+      <version>${nimbus-oauth.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>jakarta.xml.bind</groupId>
+      <artifactId>jakarta.xml.bind-api</artifactId>
+      <version>4.0.4</version>
+    </dependency>
+    <dependency>
+      <groupId>jakarta.activation</groupId>
+      <artifactId>jakarta.activation-api</artifactId>
+      <version>2.1.2</version>
+    </dependency>
+    <dependency>
+      <groupId>org.glassfish.jaxb</groupId>
+      <artifactId>jaxb-runtime</artifactId>
+      <version>${jaxb-runtime.version}</version>
+    </dependency>
   </dependencies>
   <build>
     <plugins>