Skip to content

Commit ad55d58

Browse files
difindifin
authored
HIVE-29234: Iceberg: Validate HMS REST Catalog Client with OAuth2 (#6124)
1 parent abcca13 commit ad55d58

File tree

11 files changed

+317
-88
lines changed

11 files changed

+317
-88
lines changed

‎itests/hive-iceberg/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,12 @@
4646
<classifier>tests</classifier>
4747
<version>${project.version}</version>
4848
</dependency>
49+
<dependency>
50+
<groupId>org.keycloak</groupId>
51+
<artifactId>keycloak-admin-client</artifactId>
52+
<version>${keycloak.version}</version>
53+
<scope>test</scope>
54+
</dependency>
4955
<dependency>
5056
<groupId>org.apache.hive</groupId>
5157
<artifactId>hive-standalone-metastore-common</artifactId>

‎itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientIT.java‎ renamed to ‎itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java‎

Lines changed: 51 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@
2323
import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
2424
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
2525
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
26-
import org.apache.hadoop.hive.metastore.ServletSecurity;
2726
import org.apache.hadoop.hive.metastore.api.Database;
2827
import org.apache.hadoop.hive.metastore.api.FieldSchema;
28+
import org.apache.hadoop.hive.metastore.api.GetTableRequest;
2929
import org.apache.hadoop.hive.metastore.api.MetaException;
3030
import org.apache.hadoop.hive.metastore.api.PrincipalType;
3131
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
@@ -46,59 +46,54 @@
4646
import org.apache.iceberg.hive.CatalogUtils;
4747
import org.apache.iceberg.hive.HiveSchemaUtil;
4848
import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
49-
import org.junit.jupiter.api.AfterAll;
49+
import org.junit.jupiter.api.AfterEach;
5050
import org.junit.jupiter.api.Assertions;
51-
import org.junit.jupiter.api.BeforeAll;
51+
import org.junit.jupiter.api.BeforeEach;
5252
import org.junit.jupiter.api.Test;
53-
import org.junit.jupiter.api.TestInstance;
54-
import org.junit.jupiter.api.extension.RegisterExtension;
55-
56-
import java.util.Collections;
57-
import java.util.Map;
5853

5954
import java.util.Arrays;
55+
import java.util.Collections;
6056
import java.util.List;
57+
import java.util.Map;
6158

6259
import static org.junit.jupiter.api.Assertions.assertThrows;
6360

6461
/*
65-
* This test is an integration test for the hive-iceberg REST Catalog client and HMS REST Catalog Server.
66-
* It uses the HiveMetaStoreClient backed by hive-iceberg REST catalog adapter to connect to the HMS RESTCatalog Server.
62+
* This is an integration test for the HiveMetaStoreClient and HMS REST Catalog Server. It creates and uses the
63+
* HMS IMetaStoreClient backed by HiveMetaStoreClient adapter to connect to the HMS RESTCatalog Server.
6764
* The flow is as follows:
6865
* Hive ql wrapper --> HiveMetaStoreClient --> HiveRESTCatalogClient --> HMS RESTCatalog Server --> HMS
6966
*/
70-
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
71-
public class TestHiveRESTCatalogClientIT {
72-
73-
private static final String DB_NAME = "ice_db";
74-
private static final String TABLE_NAME = "ice_tbl";
75-
private static final String CATALOG_NAME = "ice01";
76-
private static final String HIVE_ICEBERG_STORAGE_HANDLER = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
67+
public abstract class TestHiveRESTCatalogClientITBase {
68+
69+
static final String DB_NAME = "ice_db";
70+
static final String TABLE_NAME = "ice_tbl";
71+
static final String CATALOG_NAME = "ice01";
72+
static final String HIVE_ICEBERG_STORAGE_HANDLER = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
73+
static final String REST_CATALOG_PREFIX = String.format("%s%s.", CatalogUtils.CATALOG_CONFIG_PREFIX, CATALOG_NAME);
74+
75+
HiveConf hiveConf;
76+
Configuration conf;
77+
Hive hive;
78+
IMetaStoreClient msClient;
7779

78-
private Configuration conf;
79-
private HiveConf hiveConf;
80-
private Hive hive;
81-
82-
private IMetaStoreClient msClient;
83-
84-
@RegisterExtension
85-
private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
86-
HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.NONE)
87-
.addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
88-
.build();
80+
abstract HiveRESTCatalogServerExtension getHiveRESTCatalogServerExtension();
8981

90-
@BeforeAll
91-
public void setup() throws Exception {
92-
// Starting msClient with Iceberg REST Catalog client underneath
93-
String restCatalogPrefix = String.format("%s%s.", CatalogUtils.CATALOG_CONFIG_PREFIX, CATALOG_NAME);
82+
public void setupConf() {
83+
HiveRESTCatalogServerExtension restCatalogExtension = getHiveRESTCatalogServerExtension();
9484

95-
conf = REST_CATALOG_EXTENSION.getConf();
85+
conf = restCatalogExtension.getConf();
9686

9787
MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METASTORE_CLIENT_IMPL,
9888
"org.apache.iceberg.hive.client.HiveRESTCatalogClient");
9989
conf.set(MetastoreConf.ConfVars.CATALOG_DEFAULT.getVarname(), CATALOG_NAME);
100-
conf.set(restCatalogPrefix + "uri", REST_CATALOG_EXTENSION.getRestEndpoint());
101-
conf.set(restCatalogPrefix + "type", CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
90+
conf.set(REST_CATALOG_PREFIX + "uri", restCatalogExtension.getRestEndpoint());
91+
conf.set(REST_CATALOG_PREFIX + "type", CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
92+
}
93+
94+
@BeforeEach
95+
void setup() throws Exception {
96+
setupConf();
10297

10398
HiveMetaHookLoader hookLoader = tbl -> {
10499
HiveStorageHandler storageHandler;
@@ -109,18 +104,19 @@ public void setup() throws Exception {
109104
}
110105
return storageHandler == null ? null : storageHandler.getMetaHook();
111106
};
112-
107+
113108
msClient = new HiveMetaStoreClient(conf, hookLoader);
114109
hiveConf = new HiveConf(conf, HiveConf.class);
115110
hive = Hive.get(hiveConf);
116111
}
117112

118-
@AfterAll public void tearDown() {
113+
@AfterEach
114+
public void tearDown() {
119115
if (msClient != null) {
120116
msClient.close();
121117
}
122118
}
123-
119+
124120
@Test
125121
public void testIceberg() throws Exception {
126122

@@ -142,7 +138,7 @@ public void testIceberg() throws Exception {
142138
// --- Get Databases ---
143139
List<String> dbs = msClient.getDatabases(CATALOG_NAME, "ice_*");
144140
Assertions.assertEquals(1, dbs.size());
145-
Assertions.assertEquals(DB_NAME, dbs.get(0));
141+
Assertions.assertEquals(DB_NAME, dbs.getFirst());
146142

147143
// --- Get All Databases ---
148144
List<String> allDbs = msClient.getAllDatabases(CATALOG_NAME);
@@ -151,7 +147,7 @@ public void testIceberg() throws Exception {
151147
Assertions.assertTrue(allDbs.contains(DB_NAME));
152148

153149
// --- Create Table ---
154-
org.apache.hadoop.hive.metastore.api.Table tTable = createPartitionedTable(msClient,
150+
Table tTable = createPartitionedTable(msClient,
155151
CATALOG_NAME, DB_NAME, TABLE_NAME, new java.util.HashMap<>());
156152
Assertions.assertNotNull(tTable);
157153
Assertions.assertEquals(HiveMetaHook.ICEBERG, tTable.getParameters().get(HiveMetaHook.TABLE_TYPE));
@@ -166,7 +162,12 @@ public void testIceberg() throws Exception {
166162
Assertions.assertTrue(msClient.tableExists(CATALOG_NAME, DB_NAME, TABLE_NAME));
167163

168164
// --- Get Table ---
169-
org.apache.hadoop.hive.metastore.api.Table table = msClient.getTable(CATALOG_NAME, DB_NAME, TABLE_NAME);
165+
GetTableRequest getTableRequest = new GetTableRequest();
166+
getTableRequest.setCatName(CATALOG_NAME);
167+
getTableRequest.setDbName(DB_NAME);
168+
getTableRequest.setTblName(TABLE_NAME);
169+
170+
Table table = msClient.getTable(getTableRequest);
170171
Assertions.assertEquals(DB_NAME, table.getDbName());
171172
Assertions.assertEquals(TABLE_NAME, table.getTableName());
172173
Assertions.assertEquals(HIVE_ICEBERG_STORAGE_HANDLER, table.getParameters().get("storage_handler"));
@@ -193,8 +194,8 @@ public void testIceberg() throws Exception {
193194
Assertions.assertFalse(msClient.getAllDatabases(CATALOG_NAME).contains(DB_NAME));
194195
}
195196

196-
private static Table createPartitionedTable(IMetaStoreClient db, String catName, String dbName, String tableName,
197-
Map<String, String> tableParameters) throws Exception {
197+
private static Table createPartitionedTable(IMetaStoreClient db, String catName, String dbName, String tableName,
198+
Map<String, String> tableParameters) throws Exception {
198199
db.dropTable(catName, dbName, tableName);
199200
Table table = new Table();
200201
table.setCatName(catName);
@@ -222,6 +223,12 @@ private static Table createPartitionedTable(IMetaStoreClient db, String catName,
222223
table.getParameters().put(TableProperties.DEFAULT_PARTITION_SPEC, specString);
223224

224225
db.createTable(table);
225-
return db.getTable(catName, dbName, tableName);
226+
227+
GetTableRequest getTableRequest = new GetTableRequest();
228+
getTableRequest.setCatName(CATALOG_NAME);
229+
getTableRequest.setDbName(DB_NAME);
230+
getTableRequest.setTblName(TABLE_NAME);
231+
232+
return db.getTable(getTableRequest);
226233
}
227234
}

‎itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITNoAuth.java‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hive;
19+
20+
import org.apache.hadoop.hive.metastore.ServletSecurity;
21+
import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
22+
import org.junit.jupiter.api.TestInstance;
23+
import org.junit.jupiter.api.extension.RegisterExtension;
24+
25+
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
26+
public class TestHiveRESTCatalogClientITNoAuth extends TestHiveRESTCatalogClientITBase {
27+
28+
@RegisterExtension
29+
private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
30+
HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.NONE)
31+
.addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
32+
.build();
33+
34+
@Override
35+
HiveRESTCatalogServerExtension getHiveRESTCatalogServerExtension() {
36+
return REST_CATALOG_EXTENSION;
37+
}
38+
}

‎itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITOauth2.java‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
package org.apache.hive;
19+
20+
import org.apache.hadoop.hive.metastore.ServletSecurity;
21+
import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
22+
import org.junit.jupiter.api.TestInstance;
23+
import org.junit.jupiter.api.extension.RegisterExtension;
24+
25+
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
26+
public class TestHiveRESTCatalogClientITOauth2 extends TestHiveRESTCatalogClientITBase {
27+
28+
@RegisterExtension
29+
private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
30+
HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.OAUTH2)
31+
.addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
32+
.build();
33+
34+
@Override
35+
public void setupConf() {
36+
super.setupConf();
37+
38+
// Oauth2 properties
39+
conf.set(REST_CATALOG_PREFIX + "rest.auth.type", "oauth2");
40+
conf.set(REST_CATALOG_PREFIX + "oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint());
41+
conf.set(REST_CATALOG_PREFIX + "credential", REST_CATALOG_EXTENSION.getOAuth2ClientCredential());
42+
}
43+
44+
@Override
45+
HiveRESTCatalogServerExtension getHiveRESTCatalogServerExtension() {
46+
return REST_CATALOG_EXTENSION;
47+
}
48+
}

‎itests/qtest-iceberg/pom.xml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -480,6 +480,37 @@
480480
<artifactId>testcontainers</artifactId>
481481
<scope>test</scope>
482482
</dependency>
483+
<dependency>
484+
<groupId>org.keycloak</groupId>
485+
<artifactId>keycloak-admin-client</artifactId>
486+
<version>${keycloak.version}</version>
487+
<scope>test</scope>
488+
</dependency>
489+
<dependency>
490+
<groupId>jakarta.annotation</groupId>
491+
<artifactId>jakarta.annotation-api</artifactId>
492+
<version>2.1.1</version>
493+
</dependency>
494+
<dependency>
495+
<groupId>com.nimbusds</groupId>
496+
<artifactId>oauth2-oidc-sdk</artifactId>
497+
<version>${nimbus-oauth.version}</version>
498+
</dependency>
499+
<dependency>
500+
<groupId>jakarta.xml.bind</groupId>
501+
<artifactId>jakarta.xml.bind-api</artifactId>
502+
<version>4.0.4</version>
503+
</dependency>
504+
<dependency>
505+
<groupId>jakarta.activation</groupId>
506+
<artifactId>jakarta.activation-api</artifactId>
507+
<version>2.1.2</version>
508+
</dependency>
509+
<dependency>
510+
<groupId>org.glassfish.jaxb</groupId>
511+
<artifactId>jaxb-runtime</artifactId>
512+
<version>${jaxb-runtime.version}</version>
513+
</dependency>
483514
</dependencies>
484515
<build>
485516
<plugins>

0 commit comments

Comments
 (0)