Skip to content

Commit 8495346

Browse files
author
John Sichi
committed
HIVE-1264. Make Hive work with Hadoop security
(Todd Lipcon via jvs) git-svn-id: https://svn.apache.org/repos/asf/hadoop/hive/trunk@1021549 13f79535-47bb-0310-9956-ffa450edef68
1 parent f7b856d commit 8495346

File tree

19 files changed

+626
-34
lines changed

19 files changed

+626
-34
lines changed

‎CHANGES.txt

+3
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,9 @@ Trunk - Unreleased
181181
HIVE-1697. Migration scripts should increase size of PARAM_VALUE in
182182
PARTITION_PARAMS (Paul Yang via namit)
183183

184+
HIVE-1264. Make Hive work with Hadoop security
185+
(Todd Lipcon via jvs)
186+
184187
OPTIMIZATIONS
185188

186189
BUG FIXES

‎build-common.xml

+4-2
Original file line numberDiff line numberDiff line change
@@ -218,8 +218,10 @@
218218

219219
<!-- the normal classpath -->
220220
<path id="common-classpath">
221-
<pathelement location="${hadoop.jar}"/>
222-
<pathelement location="${hadoop.tools.jar}"/>
221+
<pathelement location="${hadoop.oldstyle-name.jar}"/>
222+
<pathelement location="${hadoop.oldstyle-name.tools.jar}"/>
223+
<pathelement location="${hadoop.newstyle-name.jar}"/>
224+
<pathelement location="${hadoop.newstyle-name.tools.jar}"/>
223225
<pathelement location="${build.dir.hive}/classes"/>
224226
<fileset dir="${build.dir.hive}" includes="*/*.jar"/>
225227
<fileset dir="${hive.root}/lib" includes="*.jar"/>

‎build.properties

+14-3
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,15 @@ build.dir.hadoop=${build.dir.hive}/hadoopcore
2020
hadoop.version.ant-internal=${hadoop.version}
2121
hadoop.root.default=${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}
2222
hadoop.root=${hadoop.root.default}
23-
hadoop.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-core.jar
24-
hadoop.tools.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-tools.jar
25-
hadoop.test.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-test.jar
23+
# Newer versions of Hadoop name the jar as hadoop-{core,test}-VERSION instead of hadoop-VERSION-{core,test}
24+
# We will add both styles to the classpath and it will pick up whichever is there
25+
hadoop.oldstyle-name.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-core.jar
26+
hadoop.oldstyle-name.tools.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-tools.jar
27+
hadoop.oldstyle-name.test.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-test.jar
28+
hadoop.newstyle-name.jar=${hadoop.root}/hadoop-core-${hadoop.version.ant-internal}.jar
29+
hadoop.newstyle-name.test.jar=${hadoop.root}/hadoop-test-${hadoop.version.ant-internal}.jar
30+
hadoop.newstyle-name.tools.jar=${hadoop.root}/hadoop-tools-${hadoop.version.ant-internal}.jar
31+
2632
jetty.test.jar=${hadoop.root}/lib/jetty-5.1.4.jar
2733
servlet.test.jar=${hadoop.root}/lib/servlet-api.jar
2834
jasper.test.jar=${hadoop.root}/lib/jetty-ext/jasper-runtime.jar
@@ -34,3 +40,8 @@ common.jar=${hadoop.root}/lib/commons-httpclient-3.0.1.jar
3440
# Data nucleus repository - needed for jdo2-api-2.3-ec.jar download
3541
#
3642
datanucleus.repo=http://www.datanucleus.org/downloads/maven2
43+
44+
# URLs pointing to a built tarball of a secure hadoop release
45+
hadoop.security.url=http://mirror.facebook.net/facebook/hive-deps/hadoop/core/hadoop-0.20.3-CDH3-SNAPSHOT/hadoop-0.20.3-CDH3-SNAPSHOT.tar.gz
46+
hadoop.security.version=0.20.3-CDH3-SNAPSHOT
47+

‎build.xml

+2-1
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,8 @@
112112

113113
<!-- the normal classpath -->
114114
<path id="common-classpath">
115-
<pathelement location="${hadoop.jar}"/>
115+
<pathelement location="${hadoop.oldstyle-name.jar}"/>
116+
<pathelement location="${hadoop.newstyle-name.jar}"/>
116117
<pathelement location="${build.dir.hive}/classes"/>
117118
<fileset dir="${hive.root}" includes="hive-*.jar"/>
118119
<fileset dir="${hive.root}/lib" includes="*.jar"/>

‎common/src/java/org/apache/hadoop/hive/conf/HiveConf.java

+4-6
Original file line numberDiff line numberDiff line change
@@ -591,13 +591,11 @@ public void setAuxJars(String auxJars) {
591591
*/
592592
public String getUser() throws IOException {
593593
try {
594-
UserGroupInformation ugi = UserGroupInformation.readFrom(this);
595-
if (ugi == null) {
596-
ugi = UserGroupInformation.login(this);
597-
}
594+
UserGroupInformation ugi = ShimLoader.getHadoopShims()
595+
.getUGIForConf(this);
598596
return ugi.getUserName();
599-
} catch (LoginException e) {
600-
throw (IOException) new IOException().initCause(e);
597+
} catch (LoginException le) {
598+
throw new IOException(le);
601599
}
602600
}
603601

‎contrib/build.xml

-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
<pathelement location="${test.src.data.dir}/conf"/>
3737
<pathelement location="${hive.conf.dir}"/>
3838
<pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
39-
<pathelement location="${hadoop.test.jar}"/>
4039
<pathelement location="${jetty.test.jar}"/>
4140
<pathelement location="${servlet.test.jar}"/>
4241
<pathelement location="${jasper.test.jar}"/>

‎hbase-handler/build.xml

+2-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@
3737
<pathelement location="${test.src.data.dir}/conf"/>
3838
<pathelement location="${hive.conf.dir}"/>
3939
<pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
40-
<pathelement location="${hadoop.test.jar}"/>
40+
<pathelement location="${hadoop.oldstyle-name.test.jar}"/>
41+
<pathelement location="${hadoop.newstyle-name.test.jar}"/>
4142
<pathelement location="${jetty.test.jar}"/>
4243
<pathelement location="${servlet.test.jar}"/>
4344
<pathelement location="${jasper.test.jar}"/>

‎ql/build.xml

+2-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@
4040
<pathelement location="${test.src.data.dir}/conf"/>
4141
<pathelement location="${hive.conf.dir}"/>
4242
<pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
43-
<pathelement location="${hadoop.test.jar}"/>
43+
<pathelement location="${hadoop.oldstyle-name.test.jar}"/>
44+
<pathelement location="${hadoop.newstyle-name.test.jar}"/>
4445
<pathelement location="${jetty.test.jar}"/>
4546
<pathelement location="${servlet.test.jar}"/>
4647
<pathelement location="${jasper.test.jar}"/>

‎ql/src/java/org/apache/hadoop/hive/ql/Driver.java

+3-15
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,10 @@
8585
import org.apache.hadoop.hive.ql.session.SessionState;
8686
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
8787
import org.apache.hadoop.hive.serde2.ByteStream;
88+
import org.apache.hadoop.hive.shims.ShimLoader;
8889
import org.apache.hadoop.mapred.ClusterStatus;
8990
import org.apache.hadoop.mapred.JobClient;
9091
import org.apache.hadoop.mapred.JobConf;
91-
import org.apache.hadoop.security.UnixUserGroupInformation;
9292
import org.apache.hadoop.util.ReflectionUtils;
9393

9494
public class Driver implements CommandProcessor {
@@ -285,21 +285,11 @@ public boolean hasReduceTasks(List<Task<? extends Serializable>> tasks) {
285285
*/
286286
public Driver(HiveConf conf) {
287287
this.conf = conf;
288-
try {
289-
UnixUserGroupInformation.login(conf, true);
290-
} catch (Exception e) {
291-
LOG.warn("Ignoring " + e.getMessage());
292-
}
293288
}
294289

295290
public Driver() {
296291
if (SessionState.get() != null) {
297292
conf = SessionState.get().getConf();
298-
try {
299-
UnixUserGroupInformation.login(conf, true);
300-
} catch (Exception e) {
301-
LOG.warn("Ignoring " + e.getMessage());
302-
}
303293
}
304294
}
305295

@@ -739,8 +729,7 @@ public int execute() {
739729
// Get all the pre execution hooks and execute them.
740730
for (PreExecute peh : getPreExecHooks()) {
741731
peh.run(SessionState.get(), plan.getInputs(), plan.getOutputs(),
742-
UnixUserGroupInformation.readFromConf(conf,
743-
UnixUserGroupInformation.UGI_PROPERTY_NAME));
732+
ShimLoader.getHadoopShims().getUGIForConf(conf));
744733
}
745734

746735
int jobs = Utilities.getMRTasks(plan.getRootTasks()).size();
@@ -822,8 +811,7 @@ public int execute() {
822811
for (PostExecute peh : getPostExecHooks()) {
823812
peh.run(SessionState.get(), plan.getInputs(), plan.getOutputs(),
824813
(SessionState.get() != null ? SessionState.get().getLineageState().getLineageInfo() : null),
825-
UnixUserGroupInformation.readFromConf(conf,
826-
UnixUserGroupInformation.UGI_PROPERTY_NAME));
814+
ShimLoader.getHadoopShims().getUGIForConf(conf));
827815
}
828816

829817
if (SessionState.get() != null) {

‎shims/build.xml

+10-3
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,12 @@ to call at top-level: ant deploy-contrib compile-core-test
2626
<import file="../build-common.xml"/>
2727

2828
<path id="classpath">
29-
<pathelement location="${hadoop.jar}"/>
30-
<pathelement location="${hadoop.tools.jar}"/>
31-
<pathelement location="${hadoop.test.jar}"/>
29+
<pathelement location="${hadoop.oldstyle-name.jar}"/>
30+
<pathelement location="${hadoop.oldstyle-name.tools.jar}"/>
31+
<pathelement location="${hadoop.oldstyle-name.test.jar}"/>
32+
<pathelement location="${hadoop.newstyle-name.jar}"/>
33+
<pathelement location="${hadoop.newstyle-name.test.jar}"/>
34+
<pathelement location="${hadoop.newstyle-name.tools.jar}"/>
3235
<fileset dir="${hadoop.root}/lib">
3336
<include name="**/*.jar" />
3437
<exclude name="**/excluded/" />
@@ -66,6 +69,10 @@ to call at top-level: ant deploy-contrib compile-core-test
6669
<antcall target="build_shims" inheritRefs="false" inheritAll="false">
6770
<param name="hadoop.version.ant-internal" value="0.20.0" />
6871
</antcall>
72+
<antcall target="build_shims" inheritRefs="false" inheritAll="false">
73+
<param name="hadoop.version.ant-internal" value="${hadoop.security.version}" />
74+
<param name="hadoop.version.ant-internal.prefix" value="0.20S" />
75+
</antcall>
6976
<getversionpref property="hadoop.version.ant-internal.prefix" input="${hadoop.version}" />
7077
<javac
7178
encoding="${build.encoding}"

‎shims/ivy.xml

+4
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@
1313
<dependency org="hadoop" name="core" rev="0.20.0">
1414
<artifact name="hadoop" type="source" ext="tar.gz"/>
1515
</dependency>
16+
<dependency org="hadoop" name="core" rev="${hadoop.security.version}">
17+
<artifact name="hadoop" type="source" ext="tar.gz"
18+
url="${hadoop.security.url}" />
19+
</dependency>
1620
<conflict manager="all" />
1721
</dependencies>
1822
</ivy-module>

‎shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java

+12
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@
2929
import org.apache.hadoop.mapred.RunningJob;
3030
import org.apache.hadoop.mapred.TaskCompletionEvent;
3131
import org.apache.hadoop.mapred.lib.NullOutputFormat;
32+
import org.apache.hadoop.security.UserGroupInformation;
33+
import org.apache.hadoop.security.UnixUserGroupInformation;
34+
import javax.security.auth.login.LoginException;
3235

3336
import java.io.IOException;
3437

@@ -134,4 +137,13 @@ public int createHadoopArchive(Configuration conf, Path parentDir, Path destDir,
134137
throw new RuntimeException("Not implemented in this Hadoop version");
135138
}
136139

140+
@Override
141+
public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException {
142+
UserGroupInformation ugi =
143+
UnixUserGroupInformation.readFromConf(conf, UnixUserGroupInformation.UGI_PROPERTY_NAME);
144+
if(ugi == null) {
145+
ugi = UserGroupInformation.login(conf);
146+
}
147+
return ugi;
148+
}
137149
}

‎shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java

+13
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@
3131
import org.apache.hadoop.mapred.TaskAttemptID;
3232
import org.apache.hadoop.mapred.TaskCompletionEvent;
3333
import org.apache.hadoop.mapred.lib.NullOutputFormat;
34+
import org.apache.hadoop.security.UserGroupInformation;
35+
import org.apache.hadoop.security.UnixUserGroupInformation;
36+
import javax.security.auth.login.LoginException;
3437

3538
import java.io.IOException;
3639

@@ -137,4 +140,14 @@ public int createHadoopArchive(Configuration conf, Path parentDir, Path destDir,
137140
public void setNullOutputFormat(JobConf conf) {
138141
conf.setOutputFormat(NullOutputFormat.class);
139142
}
143+
144+
@Override
145+
public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException {
146+
UserGroupInformation ugi =
147+
UnixUserGroupInformation.readFromConf(conf, UnixUserGroupInformation.UGI_PROPERTY_NAME);
148+
if(ugi == null) {
149+
ugi = UserGroupInformation.login(conf);
150+
}
151+
return ugi;
152+
}
140153
}

‎shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java

+14
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@
3737
import org.apache.hadoop.mapred.TaskAttemptContext;
3838
import org.apache.hadoop.mapred.JobContext;
3939
import org.apache.hadoop.mapred.lib.NullOutputFormat;
40+
import org.apache.hadoop.security.UserGroupInformation;
41+
import org.apache.hadoop.security.UnixUserGroupInformation;
42+
import javax.security.auth.login.LoginException;
4043

4144
import java.io.IOException;
4245
import java.lang.reflect.Constructor;
@@ -512,4 +515,15 @@ public void setNullOutputFormat(JobConf conf) {
512515
// but can be backported. So we disable setup/cleanup in all versions >= 0.19
513516
conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
514517
}
518+
519+
520+
@Override
521+
public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException {
522+
UserGroupInformation ugi =
523+
UnixUserGroupInformation.readFromConf(conf, UnixUserGroupInformation.UGI_PROPERTY_NAME);
524+
if(ugi == null) {
525+
ugi = UserGroupInformation.login(conf);
526+
}
527+
return ugi;
528+
}
515529
}

‎shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java

+13
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import java.lang.reflect.Constructor;
2424
import java.util.ArrayList;
2525
import java.util.List;
26+
import javax.security.auth.login.LoginException;
2627

2728
import org.apache.hadoop.conf.Configuration;
2829
import org.apache.hadoop.fs.FileStatus;
@@ -48,6 +49,8 @@
4849
import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
4950
import org.apache.hadoop.mapred.lib.CombineFileSplit;
5051
import org.apache.hadoop.mapred.lib.NullOutputFormat;
52+
import org.apache.hadoop.security.UserGroupInformation;
53+
import org.apache.hadoop.security.UnixUserGroupInformation;
5154
import org.apache.hadoop.tools.HadoopArchives;
5255
import org.apache.hadoop.util.ToolRunner;
5356

@@ -436,4 +439,14 @@ public void setNullOutputFormat(JobConf conf) {
436439
// but can be backported. So we disable setup/cleanup in all versions >= 0.19
437440
conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
438441
}
442+
443+
@Override
444+
public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException {
445+
UserGroupInformation ugi =
446+
UnixUserGroupInformation.readFromConf(conf, UnixUserGroupInformation.UGI_PROPERTY_NAME);
447+
if(ugi == null) {
448+
ugi = UserGroupInformation.login(conf);
449+
}
450+
return ugi;
451+
}
439452
}

0 commit comments

Comments
 (0)