Skip to content

Commit 494ecda

Browse files
unidevelunidevel
committed
Add docker files for hive4.0-hive
1 parent 3f7be0e commit 494ecda

File tree

19 files changed

+339
-0
lines changed

19 files changed

+339
-0
lines changed

‎etc/compose/hive4.0-hive/docker-compose.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
services:
2+
hadoop-master:
3+
hostname: hadoop-master
4+
image: prestodb/hive4.0-hive:latest
5+

‎prestodb/hive4.0-hive/Dockerfile‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# Licensed under the Apache License, Version 2.0 (the "License");
2+
# you may not use this file except in compliance with the License.
3+
# You may obtain a copy of the License at
4+
#
5+
# http://www.apache.org/licenses/LICENSE-2.0
6+
#
7+
# Unless required by applicable law or agreed to in writing, software
8+
# distributed under the License is distributed on an "AS IS" BASIS,
9+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10+
# See the License for the specific language governing permissions and
11+
# limitations under the License.
12+
13+
FROM prestodb/centos7-oj8:latest
14+
LABEL maintainer="Presto community <https://prestodb.io/community.html>"
15+
16+
ARG HADOOP_VERSION=3.4.1
17+
ARG HIVE_VERSION=4.0.1
18+
ARG MYSQL_CONNECTOR_VERSION=8.0.24
19+
ARG AWS_SDK_VERSION=1.12.782
20+
21+
ENV HADOOP_HOME=/opt/hadoop
22+
ENV HIVE_HOME=/opt/hive
23+
ENV HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/tools/lib/*
24+
ENV PATH=${HIVE_HOME}/bin:${HADOOP_HOME}/bin:${PATH}
25+
26+
# Copy configuration files
27+
COPY ./files /tmp/files/
28+
29+
RUN yum install -y \
30+
mariadb-server \
31+
openssh \
32+
openssh-clients \
33+
openssh-server \
34+
psmisc \
35+
which && \
36+
# setup ssh server for sock proxy
37+
ssh-keygen -t rsa -b 4096 -C "[email protected]" -N "" -f /root/.ssh/id_rsa && \
38+
ssh-keygen -t rsa -b 4096 -N "" -f /etc/ssh/ssh_host_rsa_key && \
39+
cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys && \
40+
passwd --unlock root && \
41+
# install hadoop
42+
HADOOP_BINARY_PATH=https://dlcdn.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz && \
43+
curl -o /tmp/hadoop.tar.gz --url $HADOOP_BINARY_PATH && \
44+
tar xzf /tmp/hadoop.tar.gz --directory /opt && mv /opt/hadoop-${HADOOP_VERSION} /opt/hadoop && \
45+
# install hive
46+
HIVE_BINARY_PATH=https://dlcdn.apache.org/hive/hive-$HIVE_VERSION/apache-hive-$HIVE_VERSION-bin.tar.gz && \
47+
curl -o /tmp/hive.tar.gz --url $HIVE_BINARY_PATH && \
48+
tar xzf /tmp/hive.tar.gz --directory /opt && mv /opt/apache-hive-${HIVE_VERSION}-bin /opt/hive && \
49+
# install mysql connector and aws s3 sdk
50+
mkdir /opt/hive/auxlib && \
51+
curl -o /opt/hive/auxlib/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar https://repo1.maven.org/maven2/mysql/mysql-connector-java/$MYSQL_CONNECTOR_VERSION/mysql-connector-java-$MYSQL_CONNECTOR_VERSION.jar && \
52+
curl -o /opt/hive/auxlib/aws-java-sdk-core-$AWS_SDK_VERSION.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-core/$AWS_SDK_VERSION/aws-java-sdk-core-$AWS_SDK_VERSION.jar && \
53+
curl -o /opt/hive/auxlib/aws-java-sdk-s3-$AWS_SDK_VERSION.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-s3/$AWS_SDK_VERSION/aws-java-sdk-s3-$AWS_SDK_VERSION.jar && \
54+
# copy configurations
55+
cp -a /tmp/files/root/* /root && \
56+
cp -a /tmp/files/etc/* /etc && \
57+
cp -a /tmp/files/opt/* /opt && \
58+
chown -R root:root /root && \
59+
chmod 0700 /root /root/.ssh && \
60+
# setup hadoop and hive
61+
/root/setup.sh && \
62+
# cleanup
63+
yum -q clean all && rm -rf /var/cache/yum && rm -rf /tmp/* /var/tmp/*
64+
65+
# HDFS port
66+
EXPOSE 9000 9870
67+
68+
# HIVE Metastore port
69+
EXPOSE 9083 10000
70+
71+
EXPOSE 1180
72+
73+
CMD /root/entrypoint.sh

‎prestodb/hive4.0-hive/files/etc/hadoop-init.d/init-hdfs.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash -x
2+
3+
echo 'N' | hdfs namenode -format
4+
sed -i -e "s|hdfs://localhost|hdfs://$(hostname)|g" /opt/hadoop/etc/hadoop/core-site.xml
5+
hdfs namenode &
6+
sleep 10 && hdfs dfs -mkdir -p /user/hive/warehouse && killall java

‎prestodb/hive4.0-hive/files/etc/hadoop-init.d/set-aws-creds.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash -ex
2+
3+
if [[ -n "${AWS_ACCESS_KEY_ID}" ]]
4+
then
5+
echo "Setting AWS keys"
6+
sed -i -e "s|\"Use AWS_ACCESS_KEY_ID .*\"|${AWS_ACCESS_KEY_ID}|g" \
7+
-e "s|\"Use AWS_SECRET_ACCESS_KEY .*\"|${AWS_SECRET_ACCESS_KEY}|g" \
8+
/opt/hive/conf/hive-site.xml
9+
fi

‎prestodb/hive4.0-hive/files/etc/supervisord.conf‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
[supervisord]
2+
logfile = /var/log/supervisord.log
3+
logfile_maxbytes = 50MB
4+
logfile_backups=10
5+
loglevel = info
6+
pidfile = /var/run/supervisord.pid
7+
nodaemon = true
8+
directory = /tmp
9+
strip_ansi = false
10+
11+
[unix_http_server]
12+
file = /tmp/supervisor.sock
13+
14+
[rpcinterface:supervisor]
15+
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
16+
17+
[supervisorctl]
18+
serverurl = unix:///tmp/supervisor.sock
19+
20+
[include]
21+
files = /etc/supervisord.d/*.conf

‎prestodb/hive4.0-hive/files/etc/supervisord.d/hdfs-datanode.conf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[program:hdfs-datanode]
2+
command=hdfs datanode
3+
startsecs=2
4+
stopwaitsecs=10
5+
user=root
6+
redirect_stderr=true
7+
stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-datanode.log
8+
autostart=true

‎prestodb/hive4.0-hive/files/etc/supervisord.d/hdfs-namenode.conf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[program:hdfs-namenode]
2+
command=hdfs namenode
3+
startsecs=2
4+
stopwaitsecs=10
5+
user=root
6+
redirect_stderr=true
7+
stdout_logfile=/var/log/hadoop-hdfs/hadoop-hdfs-namenode.log
8+
autostart=true

‎prestodb/hive4.0-hive/files/etc/supervisord.d/hive-metastore.conf‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[program:hive-metastore]
2+
# Add `--debug:port=5006` for debugging
3+
command=hive --service metastore
4+
startsecs=2
5+
stopwaitsecs=10
6+
user=root
7+
redirect_stderr=true
8+
stdout_logfile=/var/log/hive/hive-metastore.log
9+
autostart=true

‎prestodb/hive4.0-hive/files/etc/supervisord.d/hive-server2.conf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[program:hive-server2]
2+
command=hive --service hiveserver2
3+
startsecs=2
4+
stopwaitsecs=10
5+
user=root
6+
redirect_stderr=true
7+
stdout_logfile=/var/log/hive/hive-server2.log
8+
autostart=true

‎prestodb/hive4.0-hive/files/etc/supervisord.d/mysql-metastore.conf‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[program:mysql-metastore]
2+
command=/usr/bin/pidproxy /var/run/mysqld/mysqld.pid /usr/bin/mysqld_safe
3+
startsecs=2
4+
stopwaitsecs=10
5+
user=mysql
6+
redirect_stderr=true
7+
stdout_logfile=/var/log/mysql/mysql.log
8+
autostart=true

0 commit comments

Comments
 (0)