Skip to content

Commit bd82007

Browse files
gprasad09gprasad09
committed
Revised version on 03/1/42023
1 parent 3464eb2 commit bd82007

16 files changed

+283
-16
lines changed

Diff for: ‎config_files/bashrc

+103
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# ~/.bashrc: executed by bash(1) for non-login shells.
2+
# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc)
3+
# for examples
4+
5+
# If not running interactively, don't do anything
6+
[ -z "$PS1" ] && return
7+
8+
# don't put duplicate lines in the history. See bash(1) for more options
9+
# ... or force ignoredups and ignorespace
10+
#Add Spark HOME for pyspark
11+
export SPARK_HOME=/usr/local/spark
12+
export PATH=$SPARK_HOME/bin:$PATH
13+
14+
HISTCONTROL=ignoredups:ignorespace
15+
16+
# append to the history file, don't overwrite it
17+
shopt -s histappend
18+
19+
# for setting history length see HISTSIZE and HISTFILESIZE in bash(1)
20+
HISTSIZE=1000
21+
HISTFILESIZE=2000
22+
23+
# check the window size after each command and, if necessary,
24+
# update the values of LINES and COLUMNS.
25+
shopt -s checkwinsize
26+
27+
# make less more friendly for non-text input files, see lesspipe(1)
28+
[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)"
29+
30+
# set variable identifying the chroot you work in (used in the prompt below)
31+
if [ -z "$debian_chroot" ] && [ -r /etc/debian_chroot ]; then
32+
debian_chroot=$(cat /etc/debian_chroot)
33+
fi
34+
35+
# set a fancy prompt (non-color, unless we know we "want" color)
36+
case "$TERM" in
37+
xterm-color) color_prompt=yes;;
38+
esac
39+
40+
# uncomment for a colored prompt, if the terminal has the capability; turned
41+
# off by default to not distract the user: the focus in a terminal window
42+
# should be on the output of commands, not on the prompt
43+
#force_color_prompt=yes
44+
45+
if [ -n "$force_color_prompt" ]; then
46+
if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then
47+
# We have color support; assume it's compliant with Ecma-48
48+
# (ISO/IEC-6429). (Lack of such support is extremely rare, and such
49+
# a case would tend to support setf rather than setaf.)
50+
color_prompt=yes
51+
else
52+
color_prompt=
53+
fi
54+
fi
55+
56+
if [ "$color_prompt" = yes ]; then
57+
PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
58+
else
59+
PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ '
60+
fi
61+
unset color_prompt force_color_prompt
62+
63+
# If this is an xterm set the title to user@host:dir
64+
case "$TERM" in
65+
xterm*|rxvt*)
66+
PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
67+
;;
68+
*)
69+
;;
70+
esac
71+
72+
# enable color support of ls and also add handy aliases
73+
if [ -x /usr/bin/dircolors ]; then
74+
test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
75+
alias ls='ls --color=auto'
76+
#alias dir='dir --color=auto'
77+
#alias vdir='vdir --color=auto'
78+
79+
alias grep='grep --color=auto'
80+
alias fgrep='fgrep --color=auto'
81+
alias egrep='egrep --color=auto'
82+
fi
83+
84+
# some more ls aliases
85+
alias ll='ls -alF'
86+
alias la='ls -A'
87+
alias l='ls -CF'
88+
89+
# Alias definitions.
90+
# You may want to put all your additions into a separate file like
91+
# ~/.bash_aliases, instead of adding them here directly.
92+
# See /usr/share/doc/bash-doc/examples in the bash-doc package.
93+
94+
if [ -f ~/.bash_aliases ]; then
95+
. ~/.bash_aliases
96+
fi
97+
98+
# enable programmable completion features (you don't need to enable
99+
# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
100+
# sources /etc/bash.bashrc).
101+
#if [ -f /etc/bash_completion ] && ! shopt -oq posix; then
102+
# . /etc/bash_completion
103+
#fi

Diff for: ‎config_files/bashrc.tmp

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
# ~/.bashrc: executed by bash(1) for non-login shells.
2+
# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc)
3+
# for examples
4+
5+
# If not running interactively, don't do anything
6+
[ -z "$PS1" ] && return
7+
8+
# don't put duplicate lines in the history. See bash(1) for more options
9+
# ... or force ignoredups and ignorespace
10+
HISTCONTROL=ignoredups:ignorespace
11+
12+
# append to the history file, don't overwrite it
13+
shopt -s histappend
14+
15+
# for setting history length see HISTSIZE and HISTFILESIZE in bash(1)
16+
HISTSIZE=1000
17+
HISTFILESIZE=2000
18+
19+
# check the window size after each command and, if necessary,
20+
# update the values of LINES and COLUMNS.
21+
shopt -s checkwinsize
22+
23+
# make less more friendly for non-text input files, see lesspipe(1)
24+
[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)"
25+
26+
# set variable identifying the chroot you work in (used in the prompt below)
27+
if [ -z "$debian_chroot" ] && [ -r /etc/debian_chroot ]; then
28+
debian_chroot=$(cat /etc/debian_chroot)
29+
fi
30+
31+
# set a fancy prompt (non-color, unless we know we "want" color)
32+
case "$TERM" in
33+
xterm-color) color_prompt=yes;;
34+
esac
35+
36+
# uncomment for a colored prompt, if the terminal has the capability; turned
37+
# off by default to not distract the user: the focus in a terminal window
38+
# should be on the output of commands, not on the prompt
39+
#force_color_prompt=yes
40+
41+
if [ -n "$force_color_prompt" ]; then
42+
if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then
43+
# We have color support; assume it's compliant with Ecma-48
44+
# (ISO/IEC-6429). (Lack of such support is extremely rare, and such
45+
# a case would tend to support setf rather than setaf.)
46+
color_prompt=yes
47+
else
48+
color_prompt=
49+
fi
50+
fi
51+
52+
if [ "$color_prompt" = yes ]; then
53+
PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
54+
else
55+
PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ '
56+
fi
57+
unset color_prompt force_color_prompt
58+
59+
# If this is an xterm set the title to user@host:dir
60+
case "$TERM" in
61+
xterm*|rxvt*)
62+
PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
63+
;;
64+
*)
65+
;;
66+
esac
67+
68+
# enable color support of ls and also add handy aliases
69+
if [ -x /usr/bin/dircolors ]; then
70+
test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
71+
alias ls='ls --color=auto'
72+
#alias dir='dir --color=auto'
73+
#alias vdir='vdir --color=auto'
74+
75+
alias grep='grep --color=auto'
76+
alias fgrep='fgrep --color=auto'
77+
alias egrep='egrep --color=auto'
78+
fi
79+
80+
# some more ls aliases
81+
alias ll='ls -alF'
82+
alias la='ls -A'
83+
alias l='ls -CF'
84+
85+
# Alias definitions.
86+
# You may want to put all your additions into a separate file like
87+
# ~/.bash_aliases, instead of adding them here directly.
88+
# See /usr/share/doc/bash-doc/examples in the bash-doc package.
89+
90+
if [ -f ~/.bash_aliases ]; then
91+
. ~/.bash_aliases
92+
fi
93+
94+
# enable programmable completion features (you don't need to enable
95+
# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
96+
# sources /etc/bash.bashrc).
97+
#if [ -f /etc/bash_completion ] && ! shopt -oq posix; then
98+
# . /etc/bash_completion
99+
#fi
100+
101+
#Modified by Spark Config
102+
#add Spark HOME for pyspark
103+
export SPARK_HOME=/usr/local/spark
104+
Export PATH=$SPARK_HOME/bin:$PATH
105+
106+
#Modified by Spark Config
107+
#add Spark HOME for pyspark
108+
export SPARK_HOME=/usr/local/spark
109+
Export PATH=$SPARK_HOME/bin:$PATH

Diff for: ‎config_files/pyspark.txt

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
2+
#Modified by Spark Config
3+
#add Spark HOME for pyspark
4+
export SPARK_HOME=/usr/local/spark
5+
Export PATH=$SPARK_HOME/bin:$PATH

Diff for: ‎config_files/slaves

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
2+
sp-manager
3+
4+
sp-worker1
5+
6+
sp-worker2
7+
8+
sp-worker3
9+
10+
sp-worker4
11+
12+
sp-worker5
13+
14+
sp-worker6

Diff for: ‎docker_files/docker-compose_master.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ services:
1616
#command: ./sbin/stop-all.sh && ./sbin/start-all.sh
1717
volumes:
1818
#- ApacheSpark-vol:/usr/share/ApacheSpark/shared #mount type =volume
19-
- ./data:/usr/share/ApacheSpark/data #mount type =bind
19+
- ~/data:/usr/share/ApacheSpark/data #mount type =bind
2020
worker1:
2121
container_name: "sp-worker5"
2222
image: "gprasad09/spark_withuser:latest"
@@ -44,4 +44,4 @@ services:
4444
ports:
4545
- "8083:8081"
4646
#volumes:
47-
# ApacheSpark-vol: #mount type =volume
47+
# ApacheSpark-vol: #mount type =volume

Diff for: ‎docker_files/docker-stack.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ services:
1717
#command: ./sbin/stop-all.sh && ./sbin/start-all.sh
1818
volumes:
1919
#- ApacheSpark-vol:/usr/share/ApacheSpark/shared #mount type =volume
20-
- ./data:/usr/share/ApacheSpark/data #mount type =bind
20+
- ~/data:/usr/share/ApacheSpark/data #mount type =bind with a folder 'data' created on home directory of host machine
2121
deploy:
2222
mode: replicated
2323
replicas: 1
@@ -45,4 +45,4 @@ services:
4545
limits:
4646
cpus: "4"
4747
restart_policy:
48-
condition: on-failure
48+
condition: on-failure

Diff for: ‎sh_scripts/scripts_compose/start.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
#!bin/bash
1+
!bin/bash
22

3-
docker compose start
3+
docker compose -f ~/Apache-Spark-Cluster-Project/docker_files/docker-compose_master.yml start
44

5-
sh_scripts/start_spark.sh
5+
~/Apache-Spark-Cluster-Project/sh_scripts/scripts_compose/start_spark.sh

Diff for: ‎sh_scripts/scripts_compose/stop.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!bin/bash
22

3-
sh_scripts/stop_spark.sh
3+
~/Apache-Spark-Cluster-Project/sh_scripts/scripts_compose/stop_spark.sh
44

5-
docker compose stop
5+
docker compose -f ~/Apache-Spark-Cluster-Project/docker_files/docker-compose_master.yml stop
66

Diff for: ‎sh_scripts/scripts_swarm/build_stack.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/bash
2+
3+
docker stack deploy -c ~/Apache-Spark-Cluster-Project/docker_files/docker-stack.yml sparkstack #Build stack which create service and overlay network

Diff for: ‎sh_scripts/scripts_swarm/get_manager_id.sh

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
3+
manager=$(docker ps --filter "name=sparkstack_manager.1" -q )
4+
5+
echo $manager
6+

Diff for: ‎sh_scripts/scripts_swarm/prepare_master.sh

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
cd ~/Apache-Spark-Cluster-Project/config_files
4+
5+
manager=$(./get_manager_id.sh) #get container id of master
6+
7+
docker cp $manager:/root/.bashrc ./bashrc.tmp #copy ~/.bashrc from the container to host machine
8+
9+
cat pyspark.txt>>bashrc.tmp #append configuration lines for pyspark setting
10+
11+
docker cp ./bashrc.tmp $manager:/root/.bashrc #restore updated version of ~/.bashrc to master container

Diff for: ‎sh_scripts/scripts_swarm/star_spark.sh

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
3+
manager=$(docker ps --filter "name=sparkstack_manager.1" -q )
4+
5+
docker exec -it $manager sh -c "./sbin/stop-all.sh && ./sbin/start-all.sh"
6+

Diff for: ‎sh_scripts/scripts_swarm/start_spark.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
docker stack deploy -c docker-stack.yml sparkstack #Build stack which create service and overlay network
1+
#!/bin/bash
22

3-
manager=$(docker ps --filter "name=sparkstack_manager.1" -q )
3+
manager=$(./get_manager_id.sh)
44

5-
docker exec -it $manager sh -c "./sbin/stop-all.sh && ./sbin/start-all.sh"
5+
docker exec -it $manager sh -c "./sbin/stop-all.sh && ./sbin/start-all.sh"

Diff for: ‎sh_scripts/scripts_swarm/stop_spark.sh

+6
Original file line numberDiff line numberDiff line change
@@ -1 +1,7 @@
1+
#!/bin/bash
2+
3+
manager=$(./get_manager_id.sh)
4+
15
docker exec -it $manager sh -c "./sbin/stop-all.sh"
6+
7+
docker stack rm sparkstack

Diff for: ‎start.sh

+6-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1-
!bin/bash
1+
#!/bin/bash
22

3-
sh_scripts/scripts_swarm/start_spark.sh
3+
./sh_scripts/scripts_swarm/build_stack.sh
4+
5+
sleep 3 #wait till containers are created
6+
7+
./sh_scripts/scripts_swarm/start_spark.sh

Diff for: ‎stop.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!bin/bash
1+
#!/bin/bash
22

3-
sh_scripts/scripts_swarm/stop_spark.sh
3+
./sh_scripts/scripts_swarm/stop_spark.sh
44

0 commit comments

Comments
 (0)