lxdctl

#!/bin/bash
OS=el7
ARCH=amd64
OS_TYPE=rpm
IMG="images:centos/7/$ARCH"
PACKAGES=''
PYTHON_INT=/usr/bin/python2.7
NUM_NODES=3
DESTROY=0
LXD_PRIVILEGED_CONTAINERS=''
NAMESPACE=''
SNAPSHOT=''


getip() {
  NODE="$1"
  [ "x$NODE" = "xnode0" ] && NODE=default
  IP="$(sed -ne '/\<'$NODE'\>/ {s/^.*ansible_host=//;s/ .*$//;p}' ${NAMESPACE}ansible_hosts 2>/dev/null | head -n 1)"
  if [ "x$IP" = "x" ] ; then
    IP="$(sed -ne '/\<'$NODE'\>/ {s/ .*$//;p}' configs/${NAMESPACE}hosts 2>/dev/null|head -n 1)"
  fi
}

getcontainer() {
  NODE="$1"
  [ "x$NODE" = "xnode0" ] && NODE=default
  C="$NAMESPACE$(sed -rne '/\<'$NODE'\>/ {s/^([^ ]+)[ ].*$/\1/;p}' ${NAMESPACE}ansible_hosts 2>/dev/null)"
  CONTAINER=${C//./-}
}


setupOS() {
  if [ "x$1" == "xel8" ] ; then
    IMG="images:centos/8/$ARCH"
    PACKAGES="$PACKAGES python3 tar"
    PYTHON_INT=/usr/bin/python3
    OS_TYPE=rpm
  elif [ "x$1" == "xoel7" ] ; then
    IMG="images:oracle/7/$ARCH"
    PACKAGES=''
    PYTHON_INT=/usr/bin/python2.7
    OS_TYPE=rpm
  elif [ "x$1" == "xoel8" ] ; then
    IMG="images:oracle/8/$ARCH"
    PACKAGES="$PACKAGES oracle-epel-release-el8 python3 tar"
    PYTHON_INT=/usr/bin/python3
    OS_TYPE=rpm
  elif [ "x$1" == "xxenial" ] ; then
    IMG="images:ubuntu/xenial/$ARCH"
    PYTHON_INT=/usr/bin/python3
    OS_TYPE=deb
  elif [ "x$1" == "xbionic" ] ; then
    IMG="images:ubuntu/bionic/$ARCH"
    PYTHON_INT=/usr/bin/python3
    OS_TYPE=deb
  elif [ "x$1" == "xfocal" ] ; then
    IMG="images:ubuntu/focal/$ARCH"
    PYTHON_INT=/usr/bin/python3
    OS_TYPE=deb
  elif [ "x$1" == "xstretch" ] ; then
    IMG="images:debian/9/$ARCH"
    PYTHON_INT=/usr/bin/python3
    PACKAGES="$PACKAGES python3"
    OS_TYPE=deb
  elif [ "x$1" == "xbuster" ] ; then
    IMG="images:debian/10/$ARCH"
    PYTHON_INT=/usr/bin/python3
    PACKAGES="$PACKAGES python3"
    OS_TYPE=deb
  else # default CentOS 7
    IMG="images:centos/7/$ARCH"
    PACKAGES=''
    PYTHON_INT=/usr/bin/python2.7
    OS_TYPE=rpm
  fi
}

if [ "x$1" = "x--namespace" ] ;then
      NAMESPACE="$2-"
      shift 2
fi

if [ "x$1" = "xssh" ] ; then
  shift
  getip "$1"
  shift
  if [ "x$IP" = "x" ] ; then
    echo "No such host, try to deploy or add it"
    exit 1
  fi
  [ -t 0 -a -t 1 ] && INTERACTIVE=-t 
  exec ssh -F ${NAMESPACE}ssh_config -o LogLevel=error -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i secret/id_rsa $INTERACTIVE root@$IP "$@"
  exit 0
fi
if [ "x$1" = "xscp" ] ; then
  shift
  getip "$1"
  shift
  exec scp ${NAMESPACE}ssh_config -o LogLevel=error -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i secret/id_rsa root@$IP:"$1" "$2"
fi
if [ "x$1" = "xip" ] ; then
  shift
  getip "$1"
  shift
  if [ "x$IP" = "x" ] ; then
    exit 1
  fi
  echo $IP
  exit 0
fi


# read arguments
opts=$(getopt \
    --longoptions "pmm:,pmm-port:,os:,destroy,k8s,samba:,nodes:,hostname:,vm:,mem:,cpu:,only-node:,privileged,namespace:,snapshot:,cache:,has-cache:,fix-ip" \
    --name "$(basename "$0")" \
    --options "" \
    -- "$@"
)

declare -A HOSTNAMES
declare -A OSes
declare -A VMs
declare -A MEM_LIMITs
declare -A CPU_LIMITs
declare -A CACHE_IMAGES

eval set --$opts
while [[ $# -gt 0 ]]; do
  case "$1" in
    --pmm)
      PMM=$2
      shift 2
      ;;
    --nodes)
      NUM_NODES=$2
      shift 2
      ;;
    --only-node)
      ONLY_NODE=$2
      shift 2
      ;;
    --namespace)
      NAMESPACE="$2-"
      shift 2
      ;;
    --pmm-port)
      PMM_PORT=$2
      shift 2
      ;;
    --os)
      NODE_NAME=$(echo "$2"|cut -d= -f 1)
      NODE_OS=$(echo "$2"|cut -d= -f 2)
      if [[ "x$NODE_OS" != "x" && "$xNODE_NAME" != "x" && "$2" == *=* ]] ; then
        OSes[$USER.$NODE_NAME]="$NODE_OS"
      else
        OS=$2
      fi

      shift 2
      ;;
    --cache)
      NODE_NAME=$(echo "$2"|cut -d= -f 1)
      NODE_CACHE_IMG=$(echo "$2"|cut -d= -f 2)
      if [[ "x$NODE_NAME" != "x" && "$xNODE_CACHE_IMG" != "x" && "$2" == *=* ]] ; then
        CACHE_IMAGES[$USER.$NODE_NAME]="$NODE_CACHE_IMG"
      fi

      shift 2
      ;;
    --has-cache)
      HAS_CACHE_CHECK=$2
      shift 2
      ;;

    --vm)
      NODE_NAME="$2"
      VMs[$USER.$NODE_NAME]="--vm"
      shift 2
      ;;
    --mem)
      NODE_NAME=$( echo "$2" | cut -d= -f 1)
      NODE_RAM=$( echo "$2" | cut -d= -f 2)
      MEM_LIMITs[$USER.$NODE_NAME]="-c limits.memory=$NODE_RAM"
      shift 2
      ;;
    --cpu)
      NODE_NAME=$( echo "$2" | cut -d= -f 1)
      NODE_CPU=$( echo "$2" | cut -d= -f 2)
      CPU_LIMITs[$USER.$NODE_NAME]="-c limits.cpu=$NODE_CPU"
      shift 2
      ;;
    --k8s)
      K8S=1
      shift
      ;;
    --samba)
      SAMBA_NODE=$2
      shift 2
      ;;
    --snapshot)
      SNAPSHOT=$2
      shift 2
      ;;
    --hostname)
      NODE_NAME=$(echo "$2"|cut -d= -f 1)
      NODE_HOST=$(echo "$2"|cut -d= -f 2)
      HOSTNAMES[$USER.$NODE_NAME]="$NODE_HOST"
      shift 2
      ;;
    --privileged)
      LXD_PRIVILEGED_CONTAINERS='-c security.nesting=true -c security.privileged=true'
      shift
      ;;
    --destroy)
      DESTROY=1
      shift
      ;;
    --fix-ip)
      FIX_IP=1
      shift
      ;;
      *)
      break
      ;;
  esac
done

if [ "x$1" = "x--" ] ; then
  shift
fi

if [ "x$HAS_CACHE_CHECK" != "x" ] ; then
  if lxc image list --format yaml ${USER}-${OS}-$HAS_CACHE_CHECK | grep -q ${USER}-${OS}-$HAS_CACHE_CHECK ; then
    exit 0
  fi
  exit 1
fi

if [ "x$SNAPSHOT" != "x" ] ; then
  i="$NAMESPACE$USER.$1"
  c=${i//./-}
  lxc stop $c
  lxc publish $c --alias ${USER}-${OS}-${SNAPSHOT}
  lxc start $c

  exit 0
fi

if [ "x$FIX_IP" != "x" ] ; then
  if [ ! -f ${NAMESPACE}ansible_hosts ]  ; then
    echo "Not deployed"
    exit 1
  fi
  for NODE in $(awk '{print $1}' ${NAMESPACE}ansible_hosts |rev|cut -d. -f 1|rev) ; do
    C="$NAMESPACE$USER.$NODE";
    CONTAINER="${C//./-}";
    CIP=$( lxc exec "$CONTAINER" node_ip.sh 2>/dev/null )
    [ "x$CIP" == "x" ] && continue

    getip "$NODE"
    [ "x$IP" = "x" ] && continue

    if [[ "$IP" != "$CIP" ]] ; then
      if [[ "$NODE" == default ]] ; then
        sed -i -re '/\.(default|node0)\>/{s/ansible_host=\S+/ansible_host='$CIP'/}' ${NAMESPACE}ansible_hosts
      else
        sed -i -re '/\.('$NODE')\>/{s/ansible_host=\S+/ansible_host='$CIP'/}' ${NAMESPACE}ansible_hosts
      fi
      sed -i -e "s/$IP/$CIP/" configs/${NAMESPACE}hosts
      echo "Fixed IP for $NODE: $CIP" ;
    fi
  done
  exit 0
fi

if [ "x$1" = "xlist" ] ; then
  lxc ls ${NAMESPACE//./-}${USER//./-}
  exit 0
fi
if [ "x$1" = "xlist-caches" ] ; then
  lxc image ls local: $USER
  exit 0
fi

if [ "x$1" = "xdestroy" ] || [ "x$1" = "x--destroy" ] || [ $DESTROY -eq 1 ] ; then
  [ $DESTROY -eq 1 ] || shift
  if [ "x$1" = "x" ] ; then
    # delete everything by default
    lxc ls ${NAMESPACE//./-}${USER//./-} --format=yaml|grep "name: ${NAMESPACE//./-}${USER//./-}"|awk -F: '{print $2}'|xargs --no-run-if-empty lxc delete --force
    rm -f ${NAMESPACE}ansible_hosts configs/${NAMESPACE}hosts ${NAMESPACE}ssh_config
  else
    while [[ $# -gt 0 ]]; do
      i="$NAMESPACE$USER.$1"
      shift
      c=${i//./-}
      lxc delete --force "$c"
      sed -i -e "/$i/d" ${NAMESPACE}ansible_hosts
      tr "\n" "\r" < ${NAMESPACE}ssh_config  | sed -e "s/Host ${i/$USER./}[^0-9].*id_rsa//" | tr "\r" "\n" > ${NAMESPACE}ssh_config_
      mv ${NAMESPACE}ssh_config_ ${NAMESPACE}ssh_config
    done
  fi
  exit 0
fi

if [ "x$1" = "xmount" ] ; then
  HOSTDIR="$2"
  NODE=$(echo "$3" | cut -d: -f 1 )
  DST=$(echo "$3" | cut -d: -f 2- )
  getcontainer "$NODE"

  lxc config device add $CONTAINER shared_dir disk source="$HOSTDIR" path="$DST"
  exit 0
fi



if [ "x$LXD_PROFILE" != "x" ] ; then
  LXD_PROFILE="--profile $LXD_PROFILE"
fi

if [ "x$ONLY_NODE" != "x" ] ; then
  NUM_NODES=20
else
  :> ${NAMESPACE}ansible_hosts
  :> configs/${NAMESPACE}hosts
fi

test -f secret/id_rsa || ssh-keygen -t rsa -f secret/id_rsa -P '' && chmod 0600 secret/id_rsa
test -f secret/rs0-keyfile || openssl rand -base64 756 > secret/rs0-keyfile

N=0
for i in ${USER}.default $(seq 1 $(($NUM_NODES-1))|sed -e s/^/${USER}.node/); do
  if [ "x$ONLY_NODE" != "x" ] && [ "x${USER}.$ONLY_NODE" != "x$i" ] ; then
      continue
  fi

  if [ "x${OSes[$i]}" != "x" ] ; then
    setupOS ${OSes[$i]} 
    NODE_OS=${OSes[$i]}
  else
    setupOS $OS
    NODE_OS=$OS
  fi

  c=$NAMESPACE${i//./-}
  if [ "x${VMs[$i]}" != "x" ] ; then
    # set disk size to 10G
    lxc profile device set default root size $((10*1024*1024*1024))B
    CONT_PRIV=' -c security.secureboot=false '
  else
    CONT_PRIV="$LXD_PRIVILEGED_CONTAINERS"
  fi
  if [ "x${CACHE_IMAGES[$i]}" != "x" ] && lxc image list --format yaml ${USER}-${NODE_OS}-${CACHE_IMAGES[$i]} | grep -q ${USER}-${NODE_OS}-${CACHE_IMAGES[$i]} ; then
    lxc launch $LXD_PROFILE ${USER}-${NODE_OS}-${CACHE_IMAGES[$i]} $c $CONT_PRIV ${VMs[$i]} ${MEM_LIMITs[$i]} ${CPU_LIMITs[$i]}
    until lxc exec $c true ; do sleep 2; done
  elif [[ $ANYDBVER_CACHE_OS_IMG == 1 ]] && lxc image list --format yaml ${USER}-${NODE_OS}-empty | grep -q ${USER}-${NODE_OS}-empty ; then
    lxc launch $LXD_PROFILE ${USER}-${NODE_OS}-empty $c $CONT_PRIV ${VMs[$i]} ${MEM_LIMITs[$i]} ${CPU_LIMITs[$i]}
    until lxc exec $c true ; do sleep 2; done
  else
    lxc launch $LXD_PROFILE $IMG $c $CONT_PRIV ${VMs[$i]} ${MEM_LIMITs[$i]} ${CPU_LIMITs[$i]}
    until lxc exec $c true ; do sleep 2; done
    lxc exec $c -- mkdir /root/.ssh
    lxc file push secret/id_rsa.pub $c/root/.ssh/authorized_keys
    lxc file push tools/node_ip.sh $c/usr/bin/node_ip.sh
    lxc file push tools/fix_el8_ip.sh $c/usr/bin/fix_el8_ip.sh
    if [[ $OS_TYPE = deb ]] ; then
       lxc exec $c -- bash -c "export LANG=C;test -f /usr/bin/rsync || (until ping  -c 1 1.1.1.1 &>/dev/null ; do sleep 1; done; apt-get update > /dev/null; for pkg in sudo openssh-server rsync eatmydata curl $PACKAGES; do apt-get install -qq -o=Dpkg::Use-Pty=0 -y \$pkg &> /dev/null ; done); chmod -R og-rwx /root/.ssh;sed -i -e 's/#UseDNS yes/UseDNS no/' -e 's/#PermitRootLogin.*$/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config;sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd; chmod 0600 /root/.ssh/authorized_keys; chmod 0700 /root/.ssh; chown root:root -R /root/.ssh"
      lxc exec $c -- bash -c "systemctl enable ssh;systemctl restart ssh &>/dev/null"
    else
      lxc exec $c -- bash -c "export LANG=C;bash /usr/bin/fix_el8_ip.sh;test -f /usr/bin/rsync || (until ping  -c 1 1.1.1.1 &>/dev/null ; do sleep 1; done; for pkg in sudo openssh-server iproute rsync $PACKAGES; do yum install -q -y \$pkg ; done); chmod -R og-rwx /root/.ssh;sed -i -e 's/#UseDNS yes/UseDNS no/' -e 's/#PermitRootLogin.*$/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config;sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd; chmod 0600 /root/.ssh/authorized_keys; chmod 0700 /root/.ssh; chown root:root -R /root/.ssh; yum install -q -y https://repo.percona.com/yum/release/7/RPMS/x86_64/libeatmydata-0.1-00.21.el7.centos.x86_64.rpm"
      lxc exec $c -- bash -c "systemctl enable sshd;systemctl restart sshd"
    fi
    if [[ $ANYDBVER_CACHE_OS_IMG == 1 ]] ; then
      lxc stop $c
      lxc publish $c --alias ${USER}-${NODE_OS}-empty
      lxc start $c
    fi
  fi

  ((N=N+1))
  if [ $N -eq $NUM_NODES ] ; then break ; fi
done

for i in ${USER}.default $(seq 1 $(($NUM_NODES-1))|sed -e s/^/${USER}.node/); do
  if [ "x$ONLY_NODE" != "x" ] && [ "x${USER}.$ONLY_NODE" != "x$i" ] ; then
      continue
  fi
  if [ "x${OSes[$i]}" != "x" ] ; then
    setupOS ${OSes[$i]} 
    NODE_OS=${OSes[$i]}
  else
    setupOS $OS
    NODE_OS=$OS
  fi

  c=$NAMESPACE${i//./-}
  IP='127.0.0.1'
  while [ "x$IP" = 'x127.0.0.1' ] ; do
    IP=$(lxc exec $c -- /bin/bash /usr/bin/node_ip.sh)
  done
  echo "$i ansible_connection=ssh ansible_user=root ansible_ssh_private_key_file=secret/id_rsa ansible_host=$IP ansible_python_interpreter=$PYTHON_INT ansible_ssh_common_args='-o StrictHostKeyChecking=no'" >> ${NAMESPACE}ansible_hosts

  if [ "x$i" = "x${USER}.default" ] ; then
    echo "${USER}.node0 ansible_connection=ssh ansible_user=root ansible_ssh_private_key_file=secret/id_rsa ansible_host=$IP ansible_python_interpreter=$PYTHON_INT ansible_ssh_common_args='-o StrictHostKeyChecking=no'" >> ${NAMESPACE}ansible_hosts
  fi

  if [ "x${HOSTNAMES[$i]}" != "x" ] ; then
    lxc exec $c -- hostnamectl set-hostname "${HOSTNAMES[$i]}"
  fi

  if [ "x${HOSTNAMES[$i]}" != "x" ] ; then
    SHORT_NAME=$(echo "${HOSTNAMES[$i]}" | cut -d. -f 1)
    echo "$IP ${HOSTNAMES[$i]} $SHORT_NAME" >> configs/${NAMESPACE}hosts
  fi

  cat >> ${NAMESPACE}ssh_config <<EOF
Host ${i/$USER./} ${HOSTNAMES[$i]} $SHORT_NAME
   User root
   HostName $IP
   StrictHostKeyChecking no
   UserKnownHostsFile /dev/null
   IdentityFile $PWD/secret/id_rsa
EOF

done


for i in ${USER}.default $(seq 1 $(($NUM_NODES-1))|sed -e s/^/${USER}.node/); do
  if [ "x$ONLY_NODE" != "x" ] && [ "x${USER}.$ONLY_NODE" != "x$i" ] ; then
      continue
  fi
  i="$NAMESPACE$i"
  c=${i//./-}
  lxc exec $c -- bash -c "cat >> /etc/hosts" < configs/${NAMESPACE}hosts
done