From c735f031051f6972ed201ce644e6bfd839ff7458 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 09:10:03 -0400 Subject: [PATCH 01/48] Strip out argyle/provisioning pieces of the fabfile. --- fabfile.py | 137 ++----------------------------------------- requirements/dev.txt | 4 +- 2 files changed, 6 insertions(+), 135 deletions(-) diff --git a/fabfile.py b/fabfile.py index fd5bf25..f7b63eb 100644 --- a/fabfile.py +++ b/fabfile.py @@ -1,15 +1,8 @@ -import ConfigParser import os import re -from argyle import rabbitmq, postgres, nginx, system -from argyle.base import upload_template -from argyle.postgres import create_db_user, create_db -from argyle.supervisor import supervisor_command, upload_supervisor_app_conf -from argyle.system import service_command, start_service, stop_service, restart_service - from fabric.api import cd, env, get, hide, local, put, require, run, settings, sudo, task -from fabric.contrib import files, console +from fabric.contrib import files # Directory structure PROJECT_ROOT = os.path.dirname(__file__) @@ -20,14 +13,9 @@ env.repo = u'' # FIXME: Add repo URL env.shell = '/bin/bash -c' env.disable_known_hosts = True -env.ssh_port = 2222 +env.port = 2222 env.forward_agent = True -# Additional settings for argyle -env.ARGYLE_TEMPLATE_DIRS = ( - os.path.join(CONF_ROOT, 'templates') -) - @task def vagrant(): @@ -60,130 +48,15 @@ def setup_path(): env.home = '/home/%(project_user)s/' % env env.root = os.path.join(env.home, 'www', env.environment) env.code_root = os.path.join(env.root, env.project) - env.project_root = os.path.join(env.code_root, env.project) env.virtualenv_root = os.path.join(env.root, 'env') - env.log_dir = os.path.join(env.root, 'log') env.db = '%s_%s' % (env.project, env.environment) - env.vhost = '%s_%s' % (env.project, env.environment) env.settings = '%(project)s.settings.%(environment)s' % env @task -def create_users(): - """Create project user and developer users.""" - ssh_dir = u"/home/%s/.ssh" % env.project_user - system.create_user(env.project_user, groups=['www-data', 'login', ]) - sudo('mkdir -p %s' % ssh_dir) - user_dir = os.path.join(CONF_ROOT, "users") - for username in os.listdir(user_dir): - key_file = os.path.normpath(os.path.join(user_dir, username)) - system.create_user(username, groups=['dev', 'login', ], key_file=key_file) - with open(key_file, 'rt') as f: - ssh_key = f.read() - # Add ssh key for project user - files.append('%s/authorized_keys' % ssh_dir, ssh_key, use_sudo=True) - files.append(u'/etc/sudoers', r'%dev ALL=(ALL) NOPASSWD:ALL', use_sudo=True) - sudo('chown -R %s:%s %s' % (env.project_user, env.project_user, ssh_dir)) - - -@task -def configure_ssh(): - """ - Change sshd_config defaults: - Change default port - Disable root login - Disable password login - Restrict to only login group - """ - ssh_config = u'/etc/ssh/sshd_config' - files.sed(ssh_config, u"Port 22$", u"Port %s" % env.ssh_port, use_sudo=True) - files.sed(ssh_config, u"PermitRootLogin yes", u"PermitRootLogin no", use_sudo=True) - files.append(ssh_config, u"AllowGroups login", use_sudo=True) - files.append(ssh_config, u"PasswordAuthentication no", use_sudo=True) - service_command(u'ssh', u'reload') - - -@task -def install_packages(*roles): - """Install packages for the given roles.""" - config_file = os.path.join(CONF_ROOT, u'packages.conf') - config = ConfigParser.SafeConfigParser() - config.read(config_file) - for role in roles: - if config.has_section(role): - # Get ppas - if config.has_option(role, 'ppas'): - for ppa in config.get(role, 'ppas').split(' '): - system.add_ppa(ppa, update=False) - # Get sources - if config.has_option(role, 'sources'): - for section in config.get(role, 'sources').split(' '): - source = config.get(section, 'source') - key = config.get(section, 'key') - system.add_apt_source(source=source, key=key, update=False) - sudo(u"apt-get update") - sudo(u"apt-get install -y %s" % config.get(role, 'packages')) - sudo(u"apt-get upgrade -y") - - -@task -def setup_server(*roles): - """Install packages and add configurations for server given roles.""" - require('environment') - # Set server locale - sudo('/usr/sbin/update-locale LANG=en_US.UTF-8') - roles = list(roles) - if roles == ['all', ]: - roles = SERVER_ROLES - if 'base' not in roles: - roles.insert(0, 'base') - install_packages(*roles) - if 'db' in roles: - if console.confirm(u"Do you want to reset the Postgres cluster?.", default=False): - # Ensure the cluster is using UTF-8 - pg_version = postgres.detect_version() - sudo('pg_dropcluster --stop %s main' % pg_version, user='postgres') - sudo('pg_createcluster --start -e UTF-8 --locale en_US.UTF-8 %s main' % pg_version, - user='postgres') - postgres.create_db_user(username=env.project_user) - postgres.create_db(name=env.db, owner=env.project_user) - if 'app' in roles: - # Create project directories and install Python requirements - project_run('mkdir -p %(root)s' % env) - project_run('mkdir -p %(log_dir)s' % env) - # FIXME: update to SSH as normal user and use sudo - # we ssh as the project_user here to maintain ssh agent - # forwarding, because it doesn't work with sudo. read: - # http://serverfault.com/questions/107187/sudo-su-username-while-keeping-ssh-key-forwarding - with settings(user=env.project_user): - # TODO: Add known hosts prior to clone. - # i.e. ssh -o StrictHostKeyChecking=no git@github.com - run('git clone %(repo)s %(code_root)s' % env) - with cd(env.code_root): - run('git checkout %(branch)s' % env) - # Install and create virtualenv - with settings(hide('everything'), warn_only=True): - test_for_pip = run('which pip') - if not test_for_pip: - sudo("easy_install -U pip") - with settings(hide('everything'), warn_only=True): - test_for_virtualenv = run('which virtualenv') - if not test_for_virtualenv: - sudo("pip install -U virtualenv") - project_run('virtualenv -p python2.6 --clear --distribute %s' % env.virtualenv_root) - path_file = os.path.join(env.virtualenv_root, 'lib', 'python2.6', 'site-packages', 'project.pth') - files.append(path_file, env.code_root, use_sudo=True) - sudo('chown %s:%s %s' % (env.project_user, env.project_user, path_file)) - sudo('npm install less -g') - update_requirements() - upload_supervisor_app_conf(app_name=u'gunicorn') - upload_supervisor_app_conf(app_name=u'group') - # Restart services to pickup changes - supervisor_command('reload') - supervisor_command('restart %(environment)s:*' % env) - if 'lb' in roles: - nginx.remove_default_site() - nginx.upload_nginx_site_conf(site_name=u'%(project)s-%(environment)s.conf' % env) +def supervisor_command(command): + """Run a supervisorctl command.""" + sudo(u'supervisorctl %s' % command) def project_run(cmd): diff --git a/requirements/dev.txt b/requirements/dev.txt index ef218f7..4464426 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,6 +4,4 @@ coverage==3.5.1 pylint==0.25.1 fabric==1.4.3 ssh==1.7.14 -pycrypto==2.6.0 -Jinja2==2.6 -argyle==0.2.0 +pycrypto==2.6.0 \ No newline at end of file From 6c1bb1d6b45f4a77e52cd0e02d30ec3ca8e48f43 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 09:39:51 -0400 Subject: [PATCH 02/48] Adding provision command which bootstraps Salt mininion. --- conf/bootstrap-salt.sh | 2762 ++++++++++++++++++++++++++++++++++++++++ conf/minion.conf | 14 + fabfile.py | 24 +- 3 files changed, 2799 insertions(+), 1 deletion(-) create mode 100644 conf/bootstrap-salt.sh create mode 100644 conf/minion.conf diff --git a/conf/bootstrap-salt.sh b/conf/bootstrap-salt.sh new file mode 100644 index 0000000..d34a242 --- /dev/null +++ b/conf/bootstrap-salt.sh @@ -0,0 +1,2762 @@ +#!/bin/sh - +#=============================================================================== +# vim: softtabstop=4 shiftwidth=4 expandtab fenc=utf-8 spell spelllang=en +#=============================================================================== +# +# FILE: bootstrap-salt.sh +# +# DESCRIPTION: Bootstrap salt installation for various systems/distributions +# +# BUGS: https://github.com/saltstack/salty-vagrant/issues +# AUTHOR: Pedro Algarvio (s0undt3ch), pedro@algarvio.me +# Alec Koumjian (akoumjian), akoumjian@gmail.com +# Geoff Garside (geoffgarside), geoff@geoffgarside.co.uk +# LICENSE: Apache 2.0 +# ORGANIZATION: Salt Stack (saltstack.org) +# CREATED: 10/15/2012 09:49:37 PM WEST +#=============================================================================== +set -o nounset # Treat unset variables as an error +ScriptVersion="1.5.4" +ScriptName="bootstrap-salt.sh" + +#=============================================================================== +# Environment variables taken into account. +#------------------------------------------------------------------------------- +# * BS_COLORS: If 0 disables colour support +# * BS_PIP_ALLOWED: If 1 enable pip based installations(if needed) +# * BS_ECHO_DEBUG: If 1 enable debug echo which can also be set by -D +# * BS_SALT_ETC_DIR: Defaults to /etc/salt +# * BS_FORCE_OVERWRITE: Force overriding copied files(config, init.d, etc) +#=============================================================================== + + +#=============================================================================== +# LET THE BLACK MAGIC BEGIN!!!! +#=============================================================================== + + +# Bootstrap script truth values +BS_TRUE=1 +BS_FALSE=0 + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __detect_color_support +# DESCRIPTION: Try to detect color support. +#------------------------------------------------------------------------------- +COLORS=${BS_COLORS:-$(tput colors 2>/dev/null || echo 0)} +__detect_color_support() { + if [ $? -eq 0 ] && [ "$COLORS" -gt 2 ]; then + RC="\033[1;31m" + GC="\033[1;32m" + BC="\033[1;34m" + YC="\033[1;33m" + EC="\033[0m" + else + RC="" + GC="" + BC="" + YC="" + EC="" + fi +} +__detect_color_support + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: echoerr +# DESCRIPTION: Echo errors to stderr. +#------------------------------------------------------------------------------- +echoerror() { + printf "${RC} * ERROR${EC}: $@\n" 1>&2; +} + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: echoinfo +# DESCRIPTION: Echo information to stdout. +#------------------------------------------------------------------------------- +echoinfo() { + printf "${GC} * INFO${EC}: %s\n" "$@"; +} + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: echowarn +# DESCRIPTION: Echo warning informations to stdout. +#------------------------------------------------------------------------------- +echowarn() { + printf "${YC} * WARN${EC}: %s\n" "$@"; +} + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: echodebug +# DESCRIPTION: Echo debug information to stdout. +#------------------------------------------------------------------------------- +echodebug() { + if [ $ECHO_DEBUG -eq $BS_TRUE ]; then + printf "${BC} * DEBUG${EC}: %s\n" "$@"; + fi +} + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: check_pip_allowed +# DESCRIPTION: Simple function to let the users know that -P needs to be +# used. +#------------------------------------------------------------------------------- +check_pip_allowed() { + if [ $PIP_ALLOWED -eq $BS_FALSE ]; then + echoerror "pip based installations were not allowed. Retry using '-P'" + usage + exit 1 + fi +} + +#=== FUNCTION ================================================================ +# NAME: usage +# DESCRIPTION: Display usage information. +#=============================================================================== +usage() { + cat << EOT + + Usage : ${ScriptName} [options] + + Installation types: + - stable (default) + - daily (ubuntu specific) + - git + + Examples: + $ ${ScriptName} + $ ${ScriptName} stable + $ ${ScriptName} daily + $ ${ScriptName} git + $ ${ScriptName} git develop + $ ${ScriptName} git 8c3fadf15ec183e5ce8c63739850d543617e4357 + + Options: + -h Display this message + -v Display script version + -n No colours. + -D Show debug output. + -c Temporary configuration directory + -k Temporary directory holding the minion keys which will pre-seed + the master. + -M Also install salt-master + -S Also install salt-syndic + -N Do not install salt-minion + -C Only run the configuration function. This option automaticaly + bypasses any installation. + -P Allow pip based installations. On some distributions the required salt + packages or it's dependencies are not available as a package for that + distribution. Using this flag allows the script to use pip as a last + resort method. NOTE: This works for functions which actually implement + pip based installations. + -F Allow copied files to overwrite existing(config, init.d, etc) + +EOT +} # ---------- end of function usage ---------- + +#----------------------------------------------------------------------- +# Handle command line arguments +#----------------------------------------------------------------------- +TEMP_CONFIG_DIR="null" +TEMP_KEYS_DIR="null" +INSTALL_MASTER=$BS_FALSE +INSTALL_SYNDIC=$BS_FALSE +INSTALL_MINION=$BS_TRUE +ECHO_DEBUG=${BS_ECHO_DEBUG:-$BS_FALSE} +CONFIG_ONLY=$BS_FALSE +PIP_ALLOWED=${BS_PIP_ALLOWED:-$BS_FALSE} +SALT_ETC_DIR=${BS_SALT_ETC_DIR:-/etc/salt} +FORCE_OVERWRITE=${BS_FORCE_OVERWRITE:-$BS_FALSE} + +while getopts ":hvnDc:k:MSNCP" opt +do + case "${opt}" in + + h ) usage; exit 0 ;; + + v ) echo "$0 -- Version $ScriptVersion"; exit 0 ;; + n ) COLORS=0; __detect_color_support ;; + D ) ECHO_DEBUG=$BS_TRUE ;; + c ) TEMP_CONFIG_DIR="$OPTARG" + # If the configuration directory does not exist, error out + if [ ! -d "$TEMP_CONFIG_DIR" ]; then + echoerror "The configuration directory ${TEMP_CONFIG_DIR} does not exist." + exit 1 + fi + ;; + k ) TEMP_KEYS_DIR="$OPTARG" + # If the configuration directory does not exist, error out + if [ ! -d "$TEMP_KEYS_DIR" ]; then + echoerror "The pre-seed keys directory ${TEMP_KEYS_DIR} does not exist." + exit 1 + fi + ;; + M ) INSTALL_MASTER=$BS_TRUE ;; + S ) INSTALL_SYNDIC=$BS_TRUE ;; + N ) INSTALL_MINION=$BS_FALSE ;; + C ) CONFIG_ONLY=$BS_TRUE ;; + P ) PIP_ALLOWED=$BS_TRUE ;; + F ) FORCE_OVERWRITE=$BS_TRUE ;; + + \?) echo + echoerror "Option does not exist : $OPTARG" + usage + exit 1 + ;; + + esac # --- end of case --- +done +shift $(($OPTIND-1)) + + +__check_unparsed_options() { + shellopts="$1" + unparsed_options=$( echo "$shellopts" | grep -E '[-]+[[:alnum:]]' ) + if [ "x$unparsed_options" != "x" ]; then + usage + echo + echoerror "options are only allowed before install arguments" + echo + exit 1 + fi +} + + +# Check that we're actually installing one of minion/master/syndic +if [ $INSTALL_MINION -eq $BS_FALSE ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && [ $CONFIG_ONLY -eq $BS_FALSE ]; then + echowarn "Nothing to install or configure" + exit 0 +fi + +if [ $CONFIG_ONLY -eq $BS_TRUE ] && [ "$TEMP_CONFIG_DIR" = "null" ]; then + echoerror "In order to run the script in configuration only mode you also need to provide the configuration directory." + exit 1 +fi + +# Define installation type +if [ "$#" -eq 0 ];then + ITYPE="stable" +else + __check_unparsed_options "$*" + ITYPE=$1 + shift +fi + +# Check installation type +if [ "$ITYPE" != "stable" ] && [ "$ITYPE" != "daily" ] && [ "$ITYPE" != "git" ]; then + echoerror "Installation type \"$ITYPE\" is not known..." + exit 1 +fi + +# If doing a git install, check what branch/tag/sha will be checked out +if [ $ITYPE = "git" ]; then + if [ "$#" -eq 0 ];then + GIT_REV="master" + else + __check_unparsed_options "$*" + GIT_REV="$1" + shift + fi +fi + +# Check for any unparsed arguments. Should be an error. +if [ "$#" -gt 0 ]; then + __check_unparsed_options "$*" + usage + echo + echoerror "Too many arguments." + exit 1 +fi + +# Root permissions are required to run this script +if [ $(whoami) != "root" ] ; then + echoerror "Salt requires root privileges to install. Please re-run this script as root." + exit 1 +fi + +CALLER=$(echo `ps -a -o pid,args | grep $$ | grep -v grep | tr -s ' '` | cut -d ' ' -f 2) +if [ "${CALLER}x" = "${0}x" ]; then + CALLER="PIPED THROUGH" +fi +echoinfo "${CALLER} ${0} -- Version ${ScriptVersion}" +#echowarn "Running the unstable version of ${ScriptName}" + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __exit_cleanup +# DESCRIPTION: Cleanup any leftovers after script has ended +# +# +# http://www.unix.com/man-page/POSIX/1posix/trap/ +# +# Signal Number Signal Name +# 1 SIGHUP +# 2 SIGINT +# 3 SIGQUIT +# 6 SIGABRT +# 9 SIGKILL +# 14 SIGALRM +# 15 SIGTERM +#------------------------------------------------------------------------------- +__exit_cleanup() { + EXIT_CODE=$? + + # Remove the logging pipe when the script exits + echodebug "Removing the logging pipe $LOGPIPE" + rm -f $LOGPIPE + + # Kill tee when exiting, CentOS, at least requires this + TEE_PID=$(ps ax | grep tee | grep $LOGFILE | awk '{print $1}') + + [ "x$TEE_PID" = "x" ] && exit $EXIT_CODE + + echodebug "Killing logging pipe tee's with pid(s): $TEE_PID" + + # We need to trap errors since killing tee will cause a 127 errno + # We also do this as late as possible so we don't "mis-catch" other errors + __trap_errors() { + echoinfo "Errors Trapped: $EXIT_CODE" + # Exit with the "original" exit code, not the trapped code + exit $EXIT_CODE + } + trap "__trap_errors" INT QUIT ABRT KILL QUIT TERM + + # Now we're "good" to kill tee + kill -s TERM $TEE_PID + + # In case the 127 errno is not triggered, exit with the "original" exit code + exit $EXIT_CODE +} +trap "__exit_cleanup" EXIT INT + + +# Define our logging file and pipe paths +LOGFILE="/tmp/$( echo $ScriptName | sed s/.sh/.log/g )" +LOGPIPE="/tmp/$( echo $ScriptName | sed s/.sh/.logpipe/g )" + +# Create our logging pipe +# On FreeBSD we have to use mkfifo instead of mknod +mknod $LOGPIPE p >/dev/null 2>&1 || mkfifo $LOGPIPE >/dev/null 2>&1 +if [ $? -ne 0 ]; then + echoerror "Failed to create the named pipe required to log" + exit 1 +fi + +# What ever is written to the logpipe gets written to the logfile +tee < $LOGPIPE $LOGFILE & + +# Close STDOUT, reopen it directing it to the logpipe +exec 1>&- +exec 1>$LOGPIPE +# Close STDERR, reopen it directing it to the logpipe +exec 2>&- +exec 2>$LOGPIPE + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __gather_hardware_info +# DESCRIPTION: Discover hardware information +#------------------------------------------------------------------------------- +__gather_hardware_info() { + if [ -f /proc/cpuinfo ]; then + CPU_VENDOR_ID=$(awk '/vendor_id|Processor/ {sub(/-.*$/,"",$3); print $3; exit}' /proc/cpuinfo ) + elif [ -f /usr/bin/kstat ]; then + # SmartOS. + # Solaris!? + # This has only been tested for a GenuineIntel CPU + CPU_VENDOR_ID=$(/usr/bin/kstat -p cpu_info:0:cpu_info0:vendor_id | awk '{print $2}') + else + CPU_VENDOR_ID=$( sysctl -n hw.model ) + fi + CPU_VENDOR_ID_L=$( echo $CPU_VENDOR_ID | tr '[:upper:]' '[:lower:]' ) + CPU_ARCH=$(uname -m 2>/dev/null || uname -p 2>/dev/null || echo "unknown") + CPU_ARCH_L=$( echo $CPU_ARCH | tr '[:upper:]' '[:lower:]' ) + +} +__gather_hardware_info + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __gather_os_info +# DESCRIPTION: Discover operating system information +#------------------------------------------------------------------------------- +__gather_os_info() { + OS_NAME=$(uname -s 2>/dev/null) + OS_NAME_L=$( echo $OS_NAME | tr '[:upper:]' '[:lower:]' ) + OS_VERSION=$(uname -r) + OS_VERSION_L=$( echo $OS_VERSION | tr '[:upper:]' '[:lower:]' ) +} +__gather_os_info + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __parse_version_string +# DESCRIPTION: Parse version strings ignoring the revision. +# MAJOR.MINOR.REVISION becomes MAJOR.MINOR +#------------------------------------------------------------------------------- +__parse_version_string() { + VERSION_STRING="$1" + PARSED_VERSION=$( + echo $VERSION_STRING | + sed -e 's/^/#/' \ + -e 's/^#[^0-9]*\([0-9][0-9]*\.[0-9][0-9]*\)\(\.[0-9][0-9]*\).*$/\1/' \ + -e 's/^#[^0-9]*\([0-9][0-9]*\.[0-9][0-9]*\).*$/\1/' \ + -e 's/^#[^0-9]*\([0-9][0-9]*\).*$/\1/' \ + -e 's/^#.*$//' + ) + echo $PARSED_VERSION +} + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __unquote_string +# DESCRIPTION: Strip single or double quotes from the provided string. +#------------------------------------------------------------------------------- +__unquote_string() { + echo $@ | sed "s/^\([\"']\)\(.*\)\1\$/\2/g" +} + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __camelcase_split +# DESCRIPTION: Convert CamelCased strings to Camel_Cased +#------------------------------------------------------------------------------- +__camelcase_split() { + echo $@ | sed -r 's/([^A-Z-])([A-Z])/\1 \2/g' +} + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __strip_duplicates +# DESCRIPTION: Strip duplicate strings +#------------------------------------------------------------------------------- +__strip_duplicates() { + echo $@ | tr -s '[:space:]' '\n' | awk '!x[$0]++' +} + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __sort_release_files +# DESCRIPTION: Custom sort function. Alphabetical or numerical sort is not +# enough. +#------------------------------------------------------------------------------- +__sort_release_files() { + KNOWN_RELEASE_FILES=$(echo "(arch|centos|debian|ubuntu|fedora|redhat|suse|\ + mandrake|mandriva|gentoo|slackware|turbolinux|unitedlinux|lsb|system|\ + os)(-|_)(release|version)" | sed -r 's:[[:space:]]::g') + primary_release_files="" + secondary_release_files="" + # Sort know VS un-known files first + for release_file in $(echo $@ | sed -r 's:[[:space:]]:\n:g' | sort --unique --ignore-case); do + match=$(echo $release_file | egrep -i ${KNOWN_RELEASE_FILES}) + if [ "x${match}" != "x" ]; then + primary_release_files="${primary_release_files} ${release_file}" + else + secondary_release_files="${secondary_release_files} ${release_file}" + fi + done + + # Now let's sort by know files importance, max important goes last in the max_prio list + max_prio="redhat-release centos-release" + for entry in $max_prio; do + if [ "x$(echo ${primary_release_files} | grep $entry)" != "x" ]; then + primary_release_files=$(echo ${primary_release_files} | sed -e "s:\(.*\)\($entry\)\(.*\):\2 \1 \3:g") + fi + done + # Now, least important goes last in the min_prio list + min_prio="lsb-release" + for entry in $max_prio; do + if [ "x$(echo ${primary_release_files} | grep $entry)" != "x" ]; then + primary_release_files=$(echo ${primary_release_files} | sed -e "s:\(.*\)\($entry\)\(.*\):\1 \3 \2:g") + fi + done + + # Echo the results collapsing multiple white-space into a single white-space + echo "${primary_release_files} ${secondary_release_files}" | sed -r 's:[[:space:]]:\n:g' +} + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __gather_linux_system_info +# DESCRIPTION: Discover Linux system information +#------------------------------------------------------------------------------- +__gather_linux_system_info() { + DISTRO_NAME="" + DISTRO_VERSION="" + + # Let's test if the lsb_release binary is available + rv=$(lsb_release >/dev/null 2>&1) + if [ $? -eq 0 ]; then + DISTRO_NAME=$(lsb_release -si) + if [ "x$(echo "$DISTRO_NAME" | grep RedHat)" != "x" ]; then + # Let's convert CamelCase to Camel Case + DISTRO_NAME=$(__camelcase_split "$DISTRO_NAME") + fi + if [ "${DISTRO_NAME}" = "openSUSE project" ]; then + # lsb_release -si returns "openSUSE project" on openSUSE 12.3 + DISTRO_NAME="opensuse" + fi + rv=$(lsb_release -sr) + [ "${rv}x" != "x" ] && DISTRO_VERSION=$(__parse_version_string "$rv") + elif [ -f /etc/lsb-release ]; then + # We don't have the lsb_release binary, though, we do have the file it parses + DISTRO_NAME=$(grep DISTRIB_ID /etc/lsb-release | sed -e 's/.*=//') + rv=$(grep DISTRIB_RELEASE /etc/lsb-release | sed -e 's/.*=//') + [ "${rv}x" != "x" ] && DISTRO_VERSION=$(__parse_version_string "$rv") + fi + + if [ "x$DISTRO_NAME" != "x" ] && [ "x$DISTRO_VERSION" != "x" ]; then + # We already have the distribution name and version + return + fi + + for rsource in $(__sort_release_files $( + cd /etc && /bin/ls *[_-]release *[_-]version 2>/dev/null | env -i sort | \ + sed -e '/^redhat-release$/d' -e '/^lsb-release$/d'; \ + echo redhat-release lsb-release + )); do + + [ -L "/etc/${rsource}" ] && continue # Don't follow symlinks + [ ! -f "/etc/${rsource}" ] && continue # Does not exist + + n=$(echo ${rsource} | sed -e 's/[_-]release$//' -e 's/[_-]version$//') + rv=$( (grep VERSION /etc/${rsource}; cat /etc/${rsource}) | grep '[0-9]' | sed -e 'q' ) + [ "${rv}x" = "x" ] && continue # There's no version information. Continue to next rsource + v=$(__parse_version_string "$rv") + case $(echo ${n} | tr '[:upper:]' '[:lower:]') in + redhat ) + if [ ".$(egrep 'CentOS' /etc/${rsource})" != . ]; then + n="CentOS" + elif [ ".$(egrep 'Red Hat Enterprise Linux' /etc/${rsource})" != . ]; then + n="ed at nterprise inux" + else + n="ed at inux" + fi + ;; + arch ) n="Arch Linux" ;; + centos ) n="CentOS" ;; + debian ) n="Debian" ;; + ubuntu ) n="Ubuntu" ;; + fedora ) n="Fedora" ;; + suse ) n="SUSE" ;; + mandrake*|mandriva ) n="Mandriva" ;; + gentoo ) n="Gentoo" ;; + slackware ) n="Slackware" ;; + turbolinux ) n="TurboLinux" ;; + unitedlinux ) n="UnitedLinux" ;; + system ) + while read -r line; do + [ "${n}x" != "systemx" ] && break + case "$line" in + *Amazon*Linux*AMI*) + n="Amazon Linux AMI" + break + esac + done < /etc/${rsource} + ;; + os ) + nn=$(__unquote_string $(grep '^ID=' /etc/os-release | sed -e 's/^ID=\(.*\)$/\1/g')) + rv=$(__unquote_string $(grep '^VERSION_ID=' /etc/os-release | sed -e 's/^VERSION_ID=\(.*\)$/\1/g')) + [ "${rv}x" != "x" ] && v=$(__parse_version_string "$rv") || v="" + case $(echo ${nn} | tr '[:upper:]' '[:lower:]') in + arch ) + n="Arch Linux" + v="" # Arch Linux does not provide a version. + ;; + debian ) + n="Debian" + if [ "${v}x" = "x" ]; then + if [ "$(cat /etc/debian_version)" = "wheezy/sid" ]; then + # I've found an EC2 wheezy image which did not tell its version + v=$(__parse_version_string "7.0") + fi + else + echowarn "Unable to parse the Debian Version" + fi + ;; + * ) + n=${nn} + ;; + esac + ;; + * ) n="${n}" ; + esac + DISTRO_NAME=$n + DISTRO_VERSION=$v + break + done +} + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __gather_sunos_system_info +# DESCRIPTION: Discover SunOS system info +#------------------------------------------------------------------------------- +__gather_sunos_system_info() { + if [ -f /sbin/uname ]; then + DISTRO_VERSION=$(/sbin/uname -X | awk '/[kK][eE][rR][nN][eE][lL][iI][dD]/ { print $3}') + fi + + DISTRO_NAME="" + if [ -f /etc/release ]; then + while read -r line; do + [ "${DISTRO_NAME}x" != "x" ] && break + case "$line" in + *OpenIndiana*oi_[0-9]*) + DISTRO_NAME="OpenIndiana" + DISTRO_VERSION=$(echo "$line" | sed -nr "s/OpenIndiana(.*)oi_([[:digit:]]+)(.*)/\2/p") + break + ;; + *OpenSolaris*snv_[0-9]*) + DISTRO_NAME="OpenSolaris" + DISTRO_VERSION=$(echo "$line" | sed -nr "s/OpenSolaris(.*)snv_([[:digit:]]+)(.*)/\2/p") + break + ;; + *Oracle*Solaris*[0-9]*) + DISTRO_NAME="Oracle Solaris" + DISTRO_VERSION=$(echo "$line" | sed -nr "s/(Oracle Solaris) ([[:digit:]]+)(.*)/\2/p") + break + ;; + *Solaris*) + DISTRO_NAME="Solaris" + break + ;; + *NexentaCore*) + DISTRO_NAME="Nexenta Core" + break + ;; + *SmartOS*) + DISTRO_NAME="SmartOS" + break + ;; + esac + done < /etc/release + fi + + if [ "${DISTRO_NAME}x" = "x" ]; then + DISTRO_NAME="Solaris" + DISTRO_VERSION=$( + echo "${OS_VERSION}" | + sed -e 's;^4\.;1.;' \ + -e 's;^5\.\([0-6]\)[^0-9]*$;2.\1;' \ + -e 's;^5\.\([0-9][0-9]*\).*;\1;' + ) + fi +} + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __gather_bsd_system_info +# DESCRIPTION: Discover OpenBSD, NetBSD and FreeBSD systems information +#------------------------------------------------------------------------------- +__gather_bsd_system_info() { + DISTRO_NAME=${OS_NAME} + DISTRO_VERSION=$(echo "${OS_VERSION}" | sed -e 's;[()];;' -e 's/-.*$//') +} + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __gather_system_info +# DESCRIPTION: Discover which system and distribution we are running. +#------------------------------------------------------------------------------- +__gather_system_info() { + case ${OS_NAME_L} in + linux ) + __gather_linux_system_info + ;; + sunos ) + __gather_sunos_system_info + ;; + openbsd|freebsd|netbsd ) + __gather_bsd_system_info + ;; + * ) + echoerror "${OS_NAME} not supported."; + exit 1 + ;; + esac + +} +__gather_system_info + + +echo +echoinfo "System Information:" +echoinfo " CPU: ${CPU_VENDOR_ID}" +echoinfo " CPU Arch: ${CPU_ARCH}" +echoinfo " OS Name: ${OS_NAME}" +echoinfo " OS Version: ${OS_VERSION}" +echoinfo " Distribution: ${DISTRO_NAME} ${DISTRO_VERSION}" +echo + +# Let users know what's going to be installed/configured +if [ $INSTALL_MINION -eq $BS_TRUE ]; then + if [ $CONFIG_ONLY -eq $BS_FALSE ]; then + echoinfo "Installing minion" + else + echoinfo "Configuring minion" + fi +fi + +if [ $INSTALL_MASTER -eq $BS_TRUE ]; then + if [ $CONFIG_ONLY -eq $BS_FALSE ]; then + echoinfo "Installing master" + else + echoinfo "Configuring master" + fi +fi + +if [ $INSTALL_SYNDIC -eq $BS_TRUE ]; then + if [ $CONFIG_ONLY -eq $BS_FALSE ]; then + echoinfo "Installing syndic" + else + echoinfo "Configuring syndic" + fi +fi + +# Simplify version naming on functions +if [ "x${DISTRO_VERSION}" = "x" ]; then + DISTRO_MAJOR_VERSION="" + DISTRO_MINOR_VERSION="" + PREFIXED_DISTRO_MAJOR_VERSION="" + PREFIXED_DISTRO_MINOR_VERSION="" +else + DISTRO_MAJOR_VERSION="$(echo $DISTRO_VERSION | sed 's/^\([0-9]*\).*/\1/g')" + DISTRO_MINOR_VERSION="$(echo $DISTRO_VERSION | sed 's/^\([0-9]*\).\([0-9]*\).*/\2/g')" + PREFIXED_DISTRO_MAJOR_VERSION="_${DISTRO_MAJOR_VERSION}" + if [ "${PREFIXED_DISTRO_MAJOR_VERSION}" = "_" ]; then + PREFIXED_DISTRO_MAJOR_VERSION="" + fi + PREFIXED_DISTRO_MINOR_VERSION="_${DISTRO_MINOR_VERSION}" + if [ "${PREFIXED_DISTRO_MINOR_VERSION}" = "_" ]; then + PREFIXED_DISTRO_MINOR_VERSION="" + fi +fi +# Simplify distro name naming on functions +DISTRO_NAME_L=$(echo $DISTRO_NAME | tr '[:upper:]' '[:lower:]' | sed 's/[^a-zA-Z0-9_ ]//g' | sed -re 's/([[:space:]])+/_/g') + + +# Only Ubuntu has daily packages, let's let users know about that +if ([ "${DISTRO_NAME_L}" != "ubuntu" ] && [ $ITYPE = "daily" ]) && \ + ([ "${DISTRO_NAME_L}" != "trisquel" ] && [ $ITYPE = "daily" ]); then + echoerror "${DISTRO_NAME} does not have daily packages support" + exit 1 +fi + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __function_defined +# DESCRIPTION: Checks if a function is defined within this scripts scope +# PARAMETERS: function name +# RETURNS: 0 or 1 as in defined or not defined +#------------------------------------------------------------------------------- +__function_defined() { + FUNC_NAME=$1 + if [ "$(command -v $FUNC_NAME)x" != "x" ]; then + echoinfo "Found function $FUNC_NAME" + return 0 + fi + echodebug "$FUNC_NAME not found...." + return 1 +} + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __git_clone_and_checkout +# DESCRIPTION: (DRY) Helper function to clone and checkout salt to a +# specific revision. +#------------------------------------------------------------------------------- +__git_clone_and_checkout() { + SALT_GIT_CHECKOUT_DIR=/tmp/git/salt + [ -d /tmp/git ] || mkdir /tmp/git + cd /tmp/git + if [ -d $SALT_GIT_CHECKOUT_DIR ]; then + cd $SALT_GIT_CHECKOUT_DIR + git fetch || return 1 + # Tags are needed because of salt's versioning, also fetch that + git fetch --tags || return 1 + git reset --hard $GIT_REV || return 1 + + # Just calling `git reset --hard $GIT_REV` on a branch name that has + # already been checked out will not update that branch to the upstream + # HEAD; instead it will simply reset to itself. Check the ref to see + # if it is a branch name, check out the branch, and pull in the + # changes. + git branch -a | grep -q ${GIT_REV} + if [ $? -eq 0 ]; then + git pull --rebase || return 1 + fi + else + git clone https://github.com/saltstack/salt.git salt || return 1 + cd $SALT_GIT_CHECKOUT_DIR + git checkout $GIT_REV || return 1 + fi + return 0 +} + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: __apt_get_noinput +# DESCRIPTION: (DRY) apt-get install with noinput options +#------------------------------------------------------------------------------- +__apt_get_noinput() { + apt-get install -y -o DPkg::Options::=--force-confold $@; return $? +} + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: copyfile +# DESCRIPTION: Simple function to copy files. Overrides if asked. +#------------------------------------------------------------------------------- +copyfile() { + overwrite=$FORCE_OVERWRITE + if [ $# -eq 2 ]; then + sfile=$1 + dfile=$2 + elif [ $# -eq 3 ]; then + sfile=$1 + dfile=$2 + overwrite=$3 + else + echoerror "Wrong number of arguments for copyfile()" + echoinfo "USAGE: copyfile OR copyfile " + exit 1 + fi + + # Does the source file exist? + if [ ! -f "$sfile" ]; then + echowarn "$sfile does not exist!" + return 1 + fi + + if [ ! -f "$dfile" ]; then + # The destination file does not exist, copy + echodebug "Copying $sfile to $dfile" + cp "$sfile" "$dfile" || return 1 + elif [ -f "$dfile" ] && [ $overwrite -eq $BS_TRUE ]; then + # The destination exist and we're overwriting + echodebug "Overriding $dfile with $sfile" + cp -f "$sfile" "$dfile" || return 2 + elif [ -f "$dfile" ] && [ $overwrite -ne $BS_TRUE ]; then + echodebug "Not overriding $dfile with $sfile" + fi + return 0 +} + + +#--- FUNCTION ---------------------------------------------------------------- +# NAME: movefile +# DESCRIPTION: Simple function to move files. Overrides if asked. +#------------------------------------------------------------------------------- +movefile() { + overwrite=$FORCE_OVERWRITE + if [ $# -eq 2 ]; then + sfile=$1 + dfile=$2 + elif [ $# -eq 3 ]; then + sfile=$1 + dfile=$2 + overwrite=$3 + else + echoerror "Wrong number of arguments for movefile()" + echoinfo "USAGE: movefile OR movefile " + exit 1 + fi + + # Does the source file exist? + if [ ! -f "$sfile" ]; then + echowarn "$sfile does not exist!" + return 1 + fi + + if [ ! -f "$dfile" ]; then + # The destination file does not exist, copy + echodebug "Moving $sfile to $dfile" + mv "$sfile" "$dfile" || return 1 + elif [ -f "$dfile" ] && [ $overwrite -eq $BS_TRUE ]; then + # The destination exist and we're overwriting + echodebug "Overriding $dfile with $sfile" + mv -f "$sfile" "$dfile" || return 1 + elif [ -f "$dfile" ] && [ $overwrite -ne $BS_TRUE ]; then + echodebug "Not overriding $dfile with $sfile" + fi + + return 0 +} + +############################################################################## +# +# Distribution install functions +# +# In order to install salt for a distribution you need to define: +# +# To Install Dependencies, which is required, one of: +# 1. install____deps +# 2. install_____deps +# 3. install___deps +# 4 install____deps +# 5. install___deps +# 6. install__deps +# +# Optionally, define a salt configuration function, which will be called if +# the -c (config-dir) option is passed. One of: +# 1. config____salt +# 2. config_____salt +# 3. config___salt +# 4 config____salt +# 5. config___salt +# 6. config__salt +# 7. config_salt [THIS ONE IS ALREADY DEFINED AS THE DEFAULT] +# +# Optionally, define a salt master pre-seed function, which will be called if +# the -k (pre-seed master keys) option is passed. One of: +# 1. pressed____master +# 2. pressed_____master +# 3. pressed___master +# 4 pressed____master +# 5. pressed___master +# 6. pressed__master +# 7. pressed_master [THIS ONE IS ALREADY DEFINED AS THE DEFAULT] +# +# To install salt, which, of course, is required, one of: +# 1. install___ +# 2. install____ +# 3. install__ +# +# Optionally, define a post install function, one of: +# 1. install____post +# 2. install_____post +# 3. install___post +# 4 install____post +# 5. install___post +# 6. install__post +# +# Optionally, define a start daemons function, one of: +# 1. install____restart_daemons +# 2. install_____restart_daemons +# 3. install___restart_daemons +# 4 install____restart_daemons +# 5. install___restart_daemons +# 6. install__restart_daemons +# +# NOTE: The start daemons function should be able to restart any daemons +# which are running, or start if they're not running. +# +############################################################################## + + +############################################################################## +# +# Ubuntu Install Functions +# +install_ubuntu_deps() { + apt-get update + if [ $DISTRO_MAJOR_VERSION -eq 12 ] && [ $DISTRO_MINOR_VERSION -gt 04 ] || [ $DISTRO_MAJOR_VERSION -gt 12 ]; then + # Above Ubuntu 12.04 add-apt-repository is in a different package + __apt_get_noinput software-properties-common || return 1 + else + __apt_get_noinput python-software-properties || return 1 + fi + if [ $DISTRO_MAJOR_VERSION -lt 11 ] && [ $DISTRO_MINOR_VERSION -lt 10 ]; then + add-apt-repository ppa:saltstack/salt || return 1 + else + add-apt-repository -y ppa:saltstack/salt || return 1 + fi + apt-get update + return 0 +} + +install_ubuntu_daily_deps() { + apt-get update + if [ $DISTRO_MAJOR_VERSION -eq 12 ] && [ $DISTRO_MINOR_VERSION -gt 04 ] || [ $DISTRO_MAJOR_VERSION -gt 12 ]; then + # Above Ubuntu 12.04 add-apt-repository is in a different package + __apt_get_noinput software-properties-common || return 1 + else + __apt_get_noinput python-software-properties || return 1 + fi + if [ $DISTRO_MAJOR_VERSION -lt 11 ] && [ $DISTRO_MINOR_VERSION -lt 10 ]; then + add-apt-repository ppa:saltstack/salt-daily || return 1 + else + add-apt-repository -y ppa:saltstack/salt-daily || return 1 + fi + apt-get update + return 0 +} + +install_ubuntu_11_10_deps() { + apt-get update + __apt_get_noinput python-software-properties || return 1 + add-apt-repository -y 'deb http://us.archive.ubuntu.com/ubuntu/ oneiric universe' || return 1 + add-apt-repository -y ppa:saltstack/salt || return 1 + apt-get update + return 0 +} + +install_ubuntu_git_deps() { + install_ubuntu_deps || return 1 + __apt_get_noinput git-core python-yaml python-m2crypto python-crypto \ + msgpack-python python-zmq python-jinja2 || return 1 + + __git_clone_and_checkout || return 1 + + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + TEMP_CONFIG_DIR="${SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_ubuntu_11_10_post() { + add-apt-repository -y --remove 'deb http://us.archive.ubuntu.com/ubuntu/ oneiric universe' || return 1 + return 0 +} + +install_ubuntu_stable() { + packages="" + if [ $INSTALL_MINION -eq $BS_TRUE ]; then + packages="${packages} salt-minion" + fi + if [ $INSTALL_MASTER -eq $BS_TRUE ]; then + packages="${packages} salt-master" + fi + if [ $INSTALL_SYNDIC -eq $BS_TRUE ]; then + packages="${packages} salt-syndic" + fi + __apt_get_noinput ${packages} || return 1 + return 0 +} + +install_ubuntu_daily() { + install_ubuntu_stable || return 1 + return 0 +} + +install_ubuntu_git() { + python setup.py install --install-layout=deb || return 1 + return 0 +} + +install_ubuntu_git_post() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ -f /sbin/initctl ]; then + # We have upstart support + echodebug "There's upstart support" + /sbin/initctl status salt-$fname > /dev/null 2>&1 + + if [ $? -eq 1 ]; then + # upstart does not know about our service, let's copy the proper file + echowarn "Upstart does not apparently know anything about salt-$fname" + echodebug "Copying ${SALT_GIT_CHECKOUT_DIR}/pkg/salt-$fname.upstart to /etc/init/salt-$fname.conf" + copyfile ${SALT_GIT_CHECKOUT_DIR}/pkg/salt-$fname.upstart /etc/init/salt-$fname.conf + fi + # No upstart support in Ubuntu!? + elif [ -f ${SALT_GIT_CHECKOUT_DIR}/debian/salt-$fname.init ]; then + echodebug "There's NO upstart support!?" + echodebug "Copying ${SALT_GIT_CHECKOUT_DIR}/debian/salt-$fname.init to /etc/init.d/salt-$fname" + copyfile ${SALT_GIT_CHECKOUT_DIR}/debian/salt-$fname.init /etc/init.d/salt-$fname + chmod +x /etc/init.d/salt-$fname + update-rc.d salt-$fname defaults + else + echoerror "Neither upstart not init.d was setup for salt-$fname" + fi + done +} + +install_ubuntu_restart_daemons() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ -f /sbin/initctl ]; then + echodebug "There's upstart support while checking salt-$fname" + status salt-$fname || echowarn "Upstart does not apparently know anything about salt-$fname" + sleep 1 + if [ $? -eq 0 ]; then + echodebug "Upstart apparently knows about salt-$fname" + # upstart knows about this service, let's stop and start it. + # We could restart but earlier versions of the upstart script + # did not support restart, so, it's safer this way + + # Is it running??? + status salt-$fname | grep -q running + # If it is, stop it + if [ $? -eq 0 ]; then + sleep 1 + stop salt-$fname || (echodebug "Failed to stop salt-$fname" && return 1) + fi + # Now start it + sleep 1 + start salt-$fname + [ $? -eq 0 ] && continue + # We failed to start the service, let's test the SysV code bellow + echodebug "Failed to start salt-$fname" + fi + fi + + if [ ! -f /etc/init.d/salt-$fname ]; then + echoerror "No init.d support for salt-$fname was found" + return 1 + fi + + /etc/init.d/salt-$fname stop > /dev/null 2>&1 + /etc/init.d/salt-$fname start + done + return 0 +} +# +# End of Ubuntu Install Functions +# +############################################################################## + +############################################################################## +# +# Trisquel(Ubuntu) Install Functions +# +# Trisquel 6.0 is based on Ubuntu 12.04 +# +install_trisquel_6_stable_deps() { + apt-get update + __apt_get_noinput python-software-properties || return 1 + add-apt-repository -y ppa:saltstack/salt || return 1 + apt-get update + return 0 +} + +install_trisquel_6_daily_deps() { + apt-get update + __apt_get_noinput python-software-properties || return 1 + add-apt-repository -y ppa:saltstack/salt-daily || return 1 + apt-get update + return 0 +} + +install_trisquel_6_git_deps() { + install_trisquel_6_stable_deps || return 1 + __apt_get_noinput git-core python-yaml python-m2crypto python-crypto \ + msgpack-python python-zmq python-jinja2 || return 1 + + __git_clone_and_checkout || return 1 + + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + TEMP_CONFIG_DIR="${SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_trisquel_6_stable() { + install_ubuntu_stable || return 1 + return 0 +} + +install_trisquel_6_daily() { + install_ubuntu_daily || return 1 + return 0 +} + +install_trisquel_6_git() { + install_ubuntu_git || return 1 + return 0 +} + +install_trisquel_git_post() { + install_ubuntu_git_post || return 1 + return 0 +} + +install_trisquel_restart_daemons() { + install_ubuntu_restart_daemons || return 1 + return 0 +} +# +# End of Tristel(Ubuntu) Install Functions +# +############################################################################## + +############################################################################## +# +# Debian Install Functions +# +install_debian_deps() { + # No user interaction, libc6 restart services for example + export DEBIAN_FRONTEND=noninteractive + + apt-get update +} + +install_debian_6_deps() { + check_pip_allowed + echowarn "PyZMQ will be installed from PyPi in order to compile it against ZMQ3" + echowarn "This is required for long term stable minion connections to the master." + + # No user interaction, libc6 restart services for example + export DEBIAN_FRONTEND=noninteractive + + if [ "x$(grep -R 'backports.debian.org' /etc/apt)" = "x" ]; then + echo "deb http://backports.debian.org/debian-backports squeeze-backports main" >> \ + /etc/apt/sources.list.d/backports.list + fi + + if [ ! -f /etc/apt/preferences.d/local-salt-backport.pref ]; then + # Add madduck's repo since squeeze packages have been deprecated + for fname in salt-common salt-master salt-minion salt-syndic salt-doc; do + echo "Package: $fname" + echo "Pin: release a=squeeze-backports" + echo "Pin-Priority: 600" + echo + done > /etc/apt/preferences.d/local-salt-backport.pref + + cat <<_eof > /etc/apt/sources.list.d/local-madduck-backports.list +deb http://debian.madduck.net/repo squeeze-backports main +deb-src http://debian.madduck.net/repo squeeze-backports main +_eof + + wget -q http://debian.madduck.net/repo/gpg/archive.key -O - | apt-key add - || return 1 + fi + + if [ ! -f /etc/apt/sources.list.d/debian-experimental.list ]; then + cat <<_eof > /etc/apt/sources.list.d/debian-experimental.list +deb http://ftp.debian.org/debian experimental main +deb-src http://ftp.debian.org/debian experimental main +_eof + + cat <<_eof > /etc/apt/preferences.d/libzmq3-debian-experimental.pref +Package: libzmq3 +Pin: release a=experimental +Pin-Priority: 800 + +Package: libzmq3-dev +Pin: release a=experimental +Pin-Priority: 800 +_eof + fi + + apt-get update + __apt_get_noinput -t experimental libzmq3 libzmq3-dev || return 1 + __apt_get_noinput build-essential python-dev python-pip || return 1 + return 0 +} + +install_debian_7_deps() { + check_pip_allowed + echowarn "PyZMQ will be installed from PyPi in order to compile it against ZMQ3" + echowarn "This is required for long term stable minion connections to the master." + + if [ ! -f /etc/apt/sources.list.d/debian-experimental.list ]; then + cat <<_eof > /etc/apt/sources.list.d/debian-experimental.list +deb http://ftp.debian.org/debian experimental main +deb-src http://ftp.debian.org/debian experimental main +_eof + + cat <<_eof > /etc/apt/preferences.d/libzmq3-debian-experimental.pref +Package: libzmq3 +Pin: release a=experimental +Pin-Priority: 800 + +Package: libzmq3-dev +Pin: release a=experimental +Pin-Priority: 800 +_eof + fi + + apt-get update + __apt_get_noinput -t experimental libzmq3 libzmq3-dev || return 1 + __apt_get_noinput build-essential python-dev python-pip || return 1 + return 0 +} + +install_debian_git_deps() { + check_pip_allowed + echowarn "PyZMQ will be installed from PyPi in order to compile it against ZMQ3" + echowarn "This is required for long term stable minion connections to the master." + + # No user interaction, libc6 restart services for example + export DEBIAN_FRONTEND=noninteractive + + apt-get update + __apt_get_noinput lsb-release python python-pkg-resources python-crypto \ + python-jinja2 python-m2crypto python-yaml msgpack-python python-pip \ + git || return 1 + + __git_clone_and_checkout || return 1 + + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + TEMP_CONFIG_DIR="${SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_debian_6_git_deps() { + install_debian_6_deps || return 1 # Add backports + install_debian_git_deps || return 1 # Grab the actual deps + return 0 +} + +install_debian_7_git_deps() { + install_debian_7_deps || return 1 # Add experimental repository for ZMQ3 + install_debian_git_deps || return 1 # Grab the actual deps + return 0 +} + +__install_debian_stable() { + check_pip_allowed + packages="" + if [ $INSTALL_MINION -eq $BS_TRUE ]; then + packages="${packages} salt-minion" + fi + if [ $INSTALL_MASTER -eq $BS_TRUE ]; then + packages="${packages} salt-master" + fi + if [ $INSTALL_SYNDIC -eq $BS_TRUE ]; then + packages="${packages} salt-syndic" + fi + __apt_get_noinput ${packages} || return 1 + + # Building pyzmq from source to build it against libzmq3. + # Should override current installation + pip install -U pyzmq || return 1 + + return 0 +} + + +install_debian_6_stable() { + __install_debian_stable || return 1 + return 0 +} + +install_debian_git() { + python setup.py install --install-layout=deb || return 1 + + # Building pyzmq from source to build it against libzmq3. + # Should override current installation + pip install -U pyzmq || return 1 + return 0 +} + +install_debian_6_git() { + install_debian_git || return 1 + return 0 +} + +install_debian_7_git() { + install_debian_git || return 1 + return 0 +} + +install_debian_git_post() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ -f ${SALT_GIT_CHECKOUT_DIR}/debian/salt-$fname.init ]; then + copyfile ${SALT_GIT_CHECKOUT_DIR}/debian/salt-$fname.init /etc/init.d/salt-$fname + fi + chmod +x /etc/init.d/salt-$fname + update-rc.d salt-$fname defaults + done +} + +install_debian_restart_daemons() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + /etc/init.d/salt-$fname stop > /dev/null 2>&1 + /etc/init.d/salt-$fname start + done +} +# +# Ended Debian Install Functions +# +############################################################################## + +############################################################################## +# +# Fedora Install Functions +# +install_fedora_deps() { + yum install -y PyYAML libyaml m2crypto python-crypto python-jinja2 \ + python-msgpack python-zmq || return 1 + return 0 +} + +install_fedora_stable() { + packages="" + if [ $INSTALL_MINION -eq $BS_TRUE ]; then + packages="${packages} salt-minion" + fi + if [ $INSTALL_MASTER -eq $BS_TRUE ] || [ $INSTALL_SYNDIC -eq $BS_TRUE ]; then + packages="${packages} salt-master" + fi + yum install -y ${packages} || return 1 + return 0 +} + +install_fedora_git_deps() { + install_fedora_deps || return 1 + yum install -y git || return 1 + + __git_clone_and_checkout || return 1 + + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + TEMP_CONFIG_DIR="${SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_fedora_git() { + python setup.py install || return 1 + return 0 +} + +install_fedora_git_post() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + copyfile ${SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-$fname.service /lib/systemd/system/salt-$fname.service + + systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) + sleep 0.1 + systemctl daemon-reload + done +} + +install_fedora_restart_daemons() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + systemctl stop salt-$fname > /dev/null 2>&1 + systemctl start salt-$fname.service + done +} +# +# Ended Fedora Install Functions +# +############################################################################## + +############################################################################## +# +# CentOS Install Functions +# +install_centos_stable_deps() { + if [ $CPU_ARCH_L = "i686" ]; then + EPEL_ARCH="i386" + else + EPEL_ARCH=$CPU_ARCH_L + fi + if [ $DISTRO_MAJOR_VERSION -eq 5 ]; then + rpm -Uvh --force http://mirrors.kernel.org/fedora-epel/5/${EPEL_ARCH}/epel-release-5-4.noarch.rpm || return 1 + elif [ $DISTRO_MAJOR_VERSION -eq 6 ]; then + rpm -Uvh --force http://mirrors.kernel.org/fedora-epel/6/${EPEL_ARCH}/epel-release-6-8.noarch.rpm || return 1 + else + echoerror "Failed add EPEL repository support." + return 1 + fi + + yum -y update || return 1 + + if [ $DISTRO_MAJOR_VERSION -eq 5 ]; then + yum -y install PyYAML python26-m2crypto m2crypto python26 \ + python26-crypto python26-msgpack python26-zmq \ + python26-jinja2 --enablerepo=epel || return 1 + else + yum -y install PyYAML m2crypto python-crypto python-msgpack \ + python-zmq python-jinja2 --enablerepo=epel || return 1 + fi + return 0 +} + +install_centos_stable() { + packages="" + if [ $INSTALL_MINION -eq $BS_TRUE ]; then + packages="${packages} salt-minion" + fi + if [ $INSTALL_MASTER -eq $BS_TRUE ] || [ $INSTALL_SYNDIC -eq $BS_TRUE ]; then + packages="${packages} salt-master" + fi + yum -y install ${packages} --enablerepo=epel || return 1 + return 0 +} + +install_centos_stable_post() { + for fname in minion master syndic; do + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ ! -f /sbin/initctl ] && [ -f /etc/init.d/salt-$fname ]; then + # Still in SysV init!? + /sbin/chkconfig salt-$fname on + fi + done +} + +install_centos_git_deps() { + install_centos_stable_deps || return 1 + yum -y install git --enablerepo=epel || return 1 + + __git_clone_and_checkout || return 1 + + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + TEMP_CONFIG_DIR="${SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_centos_git() { + if [ $DISTRO_MAJOR_VERSION -eq 5 ]; then + python2.6 setup.py install || return 1 + else + python2 setup.py install || return 1 + fi + return 0 +} + +install_centos_git_post() { + for fname in master minion syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ -f /sbin/initctl ]; then + # We have upstart support + /sbin/initctl status salt-$fname > /dev/null 2>&1 + if [ $? -eq 1 ]; then + # upstart does not know about our service, let's copy the proper file + copyfile ${SALT_GIT_CHECKOUT_DIR}/pkg/salt-$fname.upstart /etc/init/salt-$fname.conf + fi + # Still in SysV init?! + elif [ ! -f /etc/init.d/salt-$fname ]; then + copyfile ${SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-${fname} /etc/init.d/ + chmod +x /etc/init.d/salt-${fname} + /sbin/chkconfig salt-${fname} on + fi + done +} + +install_centos_restart_daemons() { + for fname in minion master syndic; do + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ -f /sbin/initctl ]; then + # We have upstart support + /sbin/initctl status salt-$fname > /dev/null 2>&1 + if [ $? -eq 0 ]; then + # upstart knows about this service. + # Let's try to stop it, and then start it + /sbin/initctl stop salt-$fname > /dev/null 2>&1 + /sbin/initctl start salt-$fname > /dev/null 2>&1 + # Restart service + [ $? -eq 0 ] && continue + # We failed to start the service, let's test the SysV code bellow + fi + fi + + if [ -f /etc/init.d/salt-$fname ]; then + # Still in SysV init!? + /etc/init.d/salt-$fname stop > /dev/null 2>&1 + /etc/init.d/salt-$fname start + fi + done +} +# +# Ended CentOS Install Functions +# +############################################################################## + +############################################################################## +# +# RedHat Install Functions +# +install_red_hat_linux_stable_deps() { + install_centos_stable_deps || return 1 + return 0 +} + +install_red_hat_linux_git_deps() { + install_centos_git_deps || return 1 + return 0 +} + +install_red_hat_enterprise_linux_stable_deps() { + install_red_hat_linux_stable_deps || return 1 + return 0 +} + +install_red_hat_enterprise_linux_git_deps() { + install_red_hat_linux_git_deps || return 1 + return 0 +} + +install_red_hat_enterprise_server_stable_deps() { + install_red_hat_linux_stable_deps || return 1 + return 0 +} + +install_red_hat_enterprise_server_git_deps() { + install_red_hat_linux_git_deps || return 1 + return 0 +} + +install_red_hat_linux_stable() { + install_centos_stable || return 1 + return 0 +} + +install_red_hat_linux_git() { + install_centos_git || return 1 + return 0 +} + +install_red_hat_enterprise_linux_stable() { + install_red_hat_linux_stable || return 1 + return 0 +} + +install_red_hat_enterprise_linux_git() { + install_red_hat_linux_git || return 1 + return 0 +} + +install_red_hat_enterprise_server_stable() { + install_red_hat_linux_stable || return 1 + return 0 +} + +install_red_hat_enterprise_server_git() { + install_red_hat_linux_git || return 1 + return 0 +} + +install_red_hat_linux_stable_post() { + install_centos_stable_post || return 1 + return 0 +} + +install_red_hat_linux_restart_daemons() { + install_centos_restart_daemons || return 1 + return 0 +} + +install_red_hat_linux_git_post() { + install_centos_git_post || return 1 + return 0 +} + +install_red_hat_enterprise_linux_stable_post() { + install_red_hat_linux_stable_post || return 1 + return 0 +} + +install_red_hat_enterprise_linux_restart_daemons() { + install_red_hat_linux_restart_daemons || return 1 + return 0 +} + +install_red_hat_enterprise_linux_git_post() { + install_red_hat_linux_git_post || return 1 + return 0 +} + +install_red_hat_enterprise_server_stable_post() { + install_red_hat_linux_stable_post || return 1 + return 0 +} + +install_red_hat_enterprise_server_restart_daemons() { + install_red_hat_linux_restart_daemons || return 1 + return 0 +} + +install_red_hat_enterprise_server_git_post() { + install_red_hat_linux_git_post || return 1 + return 0 +} +# +# Ended RedHat Install Functions +# +############################################################################## + +############################################################################## +# +# Amazon Linux AMI Install Functions +# +install_amazon_linux_ami_deps() { + # Acording to http://aws.amazon.com/amazon-linux-ami/faqs/#epel we should + # enable the EPEL 6 repo + if [ $CPU_ARCH_L = "i686" ]; then + EPEL_ARCH="i386" + else + EPEL_ARCH=$CPU_ARCH_L + fi + rpm -Uvh --force http://mirrors.kernel.org/fedora-epel/6/${EPEL_ARCH}/epel-release-6-8.noarch.rpm || return 1 + yum -y update || return 1 + yum -y install PyYAML m2crypto python-crypto python-msgpack python-zmq \ + python-ordereddict python-jinja2 --enablerepo=epel || return 1 +} + +install_amazon_linux_ami_git_deps() { + install_amazon_linux_ami_deps || return 1 + yum -y install git --enablerepo=epel || return 1 + + __git_clone_and_checkout || return 1 + + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + TEMP_CONFIG_DIR="${SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_amazon_linux_ami_stable() { + install_centos_stable || return 1 + return 0 +} + +install_amazon_linux_ami_stable_post() { + install_centos_stable_post || return 1 + return 0 +} + +install_amazon_linux_ami_restart_daemons() { + install_centos_restart_daemons || return 1 + return 0 +} + +install_amazon_linux_ami_git() { + install_centos_git || return 1 + return 0 +} + +install_amazon_linux_ami_git_post() { + install_centos_git_post || return 1 + return 0 +} +# +# Ended Amazon Linux AMI Install Functions +# +############################################################################## + +############################################################################## +# +# Arch Install Functions +# +install_arch_linux_stable_deps() { + grep '\[salt\]' /etc/pacman.conf >/dev/null 2>&1 || echo '[salt] +Server = http://intothesaltmine.org/archlinux +' >> /etc/pacman.conf +} + +install_arch_linux_git_deps() { + grep '\[salt\]' /etc/pacman.conf >/dev/null 2>&1 || echo '[salt] +Server = http://intothesaltmine.org/archlinux +' >> /etc/pacman.conf + + pacman -Sy --noconfirm pacman || return 1 + pacman -Sy --noconfirm git python2-crypto python2-distribute \ + python2-jinja python2-m2crypto python2-markupsafe python2-msgpack \ + python2-psutil python2-yaml python2-pyzmq zeromq || return 1 + + __git_clone_and_checkout || return 1 + + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + TEMP_CONFIG_DIR="${SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_arch_linux_stable() { + pacman -Sy --noconfirm pacman || return 1 + pacman -Syu --noconfirm salt || return 1 + return 0 +} + +install_arch_linux_git() { + python2 setup.py install || return 1 + return 0 +} + +install_arch_linux_post() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ -f /usr/bin/systemctl ]; then + # Using systemd + /usr/bin/systemctl is-enabled salt-$fname.service > /dev/null 2>&1 || ( + /usr/bin/systemctl preset salt-$fname.service > /dev/null 2>&1 && + /usr/bin/systemctl enable salt-$fname.service > /dev/null 2>&1 + ) + sleep 0.1 + /usr/bin/systemctl daemon-reload + continue + fi + + # XXX: How do we enable old Arch init.d scripts? + done +} + +install_arch_linux_git_post() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ -f /usr/bin/systemctl ]; then + copyfile ${SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-$fname.service /lib/systemd/system/salt-$fname.service + + /usr/bin/systemctl is-enabled salt-$fname.service > /dev/null 2>&1 || ( + /usr/bin/systemctl preset salt-$fname.service > /dev/null 2>&1 && + /usr/bin/systemctl enable salt-$fname.service > /dev/null 2>&1 + ) + sleep 0.1 + /usr/bin/systemctl daemon-reload + continue + fi + + # SysV init!? + copyfile ${SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-$fname /etc/rc.d/init.d/salt-$fname + chmod +x /etc/rc.d/init.d/salt-$fname + done +} + +install_arch_linux_restart_daemons() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ -f /usr/bin/systemctl ]; then + /usr/bin/systemctl stop salt-$fname.service > /dev/null 2>&1 + /usr/bin/systemctl start salt-$fname.service + continue + fi + /etc/rc.d/salt-$fname stop > /dev/null 2>&1 + /etc/rc.d/salt-$fname start + done +} +# +# Ended Arch Install Functions +# +############################################################################## + +############################################################################## +# +# FreeBSD Install Functions +# +__freebsd_get_packagesite() { + if [ $CPU_ARCH_L = "amd64" ]; then + BSD_ARCH="x86:64" + elif [ $CPU_ARCH_L = "x86_64" ]; then + BSD_ARCH="x86:64" + elif [ $CPU_ARCH_L = "i386" ]; then + BSD_ARCH="x86:32" + elif [ $CPU_ARCH_L = "i686" ]; then + BSD_ARCH="x86:32" + fi + + # Since the variable might not be set, don't, momentarily treat it as a failure + set +o nounset + + if [ "x${PACKAGESITE}" = "x" ]; then + echowarn "The environment variable PACKAGESITE is not set." + echowarn "The installation will, most likely fail since pkgbeta.freebsd.org does not yet contain any packages" + fi + BS_PACKAGESITE=${PACKAGESITE:-"http://pkgbeta.freebsd.org/freebsd:${DISTRO_MAJOR_VERSION}:${BSD_ARCH}/latest"} + + # Treat unset variables as errors once more + set -o nounset +} + +install_freebsd_9_stable_deps() { + __freebsd_get_packagesite + + fetch "${BS_PACKAGESITE}/Latest/pkg.txz" || return 1 + tar xf ./pkg.txz -s ",/.*/,,g" "*/pkg-static" || return 1 + ./pkg-static add ./pkg.txz || return 1 + /usr/local/sbin/pkg2ng || return 1 + echo "PACKAGESITE: ${BS_PACKAGESITE}" > /usr/local/etc/pkg.conf + + /usr/local/sbin/pkg install -y swig || return 1 + return 0 +} + +install_freebsd_git_deps() { + __freebsd_get_packagesite + + fetch "${BS_PACKAGESITE}/Latest/pkg.txz" || return 1 + tar xf ./pkg.txz -s ",/.*/,,g" "*/pkg-static" || return 1 + ./pkg-static add ./pkg.txz || return 1 + /usr/local/sbin/pkg2ng || return 1 + echo "PACKAGESITE: ${BS_PACKAGESITE}" > /usr/local/etc/pkg.conf + + /usr/local/sbin/pkg install -y swig || return 1 + + __git_clone_and_checkout || return 1 + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + TEMP_CONFIG_DIR="${SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_freebsd_9_stable() { + /usr/local/sbin/pkg install -y sysutils/py-salt || return 1 + return 0 +} + +install_freebsd_git() { + /usr/local/sbin/pkg install -y git sysutils/py-salt || return 1 + /usr/local/sbin/pkg delete -y sysutils/py-salt || return 1 + + /usr/local/bin/python setup.py install || return 1 + return 0 +} + +install_freebsd_9_stable_post() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + enable_string="salt_${fname}_enable=\"YES\"" + grep "$enable_string" /etc/rc.conf >/dev/null 2>&1 + [ $? -eq 1 ] && echo "$enable_string" >> /etc/rc.conf + + [ -f /usr/local/etc/salt/${fname}.sample ] && copyfile /usr/local/etc/salt/${fname}.sample /usr/local/etc/salt/${fname} + + if [ $fname = "minion" ] ; then + grep "salt_minion_paths" /etc/rc.conf >/dev/null 2>&1 + [ $? -eq 1 ] && echo "salt_minion_paths=\"/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin\"" >> /etc/rc.conf + fi + + done +} + +install_freebsd_git_post() { + install_freebsd_9_stable_post || return 1 + return 0 +} + +install_freebsd_restart_daemons() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + service salt_$fname stop > /dev/null 2>&1 + service salt_$fname start + done +} +# +# Ended FreeBSD Install Functions +# +############################################################################## + +############################################################################## +# +# SmartOS Install Functions +# +install_smartos_deps() { + check_pip_allowed + echowarn "PyZMQ will be installed using pip" + + ZEROMQ_VERSION='3.2.2' + pkgin -y in libtool-base autoconf automake libuuid gcc-compiler gmake \ + python27 py27-pip py27-setuptools py27-yaml py27-crypto swig || return 1 + [ -d zeromq-${ZEROMQ_VERSION} ] || ( + wget http://download.zeromq.org/zeromq-${ZEROMQ_VERSION}.tar.gz && + tar -xvf zeromq-${ZEROMQ_VERSION}.tar.gz + ) + cd zeromq-${ZEROMQ_VERSION} + ./configure || return 1 + make || return 1 + make install || return 1 + + pip-2.7 install pyzmq || return 1 + + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + # Let's set the configuration directory to /tmp + TEMP_CONFIG_DIR="/tmp" + CONFIG_SALT_FUNC="config_salt" + + # Let's download, since they were not provided, the default configuration files + if [ ! -f /etc/salt/minion ] && [ ! -f $TEMP_CONFIG_DIR/minion ]; then + curl -sk -o $TEMP_CONFIG_DIR/minion -L \ + https://raw.github.com/saltstack/salt/develop/conf/minion || return 1 + fi + if [ ! -f /etc/salt/master ] && [ ! -f $TEMP_CONFIG_DIR/master ]; then + curl -sk -o $TEMP_CONFIG_DIR/master -L \ + https://raw.github.com/saltstack/salt/develop/conf/master || return 1 + fi + fi + + return 0 + +} + +install_smartos_git_deps() { + install_smartos_deps || return 1 + pkgin -y in scmgit || return 1 + + __git_clone_and_checkout || return 1 + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + TEMP_CONFIG_DIR="${SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_smartos_stable() { + USE_SETUPTOOLS=1 pip-2.7 install salt || return 1 + return 0 +} + +install_smartos_git() { + # Use setuptools in order to also install dependencies + USE_SETUPTOOLS=1 /opt/local/bin/python setup.py install || return 1 + return 0 +} + +install_smartos_post() { + # Install manifest files if needed. + for fname in minion master syndic; do + svcs network/salt-$fname > /dev/null 2>&1 + if [ $? -eq 1 ]; then + if [ ! -f $TEMP_CONFIG_DIR/salt-$fname.xml ]; then + curl -sk -o $TEMP_CONFIG_DIR/salt-$fname.xml -L https://raw.github.com/saltstack/salt/develop/pkg/solaris/salt-$fname.xml + fi + svccfg import $TEMP_CONFIG_DIR/salt-$fname.xml + fi + done +} + +install_smartos_git_post() { + # Install manifest files if needed. + for fname in minion master syndic; do + svcs network/salt-$fname > /dev/null 2>&1 + if [ $? -eq 1 ]; then + svccfg import ${SALT_GIT_CHECKOUT_DIR}/pkg/solaris/salt-$fname.xml + fi + done +} + +install_smartos_restart_daemons() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + # Stop if running && Start service + svcadm disable salt-$fname > /dev/null 2>&1 + svcadm enable salt-$fname + done +} +# +# Ended SmartOS Install Functions +# +############################################################################## + +############################################################################## +# +# openSUSE Install Functions. +# +install_opensuse_stable_deps() { + DISTRO_REPO="openSUSE_${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" + + # Is the repository already known + $(zypper repos | grep devel_languages_python >/dev/null 2>&1) + if [ $? -eq 1 ]; then + # zypper does not yet know nothing about devel_languages_python + zypper --non-interactive addrepo --refresh \ + http://download.opensuse.org/repositories/devel:/languages:/python/${DISTRO_REPO}/devel:languages:python.repo || return 1 + fi + + zypper --gpg-auto-import-keys --non-interactive refresh || return 1 + zypper --non-interactive install --auto-agree-with-licenses libzmq3 python \ + python-Jinja2 python-M2Crypto python-PyYAML python-msgpack-python \ + python-pycrypto python-pyzmq || return 1 + return 0 +} + +install_opensuse_git_deps() { + install_opensuse_stable_deps || return 1 + zypper --non-interactive install --auto-agree-with-licenses git || return 1 + + __git_clone_and_checkout || return 1 + + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + TEMP_CONFIG_DIR="${SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_opensuse_stable() { + packages="" + if [ $INSTALL_MINION -eq $BS_TRUE ]; then + packages="${packages} salt-minion" + fi + if [ $INSTALL_MASTER -eq $BS_TRUE ]; then + packages="${packages} salt-master" + fi + if [ $INSTALL_SYNDIC -eq $BS_TRUE ]; then + packages="${packages} salt-syndic" + fi + zypper --non-interactive install --auto-agree-with-licenses $packages || return 1 + return 0 +} + +install_opensuse_git() { + python setup.py install --prefix=/usr || return 1 + return 0 +} + +install_opensuse_stable_post() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ -f /bin/systemctl ]; then + systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) + sleep 0.1 + systemctl daemon-reload + continue + fi + + /sbin/chkconfig --add salt-$fname + /sbin/chkconfig salt-$fname on + + done +} + +install_opensuse_git_post() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ -f /bin/systemctl ]; then + copyfile ${SALT_GIT_CHECKOUT_DIR}/pkg/salt-$fname.service /lib/systemd/system/salt-$fname.service + continue + fi + + copyfile ${SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-$fname /etc/init.d/salt-$fname + chmod +x /etc/init.d/salt-$fname + + done + + install_opensuse_stable_post +} + +install_opensuse_restart_daemons() { + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ -f /bin/systemctl ]; then + systemctl stop salt-$fname > /dev/null 2>&1 + systemctl start salt-$fname.service + continue + fi + + service salt-$fname stop > /dev/null 2>&1 + service salt-$fname start + + done +} +# +# End of openSUSE Install Functions. +# +############################################################################## + +############################################################################## +# +# SuSE Install Functions. +# +install_suse_11_stable_deps() { + SUSE_PATCHLEVEL=$(awk '/PATCHLEVEL/ {print $3}' /etc/SuSE-release ) + if [ "x${SUSE_PATCHLEVEL}" != "x" ]; then + DISTRO_PATCHLEVEL="_SP${SUSE_PATCHLEVEL}" + fi + DISTRO_REPO="SLE_${DISTRO_MAJOR_VERSION}${DISTRO_PATCHLEVEL}" + + # Is the repository already known + $(zypper repos | grep devel_languages_python >/dev/null 2>&1) + if [ $? -eq 1 ]; then + # zypper does not yet know nothing about devel_languages_python + zypper --non-interactive addrepo --refresh \ + http://download.opensuse.org/repositories/devel:/languages:/python/${DISTRO_REPO}/devel:languages:python.repo || return 1 + fi + + zypper --gpg-auto-import-keys --non-interactive refresh || return 1 + if [ $SUSE_PATCHLEVEL -eq 1 ]; then + check_pip_allowed + echowarn "PyYaml will be installed using pip" + zypper --non-interactive install --auto-agree-with-licenses libzmq3 python \ + python-Jinja2 'python-M2Crypto>=0.21' python-msgpack-python \ + python-pycrypto python-pyzmq python-pip || return 1 + # There's no python-PyYaml in SP1, let's install it using pip + pip install PyYaml || return 1 + else + zypper --non-interactive install --auto-agree-with-licenses libzmq3 python \ + python-Jinja2 'python-M2Crypto>=0.21' python-PyYAML python-msgpack-python \ + python-pycrypto python-pyzmq || return 1 + fi + + # PIP based installs need to copy configuration files "by hand". + if [ $SUSE_PATCHLEVEL -eq 1 ]; then + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + # Let's set the configuration directory to /tmp + TEMP_CONFIG_DIR="/tmp" + CONFIG_SALT_FUNC="config_salt" + + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + # Syndic uses the same configuration file as the master + [ $fname = "syndic" ] && fname=master + + # Let's download, since they were not provided, the default configuration files + if [ ! -f /etc/salt/$fname ] && [ ! -f $TEMP_CONFIG_DIR/$fname ]; then + curl -sk -o $TEMP_CONFIG_DIR/$fname -L \ + https://raw.github.com/saltstack/salt/develop/conf/$fname || return 1 + fi + done + fi + fi + return 0 +} + +install_suse_11_git_deps() { + install_suse_11_stable_deps || return 1 + zypper --non-interactive install --auto-agree-with-licenses git || return 1 + + __git_clone_and_checkout || return 1 + + # Let's trigger config_salt() + if [ "$TEMP_CONFIG_DIR" = "null" ]; then + TEMP_CONFIG_DIR="${SALT_GIT_CHECKOUT_DIR}/conf/" + CONFIG_SALT_FUNC="config_salt" + fi + + return 0 +} + +install_suse_11_stable() { + if [ $SUSE_PATCHLEVEL -gt 1 ]; then + install_opensuse_stable || return 1 + else + # USE_SETUPTOOLS=1 To work around + # error: option --single-version-externally-managed not recognized + USE_SETUPTOOLS=1 pip install salt || return 1 + fi + return 0 +} + +install_suse_11_git() { + install_opensuse_git || return 1 + return 0 +} + +install_suse_11_stable_post() { + if [ $SUSE_PATCHLEVEL -gt 1 ]; then + install_opensuse_stable_post || return 1 + else + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ -f /bin/systemctl ]; then + curl -k -L https://github.com/saltstack/salt/raw/develop/pkg/salt-$fname.service \ + -o /lib/systemd/system/salt-$fname.service || return 1 + continue + fi + + curl -k -L https://github.com/saltstack/salt/raw/develop/pkg/rpm/salt-$fname \ + -o /etc/init.d/salt-$fname || return 1 + chmod +x /etc/init.d/salt-$fname + + done + fi + return 0 +} + +install_suse_11_git_post() { + install_opensuse_git_post || return 1 + return 0 +} + +install_suse_11_restart_daemons() { + install_opensuse_restart_daemons || return 1 + return 0 +} +# +# End of SuSE Install Functions. +# +############################################################################## + +############################################################################## +# +# Default minion configuration function. Matches ANY distribution as long as +# the -c options is passed. +# +config_salt() { + # If the configuration directory is not passed, return + [ "$TEMP_CONFIG_DIR" = "null" ] && return + + CONFIGURED_ANYTHING=$BS_FALSE + + PKI_DIR=$SALT_ETC_DIR/pki + + # Let's create the necessary directories + [ -d $SALT_ETC_DIR ] || mkdir $SALT_ETC_DIR || return 1 + [ -d $PKI_DIR ] || mkdir -p $PKI_DIR && chmod 700 $PKI_DIR || return 1 + + if [ $INSTALL_MINION -eq $BS_TRUE ]; then + # Create the PKI directory + [ -d $PKI_DIR/minion ] || mkdir -p $PKI_DIR/minion && chmod 700 $PKI_DIR/minion || return 1 + + # Copy the minions configuration if found + if [ -f "$TEMP_CONFIG_DIR/minion" ]; then + mv "$TEMP_CONFIG_DIR/minion" /etc/salt || return 1 + CONFIGURED_ANYTHING=$BS_TRUE + fi + + # Copy the minion's keys if found + if [ -f "$TEMP_CONFIG_DIR/minion.pem" ]; then + mv "$TEMP_CONFIG_DIR/minion.pem" $PKI_DIR/minion/ || return 1 + chmod 400 $PKI_DIR/minion/minion.pem || return 1 + CONFIGURED_ANYTHING=$BS_TRUE + fi + if [ -f "$TEMP_CONFIG_DIR/minion.pub" ]; then + mv "$TEMP_CONFIG_DIR/minion.pub" $PKI_DIR/minion/ || return 1 + chmod 664 $PKI_DIR/minion/minion.pub || return 1 + CONFIGURED_ANYTHING=$BS_TRUE + fi + fi + + + if [ $INSTALL_MASTER -eq $BS_TRUE ] || [ $INSTALL_SYNDIC -eq $BS_TRUE ]; then + # Create the PKI directory + [ -d $PKI_DIR/master ] || mkdir -p $PKI_DIR/master && chmod 700 $PKI_DIR/master || return 1 + + # Copy the masters configuration if found + if [ -f "$TEMP_CONFIG_DIR/master" ]; then + mv "$TEMP_CONFIG_DIR/master" /etc/salt || return 1 + CONFIGURED_ANYTHING=$BS_TRUE + fi + + # Copy the master's keys if found + if [ -f "$TEMP_CONFIG_DIR/master.pem" ]; then + mv "$TEMP_CONFIG_DIR/master.pem" $PKI_DIR/master/ || return 1 + chmod 400 $PKI_DIR/master/master.pem || return 1 + CONFIGURED_ANYTHING=$BS_TRUE + fi + if [ -f "$TEMP_CONFIG_DIR/master.pub" ]; then + mv "$TEMP_CONFIG_DIR/master.pub" $PKI_DIR/master/ || return 1 + chmod 664 $PKI_DIR/master/master.pub || return 1 + CONFIGURED_ANYTHING=$BS_TRUE + fi + fi + + if [ $CONFIG_ONLY -eq $BS_TRUE ] && [ $CONFIGURED_ANYTHING -eq $BS_FALSE ]; then + echowarn "No configuration or keys were copied over. No configuration was done!" + exit 0 + fi + return 0 +} +# +# Ended Default Configuration function +# +############################################################################## + + +############################################################################## +# +# Default salt master minion keys pre-seed function. Matches ANY distribution +# as long as the -k option is passed. +# +preseed_master() { + # Create the PKI directory + [ -d $PKI_DIR/minions ] || mkdir -p $PKI_DIR/minions && chmod 700 $PKI_DIR/minions || return 1 + + for keyfile in $(ls $TEMP_KEYS_DIR); do + src_keyfile="${TEMP_KEYS_DIR}/${keyfile}" + dst_keyfile="${PKI_DIR}/minions/${keyfile}" + + # If it's not a file, skip to the next + [ ! -f $keyfile_path ] && continue + + movefile "$src_keyfile" "$dst_keyfile" || return 1 + chmod 664 $dst_keyfile || return 1 + done + + return 0 +} +# +# Ended Default Salt Master Pre-Seed minion keys function +# +############################################################################## + + +############################################################################## +# +# This function checks if all of the installed daemons are running or not. +# +daemons_running() { + FAILED_DAEMONS=0 + for fname in minion master syndic; do + + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ "x$(ps aux | grep -v grep | grep salt-$fname)" = "x" ]; then + echoerror "salt-$fname was not found running" + FAILED_DAEMONS=$(expr $FAILED_DAEMONS + 1) + fi + done + return $FAILED_DAEMONS +} +# +# Ended daemons running check function +# +############################################################################## + + +#============================================================================= +# LET'S PROCEED WITH OUR INSTALLATION +#============================================================================= +# Let's get the dependencies install function +DEP_FUNC_NAMES="install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_${ITYPE}_deps" +DEP_FUNC_NAMES="$DEP_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_${ITYPE}_deps" +DEP_FUNC_NAMES="$DEP_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_deps" +DEP_FUNC_NAMES="$DEP_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_deps" +DEP_FUNC_NAMES="$DEP_FUNC_NAMES install_${DISTRO_NAME_L}_${ITYPE}_deps" +DEP_FUNC_NAMES="$DEP_FUNC_NAMES install_${DISTRO_NAME_L}_deps" + +DEPS_INSTALL_FUNC="null" +for DEP_FUNC_NAME in $(__strip_duplicates $DEP_FUNC_NAMES); do + if __function_defined $DEP_FUNC_NAME; then + DEPS_INSTALL_FUNC=$DEP_FUNC_NAME + break + fi +done + + +# Let's get the minion config function +CONFIG_SALT_FUNC="null" +if [ "$TEMP_CONFIG_DIR" != "null" ]; then + + CONFIG_FUNC_NAMES="config_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_${ITYPE}_salt" + CONFIG_FUNC_NAMES="$CONFIG_FUNC_NAMES config_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_${ITYPE}_salt" + CONFIG_FUNC_NAMES="$CONFIG_FUNC_NAMES config_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_salt" + CONFIG_FUNC_NAMES="$CONFIG_FUNC_NAMES config_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_salt" + CONFIG_FUNC_NAMES="$CONFIG_FUNC_NAMES config_${DISTRO_NAME_L}_${ITYPE}_salt" + CONFIG_FUNC_NAMES="$CONFIG_FUNC_NAMES config_${DISTRO_NAME_L}_salt" + CONFIG_FUNC_NAMES="$CONFIG_FUNC_NAMES config_salt" + + for FUNC_NAME in $(__strip_duplicates $CONFIG_FUNC_NAMES); do + if __function_defined $FUNC_NAME; then + CONFIG_SALT_FUNC=$FUNC_NAME + break + fi + done +fi + + +# Let's get the pre-seed master function +PRESEED_MASTER_FUNC="null" +if [ "$TEMP_CONFIG_DIR" != "null" ]; then + + PRESEED_FUNC_NAMES="preseed_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_${ITYPE}_master" + PRESEED_FUNC_NAMES="$PRESEED_FUNC_NAMES preseed_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_${ITYPE}_master" + PRESEED_FUNC_NAMES="$PRESEED_FUNC_NAMES preseed_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_master" + PRESEED_FUNC_NAMES="$PRESEED_FUNC_NAMES preseed_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_master" + PRESEED_FUNC_NAMES="$PRESEED_FUNC_NAMES preseed_${DISTRO_NAME_L}_${ITYPE}_master" + PRESEED_FUNC_NAMES="$PRESEED_FUNC_NAMES preseed_${DISTRO_NAME_L}_master" + PRESEED_FUNC_NAMES="$PRESEED_FUNC_NAMES preseed_master" + + for FUNC_NAME in $(__strip_duplicates $PRESEED_FUNC_NAMES); do + if __function_defined $FUNC_NAME; then + PRESEED_MASTER_FUNC=$FUNC_NAME + break + fi + done +fi + + +# Let's get the install function +INSTALL_FUNC_NAMES="install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_${ITYPE}" +INSTALL_FUNC_NAMES="$INSTALL_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_${ITYPE}" +INSTALL_FUNC_NAMES="$INSTALL_FUNC_NAMES install_${DISTRO_NAME_L}_${ITYPE}" + +INSTALL_FUNC="null" +for FUNC_NAME in $(__strip_duplicates $INSTALL_FUNC_NAMES); do + if __function_defined $FUNC_NAME; then + INSTALL_FUNC=$FUNC_NAME + break + fi +done + + +# Let's get the post install function +POST_FUNC_NAMES="install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_${ITYPE}_post" +POST_FUNC_NAMES="$POST_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_${ITYPE}_post" +POST_FUNC_NAMES="$POST_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_post" +POST_FUNC_NAMES="$POST_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_post" +POST_FUNC_NAMES="$POST_FUNC_NAMES install_${DISTRO_NAME_L}_${ITYPE}_post" +POST_FUNC_NAMES="$POST_FUNC_NAMES install_${DISTRO_NAME_L}_post" + + +POST_INSTALL_FUNC="null" +for FUNC_NAME in $(__strip_duplicates $POST_FUNC_NAMES); do + if __function_defined $FUNC_NAME; then + POST_INSTALL_FUNC=$FUNC_NAME + break + fi +done + + +# Let's get the start daemons install function +STARTDAEMONS_FUNC_NAMES="install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_${ITYPE}_restart_daemons" +STARTDAEMONS_FUNC_NAMES="$STARTDAEMONS_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_${ITYPE}_restart_daemons" +STARTDAEMONS_FUNC_NAMES="$STARTDAEMONS_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_restart_daemons" +STARTDAEMONS_FUNC_NAMES="$STARTDAEMONS_FUNC_NAMES install_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_restart_daemons" +STARTDAEMONS_FUNC_NAMES="$STARTDAEMONS_FUNC_NAMES install_${DISTRO_NAME_L}_${ITYPE}_restart_daemons" +STARTDAEMONS_FUNC_NAMES="$STARTDAEMONS_FUNC_NAMES install_${DISTRO_NAME_L}_restart_daemons" + +STARTDAEMONS_INSTALL_FUNC="null" +for FUNC_NAME in $(__strip_duplicates $STARTDAEMONS_FUNC_NAMES); do + if __function_defined $FUNC_NAME; then + STARTDAEMONS_INSTALL_FUNC=$FUNC_NAME + break + fi +done + + +# Let's get the daemons running check function. +DAEMONS_RUNNING_FUNC="null" +DAEMONS_RUNNING_FUNC_NAMES="daemons_running_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}_${ITYPE}" +DAEMONS_RUNNING_FUNC_NAMES="$DAEMONS_RUNNING_FUNC_NAMES daemons_running_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}_${ITYPE}" +DAEMONS_RUNNING_FUNC_NAMES="$DAEMONS_RUNNING_FUNC_NAMES daemons_running_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}" +DAEMONS_RUNNING_FUNC_NAMES="$DAEMONS_RUNNING_FUNC_NAMES daemons_running_${DISTRO_NAME_L}${PREFIXED_DISTRO_MAJOR_VERSION}${PREFIXED_DISTRO_MINOR_VERSION}" +DAEMONS_RUNNING_FUNC_NAMES="$DAEMONS_RUNNING_FUNC_NAMES daemons_running_${DISTRO_NAME_L}_${ITYPE}" +DAEMONS_RUNNING_FUNC_NAMES="$DAEMONS_RUNNING_FUNC_NAMES daemons_running_${DISTRO_NAME_L}" +DAEMONS_RUNNING_FUNC_NAMES="$DAEMONS_RUNNING_FUNC_NAMES daemons_running" + +for FUNC_NAME in $(__strip_duplicates $DAEMONS_RUNNING_FUNC_NAMES); do + if __function_defined $FUNC_NAME; then + DAEMONS_RUNNING_FUNC=$FUNC_NAME + break + fi +done + + + +if [ $DEPS_INSTALL_FUNC = "null" ]; then + echoerror "No dependencies installation function found. Exiting..." + exit 1 +fi + +if [ $INSTALL_FUNC = "null" ]; then + echoerror "No installation function found. Exiting..." + exit 1 +fi + + +# Install dependencies +if [ $CONFIG_ONLY -eq $BS_FALSE ]; then + # Only execute function is not in config mode only + echoinfo "Running ${DEPS_INSTALL_FUNC}()" + $DEPS_INSTALL_FUNC + if [ $? -ne 0 ]; then + echoerror "Failed to run ${DEPS_INSTALL_FUNC}()!!!" + exit 1 + fi +fi + + +# Configure Salt +if [ "$TEMP_CONFIG_DIR" != "null" ] && [ "$CONFIG_SALT_FUNC" != "null" ]; then + echoinfo "Running ${CONFIG_SALT_FUNC}()" + $CONFIG_SALT_FUNC + if [ $? -ne 0 ]; then + echoerror "Failed to run ${CONFIG_SALT_FUNC}()!!!" + exit 1 + fi +fi + + +# Pre-Seed master keys +if [ "$TEMP_KEYS_DIR" != "null" ] && [ "$PRESEED_MASTER_FUNC" != "null" ]; then + echoinfo "Running ${PRESEED_MASTER_FUNC}()" + $PRESEED_MASTER_FUNC + if [ $? -ne 0 ]; then + echoerror "Failed to run ${PRESEED_MASTER_FUNC}()!!!" + exit 1 + fi +fi + + +# Install Salt +if [ $CONFIG_ONLY -eq $BS_FALSE ]; then + # Only execute function is not in config mode only + echoinfo "Running ${INSTALL_FUNC}()" + $INSTALL_FUNC + if [ $? -ne 0 ]; then + echoerror "Failed to run ${INSTALL_FUNC}()!!!" + exit 1 + fi +fi + + +# Run any post install function, Only execute function is not in config mode only +if [ $CONFIG_ONLY -eq $BS_FALSE ] && [ "$POST_INSTALL_FUNC" != "null" ]; then + echoinfo "Running ${POST_INSTALL_FUNC}()" + $POST_INSTALL_FUNC + if [ $? -ne 0 ]; then + echoerror "Failed to run ${POST_INSTALL_FUNC}()!!!" + exit 1 + fi +fi + + +# Run any start daemons function +if [ "$STARTDAEMONS_INSTALL_FUNC" != "null" ]; then + echoinfo "Running ${STARTDAEMONS_INSTALL_FUNC}()" + $STARTDAEMONS_INSTALL_FUNC + if [ $? -ne 0 ]; then + echoerror "Failed to run ${STARTDAEMONS_INSTALL_FUNC}()!!!" + exit 1 + fi +fi + +# Check if the installed daemons are running or not +if [ "$DAEMONS_RUNNING_FUNC" != "null" ]; then + sleep 3 # Sleep a little bit to let daemons start + echoinfo "Running ${DAEMONS_RUNNING_FUNC}()" + $DAEMONS_RUNNING_FUNC + if [ $? -ne 0 ]; then + echoerror "Failed to run ${DAEMONS_RUNNING_FUNC}()!!!" + + for fname in minion master syndic; do + # Skip if not meant to be installed + [ $fname = "minion" ] && [ $INSTALL_MINION -eq $BS_FALSE ] && continue + [ $fname = "master" ] && [ $INSTALL_MASTER -eq $BS_FALSE ] && continue + [ $fname = "syndic" ] && [ $INSTALL_SYNDIC -eq $BS_FALSE ] && continue + + if [ $ECHO_DEBUG -eq $BS_FALSE ]; then + echoerror "salt-$fname was not found running. Pass '-D' for additional debugging information..." + continue + fi + + + [ ! $SALT_ETC_DIR/$fname ] && [ $fname != "syndic" ] && echodebug "$SALT_ETC_DIR/$fname does not exist" + + echodebug "Running salt-$fname by hand outputs: $(nohup salt-$fname -l debug)" + + [ ! -f /var/log/salt/$fname ] && echodebug "/var/log/salt/$fname does not exist. Can't cat its contents!" && continue + + echodebug "DEAMON LOGS for $fname:" + echodebug "$(cat /var/log/salt/$fname)" + echo + done + + echodebug "Running Processes:" + echodebug "$(ps auxwww)" + + exit 1 + fi +fi + + +# Done! +if [ $CONFIG_ONLY -eq $BS_FALSE ]; then + echoinfo "Salt installed!" +else + echoinfo "Salt configured" +fi +exit 0 \ No newline at end of file diff --git a/conf/minion.conf b/conf/minion.conf new file mode 100644 index 0000000..39bf083 --- /dev/null +++ b/conf/minion.conf @@ -0,0 +1,14 @@ +## Minimal masterless minion configuration ## +## All other settings take the defaults ## + +master: localhost + +file_client: local + +file_roots: + base: + - /srv/local + +pillar_roots: + base: + - /srv/pillar \ No newline at end of file diff --git a/fabfile.py b/fabfile.py index f7b63eb..6a6b559 100644 --- a/fabfile.py +++ b/fabfile.py @@ -2,7 +2,7 @@ import re from fabric.api import cd, env, get, hide, local, put, require, run, settings, sudo, task -from fabric.contrib import files +from fabric.contrib import files, project # Directory structure PROJECT_ROOT = os.path.dirname(__file__) @@ -53,6 +53,28 @@ def setup_path(): env.settings = '%(project)s.settings.%(environment)s' % env +@task +def provision(): + """Provision server with masterless Salt minion.""" + # Install salt minion + with hide('running', 'stdout', 'stderr'): + installed = run('which salt-call') + if not installed: + bootstrap_file = os.path.join(CONF_ROOT, 'bootstrap-salt.sh') + put(bootstrap_file, '/tmp/bootstrap-salt.sh') + sudo('sh /tmp/bootstrap-salt.sh') + # Rsync local states and pillars + minion_file = os.path.join(CONF_ROOT, 'minion.conf') + put(minion_file, '/etc/salt/minion', use_sudo=True) + salt_root = os.path.join(CONF_ROOT, 'salt') + project.rsync_project(local_dir=salt_root, remote_dir='/tmp/salt', delete=True) + sudo('rm -rf /srv/*') + sudo('mv /tmp/salt/* /srv/') + sudo('chown root:root -R /srv/') + # Update to highstate + sudo('salt-call --local state.highstate') + + @task def supervisor_command(command): """Run a supervisorctl command.""" From b74d7150fffff244f85a2ed2dd9ccd1b9282bdf2 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 09:45:55 -0400 Subject: [PATCH 03/48] Update deploy to handle the initial clone. Don't SSH as the project user and instead update the project directory permissions. See #28 --- fabfile.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/fabfile.py b/fabfile.py index 6a6b559..716484d 100644 --- a/fabfile.py +++ b/fabfile.py @@ -139,18 +139,26 @@ def deploy(branch=None): env.branch = branch requirements = False migrations = False - # Fetch latest changes - with cd(env.code_root): - with settings(user=env.project_user): + if files.exists(env.code_root): + # Fetch latest changes + with cd(env.code_root): run('git fetch origin') - # Look for new requirements or migrations - changes = run("git diff origin/%(branch)s --stat-name-width=9999" % env) - requirements = match_changes(changes, r"requirements/") - migrations = match_changes(changes, r"/migrations/") - if requirements or migrations: - supervisor_command('stop %(environment)s:*' % env) - with settings(user=env.project_user): - run("git reset --hard origin/%(branch)s" % env) + # Look for new requirements or migrations + changes = run("git diff origin/%(branch)s --stat-name-width=9999" % env) + requirements = match_changes(changes, r"requirements/") + migrations = match_changes(changes, r"/migrations/") + if requirements or migrations: + supervisor_command('stop %(environment)s:*' % env) + with settings(user=env.project_user): + run("git reset --hard origin/%(branch)s" % env) + else: + # Initial clone + run('git clone %(repo)s %(code_root)s' % env) + with cd(env.code_root): + run('git checkout %(branch)s' % env) + requirements = True + migrations = True + sudo('chown %(project_user)s:%(project_user)s -R %(code_root)s' % env) if requirements: update_requirements() # New requirements might need new tables/migrations From 4361f3ee9cb96dbf198a4d0ca1eb03c1034b593d Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 10:16:37 -0400 Subject: [PATCH 04/48] No longer using users here. Users will be created/managed by Salt. --- conf/users/REMOVE | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 conf/users/REMOVE diff --git a/conf/users/REMOVE b/conf/users/REMOVE deleted file mode 100644 index 99b01d2..0000000 --- a/conf/users/REMOVE +++ /dev/null @@ -1,3 +0,0 @@ -This directory should contain the public ssh keys for the developers working on -the project. The name of the file should be the name of the user which will be -created by ``fab create_users``. From 16df2faf5295370809edfa7e98ed7f62742a72a0 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 10:41:32 -0400 Subject: [PATCH 05/48] Pull in common states from margarita repo. Add additional directory to minion file roots. --- conf/minion.conf | 1 + fabfile.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/conf/minion.conf b/conf/minion.conf index 39bf083..1583454 100644 --- a/conf/minion.conf +++ b/conf/minion.conf @@ -8,6 +8,7 @@ file_client: local file_roots: base: - /srv/local + - /srv/common pillar_roots: base: diff --git a/fabfile.py b/fabfile.py index 716484d..8d5efbb 100644 --- a/fabfile.py +++ b/fabfile.py @@ -70,6 +70,11 @@ def provision(): project.rsync_project(local_dir=salt_root, remote_dir='/tmp/salt', delete=True) sudo('rm -rf /srv/*') sudo('mv /tmp/salt/* /srv/') + # Pull common states + # TOOD: Need a way to pin these states to a particular branch/tag + sudo('rm -rf /tmp/common/') + run('git clone git://github.com/caktus/margarita.git /tmp/common/') + sudo('mv /tmp/common/ /srv/common/') sudo('chown root:root -R /srv/') # Update to highstate sudo('salt-call --local state.highstate') From 551f3b8f591573b8ea618ec309facbce4618a9dc Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 11:49:23 -0400 Subject: [PATCH 06/48] Set the minion id and configure the pillar tree based on the current environment. --- conf/minion.conf | 5 ++++- fabfile.py | 13 +++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/conf/minion.conf b/conf/minion.conf index 1583454..4ee950c 100644 --- a/conf/minion.conf +++ b/conf/minion.conf @@ -1,6 +1,8 @@ ## Minimal masterless minion configuration ## ## All other settings take the defaults ## +id: %(project)s-%(environment)s + master: localhost file_client: local @@ -12,4 +14,5 @@ file_roots: pillar_roots: base: - - /srv/pillar \ No newline at end of file + - /srv/pillar + - /src/pillar/%(environment)s \ No newline at end of file diff --git a/fabfile.py b/fabfile.py index 8d5efbb..8e9f080 100644 --- a/fabfile.py +++ b/fabfile.py @@ -54,8 +54,9 @@ def setup_path(): @task -def provision(): +def provision(common='master'): """Provision server with masterless Salt minion.""" + require('environment') # Install salt minion with hide('running', 'stdout', 'stderr'): installed = run('which salt-call') @@ -65,15 +66,19 @@ def provision(): sudo('sh /tmp/bootstrap-salt.sh') # Rsync local states and pillars minion_file = os.path.join(CONF_ROOT, 'minion.conf') - put(minion_file, '/etc/salt/minion', use_sudo=True) + files.upload_template(minion_file, '/etc/salt/minion', use_sudo=True, context=env) salt_root = os.path.join(CONF_ROOT, 'salt') - project.rsync_project(local_dir=salt_root, remote_dir='/tmp/salt', delete=True) + environments = ['staging', 'production'] + # Only include current environment's pillar tree + exclude = [os.path.join(salt_root, 'pillar', e) for e in environments if e != env.environment] + project.rsync_project(local_dir=salt_root, remote_dir='/tmp/salt', delete=True, exclude=exclude) sudo('rm -rf /srv/*') sudo('mv /tmp/salt/* /srv/') # Pull common states - # TOOD: Need a way to pin these states to a particular branch/tag sudo('rm -rf /tmp/common/') run('git clone git://github.com/caktus/margarita.git /tmp/common/') + with cd('/tmp/common/'): + run('git checkout %s' % common) sudo('mv /tmp/common/ /srv/common/') sudo('chown root:root -R /srv/') # Update to highstate From da434fd53b9e2597708ff7f1a7dba9cc755e4da8 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 12:59:48 -0400 Subject: [PATCH 07/48] Starting on local Salt states. --- conf/local/project/db.sls | 2 ++ conf/local/project/web.sls | 7 +++++++ conf/local/top.sls | 9 +++++++++ 3 files changed, 18 insertions(+) create mode 100644 conf/local/project/db.sls create mode 100644 conf/local/project/web.sls create mode 100644 conf/local/top.sls diff --git a/conf/local/project/db.sls b/conf/local/project/db.sls new file mode 100644 index 0000000..22b2209 --- /dev/null +++ b/conf/local/project/db.sls @@ -0,0 +1,2 @@ +include: + - postgres \ No newline at end of file diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls new file mode 100644 index 0000000..2255234 --- /dev/null +++ b/conf/local/project/web.sls @@ -0,0 +1,7 @@ +include: + - memcached + - postfix + - version-control + - nginx + - python + - supervisor \ No newline at end of file diff --git a/conf/local/top.sls b/conf/local/top.sls new file mode 100644 index 0000000..2aa27ac --- /dev/null +++ b/conf/local/top.sls @@ -0,0 +1,9 @@ +base: + '*': + - base + - sshd + - sshd.github + - locale.utf8 + - users.devs + - project.web + - project.db \ No newline at end of file From 9c6e17091b2e4127a3cef76b317200afb487181b Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 13:03:04 -0400 Subject: [PATCH 08/48] Adding environment pillars. --- conf/pillar/production/env.sls | 4 ++++ conf/pillar/staging/env.sls | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 conf/pillar/production/env.sls create mode 100644 conf/pillar/staging/env.sls diff --git a/conf/pillar/production/env.sls b/conf/pillar/production/env.sls new file mode 100644 index 0000000..211e069 --- /dev/null +++ b/conf/pillar/production/env.sls @@ -0,0 +1,4 @@ +environment: production + +# FIXME: Change to match production domain name +domain: example.com \ No newline at end of file diff --git a/conf/pillar/staging/env.sls b/conf/pillar/staging/env.sls new file mode 100644 index 0000000..c4849c3 --- /dev/null +++ b/conf/pillar/staging/env.sls @@ -0,0 +1,4 @@ +environment: staging + +# FIXME: Change to match staging domain name +domain: staging.example.com \ No newline at end of file From 7673f879d110e238fe1db180034abd81936e53bd Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 13:12:30 -0400 Subject: [PATCH 09/48] Project and dev user pillars. Create web related directories in the web state. --- conf/local/project/web.sls | 24 +++++++++++++++++++++++- conf/pillar/devs.sls | 6 ++++++ conf/pillar/project.sls | 2 ++ 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 conf/pillar/devs.sls create mode 100644 conf/pillar/project.sls diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index 2255234..4417e50 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -4,4 +4,26 @@ include: - version-control - nginx - python - - supervisor \ No newline at end of file + - supervisor + +project_user: + user.present: + - name: {{ pillar['project_name'] }} + - groups: [www-data] + +/var/www/: + file.directory: + - user: {{ pillar['project_name'] }} + - group: www-data + - makedirs: True + +/var/www/log/: + file.directory: + - user: {{ pillar['project_name'] }} + - group: www-data + - makedirs: True + +/home/www/env/: + virtualenv.managed: + - no_site_packages: True + - distribute: True \ No newline at end of file diff --git a/conf/pillar/devs.sls b/conf/pillar/devs.sls new file mode 100644 index 0000000..5db0a29 --- /dev/null +++ b/conf/pillar/devs.sls @@ -0,0 +1,6 @@ +users: + # This should be a list of developer users and thier public SSH keys + # example-user: + # groups: [admin, login] + # public_key: + # - ssh-dss AAAAB3NzaC1kc3MAAACBAP/dCNcAJED+pBsEwH01E4NU2xrvoB6H5gXkvQHWIKUuMF3MWXgSGhKpgVqLJKh+d0gwuKyI9344HM5dFs4z3E0JhI7Fg4uXIYu1SwuqnxO+D18WLVGt4gCn57JCjLy/c8LJWAHJWFb2v9t4fayC+oBiyEvpjI6VYIJnSvO3D4tjAAAAFQCNzcKi0sehN1Jw+zB6ccMlHt5E6wAAAIEAnW18UHG/O+RIkJazTJ7qFlOb79RS1nnvnHAvtfuiAPIBmeJcKoZkiQzeBYtFereSRHmSug9DsqHK6C5PrP36UMZYhDkqqp5gpJexmokI2kt3AVxJwro7cjy6Tq+0yt+lwqH4JEblybk7yPeRNC1ihnp2CSipC5LP1PydIcgN9/UAAACAeH1OxUzgCfpM06cfKL57gtjIS34ryCdkT2oYfYOANa8vahN2JqxB004o+z2CnQ9DkTqzzf9jUYI/qal19+zYhn8Bd/FdPVp+VTfRpR17fQKuTWrnF7g6jNVN2ltwHo6o99vrCzjHhJHZ2EXOODzAUrACptyfQv/ZCutkjAg44YE= copelco@montgomery.local diff --git a/conf/pillar/project.sls b/conf/pillar/project.sls new file mode 100644 index 0000000..4a921bb --- /dev/null +++ b/conf/pillar/project.sls @@ -0,0 +1,2 @@ +# FIXME: Change to match project name +project_name: example \ No newline at end of file From 3982f5239b59cd131441a76be462fab4312523d3 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 13:14:37 -0400 Subject: [PATCH 10/48] Adding top pillar to include project name, dev users, current environment and environment secrets. --- conf/pillar/top.sls | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 conf/pillar/top.sls diff --git a/conf/pillar/top.sls b/conf/pillar/top.sls new file mode 100644 index 0000000..57e5196 --- /dev/null +++ b/conf/pillar/top.sls @@ -0,0 +1,6 @@ +base: + "*": + - project + - devs + - env + - secrets \ No newline at end of file From 65f3b0b80d01ca201246c8babc243c19f18b83bb Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 13:19:06 -0400 Subject: [PATCH 11/48] Update root directory to match web state. --- fabfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabfile.py b/fabfile.py index 8e9f080..05ae094 100644 --- a/fabfile.py +++ b/fabfile.py @@ -46,7 +46,7 @@ def production(): def setup_path(): env.home = '/home/%(project_user)s/' % env - env.root = os.path.join(env.home, 'www', env.environment) + env.root = '/var/www/' env.code_root = os.path.join(env.root, env.project) env.virtualenv_root = os.path.join(env.root, 'env') env.db = '%s_%s' % (env.project, env.environment) From 1d1991d11d42f3beeaceaa9121971f848f687265 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Fri, 26 Apr 2013 13:32:30 -0400 Subject: [PATCH 12/48] add nginx conf --- conf/local/project/nginx.conf | 55 +++++++++++++++++++++++++++++++++++ conf/local/project/web.sls | 20 ++++++++++++- 2 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 conf/local/project/nginx.conf diff --git a/conf/local/project/nginx.conf b/conf/local/project/nginx.conf new file mode 100644 index 0000000..b0e3aea --- /dev/null +++ b/conf/local/project/nginx.conf @@ -0,0 +1,55 @@ +upstream django_server { + server 127.0.0.1:{{ server_port|default('8000') }} fail_timeout=0; +} + +server { + listen 80; + server_name {{ pillar['domain'] }}; + root {{ code_root }}/public; + + keepalive_timeout 5; + + access_log {{ log_dir }}/access.log; + error_log {{ log_dir }}/error.log; + + if ($host !~* ^({{ pillar['domain'] }}|www.{{ pillar['domain'] }})$) { + # Deny non-matching Host headers + return 444; + } + + location /robots.txt { + alias {{ code_root }}/public/robots.txt; + } + + location /media { + alias {{ code_root }}/public/media; + } + + location /static { + alias {{ code_root }}/public/static; + expires max; + } + + error_page 502 503 504 /502.html; + location /502.html { + alias {{ code_root }}/public/502.html; + } + + location / { + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Protocol http; + proxy_set_header Host $http_host; + proxy_redirect off; + proxy_buffering on; + proxy_intercept_errors on; + proxy_pass http://django_server; + } +} + +{# redirect other server names to the real one (http://nginx.org/en/docs/http/converting_rewrite_rules.html under "A redirect to a main site") #} +server { + listen 80 default_server; + server_name _; + return 301 http://{{ pillar['domain'] }}$request_uri; +} diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index 4417e50..c29a99b 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -26,4 +26,22 @@ project_user: /home/www/env/: virtualenv.managed: - no_site_packages: True - - distribute: True \ No newline at end of file + - distribute: True + +nginx_log: + file.managed: + - name: /var/www/log/error.log + - user: {{ pillar['project_name'] }} + +/etc/nginx/sites-enabled/{{ pillar['project_name'] }}.conf: + file.managed: + - source: salt://project/nginx.conf + - user: root + - group: root + - mode: 644 + - template: jinja + - context: + code_root: "/var/www/{{ pillar['project_name']}}" + log_dir: "/var/www/log" + - require: + - file: nginx_log From 4c7d4070ce78862e0e04964ab46887f963131f95 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Fri, 26 Apr 2013 14:01:56 -0400 Subject: [PATCH 13/48] add supervisor conf --- conf/local/project/supervisor/group.conf | 2 ++ conf/local/project/supervisor/gunicorn.conf | 9 +++++++ conf/local/project/web.sls | 28 +++++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 conf/local/project/supervisor/group.conf create mode 100644 conf/local/project/supervisor/gunicorn.conf diff --git a/conf/local/project/supervisor/group.conf b/conf/local/project/supervisor/group.conf new file mode 100644 index 0000000..26ac3c5 --- /dev/null +++ b/conf/local/project/supervisor/group.conf @@ -0,0 +1,2 @@ +[group:{{ pillar['environment'] }}] +programs=server diff --git a/conf/local/project/supervisor/gunicorn.conf b/conf/local/project/supervisor/gunicorn.conf new file mode 100644 index 0000000..1ff72d4 --- /dev/null +++ b/conf/local/project/supervisor/gunicorn.conf @@ -0,0 +1,9 @@ +[program:server] +process_name=%(program_name)s +command={{ virtualenv_root }}/bin/django-admin.py run_gunicorn --bind=127.0.0.1:{{ server_port|default('8000') }} --workers=3 --worker-class=gevent --preload --settings={{ settings }} +user={{ pillar['project_name'] }} +autostart=true +autorestart=true +stdout_logfile={{ log_dir }}/gunicorn.log +redirect_stderr=true +stderr_logfile={{ log_dir }}/gunicorn.error.log diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index c29a99b..bbcc8e2 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -45,3 +45,31 @@ nginx_log: log_dir: "/var/www/log" - require: - file: nginx_log + +/etc/supervisor/conf.d/group.conf: + file.managed: + - source: salt://project/supervisor/group.conf + - user: root + - group: root + - mode: 644 + - template: jinja + - context: + code_root: "/var/www/{{ pillar['project_name']}}" + log_dir: "/var/www/log" + - require: + - file: nginx_log + +/etc/supervisor/conf.d/gunicorn.conf: + file.managed: + - source: salt://project/supervisor/gunicorn.conf + - user: root + - group: root + - mode: 644 + - template: jinja + - context: + code_root: "/var/www/{{ pillar['project_name']}}" + log_dir: "/var/www/log" + virtualenv_root: "/var/www/env" + settings: "{{ pillar['project_name']}}.settings.{{ pillar['environment'] }}" + - require: + - file: nginx_log From 7b737781ba5108d60e6f54d1b4b7e353e0f5df18 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 14:44:21 -0400 Subject: [PATCH 14/48] Remove old templates and package declarations. --- conf/packages.conf | 14 ------- conf/templates/nginx/site.conf | 56 ------------------------- conf/templates/supervisor/group.conf | 2 - conf/templates/supervisor/gunicorn.conf | 10 ----- 4 files changed, 82 deletions(-) delete mode 100644 conf/packages.conf delete mode 100644 conf/templates/nginx/site.conf delete mode 100644 conf/templates/supervisor/group.conf delete mode 100644 conf/templates/supervisor/gunicorn.conf diff --git a/conf/packages.conf b/conf/packages.conf deleted file mode 100644 index 0752523..0000000 --- a/conf/packages.conf +++ /dev/null @@ -1,14 +0,0 @@ -[base] -packages = python-software-properties dpkg-dev wget build-essential git-core subversion mercurial vim ntp - -[app] -packages = python2.6 python-all-dev python-setuptools libpq-dev libevent-1.4-2 libevent-core-1.4-2 libevent-extra-1.4-2 libevent-dev memcached libjpeg62 libjpeg62-dev zlib1g-dev supervisor postfix nodejs npm -ppas = ppa:pitti/postgresql ppa:chris-lea/node.js - -[lb] -packages = nginx -ppas = ppa:nginx/stable - -[db] -packages = postgresql-9.1 postgresql-contrib-9.1 postgresql-server-dev-9.1 postgresql-client-9.1 libpq-dev -ppas = ppa:pitti/postgresql diff --git a/conf/templates/nginx/site.conf b/conf/templates/nginx/site.conf deleted file mode 100644 index 160a02c..0000000 --- a/conf/templates/nginx/site.conf +++ /dev/null @@ -1,56 +0,0 @@ -upstream django_server { - server 127.0.0.1:{{ server_port|default('8000') }} fail_timeout=0; -} - - -server { - listen 80; - server_name {{ server_name }}; - root {{ code_root }}/public; - - keepalive_timeout 5; - - access_log {{ log_dir }}/access.log; - error_log {{ log_dir }}/error.log; - - if ($host !~* ^({{ server_name }}|www.{{ server_name }})$) { - # Deny non-matching Host headers - return 444; - } - - location /robots.txt { - alias {{ code_root }}/public/robots.txt; - } - - location /media { - alias {{ code_root }}/public/media; - } - - location /static { - alias {{ code_root }}/public/static; - expires max; - } - - error_page 502 503 504 /502.html; - location /502.html { - alias {{ code_root }}/public/502.html; - } - - location / { - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Protocol http; - proxy_set_header Host $http_host; - proxy_redirect off; - proxy_buffering on; - proxy_intercept_errors on; - proxy_pass http://django_server; - } -} - -{# redirect other server names to the real one (http://nginx.org/en/docs/http/converting_rewrite_rules.html under "A redirect to a main site") #} -server { - listen 80 default_server; - server_name _; - return 301 http://{{ server_name }}$request_uri; -} diff --git a/conf/templates/supervisor/group.conf b/conf/templates/supervisor/group.conf deleted file mode 100644 index 4d37708..0000000 --- a/conf/templates/supervisor/group.conf +++ /dev/null @@ -1,2 +0,0 @@ -[group:{{ environment }}] -programs=server diff --git a/conf/templates/supervisor/gunicorn.conf b/conf/templates/supervisor/gunicorn.conf deleted file mode 100644 index 8088f9b..0000000 --- a/conf/templates/supervisor/gunicorn.conf +++ /dev/null @@ -1,10 +0,0 @@ -[program:server] -process_name=%(program_name)s -command={{ virtualenv_root }}/bin/django-admin.py run_gunicorn --bind=127.0.0.1:{{ server_port|default('8000') }} --workers=3 --worker-class=gevent --preload --settings={{ settings }} -directory={{ project_root }} -user={{ project_user }} -autostart=true -autorestart=true -stdout_logfile={{ log_dir }}/gunicorn.log -redirect_stderr=true -stderr_logfile={{ log_dir }}/gunicorn.error.log From 1ba7d2db58f26f20bb633d7b10fd8fb02eb2196d Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 14:52:23 -0400 Subject: [PATCH 15/48] Adding example secrets file and ignoring the actual files. --- .gitignore | 1 + conf/pillar/production/secrets.ex | 3 +++ conf/pillar/staging/secrets.ex | 3 +++ 3 files changed, 7 insertions(+) create mode 100644 conf/pillar/production/secrets.ex create mode 100644 conf/pillar/staging/secrets.ex diff --git a/.gitignore b/.gitignore index 63472c8..60e2a1a 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ */settings/local.py public/static/ public/media/ +secrets.sls \ No newline at end of file diff --git a/conf/pillar/production/secrets.ex b/conf/pillar/production/secrets.ex new file mode 100644 index 0000000..5bd472d --- /dev/null +++ b/conf/pillar/production/secrets.ex @@ -0,0 +1,3 @@ +# This file should be renamed to secrets.sls +secrets: + DB_PASSWORD: 'XXXXXX' \ No newline at end of file diff --git a/conf/pillar/staging/secrets.ex b/conf/pillar/staging/secrets.ex new file mode 100644 index 0000000..5bd472d --- /dev/null +++ b/conf/pillar/staging/secrets.ex @@ -0,0 +1,3 @@ +# This file should be renamed to secrets.sls +secrets: + DB_PASSWORD: 'XXXXXX' \ No newline at end of file From 20582e64474d2222d660d4003b3e41f3004191c7 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 14:52:56 -0400 Subject: [PATCH 16/48] Update Vagrantfile for 1.0 and Ubuntu 12.04 --- Vagrantfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 62217b7..7643e19 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,6 +1,6 @@ Vagrant::Config.run do |config| - config.vm.box = "lucid32" - config.vm.box_url = "http://files.vagrantup.com/lucid32.box" + config.vm.box = "precise32" + config.vm.box_url = "http://files.vagrantup.com/precise32.box" config.vm.forward_port 80, 8080 config.vm.network :hostonly, "33.33.33.10" -end +end \ No newline at end of file From b35f5696b39a69bd2a36b4879e73846791efe0a5 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 14:54:08 -0400 Subject: [PATCH 17/48] Updating local db state file to create project user/db. --- conf/local/project/db.sls | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/local/project/db.sls b/conf/local/project/db.sls index 22b2209..f202dd3 100644 --- a/conf/local/project/db.sls +++ b/conf/local/project/db.sls @@ -1,2 +1,2 @@ include: - - postgres \ No newline at end of file + - postgresql.project \ No newline at end of file From 08b2b84116a663c7413f0bfac3b9361af72aedbc Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 15:11:23 -0400 Subject: [PATCH 18/48] Updating README notes on new provisioning. --- README.rst | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/README.rst b/README.rst index ea7ff64..13250de 100644 --- a/README.rst +++ b/README.rst @@ -81,38 +81,32 @@ Server Provisioning The first step in creating a new server is to create users on the remote server. You will need root user access with passwordless sudo. How you specify this user will vary based on the hosting provider. EC2 and Vagrant use a private key file. Rackspace and -Linode use a user/password combination. +Linode use a user/password combination. -1. For each developer, put a file in the ``conf/users`` directory - containing their public ssh key, and named exactly the same as the - user to create on the server, which should be the same as the userid - on the local development system. (E.g. for user "dickens", the filename - must be "dickens", not "dickens.pub" or "user_dickens".) +1. For each developer, add a user record along with their SSH key into the ``conf/pillar/devs.sls``. -2. Run this command to create users on the server:: +2. Set the project name in ``conf/pillar/project.sls`` and the environment's domain in + ``conf/pillar//env.sls`` if it has not already been set. - fab -H -u create_users +3. Add any environment secrets to the ``conf/pillar//secrets.sls``. This file is not in the source + control by default but there are example ``secrets.ex`` files to use as a starting point. - This will create a project user and users for all the developers. +4. Provision the box using the Salt bootstrap:: -3. Lock down SSH connections: disable password login and move - the default port from 22 to ``env.ssh_port``:: + fab -H -u provision - fab -H configure_ssh + This will provision the box for the initial deploy (create users and install/configure necessary pacakges). -4. Add the IP to the appropriate environment - function and provision it for its role. You can provision a new server with the - ``setup_server`` fab command. It takes a list of roles for this server - ('app', 'db', 'lb') or you can say 'all':: +5. Add the IP to the appropriate environment function. You can now run the initial deploy:: - fab staging setup_server:all + fab deploy Vagrant Testing ------------------------ You can test the provisioning/deployment using `Vagrant `_. -Using the Vagrantfile you can start up the VM. This requires the ``lucid32`` box:: +Using the Vagrantfile you can start up the VM. This requires the ``precise32`` box:: vagrant up @@ -123,8 +117,7 @@ use these commands to create the users. The location of the key file may vary on your system. Running ``locate keys/vagrant`` might help find it:: - fab -H 33.33.33.10 -u vagrant -i /usr/lib/ruby/gems/1.8/gems/vagrant-1.0.2/keys/vagrant create_users - fab vagrant setup_server:all + fab -H 33.33.33.10 -u vagrant -i /usr/lib/ruby/gems/1.8/gems/vagrant-1.0.2/keys/vagrant vagrant provision fab vagrant deploy It is not necessary to reconfigure the SSH settings on the vagrant box. From 8e46ff1d35f2158ebcc6e707cff3feff40163b13 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 26 Apr 2013 15:16:45 -0400 Subject: [PATCH 19/48] Another minor README note. --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 13250de..208ecb2 100644 --- a/README.rst +++ b/README.rst @@ -114,7 +114,8 @@ With the VM up and running, you can create the necessary users. Put the developers' keys in ``conf/users`` as before, then use these commands to create the users. The location of the key file (/usr/lib/ruby/gems/1.8/gems/vagrant-1.0.2/keys/vagrant) -may vary on your system. Running ``locate keys/vagrant`` might +may vary on your system. If installed via apt then this may be located in +/usr/share/vagrant/keys/vagrant. Running ``locate keys/vagrant`` might help find it:: fab -H 33.33.33.10 -u vagrant -i /usr/lib/ruby/gems/1.8/gems/vagrant-1.0.2/keys/vagrant vagrant provision From 6a3fb4847caa1499ab4c594ce37b24318d620ab7 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Fri, 26 Apr 2013 15:33:49 -0400 Subject: [PATCH 20/48] don't die on which calls; install git if it doesn't exist; fix salt_root --- fabfile.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/fabfile.py b/fabfile.py index 05ae094..d132a9a 100644 --- a/fabfile.py +++ b/fabfile.py @@ -58,8 +58,9 @@ def provision(common='master'): """Provision server with masterless Salt minion.""" require('environment') # Install salt minion - with hide('running', 'stdout', 'stderr'): - installed = run('which salt-call') + with settings(warn_only=True): + with hide('running', 'stdout', 'stderr'): + installed = run('which salt-call') if not installed: bootstrap_file = os.path.join(CONF_ROOT, 'bootstrap-salt.sh') put(bootstrap_file, '/tmp/bootstrap-salt.sh') @@ -67,7 +68,7 @@ def provision(common='master'): # Rsync local states and pillars minion_file = os.path.join(CONF_ROOT, 'minion.conf') files.upload_template(minion_file, '/etc/salt/minion', use_sudo=True, context=env) - salt_root = os.path.join(CONF_ROOT, 'salt') + salt_root = CONF_ROOT if CONF_ROOT.endswith('/') else CONF_ROOT + '/' environments = ['staging', 'production'] # Only include current environment's pillar tree exclude = [os.path.join(salt_root, 'pillar', e) for e in environments if e != env.environment] @@ -76,6 +77,11 @@ def provision(common='master'): sudo('mv /tmp/salt/* /srv/') # Pull common states sudo('rm -rf /tmp/common/') + with settings(warn_only=True): + with hide('running', 'stdout', 'stderr'): + installed = run('which git') + if not installed: + sudo('apt-get install git-core -q -y') run('git clone git://github.com/caktus/margarita.git /tmp/common/') with cd('/tmp/common/'): run('git checkout %s' % common) From 7bff7d95b1530be52b0e0d8e35eca20b6dfb6c4d Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Fri, 26 Apr 2013 15:34:33 -0400 Subject: [PATCH 21/48] fix indent of example --- conf/pillar/devs.sls | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conf/pillar/devs.sls b/conf/pillar/devs.sls index 5db0a29..4512b1b 100644 --- a/conf/pillar/devs.sls +++ b/conf/pillar/devs.sls @@ -2,5 +2,5 @@ users: # This should be a list of developer users and thier public SSH keys # example-user: # groups: [admin, login] - # public_key: - # - ssh-dss AAAAB3NzaC1kc3MAAACBAP/dCNcAJED+pBsEwH01E4NU2xrvoB6H5gXkvQHWIKUuMF3MWXgSGhKpgVqLJKh+d0gwuKyI9344HM5dFs4z3E0JhI7Fg4uXIYu1SwuqnxO+D18WLVGt4gCn57JCjLy/c8LJWAHJWFb2v9t4fayC+oBiyEvpjI6VYIJnSvO3D4tjAAAAFQCNzcKi0sehN1Jw+zB6ccMlHt5E6wAAAIEAnW18UHG/O+RIkJazTJ7qFlOb79RS1nnvnHAvtfuiAPIBmeJcKoZkiQzeBYtFereSRHmSug9DsqHK6C5PrP36UMZYhDkqqp5gpJexmokI2kt3AVxJwro7cjy6Tq+0yt+lwqH4JEblybk7yPeRNC1ihnp2CSipC5LP1PydIcgN9/UAAACAeH1OxUzgCfpM06cfKL57gtjIS34ryCdkT2oYfYOANa8vahN2JqxB004o+z2CnQ9DkTqzzf9jUYI/qal19+zYhn8Bd/FdPVp+VTfRpR17fQKuTWrnF7g6jNVN2ltwHo6o99vrCzjHhJHZ2EXOODzAUrACptyfQv/ZCutkjAg44YE= copelco@montgomery.local + # public_key: + # - ssh-dss AAAAB3NzaC1kc3MAAACBAP/dCNcAJED+pBsEwH01E4NU2xrvoB6H5gXkvQHWIKUuMF3MWXgSGhKpgVqLJKh+d0gwuKyI9344HM5dFs4z3E0JhI7Fg4uXIYu1SwuqnxO+D18WLVGt4gCn57JCjLy/c8LJWAHJWFb2v9t4fayC+oBiyEvpjI6VYIJnSvO3D4tjAAAAFQCNzcKi0sehN1Jw+zB6ccMlHt5E6wAAAIEAnW18UHG/O+RIkJazTJ7qFlOb79RS1nnvnHAvtfuiAPIBmeJcKoZkiQzeBYtFereSRHmSug9DsqHK6C5PrP36UMZYhDkqqp5gpJexmokI2kt3AVxJwro7cjy6Tq+0yt+lwqH4JEblybk7yPeRNC1ihnp2CSipC5LP1PydIcgN9/UAAACAeH1OxUzgCfpM06cfKL57gtjIS34ryCdkT2oYfYOANa8vahN2JqxB004o+z2CnQ9DkTqzzf9jUYI/qal19+zYhn8Bd/FdPVp+VTfRpR17fQKuTWrnF7g6jNVN2ltwHo6o99vrCzjHhJHZ2EXOODzAUrACptyfQv/ZCutkjAg44YE= copelco@montgomery.local From 8357f0a13454512cb8ef166d059fa57391d24f7a Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Sat, 27 Apr 2013 11:32:48 -0400 Subject: [PATCH 22/48] update Vagrantfile to work with new vagrant --- Vagrantfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 7643e19..8cc589b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,6 +1,7 @@ -Vagrant::Config.run do |config| +Vagrant.configure("2") do |config| + # Every Vagrant virtual environment requires a box to build off of. config.vm.box = "precise32" config.vm.box_url = "http://files.vagrantup.com/precise32.box" - config.vm.forward_port 80, 8080 - config.vm.network :hostonly, "33.33.33.10" -end \ No newline at end of file + config.vm.network :forwarded_port, guest: 80, host: 8089 + config.vm.network :private_network, ip: "33.33.33.10" +end From 200d43ce23152b1535f906ff3bdc6d090d72673d Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Sat, 27 Apr 2013 11:33:07 -0400 Subject: [PATCH 23/48] fix virtualenv path; append source to activate script --- conf/local/project/web.sls | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index bbcc8e2..16fc53e 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -23,11 +23,17 @@ project_user: - group: www-data - makedirs: True -/home/www/env/: +/var/www/env/: virtualenv.managed: - no_site_packages: True - distribute: True +/var/www/env/bin/activate: + file.append: + - text: source /var/www/env/bin/secrets + - require: + virtualenv: /var/www/env/ + nginx_log: file.managed: - name: /var/www/log/error.log From 5dc4b32aa3e885b5b8806fc2843985cb48b346b8 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Sat, 27 Apr 2013 12:24:31 -0400 Subject: [PATCH 24/48] add vagrant user --- conf/local/top.sls | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conf/local/top.sls b/conf/local/top.sls index 2aa27ac..dbc8776 100644 --- a/conf/local/top.sls +++ b/conf/local/top.sls @@ -1,9 +1,10 @@ base: '*': - base + - vagrant.user + - users.devs - sshd - sshd.github - locale.utf8 - - users.devs - project.web - - project.db \ No newline at end of file + - project.db From 22b2ad3082c4eac3707756a736e614ae95e7be92 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Sat, 27 Apr 2013 12:26:43 -0400 Subject: [PATCH 25/48] use info logging and pty=False --- fabfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabfile.py b/fabfile.py index d132a9a..94767e7 100644 --- a/fabfile.py +++ b/fabfile.py @@ -88,7 +88,7 @@ def provision(common='master'): sudo('mv /tmp/common/ /srv/common/') sudo('chown root:root -R /srv/') # Update to highstate - sudo('salt-call --local state.highstate') + sudo('salt-call --local state.highstate -l info', pty=False) @task From 3665a49ba4a0c30714d2b65325eb3c5760c0dc48 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Sat, 27 Apr 2013 17:21:17 -0400 Subject: [PATCH 26/48] use daily version --- fabfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabfile.py b/fabfile.py index 94767e7..932d9e4 100644 --- a/fabfile.py +++ b/fabfile.py @@ -64,7 +64,7 @@ def provision(common='master'): if not installed: bootstrap_file = os.path.join(CONF_ROOT, 'bootstrap-salt.sh') put(bootstrap_file, '/tmp/bootstrap-salt.sh') - sudo('sh /tmp/bootstrap-salt.sh') + sudo('sh /tmp/bootstrap-salt.sh daily') # Rsync local states and pillars minion_file = os.path.join(CONF_ROOT, 'minion.conf') files.upload_template(minion_file, '/etc/salt/minion', use_sudo=True, context=env) From 4eaf5cf397a283ce8831c179c4b06e383a5e5c4a Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Mon, 29 Apr 2013 11:56:39 -0400 Subject: [PATCH 27/48] Upgrade to the latest Fabric. --- requirements/dev.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 4464426..2d25beb 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,6 +2,6 @@ django-debug-toolbar==0.9.4 coverage==3.5.1 pylint==0.25.1 -fabric==1.4.3 -ssh==1.7.14 +fabric==1.6.0 +paramiko==1.10.1 pycrypto==2.6.0 \ No newline at end of file From 6944a8de9ebafb9a9ac5236a01d99ff6bc57297b Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Mon, 29 Apr 2013 11:57:12 -0400 Subject: [PATCH 28/48] Rsync excludes are always assumed to be relative. --- fabfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fabfile.py b/fabfile.py index 932d9e4..e828ec0 100644 --- a/fabfile.py +++ b/fabfile.py @@ -71,7 +71,7 @@ def provision(common='master'): salt_root = CONF_ROOT if CONF_ROOT.endswith('/') else CONF_ROOT + '/' environments = ['staging', 'production'] # Only include current environment's pillar tree - exclude = [os.path.join(salt_root, 'pillar', e) for e in environments if e != env.environment] + exclude = [os.path.join('pillar', e) for e in environments if e != env.environment] project.rsync_project(local_dir=salt_root, remote_dir='/tmp/salt', delete=True, exclude=exclude) sudo('rm -rf /srv/*') sudo('mv /tmp/salt/* /srv/') From 656d8c01d4b2912b11308316ab4d20ef3b4e3fc9 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Mon, 29 Apr 2013 11:57:52 -0400 Subject: [PATCH 29/48] add env_secrets state --- conf/local/project/env_secrets.jinja2 | 3 +++ conf/local/project/supervisor/gunicorn.conf | 2 +- conf/local/project/web.sls | 12 ++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 conf/local/project/env_secrets.jinja2 diff --git a/conf/local/project/env_secrets.jinja2 b/conf/local/project/env_secrets.jinja2 new file mode 100644 index 0000000..cb2d060 --- /dev/null +++ b/conf/local/project/env_secrets.jinja2 @@ -0,0 +1,3 @@ +{% for key, val in pillar['secrets'].iteritems() %} +export {{ key }}={{ val }} +{% endfor %} diff --git a/conf/local/project/supervisor/gunicorn.conf b/conf/local/project/supervisor/gunicorn.conf index 1ff72d4..40c863d 100644 --- a/conf/local/project/supervisor/gunicorn.conf +++ b/conf/local/project/supervisor/gunicorn.conf @@ -1,6 +1,6 @@ [program:server] process_name=%(program_name)s -command={{ virtualenv_root }}/bin/django-admin.py run_gunicorn --bind=127.0.0.1:{{ server_port|default('8000') }} --workers=3 --worker-class=gevent --preload --settings={{ settings }} +command={{ virtualenv_root }}/bin/activate && {{ virtualenv_root }}/bin/django-admin.py run_gunicorn --bind=127.0.0.1:{{ server_port|default('8000') }} --workers=3 --worker-class=gevent --preload --settings={{ settings }} user={{ pillar['project_name'] }} autostart=true autorestart=true diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index 16fc53e..9decba2 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -9,6 +9,7 @@ include: project_user: user.present: - name: {{ pillar['project_name'] }} + - remove_groups: False - groups: [www-data] /var/www/: @@ -27,6 +28,8 @@ project_user: virtualenv.managed: - no_site_packages: True - distribute: True + - require: + - pip: virtualenv /var/www/env/bin/activate: file.append: @@ -34,6 +37,15 @@ project_user: - require: virtualenv: /var/www/env/ +/var/www/env/bin/secrets: + file.managed: + - source: salt://project/env_secrets.jinja2 + - user: {{ pillar['project_name'] }} + - group: {{ pillar['project_name'] }} + - template: jinja + - require: + - file: /var/www/env/bin/activate + nginx_log: file.managed: - name: /var/www/log/error.log From b1d79e92afc7093c226c55e198692f35e19a8f85 Mon Sep 17 00:00:00 2001 From: Colin Copeland Date: Mon, 29 Apr 2013 12:32:21 -0400 Subject: [PATCH 30/48] fix pillar top to load secrets properly --- conf/pillar/top.sls | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/conf/pillar/top.sls b/conf/pillar/top.sls index 57e5196..f614725 100644 --- a/conf/pillar/top.sls +++ b/conf/pillar/top.sls @@ -2,5 +2,9 @@ base: "*": - project - devs - - env - - secrets \ No newline at end of file + "*-staging": + - staging.env + - staging.secrets + "*-production": + - production.env + - production.secrets From a1e4ef08287de26e7d07924fd12aa203188f5d9c Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Mon, 29 Apr 2013 13:04:21 -0400 Subject: [PATCH 31/48] Fix requirement in the activate script state and remove the nested pillar directory since that is now handled by the top.sls --- conf/local/project/web.sls | 2 +- conf/minion.conf | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index 9decba2..e960f01 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -35,7 +35,7 @@ project_user: file.append: - text: source /var/www/env/bin/secrets - require: - virtualenv: /var/www/env/ + - virtualenv: /var/www/env/ /var/www/env/bin/secrets: file.managed: diff --git a/conf/minion.conf b/conf/minion.conf index 4ee950c..43d3abc 100644 --- a/conf/minion.conf +++ b/conf/minion.conf @@ -14,5 +14,4 @@ file_roots: pillar_roots: base: - - /srv/pillar - - /src/pillar/%(environment)s \ No newline at end of file + - /srv/pillar \ No newline at end of file From 3c104c8257f2de931d6db4dea6f48aa7e31f4483 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Mon, 29 Apr 2013 13:15:59 -0400 Subject: [PATCH 32/48] Clean up tmp directories on the provision and don't set the default port. --- fabfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fabfile.py b/fabfile.py index e828ec0..eb052f8 100644 --- a/fabfile.py +++ b/fabfile.py @@ -13,7 +13,6 @@ env.repo = u'' # FIXME: Add repo URL env.shell = '/bin/bash -c' env.disable_known_hosts = True -env.port = 2222 env.forward_agent = True @@ -75,6 +74,7 @@ def provision(common='master'): project.rsync_project(local_dir=salt_root, remote_dir='/tmp/salt', delete=True, exclude=exclude) sudo('rm -rf /srv/*') sudo('mv /tmp/salt/* /srv/') + sudo('rm -rf /tmp/salt/') # Pull common states sudo('rm -rf /tmp/common/') with settings(warn_only=True): @@ -86,6 +86,7 @@ def provision(common='master'): with cd('/tmp/common/'): run('git checkout %s' % common) sudo('mv /tmp/common/ /srv/common/') + sudo('rm -rf /tmp/common/') sudo('chown root:root -R /srv/') # Update to highstate sudo('salt-call --local state.highstate -l info', pty=False) From b08c0d3abed57d6589c583a2be11c95de2714e6a Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 3 May 2013 09:30:39 -0400 Subject: [PATCH 33/48] Update directory permissions for deployment. --- conf/local/project/web.sls | 26 +++++++++++++++++++++++++- fabfile.py | 3 +-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index e960f01..c075ac3 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -15,14 +15,29 @@ project_user: /var/www/: file.directory: - user: {{ pillar['project_name'] }} - - group: www-data + - group: admin + - mode: 775 - makedirs: True + - require: + - user: project_user /var/www/log/: file.directory: - user: {{ pillar['project_name'] }} - group: www-data + - mode: 775 - makedirs: True + - require: + - file: /var/www/ + +/var/www/public/: + file.directory: + - user: {{ pillar['project_name'] }} + - group: www-data + - mode: 775 + - makedirs: True + - require: + - file: /var/www/ /var/www/env/: virtualenv.managed: @@ -30,6 +45,13 @@ project_user: - distribute: True - require: - pip: virtualenv + - file: /var/www/ + file.directory: + - user: {{ pillar['project_name'] }} + - group: {{ pillar['project_name'] }} + - recurse: + - user + - group /var/www/env/bin/activate: file.append: @@ -50,6 +72,8 @@ nginx_log: file.managed: - name: /var/www/log/error.log - user: {{ pillar['project_name'] }} + - require: + - file: /var/www/log/ /etc/nginx/sites-enabled/{{ pillar['project_name'] }}.conf: file.managed: diff --git a/fabfile.py b/fabfile.py index eb052f8..eaf0e9e 100644 --- a/fabfile.py +++ b/fabfile.py @@ -166,8 +166,7 @@ def deploy(branch=None): migrations = match_changes(changes, r"/migrations/") if requirements or migrations: supervisor_command('stop %(environment)s:*' % env) - with settings(user=env.project_user): - run("git reset --hard origin/%(branch)s" % env) + run("git reset --hard origin/%(branch)s" % env) else: # Initial clone run('git clone %(repo)s %(code_root)s' % env) From b9b12dddc3b5af3ab534962e1241fb5fde7f6378 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 3 May 2013 09:38:59 -0400 Subject: [PATCH 34/48] Add project path to the Python path on initial checkout. --- fabfile.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fabfile.py b/fabfile.py index eaf0e9e..e9786cd 100644 --- a/fabfile.py +++ b/fabfile.py @@ -174,6 +174,10 @@ def deploy(branch=None): run('git checkout %(branch)s' % env) requirements = True migrations = True + # Add code root to the Python path + path_file = os.path.join(env.virtualenv_root, 'lib', 'python2.7', 'site-packages', 'project.pth') + files.append(path_file, env.code_root, use_sudo=True) + sudo('chown %s:%s %s' % (env.project_user, env.project_user, path_file)) sudo('chown %(project_user)s:%(project_user)s -R %(code_root)s' % env) if requirements: update_requirements() From 2e46834f52e2ef2b935effa7c88328957f2a0159 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 3 May 2013 09:50:01 -0400 Subject: [PATCH 35/48] Update how gunicorn is run and how it gets the environment secrets. --- conf/local/project/supervisor/gunicorn.conf | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/conf/local/project/supervisor/gunicorn.conf b/conf/local/project/supervisor/gunicorn.conf index 40c863d..77144cd 100644 --- a/conf/local/project/supervisor/gunicorn.conf +++ b/conf/local/project/supervisor/gunicorn.conf @@ -1,9 +1,13 @@ [program:server] process_name=%(program_name)s -command={{ virtualenv_root }}/bin/activate && {{ virtualenv_root }}/bin/django-admin.py run_gunicorn --bind=127.0.0.1:{{ server_port|default('8000') }} --workers=3 --worker-class=gevent --preload --settings={{ settings }} +command={{ virtualenv_root }}/bin/django-admin.py run_gunicorn --bind=127.0.0.1:{{ server_port|default('8000') }} --workers=3 --worker-class=gevent --preload user={{ pillar['project_name'] }} autostart=true autorestart=true stdout_logfile={{ log_dir }}/gunicorn.log redirect_stderr=true stderr_logfile={{ log_dir }}/gunicorn.error.log +environment=DJANGO_SETTINGS_MODULE="{{ settings }}", + {%- for key, value in pillar['secrets'].iteritems() -%} + {{ key }}="{{ value }}"{%- if not loop.last -%},{%- endif -%} + {%- endfor -%} \ No newline at end of file From ab52fbc03d67995e292ad96dcd513fe4ea7b5716 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 3 May 2013 09:59:51 -0400 Subject: [PATCH 36/48] Restart Nginx/Supervisor when files change. --- conf/local/project/web.sls | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index c075ac3..23b3e95 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -88,6 +88,13 @@ nginx_log: - require: - file: nginx_log +extend: + nginx: + service: + - running + - watch: + - file: /etc/nginx/sites-enabled/{{ pillar['project_name'] }}.conf + /etc/supervisor/conf.d/group.conf: file.managed: - source: salt://project/supervisor/group.conf @@ -115,3 +122,11 @@ nginx_log: settings: "{{ pillar['project_name']}}.settings.{{ pillar['environment'] }}" - require: - file: nginx_log + +extend: + supervisor: + service: + - running + - watch: + - file: /etc/supervisor/conf.d/group.conf + - file: /etc/supervisor/conf.d/gunicorn.conf From 7554a6d258047132025d2b4310b61d8ca75dccee Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 3 May 2013 10:54:00 -0400 Subject: [PATCH 37/48] Install lessc command from NPM. --- conf/local/project/web.sls | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index 23b3e95..96d4e92 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -130,3 +130,18 @@ extend: - watch: - file: /etc/supervisor/conf.d/group.conf - file: /etc/supervisor/conf.d/gunicorn.conf + +npm: + pkg: + - installed + +less: + cmd.run: + - name: npm install less -g + - user: root + - unless: which lessc + - require: + - pkg: npm + file.symlink: + - name: /usr/bin/lessc + - target: /usr/local/bin/lessc \ No newline at end of file From f14fe4a3c5310f5c408f5cd8b1159fbb3f88e340 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 3 May 2013 13:39:27 -0400 Subject: [PATCH 38/48] Adding docs on server setup. See #30 --- README.rst | 84 +------------------------ docs/provisioning.rst | 143 ++++++++++++++++++++++++++++++++++++++++++ docs/server-setup.rst | 67 ++++++++++++++++++++ docs/vagrant.rst | 62 ++++++++++++++++++ 4 files changed, 273 insertions(+), 83 deletions(-) create mode 100644 docs/provisioning.rst create mode 100644 docs/server-setup.rst create mode 100644 docs/vagrant.rst diff --git a/README.rst b/README.rst index 208ecb2..a7f8bfb 100644 --- a/README.rst +++ b/README.rst @@ -57,92 +57,10 @@ You should now be able to run the development server:: python manage.py runserver -Setup repository ------------------------- - -Before your project can be deployed to a server, the code needs to be -accessible in a git repository. - -1. Add your project code to a git repo, hosted somewhere your server can clone it from. - -2. Edit ``fabfile.py`` near the top and insert your repo's URL. E.g., change this:: - - env.repo = u'' # FIXME: Add repo URL - - to this:: - - env.repo = u'git@github.com:account/reponame.git' - - - -Server Provisioning ------------------------- - -The first step in creating a new server is to create users on the remote server. You -will need root user access with passwordless sudo. How you specify this user will vary -based on the hosting provider. EC2 and Vagrant use a private key file. Rackspace and -Linode use a user/password combination. - -1. For each developer, add a user record along with their SSH key into the ``conf/pillar/devs.sls``. - -2. Set the project name in ``conf/pillar/project.sls`` and the environment's domain in - ``conf/pillar//env.sls`` if it has not already been set. - -3. Add any environment secrets to the ``conf/pillar//secrets.sls``. This file is not in the source - control by default but there are example ``secrets.ex`` files to use as a starting point. - -4. Provision the box using the Salt bootstrap:: - - fab -H -u provision - - This will provision the box for the initial deploy (create users and install/configure necessary pacakges). - -5. Add the IP to the appropriate environment function. You can now run the initial deploy:: - - fab deploy - - -Vagrant Testing ------------------------- - -You can test the provisioning/deployment using `Vagrant `_. -Using the Vagrantfile you can start up the VM. This requires the ``precise32`` box:: - - vagrant up - -With the VM up and running, you can create the necessary users. -Put the developers' keys in ``conf/users`` as before, then -use these commands to create the users. The location of the key file -(/usr/lib/ruby/gems/1.8/gems/vagrant-1.0.2/keys/vagrant) -may vary on your system. If installed via apt then this may be located in -/usr/share/vagrant/keys/vagrant. Running ``locate keys/vagrant`` might -help find it:: - - fab -H 33.33.33.10 -u vagrant -i /usr/lib/ruby/gems/1.8/gems/vagrant-1.0.2/keys/vagrant vagrant provision - fab vagrant deploy - -It is not necessary to reconfigure the SSH settings on the vagrant box. - -The vagrant box forwards -port 80 in the VM to port 8080 on the host box. You can view the site -by visiting localhost:8080 in your browser. - -You may also want to add:: - - 33.33.33.10 dev.example.com - -to your hosts (/etc/hosts) file. - -You can stop the VM with ``vagrant halt`` and -destroy the box completely to retest the provisioning with ``vagrant destroy``. - -For more information please review the Vagrant documentation. - - Deployment ------------------------ -For future deployments, you can deploy changes to a particular environment with +You can deploy changes to a particular environment with the ``deploy`` command. This takes an optional branch name to deploy. If the branch is not given, it will use the default branch defined for this environment in ``env.branch``:: diff --git a/docs/provisioning.rst b/docs/provisioning.rst new file mode 100644 index 0000000..f0df6f1 --- /dev/null +++ b/docs/provisioning.rst @@ -0,0 +1,143 @@ +Server Provisioning +======================== + + +Overview +------------------------ + +{{ project_name|title }} is deployed on the following stack. + +- OS: Ubuntu 12.04 LTS +- Python: 2.7 +- Database: Postgres +- Application Server: Gunicorn +- Frontend Server: Nginx +- Cache: Memcached + +These services are configured to run together on a single machine. Each environment +(``staging`` or ``production``) should run on a separate machine. `Supervisord `_ +manages the application server process. + + +Initial Setup +------------------------ + +Before your project can be deployed to a server, the code needs to be +accessible in a git repository. Once that is done you should update the ``env.repo`` in +the ``fabfile.py``. E.g., change this:: + + env.repo = u'' # FIXME: Add repo URL + +to this:: + + env.repo = u'git@github.com:account/reponame.git' + +You also need to set the project name in `conf/pillar/project.sls``. This should +match the ``env.project`` in ``fabfile.py``. For the environment you want to setup +you will need to set the ``domain`` in ``conf/pillar//env.sls``. + +You will also need add the developer's user names and SSH keys to ``conf/pillar/devs.sls``. Each +user record should match the format:: + + example-user: + groups: [admin, login] + public_key: + - ssh-rsa + +Additional developers can be added later but you will need to create at least on user for +yourself. + + +Managing Secrets +------------------------ + +Secret information such as passwords and API keys should never be committed to the +source repository. Instead aach environment manages is secrets in ``conf/pillar//secrets.sls``. +These ``secrets.sls`` files are excluded from the source control and need to be passed +to the developers out of band. There are example files given in ``conf/pillar//secrets.ex``. +They have the format:: + + secrets: + DB_PASSWORD: 'XXXXXX' + +Each key/value pair given in the ``secrets`` dictionary will be added to the OS environment +and can retrieved in the Python code via:: + + import os + + password = os.environ['DB_PASSWORD'] + +Secrets for other environments will not be available. That is the staging server +will not have access to the production secrets. As such there is no need to namespace the +secrets by their environment. + + +Setup Checklist +------------------------ + +To summarize the steps above you can use the following checklist + +- ``env.repo`` is set in ``fabfile.py`` +- Developer user names and SSH keys have been added to ``conf/pillar/devs.sls`` +- Project name has been in ``conf/pillar/project.sls`` +- Environment domain name has been set in ``conf/pillar//env.sls`` +- Environment secrets have been set in ``conf/pillar//secrets.sls`` + + +Provision +------------------------ + +Once you have completed the above steps you are ready to provision a new server +for a given environment. You will need to be able to connect to the server +as a root user. How this is done will depend on where the server is hosted. +VPS providers such as Linode will give you a username/password combination. Amazon's +EC2 uses a private key. These credentials will be passed as command line arguments.:: + + # Template of the command + fab -H -u provision + # Example of provisioning 33.33.33.10 as a staging machine + fab -H 33.33.33.10 -u root staging provision + +Behind the scenes this will rsync the states/pillars in ``conf`` over to the +server as well as check out the base states from the `margarita `_ +repo. It will then use the `masterless salt-minion `_ +to ensure the states are up to date. + +Note that because of the use of rsync it is possible to execute configuration changes which +have not yet been committed to the repo. This can be handy for testing configuration +changes and allows for the secrets to be excluded from the repo but it's a double-edged sword. +You should be sure to commit any configuration changes to the repo when they are ready. + +Once a server has been created for its environment it should be added to the ``env.hosts`` +for the given environment. In our example we would add:: + + def staging(): + env.environment = 'staging' + env.hosts = ['33.33.33.10', ] + +At this point we can run the first deploy:: + + fab staging deploy + +This will do the initial checkout of the repo source, install the Python requirements, +run syncdb/migrate and collect the static resources. + + +Updates +------------------------ + +During the life of the project you will likely need to make updates to the server +configuration. This might include new secrets add to the pillar, new developers +added to the project or new services which need to be installed. Configuration updates +can be made by calling the ``provision`` command again.:: + + # Template of the command + fab provision + # Reprovision the staging server + fab staging provision + +In this case we do not need to connect as the root user. We connect as our developer +user. We also do not need to specify the host. It will use the ``env.hosts`` previously +set for this environment. + +For more information testing the provisioning see the doc:`vagrant guide `. \ No newline at end of file diff --git a/docs/server-setup.rst b/docs/server-setup.rst new file mode 100644 index 0000000..b447a0d --- /dev/null +++ b/docs/server-setup.rst @@ -0,0 +1,67 @@ +Server Setup +======================== + + +Provisioning +------------------------ + +The server provisioning is managed using `Salt Stack `_. The base +states are managed in a `common repo `_ and additional +states specific to this project are contained within the ``conf`` directory at the root +of the repository. + +For more information see the doc:`provisioning guide `. + + +Layout +------------------------ + +Below is the server layout created by this provisioning process:: + + /var/www/ + {{ project_name }}/ + env/ + log/ + public/ + static/ + media/ + +``/var/www/{{ project_name }}/`` contains source code of the project. ``/var/www/env/`` +is the `virtualenv `_ for Python requirements. ``/var/www/log/`` +stores the Nginx, Gunicorn and other logs used by the project. ``/var/www/public/`` +holds the static resources (css/js) for the project and the uploaded user media. +``/var/www/public/static/`` and ``/var/www/public/media/`` map to the ``STATIC_ROOT`` and +``MEDIA_ROOT`` settings. + + +Deployment +------------------------ + +For deployment each developer connects to the server as their own user. Each developer +has SSH access via their public key. These users are created/managed by the Salt +provisioning. The deployment itself is automated with `Fabric `_. +To deploy a developer simply runs:: + + # Deploy updates to staging + fab staging deploy + # Deploy updates to production + fab production deploy + +Each environment (``staging`` or ``production``) is tied to a particular Git branch managed +by ``env.branch`` in the ``fabfile.py``. Deploying a different branch can be done by +passing the branch name to the deploy command:: + + # Deploy new-feature to staging + fab staging deploy:new-feature + +Developers should coordinate to ensure that they do not deploy different branches on +top of one another. + +New python requirements add to the ``requirements/`` files and new South migrations +are detected by grepping the Git diff on deploy. This works fairly well but if they +need to be manually updated that can be done via Fabric:: + + # Installs new requirements from requirements/production.txt + fab staging update_requirements + # Runs syncdb/migrate + fab staging syncdb \ No newline at end of file diff --git a/docs/vagrant.rst b/docs/vagrant.rst new file mode 100644 index 0000000..861e0f8 --- /dev/null +++ b/docs/vagrant.rst @@ -0,0 +1,62 @@ +Vagrant Testing +======================== + + +Starting the VM +------------------------ + +You can test the provisioning/deployment using `Vagrant `_. +Using the included Vagrantfile you can start up the VM. This requires Vagrant 1.2+ and +the ``precise32`` box. The box will be installed if you don't have it already.:: + + vagrant up + +The general provision workflow is the same as in the previous doc:`provisioning guide ` +so here are notes of the Vagrant specifics. + + +Provisioning the VM +------------------------ + +The ``fabfile.py`` contains a ``vagrant`` environment with the VM's IP already added. +The rest of the environment is made to match the ``staging`` environment. If you +have already configured the ``conf/pillar/staing/env.sls`` and ``conf/pillar/staing/secrets.sls`` +then you can continue provisioning the VM. + +To connect to the VM for the first time you need to use the private key which ships +with the Vagrant install. The location of the file may vary on your platform depending +on which version you installed and how it was installed. You can use ``locate`` to find it:: + + # Example locate with output + $ locate keys/vagrant + /opt/vagrant/embedded/gems/gems/vagrant-1.2.2/keys/vagrant + /opt/vagrant/embedded/gems/gems/vagrant-1.2.2/keys/vagrant.pub + +You can then call the initial provision using this key location for the ``-i`` option:: + + fab -u vagrant -i /opt/vagrant/embedded/gems/gems/vagrant-1.2.2/keys/vagrant vagrant provision + +After that has finished you can run the initial deploy:: + + fab vagrant deploy + + +Testing on the VM +------------------------ + +With the VM fully provisioned and deployed you can access the VM on localhost port 8089. Since +the Nginx configuration will only listen for the domain name in ``conf/pillar/staing/env.sls`` +you will need to modify your ``/etc/hosts`` configuration to view it. You will need to add:: + + 127.0.0.1 + +where ```` matches the domain in ``conf/pillar/staing/env.sls``. For example lets use +staging.example.com:: + + 127.0.0.1 staging.example.com + +In your browser you can now view staging.example.com:8089 and see the VM running the full +web stack. + +Note that this ``/etc/hosts`` entry will prevent you from accessing the true staging.example.com. +When your testing is complete you should remove or comment out this entry. \ No newline at end of file From 36583822c939b75f94aba1eef0dff17f067bff81 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 3 May 2013 13:39:44 -0400 Subject: [PATCH 39/48] Remove unused pieces of the fabfile. --- fabfile.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fabfile.py b/fabfile.py index e9786cd..3fac84c 100644 --- a/fabfile.py +++ b/fabfile.py @@ -7,7 +7,6 @@ # Directory structure PROJECT_ROOT = os.path.dirname(__file__) CONF_ROOT = os.path.join(PROJECT_ROOT, 'conf') -SERVER_ROLES = ['app', 'lb', 'db'] env.project = '{{ project_name }}' env.project_user = '{{ project_name }}' env.repo = u'' # FIXME: Add repo URL @@ -21,7 +20,6 @@ def vagrant(): env.environment = 'staging' env.hosts = ['33.33.33.10', ] env.branch = 'master' - env.server_name = 'dev.example.com' setup_path() @@ -30,7 +28,6 @@ def staging(): env.environment = 'staging' env.hosts = [] # FIXME: Add staging server hosts env.branch = 'master' - env.server_name = '' # FIXME: Add staging server name setup_path() @@ -39,7 +36,6 @@ def production(): env.environment = 'production' env.hosts = [] # FIXME: Add production hosts env.branch = 'master' - env.server_name = '' # FIXME: Add production server name setup_path() From 2dea0808bf5b14dc1f41a515125f6f8fafef3686 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 3 May 2013 13:48:41 -0400 Subject: [PATCH 40/48] Update settings and Nginx configuration to store static and media resources outside of the project repo. --- conf/local/project/nginx.conf | 10 +++++----- conf/local/project/web.sls | 2 +- project_name/settings/staging.py | 7 +++++++ {public => project_name/static}/502.html | 0 {public => project_name/static}/robots.txt | 0 5 files changed, 13 insertions(+), 6 deletions(-) rename {public => project_name/static}/502.html (100%) rename {public => project_name/static}/robots.txt (100%) diff --git a/conf/local/project/nginx.conf b/conf/local/project/nginx.conf index b0e3aea..c93c4c4 100644 --- a/conf/local/project/nginx.conf +++ b/conf/local/project/nginx.conf @@ -5,7 +5,7 @@ upstream django_server { server { listen 80; server_name {{ pillar['domain'] }}; - root {{ code_root }}/public; + root {{ public_root }}; keepalive_timeout 5; @@ -18,21 +18,21 @@ server { } location /robots.txt { - alias {{ code_root }}/public/robots.txt; + alias {{ public_root }}/static/robots.txt; } location /media { - alias {{ code_root }}/public/media; + alias {{ public_root }}/media; } location /static { - alias {{ code_root }}/public/static; + alias {{ public_root }}/static; expires max; } error_page 502 503 504 /502.html; location /502.html { - alias {{ code_root }}/public/502.html; + alias {{ public_root }}/static/502.html; } location / { diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index 96d4e92..d89ad0c 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -83,7 +83,7 @@ nginx_log: - mode: 644 - template: jinja - context: - code_root: "/var/www/{{ pillar['project_name']}}" + public_root: "/var/www/public" log_dir: "/var/www/log" - require: - file: nginx_log diff --git a/project_name/settings/staging.py b/project_name/settings/staging.py index f9796fd..fa10b85 100644 --- a/project_name/settings/staging.py +++ b/project_name/settings/staging.py @@ -5,6 +5,13 @@ DATABASES['default']['NAME'] = '{{ project_name }}_staging' + +PUBLIC_ROOT = '/var/www/public/' + +STATIC_ROOT = os.path.join(PUBLIC_ROOT, 'static') + +MEDIA_ROOT = os.path.join(PUBLIC_ROOT, 'media') + INSTALLED_APPS += ( 'gunicorn', ) diff --git a/public/502.html b/project_name/static/502.html similarity index 100% rename from public/502.html rename to project_name/static/502.html diff --git a/public/robots.txt b/project_name/static/robots.txt similarity index 100% rename from public/robots.txt rename to project_name/static/robots.txt From 0ea01b633c0b41a99836f798234906a9469a5924 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 3 May 2013 19:44:02 -0400 Subject: [PATCH 41/48] Abort the deploy command if env.repo has not been set. --- fabfile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fabfile.py b/fabfile.py index 3fac84c..0c1903a 100644 --- a/fabfile.py +++ b/fabfile.py @@ -3,6 +3,7 @@ from fabric.api import cd, env, get, hide, local, put, require, run, settings, sudo, task from fabric.contrib import files, project +from fabric.utils import abort # Directory structure PROJECT_ROOT = os.path.dirname(__file__) @@ -148,6 +149,8 @@ def match_changes(changes, match): def deploy(branch=None): """Deploy to a given environment.""" require('environment') + if not env.repo: + abort('env.repo is not set.') if branch is not None: env.branch = branch requirements = False From 9ee1785cbc1c09b9bfbeabfbd29dec05f1e4348c Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 3 May 2013 19:44:42 -0400 Subject: [PATCH 42/48] Include postgres version in the docs. --- docs/provisioning.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/provisioning.rst b/docs/provisioning.rst index f0df6f1..16fbc6e 100644 --- a/docs/provisioning.rst +++ b/docs/provisioning.rst @@ -9,7 +9,7 @@ Overview - OS: Ubuntu 12.04 LTS - Python: 2.7 -- Database: Postgres +- Database: Postgres 9.1 - Application Server: Gunicorn - Frontend Server: Nginx - Cache: Memcached From 804cb85540805d9c441ff2168f92f46f8acf8fb7 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 3 May 2013 19:46:34 -0400 Subject: [PATCH 43/48] Make it clear that the public SSH key should be added to the devs pillar. --- docs/provisioning.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/provisioning.rst b/docs/provisioning.rst index 16fbc6e..24d1ff2 100644 --- a/docs/provisioning.rst +++ b/docs/provisioning.rst @@ -42,7 +42,7 @@ user record should match the format:: example-user: groups: [admin, login] public_key: - - ssh-rsa + - ssh-rsa Additional developers can be added later but you will need to create at least on user for yourself. From e17dfa2f1bbc89cfe056dc5f356a50937e10e67d Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 3 May 2013 21:45:46 -0400 Subject: [PATCH 44/48] Update /var/www directory structure and related configurations to make it easier to manage multiple projects on the same machine. Still does not support mutliple environments of the same project. --- conf/local/project/nginx.conf | 4 +- conf/local/project/supervisor/group.conf | 4 +- conf/local/project/supervisor/gunicorn.conf | 2 +- conf/local/project/web.sls | 78 +++++++++++---------- docs/server-setup.rst | 12 ++-- fabfile.py | 8 +-- project_name/settings/staging.py | 2 +- 7 files changed, 58 insertions(+), 52 deletions(-) diff --git a/conf/local/project/nginx.conf b/conf/local/project/nginx.conf index c93c4c4..71acbf3 100644 --- a/conf/local/project/nginx.conf +++ b/conf/local/project/nginx.conf @@ -1,4 +1,4 @@ -upstream django_server { +upstream {{ pillar['project_name'] }} { server 127.0.0.1:{{ server_port|default('8000') }} fail_timeout=0; } @@ -43,7 +43,7 @@ server { proxy_redirect off; proxy_buffering on; proxy_intercept_errors on; - proxy_pass http://django_server; + proxy_pass http://{{ pillar['project_name'] }}; } } diff --git a/conf/local/project/supervisor/group.conf b/conf/local/project/supervisor/group.conf index 26ac3c5..6cc0cf2 100644 --- a/conf/local/project/supervisor/group.conf +++ b/conf/local/project/supervisor/group.conf @@ -1,2 +1,2 @@ -[group:{{ pillar['environment'] }}] -programs=server +[group:{{ pillar['project_name'] }}] +programs={{ pillar['project_name'] }}-server diff --git a/conf/local/project/supervisor/gunicorn.conf b/conf/local/project/supervisor/gunicorn.conf index 77144cd..82c0bcb 100644 --- a/conf/local/project/supervisor/gunicorn.conf +++ b/conf/local/project/supervisor/gunicorn.conf @@ -1,4 +1,4 @@ -[program:server] +[program:{{ pillar['project_name'] }}-server] process_name=%(program_name)s command={{ virtualenv_root }}/bin/django-admin.py run_gunicorn --bind=127.0.0.1:{{ server_port|default('8000') }} --workers=3 --worker-class=gevent --preload user={{ pillar['project_name'] }} diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index d89ad0c..a4d4089 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -12,8 +12,9 @@ project_user: - remove_groups: False - groups: [www-data] -/var/www/: +root_dir: file.directory: + - name: /var/www/{{ pillar['project_name'] }}/ - user: {{ pillar['project_name'] }} - group: admin - mode: 775 @@ -21,115 +22,120 @@ project_user: - require: - user: project_user -/var/www/log/: +log_dir: file.directory: + - name: /var/www/{{ pillar['project_name'] }}/log/ - user: {{ pillar['project_name'] }} - group: www-data - mode: 775 - makedirs: True - require: - - file: /var/www/ + - file: root_dir -/var/www/public/: +public_dir: file.directory: + - name: /var/www/{{ pillar['project_name'] }}/public/ - user: {{ pillar['project_name'] }} - group: www-data - mode: 775 - makedirs: True - require: - - file: /var/www/ + - file: root_dir -/var/www/env/: +venv: virtualenv.managed: + - name: /var/www/{{ pillar['project_name'] }}/env/ - no_site_packages: True - distribute: True - require: - pip: virtualenv - - file: /var/www/ + - file: root_dir + +venv_dir: file.directory: + - name: /var/www/{{ pillar['project_name'] }}/env/ - user: {{ pillar['project_name'] }} - group: {{ pillar['project_name'] }} - recurse: - user - group + - require: + - virtualenv: venv -/var/www/env/bin/activate: +activate: file.append: - - text: source /var/www/env/bin/secrets + - name: /var/www/{{ pillar['project_name'] }}/env/bin/activate + - text: source /var/www/{{ pillar['project_name'] }}/env/bin/secrets - require: - - virtualenv: /var/www/env/ + - virtualenv: venv -/var/www/env/bin/secrets: +secrets: file.managed: + - name: /var/www/{{ pillar['project_name'] }}/env/bin/secrets - source: salt://project/env_secrets.jinja2 - user: {{ pillar['project_name'] }} - group: {{ pillar['project_name'] }} - template: jinja - require: - - file: /var/www/env/bin/activate + - file: activate -nginx_log: - file.managed: - - name: /var/www/log/error.log - - user: {{ pillar['project_name'] }} - - require: - - file: /var/www/log/ - -/etc/nginx/sites-enabled/{{ pillar['project_name'] }}.conf: +nginx_conf: file.managed: + - name: /etc/nginx/sites-enabled/{{ pillar['project_name'] }}.conf - source: salt://project/nginx.conf - user: root - group: root - mode: 644 - template: jinja - context: - public_root: "/var/www/public" - log_dir: "/var/www/log" + public_root: "/var/www/{{ pillar['project_name']}}/public" + log_dir: "/var/www/{{ pillar['project_name']}}/log" - require: - - file: nginx_log + - pkg: nginx + - file: log_dir extend: nginx: service: - running - watch: - - file: /etc/nginx/sites-enabled/{{ pillar['project_name'] }}.conf + - file: nginx_conf -/etc/supervisor/conf.d/group.conf: +group_conf: file.managed: + - name: /etc/supervisor/conf.d/{{ pillar['project_name'] }}-group.conf - source: salt://project/supervisor/group.conf - user: root - group: root - mode: 644 - template: jinja - - context: - code_root: "/var/www/{{ pillar['project_name']}}" - log_dir: "/var/www/log" - require: - - file: nginx_log + - pkg: supervisor + - file: log_dir -/etc/supervisor/conf.d/gunicorn.conf: +gunicorn_conf: file.managed: + - name: /etc/supervisor/conf.d/{{ pillar['project_name'] }}-gunicorn.conf - source: salt://project/supervisor/gunicorn.conf - user: root - group: root - mode: 644 - template: jinja - context: - code_root: "/var/www/{{ pillar['project_name']}}" - log_dir: "/var/www/log" - virtualenv_root: "/var/www/env" + log_dir: "/var/www/{{ pillar['project_name']}}/log" + virtualenv_root: "/var/www/{{ pillar['project_name']}}/env" settings: "{{ pillar['project_name']}}.settings.{{ pillar['environment'] }}" - require: - - file: nginx_log + - pkg: supervisor + - file: log_dir extend: supervisor: service: - running - watch: - - file: /etc/supervisor/conf.d/group.conf - - file: /etc/supervisor/conf.d/gunicorn.conf + - file: group_conf + - file: gunicorn_conf npm: pkg: diff --git a/docs/server-setup.rst b/docs/server-setup.rst index b447a0d..2054589 100644 --- a/docs/server-setup.rst +++ b/docs/server-setup.rst @@ -18,19 +18,19 @@ Layout Below is the server layout created by this provisioning process:: - /var/www/ - {{ project_name }}/ + /var/www/{{ project_name }}/ + source/ env/ log/ public/ static/ media/ -``/var/www/{{ project_name }}/`` contains source code of the project. ``/var/www/env/`` -is the `virtualenv `_ for Python requirements. ``/var/www/log/`` -stores the Nginx, Gunicorn and other logs used by the project. ``/var/www/public/`` +``source`` contains source code of the project. ``env`` +is the `virtualenv `_ for Python requirements. ``log`` +stores the Nginx, Gunicorn and other logs used by the project. ``public`` holds the static resources (css/js) for the project and the uploaded user media. -``/var/www/public/static/`` and ``/var/www/public/media/`` map to the ``STATIC_ROOT`` and +``public/static/`` and ``public/media/`` map to the ``STATIC_ROOT`` and ``MEDIA_ROOT`` settings. diff --git a/fabfile.py b/fabfile.py index 0c1903a..62a85c0 100644 --- a/fabfile.py +++ b/fabfile.py @@ -42,8 +42,8 @@ def production(): def setup_path(): env.home = '/home/%(project_user)s/' % env - env.root = '/var/www/' - env.code_root = os.path.join(env.root, env.project) + env.root = os.path.join('/var/www/', env.project) + env.code_root = os.path.join(env.root, 'source') env.virtualenv_root = os.path.join(env.root, 'env') env.db = '%s_%s' % (env.project, env.environment) env.settings = '%(project)s.settings.%(environment)s' % env @@ -164,7 +164,7 @@ def deploy(branch=None): requirements = match_changes(changes, r"requirements/") migrations = match_changes(changes, r"/migrations/") if requirements or migrations: - supervisor_command('stop %(environment)s:*' % env) + supervisor_command('stop %(project)s:*' % env) run("git reset --hard origin/%(branch)s" % env) else: # Initial clone @@ -185,7 +185,7 @@ def deploy(branch=None): elif migrations: syncdb() collectstatic() - supervisor_command('restart %(environment)s:*' % env) + supervisor_command('restart %(project)s:*' % env) @task diff --git a/project_name/settings/staging.py b/project_name/settings/staging.py index fa10b85..97b8970 100644 --- a/project_name/settings/staging.py +++ b/project_name/settings/staging.py @@ -6,7 +6,7 @@ DATABASES['default']['NAME'] = '{{ project_name }}_staging' -PUBLIC_ROOT = '/var/www/public/' +PUBLIC_ROOT = '/var/www/{{ project_name }}/public/' STATIC_ROOT = os.path.join(PUBLIC_ROOT, 'static') From e8a9067f446e54507b7fda1a8094fe1750bd8631 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Mon, 6 May 2013 14:53:39 -0400 Subject: [PATCH 45/48] Can't have duplicate extend declarations. --- conf/local/project/web.sls | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index a4d4089..e0eb2e5 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -94,13 +94,6 @@ nginx_conf: - pkg: nginx - file: log_dir -extend: - nginx: - service: - - running - - watch: - - file: nginx_conf - group_conf: file.managed: - name: /etc/supervisor/conf.d/{{ pillar['project_name'] }}-group.conf @@ -130,6 +123,12 @@ gunicorn_conf: - file: log_dir extend: + nginx: + service: + - running + - watch: + - file: nginx_conf + supervisor: service: - running From 9a48dd0e95f6d7caa2441e8885b40d92dcca18b9 Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Mon, 6 May 2013 15:27:02 -0400 Subject: [PATCH 46/48] Capture JSON output of the highstate call and look for states which did not execute properly. --- fabfile.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/fabfile.py b/fabfile.py index 62a85c0..6a4e0af 100644 --- a/fabfile.py +++ b/fabfile.py @@ -2,8 +2,9 @@ import re from fabric.api import cd, env, get, hide, local, put, require, run, settings, sudo, task +from fabric.color import red from fabric.contrib import files, project -from fabric.utils import abort +from fabric.utils import abort, error # Directory structure PROJECT_ROOT = os.path.dirname(__file__) @@ -86,7 +87,18 @@ def provision(common='master'): sudo('rm -rf /tmp/common/') sudo('chown root:root -R /srv/') # Update to highstate - sudo('salt-call --local state.highstate -l info', pty=False) + with settings(warn_only=True): + sudo('salt-call --local state.highstate -l info --out json > /tmp/output.json') + get('/tmp/output.json', 'output.json') + with open('output.json', 'r') as f: + try: + results = json.load(f) + except (TypeError, ValueError) as e: + error(u'Non-JSON output from salt-call', exception=e) + else: + for state, result in results['local'].items(): + if not result["result"]: + print red(u'Error with %(name)s state: %(comment)s' % result) @task From 0a1ec5186b99835df5169eda5b816df0f8653e4b Mon Sep 17 00:00:00 2001 From: Dan Poirier Date: Mon, 6 May 2013 16:45:28 -0400 Subject: [PATCH 47/48] Typos and other trival copy-editing Branch: salt-provision --- docs/provisioning.rst | 20 ++++++++++---------- docs/server-setup.rst | 10 +++++----- docs/vagrant.rst | 14 +++++++------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/provisioning.rst b/docs/provisioning.rst index 24d1ff2..56871c1 100644 --- a/docs/provisioning.rst +++ b/docs/provisioning.rst @@ -44,7 +44,7 @@ user record should match the format:: public_key: - ssh-rsa -Additional developers can be added later but you will need to create at least on user for +Additional developers can be added later, but you will need to create at least one user for yourself. @@ -52,7 +52,7 @@ Managing Secrets ------------------------ Secret information such as passwords and API keys should never be committed to the -source repository. Instead aach environment manages is secrets in ``conf/pillar//secrets.sls``. +source repository. Instead, each environment manages its secrets in ``conf/pillar//secrets.sls``. These ``secrets.sls`` files are excluded from the source control and need to be passed to the developers out of band. There are example files given in ``conf/pillar//secrets.ex``. They have the format:: @@ -67,7 +67,7 @@ and can retrieved in the Python code via:: password = os.environ['DB_PASSWORD'] -Secrets for other environments will not be available. That is the staging server +Secrets for other environments will not be available. That is, the staging server will not have access to the production secrets. As such there is no need to namespace the secrets by their environment. @@ -75,7 +75,7 @@ secrets by their environment. Setup Checklist ------------------------ -To summarize the steps above you can use the following checklist +To summarize the steps above, you can use the following checklist - ``env.repo`` is set in ``fabfile.py`` - Developer user names and SSH keys have been added to ``conf/pillar/devs.sls`` @@ -87,7 +87,7 @@ To summarize the steps above you can use the following checklist Provision ------------------------ -Once you have completed the above steps you are ready to provision a new server +Once you have completed the above steps, you are ready to provision a new server for a given environment. You will need to be able to connect to the server as a root user. How this is done will depend on where the server is hosted. VPS providers such as Linode will give you a username/password combination. Amazon's @@ -105,7 +105,7 @@ to ensure the states are up to date. Note that because of the use of rsync it is possible to execute configuration changes which have not yet been committed to the repo. This can be handy for testing configuration -changes and allows for the secrets to be excluded from the repo but it's a double-edged sword. +changes and allows for the secrets to be excluded from the repo, but it's a double-edged sword. You should be sure to commit any configuration changes to the repo when they are ready. Once a server has been created for its environment it should be added to the ``env.hosts`` @@ -120,15 +120,15 @@ At this point we can run the first deploy:: fab staging deploy This will do the initial checkout of the repo source, install the Python requirements, -run syncdb/migrate and collect the static resources. +run syncdb/migrate, and collect the static resources. Updates ------------------------ During the life of the project you will likely need to make updates to the server -configuration. This might include new secrets add to the pillar, new developers -added to the project or new services which need to be installed. Configuration updates +configuration. This might include new secrets added to the pillar, new developers +added to the project, or new services which need to be installed. Configuration updates can be made by calling the ``provision`` command again.:: # Template of the command @@ -140,4 +140,4 @@ In this case we do not need to connect as the root user. We connect as our devel user. We also do not need to specify the host. It will use the ``env.hosts`` previously set for this environment. -For more information testing the provisioning see the doc:`vagrant guide `. \ No newline at end of file +For more information testing the provisioning see the doc:`vagrant guide `. diff --git a/docs/server-setup.rst b/docs/server-setup.rst index 2054589..4bbcbac 100644 --- a/docs/server-setup.rst +++ b/docs/server-setup.rst @@ -26,7 +26,7 @@ Below is the server layout created by this provisioning process:: static/ media/ -``source`` contains source code of the project. ``env`` +``source`` contains the source code of the project. ``env`` is the `virtualenv `_ for Python requirements. ``log`` stores the Nginx, Gunicorn and other logs used by the project. ``public`` holds the static resources (css/js) for the project and the uploaded user media. @@ -37,10 +37,10 @@ holds the static resources (css/js) for the project and the uploaded user media. Deployment ------------------------ -For deployment each developer connects to the server as their own user. Each developer +For deployment, each developer connects to the server as their own user. Each developer has SSH access via their public key. These users are created/managed by the Salt provisioning. The deployment itself is automated with `Fabric `_. -To deploy a developer simply runs:: +To deploy, a developer simply runs:: # Deploy updates to staging fab staging deploy @@ -58,10 +58,10 @@ Developers should coordinate to ensure that they do not deploy different branche top of one another. New python requirements add to the ``requirements/`` files and new South migrations -are detected by grepping the Git diff on deploy. This works fairly well but if they +are detected by grepping the Git diff on deploy. This works fairly well, but if they need to be manually updated that can be done via Fabric:: # Installs new requirements from requirements/production.txt fab staging update_requirements # Runs syncdb/migrate - fab staging syncdb \ No newline at end of file + fab staging syncdb diff --git a/docs/vagrant.rst b/docs/vagrant.rst index 861e0f8..d541751 100644 --- a/docs/vagrant.rst +++ b/docs/vagrant.rst @@ -11,7 +11,7 @@ the ``precise32`` box. The box will be installed if you don't have it already.:: vagrant up -The general provision workflow is the same as in the previous doc:`provisioning guide ` +The general provision workflow is the same as in the previous :doc:`provisioning guide ` so here are notes of the Vagrant specifics. @@ -20,10 +20,10 @@ Provisioning the VM The ``fabfile.py`` contains a ``vagrant`` environment with the VM's IP already added. The rest of the environment is made to match the ``staging`` environment. If you -have already configured the ``conf/pillar/staing/env.sls`` and ``conf/pillar/staing/secrets.sls`` +have already configured the ``conf/pillar/staging/env.sls`` and ``conf/pillar/staging/secrets.sls`` then you can continue provisioning the VM. -To connect to the VM for the first time you need to use the private key which ships +To connect to the VM for the first time, you need to use the private key which ships with the Vagrant install. The location of the file may vary on your platform depending on which version you installed and how it was installed. You can use ``locate`` to find it:: @@ -44,13 +44,13 @@ After that has finished you can run the initial deploy:: Testing on the VM ------------------------ -With the VM fully provisioned and deployed you can access the VM on localhost port 8089. Since -the Nginx configuration will only listen for the domain name in ``conf/pillar/staing/env.sls`` +With the VM fully provisioned and deployed, you can access the VM on localhost port 8089. Since +the Nginx configuration will only listen for the domain name in ``conf/pillar/staging/env.sls``, you will need to modify your ``/etc/hosts`` configuration to view it. You will need to add:: 127.0.0.1 -where ```` matches the domain in ``conf/pillar/staing/env.sls``. For example lets use +where ```` matches the domain in ``conf/pillar/staging/env.sls``. For example, let's use staging.example.com:: 127.0.0.1 staging.example.com @@ -59,4 +59,4 @@ In your browser you can now view staging.example.com:8089 and see the VM running web stack. Note that this ``/etc/hosts`` entry will prevent you from accessing the true staging.example.com. -When your testing is complete you should remove or comment out this entry. \ No newline at end of file +When your testing is complete, you should remove or comment out this entry. From 10c598d868ba118af982c91ce7386a1fe1158d8a Mon Sep 17 00:00:00 2001 From: Mark Lavin Date: Fri, 10 May 2013 15:37:05 -0400 Subject: [PATCH 48/48] Open HTTP and HTTPS ports on the webserver by default. Fixes #29 --- conf/local/project/web.sls | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/conf/local/project/web.sls b/conf/local/project/web.sls index e0eb2e5..6466b72 100644 --- a/conf/local/project/web.sls +++ b/conf/local/project/web.sls @@ -5,6 +5,14 @@ include: - nginx - python - supervisor + - ufw + +http_firewall: + ufw.allow: + - names: + - '80' + - '443' + - enabled: true project_user: user.present: