Release 0.4

MAINTAINERS.md

@@ -12,7 +12,7 @@ least one maintainer on relevant issues and PRs.
 
 * GCE: [Brendan Burns](https://github.com/brendandburns), [Joe Beda](https://github.com/jbeda), [Daniel Smith](https://github.com/lavalamp), [Tim Hockin](https://github.com/thockin)
 * Azure: [Jeff Mendoza](https://github.com/jeffmendoza)
-* VSphere: [Pieter Noordhuis](https://github.com/pietern)
+* vSphere: [Pieter Noordhuis](https://github.com/pietern)
 * Rackspace: [Ryan Richard](https://github.com/doublerr)
 * oVirt: [Federico Simoncelli](https://github.com/simon3z)
 * Local: [Derek Carr](https://github.com/derekwaynecarr)

README.md

@@ -23,11 +23,12 @@ While the concepts and architecture in Kubernetes represent years of experience
   * [CoreOS](docs/getting-started-guides/coreos.md)
   * [OpenStack](https://developer.rackspace.com/blog/running-coreos-and-kubernetes/)
   * [CloudStack](docs/getting-started-guides/cloudstack.md)
+  * [Rackspace](docs/getting-started-guides/rackspace.md)
+  * [vSphere](docs/getting-started-guides/vsphere.md)
+
 * The following clouds are currently broken at Kubernetes head.  Please sync your client to `v0.3` (`git checkout v0.3`) to use these:
   * [Locally](docs/getting-started-guides/locally.md)
-  * [vSphere](docs/getting-started-guides/vsphere.md)
   * [Microsoft Azure](docs/getting-started-guides/azure.md)
-  * [Rackspace](docs/getting-started-guides/rackspace.md)
 * [Kubernetes 101](https://github.com/GoogleCloudPlatform/kubernetes/tree/master/examples/walkthrough)
 * [kubecfg command line tool](https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/cli.md)
 * [Kubernetes API Documentation](http://cdn.rawgit.com/GoogleCloudPlatform/kubernetes/31a0daae3627c91bc96e1f02a6344cd76e294791/api/kubernetes.html)

build/README.md

@@ -51,11 +51,11 @@ Env Variable | Default | Description
 `KUBE_SKIP_CONFIRMATIONS` | `n` | If `y` then no questions are asked and the scripts just continue.
 `KUBE_GCS_UPLOAD_RELEASE` | `n` | Upload release artifacts to GCS
 `KUBE_GCS_RELEASE_BUCKET` | `kubernetes-releases-${project_hash}` | The bucket to upload releases to
-`KUBE_GCS_RELEASE_PREFIX` | `devel/` | The path under the release bucket to put releases
+`KUBE_GCS_RELEASE_PREFIX` | `devel` | The path under the release bucket to put releases
 `KUBE_GCS_MAKE_PUBLIC` | `y` | Make GCS links readable from anywhere
 `KUBE_GCS_NO_CACHING` | `y` | Disable HTTP caching of GCS release artifacts.  By default GCS will cache public objects for up to an hour.  When doing "devel" releases this can cause problems.
 `KUBE_BUILD_RUN_IMAGES` | `n` | *Experimental* Build Docker images for running most server components.
-`KUBE_GCS_DOCKER_REG_PREFIX` | `docker-reg/` | *Experimental* When uploading docker images, the bucket that backs the registry.
+`KUBE_GCS_DOCKER_REG_PREFIX` | `docker-reg` | *Experimental* When uploading docker images, the bucket that backs the registry.
 
 ## Basic Flow
 

build/common.sh

@@ -35,9 +35,8 @@ readonly KUBE_GCS_UPLOAD_RELEASE="${KUBE_GCS_UPLOAD_RELEASE:-n}"
 readonly KUBE_GCS_NO_CACHING="${KUBE_GCS_NO_CACHING:-y}"
 readonly KUBE_GCS_MAKE_PUBLIC="${KUBE_GCS_MAKE_PUBLIC:-y}"
 # KUBE_GCS_RELEASE_BUCKET default: kubernetes-releases-${project_hash}
-# KUBE_GCS_RELEASE_PREFIX default: devel/
-# KUBE_GCS_DOCKER_REG_PREFIX default: docker-reg/
-
+readonly KUBE_GCS_RELEASE_PREFIX=${KUBE_GCS_RELEASE_PREFIX-devel}/
+readonly KUBE_GCS_DOCKER_REG_PREFIX=${KUBE_GCS_DOCKER_REG_PREFIX-docker-reg}/
 
 
 # Constants
@@ -370,35 +369,38 @@ function kube::build::run_build_command() {
   docker rm "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true
 }
 
+# Test if the output directory is remote (and can only be accessed through
+# docker) or if it is "local" and we can access the output without going through
+# docker.
+function kube::build::is_output_remote() {
+  rm -f "${LOCAL_OUTPUT_BUILD}/test_for_remote"
+  kube::build::run_build_command touch "${REMOTE_OUTPUT_DIR}/test_for_remote"
+
+  [[ ! -e "${LOCAL_OUTPUT_BUILD}/test_for_remote" ]]
+}
+
 # If the Docker server is remote, copy the results back out.
 function kube::build::copy_output() {
-  if kube::build::is_osx; then
-    # When we are on the Mac with boot2docker we need to copy the results back
-    # out.  Ideally we would leave the container around and use 'docker cp' to
-    # copy the results out.  However, that doesn't work for mounted volumes
-    # currently (https://github.com/dotcloud/docker/issues/1992).  And it is
-    # just plain broken (https://github.com/dotcloud/docker/issues/6483).
+  if kube::build::is_output_remote; then
+    # When we are on the Mac with boot2docker (or to a remote Docker in any
+    # other situation) we need to copy the results back out.  Ideally we would
+    # leave the container around and use 'docker cp' to copy the results out.
+    # However, that doesn't work for mounted volumes currently
+    # (https://github.com/dotcloud/docker/issues/1992).  And it is just plain
+    # broken (https://github.com/dotcloud/docker/issues/6483).
     #
     # The easiest thing I (jbeda) could figure out was to launch another
     # container pointed at the same volume, tar the output directory and ship
-    # that tar over stdou.
-    local -ra docker_cmd=(
-      docker run -a stdout "--name=${KUBE_BUILD_CONTAINER_NAME}"
-      "${DOCKER_MOUNT_ARGS[@]}" "${KUBE_BUILD_IMAGE}")
-
-    # Kill any leftover container
-    docker rm "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true
+    # that tar over stdout.
 
     echo "+++ Syncing back _output directory from boot2docker VM"
     rm -rf "${LOCAL_OUTPUT_BUILD}"
     mkdir -p "${LOCAL_OUTPUT_BUILD}"
-    "${docker_cmd[@]}" sh -c "tar c -C ${REMOTE_OUTPUT_DIR} . ; sleep 1"  \
-      | tar xv -C "${LOCAL_OUTPUT_BUILD}"
 
-    # Remove the container after we run.  '--rm' might be appropriate but it
-    # appears that sometimes it fails. See
-    # https://github.com/docker/docker/issues/3968
-    docker rm "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true
+    # The '</dev/null' here makes us run docker in a "non-interactive" mode. Not
+    # doing this corrupts the output stream.
+    kube::build::run_build_command sh -c "tar c -C ${REMOTE_OUTPUT_DIR} . ; sleep 1" </dev/null \
+      | tar xv -C "${LOCAL_OUTPUT_BUILD}"
 
     # I (jbeda) also tried getting rsync working using 'docker run' as the
     # 'remote shell'.  This mostly worked but there was a hang when
@@ -407,6 +409,8 @@ function kube::build::copy_output() {
     # local DOCKER="docker run -i --rm --name=${KUBE_BUILD_CONTAINER_NAME} ${DOCKER_MOUNT} ${KUBE_BUILD_IMAGE}"
     # DOCKER+=" bash -c 'shift ; exec \"\$@\"' --"
     # rsync --blocking-io -av -e "${DOCKER}" foo:${REMOTE_OUTPUT_DIR}/ ${LOCAL_OUTPUT_BUILD}
+  else
+    echo "+++ Output directory is local.  No need to copy results out."
   fi
 }
 
@@ -444,7 +448,7 @@ function kube::release::package_client_tarballs() {
       "${release_stage}/client/bin/"
 
     local package_name="${RELEASE_DIR}/kubernetes-client-${platform_tag}.tar.gz"
-    tar czf "${package_name}" -C "${release_stage}/.." .
+    kube::release::create_tarball "${package_name}" "${release_stage}/.."
   done
 }
 
@@ -466,7 +470,7 @@ function kube::release::package_server_tarballs() {
       "${release_stage}/server/bin/"
 
     local package_name="${RELEASE_DIR}/kubernetes-server-${platform_tag}.tar.gz"
-    tar czf "${package_name}" -C "${release_stage}/.." .
+    kube::release::create_tarball "${package_name}" "${release_stage}/.."
   done
 }
 
@@ -482,7 +486,7 @@ function kube::release::package_salt_tarball() {
   cp -R "${KUBE_ROOT}/cluster/saltbase" "${release_stage}/"
 
   local package_name="${RELEASE_DIR}/kubernetes-salt.tar.gz"
-  tar czf "${package_name}" -C "${release_stage}/.." .
+  kube::release::create_tarball "${package_name}" "${release_stage}/.."
 }
 
 # This is all the stuff you need to run/install kubernetes.  This includes:
@@ -517,9 +521,33 @@ function kube::release::package_full_tarball() {
   cp "${KUBE_ROOT}/Vagrantfile" "${release_stage}/"
 
   local package_name="${RELEASE_DIR}/kubernetes.tar.gz"
-  tar czf "${package_name}" -C "${release_stage}/.." .
+  kube::release::create_tarball "${package_name}" "${release_stage}/.."
 }
 
+# Build a release tarball.  $1 is the output tar name.  $2 is the base directory
+# of the files to be packaged.  This assumes that ${2}/kubernetes is what is
+# being packaged.
+function kube::release::create_tarball() {
+  local tarfile=$1
+  local stagingdir=$2
+
+  # Find gnu tar if it is available
+  local tar=tar
+  if which gtar > /dev/null; then
+    tar=gtar
+  fi
+
+  local tar_cmd=("$tar" "czf" "${tarfile}" "-C" "${stagingdir}" "kubernetes")
+  if "$tar" --version | grep -q GNU; then
+    tar_cmd=("${tar_cmd[@]}" "--owner=0" "--group=0")
+  else
+    echo "  !!! GNU tar not available.  User names will be embedded in output and"
+    echo "      release tars are not official. Build on Linux or install GNU tar"
+    echo "      on Mac OS X (brew install gnu-tar)"
+  fi
+
+  "${tar_cmd[@]}"
+}
 
 # ---------------------------------------------------------------------------
 # GCS Release
@@ -530,7 +558,7 @@ function kube::release::gcs::release() {
   kube::release::gcs::verify_prereqs
   kube::release::gcs::ensure_release_bucket
   kube::release::gcs::push_images
-  kube::release::gcs::copy_release_tarballs
+  kube::release::gcs::copy_release_artifacts
 }
 
 # Verify things are set up for uploading to GCS
@@ -569,8 +597,6 @@ function kube::release::gcs::ensure_release_bucket() {
   local project_hash
   project_hash=$(kube::build::short_hash "$GCLOUD_PROJECT")
   KUBE_GCS_RELEASE_BUCKET=${KUBE_GCS_RELEASE_BUCKET-kubernetes-releases-${project_hash}}
-  KUBE_GCS_RELEASE_PREFIX=${KUBE_GCS_RELEASE_PREFIX-devel/}
-  KUBE_GCS_DOCKER_REG_PREFIX=${KUBE_GCS_DOCKER_REG_PREFIX-docker-reg/}
 
   if ! gsutil ls "gs://${KUBE_GCS_RELEASE_BUCKET}" >/dev/null 2>&1 ; then
     echo "Creating Google Cloud Storage bucket: $RELEASE_BUCKET"
@@ -631,7 +657,7 @@ function kube::release::gcs::push_images() {
   done
 }
 
-function kube::release::gcs::copy_release_tarballs() {
+function kube::release::gcs::copy_release_artifacts() {
   # TODO: This isn't atomic.  There will be points in time where there will be
   # no active release.  Also, if something fails, the release could be half-
   # copied.  The real way to do this would perhaps to have some sort of release
@@ -643,17 +669,27 @@ function kube::release::gcs::copy_release_tarballs() {
     gcs_options=("-h" "Cache-Control:private, max-age=0")
   fi
 
-  echo "+++ Copying client tarballs to ${gcs_destination}"
+  echo "+++ Copying release artifacts to ${gcs_destination}"
 
   # First delete all objects at the destination
   gsutil -q rm -f -R "${gcs_destination}" >/dev/null 2>&1 || true
 
   # Now upload everything in release directory
-  gsutil -m "${gcs_options[@]-}" cp -r "${RELEASE_DIR}"/* "${gcs_destination}" >/dev/null 2>&1
+  gsutil -m "${gcs_options[@]+${gcs_options[@]}}" cp -r "${RELEASE_DIR}"/* "${gcs_destination}"
+
+  # Having the "template" scripts from the GCE cluster deploy hosted with the
+  # release is useful for GKE.  Copy everything from that directory up also.
+  gsutil -m "${gcs_options[@]+${gcs_options[@]}}" cp "${KUBE_ROOT}/cluster/gce/templates/*.sh" "${gcs_destination}extra/gce-templates/"
+
+  # TODO(jbeda): Generate an HTML page with links for this release so it is easy
+  # to see it.  For extra credit, generate a dynamic page that builds up the
+  # release list using the GCS JSON API.  Use Angular and Bootstrap for extra
+  # extra credit.
 
   if [[ ${KUBE_GCS_MAKE_PUBLIC} =~ ^[yY]$ ]]; then
+    echo "+++ Marking all uploaded objects public"
     gsutil acl ch -R -g all:R "${gcs_destination}" >/dev/null 2>&1
   fi
 
-  gsutil ls -lh "${gcs_destination}"
+  gsutil ls -lhr "${gcs_destination}"
 }

cluster/gce/templates/README.md

@@ -0,0 +1,12 @@
+# Updating Salt debs
+
+We are caching all of the salt debs in GCS for speed and reliability.
+
+To update them, follow this simple N step process:
+
+1. Start up a new base image without salt installed.  SSH into this image.
+2. Install salt via their recommended method: `curl -L https://bootstrap.saltstack.com | sudo sh -s -- -M -X`
+3. Find and download the debs that originated at the saltstack.com repo: `aptitude search --disable-columns -F "%p %V" "?installed?origin(saltstack.com)" | xargs aptitude download`
+4. Upload these to GCS: `gsutil cp *.deb gs://kubernetes-release/salt/`
+5. Make sure that everything is publicly readable: `gsutil acl ch -R -g all:R gs://kubernetes-release/salt/`
+6. Test things well :)

cluster/gce/templates/common.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Copyright 2014 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Retry a download until we get it.
+#
+# $1 is the URL to download
+download-or-bust() {
+  until [[ -e "${1##*/}" ]]; do
+    echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
+    curl --ipv4 -LO --connect-timeout 20 --retry 6 --retry-delay 10 "$1"
+  done
+}
+
+# Install salt from GCS.  See README.md for instructions on how to update these
+# debs.
+#
+# $1 If set to --master, also install the master
+install-salt() {
+  apt-get update
+
+  mkdir -p /var/cache/salt-install
+  cd /var/cache/salt-install
+
+  TARS=(
+    libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb
+    python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb
+    salt-common_2014.1.13+ds-1~bpo70+1_all.deb
+    salt-minion_2014.1.13+ds-1~bpo70+1_all.deb
+  )
+  if [[ ${1-} == '--master' ]]; then
+    TARS+=(salt-master_2014.1.13+ds-1~bpo70+1_all.deb)
+  fi
+  URL_BASE="https://storage.googleapis.com/kubernetes-release/salt"
+
+  for tar in "${TARS[@]}"; do
+    download-or-bust "${URL_BASE}/${tar}"
+    dpkg -i "${tar}"
+  done
+
+  # This will install any of the unmet dependencies from above.
+  apt-get install -f -y
+
+}

cluster/gce/templates/download-release.sh

@@ -22,10 +22,10 @@
 
 
 echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)"
-gsutil cp "$SERVER_BINARY_TAR_URL" .
+download-or-bust "$SERVER_BINARY_TAR_URL"
 
 echo "Downloading binary release tar ($SALT_TAR_URL)"
-gsutil cp "$SALT_TAR_URL" .
+download-or-bust "$SALT_TAR_URL"
 
 echo "Unpacking Salt tree"
 rm -rf kubernetes

cluster/gce/templates/salt-master.sh

@@ -21,17 +21,18 @@ sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d
 mkdir -p /etc/salt/minion.d
 echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf
 
+cat <<EOF >/etc/salt/minion.d/log-level-debug.conf
+log_level: debug
+log_level_logfile: debug
+EOF
+
 cat <<EOF >/etc/salt/minion.d/grains.conf
 grains:
   roles:
     - kubernetes-master
   cloud: gce
 EOF
 
-cat <<EOF >/srv/pillar/cluster-params.sls
-node_instance_prefix: $NODE_INSTANCE_PREFIX
-EOF
-
 # Auto accept all keys from minions that try to join
 mkdir -p /etc/salt/master.d
 cat <<EOF >/etc/salt/master.d/auto-accept.conf
@@ -42,20 +43,18 @@ cat <<EOF >/etc/salt/master.d/reactor.conf
 # React to new minions starting by running highstate on them.
 reactor:
   - 'salt/minion/*/start':
-    - /srv/reactor/start.sls
+    - /srv/reactor/highstate-new.sls
 EOF
 
-mkdir -p /srv/salt/nginx
-echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd
+cat <<EOF >/etc/salt/master.d/log-level-debug.d
+log_level: debug
+log_level_logfile: debug
+EOF
 
-# Install Salt
-#
-# We specify -X to avoid a race condition that can cause minion failure to
-# install.  See https://github.com/saltstack/salt-bootstrap/issues/270
-#
-# -M installs the master
-set +x
-curl -L --connect-timeout 20 --retry 6 --retry-delay 10 http://bootstrap.saltstack.com | sh -s -- -M -X
-set -x
+install-salt --master
 
-echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd
+# Wait a few minutes and trigger another Salt run to better recover from
+# any transient errors.
+echo "Sleeping 180"
+sleep 180
+salt-call state.highstate || true