diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 4b8a7ca085433..59233ba859a00 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -12,7 +12,7 @@ least one maintainer on relevant issues and PRs. * GCE: [Brendan Burns](https://github.com/brendandburns), [Joe Beda](https://github.com/jbeda), [Daniel Smith](https://github.com/lavalamp), [Tim Hockin](https://github.com/thockin) * Azure: [Jeff Mendoza](https://github.com/jeffmendoza) -* VSphere: [Pieter Noordhuis](https://github.com/pietern) +* vSphere: [Pieter Noordhuis](https://github.com/pietern) * Rackspace: [Ryan Richard](https://github.com/doublerr) * oVirt: [Federico Simoncelli](https://github.com/simon3z) * Local: [Derek Carr](https://github.com/derekwaynecarr) diff --git a/README.md b/README.md index d83451cc4a5ab..953026c52a8ad 100644 --- a/README.md +++ b/README.md @@ -23,11 +23,12 @@ While the concepts and architecture in Kubernetes represent years of experience * [CoreOS](docs/getting-started-guides/coreos.md) * [OpenStack](https://developer.rackspace.com/blog/running-coreos-and-kubernetes/) * [CloudStack](docs/getting-started-guides/cloudstack.md) + * [Rackspace](docs/getting-started-guides/rackspace.md) + * [vSphere](docs/getting-started-guides/vsphere.md) + * The following clouds are currently broken at Kubernetes head. Please sync your client to `v0.3` (`git checkout v0.3`) to use these: * [Locally](docs/getting-started-guides/locally.md) - * [vSphere](docs/getting-started-guides/vsphere.md) * [Microsoft Azure](docs/getting-started-guides/azure.md) - * [Rackspace](docs/getting-started-guides/rackspace.md) * [Kubernetes 101](https://github.com/GoogleCloudPlatform/kubernetes/tree/master/examples/walkthrough) * [kubecfg command line tool](https://github.com/GoogleCloudPlatform/kubernetes/blob/master/docs/cli.md) * [Kubernetes API Documentation](http://cdn.rawgit.com/GoogleCloudPlatform/kubernetes/31a0daae3627c91bc96e1f02a6344cd76e294791/api/kubernetes.html) diff --git a/build/README.md b/build/README.md index 49a5d0dd9dbff..c2b2fadd90542 100644 --- a/build/README.md +++ b/build/README.md @@ -51,11 +51,11 @@ Env Variable | Default | Description `KUBE_SKIP_CONFIRMATIONS` | `n` | If `y` then no questions are asked and the scripts just continue. `KUBE_GCS_UPLOAD_RELEASE` | `n` | Upload release artifacts to GCS `KUBE_GCS_RELEASE_BUCKET` | `kubernetes-releases-${project_hash}` | The bucket to upload releases to -`KUBE_GCS_RELEASE_PREFIX` | `devel/` | The path under the release bucket to put releases +`KUBE_GCS_RELEASE_PREFIX` | `devel` | The path under the release bucket to put releases `KUBE_GCS_MAKE_PUBLIC` | `y` | Make GCS links readable from anywhere `KUBE_GCS_NO_CACHING` | `y` | Disable HTTP caching of GCS release artifacts. By default GCS will cache public objects for up to an hour. When doing "devel" releases this can cause problems. `KUBE_BUILD_RUN_IMAGES` | `n` | *Experimental* Build Docker images for running most server components. -`KUBE_GCS_DOCKER_REG_PREFIX` | `docker-reg/` | *Experimental* When uploading docker images, the bucket that backs the registry. +`KUBE_GCS_DOCKER_REG_PREFIX` | `docker-reg` | *Experimental* When uploading docker images, the bucket that backs the registry. ## Basic Flow diff --git a/build/common.sh b/build/common.sh index 42348d43ed6b0..c1791daa3196d 100644 --- a/build/common.sh +++ b/build/common.sh @@ -35,9 +35,8 @@ readonly KUBE_GCS_UPLOAD_RELEASE="${KUBE_GCS_UPLOAD_RELEASE:-n}" readonly KUBE_GCS_NO_CACHING="${KUBE_GCS_NO_CACHING:-y}" readonly KUBE_GCS_MAKE_PUBLIC="${KUBE_GCS_MAKE_PUBLIC:-y}" # KUBE_GCS_RELEASE_BUCKET default: kubernetes-releases-${project_hash} -# KUBE_GCS_RELEASE_PREFIX default: devel/ -# KUBE_GCS_DOCKER_REG_PREFIX default: docker-reg/ - +readonly KUBE_GCS_RELEASE_PREFIX=${KUBE_GCS_RELEASE_PREFIX-devel}/ +readonly KUBE_GCS_DOCKER_REG_PREFIX=${KUBE_GCS_DOCKER_REG_PREFIX-docker-reg}/ # Constants @@ -370,35 +369,38 @@ function kube::build::run_build_command() { docker rm "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true } +# Test if the output directory is remote (and can only be accessed through +# docker) or if it is "local" and we can access the output without going through +# docker. +function kube::build::is_output_remote() { + rm -f "${LOCAL_OUTPUT_BUILD}/test_for_remote" + kube::build::run_build_command touch "${REMOTE_OUTPUT_DIR}/test_for_remote" + + [[ ! -e "${LOCAL_OUTPUT_BUILD}/test_for_remote" ]] +} + # If the Docker server is remote, copy the results back out. function kube::build::copy_output() { - if kube::build::is_osx; then - # When we are on the Mac with boot2docker we need to copy the results back - # out. Ideally we would leave the container around and use 'docker cp' to - # copy the results out. However, that doesn't work for mounted volumes - # currently (https://github.com/dotcloud/docker/issues/1992). And it is - # just plain broken (https://github.com/dotcloud/docker/issues/6483). + if kube::build::is_output_remote; then + # When we are on the Mac with boot2docker (or to a remote Docker in any + # other situation) we need to copy the results back out. Ideally we would + # leave the container around and use 'docker cp' to copy the results out. + # However, that doesn't work for mounted volumes currently + # (https://github.com/dotcloud/docker/issues/1992). And it is just plain + # broken (https://github.com/dotcloud/docker/issues/6483). # # The easiest thing I (jbeda) could figure out was to launch another # container pointed at the same volume, tar the output directory and ship - # that tar over stdou. - local -ra docker_cmd=( - docker run -a stdout "--name=${KUBE_BUILD_CONTAINER_NAME}" - "${DOCKER_MOUNT_ARGS[@]}" "${KUBE_BUILD_IMAGE}") - - # Kill any leftover container - docker rm "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true + # that tar over stdout. echo "+++ Syncing back _output directory from boot2docker VM" rm -rf "${LOCAL_OUTPUT_BUILD}" mkdir -p "${LOCAL_OUTPUT_BUILD}" - "${docker_cmd[@]}" sh -c "tar c -C ${REMOTE_OUTPUT_DIR} . ; sleep 1" \ - | tar xv -C "${LOCAL_OUTPUT_BUILD}" - # Remove the container after we run. '--rm' might be appropriate but it - # appears that sometimes it fails. See - # https://github.com/docker/docker/issues/3968 - docker rm "${KUBE_BUILD_CONTAINER_NAME}" >/dev/null 2>&1 || true + # The ' /dev/null; then + tar=gtar + fi + + local tar_cmd=("$tar" "czf" "${tarfile}" "-C" "${stagingdir}" "kubernetes") + if "$tar" --version | grep -q GNU; then + tar_cmd=("${tar_cmd[@]}" "--owner=0" "--group=0") + else + echo " !!! GNU tar not available. User names will be embedded in output and" + echo " release tars are not official. Build on Linux or install GNU tar" + echo " on Mac OS X (brew install gnu-tar)" + fi + + "${tar_cmd[@]}" +} # --------------------------------------------------------------------------- # GCS Release @@ -530,7 +558,7 @@ function kube::release::gcs::release() { kube::release::gcs::verify_prereqs kube::release::gcs::ensure_release_bucket kube::release::gcs::push_images - kube::release::gcs::copy_release_tarballs + kube::release::gcs::copy_release_artifacts } # Verify things are set up for uploading to GCS @@ -569,8 +597,6 @@ function kube::release::gcs::ensure_release_bucket() { local project_hash project_hash=$(kube::build::short_hash "$GCLOUD_PROJECT") KUBE_GCS_RELEASE_BUCKET=${KUBE_GCS_RELEASE_BUCKET-kubernetes-releases-${project_hash}} - KUBE_GCS_RELEASE_PREFIX=${KUBE_GCS_RELEASE_PREFIX-devel/} - KUBE_GCS_DOCKER_REG_PREFIX=${KUBE_GCS_DOCKER_REG_PREFIX-docker-reg/} if ! gsutil ls "gs://${KUBE_GCS_RELEASE_BUCKET}" >/dev/null 2>&1 ; then echo "Creating Google Cloud Storage bucket: $RELEASE_BUCKET" @@ -631,7 +657,7 @@ function kube::release::gcs::push_images() { done } -function kube::release::gcs::copy_release_tarballs() { +function kube::release::gcs::copy_release_artifacts() { # TODO: This isn't atomic. There will be points in time where there will be # no active release. Also, if something fails, the release could be half- # copied. The real way to do this would perhaps to have some sort of release @@ -643,17 +669,27 @@ function kube::release::gcs::copy_release_tarballs() { gcs_options=("-h" "Cache-Control:private, max-age=0") fi - echo "+++ Copying client tarballs to ${gcs_destination}" + echo "+++ Copying release artifacts to ${gcs_destination}" # First delete all objects at the destination gsutil -q rm -f -R "${gcs_destination}" >/dev/null 2>&1 || true # Now upload everything in release directory - gsutil -m "${gcs_options[@]-}" cp -r "${RELEASE_DIR}"/* "${gcs_destination}" >/dev/null 2>&1 + gsutil -m "${gcs_options[@]+${gcs_options[@]}}" cp -r "${RELEASE_DIR}"/* "${gcs_destination}" + + # Having the "template" scripts from the GCE cluster deploy hosted with the + # release is useful for GKE. Copy everything from that directory up also. + gsutil -m "${gcs_options[@]+${gcs_options[@]}}" cp "${KUBE_ROOT}/cluster/gce/templates/*.sh" "${gcs_destination}extra/gce-templates/" + + # TODO(jbeda): Generate an HTML page with links for this release so it is easy + # to see it. For extra credit, generate a dynamic page that builds up the + # release list using the GCS JSON API. Use Angular and Bootstrap for extra + # extra credit. if [[ ${KUBE_GCS_MAKE_PUBLIC} =~ ^[yY]$ ]]; then + echo "+++ Marking all uploaded objects public" gsutil acl ch -R -g all:R "${gcs_destination}" >/dev/null 2>&1 fi - gsutil ls -lh "${gcs_destination}" + gsutil ls -lhr "${gcs_destination}" } diff --git a/cluster/gce/templates/README.md b/cluster/gce/templates/README.md new file mode 100644 index 0000000000000..967e0ffe3dbdb --- /dev/null +++ b/cluster/gce/templates/README.md @@ -0,0 +1,12 @@ +# Updating Salt debs + +We are caching all of the salt debs in GCS for speed and reliability. + +To update them, follow this simple N step process: + +1. Start up a new base image without salt installed. SSH into this image. +2. Install salt via their recommended method: `curl -L https://bootstrap.saltstack.com | sudo sh -s -- -M -X` +3. Find and download the debs that originated at the saltstack.com repo: `aptitude search --disable-columns -F "%p %V" "?installed?origin(saltstack.com)" | xargs aptitude download` +4. Upload these to GCS: `gsutil cp *.deb gs://kubernetes-release/salt/` +5. Make sure that everything is publicly readable: `gsutil acl ch -R -g all:R gs://kubernetes-release/salt/` +6. Test things well :) diff --git a/cluster/gce/templates/common.sh b/cluster/gce/templates/common.sh new file mode 100644 index 0000000000000..784f58685d6a9 --- /dev/null +++ b/cluster/gce/templates/common.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Retry a download until we get it. +# +# $1 is the URL to download +download-or-bust() { + until [[ -e "${1##*/}" ]]; do + echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)" + curl --ipv4 -LO --connect-timeout 20 --retry 6 --retry-delay 10 "$1" + done +} + +# Install salt from GCS. See README.md for instructions on how to update these +# debs. +# +# $1 If set to --master, also install the master +install-salt() { + apt-get update + + mkdir -p /var/cache/salt-install + cd /var/cache/salt-install + + TARS=( + libzmq3_3.2.3+dfsg-1~bpo70~dst+1_amd64.deb + python-zmq_13.1.0-1~bpo70~dst+1_amd64.deb + salt-common_2014.1.13+ds-1~bpo70+1_all.deb + salt-minion_2014.1.13+ds-1~bpo70+1_all.deb + ) + if [[ ${1-} == '--master' ]]; then + TARS+=(salt-master_2014.1.13+ds-1~bpo70+1_all.deb) + fi + URL_BASE="https://storage.googleapis.com/kubernetes-release/salt" + + for tar in "${TARS[@]}"; do + download-or-bust "${URL_BASE}/${tar}" + dpkg -i "${tar}" + done + + # This will install any of the unmet dependencies from above. + apt-get install -f -y + +} diff --git a/cluster/gce/templates/download-release.sh b/cluster/gce/templates/download-release.sh index 3982dd0f50eb7..5fadefdb26ca8 100755 --- a/cluster/gce/templates/download-release.sh +++ b/cluster/gce/templates/download-release.sh @@ -22,10 +22,10 @@ echo "Downloading binary release tar ($SERVER_BINARY_TAR_URL)" -gsutil cp "$SERVER_BINARY_TAR_URL" . +download-or-bust "$SERVER_BINARY_TAR_URL" echo "Downloading binary release tar ($SALT_TAR_URL)" -gsutil cp "$SALT_TAR_URL" . +download-or-bust "$SALT_TAR_URL" echo "Unpacking Salt tree" rm -rf kubernetes diff --git a/cluster/gce/templates/salt-master.sh b/cluster/gce/templates/salt-master.sh index 132497774da09..c8766cb821070 100755 --- a/cluster/gce/templates/salt-master.sh +++ b/cluster/gce/templates/salt-master.sh @@ -21,6 +21,11 @@ sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d mkdir -p /etc/salt/minion.d echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf +cat </etc/salt/minion.d/log-level-debug.conf +log_level: debug +log_level_logfile: debug +EOF + cat </etc/salt/minion.d/grains.conf grains: roles: @@ -28,10 +33,6 @@ grains: cloud: gce EOF -cat </srv/pillar/cluster-params.sls -node_instance_prefix: $NODE_INSTANCE_PREFIX -EOF - # Auto accept all keys from minions that try to join mkdir -p /etc/salt/master.d cat </etc/salt/master.d/auto-accept.conf @@ -42,20 +43,18 @@ cat </etc/salt/master.d/reactor.conf # React to new minions starting by running highstate on them. reactor: - 'salt/minion/*/start': - - /srv/reactor/start.sls + - /srv/reactor/highstate-new.sls EOF -mkdir -p /srv/salt/nginx -echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd +cat </etc/salt/master.d/log-level-debug.d +log_level: debug +log_level_logfile: debug +EOF -# Install Salt -# -# We specify -X to avoid a race condition that can cause minion failure to -# install. See https://github.com/saltstack/salt-bootstrap/issues/270 -# -# -M installs the master -set +x -curl -L --connect-timeout 20 --retry 6 --retry-delay 10 http://bootstrap.saltstack.com | sh -s -- -M -X -set -x +install-salt --master -echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd +# Wait a few minutes and trigger another Salt run to better recover from +# any transient errors. +echo "Sleeping 180" +sleep 180 +salt-call state.highstate || true diff --git a/cluster/gce/templates/salt-minion.sh b/cluster/gce/templates/salt-minion.sh index 7cc8176f32cab..6e6e7d140c495 100755 --- a/cluster/gce/templates/salt-minion.sh +++ b/cluster/gce/templates/salt-minion.sh @@ -22,8 +22,10 @@ sed -i -e "\|^deb.*http://ftp.debian.org/debian| s/^/#/" /etc/apt/sources.list.d mkdir -p /etc/salt/minion.d echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf -# Turn on debugging for salt-minion -# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion +cat </etc/salt/minion.d/log-level-debug.conf +log_level: debug +log_level_logfile: debug +EOF # Our minions will have a pool role to distinguish them from the master. cat </etc/salt/minion.d/grains.conf @@ -34,8 +36,10 @@ grains: cloud: gce EOF -# Install Salt -# -# We specify -X to avoid a race condition that can cause minion failure to -# install. See https://github.com/saltstack/salt-bootstrap/issues/270 -curl -L --connect-timeout 20 --retry 6 --retry-delay 10 https://bootstrap.saltstack.com | sh -s -- -X +install-salt + +# Wait a few minutes and trigger another Salt run to better recover from +# any transient errors. +echo "Sleeping 180" +sleep 180 +salt-call state.highstate || true diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 1631820be106a..60769f3fc238e 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -121,10 +121,16 @@ function upload-server-tars() { local -r staging_path="${staging_bucket}/devel" echo "+++ Staging server tars to Google Storage: ${staging_path}" - SERVER_BINARY_TAR_URL="${staging_path}/${SERVER_BINARY_TAR##*/}" - gsutil -q cp "${SERVER_BINARY_TAR}" "${SERVER_BINARY_TAR_URL}" - SALT_TAR_URL="${staging_path}/${SALT_TAR##*/}" - gsutil -q cp "${SALT_TAR}" "${SALT_TAR_URL}" + local server_binary_gs_url="${staging_path}/${SERVER_BINARY_TAR##*/}" + gsutil -q -h "Cache-Control:private, max-age=0" cp "${SERVER_BINARY_TAR}" "${server_binary_gs_url}" + gsutil acl ch -g all:R "${server_binary_gs_url}" >/dev/null 2>&1 + local salt_gs_url="${staging_path}/${SALT_TAR##*/}" + gsutil -q -h "Cache-Control:private, max-age=0" cp "${SALT_TAR}" "${salt_gs_url}" + gsutil acl ch -g all:R "${salt_gs_url}" >/dev/null 2>&1 + + # Convert from gs:// URL to an https:// URL + SERVER_BINARY_TAR_URL="${server_binary_gs_url/gs:\/\//https://storage.googleapis.com/}" + SALT_TAR_URL="${salt_gs_url/gs:\/\//https://storage.googleapis.com/}" } # Detect the information about the minions @@ -263,6 +269,7 @@ function kube-up { echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'" echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'" echo "readonly MASTER_HTPASSWD='${htpasswd}'" + grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/create-dynamic-salt-files.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-master.sh" @@ -286,6 +293,7 @@ function kube-up { echo "#! /bin/bash" echo "MASTER_NAME='${MASTER_NAME}'" echo "MINION_IP_RANGE='${MINION_IP_RANGES[$i]}'" + grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/salt-minion.sh" ) > "${KUBE_TEMP}/minion-start-${i}.sh" @@ -458,6 +466,7 @@ function kube-push { echo "cd /var/cache/kubernetes-install" echo "readonly SERVER_BINARY_TAR_URL='${SERVER_BINARY_TAR_URL}'" echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'" + grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/download-release.sh" echo "echo Executing configuration" echo "sudo salt '*' mine.update" diff --git a/cluster/rackspace/cloud-config/master-cloud-config.yaml b/cluster/rackspace/cloud-config/master-cloud-config.yaml new file mode 100644 index 0000000000000..477db802f52e3 --- /dev/null +++ b/cluster/rackspace/cloud-config/master-cloud-config.yaml @@ -0,0 +1,152 @@ +#cloud-config + +write_files: + - path: /opt/bin/regen-minion-list.sh + permissions: 0755 + content: | + #!/bin/sh + m=$(echo $(etcdctl ls --recursive /corekube/minions | cut -d/ -f4 | sort) | tr ' ' ,) + echo "Found $m" + mkdir -p /run/apiserver + echo "MINIONS=$m" > /run/apiserver/minions.env + - path: /opt/bin/git-kubernetes-nginx.sh + permissions: 0755 + content: | + #!/bin/bash + git clone https://github.com/doublerr/kubernetes_nginx /opt/kubernetes_nginx + /usr/bin/cp /opt/.kubernetes_auth /opt/kubernetes_nginx/.kubernetes_auth + docker build -t kubernetes_nginx:latest /opt/kubernetes_nginx + - path: /opt/bin/download-release.sh + permissions: 0755 + content: | + #!/bin/bash + OBJECT_URL="CLOUD_FILES_URL" + echo "Downloading release ($OBJECT_URL)" + wget "${OBJECT_URL}" -O /opt/kubernetes.tar.gz + echo "Unpacking release" + rm -rf /opt/kubernetes || false + tar xzf /opt/kubernetes.tar.gz -C /opt/ + - path: /opt/.kubernetes_auth + permissions: 0600 + content: | + KUBE_USER:KUBE_PASSWORD + +coreos: + etcd: + name: kubernetes-master + discovery: https://discovery.etcd.io/DISCOVERY_ID + addr: $private_ipv4:4001 + peer-addr: $private_ipv4:7001 + peer-bind-addr: $private_ipv4:7001 + + fleet: + public-ip: $private_ipv4 + metadata: kubernetes_role=master + + update: + reboot-strategy: etcd-lock + + units: + - name: etcd.service + command: start + - name: fleet.service + command: start + - name: download-release.service + command: start + content: | + [Unit] + Description=Downloads Kubernetes Release + After=network-online.target + Requires=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/bin/bash /opt/bin/download-release.sh + - name: master-apiserver.service + command: start + content: | + [Unit] + Description=Kubernetes API Server + Documentation=https://github.com/GoogleCloudPlatform/kubernetes + After=network-online.target + Requires=network-online.target + After=minion-finder.service + Requires=minion-finder.service + After=download-release.service + Requires=download-release.service + [Service] + EnvironmentFile=-/run/apiserver/minions.env + ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/apiserver /opt/bin/apiserver + ExecStart=/opt/bin/apiserver --address=127.0.0.1 --port=8080 --machines=${MINIONS} --etcd_servers=http://127.0.0.1:4001 --portal_net=PORTAL_NET --logtostderr=true + Restart=always + RestartSec=2 + - name: master-apiserver-sighup.path + command: start + content: | + [Path] + PathChanged=/run/apiserver/minions.env + - name: master-apiserver-sighup.service + command: start + content: | + [Service] + ExecStart=/usr/bin/pkill -SIGHUP -f apiserver + - name: minion-finder.service + command: start + content: | + [Unit] + Description=Kubernetes Minion finder + After=network-online.target + Requires=network-online.target + After=etcd.service + Requires=etcd.service + [Service] + ExecStartPre=/opt/bin/regen-minion-list.sh + ExecStart=/usr/bin/etcdctl exec-watch --recursive /corekube/minions -- /opt/bin/regen-minion-list.sh + Restart=always + RestartSec=30 + - name: master-controller-manager.service + command: start + content: | + [Unit] + Description=Kubernetes Controller Manager + Documentation=https://github.com/GoogleCloudPlatform/kubernetes + After=network-online.target + Requires=network-online.target + After=master-apiserver.service + Requires=master-apiserver.service + [Service] + ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/controller-manager /opt/bin/controller-manager + ExecStart=/opt/bin/controller-manager --master=127.0.0.1:8080 --logtostderr=true + Restart=always + RestartSec=2 + - name: master-scheduler.service + command: start + content: | + [Unit] + Description=Kubernetes Scheduler + Documentation=https://github.com/GoogleCloudPlatform/kubernetes + After=network-online.target + Requires=network-online.target + After=master-apiserver.service + Requires=master-apiserver.service + [Service] + ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/scheduler /opt/bin/scheduler + ExecStart=/opt/bin/scheduler --master=127.0.0.1:8080 --logtostderr=true + Restart=always + RestartSec=10 + #Running nginx service with --net="host" is a necessary evil until running all k8s services in docker. + - name: kubernetes-nginx.service + command: start + content: | + [Unit] + Description=Kubernetes Nginx Service + After=network-online.target + Requires=network-online.target + After=docker.service + Requires=docker.service + [Service] + ExecStartPre=/opt/bin/git-kubernetes-nginx.sh + ExecStart=/usr/bin/docker run --rm --net="host" -p "443:443" -t --name "kubernetes_nginx" kubernetes_nginx + ExecStop=/usr/bin/docker stop kubernetes_nginx + Restart=always + RestartSec=15 diff --git a/cluster/rackspace/cloud-config/minion-cloud-config.yaml b/cluster/rackspace/cloud-config/minion-cloud-config.yaml new file mode 100644 index 0000000000000..40c30dbbaa24f --- /dev/null +++ b/cluster/rackspace/cloud-config/minion-cloud-config.yaml @@ -0,0 +1,224 @@ +#cloud-config + +write_files: + - path: /opt/bin/kube-net-update.sh + permissions: 0755 + content: | + #!/bin/sh + set -x -e + nh=${ETCD_WATCH_KEY##*/} + net=$ETCD_WATCH_VALUE + case $ETCD_WATCH_ACTION in + set) ip route replace $net via $nh dev eth2 metric 900 ;; + expire) ip route del $net via $nh metric 900 ;; + esac + - path: /opt/bin/download-release.sh + permissions: 0755 + content: | + #!/bin/bash + OBJECT_URL="CLOUD_FILES_URL" + echo "Downloading release ($OBJECT_URL)" + wget "${OBJECT_URL}" -O /opt/kubernetes.tar.gz + echo "Unpacking release" + rm -rf /opt/kubernetes || false + tar xzf /opt/kubernetes.tar.gz -C /opt/ + - path: /opt/kubernetes-manifests/cadvisor.manifest + permissions: 0755 + content: | + version: v1beta2 + id: cadvisor-agent + containers: + - name: cadvisor + image: google/cadvisor:latest + ports: + - name: http + containerPort: 8080 + hostPort: 4194 + volumeMounts: + - name: varrun + mountPath: /var/run + readOnly: false + - name: varlibdocker + mountPath: /var/lib/docker + readOnly: true + - name: cgroups + mountPath: /sys/fs/cgroup + readOnly: true + volumes: + - name: varrun + source: + hostDir: + path: /var/run + - name: varlibdocker + source: + hostDir: + path: /var/lib/docker + - name: cgroups + source: + hostDir: + path: /sys/fs/cgroup + +coreos: + etcd: + name: kubernetes-minion-INDEX + discovery: https://discovery.etcd.io/DISCOVERY_ID + addr: $private_ipv4:4001 + peer-addr: $private_ipv4:7001 + peer-bind-addr: $private_ipv4:7001 + + fleet: + public-ip: $private_ipv4 + metadata: kubernetes_role=minion + + update: + reboot-strategy: etcd-lock + + units: + - name: etcd.service + command: start + - name: fleet.service + command: start + - name: download-release.service + command: start + content: | + [Unit] + Description=Downloads Kubernetes Release + After=network-online.target + Requires=network-online.target + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/bin/bash /opt/bin/download-release.sh + - name: minion-kubelet.service + command: start + content: | + [Unit] + Description=Kubernetes Kubelet + Documentation=https://github.com/GoogleCloudPlatform/kubernetes + After=network-online.target + Requires=network-online.target + After=docker.service + Requires=docker.service + After=download-release.service + Requires=download-release.service + [Service] + ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/kubelet /opt/bin/kubelet + ExecStart=/opt/bin/kubelet --address=$private_ipv4 --hostname_override=$private_ipv4 --etcd_servers=http://127.0.0.1:4001 --logtostderr=true --config=/opt/kubernetes-manifests + Restart=always + RestartSec=2 + - name: minion-proxy.service + command: start + content: | + [Unit] + Description=Kubernetes Proxy + Documentation=https://github.com/GoogleCloudPlatform/kubernetes + After=network-online.target + Requires=network-online.target + After=docker.service + Requires=docker.service + After=download-release.service + Requires=download-release.service + [Service] + ExecStartPre=/usr/bin/ln -sf /opt/kubernetes/server/bin/proxy /opt/bin/proxy + ExecStart=/opt/bin/proxy --bind_address=$private_ipv4 --etcd_servers=http://127.0.0.1:4001 --logtostderr=true + Restart=always + RestartSec=2 + - name: minion-advertiser.service + command: start + content: | + [Unit] + Description=Kubernetes Minion Advertiser + After=etcd.service + Requires=etcd.service + After=minion-kubelet.service + [Service] + ExecStart=/bin/sh -c 'while :; do etcdctl set /corekube/minions/$private_ipv4 $private_ipv4 --ttl 300; sleep 120; done' + Restart=always + RestartSec=120 + - name: net-advertiser.service + command: start + content: | + [Unit] + Description=Kubernetes Network Advertiser + After=etcd.service + Requires=etcd.service + After=minion-kubelet.service + [Service] + ExecStart=/bin/sh -c 'eth2_ip=$$(ip -o -f inet a show dev eth2 | sed "s/.* inet \([0-9.]\+\).*/\1/"); while :; do etcdctl set /corekube/net/$$eth2_ip 10.240.INDEX.0/24 --ttl 300; sleep 120; done' + Restart=always + RestartSec=120 + - name: net-router.service + command: start + content: | + [Unit] + Description=Kubernetes Network Router + After=etcd.service + Requires=etcd.service + After=minion-kubelet.service + [Service] + ExecStart=/usr/bin/etcdctl exec-watch --recursive /corekube/net -- /opt/bin/kube-net-update.sh + Restart=always + RestartSec=120 + - name: cbr0.netdev + command: start + content: | + [NetDev] + Kind=bridge + Name=cbr0 + - name: cbr0.network + command: start + content: | + [Match] + Name=cbr0 + + [Network] + Address=10.240.INDEX.1/24 + - name: nat.service + command: start + content: | + [Unit] + Description=NAT container->outside traffic + + [Service] + ExecStart=/usr/sbin/iptables -t nat -A POSTROUTING -o eth0 -s 10.240.INDEX.0/24 -j MASQUERADE + ExecStart=/usr/sbin/iptables -t nat -A POSTROUTING -o eth1 -s 10.240.INDEX.0/24 -j MASQUERADE + RemainAfterExit=yes + Type=oneshot + - name: docker.service + command: start + content: | + [Unit] + After=network.target + Description=Docker Application Container Engine + Documentation=http://docs.docker.io + + [Service] + ExecStartPre=/bin/mount --make-rprivate / + ExecStart=/usr/bin/docker -d -s=btrfs -H fd:// -b cbr0 --iptables=false + Restart=always + RestartSec=30 + + [Install] + WantedBy=multi-user.target + - name: format-data.service + command: start + content: | + [Unit] + Description=Formats data drive + [Service] + Type=oneshot + RemainAfterExit=yes + ExecStart=/usr/sbin/wipefs -f /dev/xvde1 + ExecStart=/usr/sbin/mkfs.btrfs -f /dev/xvde1 + - name: var-lib-docker-volumes.mount + command: start + content: | + [Unit] + Description=Mount data drive to /var/lib/docker/volumes + Requires=format-data.service + After=format-data.service + Before=docker.service + [Mount] + What=/dev/xvde1 + Where=/var/lib/docker/volumes + Type=btrfs diff --git a/icebox/cluster/rackspace/config-default.sh b/cluster/rackspace/config-default.sh similarity index 84% rename from icebox/cluster/rackspace/config-default.sh rename to cluster/rackspace/config-default.sh index 4febe5771022d..635468dc8a646 100644 --- a/icebox/cluster/rackspace/config-default.sh +++ b/cluster/rackspace/config-default.sh @@ -19,7 +19,7 @@ # KUBE_IMAGE, KUBE_MASTER_FLAVOR, KUBE_MINION_FLAVOR, NUM_MINIONS, NOVA_NETWORK and SSH_KEY_NAME # Shared -KUBE_IMAGE="${KUBE_IMAGE-255df5fb-e3d4-45a3-9a07-c976debf7c14}" # Ubuntu 14.04 LTS (Trusty Tahr) (PVHVM) +KUBE_IMAGE="${KUBE_IMAGE-b63e1435-a46f-4726-b984-e3f15ae92753}" # CoreOS(Beta) SSH_KEY_NAME="${SSH_KEY_NAME-id_kubernetes}" NOVA_NETWORK_LABEL="kubernetes-pool-net" NOVA_NETWORK_CIDR="${NOVA_NETWORK-192.168.0.0/24}" @@ -28,11 +28,11 @@ INSTANCE_PREFIX="kubernetes" # Master KUBE_MASTER_FLAVOR="${KUBE_MASTER_FLAVOR-performance1-1}" MASTER_NAME="${INSTANCE_PREFIX}-master" -MASTER_TAG="tag=${INSTANCE_PREFIX}-master" +MASTER_TAG="tags=${INSTANCE_PREFIX}-master" # Minion -KUBE_MINION_FLAVOR="${KUBE_MINION_FLAVOR-performance1-1}" +KUBE_MINION_FLAVOR="${KUBE_MINION_FLAVOR-performance1-2}" RAX_NUM_MINIONS="${RAX_NUM_MINIONS-4}" -MINION_TAG="tag=${INSTANCE_PREFIX}-minion" +MINION_TAG="tags=${INSTANCE_PREFIX}-minion" MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${RAX_NUM_MINIONS}})) KUBE_NETWORK=($(eval echo "10.240.{1..${RAX_NUM_MINIONS}}.0/24")) diff --git a/icebox/cluster/rackspace/kube-up.sh b/cluster/rackspace/kube-up.sh similarity index 100% rename from icebox/cluster/rackspace/kube-up.sh rename to cluster/rackspace/kube-up.sh diff --git a/icebox/cluster/rackspace/util.sh b/cluster/rackspace/util.sh similarity index 58% rename from icebox/cluster/rackspace/util.sh rename to cluster/rackspace/util.sh index fc667e13632a6..86f898f273875 100644 --- a/icebox/cluster/rackspace/util.sh +++ b/cluster/rackspace/util.sh @@ -18,16 +18,61 @@ # Use the config file specified in $KUBE_CONFIG_FILE, or default to # config-default.sh. +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. source $(dirname ${BASH_SOURCE})/${KUBE_CONFIG_FILE-"config-default.sh"} verify-prereqs() { # Make sure that prerequisites are installed. - for x in nova; do + for x in nova swiftly; do if [ "$(which $x)" == "" ]; then echo "cluster/rackspace/util.sh: Can't find $x in PATH, please fix and retry." exit 1 fi done + + if [[ -z "${OS_AUTH_URL-}" ]]; then + echo "cluster/rackspace/util.sh: OS_AUTH_URL not set." + echo -e "\texport OS_AUTH_URL=https://identity.api.rackspacecloud.com/v2.0/" + return 1 + fi + + if [[ -z "${OS_USERNAME-}" ]]; then + echo "cluster/rackspace/util.sh: OS_USERNAME not set." + echo -e "\texport OS_USERNAME=myusername" + return 1 + fi + + if [[ -z "${OS_PASSWORD-}" ]]; then + echo "cluster/rackspace/util.sh: OS_PASSWORD not set." + echo -e "\texport OS_PASSWORD=myapikey" + return 1 + fi +} + +# Ensure that we have a password created for validating to the master. Will +# read from $HOME/.kubernetres_auth if available. +# +# Vars set: +# KUBE_USER +# KUBE_PASSWORD +get-password() { + local file="$HOME/.kubernetes_auth" + if [[ -r "$file" ]]; then + KUBE_USER=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["User"]') + KUBE_PASSWORD=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["Password"]') + return + fi + KUBE_USER=admin + KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))') + + # Store password for reuse. + cat << EOF > "$file" +{ + "User": "$KUBE_USER", + "Password": "$KUBE_PASSWORD" +} +EOF + chmod 0600 "$file" } rax-ssh-key() { @@ -45,35 +90,72 @@ rax-ssh-key() { fi } -find-object-url() { - if [ -n "$1" ]; then - CONTAINER=$1 - else - local RELEASE_CONFIG_SCRIPT=$(dirname $0)/../release/rackspace/config.sh - if [ -f $(dirname $0)/../release/rackspace/config.sh ]; then - . $RELEASE_CONFIG_SCRIPT - fi +find-release-tars() { + SERVER_BINARY_TAR="${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz" + RELEASE_DIR="${KUBE_ROOT}/server/" + if [[ ! -f "$SERVER_BINARY_TAR" ]]; then + SERVER_BINARY_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz" + RELEASE_DIR="${KUBE_ROOT}/_output/release-tars/" fi + if [[ ! -f "$SERVER_BINARY_TAR" ]]; then + echo "!!! Cannot find kubernetes-server-linux-amd64.tar.gz" + exit 1 + fi +} + +rackspace-set-vars() { - TEMP_URL=$(swiftly -A ${OS_AUTH_URL} -U ${OS_USERNAME} -K ${OS_PASSWORD} tempurl GET $1/$2) + CLOUDFILES_CONTAINER="kubernetes-releases-${OS_USERNAME}" + CONTAINER_PREFIX=${CONTAINER_PREFIX-devel/} + find-release-tars +} + +# Retrieves a tempurl from cloudfiles to make the release object publicly accessible temporarily. +find-object-url() { + + rackspace-set-vars + + KUBE_TAR=${CLOUDFILES_CONTAINER}/${CONTAINER_PREFIX}/kubernetes-server-linux-amd64.tar.gz + + RELEASE_TMP_URL=$(swiftly -A ${OS_AUTH_URL} -U ${OS_USERNAME} -K ${OS_PASSWORD} tempurl GET ${KUBE_TAR}) echo "cluster/rackspace/util.sh: Object temp URL:" - echo -e "\t${TEMP_URL}" + echo -e "\t${RELEASE_TMP_URL}" + +} + +ensure_dev_container() { + SWIFTLY_CMD="swiftly -A ${OS_AUTH_URL} -U ${OS_USERNAME} -K ${OS_PASSWORD}" + + if ! ${SWIFTLY_CMD} get ${CLOUDFILES_CONTAINER} > /dev/null 2>&1 ; then + echo "cluster/rackspace/util.sh: Container doesn't exist. Creating container ${KUBE_RACKSPACE_RELEASE_BUCKET}" + ${SWIFTLY_CMD} put ${CLOUDFILES_CONTAINER} > /dev/null 2>&1 + fi +} + +# Copy kubernetes-server-linux-amd64.tar.gz to cloud files object store +copy_dev_tarballs() { + + echo "cluster/rackspace/util.sh: Uploading to Cloud Files" + ${SWIFTLY_CMD} put -i ${RELEASE_DIR}/kubernetes-server-linux-amd64.tar.gz \ + ${CLOUDFILES_CONTAINER}/${CONTAINER_PREFIX}/kubernetes-server-linux-amd64.tar.gz > /dev/null 2>&1 + + echo "Release pushed." } rax-boot-master() { - ( - echo "#! /bin/bash" - echo "OBJECT_URL=\"${TEMP_URL}\"" - echo "MASTER_HTPASSWD=${HTPASSWD}" - grep -v "^#" $(dirname $0)/templates/download-release.sh - ) > ${KUBE_TEMP}/masterStart.sh + DISCOVERY_URL=$(curl https://discovery.etcd.io/new) + DISCOVERY_ID=$(echo "${DISCOVERY_URL}" | cut -f 4 -d /) + echo "cluster/rackspace/util.sh: etcd discovery URL: ${DISCOVERY_URL}" # Copy cloud-config to KUBE_TEMP and work some sed magic - sed -e "s/KUBE_MASTER/$MASTER_NAME/g" \ - -e "s/MASTER_HTPASSWD/$HTPASSWD/" \ - $(dirname $0)/cloud-config/master-cloud-config.yaml > $KUBE_TEMP/master-cloud-config.yaml + sed -e "s|DISCOVERY_ID|${DISCOVERY_ID}|" \ + -e "s|CLOUD_FILES_URL|${RELEASE_TMP_URL//&/\&}|" \ + -e "s|KUBE_USER|${KUBE_USER}|" \ + -e "s|KUBE_PASSWORD|${KUBE_PASSWORD}|" \ + -e "s|PORTAL_NET|${PORTAL_NET}|" \ + $(dirname $0)/rackspace/cloud-config/master-cloud-config.yaml > $KUBE_TEMP/master-cloud-config.yaml MASTER_BOOT_CMD="nova boot \ @@ -81,9 +163,9 @@ rax-boot-master() { --flavor ${KUBE_MASTER_FLAVOR} \ --image ${KUBE_IMAGE} \ --meta ${MASTER_TAG} \ +--meta ETCD=${DISCOVERY_ID} \ --user-data ${KUBE_TEMP}/master-cloud-config.yaml \ --config-drive true \ ---file /root/masterStart.sh=${KUBE_TEMP}/masterStart.sh \ --nic net-id=${NETWORK_UUID} \ ${MASTER_NAME}" @@ -94,28 +176,25 @@ ${MASTER_NAME}" rax-boot-minions() { - cp $(dirname $0)/cloud-config/minion-cloud-config.yaml \ + cp $(dirname $0)/rackspace/cloud-config/minion-cloud-config.yaml \ ${KUBE_TEMP}/minion-cloud-config.yaml for (( i=0; i<${#MINION_NAMES[@]}; i++)); do - ( - echo "#! /bin/bash" - echo "MASTER_NAME=${MASTER_IP}" - echo "MINION_IP_RANGE=${KUBE_NETWORK[$i]}" - echo "NUM_MINIONS=${RAX_NUM_MINIONS}" - grep -v "^#" $(dirname $0)/templates/salt-minion.sh - ) > ${KUBE_TEMP}/minionStart${i}.sh + sed -e "s|DISCOVERY_ID|${DISCOVERY_ID}|" \ + -e "s|INDEX|$((i + 1))|g" \ + -e "s|CLOUD_FILES_URL|${RELEASE_TMP_URL//&/\&}|" \ + $(dirname $0)/rackspace/cloud-config/minion-cloud-config.yaml > $KUBE_TEMP/minion-cloud-config-$(($i + 1)).yaml + MINION_BOOT_CMD="nova boot \ --key-name ${SSH_KEY_NAME} \ --flavor ${KUBE_MINION_FLAVOR} \ --image ${KUBE_IMAGE} \ --meta ${MINION_TAG} \ ---user-data ${KUBE_TEMP}/minion-cloud-config.yaml \ +--user-data ${KUBE_TEMP}/minion-cloud-config-$(( i +1 )).yaml \ --config-drive true \ --nic net-id=${NETWORK_UUID} \ ---file=/root/minionStart.sh=${KUBE_TEMP}/minionStart${i}.sh \ ${MINION_NAMES[$i]}" echo "cluster/rackspace/util.sh: Booting ${MINION_NAMES[$i]} with following command:" @@ -169,21 +248,22 @@ detect-master-nova-net() { kube-up() { SCRIPT_DIR=$(CDPATH="" cd $(dirname $0); pwd) - source $(dirname $0)/../gce/util.sh - source $(dirname $0)/util.sh - source $(dirname $0)/../../release/rackspace/config.sh + + rackspace-set-vars + ensure_dev_container + copy_dev_tarballs # Find the release to use. Generally it will be passed when doing a 'prod' # install and will default to the release/config.sh version when doing a # developer up. - find-object-url $CONTAINER output/release/$TAR_FILE + find-object-url # Create a temp directory to hold scripts that will be uploaded to master/minions KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX) trap "rm -rf ${KUBE_TEMP}" EXIT get-password - python $(dirname $0)/../../third_party/htpasswd/htpasswd.py -b -c ${KUBE_TEMP}/htpasswd $user $passwd + python $(dirname $0)/../third_party/htpasswd/htpasswd.py -b -c ${KUBE_TEMP}/htpasswd $KUBE_USER $KUBE_PASSWORD HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd) rax-nova-network @@ -195,11 +275,6 @@ kube-up() { echo "cluster/rackspace/util.sh: Starting Cloud Servers" rax-boot-master - # a bit of a hack to wait until master is has an IP from the extra network - echo "cluster/rackspace/util.sh: sleeping 35 seconds" - sleep 35 - - detect-master-nova-net $NOVA_NETWORK_LABEL rax-boot-minions FAIL=0 @@ -222,20 +297,16 @@ kube-up() { echo #This will fail until apiserver salt is updated - until $(curl --insecure --user ${user}:${passwd} --max-time 5 \ + until $(curl --insecure --user ${KUBE_USER}:${KUBE_PASSWORD} --max-time 5 \ --fail --output /dev/null --silent https://${KUBE_MASTER_IP}/api/v1beta1/pods); do printf "." sleep 2 done echo "Kubernetes cluster created." - echo "Sanity checking cluster..." - - sleep 5 # Don't bail on errors, we want to be able to print some info. set +e - sleep 45 detect-minions diff --git a/cluster/saltbase/reactor/highstate-masters.sls b/cluster/saltbase/reactor/highstate-masters.sls new file mode 100644 index 0000000000000..246f7ebce9fda --- /dev/null +++ b/cluster/saltbase/reactor/highstate-masters.sls @@ -0,0 +1,10 @@ +# This runs highstate on the master node(s). +# +# Some of the cluster deployment scripts pass the list of minion addresses to +# the apiserver as a command line argument. This list needs to be updated if a +# new minion is started, so run highstate on the master(s) when this happens. +# +highstate_master: + cmd.state.highstate: + - tgt: 'roles:kubernetes-master' + - expr_form: grain diff --git a/cluster/saltbase/reactor/highstate-minions.sls b/cluster/saltbase/reactor/highstate-minions.sls new file mode 100644 index 0000000000000..7bd00d028570c --- /dev/null +++ b/cluster/saltbase/reactor/highstate-minions.sls @@ -0,0 +1,11 @@ +# This runs highstate on the minion nodes. +# +# Some of the cluster deployment scripts use the list of minions on the minions +# themselves (for example: every minion is configured with static routes to +# every other minion on a vSphere deployment). To propagate changes throughout +# the pool, run highstate on all minions whenever a single minion starts. +# +highstate_minions: + cmd.state.highstate: + - tgt: 'roles:kubernetes-pool' + - expr_form: grain diff --git a/cluster/saltbase/reactor/highstate-new.sls b/cluster/saltbase/reactor/highstate-new.sls new file mode 100644 index 0000000000000..825749dc27531 --- /dev/null +++ b/cluster/saltbase/reactor/highstate-new.sls @@ -0,0 +1,4 @@ +# This runs highstate only on the NEW node, regardless of type. +highstate_new: + cmd.state.highstate: + - tgt: {{ data['id'] }} diff --git a/cluster/saltbase/reactor/start.sls b/cluster/saltbase/reactor/start.sls deleted file mode 100644 index ea1c906e977ec..0000000000000 --- a/cluster/saltbase/reactor/start.sls +++ /dev/null @@ -1,5 +0,0 @@ - -# This runs highstate on the target node -highstate_run: - cmd.state.highstate: - - tgt: {{ data['id'] }} diff --git a/cluster/saltbase/salt/apiserver/default b/cluster/saltbase/salt/apiserver/default index d84085fe87803..e3264dc5db574 100644 --- a/cluster/saltbase/salt/apiserver/default +++ b/cluster/saltbase/salt/apiserver/default @@ -45,15 +45,6 @@ {% set machines = "-machines=$(echo ${MACHINE_IPS[@]} | xargs -n1 echo | paste -sd,)" %} {% set minion_regexp = "" %} {% endif %} -{%- if grains.cloud == 'rackspace' %} - {%- set ip_addrs = [] %} - {%- for addrs in salt['mine.get']('roles:kubernetes-pool', 'grains.items', expr_form='grain').values() %} - {%- do ip_addrs.append(addrs.ip_interfaces.eth2[0]) %} - {%- endfor %} - MACHINES="{{ip_addrs|join(',')}}" - {%- set machines = "-machines=$MACHINES" %} - {%- set minion_regexp = "" %} -{% endif %} {% endif %} DAEMON_ARGS="{{daemon_args}} {{address}} {{machines}} {{etcd_servers}} {{ minion_regexp }} {{ cloud_provider }} --allow_privileged={{pillar['allow_privileged']}}" diff --git a/cluster/saltbase/salt/base.sls b/cluster/saltbase/salt/base.sls index e0418f7acc57b..cad04ee75f068 100644 --- a/cluster/saltbase/salt/base.sls +++ b/cluster/saltbase/salt/base.sls @@ -1,6 +1,7 @@ pkg-core: pkg.installed: - names: + - curl {% if grains['os_family'] == 'RedHat' %} - python - git diff --git a/cluster/saltbase/salt/debian-auto-upgrades/20auto-upgrades b/cluster/saltbase/salt/debian-auto-upgrades/20auto-upgrades new file mode 100644 index 0000000000000..2bb25d7053b56 --- /dev/null +++ b/cluster/saltbase/salt/debian-auto-upgrades/20auto-upgrades @@ -0,0 +1,4 @@ +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Unattended-Upgrade "1"; + +APT::Periodic::AutocleanInterval "7"; diff --git a/cluster/saltbase/salt/debian-auto-upgrades/init.sls b/cluster/saltbase/salt/debian-auto-upgrades/init.sls new file mode 100644 index 0000000000000..79e28a6820f61 --- /dev/null +++ b/cluster/saltbase/salt/debian-auto-upgrades/init.sls @@ -0,0 +1,13 @@ +{% if grains['os_family'] == 'Debian' %} +unattended-upgrades: + pkg.installed + +'/etc/apt/apt.conf.d/20auto-upgrades': + file.managed: + - source: salt://debian-auto-upgrades/20auto-upgrades + - user: root + - group: root + - mode: 644 + - require: + - pkg: unattended-upgrades +{% endif %} diff --git a/cluster/saltbase/salt/docker/docker-defaults b/cluster/saltbase/salt/docker/docker-defaults index 98a4f5b0d74b6..1317593c27262 100644 --- a/cluster/saltbase/salt/docker/docker-defaults +++ b/cluster/saltbase/salt/docker/docker-defaults @@ -1 +1 @@ -DOCKER_OPTS="--bridge cbr0 --iptables=false -r=false" +DOCKER_OPTS="--bridge cbr0 --iptables=false --ip-masq=false -r=false" diff --git a/cluster/saltbase/salt/docker/init.sls b/cluster/saltbase/salt/docker/init.sls index 942b3d11cd938..0151a23d9c8eb 100644 --- a/cluster/saltbase/salt/docker/init.sls +++ b/cluster/saltbase/salt/docker/init.sls @@ -7,23 +7,30 @@ bridge-utils: pkg.installed -{% if grains['os_family'] != 'RedHat' %} +{% if grains.os_family == 'RedHat' %} +docker-io: + pkg: + - installed -docker-repo: - pkgrepo.managed: - - humanname: Docker Repo - - name: deb https://get.docker.com/ubuntu docker main - - key_url: https://get.docker.com/gpg +docker: + service.running: + - enable: True - require: - - pkg: pkg-core + - pkg: docker-io +{% else %} + +{% if grains.cloud is defined + and grains.cloud == 'gce' %} # The default GCE images have ip_forwarding explicitly set to 0. # Here we take care of commenting that out. /etc/sysctl.d/11-gce-network-security.conf: file.replace: - pattern: '^net.ipv4.ip_forward=0' - repl: '# net.ipv4.ip_forward=0' +{% endif %} +# TODO: This should really be based on network strategy instead of os_family net.ipv4.ip_forward: sysctl.present: - value: 1 @@ -33,41 +40,72 @@ cbr0: - cidr: {{ grains['cbr-cidr'] }} - mtu: 1460 -{% endif %} +purge-old-docker: + pkg.removed: + - pkgs: + - lxc-docker-1.2.0 -{% if grains['os_family'] == 'RedHat' %} +{{ environment_file }}: + file.managed: + - source: salt://docker/docker-defaults + - template: jinja + - user: root + - group: root + - mode: 644 + - makedirs: true -docker-io: - pkg: - - installed +# We are caching the Docker deb file in GCS for reliability and speed. To +# update this to a new version of docker, do the following: +# 1. Find new deb name with: +# curl https://get.docker.com/ubuntu/dists/docker/main/binary-amd64/Packages +# 2. Download based on that: +# curl -O https://get.docker.com/ubuntu/pool/main/<...> +# 3. Upload to GCS: +# gsutil cp gs://kubernetes-release/docker/ +# 4. Make it world readable: +# gsutil acl ch -R -g all:R gs://kubernetes-release/docker/ +# 5. Get a hash of the deb: +# shasum +# 6. Update this file with new deb name, new hash and new version +# 7. Add the old version to purge-old-docker above. -docker: - service.running: - - enable: True - - require: - - pkg: docker-io +{% set storage_base='https://storage.googleapis.com/kubernetes-release/docker/' %} +{% set deb='lxc-docker-1.3.0_1.3.0-20141016165047-c78088f_amd64.deb' %} +{% set deb_hash='sha1=99c2135e4f1f469b771226c3846e0b6accb6056a' %} +{% set docker_ver='1.3.0' %} -{% else %} +/var/cache/docker-install/{{ deb }}: + file.managed: + - source: {{ storage_base }}{{ deb }} + - source_hash: {{ deb_hash }} + - user: root + - group: root + - mode: 644 + - makedirs: true -{{ environment_file }}: +# Drop the license file into /usr/share so that everyting is crystal clear. +/usr/share/doc/docker/apache.txt: file.managed: - - source: salt://docker/docker-defaults - - template: jinja + - source: {{ storage_base }}apache2.txt + - source_hash: sha1=2b8b815229aa8a61e483fb4ba0588b8b6c491890 - user: root - group: root - mode: 644 - makedirs: true -lxc-docker: - pkg.installed +lxc-docker-{{ docker_ver }}: + pkg.installed: + - sources: + - lxc-docker-{{ docker_ver }}: /var/cache/docker-install/{{ deb }} docker: service.running: - enable: True - require: - - pkg: lxc-docker + - pkg: lxc-docker-{{ docker_ver }} - watch: - file: {{ environment_file }} - container_bridge: cbr0 + - pkg: lxc-docker-{{ docker_ver }} {% endif %} diff --git a/cluster/saltbase/salt/etcd/init.sls b/cluster/saltbase/salt/etcd/init.sls index d7359ac2f3486..cd09961a84204 100644 --- a/cluster/saltbase/salt/etcd/init.sls +++ b/cluster/saltbase/salt/etcd/init.sls @@ -1,7 +1,19 @@ +# We are caching the etcd tar file in GCS for reliability and speed. To +# update this to a new version, do the following: +# 2. Download tar file: +# curl -LO https://github.com/coreos/etcd/releases/download//etcd--linux-amd64.tar.gz +# 3. Upload to GCS (the cache control makes : +# gsutil cp gs://kubernetes-release/etcd/ +# 4. Make it world readable: +# gsutil -m acl ch -R -g all:R gs://kubernetes-release/etcd/ +# 5. Get a hash of the tar: +# shasum +# 6. Update this file with new tar version and new hash + {% set etcd_version="v0.4.6" %} -{% set etcd_tar_url="https://github.com/coreos/etcd/releases/download/%s/etcd-%s-linux-amd64.tar.gz" - | format(etcd_version, etcd_version) %} -{% set etcd_tar_hash="md5=2949e9163e59dc4f8db9ad92f3245b20" %} +{% set etcd_tar_url="https://storage.googleapis.com/kubernetes-release/etcd/etcd-%s-linux-amd64.tar.gz" + | format(etcd_version) %} +{% set etcd_tar_hash="sha1=5db514e30b9f340eda00671230d5136855ae14d7" %} etcd-tar: archive: @@ -12,7 +24,7 @@ etcd-tar: - source_hash: {{ etcd_tar_hash }} - archive_format: tar - if_missing: /usr/local/src/etcd-{{ etcd_version }}-linux-amd64 - - tar_options: z + - tar_options: xz file.directory: - name: /usr/local/src/etcd-{{ etcd_version }}-linux-amd64 - user: root diff --git a/cluster/saltbase/salt/nginx/init.sls b/cluster/saltbase/salt/nginx/init.sls index 08879167538c0..edbad35960a8f 100644 --- a/cluster/saltbase/salt/nginx/init.sls +++ b/cluster/saltbase/salt/nginx/init.sls @@ -17,6 +17,9 @@ nginx: {% if grains.cloud == 'vagrant' %} {% set cert_ip=grains.fqdn_ip4 %} {% endif %} + {% if grains.cloud == 'vsphere' %} + {% set cert_ip=grains.ip_interfaces.eth0[0] %} + {% endif %} {% endif %} # If there is a pillar defined, override any defaults. {% if pillar['cert_ip'] is defined %} @@ -34,6 +37,8 @@ nginx: - source: salt://nginx/{{certgen}} {% if cert_ip is defined %} - args: {{cert_ip}} + - require: + - pkg: curl {% endif %} - cwd: / - user: root diff --git a/cluster/saltbase/salt/sdn/init.sls b/cluster/saltbase/salt/sdn/init.sls index 2d69a53e5d712..0f0205335109a 100644 --- a/cluster/saltbase/salt/sdn/init.sls +++ b/cluster/saltbase/salt/sdn/init.sls @@ -8,7 +8,7 @@ openvswitch: sdn: cmd.wait: - - name: /vagrant/network_closure.sh + - name: /kubernetes-vagrant/network_closure.sh - watch: - pkg: docker-io - pkg: openvswitch diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index 69b383b192f17..d996bf4e140eb 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -1,6 +1,7 @@ base: '*': - base + - debian-auto-upgrades 'roles:kubernetes-pool': - match: grain diff --git a/cluster/vagrant/provision-master.sh b/cluster/vagrant/provision-master.sh index fc093c6fae418..e917f1750657e 100755 --- a/cluster/vagrant/provision-master.sh +++ b/cluster/vagrant/provision-master.sh @@ -87,7 +87,7 @@ cat </etc/salt/master.d/reactor.conf # React to new minions starting by running highstate on them. reactor: - 'salt/minion/*/start': - - /srv/reactor/start.sls + - /srv/reactor/highstate-new.sls EOF cat </etc/salt/master.d/salt-output.conf diff --git a/cluster/vagrant/provision-network.sh b/cluster/vagrant/provision-network.sh index 00cbb6c3b245b..0994b27e98736 100755 --- a/cluster/vagrant/provision-network.sh +++ b/cluster/vagrant/provision-network.sh @@ -29,7 +29,11 @@ BRIDGE_ADDRESS=${BRIDGE_BASE}.${MINION_ID}.1 BRIDGE_NETWORK=${BRIDGE_ADDRESS}/24 BRIDGE_NETMASK=255.255.255.0 NETWORK_CONF_PATH=/etc/sysconfig/network-scripts/ -POST_NETWORK_SCRIPT=/vagrant/network_closure.sh +POST_NETWORK_SCRIPT_DIR=/kubernetes-vagrant +POST_NETWORK_SCRIPT=${POST_NETWORK_SCRIPT_DIR}/network_closure.sh + +# ensure location of POST_NETWORK_SCRIPT exists +mkdir -p $POST_NETWORK_SCRIPT_DIR # add docker bridge ifcfg file cat < ${NETWORK_CONF_PATH}ifcfg-${DOCKER_BRIDGE} @@ -106,8 +110,6 @@ iptables -t nat -A POSTROUTING -s ${BRIDGE_BASE}.0.0/16 ! -d ${BRIDGE_BASE}.0.0/ # persist please iptables-save >& /etc/sysconfig/iptables -# self-destruct after doing the job -#rm -f ${POST_NETWORK_SCRIPT} EOF chmod +x ${POST_NETWORK_SCRIPT} diff --git a/cluster/validate-cluster.sh b/cluster/validate-cluster.sh index 20c6f05d7f456..d96cee3cf4164 100755 --- a/cluster/validate-cluster.sh +++ b/cluster/validate-cluster.sh @@ -36,7 +36,7 @@ MINIONS_FILE=/tmp/minions "${KUBE_ROOT}/cluster/kubecfg.sh" -template '{{range.Items}}{{.ID}}:{{end}}' list minions > ${MINIONS_FILE} # On vSphere, use minion IPs as their names -if [ "$KUBERNETES_PROVIDER" == "vsphere" ]; then +if [[ "${KUBERNETES_PROVIDER}" == "vsphere" ]]; then for (( i=0; i<${#MINION_NAMES[@]}; i++)); do MINION_NAMES[i]=${KUBE_MINION_IP_ADDRESSES[i]} done diff --git a/icebox/cluster/vsphere/config-common.sh b/cluster/vsphere/config-common.sh similarity index 76% rename from icebox/cluster/vsphere/config-common.sh rename to cluster/vsphere/config-common.sh index 0c918486a46c3..6adadef80a69f 100644 --- a/icebox/cluster/vsphere/config-common.sh +++ b/cluster/vsphere/config-common.sh @@ -14,24 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -function public-key { - local dir=${HOME}/.ssh - - for f in $HOME/.ssh/{id_{rsa,dsa},*}.pub; do - if [ -r $f ]; then - echo $f - return - fi - done - - echo "Can't find public key file..." 1>&2 - exit 1 -} - -DISK=./kube/kube.vmdk -GUEST_ID=debian7_64Guest -PUBLIC_KEY_FILE=${PUBLIC_KEY_FILE-$(public-key)} -SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null" +SSH_OPTS="-oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oLogLevel=ERROR" # These need to be set #export GOVC_URL= diff --git a/icebox/cluster/vsphere/config-default.sh b/cluster/vsphere/config-default.sh similarity index 85% rename from icebox/cluster/vsphere/config-default.sh rename to cluster/vsphere/config-default.sh index fb5e6e321c2b1..e8d9a637c9e70 100755 --- a/icebox/cluster/vsphere/config-default.sh +++ b/cluster/vsphere/config-default.sh @@ -14,10 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -source $(dirname ${BASH_SOURCE})/config-common.sh - NUM_MINIONS=4 +DISK=./kube/kube.vmdk +GUEST_ID=debian7_64Guest + INSTANCE_PREFIX=kubernetes +MASTER_TAG="${INSTANCE_PREFIX}-master" +MINION_TAG="${INSTANCE_PREFIX}-minion" MASTER_NAME="${INSTANCE_PREFIX}-master" MASTER_MEMORY_MB=1024 @@ -27,3 +30,5 @@ MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24")) MINION_MEMORY_MB=2048 MINION_CPU=1 + +PORTAL_NET="10.244.240.0/20" diff --git a/icebox/cluster/vsphere/config-test.sh b/cluster/vsphere/config-test.sh similarity index 85% rename from icebox/cluster/vsphere/config-test.sh rename to cluster/vsphere/config-test.sh index 3d36bc7cde23b..0c2b2f1608e8d 100755 --- a/icebox/cluster/vsphere/config-test.sh +++ b/cluster/vsphere/config-test.sh @@ -14,10 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -source $(dirname ${BASH_SOURCE})/config-common.sh - NUM_MINIONS=2 +DISK=./kube/kube.vmdk +GUEST_ID=debian7_64Guest + INSTANCE_PREFIX="e2e-test-${USER}" +MASTER_TAG="${INSTANCE_PREFIX}-master" +MINION_TAG="${INSTANCE_PREFIX}-minion" MASTER_NAME="${INSTANCE_PREFIX}-master" MASTER_MEMORY_MB=1024 @@ -27,3 +30,5 @@ MINION_NAMES=($(eval echo ${INSTANCE_PREFIX}-minion-{1..${NUM_MINIONS}})) MINION_IP_RANGES=($(eval echo "10.244.{1..${NUM_MINIONS}}.0/24")) MINION_MEMORY_MB=1024 MINION_CPU=1 + +PORTAL_NET="10.244.240.0/20" diff --git a/icebox/release/rackspace/config.sh b/cluster/vsphere/templates/create-dynamic-salt-files.sh old mode 100644 new mode 100755 similarity index 57% rename from icebox/release/rackspace/config.sh rename to cluster/vsphere/templates/create-dynamic-salt-files.sh index 8faf2fb11ffb5..afea82b88c874 --- a/icebox/release/rackspace/config.sh +++ b/cluster/vsphere/templates/create-dynamic-salt-files.sh @@ -1,3 +1,5 @@ +#!/bin/bash + # Copyright 2014 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,15 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -# A set of Cloud Files defaults for which Kubernetes releases will be uploaded to - -# Make sure swiftly is installed and available -if [ "$(which swiftly)" == "" ]; then - echo "release/rackspace/config.sh: Couldn't find swiftly in PATH. Please install swiftly:" - echo -e "\tpip install swiftly" - exit 1 -fi +# Create the overlay files for the salt tree. We create these in a separate +# place so that we can blow away the rest of the salt configs on a kube-push and +# re-apply these. -CONTAINER="kubernetes-releases-${OS_USERNAME}" +mkdir -p /srv/salt-overlay/pillar +cat </srv/salt-overlay/pillar/cluster-params.sls +node_instance_prefix: $NODE_INSTANCE_PREFIX +portal_net: $PORTAL_NET +EOF -TAR_FILE=master-release.tgz +mkdir -p /srv/salt-overlay/salt/nginx +echo $MASTER_HTPASSWD > /srv/salt-overlay/salt/nginx/htpasswd diff --git a/icebox/cluster/vsphere/templates/hostname.sh b/cluster/vsphere/templates/hostname.sh similarity index 100% rename from icebox/cluster/vsphere/templates/hostname.sh rename to cluster/vsphere/templates/hostname.sh diff --git a/icebox/cluster/vsphere/templates/install-release.sh b/cluster/vsphere/templates/install-release.sh similarity index 68% rename from icebox/cluster/vsphere/templates/install-release.sh rename to cluster/vsphere/templates/install-release.sh index 877f150f13a3b..5984ec1ff54eb 100755 --- a/icebox/cluster/vsphere/templates/install-release.sh +++ b/cluster/vsphere/templates/install-release.sh @@ -14,11 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Install release +# This script assumes that the environment variable SERVER_BINARY_TAR contains +# the release tar to download and unpack. It is meant to be pushed to the +# master and run. -echo "Unpacking release" -rm -rf master-release || false -tar xzf master-release.tgz +echo "Unpacking Salt tree" +rm -rf kubernetes +tar xzf "${SALT_TAR}" echo "Running release install script" -sudo master-release/src/scripts/master-release-install.sh +sudo kubernetes/saltbase/install.sh "${SERVER_BINARY_TAR}" diff --git a/icebox/cluster/vsphere/templates/salt-master.sh b/cluster/vsphere/templates/salt-master.sh similarity index 83% rename from icebox/cluster/vsphere/templates/salt-master.sh rename to cluster/vsphere/templates/salt-master.sh index 92b526fc3f2c1..df675b246c41b 100755 --- a/icebox/cluster/vsphere/templates/salt-master.sh +++ b/cluster/vsphere/templates/salt-master.sh @@ -38,23 +38,19 @@ cat </etc/salt/master.d/reactor.conf # React to new minions starting by running highstate on them. reactor: - 'salt/minion/*/start': - - /srv/reactor/start.sls + - /srv/reactor/highstate-new.sls + - /srv/reactor/highstate-masters.sls + - /srv/reactor/highstate-minions.sls EOF -mkdir -p /srv/salt/nginx -echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd - # Install Salt # # We specify -X to avoid a race condition that can cause minion failure to # install. See https://github.com/saltstack/salt-bootstrap/issues/270 # # -M installs the master -if [ ! -x /etc/init.d/salt-master ]; then - wget -q -O - https://bootstrap.saltstack.com | sh -s -- -M -X -else - /etc/init.d/salt-master restart - /etc/init.d/salt-minion restart -fi +set +x +wget -q -O - https://bootstrap.saltstack.com | sh -s -- -M -X +set -x echo $MASTER_HTPASSWD > /srv/salt/nginx/htpasswd diff --git a/icebox/cluster/vsphere/templates/salt-minion.sh b/cluster/vsphere/templates/salt-minion.sh similarity index 81% rename from icebox/cluster/vsphere/templates/salt-minion.sh rename to cluster/vsphere/templates/salt-minion.sh index 75a6aab2631c4..40983c6bfa2f6 100755 --- a/icebox/cluster/vsphere/templates/salt-minion.sh +++ b/cluster/vsphere/templates/salt-minion.sh @@ -18,14 +18,14 @@ sed -i -e "s/http.us.debian.org/mirrors.kernel.org/" /etc/apt/sources.list # Resolve hostname of master -if ! grep -q $MASTER_NAME /etc/hosts; then - echo "Adding host entry for $MASTER_NAME" - echo "$MASTER_IP $MASTER_NAME" >> /etc/hosts +if ! grep -q $KUBE_MASTER /etc/hosts; then + echo "Adding host entry for $KUBE_MASTER" + echo "$KUBE_MASTER_IP $KUBE_MASTER" >> /etc/hosts fi # Prepopulate the name of the Master mkdir -p /etc/salt/minion.d -echo "master: $MASTER_NAME" > /etc/salt/minion.d/master.conf +echo "master: $KUBE_MASTER" > /etc/salt/minion.d/master.conf # Turn on debugging for salt-minion # echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion @@ -48,8 +48,4 @@ EOF # # We specify -X to avoid a race condition that can cause minion failure to # install. See https://github.com/saltstack/salt-bootstrap/issues/270 -if [ ! -x /etc/init.d/salt-minion ]; then - wget -q -O - https://bootstrap.saltstack.com | sh -s -- -X -else - /etc/init.d/salt-minion restart -fi +wget -q -O - https://bootstrap.saltstack.com | sh -s -- -X diff --git a/cluster/vsphere/util.sh b/cluster/vsphere/util.sh new file mode 100755 index 0000000000000..7a85af1d4aa89 --- /dev/null +++ b/cluster/vsphere/util.sh @@ -0,0 +1,473 @@ +#!/bin/bash + +# Copyright 2014 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A library of helper functions and constants for the local config. + +# Use the config file specified in $KUBE_CONFIG_FILE, or default to +# config-default.sh. +KUBE_ROOT=$(dirname "${BASH_SOURCE}")/../.. +source "${KUBE_ROOT}/cluster/vsphere/config-common.sh" +source "${KUBE_ROOT}/cluster/vsphere/${KUBE_CONFIG_FILE-"config-default.sh"}" + +# Detect the IP for the master +# +# Assumed vars: +# MASTER_NAME +# Vars set: +# KUBE_MASTER +# KUBE_MASTER_IP +function detect-master { + KUBE_MASTER=${MASTER_NAME} + if [[ -z "${KUBE_MASTER_IP-}" ]]; then + KUBE_MASTER_IP=$(govc vm.ip ${MASTER_NAME}) + fi + if [[ -z "${KUBE_MASTER_IP-}" ]]; then + echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'" >&2 + exit 1 + fi + echo "Using master: $KUBE_MASTER (external IP: $KUBE_MASTER_IP)" +} + +# Detect the information about the minions +# +# Assumed vars: +# MINION_NAMES +# Vars set: +# KUBE_MINION_IP_ADDRESS (array) +function detect-minions { + KUBE_MINION_IP_ADDRESSES=() + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + local minion_ip=$(govc vm.ip ${MINION_NAMES[$i]}) + if [[ -z "${minion_ip-}" ]] ; then + echo "Did not find ${MINION_NAMES[$i]}" >&2 + else + echo "Found ${MINION_NAMES[$i]} at ${minion_ip}" + KUBE_MINION_IP_ADDRESSES+=("${minion_ip}") + fi + done + if [[ -z "${KUBE_MINION_IP_ADDRESSES-}" ]]; then + echo "Could not detect Kubernetes minion nodes. Make sure you've launched a cluster with 'kube-up.sh'" >&2 + exit 1 + fi +} + +function trap-add { + local handler="$1" + local signal="${2-EXIT}" + local cur + + cur="$(eval "sh -c 'echo \$3' -- $(trap -p ${signal})")" + if [[ -n "${cur}" ]]; then + handler="${cur}; ${handler}" + fi + + trap "${handler}" ${signal} +} + +function verify-prereqs { + which "govc" >/dev/null || { + echo "Can't find govc in PATH, please install and retry." + echo "" + echo " go install github.com/vmware/govmomi/govc" + echo "" + exit 1 + } +} + +function verify-ssh-prereqs { + local rc + + rc=0 + ssh-add -L 1> /dev/null 2> /dev/null || rc="$?" + # "Could not open a connection to your authentication agent." + if [[ "${rc}" -eq 2 ]]; then + eval "$(ssh-agent)" > /dev/null + trap-add "kill ${SSH_AGENT_PID}" EXIT + fi + + rc=0 + ssh-add -L 1> /dev/null 2> /dev/null || rc="$?" + # "The agent has no identities." + if [[ "${rc}" -eq 1 ]]; then + # Try adding one of the default identities, with or without passphrase. + ssh-add || true + fi + + # Expect at least one identity to be available. + if ! ssh-add -L 1> /dev/null 2> /dev/null; then + echo "Could not find or add an SSH identity." + echo "Please start ssh-agent, add your identity, and retry." + exit 1 + fi +} + +# Create a temp dir that'll be deleted at the end of this bash session. +# +# Vars set: +# KUBE_TEMP +function ensure-temp-dir { + if [[ -z ${KUBE_TEMP-} ]]; then + KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX) + trap-add 'rm -rf "${KUBE_TEMP}"' EXIT + fi +} + +# Verify and find the various tar files that we are going to use on the server. +# +# Vars set: +# SERVER_BINARY_TAR +# SALT_TAR +function find-release-tars { + SERVER_BINARY_TAR="${KUBE_ROOT}/server/kubernetes-server-linux-amd64.tar.gz" + if [[ ! -f "$SERVER_BINARY_TAR" ]]; then + SERVER_BINARY_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-server-linux-amd64.tar.gz" + fi + if [[ ! -f "$SERVER_BINARY_TAR" ]]; then + echo "!!! Cannot find kubernetes-server-linux-amd64.tar.gz" + exit 1 + fi + + SALT_TAR="${KUBE_ROOT}/server/kubernetes-salt.tar.gz" + if [[ ! -f "$SALT_TAR" ]]; then + SALT_TAR="${KUBE_ROOT}/_output/release-tars/kubernetes-salt.tar.gz" + fi + if [[ ! -f "$SALT_TAR" ]]; then + echo "!!! Cannot find kubernetes-salt.tar.gz" + exit 1 + fi +} + +# Take the local tar files and upload them to the master. +# +# Assumed vars: +# MASTER_NAME +# SERVER_BINARY_TAR +# SALT_TAR +function upload-server-tars { + local vm_ip + + vm_ip=$(govc vm.ip "${MASTER_NAME}") + kube-ssh ${vm_ip} "mkdir -p /home/kube/cache/kubernetes-install" + + local tar + for tar in "${SERVER_BINARY_TAR}" "${SALT_TAR}"; do + kube-scp ${vm_ip} "${tar}" "/home/kube/cache/kubernetes-install/${tar##*/}" + done +} + +# Ensure that we have a password created for validating to the master. Will +# read from $HOME/.kubernetes_auth if available. +# +# Vars set: +# KUBE_USER +# KUBE_PASSWORD +function get-password { + local file="$HOME/.kubernetes_auth" + if [[ -r "$file" ]]; then + KUBE_USER=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["User"]') + KUBE_PASSWORD=$(cat "$file" | python -c 'import json,sys;print json.load(sys.stdin)["Password"]') + return + fi + KUBE_USER=admin + KUBE_PASSWORD=$(python -c 'import string,random; print "".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16))') + + # Store password for reuse. + cat << EOF > "$file" +{ + "User": "$KUBE_USER", + "Password": "$KUBE_PASSWORD" +} +EOF + chmod 0600 "$file" +} + +# Run command over ssh +function kube-ssh { + local host="$1" + shift + ssh ${SSH_OPTS-} "kube@${host}" "$@" 2> /dev/null +} + +# Copy file over ssh +function kube-scp { + local host="$1" + local src="$2" + local dst="$3" + scp ${SSH_OPTS-} "${src}" "kube@${host}:${dst}" +} + +# Instantiate a generic kubernetes virtual machine (master or minion) +# +# Usage: +# kube-up-vm VM_NAME [options to pass to govc vm.create] +# +# Example: +# kube-up-vm "vm-name" -c 2 -m 4096 +# +# Assumed vars: +# DISK +# GUEST_ID +function kube-up-vm { + local vm_name="$1" + shift + + govc vm.create \ + -debug \ + -disk="${DISK}" \ + -g="${GUEST_ID}" \ + -link=true \ + "$@" \ + "${vm_name}" + + # Retrieve IP first, to confirm the guest operations agent is running. + govc vm.ip "${vm_name}" > /dev/null + + govc guest.mkdir \ + -vm="${vm_name}" \ + -p \ + /home/kube/.ssh + + ssh-add -L > "${KUBE_TEMP}/${vm_name}-authorized_keys" + + govc guest.upload \ + -vm="${vm_name}" \ + -f \ + "${KUBE_TEMP}/${vm_name}-authorized_keys" \ + /home/kube/.ssh/authorized_keys +} + +# Kick off a local script on a kubernetes virtual machine (master or minion) +# +# Usage: +# kube-run VM_NAME LOCAL_FILE +function kube-run { + local vm_name="$1" + local file="$2" + local dst="/tmp/$(basename "${file}")" + govc guest.upload -vm="${vm_name}" -f -perm=0755 "${file}" "${dst}" + + local vm_ip + vm_ip=$(govc vm.ip "${vm_name}") + kube-ssh ${vm_ip} "nohup sudo ${dst} < /dev/null 1> ${dst}.out 2> ${dst}.err &" +} + +# Instantiate a kubernetes cluster +# +# Assumed vars: +# KUBE_ROOT +# +function kube-up { + verify-ssh-prereqs + find-release-tars + + ensure-temp-dir + + get-password + python "${KUBE_ROOT}/third_party/htpasswd/htpasswd.py" \ + -b -c "${KUBE_TEMP}/htpasswd" "$KUBE_USER" "$KUBE_PASSWORD" + local htpasswd + htpasswd=$(cat "${KUBE_TEMP}/htpasswd") + + echo "Starting master VM (this can take a minute)..." + + ( + echo "#! /bin/bash" + echo "readonly MY_NAME=${MASTER_NAME}" + grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/hostname.sh" + echo "cd /home/kube/cache/kubernetes-install" + echo "readonly MASTER_NAME='${MASTER_NAME}'" + echo "readonly NODE_INSTANCE_PREFIX='${INSTANCE_PREFIX}-minion'" + echo "readonly PORTAL_NET='${PORTAL_NET}'" + echo "readonly SERVER_BINARY_TAR='${SERVER_BINARY_TAR##*/}'" + echo "readonly SALT_TAR='${SALT_TAR##*/}'" + echo "readonly MASTER_HTPASSWD='${htpasswd}'" + grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/create-dynamic-salt-files.sh" + grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/install-release.sh" + grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/salt-master.sh" + ) > "${KUBE_TEMP}/master-start.sh" + + kube-up-vm ${MASTER_NAME} -c ${MASTER_CPU-1} -m ${MASTER_MEMORY_MB-1024} + upload-server-tars + kube-run ${MASTER_NAME} "${KUBE_TEMP}/master-start.sh" + + # Print master IP, so user can log in for debugging. + detect-master + echo + + echo "Starting minion VMs (this can take a minute)..." + + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + ( + echo "#! /bin/bash" + echo "readonly MY_NAME=${MINION_NAMES[$i]}" + grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/hostname.sh" + echo "KUBE_MASTER=${KUBE_MASTER}" + echo "KUBE_MASTER_IP=${KUBE_MASTER_IP}" + echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}" + grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/salt-minion.sh" + ) > "${KUBE_TEMP}/minion-start-${i}.sh" + + ( + kube-up-vm "${MINION_NAMES[$i]}" -c ${MINION_CPU-1} -m ${MINION_MEMORY_MB-1024} + kube-run "${MINION_NAMES[$i]}" "${KUBE_TEMP}/minion-start-${i}.sh" + ) & + done + + local fail=0 + local job + for job in $(jobs -p); do + wait "${job}" || fail=$((fail + 1)) + done + if (( $fail != 0 )); then + echo "${fail} commands failed. Exiting." >&2 + exit 2 + fi + + # Print minion IPs, so user can log in for debugging. + detect-minions + echo + + echo "Waiting for master and minion initialization." + echo + echo " This will continually check to see if the API for kubernetes is reachable." + echo " This might loop forever if there was some uncaught error during start up." + echo + + printf "Waiting for ${KUBE_MASTER} to become available..." + until curl --insecure --user "${KUBE_USER}:${KUBE_PASSWORD}" --max-time 5 \ + --fail --output /dev/null --silent "https://${KUBE_MASTER_IP}/api/v1beta1/pods"; do + printf "." + sleep 2 + done + printf " OK\n" + + local i + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + printf "Waiting for ${MINION_NAMES[$i]} to become available..." + until curl --max-time 5 \ + --fail --output /dev/null --silent "http://${KUBE_MINION_IP_ADDRESSES[$i]}:10250/healthz"; do + printf "." + sleep 2 + done + printf " OK\n" + done + + echo + echo "Sanity checking cluster..." + + sleep 5 + + # Basic sanity checking + local i + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + # Make sure docker is installed + kube-ssh "${KUBE_MINION_IP_ADDRESSES[$i]}" which docker > /dev/null || { + echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely" >&2 + echo "to work correctly. Please run ./cluster/kube-down.sh and re-create the" >&2 + echo "cluster. (sorry!)" >&2 + exit 1 + } + done + + echo + echo "Kubernetes cluster is running. The master is running at:" + echo + echo " https://${KUBE_MASTER_IP}" + echo + echo "The user name and password to use is located in ~/.kubernetes_auth." + echo + + local kube_cert=".kubecfg.crt" + local kube_key=".kubecfg.key" + local ca_cert=".kubernetes.ca.crt" + + ( + umask 077 + + kube-ssh "${KUBE_MASTER_IP}" sudo cat /usr/share/nginx/kubecfg.crt >"${HOME}/${kube_cert}" 2>/dev/null + kube-ssh "${KUBE_MASTER_IP}" sudo cat /usr/share/nginx/kubecfg.key >"${HOME}/${kube_key}" 2>/dev/null + kube-ssh "${KUBE_MASTER_IP}" sudo cat /usr/share/nginx/ca.crt >"${HOME}/${ca_cert}" 2>/dev/null + + cat << EOF > ~/.kubernetes_auth + { + "User": "$KUBE_USER", + "Password": "$KUBE_PASSWORD", + "CAFile": "$HOME/$ca_cert", + "CertFile": "$HOME/$kube_cert", + "KeyFile": "$HOME/$kube_key" + } +EOF + + chmod 0600 ~/.kubernetes_auth "${HOME}/${kube_cert}" \ + "${HOME}/${kube_key}" "${HOME}/${ca_cert}" + ) +} + +# Delete a kubernetes cluster +function kube-down { + govc vm.destroy ${MASTER_NAME} & + + for (( i=0; i<${#MINION_NAMES[@]}; i++)); do + govc vm.destroy ${MINION_NAMES[i]} & + done + + wait +} + +# Update a kubernetes cluster with latest source +function kube-push { + verify-ssh-prereqs + find-release-tars + + detect-master + upload-server-tars + + ( + echo "#! /bin/bash" + echo "cd /home/kube/cache/kubernetes-install" + echo "readonly SERVER_BINARY_TAR='${SERVER_BINARY_TAR##*/}'" + echo "readonly SALT_TAR='${SALT_TAR##*/}'" + grep -v "^#" "${KUBE_ROOT}/cluster/vsphere/templates/install-release.sh" + echo "echo Executing configuration" + echo "sudo salt '*' mine.update" + echo "sudo salt --force-color '*' state.highstate" + ) | kube-ssh "${KUBE_MASTER_IP}" + + get-password + + echo + echo "Kubernetes cluster is running. The master is running at:" + echo + echo " https://${KUBE_MASTER_IP}" + echo + echo "The user name and password to use is located in ~/.kubernetes_auth." + echo +} + +# Execute prior to running tests to build a release if required for env +function test-build-release { + echo "TODO" +} + +# Execute prior to running tests to initialize required structure +function test-setup { + echo "TODO" +} + +# Execute after running tests to perform any required clean-up +function test-teardown { + echo "TODO" +} diff --git a/cmd/kubelet/kubelet.go b/cmd/kubelet/kubelet.go index 0b2f1cf4112d0..c32967e45e674 100644 --- a/cmd/kubelet/kubelet.go +++ b/cmd/kubelet/kubelet.go @@ -67,6 +67,8 @@ var ( registryBurst = flag.Int("registry_burst", 10, "Maximum size of a bursty pulls, temporarily allows pulls to burst to this number, while still not exceeding registry_qps. Only used if --registry_qps > 0") runonce = flag.Bool("runonce", false, "If true, exit after spawning pods from local manifests or remote urls. Exclusive with --etcd_servers and --enable-server") enableDebuggingHandlers = flag.Bool("enable_debugging_handlers", true, "Enables server endpoints for log collection and local running of containers and commands") + minimumGCAge = flag.Duration("minimum_container_ttl_duration", 0, "Minimum age for a finished container before it is garbage collected. Examples: '300ms', '10s' or '2h45m'") + maxContainerCount = flag.Int("maximum_dead_containers_per_container", 5, "Maximum number of old instances of a container to retain per container. Each container takes up some disk space. Default: 5.") ) func init() { @@ -183,7 +185,17 @@ func main() { *networkContainerImage, *syncFrequency, float32(*registryPullQPS), - *registryBurst) + *registryBurst, + *minimumGCAge, + *maxContainerCount) + go func() { + util.Forever(func() { + err := k.GarbageCollectContainers() + if err != nil { + glog.Errorf("Garbage collect failed: %v", err) + } + }, time.Minute*1) + }() go func() { defer util.HandleCrash() diff --git a/docs/getting-started-guides/binary_release.md b/docs/getting-started-guides/binary_release.md index c234f805601b6..df17a9eec22c5 100644 --- a/docs/getting-started-guides/binary_release.md +++ b/docs/getting-started-guides/binary_release.md @@ -6,7 +6,7 @@ You can either build a release from sources or download a pre-built release. If Soon, we will have a list of numbered and nightly releases. Until then, you can download a development release/snapshot from [here](http://storage.googleapis.com/kubernetes-releases-56726/devel/kubernetes.tar.gz). -Unpack this tar file on Linux or OS X. Most guides assume you are in the `kubernetes/` directory. +Unpack this tar file on Linux or OS X. Unpack this tar file on Linux or OS X, cd to the created `kubernetes/` directory, and then follow the getting started guide for your cloud. ### Building from source diff --git a/docs/getting-started-guides/rackspace.md b/docs/getting-started-guides/rackspace.md index 7e5de3340de0d..352b46ebaf5e7 100644 --- a/docs/getting-started-guides/rackspace.md +++ b/docs/getting-started-guides/rackspace.md @@ -1,37 +1,43 @@ -# WARNING -These instructions are broken at git HEAD. Please either: -* Sync back to `v0.3` with `git checkout v0.3` -* Download a [snapshot of `v0.3`](https://github.com/GoogleCloudPlatform/kubernetes/archive/v0.3.tar.gz) - # Rackspace -In general, the dev-build-and-up.sh workflow for Rackspace is the similar to GCE. The specific implementation is different mainly due to network differences between the providers: +In general, the dev-build-and-up.sh workflow for Rackspace is the similar to GCE. The specific implementation is different due to the use of CoreOS, Rackspace Cloud Files and network design. + +These scripts should be used to deploy development environments for Kubernetes. If your account leverages RackConnect or non-standard networking, these scripts will most likely not work without modification. + +NOTE: The rackspace scripts do NOT rely on `saltstack`. + +The current cluster design is inspired by: +- [corekube](https://github.com/metral/corekube/) +- [Angus Lees](https://github.com/anguslees/kube-openstack/) ## Prerequisites 1. You need to have both `nova` and `swiftly` installed. It's recommended to use a python virtualenv to install these packages into. 2. Make sure you have the appropriate environment variables set to interact with the OpenStack APIs. See [Rackspace Documentation](http://docs.rackspace.com/servers/api/v2/cs-gettingstarted/content/section_gs_install_nova.html) for more details. -3. You can test this by running `nova list` to make sure you're authenticated successfully. ## Provider: Rackspace - To use Rackspace as the provider, set the KUBERNETES_PROVIDER ENV variable: - `export KUBERNETES_PROVIDER=rackspace` and run the `hack/rackspace/dev-build-and-up.sh` script. + `export KUBERNETES_PROVIDER=rackspace` and run the `bash hack/dev-build-and-up.sh` script. -## Release -1. The kubernetes binaries will be built via the common build scripts in `release/`. There is a specific `release/rackspace` directory with scripts for the following steps: +## Build +1. The kubernetes binaries will be built via the common build scripts in `build/`. +2. If you've set the ENV `KUBERNETES_PROVIDER=rackspace`, the scripts will upload `kubernetes-server-linux-amd64.tar.gz` to Cloud Files. 2. A cloud files container will be created via the `swiftly` CLI and a temp URL will be enabled on the object. -3. The built `master-release.tar.gz` will be uploaded to this container and the URL will be passed to master/minions nodes when booted. -- NOTE: RELEASE tagging and launch scripts are not used currently. +3. The built `kubernetes-server-linux-amd64.tar.gz` will be uploaded to this container and the URL will be passed to master/minions nodes when booted. ## Cluster 1. There is a specific `cluster/rackspace` directory with the scripts for the following steps: 2. A cloud network will be created and all instances will be attached to this network. We will connect the master API and minion kubelet service via this network. 3. A SSH key will be created and uploaded if needed. This key must be used to ssh into the machines since we won't capture the password. -4. A master will be created via the `nova` CLI. A `cloud-config.yaml` is generated and provided as user-data. A basic `masterStart.sh` will be injected as a file and cloud-init will run it. -5. We sleep for 25 seconds since we need to make sure we can get the IP address of the master on the cloud network we've created to provide the minions as their salt master. -6. We then boot as many minions as defined via `$RAX_NUM_MINIONS`. We pass both a `cloud-config.yaml` as well as a `minionStart.sh`. The latter is executed via cloud-init just like on the master. +4. A master and minions will be created via the `nova` CLI. A `cloud-config.yaml` is generated and provided as user-data with the entire configuration for the systems. +5. We then boot as many minions as defined via `$RAX_NUM_MINIONS`. ## Some notes: - The scripts expect `eth2` to be the cloud network that the containers will communicate across. -- `vxlan` is required on the cloud network interface since cloud networks will filter based on MAC address. This is the workaround for the time being. -- A linux image with a recent kernel `> 13.07` is required for `vxlan`. Ubuntu 14.04 works. - A number of the items in `config-default.sh` are overridable via environment variables. -- routes must be configured on each minion so that containers and kube-proxy are able to locate containers on another system. This is due to the network design in kubernetes and the MAC address limits on Cloud Networks. Static Routes are currently leveraged until we implement a more advanced solution. +- For older versions please either: + * Sync back to `v0.3` with `git checkout v0.3` + * Download a [snapshot of `v0.3`](https://github.com/GoogleCloudPlatform/kubernetes/archive/v0.3.tar.gz) + +## Network Design +- eth0 - Public Interface used for servers/containers to reach the internet +- eth1 - ServiceNet - Intra-cluster communication (k8s, etcd, etc) communicate via this interface. The `cloud-config` files use the special CoreOS identifier `$private_ipv4` to configure the services. +- eth2 - Cloud Network - Used for k8s pods to communicate with one another. The proxy service will pass traffic via this interface. diff --git a/docs/getting-started-guides/vsphere.md b/docs/getting-started-guides/vsphere.md index fedeb421beb6a..1a32ecdf9a6a6 100644 --- a/docs/getting-started-guides/vsphere.md +++ b/docs/getting-started-guides/vsphere.md @@ -1,10 +1,10 @@ -# WARNING -These instructions are broken at git HEAD. Please either: -* Sync back to `v0.3` with `git checkout v0.3` -* Download a [snapshot of `v0.3`](https://github.com/GoogleCloudPlatform/kubernetes/archive/v0.3.tar.gz) - ## Getting started with vSphere +The example below creates a Kubernetes cluster with 4 worker node Virtual +Machines and a master Virtual Machine (i.e. 5 VMs in your cluster). This +cluster is set up and controlled from your workstation (or wherever you find +convenient). + ### Prerequisites 1. You need administrator credentials to an ESXi machine or vCenter instance. @@ -23,15 +23,7 @@ These instructions are broken at git HEAD. Please either: go get github.com/vmware/govmomi/govc ``` -5. Install godep (optional, only required when modifying package dependencies). [Instructions here](https://github.com/GoogleCloudPlatform/kubernetes#installing-godep) - -6. Get the Kubernetes source: - - ```sh - mkdir -p $GOPATH/src/github.com/GoogleCloudPlatform - git clone https://github.com/GoogleCloudPlatform/kubernetes.git - cd kubernetes - ``` +5. Get or build a [binary release](binary_release.md) ### Setup @@ -46,7 +38,7 @@ gzip -d kube.vmdk.gz Upload this VMDK to your vSphere instance: ```sh -export GOVC_URL='https://user:pass@hostname/sdk' +export GOVC_URL='user:pass@hostname' export GOVC_INSECURE=1 # If the host above uses a self-signed cert export GOVC_DATASTORE='target datastore' export GOVC_RESOURCE_POOL='resource pool or cluster with access to datastore' @@ -63,18 +55,13 @@ govc datastore.ls ./kube/ Take a look at the file `cluster/vsphere/config-common.sh` fill in the required parameters. The guest login for the image that you imported is `kube:kube`. -Now, let's continue with deploying Kubernetes: +### Starting a cluster -```sh -cd kubernetes - -# Build source -hack/build-go.sh - -# Build a release (argument is the instance prefix) -release/build-release.sh kubernetes +Now, let's continue with deploying Kubernetes. +This process takes about ~10 minutes. -# Deploy Kubernetes (takes ~5 minutes, provided everything works out) +```sh +cd kubernetes # Extracted binary release OR repository root export KUBERNETES_PROVIDER=vsphere cluster/kube-up.sh ``` @@ -84,3 +71,10 @@ Engine. Once you have successfully reached this point, your vSphere Kubernetes deployment works just as any other one! **Enjoy!** + +### Extra: debugging deployment failure + +The output of `kube-up.sh` displays the IP addresses of the VMs it deploys. You +can log into any VM as the `kube` user to poke around and figure out what is +going on (find yourself authorized with your SSH key, or use the password +`kube` otherwise). diff --git a/hack/config-go.sh b/hack/config-go.sh index ccf7fd596e69a..dded5744c2c2e 100644 --- a/hack/config-go.sh +++ b/hack/config-go.sh @@ -78,7 +78,7 @@ kube::version_ldflags() { ldflags+=(-X "${KUBE_GO_PACKAGE}/pkg/version.gitTreeState" "${KUBE_GIT_TREE_STATE}") # Use git describe to find the version based on annotated tags. - if [[ -n ${KUBE_GIT_VERSION-} ]] || KUBE_GIT_VERSION=$(git describe --abbrev=14 "${KUBE_GIT_COMMIT}^{commit}" 2>/dev/null); then + if [[ -n ${KUBE_GIT_VERSION-} ]] || KUBE_GIT_VERSION=$(git describe --tag --abbrev=14 "${KUBE_GIT_COMMIT}^{commit}" 2>/dev/null); then if [[ "${KUBE_GIT_TREE_STATE}" == "dirty" ]]; then # git describe --dirty only considers changes to existing files, but # that is problematic since new untracked .go files affect the build, diff --git a/icebox/cluster/rackspace/cloud-config/master-cloud-config.yaml b/icebox/cluster/rackspace/cloud-config/master-cloud-config.yaml deleted file mode 100644 index cec8bf2e13a6d..0000000000000 --- a/icebox/cluster/rackspace/cloud-config/master-cloud-config.yaml +++ /dev/null @@ -1,29 +0,0 @@ -#cloud-config - -write_files: -- content: | - grains: - roles: - - kubernetes-master - cloud: rackspace - etcd_servers: KUBE_MASTER - path: /etc/salt/minion.d/grains.conf -- content: | - auto_accept: True - path: /etc/salt/master.d/auto-accept.conf -- content: | - reactor: - - 'salt/minion/*/start': - - /srv/reactor/start.sls - path: /etc/salt/master.d/reactor.conf -- content: | - master: KUBE_MASTER - path: /etc/salt/minion.d/master.conf - -runcmd: - - [mkdir, -p, /etc/salt/minion.d] - - [mkdir, -p, /etc/salt/master.d] - - [mkdir, -p, /srv/salt/nginx] - - echo "MASTER_HTPASSWD" > /srv/salt/nginx/htpasswd - - [bash, /root/masterStart.sh] - - curl -L http://bootstrap.saltstack.com | sh -s -- -M -X diff --git a/icebox/cluster/rackspace/cloud-config/minion-cloud-config.yaml b/icebox/cluster/rackspace/cloud-config/minion-cloud-config.yaml deleted file mode 100644 index fab7a7b1abaf0..0000000000000 --- a/icebox/cluster/rackspace/cloud-config/minion-cloud-config.yaml +++ /dev/null @@ -1,5 +0,0 @@ -#cloud-config - -runcmd: - - [mkdir, -p, /etc/salt/minion.d] - - [bash, /root/minionStart.sh] diff --git a/icebox/cluster/rackspace/templates/download-release.sh b/icebox/cluster/rackspace/templates/download-release.sh deleted file mode 100644 index 3d036add0e8de..0000000000000 --- a/icebox/cluster/rackspace/templates/download-release.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Download and install release - -# This script assumes that the environment variable MASTER_RELEASE_TAR contains -# the release tar to download and unpack. It is meant to be pushed to the -# master and run. - -echo "Downloading release ($OBJECT_URL)" -wget $OBJECT_URL -O master-release.tgz - -echo "Unpacking release" -rm -rf master-release || false -tar xzf master-release.tgz - -echo "Running release install script" -sudo master-release/src/scripts/master-release-install.sh diff --git a/icebox/cluster/rackspace/templates/salt-minion.sh b/icebox/cluster/rackspace/templates/salt-minion.sh deleted file mode 100644 index 78daf184c9fcc..0000000000000 --- a/icebox/cluster/rackspace/templates/salt-minion.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Prepopulate the name of the Master -mkdir -p /etc/salt/minion.d -echo master: $MASTER_NAME > /etc/salt/minion.d/master.conf -# Turn on debugging for salt-minion -# echo "DAEMON_ARGS=\"\$DAEMON_ARGS --log-file-level=debug\"" > /etc/default/salt-minion -MINION_IP=$(ip -f inet a sh dev eth2 | awk -F '[ \t/]+' '/inet/ { print $3 }' ) -# Our minions will have a pool role to distinguish them from the master. -cat </etc/salt/minion.d/grains.conf -grains: - roles: - - kubernetes-pool - cbr-cidr: $MINION_IP_RANGE - minion_ip: $MINION_IP - etcd_servers: $MASTER_NAME -EOF -#Move all of this to salt -apt-get update -apt-get install bridge-utils -y -brctl addbr cbr0 -ip l set dev cbr0 up -#for loop to add routes of other minions -for i in `seq 1 $NUM_MINIONS` -do ip r a 10.240.$i.0/24 dev cbr0 -done -ip l a vxlan42 type vxlan id 42 group 239.0.0.42 dev eth2 -brctl addif cbr0 vxlan42 -# Install Salt -# -# We specify -X to avoid a race condition that can cause minion failure to -# install. See https://github.com/saltstack/salt-bootstrap/issues/270 -curl -L http://bootstrap.saltstack.com | sh -s -- -X -ip l set vxlan42 up \ No newline at end of file diff --git a/icebox/cluster/vsphere/util.sh b/icebox/cluster/vsphere/util.sh deleted file mode 100644 index 8e5fb15055d57..0000000000000 --- a/icebox/cluster/vsphere/util.sh +++ /dev/null @@ -1,306 +0,0 @@ -#!/bin/bash - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A library of helper functions and constants for the local config. - -# Use the config file specified in $KUBE_CONFIG_FILE, or default to -# config-default.sh. -source $(dirname ${BASH_SOURCE})/${KUBE_CONFIG_FILE-"config-default.sh"} - -function detect-master { - KUBE_MASTER=${MASTER_NAME} - if [ -z "$KUBE_MASTER_IP" ]; then - KUBE_MASTER_IP=$(govc vm.ip ${MASTER_NAME}) - fi - if [ -z "$KUBE_MASTER_IP" ]; then - echo "Could not detect Kubernetes master node. Make sure you've launched a cluster with 'kube-up.sh'" - exit 1 - fi - echo "Found ${KUBE_MASTER} at ${KUBE_MASTER_IP}" -} - -function detect-minions { - KUBE_MINION_IP_ADDRESSES=() - for (( i=0; i<${#MINION_NAMES[@]}; i++)); do - local minion_ip=$(govc vm.ip ${MINION_NAMES[$i]}) - echo "Found ${MINION_NAMES[$i]} at ${minion_ip}" - KUBE_MINION_IP_ADDRESSES+=("${minion_ip}") - done - if [ -z "$KUBE_MINION_IP_ADDRESSES" ]; then - echo "Could not detect Kubernetes minion nodes. Make sure you've launched a cluster with 'kube-up.sh'" - exit 1 - fi -} - -# Verify prereqs on host machine -function verify-prereqs { - if [ "$(which govc)" == "" ]; then - echo "Can't find govc in PATH, please install and retry." - echo "" - echo " go install github.com/vmware/govmomi/govc" - echo "" - exit 1 - fi -} - -# Run command over ssh -function kube-ssh { - local host=$1 - shift - ssh ${SSH_OPTS} kube@${host} "$*" 2> /dev/null -} - -# Instantiate a generic kubernetes virtual machine (master or minion) -function kube-up-vm { - local vm_name=$1 - local vm_memory=$2 - local vm_cpu=$3 - local vm_ip= - - govc vm.create \ - -debug \ - -m ${vm_memory} \ - -c ${vm_cpu} \ - -disk ${DISK} \ - -g ${GUEST_ID} \ - -link=true \ - ${vm_name} - - # Retrieve IP first, to confirm the guest operations agent is running. - vm_ip=$(govc vm.ip ${vm_name}) - - govc guest.mkdir \ - -vm ${vm_name} \ - -p \ - /home/kube/.ssh - - govc guest.upload \ - -vm ${vm_name} \ - -f \ - ${PUBLIC_KEY_FILE} \ - /home/kube/.ssh/authorized_keys -} - -# Instantiate a kubernetes cluster -function kube-up { - # Build up start up script for master - KUBE_TEMP=$(mktemp -d -t kubernetes.XXXXXX) - trap "rm -rf ${KUBE_TEMP}" EXIT - - get-password - python $(dirname $0)/../third_party/htpasswd/htpasswd.py -b -c ${KUBE_TEMP}/htpasswd $user $passwd - HTPASSWD=$(cat ${KUBE_TEMP}/htpasswd) - - echo "Starting master VM (this can take a minute)..." - - kube-up-vm ${MASTER_NAME} ${MASTER_MEMORY_MB-1024} ${MASTER_CPU-1} - - # Prints master IP, so user can log in for debugging. - detect-master - echo - - echo "Starting minion VMs (this can take a minute)..." - - for (( i=0; i<${#MINION_NAMES[@]}; i++)); do - ( - echo "#! /bin/bash" - echo "MY_NAME=${MINION_NAMES[$i]}" - grep -v "^#" $(dirname $0)/vsphere/templates/hostname.sh - echo "MASTER_NAME=${MASTER_NAME}" - echo "MASTER_IP=${KUBE_MASTER_IP}" - echo "MINION_IP_RANGE=${MINION_IP_RANGES[$i]}" - grep -v "^#" $(dirname $0)/vsphere/templates/salt-minion.sh - ) > ${KUBE_TEMP}/minion-start-${i}.sh - - ( - kube-up-vm ${MINION_NAMES[$i]} ${MINION_MEMORY_MB-1024} ${MINION_CPU-1} - - MINION_IP=$(govc vm.ip ${MINION_NAMES[$i]}) - - govc guest.upload \ - -vm ${MINION_NAMES[$i]} \ - -perm 0700 \ - -f \ - ${KUBE_TEMP}/minion-start-${i}.sh \ - /home/kube/minion-start.sh - - # Kickstart start script - kube-ssh ${MINION_IP} "nohup sudo ~/minion-start.sh < /dev/null 1> minion-start.out 2> minion-start.err &" - ) & - done - - FAIL=0 - for job in `jobs -p` - do - wait $job || let "FAIL+=1" - done - if (( $FAIL != 0 )); then - echo "${FAIL} commands failed. Exiting." - exit 2 - fi - - # Print minion IPs, so user can log in for debugging. - detect-minions - echo - - # Continue provisioning the master. - - ( - echo "#! /bin/bash" - echo "MY_NAME=${MASTER_NAME}" - grep -v "^#" $(dirname $0)/vsphere/templates/hostname.sh - echo "MASTER_NAME=${MASTER_NAME}" - echo "MASTER_HTPASSWD='${HTPASSWD}'" - grep -v "^#" $(dirname $0)/vsphere/templates/install-release.sh - grep -v "^#" $(dirname $0)/vsphere/templates/salt-master.sh - ) > ${KUBE_TEMP}/master-start.sh - - govc guest.upload \ - -vm ${MASTER_NAME} \ - -perm 0700 \ - -f \ - ${KUBE_TEMP}/master-start.sh \ - /home/kube/master-start.sh - - govc guest.upload \ - -vm ${MASTER_NAME} \ - -f \ - ./_output/release/master-release.tgz \ - /home/kube/master-release.tgz - - # Kickstart start script - kube-ssh ${KUBE_MASTER_IP} "nohup sudo ~/master-start.sh < /dev/null 1> master-start.out 2> master-start.err &" - - echo "Waiting for cluster initialization." - echo - echo " This will continually check to see if the API for kubernetes is reachable." - echo " This might loop forever if there was some uncaught error during start up." - echo - - until $(curl --insecure --user ${user}:${passwd} --max-time 5 \ - --fail --output /dev/null --silent https://${KUBE_MASTER_IP}/api/v1beta1/pods); do - printf "." - sleep 2 - done - - echo "Kubernetes cluster created." - echo - - echo "Sanity checking cluster..." - - sleep 5 - - # Don't bail on errors, we want to be able to print some info. - set +e - - # Basic sanity checking - for (( i=0; i<${#MINION_NAMES[@]}; i++)); do - # Make sure docker is installed - kube-ssh ${KUBE_MINION_IP_ADDRESSES[$i]} which docker > /dev/null - if [ "$?" != "0" ]; then - echo "Docker failed to install on ${MINION_NAMES[$i]}. Your cluster is unlikely to work correctly." - echo "Please run ./cluster/kube-down.sh and re-create the cluster. (sorry!)" - exit 1 - fi - done - - echo - echo "Kubernetes cluster is running. The master is running at:" - echo - echo " https://${KUBE_MASTER_IP}" - echo - echo "The user name and password to use is located in ~/.kubernetes_auth." - echo - echo "Security note: The server above uses a self signed certificate." - echo "This is subject to \"Man in the middle\" type attacks." - echo -} - -# Delete a kubernetes cluster -function kube-down { - govc vm.destroy ${MASTER_NAME} & - - for (( i=0; i<${#MINION_NAMES[@]}; i++)); do - govc vm.destroy ${MINION_NAMES[i]} & - done - - wait - -} - -# Update a kubernetes cluster with latest source -function kube-push { - detect-master - - govc guest.upload \ - -vm ${MASTER_NAME} \ - -f \ - ./_output/release/master-release.tgz \ - /home/kube/master-release.tgz - - ( - grep -v "^#" $(dirname $0)/vsphere/templates/install-release.sh - echo "echo Executing configuration" - echo "sudo salt '*' mine.update" - echo "sudo salt --force-color '*' state.highstate" - ) | kube-ssh ${KUBE_MASTER_IP} bash - - get-password - - echo - echo "Kubernetes cluster is updated. The master is running at:" - echo - echo " https://${KUBE_MASTER_IP}" - echo - echo "The user name and password to use is located in ~/.kubernetes_auth." - echo -} - -# Execute prior to running tests to build a release if required for env -function test-build-release { - echo "TODO" -} - -# Execute prior to running tests to initialize required structure -function test-setup { - echo "TODO" -} - -# Execute after running tests to perform any required clean-up -function test-teardown { - echo "TODO" -} - -# Set the {user} and {password} environment values required to interact with provider -function get-password { - file=${HOME}/.kubernetes_auth - if [ -e ${file} ]; then - user=$(cat $file | python -c 'import json,sys;print(json.load(sys.stdin)["User"])') - passwd=$(cat $file | python -c 'import json,sys;print(json.load(sys.stdin)["Password"])') - return - fi - user=admin - passwd=$(python -c 'import string,random; print("".join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(16)))') - - # Store password for reuse. - cat << EOF > ~/.kubernetes_auth -{ - "User": "$user", - "Password": "$passwd" -} -EOF - chmod 0600 ~/.kubernetes_auth -} diff --git a/icebox/release/rackspace/release.sh b/icebox/release/rackspace/release.sh deleted file mode 100755 index 0caac1ce178aa..0000000000000 --- a/icebox/release/rackspace/release.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -# Copyright 2014 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script will build and release Kubernetes. -# -# The main parameters to this script come from the config.sh file. This is set -# up by default for development releases. Feel free to edit it or override some -# of the variables there. - -# exit on any error -set -e - -SCRIPT_DIR=$(CDPATH="" cd $(dirname $0); pwd) - -source $SCRIPT_DIR/config.sh -KUBE_REPO_ROOT="$(cd "$(dirname "$0")/../../" && pwd -P)" - -source "${KUBE_REPO_ROOT}/cluster/kube-env.sh" -source $SCRIPT_DIR/../../cluster/rackspace/${KUBE_CONFIG_FILE-"config-default.sh"} -source $SCRIPT_DIR/../../cluster/rackspace/util.sh - -$SCRIPT_DIR/../build-release.sh $INSTANCE_PREFIX - -# Copy everything up to swift object store -echo "release/rackspace/release.sh: Uploading to Cloud Files" -if ! swiftly -A $OS_AUTH_URL -U $OS_USERNAME -K $OS_PASSWORD get $CONTAINER > /dev/null 2>&1 ; then - echo "release/rackspace/release.sh: Container doesn't exist. Creating..." - swiftly -A $OS_AUTH_URL -U $OS_USERNAME -K $OS_PASSWORD put $CONTAINER > /dev/null 2>&1 - -fi - -for x in master-release.tgz; do - swiftly -A $OS_AUTH_URL -U $OS_USERNAME -K $OS_PASSWORD put -i _output/release/$x $CONTAINER/output/release/$x > /dev/null 2>&1 -done - -echo "Release pushed." diff --git a/pkg/api/types.go b/pkg/api/types.go index ad47551e85d5b..8ac256a703086 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -307,8 +307,10 @@ type ContainerState struct { type ContainerStatus struct { // TODO(dchen1107): Should we rename PodStatus to a more generic name or have a separate states // defined for container? - State ContainerState `json:"state,omitempty" yaml:"state,omitempty"` - RestartCount int `json:"restartCount" yaml:"restartCount"` + State ContainerState `json:"state,omitempty" yaml:"state,omitempty"` + // Note that this is calculated from dead containers. But those containers are subject to + // garbage collection. This value will get capped at 5 by GC. + RestartCount int `json:"restartCount" yaml:"restartCount"` // TODO(dchen1107): Deprecated this soon once we pull entire PodStatus from node, // not just PodInfo. Now we need this to remove docker.Container from API PodIP string `json:"podIP,omitempty" yaml:"podIP,omitempty"` diff --git a/pkg/api/v1beta1/types.go b/pkg/api/v1beta1/types.go index f8c136e29ce05..d89b247c193ea 100644 --- a/pkg/api/v1beta1/types.go +++ b/pkg/api/v1beta1/types.go @@ -335,8 +335,10 @@ type ContainerState struct { type ContainerStatus struct { // TODO(dchen1107): Should we rename PodStatus to a more generic name or have a separate states // defined for container? - State ContainerState `json:"state,omitempty" yaml:"state,omitempty"` - RestartCount int `json:"restartCount" yaml:"restartCount"` + State ContainerState `json:"state,omitempty" yaml:"state,omitempty"` + // Note that this is calculated from dead containers. But those containers are subject to + // garbage collection. This value will get capped at 5 by GC. + RestartCount int `json:"restartCount" yaml:"restartCount"` // TODO(dchen1107): Deprecated this soon once we pull entire PodStatus from node, // not just PodInfo. Now we need this to remove docker.Container from API PodIP string `json:"podIP,omitempty" yaml:"podIP,omitempty"` diff --git a/pkg/api/v1beta2/types.go b/pkg/api/v1beta2/types.go index 93f2710bd1b45..47fd02af2490e 100644 --- a/pkg/api/v1beta2/types.go +++ b/pkg/api/v1beta2/types.go @@ -300,8 +300,10 @@ type ContainerState struct { type ContainerStatus struct { // TODO(dchen1107): Should we rename PodStatus to a more generic name or have a separate states // defined for container? - State ContainerState `json:"state,omitempty" yaml:"state,omitempty"` - RestartCount int `json:"restartCount" yaml:"restartCount"` + State ContainerState `json:"state,omitempty" yaml:"state,omitempty"` + // Note that this is calculated from dead containers. But those containers are subject to + // garbage collection. This value will get capped at 5 by GC. + RestartCount int `json:"restartCount" yaml:"restartCount"` // TODO(dchen1107): Deprecated this soon once we pull entire PodStatus from node, // not just PodInfo. Now we need this to remove docker.Container from API PodIP string `json:"podIP,omitempty" yaml:"podIP,omitempty"` diff --git a/pkg/api/v1beta3/types.go b/pkg/api/v1beta3/types.go index f2b89fbd2c265..21397c50af79f 100644 --- a/pkg/api/v1beta3/types.go +++ b/pkg/api/v1beta3/types.go @@ -399,8 +399,10 @@ type ContainerState struct { type ContainerStatus struct { // TODO(dchen1107): Should we rename PodStatus to a more generic name or have a separate states // defined for container? - State ContainerState `json:"state,omitempty" yaml:"state,omitempty"` - RestartCount int `json:"restartCount" yaml:"restartCount"` + State ContainerState `json:"state,omitempty" yaml:"state,omitempty"` + // Note that this is calculated from dead containers. But those containers are subject to + // garbage collection. This value will get capped at 5 by GC. + RestartCount int `json:"restartCount" yaml:"restartCount"` // TODO(dchen1107): Introduce our own NetworkSettings struct here? // TODO(dchen1107): Which image the container is running with? // TODO(dchen1107): Once we have done with integration with cadvisor, resource diff --git a/pkg/kubelet/dockertools/docker.go b/pkg/kubelet/dockertools/docker.go index 6dc70d300c688..2f1bb1282e382 100644 --- a/pkg/kubelet/dockertools/docker.go +++ b/pkg/kubelet/dockertools/docker.go @@ -47,6 +47,7 @@ type DockerInterface interface { CreateContainer(docker.CreateContainerOptions) (*docker.Container, error) StartContainer(id string, hostConfig *docker.HostConfig) error StopContainer(id string, timeout uint) error + RemoveContainer(opts docker.RemoveContainerOptions) error InspectImage(image string) (*docker.Image, error) PullImage(opts docker.PullImageOptions, auth docker.AuthConfiguration) error Logs(opts docker.LogsOptions) error diff --git a/pkg/kubelet/dockertools/fake_docker_client.go b/pkg/kubelet/dockertools/fake_docker_client.go index 10c80335a6658..7820b39d77716 100644 --- a/pkg/kubelet/dockertools/fake_docker_client.go +++ b/pkg/kubelet/dockertools/fake_docker_client.go @@ -29,12 +29,15 @@ type FakeDockerClient struct { sync.Mutex ContainerList []docker.APIContainers Container *docker.Container + ContainerMap map[string]*docker.Container Image *docker.Image Err error called []string Stopped []string pulled []string Created []string + Removed []string + VersionInfo docker.Env } func (f *FakeDockerClient) clearCalls() { @@ -69,6 +72,11 @@ func (f *FakeDockerClient) InspectContainer(id string) (*docker.Container, error f.Lock() defer f.Unlock() f.called = append(f.called, "inspect_container") + if f.ContainerMap != nil { + if container, ok := f.ContainerMap[id]; ok { + return container, f.Err + } + } return f.Container, f.Err } @@ -121,6 +129,14 @@ func (f *FakeDockerClient) StopContainer(id string, timeout uint) error { return f.Err } +func (f *FakeDockerClient) RemoveContainer(opts docker.RemoveContainerOptions) error { + f.Lock() + defer f.Unlock() + f.called = append(f.called, "remove") + f.Removed = append(f.Removed, opts.ID) + return f.Err +} + // Logs is a test-spy implementation of DockerInterface.Logs. // It adds an entry "logs" to the internal method call record. func (f *FakeDockerClient) Logs(opts docker.LogsOptions) error { diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go index 32e6b02e385f7..18d4bc8affc98 100644 --- a/pkg/kubelet/kubelet.go +++ b/pkg/kubelet/kubelet.go @@ -22,6 +22,7 @@ import ( "io" "net/http" "path" + "sort" "strconv" "strings" "sync" @@ -69,7 +70,9 @@ func NewMainKubelet( ni string, ri time.Duration, pullQPS float32, - pullBurst int) *Kubelet { + pullBurst int, + minimumGCAge time.Duration, + maxContainerCount int) *Kubelet { return &Kubelet{ hostname: hn, dockerClient: dc, @@ -82,6 +85,8 @@ func NewMainKubelet( httpClient: &http.Client{}, pullQPS: pullQPS, pullBurst: pullBurst, + minimumGCAge: minimumGCAge, + maxContainerCount: maxContainerCount, } } @@ -133,6 +138,68 @@ type Kubelet struct { // Optional, no statistics will be available if omitted cadvisorClient CadvisorInterface cadvisorLock sync.RWMutex + + // Optional, minimum age required for garbage collection. If zero, no limit. + minimumGCAge time.Duration + maxContainerCount int +} + +type ByCreated []*docker.Container + +func (a ByCreated) Len() int { return len(a) } +func (a ByCreated) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a ByCreated) Less(i, j int) bool { return a[i].Created.After(a[j].Created) } + +// TODO: these removals are racy, we should make dockerclient threadsafe across List/Inspect transactions. +func (kl *Kubelet) purgeOldest(ids []string) error { + dockerData := []*docker.Container{} + for _, id := range ids { + data, err := kl.dockerClient.InspectContainer(id) + if err != nil { + return err + } + if !data.State.Running && (kl.minimumGCAge == 0 || time.Now().Sub(data.State.FinishedAt) > kl.minimumGCAge) { + dockerData = append(dockerData, data) + } + } + sort.Sort(ByCreated(dockerData)) + if len(dockerData) <= kl.maxContainerCount { + return nil + } + dockerData = dockerData[kl.maxContainerCount:] + for _, data := range dockerData { + if err := kl.dockerClient.RemoveContainer(docker.RemoveContainerOptions{ID: data.ID}); err != nil { + return err + } + } + + return nil +} + +// TODO: Also enforce a maximum total number of containers. +func (kl *Kubelet) GarbageCollectContainers() error { + if kl.maxContainerCount == 0 { + return nil + } + containers, err := dockertools.GetKubeletDockerContainers(kl.dockerClient, true) + if err != nil { + return err + } + uuidToIDMap := map[string][]string{} + for _, container := range containers { + _, uuid, name, _ := dockertools.ParseDockerName(container.ID) + uuidName := uuid + "." + name + uuidToIDMap[uuidName] = append(uuidToIDMap[uuidName], container.ID) + } + for _, list := range uuidToIDMap { + if len(list) <= kl.maxContainerCount { + continue + } + if err := kl.purgeOldest(list); err != nil { + return err + } + } + return nil } // SetCadvisorClient sets the cadvisor client in a thread-safe way. diff --git a/pkg/kubelet/kubelet_test.go b/pkg/kubelet/kubelet_test.go index df323fccc262c..03916795c1d8d 100644 --- a/pkg/kubelet/kubelet_test.go +++ b/pkg/kubelet/kubelet_test.go @@ -1162,3 +1162,297 @@ func TestSyncPodEventHandlerFails(t *testing.T) { t.Errorf("Wrong containers were stopped: %v", fakeDocker.Stopped) } } + +func TestKubeletGarbageCollection(t *testing.T) { + tests := []struct { + containers []docker.APIContainers + containerDetails map[string]*docker.Container + expectedRemoved []string + }{ + { + containers: []docker.APIContainers{ + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "1876", + }, + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "2876", + }, + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "3876", + }, + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "4876", + }, + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "5876", + }, + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "6876", + }, + }, + containerDetails: map[string]*docker.Container{ + "1876": { + State: docker.State{ + Running: false, + }, + ID: "1876", + Created: time.Now(), + }, + }, + expectedRemoved: []string{"1876"}, + }, + { + containers: []docker.APIContainers{ + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "1876", + }, + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "2876", + }, + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "3876", + }, + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "4876", + }, + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "5876", + }, + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "6876", + }, + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "7876", + }, + }, + containerDetails: map[string]*docker.Container{ + "1876": { + State: docker.State{ + Running: true, + }, + ID: "1876", + Created: time.Now(), + }, + "2876": { + State: docker.State{ + Running: false, + }, + ID: "2876", + Created: time.Now(), + }, + }, + expectedRemoved: []string{"2876"}, + }, + { + containers: []docker.APIContainers{ + { + // network container + Names: []string{"/k8s_net_foo.new.test_.deadbeef"}, + ID: "1876", + }, + }, + }, + } + for _, test := range tests { + kubelet, _, fakeDocker := newTestKubelet(t) + kubelet.maxContainerCount = 5 + fakeDocker.ContainerList = test.containers + fakeDocker.ContainerMap = test.containerDetails + fakeDocker.Container = &docker.Container{ID: "error", Created: time.Now()} + err := kubelet.GarbageCollectContainers() + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !reflect.DeepEqual(fakeDocker.Removed, test.expectedRemoved) { + t.Errorf("expected: %v, got: %v", test.expectedRemoved, fakeDocker.Removed) + } + } +} + +func TestPurgeOldest(t *testing.T) { + created := time.Now() + tests := []struct { + ids []string + containerDetails map[string]*docker.Container + expectedRemoved []string + }{ + { + ids: []string{"1", "2", "3", "4", "5"}, + containerDetails: map[string]*docker.Container{ + "1": { + State: docker.State{ + Running: true, + }, + ID: "1", + Created: created, + }, + "2": { + State: docker.State{ + Running: false, + }, + ID: "2", + Created: created.Add(time.Second), + }, + "3": { + State: docker.State{ + Running: false, + }, + ID: "3", + Created: created.Add(time.Second), + }, + "4": { + State: docker.State{ + Running: false, + }, + ID: "4", + Created: created.Add(time.Second), + }, + "5": { + State: docker.State{ + Running: false, + }, + ID: "5", + Created: created.Add(time.Second), + }, + }, + }, + { + ids: []string{"1", "2", "3", "4", "5", "6"}, + containerDetails: map[string]*docker.Container{ + "1": { + State: docker.State{ + Running: false, + }, + ID: "1", + Created: created.Add(time.Second), + }, + "2": { + State: docker.State{ + Running: false, + }, + ID: "2", + Created: created.Add(time.Millisecond), + }, + "3": { + State: docker.State{ + Running: false, + }, + ID: "3", + Created: created.Add(time.Second), + }, + "4": { + State: docker.State{ + Running: false, + }, + ID: "4", + Created: created.Add(time.Second), + }, + "5": { + State: docker.State{ + Running: false, + }, + ID: "5", + Created: created.Add(time.Second), + }, + "6": { + State: docker.State{ + Running: false, + }, + ID: "6", + Created: created.Add(time.Second), + }, + }, + expectedRemoved: []string{"2"}, + }, + { + ids: []string{"1", "2", "3", "4", "5", "6", "7"}, + containerDetails: map[string]*docker.Container{ + "1": { + State: docker.State{ + Running: false, + }, + ID: "1", + Created: created.Add(time.Second), + }, + "2": { + State: docker.State{ + Running: false, + }, + ID: "2", + Created: created.Add(time.Millisecond), + }, + "3": { + State: docker.State{ + Running: false, + }, + ID: "3", + Created: created.Add(time.Second), + }, + "4": { + State: docker.State{ + Running: false, + }, + ID: "4", + Created: created.Add(time.Second), + }, + "5": { + State: docker.State{ + Running: false, + }, + ID: "5", + Created: created.Add(time.Second), + }, + "6": { + State: docker.State{ + Running: false, + }, + ID: "6", + Created: created.Add(time.Microsecond), + }, + "7": { + State: docker.State{ + Running: false, + }, + ID: "7", + Created: created.Add(time.Second), + }, + }, + expectedRemoved: []string{"2", "6"}, + }, + } + for _, test := range tests { + kubelet, _, fakeDocker := newTestKubelet(t) + kubelet.maxContainerCount = 5 + fakeDocker.ContainerMap = test.containerDetails + kubelet.purgeOldest(test.ids) + if !reflect.DeepEqual(fakeDocker.Removed, test.expectedRemoved) { + t.Errorf("expected: %v, got: %v", test.expectedRemoved, fakeDocker.Removed) + } + } +} diff --git a/pkg/version/base.go b/pkg/version/base.go index ea21e837dc240..3ecc40bc50486 100644 --- a/pkg/version/base.go +++ b/pkg/version/base.go @@ -36,8 +36,8 @@ package version var ( // TODO: Deprecate gitMajor and gitMinor, use only gitVersion instead. gitMajor string = "0" // major version, always numeric - gitMinor string = "4+" // minor version, numeric possibly followed by "+" - gitVersion string = "v0.4-dev" // version from git, output of $(git describe) + gitMinor string = "4.4+" // minor version, numeric possibly followed by "+" + gitVersion string = "v0.4.4-dev" // version from git, output of $(git describe) gitCommit string = "" // sha1 from git, output of $(git rev-parse HEAD) gitTreeState string = "not a git tree" // state of git tree, either "clean" or "dirty" )