From 7352ba0c8aada5cac46e3a4c1692ffcb6d1d8d9a Mon Sep 17 00:00:00 2001
From: Madhav Puri <madhav.puri@gmail.com>
Date: Tue, 22 Mar 2016 21:02:39 -0700
Subject: [PATCH 1/5] Makefile enhancements

- added a help target to print help for the available targets
- added a `demo-stock` target to bring up demo setup with a stock OS image than
  our pre-provisioned contiv/centos72 image.

Signed-off-by: Madhav Puri <madhav.puri@gmail.com>
---
 Makefile    | 26 +++++++++++++++++++++-----
 Vagrantfile | 52 +++++++++++++++++++++++++++++++++-------------------
 2 files changed, 54 insertions(+), 24 deletions(-)

diff --git a/Makefile b/Makefile
index 3bdcf45..d0ee786 100644
--- a/Makefile
+++ b/Makefile
@@ -1,11 +1,27 @@
-demo: stop start
-stop:
+.PHONY: demo demo-stock help start start-stock stop svc-provision svc-cleanup
+
+.DEFAULT_GOAL := help
+
+help: ## This help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+demo: stop start ## Bring up a demo setup. Tearsdown any existing setup first.
+
+demo-stock: stop start-stock ## Bring up a demo setup with stock OS box. All packages are installed fresh. Useful for testing production like environment. Tearsdown any existing setup first.
+
+stop: ## Teardown a demo setup.
 	CONTIV_NODES=$${CONTIV_NODES:-3} vagrant destroy -f
-start:
+
+start: ## Bring up a demo setup.
 	CONTIV_NODES=$${CONTIV_NODES:-3} CONTIV_SRV_INIT=1 vagrant up
-svc-provision:
+
+start-stock: ## Bring up a demo setup with stock OS box. All packages are installed fresh. Useful for testing production like environment.
+	CONTIV_BOX="puppetlabs/centos-7.2-64-nocm" CONTIV_BOX_VERSION="1.0.1" make start
+
+svc-provision: ## Rerun ansible provisioning on the exisitng demo setup.
 	CONTIV_NODES=$${CONTIV_NODES:-3} CONTIV_SRV_INIT=1 \
 				 vagrant provision --provision-with ansible
-svc-cleanup:
+
+svc-cleanup: ## Run cleanup ansible on the existing demo setup.
 	CONTIV_NODES=$${CONTIV_NODES:-3} CONTIV_ANSIBLE_PLAYBOOK="./vendor/ansible/cleanup.yml" \
 				 vagrant provision --provision-with ansible
diff --git a/Vagrantfile b/Vagrantfile
index 8406f50..7577bae 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -16,6 +16,16 @@ if ENV['CONTIV_SRV_INIT'] then
     service_init = true
 end
 
+box = "contiv/centos72"
+if ENV['CONTIV_BOX'] then
+    box = ENV['CONTIV_BOX']
+end
+
+box_version = "0.3.0"
+if ENV['CONTIV_BOX_VERSION'] then
+    box_version = ENV['CONTIV_BOX_VERSION']
+end
+
 host_env = { }
 if ENV['CONTIV_ENV'] then
     ENV['CONTIV_ENV'].split(" ").each do |env|
@@ -73,8 +83,8 @@ EOF
 
 VAGRANTFILE_API_VERSION = "2"
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-    config.vm.box = "contiv/centos72"
-    config.vm.box_version = "0.3.0"
+    config.vm.box = box
+    config.vm.box_version = box_version
     node_ips = num_nodes.times.collect { |n| base_ip + "#{n+10}" }
     node_names = num_nodes.times.collect { |n| "cluster-node#{n+1}" }
     # this is to avoid the issue: https://github.com/mitchellh/vagrant/issues/5186
@@ -108,22 +118,26 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
                 v.customize ['modifyvm', :id, '--nictype3', 'virtio']
                 v.customize ['modifyvm', :id, '--nicpromisc2', 'allow-all']
                 v.customize ['modifyvm', :id, '--nicpromisc3', 'allow-all']
-                # create disks for ceph
-                (0..1).each do |d|
-                  disk_path = "disk-#{n}-#{d}"
-                  vdi_disk_path = disk_path + ".vdi"
-
-                  v.customize ['createhd',
-                               '--filename', disk_path,
-                               '--size', '11000']
-                  # Controller names are dependent on the VM being built.
-                  # It is set when the base box is made in our case ubuntu/trusty64.
-                  # Be careful while changing the box.
-                  v.customize ['storageattach', :id,
-                               '--storagectl', 'SATA Controller',
-                               '--port', 3 + d,
-                               '--type', 'hdd',
-                               '--medium', vdi_disk_path]
+                # XXX: creating disk doesn't work in stock centos box, remove this check
+                # once we need ceph working in stock OS demo
+                if box == "contiv/centos72" then
+                    # create disks for ceph
+                    (0..1).each do |d|
+                        disk_path = "disk-#{n}-#{d}"
+                        vdi_disk_path = disk_path + ".vdi"
+
+                        v.customize ['createhd',
+                                     '--filename', disk_path,
+                                     '--size', '11000']
+                        # Controller names are dependent on the VM being built.
+                        # It is set when the base box is made in our case ubuntu/trusty64.
+                        # Be careful while changing the box.
+                        v.customize ['storageattach', :id,
+                                     '--storagectl', 'SATA Controller',
+                                     '--port', 3 + d,
+                                     '--type', 'hdd',
+                                     '--medium', vdi_disk_path]
+                    end
                 end
             end
 
@@ -154,7 +168,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
                 ansible_groups["cluster-control"] = [node_name]
             end
 
-            if service_init
+            if service_init then
                 # Share anything in `shared` to '/shared' on the cluster hosts.
                 node.vm.synced_folder "shared", "/shared"
 

From d6f5f2b47665c2e77f2a0b61db603f33cca865a5 Mon Sep 17 00:00:00 2001
From: Madhav Puri <madhav.puri@gmail.com>
Date: Wed, 23 Mar 2016 02:26:53 -0700
Subject: [PATCH 2/5] update clusterm args and conf file location in tests

Signed-off-by: Madhav Puri <madhav.puri@gmail.com>
---
 management/src/demo/files/cli_test/clusterm      | 1 -
 management/src/demo/files/cli_test/clusterm.args | 1 +
 management/src/systemtests/cli_test.go           | 2 +-
 3 files changed, 2 insertions(+), 2 deletions(-)
 delete mode 100644 management/src/demo/files/cli_test/clusterm
 create mode 100644 management/src/demo/files/cli_test/clusterm.args

diff --git a/management/src/demo/files/cli_test/clusterm b/management/src/demo/files/cli_test/clusterm
deleted file mode 100644
index f1fa412..0000000
--- a/management/src/demo/files/cli_test/clusterm
+++ /dev/null
@@ -1 +0,0 @@
-CLUSTERM_ARGS="--config /etc/default/clusterm.conf.json"
diff --git a/management/src/demo/files/cli_test/clusterm.args b/management/src/demo/files/cli_test/clusterm.args
new file mode 100644
index 0000000..d025310
--- /dev/null
+++ b/management/src/demo/files/cli_test/clusterm.args
@@ -0,0 +1 @@
+CLUSTERM_ARGS="--config /etc/default/clusterm/clusterm.conf.json"
diff --git a/management/src/systemtests/cli_test.go b/management/src/systemtests/cli_test.go
index 7ec1105..be1d2cf 100644
--- a/management/src/systemtests/cli_test.go
+++ b/management/src/systemtests/cli_test.go
@@ -92,7 +92,7 @@ func (s *CliTestSuite) SetUpSuite(c *C) {
 	s.Assert(c, err, IsNil, Commentf("output: %s", out))
 	//provide test ansible playbooks and restart cluster-mgr
 	src := fmt.Sprintf("%s/../demo/files/cli_test/*", pwd)
-	dst := "/etc/default/"
+	dst := "/etc/default/clusterm/"
 	out, err = s.tbn1.RunCommandWithOutput(fmt.Sprintf("sudo cp -rf %s %s", src, dst))
 	s.Assert(c, err, IsNil, Commentf("output: %s", out))
 	out, err = tutils.ServiceRestartAndWaitForUp(s.tbn1, "clusterm", 30)

From 9b0a843abbdcc7bee9fdb6c12364c2dbbee95690 Mon Sep 17 00:00:00 2001
From: Madhav Puri <madhav.puri@gmail.com>
Date: Wed, 23 Mar 2016 02:45:32 -0700
Subject: [PATCH 3/5] bump up the vagrant box version

Signed-off-by: Madhav Puri <madhav.puri@gmail.com>
---
 Vagrantfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Vagrantfile b/Vagrantfile
index 7577bae..ba0c822 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -21,7 +21,7 @@ if ENV['CONTIV_BOX'] then
     box = ENV['CONTIV_BOX']
 end
 
-box_version = "0.3.0"
+box_version = "0.3.2"
 if ENV['CONTIV_BOX_VERSION'] then
     box_version = ENV['CONTIV_BOX_VERSION']
 end

From 3a3895084fbe538f6dfcec91c74130f6aba5b86a Mon Sep 17 00:00:00 2001
From: Madhav Puri <madhav.puri@gmail.com>
Date: Wed, 23 Mar 2016 23:30:30 -0700
Subject: [PATCH 4/5] remove git-branch from build docker image

this results in too many docker images piling up per PR

Also somehow docker 1.10 doesn't like build tags with / (forward slashes)

Signed-off-by: Madhav Puri <madhav.puri@gmail.com>
---
 management/src/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/management/src/Makefile b/management/src/Makefile
index c51b5d1..d4099fe 100644
--- a/management/src/Makefile
+++ b/management/src/Makefile
@@ -9,7 +9,7 @@ docker_buildargs:= \
 	$(if $(HTTPS_PROXY), --build-arg "HTTPS_PROXY=$(HTTPS_PROXY)") \
 	$(if $(http_proxy), --build-arg "http_proxy=$(http_proxy)") \
 	$(if $(https_proxy), --build-arg "https_proxy=$(https_proxy)")
-docker_img:=cluster-dev$(if $(GIT_BRANCH),:$(GIT_BRANCH))
+docker_img:=cluster-dev
 docker_run:=docker run --rm -u `id -un`:`id -un` -v `pwd`:$(work_dir) \
 	-w $(work_dir) "$(docker_img)"
 docker_run_interactive:=docker run -it --rm -u `id -un`:`id -un` -v `pwd`:$(work_dir) \

From a6ed3f0567c6e69bc3d86723e998d2c4347dfa49 Mon Sep 17 00:00:00 2001
From: Madhav Puri <madhav.puri@gmail.com>
Date: Thu, 24 Mar 2016 03:28:31 -0700
Subject: [PATCH 5/5] Squashed 'vendor/ansible/' changes from 00230d6..9ad9d15

9ad9d15 Merge pull request #140 from mapuri/collins
c141ce5 Merge pull request #139 from mapuri/serf
8598d4f Merge pull request #136 from mapuri/misc
75e9678 use collins container version in collins systemd config
a92a6ad set node-name in serf config
c08372d fix setup and cleanup logic for netmaster hosts alias
488278c Merge pull request #134 from saramach/phys-dom-ansible-changes
4e88f05 Merge pull request #132 from mapuri/misc
9700c21 Add user specified Bridge domain and Contract mode - Add defaults
ac42176 Add user specified Bridge domain and Contract mode
e80f69a Merge pull request #131 from contiv/other-volplugin-fixes
9283a68 Merge pull request #108 from vvb/mon_osd_separation
4d51a28 remove leftover diffs from PR #131
c75429a move iptable rule installation before service start
c24108b bump up the version of contiv services
8c35bea use short hostname in serf's node label
7bdd7a3 pick the etcd peer interface from hostvars
f362602 install a /32 for vip in ucarp script
7b7c678 serf fixes
253e428 Merge pull request #126 from mapuri/iptables
3fe1986 ansible/roles/docker: support for proper lvm thin provisioning; will improve performance without drawbacks like overlayfs.
3aab0fb add setup and cleanup of iptables rules to allow service traffic
5fcb251 Merge pull request #125 from mapuri/gofix
6d49ae2 Merge pull request #124 from mapuri/basecleanup
ca3ed78 cleanup /usr/local/go/ directory before installing golang there
3d578ba remove python-crypto workaround as we don't need it with pip gone
1683bef move ansible installation from base to dev and cluster-mgr roles
5db6056 move lshw installation to serf role
18eefdb move dev env specific packages from base role to dev role
1475deb fix for Golang installation task
7ecbf8f Merge pull request #123 from unclejack/vagrantfile_tweaks
dbdf7b7 Vagrantfile: add paravirt KVM & network tweaks
20e87da Merge pull request #121 from mapuri/iptables
d7989cc Merge pull request #120 from mapuri/yumfix
d66a75d pass the conf file option in clusterm systemd unit
e99cc35 remove the blanket yum upgrade task
ac7ca67 renaming compute role, commenting ceph, contiv_storage roles for now
64986c1 adding a variable which defines capability of a host
84add1b fixes to enable mon and osd separation based of a flag
771d4e2 Merge pull request #117 from saramach/phys-dom-ansible-changes
c55bda2 Merge pull request #116 from mapuri/master
90e63fa Merge pull request #115 from contiv/load_images
5dfc0ce Add support to specify APIC Physical Domain as a parameter
4de4617 roles/docker/tasks: fixups as prescribed by @mapuri
08ad31f docker: load saved images
7a21763 fix ansible task for Ubuntu to update ansible to latest

git-subtree-dir: vendor/ansible
git-subtree-split: 9ad9d158e414d80b08cdbc13e638b1d745731b0a
---
 Vagrantfile                                   |  3 +
 cleanup.yml                                   | 47 ++++++---------
 group_vars/all                                |  2 +
 roles/ansible/tasks/main.yml                  | 20 +++++++
 roles/base/tasks/redhat_tasks.yml             | 33 ++---------
 roles/base/tasks/ubuntu_tasks.yml             | 16 +----
 roles/ceph-common/handlers/main.yml           | 12 ++++
 roles/ceph-common/tasks/main.yml              | 14 +----
 roles/contiv_cluster/defaults/main.yml        |  8 ++-
 roles/contiv_cluster/files/clusterm           |  1 -
 roles/contiv_cluster/files/clusterm.args      |  1 +
 roles/contiv_cluster/files/clusterm.conf      |  3 +
 roles/contiv_cluster/meta/main.yml            |  5 ++
 roles/contiv_cluster/tasks/main.yml           | 27 +++++++--
 .../clusterm.j2}                              |  2 +-
 roles/contiv_cluster/templates/collins.j2     |  2 +-
 roles/contiv_network/defaults/main.yml        | 15 ++++-
 roles/contiv_network/tasks/cleanup.yml        | 18 ++++++
 roles/contiv_network/tasks/main.yml           | 20 ++++++-
 roles/contiv_network/tasks/ovs.yml            |  8 +++
 roles/contiv_network/tasks/ovs_cleanup.yml    |  6 ++
 roles/contiv_network/templates/aci_gw.j2      |  3 +
 roles/contiv_storage/defaults/main.yml        |  2 +-
 roles/dev/meta/main.yml                       |  1 +
 roles/dev/tasks/main.yml                      | 10 ++++
 roles/dev/tasks/os_agnostic_tasks.yml         |  5 +-
 roles/dev/tasks/redhat_tasks.yml              | 11 ++++
 roles/dev/tasks/ubuntu_tasks.yml              |  9 +++
 roles/docker/defaults/main.yml                |  4 +-
 roles/docker/tasks/cleanup.yml                |  6 ++
 roles/docker/tasks/create_docker_device.yml   | 29 +++++++++
 roles/docker/tasks/main.yml                   | 59 +++++++++++++++++--
 roles/docker/templates/docker-svc.j2          |  6 +-
 roles/docker/vars/main.yml                    |  2 +
 roles/etcd/defaults/main.yml                  |  2 +-
 roles/etcd/tasks/cleanup.yml                  |  9 +++
 roles/etcd/tasks/main.yml                     | 13 ++++
 roles/etcd/templates/etcd.j2                  |  6 +-
 roles/serf/files/serf.service                 |  1 +
 roles/serf/tasks/main.yml                     | 17 ++++++
 roles/serf/templates/serf.j2                  | 40 ++++++++++---
 roles/swarm/defaults/main.yml                 |  1 +
 roles/swarm/tasks/cleanup.yml                 |  6 ++
 roles/swarm/tasks/main.yml                    | 15 +++++
 roles/ucarp/files/ucarp/vip_up.sh             |  3 +-
 roles/ucp/defaults/main.yml                   |  9 ++-
 roles/ucp/tasks/cleanup.yml                   | 12 ++++
 roles/ucp/tasks/main.yml                      | 14 +++++
 roles/ucp/templates/ucp.j2                    |  5 +-
 site.yml                                      | 18 +++---
 50 files changed, 450 insertions(+), 131 deletions(-)
 create mode 100644 roles/ansible/tasks/main.yml
 delete mode 100644 roles/contiv_cluster/files/clusterm
 create mode 100644 roles/contiv_cluster/files/clusterm.args
 create mode 100644 roles/contiv_cluster/files/clusterm.conf
 create mode 100644 roles/contiv_cluster/meta/main.yml
 rename roles/contiv_cluster/{files/clusterm.service => templates/clusterm.j2} (78%)
 create mode 100644 roles/dev/tasks/redhat_tasks.yml
 create mode 100644 roles/dev/tasks/ubuntu_tasks.yml
 create mode 100644 roles/docker/tasks/create_docker_device.yml
 create mode 100644 roles/docker/vars/main.yml

diff --git a/Vagrantfile b/Vagrantfile
index d166bd2..4c11b45 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -43,6 +43,9 @@ Vagrant.configure(2) do |config|
         node.vm.provider "virtualbox" do |vb|
             vb.customize ['modifyvm', :id, '--memory', "4096"]
             vb.customize ["modifyvm", :id, "--cpus", "2"]
+            vb.customize ['modifyvm', :id, '--paravirtprovider', 'kvm']
+            vb.customize ['modifyvm', :id, '--natdnshostresolver1', 'on']
+            vb.customize ['modifyvm', :id, '--natdnsproxy1', 'on']
         end
 
         if ansible_groups["devtest"] == nil then
diff --git a/cleanup.yml b/cleanup.yml
index 7f1f606..61d74e0 100644
--- a/cleanup.yml
+++ b/cleanup.yml
@@ -8,34 +8,21 @@
   tasks:
     - include_vars: roles/{{ item }}/defaults/main.yml
       with_items:
-      - "etcd"
-      - "ucp"
-    - include: roles/contiv_network/tasks/cleanup.yml
-      ignore_errors: yes
-    - include: roles/contiv_storage/tasks/cleanup.yml
-      ignore_errors: yes
-    - include: roles/contiv_cluster/tasks/cleanup.yml
-      ignore_errors: yes
-    - include: roles/swarm/tasks/cleanup.yml
-      ignore_errors: yes
-    - include: roles/ucp/tasks/cleanup.yml
-      ignore_errors: yes
-    - include: roles/docker/tasks/cleanup.yml
-      ignore_errors: yes
-    - include: roles/etcd/tasks/cleanup.yml
-      ignore_errors: yes
-    - include: roles/ucarp/tasks/cleanup.yml
+        - "contiv_network"
+        - "contiv_storage"
+        - "contiv_cluster"
+        - "swarm"
+        - "ucp"
+        - "docker"
+        - "etcd"
+    - include: roles/{{ item }}/tasks/cleanup.yml
+      with_items:
+        - contiv_network
+        - contiv_storage
+        - contiv_cluster
+        - swarm
+        - ucp
+        - docker
+        - etcd
+        - ucarp
       ignore_errors: yes
-    # XXX: following syntax is much cleaner but is available only in v2.
-    # Will move to this once our packer images and hosts have consistently moved to Ansiblev2
-    #- include: roles/{{ item }}/tasks/cleanup.yml
-    #  with_items:
-    #    - contiv_network
-    #    - contiv_storage
-    #    - contiv_cluster
-    #    - swarm
-    #    - ucp
-    #    - docker
-    #    - etcd
-    #    - ucarp
-    #  ignore_errors: yes
diff --git a/group_vars/all b/group_vars/all
index 813731b..126d0a4 100644
--- a/group_vars/all
+++ b/group_vars/all
@@ -25,3 +25,5 @@ validate_certs: "yes"
 # env:
 # service_vip:
 # control_interface:
+
+host_capability: "can-run-user-containers, storage"
diff --git a/roles/ansible/tasks/main.yml b/roles/ansible/tasks/main.yml
new file mode 100644
index 0000000..1b50903
--- /dev/null
+++ b/roles/ansible/tasks/main.yml
@@ -0,0 +1,20 @@
+---
+# This role contains tasks for installing ansible
+
+- name: install ansible (redhat)
+  yum:
+    name: ansible
+    enablerepo: epel-testing
+    state: latest
+  when: ansible_os_family == "RedHat"
+
+- name: add ansible apt repository (debian)
+  apt_repository:
+    repo: ppa:ansible/ansible
+    state: present
+    validate_certs: "{{ validate_certs }}"
+  when: ansible_os_family == "Debian"
+
+- name: install ansible (debian)
+  apt: name=ansible state=latest
+  when: ansible_os_family == "Debian"
diff --git a/roles/base/tasks/redhat_tasks.yml b/roles/base/tasks/redhat_tasks.yml
index 58095f0..c5dad57 100644
--- a/roles/base/tasks/redhat_tasks.yml
+++ b/roles/base/tasks/redhat_tasks.yml
@@ -1,46 +1,23 @@
-- name: upgrade system (redhat)
-  yum:
-    update_cache: true
-    name: '*'
-    state: latest
-
 # install epel-release first to ensure the extra packages can be installed later
 - name: install epel release package (redhat)
   yum:
     name: epel-release
+    state: latest
 
-- name: install base packages (redhat)
+- name: install/upgrade base packages (redhat)
   yum:
     name: "{{ item }}"
+    update_cache: true
+    state: latest
   with_items:
     - ntp
     - unzip
     - bzip2
-    - vim
     - curl
-    - git
-    - mercurial
-    - gcc
-    - perl
     - librbd1-devel
-    - lshw
     - python-requests # XXX required by ceph repo, but it has a bad package on it
     - bash-completion
+    - kernel #keep kernel up to date
 
 - name: install and start ntp
   shell: systemctl enable ntpd
-
-- name: install python-crypto
-  yum: name=python-crypto state=present
-  register: python_crypto_result
-  ignore_errors: yes
-
-- name: remove python crypt egg file to work-around https://bugs.centos.org/view.php?id=9896&nbn=2
-  shell: rm -rf /usr/lib64/python2.7/site-packages/pycrypto-*.egg-info
-  when: '"Error unpacking rpm package python2-crypto-" in python_crypto_result.msg'
-
-- name: install ansible (redhat)
-  yum:
-    name: ansible
-    enablerepo: epel-testing
-    state: latest
diff --git a/roles/base/tasks/ubuntu_tasks.yml b/roles/base/tasks/ubuntu_tasks.yml
index 35ffdf9..4536cb9 100644
--- a/roles/base/tasks/ubuntu_tasks.yml
+++ b/roles/base/tasks/ubuntu_tasks.yml
@@ -6,25 +6,11 @@
 - name: install base packages (debian)
   apt:
     name: "{{ item }}"
+    state: latest
   with_items:
     - unzip
     - bzip2
-    - vim-nox
     - curl
     - python-software-properties
-    - git
-    - mercurial
-    - build-essential
-    - perl
     - librbd-dev
-    - lshw
     - bash-completion
-
-- name: add ansible apt repository (debian)
-  apt_repository:
-    repo: ppa:ansible/ansible
-    state: present
-    validate_certs: "{{ validate_certs }}"
-
-- name: install ansible (debian)
-  apt: name=ansible state=present
diff --git a/roles/ceph-common/handlers/main.yml b/roles/ceph-common/handlers/main.yml
index ef109b0..8080c60 100644
--- a/roles/ceph-common/handlers/main.yml
+++ b/roles/ceph-common/handlers/main.yml
@@ -3,6 +3,18 @@
   apt:
     update-cache: yes
 
+- name: check for a ceph socket
+  shell: "stat /var/run/ceph/*.asok > /dev/null 2>&1"
+  changed_when: false
+  failed_when: false
+  register: socket
+
+- name: check for a rados gateway socket
+  shell: "stat {{ rbd_client_admin_socket_path }}*.asok > /dev/null 2>&1"
+  changed_when: false
+  failed_when: false
+  register: socketrgw
+
 - name: restart ceph mons
   command: service ceph restart mon
   when:
diff --git a/roles/ceph-common/tasks/main.yml b/roles/ceph-common/tasks/main.yml
index adae3e6..9fc665c 100644
--- a/roles/ceph-common/tasks/main.yml
+++ b/roles/ceph-common/tasks/main.yml
@@ -37,18 +37,6 @@
     radosgw_frontend == 'apache' and
     rgw_group_name in group_names
 
-- name: check for a ceph socket
-  shell: "stat /var/run/ceph/*.asok > /dev/null 2>&1"
-  changed_when: false
-  failed_when: false
-  register: socket
-
-- name: check for a rados gateway socket
-  shell: "stat {{ rbd_client_admin_socket_path }}*.asok > /dev/null 2>&1"
-  changed_when: false
-  failed_when: false
-  register: socketrgw
-
 - name: create a local fetch directory if it doesn't exist
   local_action: file path={{ fetch_directory }} state=directory
   changed_when: false
@@ -86,6 +74,8 @@
     group: root
     mode: 0644
   notify:
+    - check for a ceph socket
+    - check for a rados gateway socket
     - restart ceph mons
     - restart ceph mons on ubuntu
     - restart ceph mons with systemd
diff --git a/roles/contiv_cluster/defaults/main.yml b/roles/contiv_cluster/defaults/main.yml
index 257d58e..4dcbb14 100644
--- a/roles/contiv_cluster/defaults/main.yml
+++ b/roles/contiv_cluster/defaults/main.yml
@@ -1,11 +1,15 @@
 ---
 # role variable for the cluster manager service
 
-collins_image: contiv/collins:02_25_2016
+collins_image: contiv/collins
+collins_image_version: "02_25_2016"
 collins_host_port: 9000
 collins_guest_port: 9000
 
-contiv_cluster_version: "v0.0.0-03-08-2016.00-06-26.UTC"
+clusterm_args_file: "clusterm.args"
+clusterm_conf_file: "clusterm.conf"
+
+contiv_cluster_version: "v0.0.0-03-13-2016.03-44-45.UTC"
 contiv_cluster_tar_file: "cluster-{{ contiv_cluster_version }}.tar.bz2"
 contiv_cluster_src_file: "https://github.com/contiv/cluster/releases/download/{{ contiv_cluster_version }}/{{ contiv_cluster_tar_file }}"
 contiv_cluster_dest_file: "/tmp/{{ contiv_cluster_tar_file }}"
diff --git a/roles/contiv_cluster/files/clusterm b/roles/contiv_cluster/files/clusterm
deleted file mode 100644
index 079efbd..0000000
--- a/roles/contiv_cluster/files/clusterm
+++ /dev/null
@@ -1 +0,0 @@
-CLUSTERM_ARGS=""
diff --git a/roles/contiv_cluster/files/clusterm.args b/roles/contiv_cluster/files/clusterm.args
new file mode 100644
index 0000000..5e27b80
--- /dev/null
+++ b/roles/contiv_cluster/files/clusterm.args
@@ -0,0 +1 @@
+CLUSTERM_ARGS="--config=/etc/default/clusterm/clusterm.conf"
diff --git a/roles/contiv_cluster/files/clusterm.conf b/roles/contiv_cluster/files/clusterm.conf
new file mode 100644
index 0000000..6aee63c
--- /dev/null
+++ b/roles/contiv_cluster/files/clusterm.conf
@@ -0,0 +1,3 @@
+{
+  "comment" : "empty JSON loads a default clusterm configuration. Add configuration here and restart clusterm service to load non-default configuration"
+}
diff --git a/roles/contiv_cluster/meta/main.yml b/roles/contiv_cluster/meta/main.yml
new file mode 100644
index 0000000..fd088d3
--- /dev/null
+++ b/roles/contiv_cluster/meta/main.yml
@@ -0,0 +1,5 @@
+---
+# The dependecies for cluster-mgr
+
+dependencies:
+- { role: ansible }
diff --git a/roles/contiv_cluster/tasks/main.yml b/roles/contiv_cluster/tasks/main.yml
index ca8195b..91731ed 100644
--- a/roles/contiv_cluster/tasks/main.yml
+++ b/roles/contiv_cluster/tasks/main.yml
@@ -11,8 +11,16 @@
   tags:
     - prebake-for-dev
 
+- name: check for collins image
+  shell: "docker images | grep {{ collins_image }} | grep -q {{ collins_image_version }}"
+  ignore_errors: true
+  register: collins_exists
+  tags:
+    - prebake-for-dev
+
 - name: pull collins container image
-  shell: docker pull {{ collins_image }}
+  shell: "docker pull {{ collins_image }}:{{ collins_image_version }}"
+  when: not collins_exists|success
   tags:
     - prebake-for-dev
 
@@ -33,11 +41,22 @@
     chdir: /usr/bin/
   when: download_result | changed 
 
-- name: copy environment file for clusterm
-  copy: src=clusterm dest=/etc/default/clusterm
+- name: create conf dir for clusterm
+  file:
+    name: /etc/default/clusterm/
+    state: directory
+
+- name: copy conf files for clusterm
+  copy:
+    src: "{{ item }}"
+    dest: /etc/default/clusterm/{{ item }}
+    force: yes
+  with_items:
+    - "{{ clusterm_args_file }}"
+    - "{{ clusterm_conf_file }}"
 
 - name: copy systemd units for clusterm
-  copy: src=clusterm.service dest=/etc/systemd/system/clusterm.service
+  template: src=clusterm.j2 dest=/etc/systemd/system/clusterm.service
 
 - name: start clusterm
   service: name=clusterm state=started
diff --git a/roles/contiv_cluster/files/clusterm.service b/roles/contiv_cluster/templates/clusterm.j2
similarity index 78%
rename from roles/contiv_cluster/files/clusterm.service
rename to roles/contiv_cluster/templates/clusterm.j2
index eae327f..7a7d8ee 100644
--- a/roles/contiv_cluster/files/clusterm.service
+++ b/roles/contiv_cluster/templates/clusterm.j2
@@ -3,7 +3,7 @@ Description=Clusterm
 After=auditd.service systemd-user-sessions.service time-sync.target serf.service collins.service
 
 [Service]
-EnvironmentFile=/etc/default/clusterm
+EnvironmentFile=/etc/default/clusterm/{{ clusterm_args_file }}
 ExecStart=/usr/bin/clusterm $CLUSTERM_ARGS
 Restart=on-failure
 RestartSec=10
diff --git a/roles/contiv_cluster/templates/collins.j2 b/roles/contiv_cluster/templates/collins.j2
index b0de63b..2174579 100644
--- a/roles/contiv_cluster/templates/collins.j2
+++ b/roles/contiv_cluster/templates/collins.j2
@@ -11,7 +11,7 @@ start)
     set -e
 
     /usr/bin/docker run -t -p {{ collins_host_port }}:{{ collins_guest_port }} \
-        --name collins {{ collins_image }}
+        --name collins {{ collins_image }}:{{ collins_image_version }}
     ;;
 
 stop)
diff --git a/roles/contiv_network/defaults/main.yml b/roles/contiv_network/defaults/main.yml
index 9d44582..951aaa1 100644
--- a/roles/contiv_network/defaults/main.yml
+++ b/roles/contiv_network/defaults/main.yml
@@ -7,13 +7,24 @@
 contiv_network_mode: "standalone" # Accepted values: standalone, aci
 netplugin_mode: "docker" # Accepted values: docker, kubernetes
 fwd_mode: "bridge" #Accepted values: bridge , routing
+ofnet_master_port: 9001
+ofnet_agent_port1: 9002
+ofnet_agent_port2: 9003
+netmaster_port: 9999
+gobgp_grpc_port: 8080
+bgp_port: 179
+vxlan_port: 4789
+netplugin_rule_comment: "contiv network traffic"
 
-contiv_network_version: "v0.1-03-05-2016.09-42-48.UTC"
+contiv_network_version: "v0.1-03-16-2016.13-43-59.UTC"
 contiv_network_tar_file: "netplugin-{{ contiv_network_version }}.tar.bz2"
 contiv_network_src_file: "https://github.com/contiv/netplugin/releases/download/{{ contiv_network_version }}/{{ contiv_network_tar_file }}"
 contiv_network_dest_file: "/tmp/{{ contiv_network_tar_file }}"
 
-contivctl_version: "v0.0.0-03-07-2016.23-26-25.UTC"
+contivctl_version: "v0.0.0-03-10-2016.22-13-24.UTC"
 contivctl_tar_file: "contivctl-{{ contivctl_version }}.tar.bz2"
 contivctl_src_file: "https://github.com/contiv/contivctl/releases/download/{{ contivctl_version }}/{{ contivctl_tar_file }}"
 contivctl_dest_file: "/tmp/{{ contivctl_tar_file }}"
+
+apic_epg_bridge_domain: "not_specified"
+apic_contracts_unrestricted_mode: "no"
diff --git a/roles/contiv_network/tasks/cleanup.yml b/roles/contiv_network/tasks/cleanup.yml
index 804181b..d495b0a 100644
--- a/roles/contiv_network/tasks/cleanup.yml
+++ b/roles/contiv_network/tasks/cleanup.yml
@@ -7,4 +7,22 @@
 - name: stop netplugin
   service: name=netplugin state=stopped
 
+- name: cleanup netmaster host alias
+  lineinfile:
+    dest: /etc/hosts
+    regexp: " netmaster$"
+    state: absent
+  become: true
+
+- name: cleanup iptables for contiv network control plane
+  shell: iptables -D INPUT -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ netplugin_rule_comment }} ({{ item }})"
+  become: true
+  with_items:
+    - "{{ ofnet_master_port }}"
+    - "{{ ofnet_agent_port1 }}"
+    - "{{ ofnet_agent_port2 }}"
+    - "{{ netmaster_port }}"
+    - "{{ gobgp_grpc_port }}"
+    - "{{ bgp_port }}"
+
 - include: ovs_cleanup.yml
diff --git a/roles/contiv_network/tasks/main.yml b/roles/contiv_network/tasks/main.yml
index c0a0a4d..f3e944e 100644
--- a/roles/contiv_network/tasks/main.yml
+++ b/roles/contiv_network/tasks/main.yml
@@ -17,6 +17,19 @@
 # install can be conditional based on deployment environment.
 - include: ovs.yml
 
+- name: setup iptables for contiv network control plane
+  shell: >
+      ( iptables -L INPUT | grep "{{ netplugin_rule_comment }} ({{ item }})" ) || \
+      iptables -I INPUT 1 -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ netplugin_rule_comment }} ({{ item }})"
+  become: true
+  with_items:
+    - "{{ ofnet_master_port }}"
+    - "{{ ofnet_agent_port1 }}"
+    - "{{ ofnet_agent_port2 }}"
+    - "{{ netmaster_port }}"
+    - "{{ gobgp_grpc_port }}"
+    - "{{ bgp_port }}"
+
 - name: download netmaster and netplugin
   get_url:
     validate_certs: "{{ validate_certs }}"
@@ -54,7 +67,12 @@
   shell: systemctl daemon-reload && systemctl start netplugin
 
 - name: setup netmaster host alias
-  shell: echo "{{ service_vip }} netmaster" >> /etc/hosts
+  lineinfile:
+    dest: /etc/hosts
+    line: "{{ service_vip }} netmaster"
+    regexp: " netmaster$"
+    state: present
+  become: true
 
 - name: copy environment file for netmaster
   copy: src=netmaster dest=/etc/default/netmaster
diff --git a/roles/contiv_network/tasks/ovs.yml b/roles/contiv_network/tasks/ovs.yml
index f25d061..da0e052 100644
--- a/roles/contiv_network/tasks/ovs.yml
+++ b/roles/contiv_network/tasks/ovs.yml
@@ -52,3 +52,11 @@
   with_items:
     - "tcp:127.0.0.1:6640"
     - "ptcp:6640"
+
+- name: setup iptables for vxlan vtep port
+  shell: >
+      ( iptables -L INPUT | grep "{{ netplugin_rule_comment }} ({{ item }})" ) || \
+      iptables -I INPUT 1 -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ netplugin_rule_comment }} ({{ item }})"
+  become: true
+  with_items:
+    - "{{ vxlan_port }}"
diff --git a/roles/contiv_network/tasks/ovs_cleanup.yml b/roles/contiv_network/tasks/ovs_cleanup.yml
index 2c9b0a5..4aa396d 100644
--- a/roles/contiv_network/tasks/ovs_cleanup.yml
+++ b/roles/contiv_network/tasks/ovs_cleanup.yml
@@ -24,3 +24,9 @@
   register: ports
 
 - debug: var=ports
+
+- name: cleanup iptables for vxlan vtep port
+  shell: iptables -D INPUT -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ netplugin_rule_comment }} ({{ item }})"
+  become: true
+  with_items:
+    - "{{ vxlan_port }}"
diff --git a/roles/contiv_network/templates/aci_gw.j2 b/roles/contiv_network/templates/aci_gw.j2
index 94eefe5..dc72948 100644
--- a/roles/contiv_network/templates/aci_gw.j2
+++ b/roles/contiv_network/templates/aci_gw.j2
@@ -15,6 +15,9 @@ start)
     -e "APIC_USERNAME={{ apic_username }}" \
     -e "APIC_PASSWORD={{ apic_password }}" \
     -e "APIC_LEAF_NODE={{ apic_leaf_nodes }}" \
+    -e "APIC_PHYS_DOMAIN={{ apic_phys_domain }}" \
+    -e "APIC_EPG_BRIDGE_DOMAIN={{ apic_epg_bridge_domain }}" \
+    -e "APIC_CONTRACTS_UNRESTRICTED_MODE={{ apic_contracts_unrestricted_mode }}" \
     --name=contiv-aci-gw \
     contiv/aci-gw
     ;;
diff --git a/roles/contiv_storage/defaults/main.yml b/roles/contiv_storage/defaults/main.yml
index 6a63281..afcd2c2 100644
--- a/roles/contiv_storage/defaults/main.yml
+++ b/roles/contiv_storage/defaults/main.yml
@@ -2,7 +2,7 @@
 
 # Role defaults for contiv_storage
 
-contiv_storage_version: "v0.0.0-03-07-2016.09-29-13.UTC"
+contiv_storage_version: "v0.0.0-03-19-2016.02-39-28.UTC"
 contiv_storage_tar_file: "volplugin-{{ contiv_storage_version }}.tar.bz2"
 contiv_storage_src_file: "https://github.com/contiv/volplugin/releases/download/{{ contiv_storage_version }}/{{ contiv_storage_tar_file }}"
 contiv_storage_dest_file: "/tmp/{{ contiv_storage_tar_file }}"
diff --git a/roles/dev/meta/main.yml b/roles/dev/meta/main.yml
index 517c36a..568f773 100644
--- a/roles/dev/meta/main.yml
+++ b/roles/dev/meta/main.yml
@@ -13,6 +13,7 @@
 
 dependencies:
 - { role: ceph-install, tags: 'prebake-for-dev' }
+- { role: ansible, tags: 'prebake-for-dev' }
 - { role: etcd }
 - { role: docker }
 - { role: swarm }
diff --git a/roles/dev/tasks/main.yml b/roles/dev/tasks/main.yml
index a104705..357a07b 100644
--- a/roles/dev/tasks/main.yml
+++ b/roles/dev/tasks/main.yml
@@ -13,6 +13,16 @@
 # it explicitly here
 
 
+- include: ubuntu_tasks.yml
+  when: ansible_os_family == "Debian"
+  tags:
+    - prebake-for-dev
+
+- include: redhat_tasks.yml
+  when: ansible_os_family == "RedHat"
+  tags:
+    - prebake-for-dev
+
 - include: os_agnostic_tasks.yml
   tags:
     - prebake-for-dev
diff --git a/roles/dev/tasks/os_agnostic_tasks.yml b/roles/dev/tasks/os_agnostic_tasks.yml
index 7e6e150..68c25aa 100644
--- a/roles/dev/tasks/os_agnostic_tasks.yml
+++ b/roles/dev/tasks/os_agnostic_tasks.yml
@@ -4,12 +4,13 @@
     url: https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz
     dest: /tmp/go1.6.linux-amd64.tar.gz
     force: no
+  register: download_result
 
 - name: install Golang
-  shell: tar xfvz /tmp/go1.6.linux-amd64.tar.gz
+  shell: rm -rf go/ && tar xfvz /tmp/go1.6.linux-amd64.tar.gz
   args:
     chdir: /usr/local/
-    creates: /usr/local/go/bin/go
+  when: download_result | changed
 
 - name: setup golang environment
   copy:
diff --git a/roles/dev/tasks/redhat_tasks.yml b/roles/dev/tasks/redhat_tasks.yml
new file mode 100644
index 0000000..6d81bf3
--- /dev/null
+++ b/roles/dev/tasks/redhat_tasks.yml
@@ -0,0 +1,11 @@
+- name: install/upgrade base packages (redhat)
+  yum:
+    name: "{{ item }}"
+    update_cache: true
+    state: latest
+  with_items:
+    - vim
+    - git
+    - mercurial
+    - gcc
+    - perl
diff --git a/roles/dev/tasks/ubuntu_tasks.yml b/roles/dev/tasks/ubuntu_tasks.yml
new file mode 100644
index 0000000..193cccb
--- /dev/null
+++ b/roles/dev/tasks/ubuntu_tasks.yml
@@ -0,0 +1,9 @@
+- name: install base packages (debian)
+  apt:
+    name: "{{ item }}"
+  with_items:
+    - vim-nox
+    - git
+    - mercurial
+    - build-essential
+    - perl
diff --git a/roles/docker/defaults/main.yml b/roles/docker/defaults/main.yml
index 1d767ae..ea50e66 100644
--- a/roles/docker/defaults/main.yml
+++ b/roles/docker/defaults/main.yml
@@ -2,5 +2,7 @@
 # Default values for docker role
 
 docker_api_port: 2385
-
 docker_version: 1.9.1
+docker_rule_comment: "docker api"
+docker_device: ""
+docker_device_size: "10000MB"
diff --git a/roles/docker/tasks/cleanup.yml b/roles/docker/tasks/cleanup.yml
index 4b7548f..0780220 100644
--- a/roles/docker/tasks/cleanup.yml
+++ b/roles/docker/tasks/cleanup.yml
@@ -6,3 +6,9 @@
 
 - name: stop docker tcp socket
   service: name=docker-tcp.socket state=stopped
+
+- name: cleanup iptables for docker
+  shell: iptables -D INPUT -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ docker_rule_comment }} ({{ item }})"
+  become: true
+  with_items:
+    - "{{ docker_api_port }}"
diff --git a/roles/docker/tasks/create_docker_device.yml b/roles/docker/tasks/create_docker_device.yml
new file mode 100644
index 0000000..8568da2
--- /dev/null
+++ b/roles/docker/tasks/create_docker_device.yml
@@ -0,0 +1,29 @@
+---
+
+- name: pvcreate check for {{ docker_device }}
+  shell: "pvdisplay {{ docker_device }}"
+  register: pvcreated
+  ignore_errors: true
+
+- name: pvcreate {{ docker_device }}
+  shell: "pvcreate {{ docker_device }}"
+  when: pvcreated|failed
+
+- name: vgcreate check for {{ docker_device }}
+  shell: "vgdisplay contiv"
+  register: vgcreated
+  ignore_errors: true
+
+- name: vgcreate contiv
+  shell: "vgcreate contiv {{ docker_device }}"
+  when: vgcreated|failed
+
+- name: lvcreate check for {{ docker_device }}
+  shell: "lvdisplay contiv | grep -q dockerthin"
+  register: lvcreated
+  ignore_errors: true
+
+- name: lvcreate contiv-dockerthin
+  shell: lvcreate -n dockerthin -T contiv --size {{ docker_device_size }}
+  when: lvcreated|failed
+  register: thin_provisioned
diff --git a/roles/docker/tasks/main.yml b/roles/docker/tasks/main.yml
index 45a53eb..06d80ec 100644
--- a/roles/docker/tasks/main.yml
+++ b/roles/docker/tasks/main.yml
@@ -1,5 +1,6 @@
 ---
 # This role contains tasks for configuring and starting docker service
+#
 
 - name: check docker version
   shell: docker --version
@@ -36,6 +37,14 @@
   tags:
     - prebake-for-dev
 
+- name: setup iptables for docker
+  shell: >
+      ( iptables -L INPUT | grep "{{ docker_rule_comment }} ({{ item }})" ) || \
+      iptables -I INPUT 1 -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ docker_rule_comment }} ({{ item }})"
+  become: true
+  with_items:
+    - "{{ docker_api_port }}"
+
 - name: copy systemd units for docker(enable cluster store) (debian)
   template: src=docker-svc.j2 dest=/lib/systemd/system/docker.service
   when: ansible_os_family == "Debian"
@@ -46,19 +55,36 @@
 
 - name: check docker-tcp socket state
   shell: systemctl status docker-tcp.socket | grep 'Active.*active' -o
+  ignore_errors: true
   register: docker_tcp_socket_state
 
+- include: create_docker_device.yml
+  when: docker_device != ""
+
 - name: copy systemd units for docker tcp socket settings
   template: src=docker-tcp.j2 dest=/etc/systemd/system/docker-tcp.socket
   register: docker_tcp_socket
 
   # tcp socket service requires docker service to be started after it
-- name: start docker tcp socket service
-  shell: sudo systemctl daemon-reload && sudo systemctl stop docker && sudo systemctl start docker-tcp.socket && sudo systemctl start docker
+- name: reload systemd configuration
+  shell: sudo systemctl daemon-reload
+  when: "(docker_tcp_socket | changed) or (docker_tcp_socket_state.stdout != 'Active: active')"
+
+- name: stop docker
+  service:
+    name: docker
+    state: stopped
+  when: "(docker_tcp_socket | changed) or (docker_tcp_socket_state.stdout != 'Active: active')"
+
+- name: start docker-tcp service
+  service:
+    name: docker-tcp.socket
+    state: started
   when: "(docker_tcp_socket | changed) or (docker_tcp_socket_state.stdout != 'Active: active')"
 
 - name: check docker service state
   shell: sudo systemctl status docker | grep 'Active.*active' -o
+  ignore_errors: true
   register: docker_service_state
   tags:
     - prebake-for-dev
@@ -69,13 +95,38 @@
 
 # XXX: service module doesn't do daemon-reload yet, so need to use shell module here
 # https://github.com/ansible/ansible-modules-core/issues/191
-- name: restart docker
+- name: reload docker systemd configuration
   #service: name=docker state=restarted
-  shell: sudo systemctl daemon-reload && sudo systemctl restart docker
+  shell: sudo systemctl daemon-reload
   when: "(docker_service_state.stderr | match('.*docker.service changed on disk.*')) or (docker_service_state.stdout != 'Active: active')"
   tags:
     - prebake-for-dev
 
+# XXX: this needs to happen twice after setting up the thin provisioner because
+#      of some docker bug I've not investigated.
+- name: restart docker (first time)
+  service:
+    name: docker
+    state: restarted
+  when: thin_provisioned|changed
+  ignore_errors: true
+  tags:
+    - prebake-for-dev
+
+- name: ensure docker is started
+  service:
+    name: docker
+    state: started
+  tags:
+    - prebake-for-dev
+
+- stat: path=/var/docker_images
+  register: docker_images
+
+- name: Import saved docker images
+  shell: set -e; cd /var/docker_images; images=$(ls); for i in $images; do docker load -i $i; rm $i; done
+  when: docker_images.stat.isdir is defined and docker_images.stat.isdir
+
 - name: check docker-compose version
   shell: docker-compose --version
   register: docker_compose_version
diff --git a/roles/docker/templates/docker-svc.j2 b/roles/docker/templates/docker-svc.j2
index 8511784..4a1d87f 100644
--- a/roles/docker/templates/docker-svc.j2
+++ b/roles/docker/templates/docker-svc.j2
@@ -6,7 +6,11 @@ Requires=docker.socket
 
 [Service]
 Type=notify
-ExecStart=/usr/bin/docker daemon -s overlay -H fd:// --cluster-store=etcd://localhost:{{ etcd_client_port1 }}
+{% if docker_device != "" %}
+ExecStart=/usr/bin/docker daemon --storage-opt dm.thinpooldev=/dev/mapper/contiv-dockerthin -H fd:// --cluster-store=etcd://localhost:{{ etcd_client_port1 }}
+{% else %}
+ExecStart=/usr/bin/docker daemon -H fd:// --cluster-store=etcd://localhost:{{ etcd_client_port1 }}
+{% endif %}
 MountFlags=slave
 LimitNOFILE=1048576
 LimitNPROC=1048576
diff --git a/roles/docker/vars/main.yml b/roles/docker/vars/main.yml
new file mode 100644
index 0000000..21e7e7d
--- /dev/null
+++ b/roles/docker/vars/main.yml
@@ -0,0 +1,2 @@
+---
+thin_provisioned: false
diff --git a/roles/etcd/defaults/main.yml b/roles/etcd/defaults/main.yml
index fdaa933..9f38316 100644
--- a/roles/etcd/defaults/main.yml
+++ b/roles/etcd/defaults/main.yml
@@ -6,9 +6,9 @@ etcd_client_port2: 4001
 etcd_peer_port1: 2380
 etcd_peer_port2: 7001
 etcd_peers_group: "service-master"
-etcd_peer_interface: "{{ control_interface }}"
 etcd_init_cluster: true
 etcd_tmp_filename: "/tmp/etcd.existing"
+etcd_rule_comment: "etcd traffic"
 
 # following variables are used in one or more roles, but have no good default value to pick from.
 # Leaving them as commented so that playbooks can fail early with variable not defined error.
diff --git a/roles/etcd/tasks/cleanup.yml b/roles/etcd/tasks/cleanup.yml
index e505daa..73cda0d 100644
--- a/roles/etcd/tasks/cleanup.yml
+++ b/roles/etcd/tasks/cleanup.yml
@@ -6,3 +6,12 @@
 
 - name: remove the temp etcd file
   file: name={{ etcd_tmp_filename }} state=absent
+
+- name: cleanup iptables for etcd
+  shell: iptables -D INPUT -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ etcd_rule_comment }} ({{ item }})"
+  become: true
+  with_items:
+    - "{{ etcd_client_port1 }}"
+    - "{{ etcd_client_port2 }}"
+    - "{{ etcd_peer_port1 }}"
+    - "{{ etcd_peer_port2 }}"
diff --git a/roles/etcd/tasks/main.yml b/roles/etcd/tasks/main.yml
index 4f6a4d3..9d4bb34 100644
--- a/roles/etcd/tasks/main.yml
+++ b/roles/etcd/tasks/main.yml
@@ -15,6 +15,17 @@
   tags:
     - prebake-for-dev
 
+- name: setup iptables for etcd
+  shell: >
+      ( iptables -L INPUT | grep "{{ etcd_rule_comment }} ({{ item }})" ) || \
+      iptables -I INPUT 1 -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ etcd_rule_comment }} ({{ item }})"
+  become: true
+  with_items:
+    - "{{ etcd_client_port1 }}"
+    - "{{ etcd_client_port2 }}"
+    - "{{ etcd_peer_port1 }}"
+    - "{{ etcd_peer_port2 }}"
+
 # The second part of the condition avoids reconfiguring master if it was already present in the host-group
 - name: copy the etcd start/stop script
   template: src=etcd.j2 dest=/usr/bin/etcd.sh mode=u=rwx,g=rx,o=rx
@@ -25,3 +36,5 @@
 
 - name: start etcd
   service: name=etcd state=started
+
+
diff --git a/roles/etcd/templates/etcd.j2 b/roles/etcd/templates/etcd.j2
index dbb75bb..1635a94 100644
--- a/roles/etcd/templates/etcd.j2
+++ b/roles/etcd/templates/etcd.j2
@@ -41,9 +41,9 @@ start)
         export ETCD_INITIAL_CLUSTER="
         {%- for host in groups[etcd_peers_group] -%}
         {%- if loop.last -%}
-        {{ hostvars[host]['inventory_hostname'] }}=http://{{ hostvars[host]['ansible_' + etcd_peer_interface]['ipv4']['address'] }}:{{ etcd_peer_port1 }},{{ hostvars[host]['inventory_hostname'] }}=http://{{ hostvars[host]['ansible_' + etcd_peer_interface]['ipv4']['address'] }}:{{ etcd_peer_port2 }}
+        {{ hostvars[host]['inventory_hostname'] }}=http://{{ hostvars[host]['ansible_' + hostvars[host]['control_interface']]['ipv4']['address'] }}:{{ etcd_peer_port1 }},{{ hostvars[host]['inventory_hostname'] }}=http://{{ hostvars[host]['ansible_' + hostvars[host]['control_interface']]['ipv4']['address'] }}:{{ etcd_peer_port2 }}
         {%- else -%}
-        {{ hostvars[host]['inventory_hostname'] }}=http://{{ hostvars[host]['ansible_' + etcd_peer_interface]['ipv4']['address'] }}:{{ etcd_peer_port1 }},{{ hostvars[host]['inventory_hostname'] }}=http://{{ hostvars[host]['ansible_' + etcd_peer_interface]['ipv4']['address'] }}:{{ etcd_peer_port2 }},
+        {{ hostvars[host]['inventory_hostname'] }}=http://{{ hostvars[host]['ansible_' + hostvars[host]['control_interface']]['ipv4']['address'] }}:{{ etcd_peer_port1 }},{{ hostvars[host]['inventory_hostname'] }}=http://{{ hostvars[host]['ansible_' + hostvars[host]['control_interface']]['ipv4']['address'] }}:{{ etcd_peer_port2 }},
         {%- endif -%}
         {% endfor -%}
         "
@@ -59,7 +59,7 @@ start)
             {%- endif %}
         {% endfor -%}
         {% if peers %}
-            {% set peer_addr=hostvars[peers[0]]['ansible_' + etcd_peer_interface]['ipv4']['address'] -%}
+            {% set peer_addr=hostvars[peers[0]]['ansible_' + hostvars[peers[0]]['control_interface']]['ipv4']['address'] -%}
             {{ add_member(peer_addr=peer_addr) }}
         {%- else -%}
         {# This condition shall not arise, so fail early #}
diff --git a/roles/serf/files/serf.service b/roles/serf/files/serf.service
index 4b4e874..d502b4c 100644
--- a/roles/serf/files/serf.service
+++ b/roles/serf/files/serf.service
@@ -4,6 +4,7 @@ After=auditd.service systemd-user-sessions.service time-sync.target
 
 [Service]
 ExecStart=/usr/bin/serf.sh start
+ExecStop=/usr/bin/serf.sh stop
 Restart=on-failure
 RestartSec=10
 KillMode=control-group
diff --git a/roles/serf/tasks/main.yml b/roles/serf/tasks/main.yml
index 3d525f8..3e8cffb 100644
--- a/roles/serf/tasks/main.yml
+++ b/roles/serf/tasks/main.yml
@@ -1,6 +1,23 @@
 ---
 # This role contains tasks for configuring and starting serf service
 
+- name: install lshw (debian)
+  apt:
+    name: "{{ item }}"
+    state: latest
+  with_items:
+    - lshw
+  when: ansible_os_family == "Debian"
+
+- name: install/upgrade base packages (redhat)
+  yum:
+    name: "{{ item }}"
+    update_cache: true
+    state: latest
+  with_items:
+    - lshw
+  when: ansible_os_family == "RedHat"
+
 - name: download serf binary
   get_url:
     validate_certs: "{{ validate_certs }}"
diff --git a/roles/serf/templates/serf.j2 b/roles/serf/templates/serf.j2
index 162341a..61edcf4 100644
--- a/roles/serf/templates/serf.j2
+++ b/roles/serf/templates/serf.j2
@@ -6,19 +6,38 @@ if [ $# -ne 1 ]; then
     exit 1
 fi
 
+{% set mdns_sport_comment="'serf discovery sport'" -%}
+{%- set mdns_sport_rule="-p udp --sport 5353 -i " + 
+        serf_discovery_interface + 
+        " -j ACCEPT -m comment --comment " +
+        mdns_sport_comment -%}
+{%- set mdns_dport_comment="'serf discovery dport'" -%}
+{%- set mdns_dport_rule="-p udp --dport 5353 -i " +
+        serf_discovery_interface +
+        " -j ACCEPT -m comment --comment " +
+        mdns_dport_comment -%}
+{%- set serf_tcp_comment="'serf control'" -%}
+{%- set serf_tcp_rule="-p tcp --dport 7946 -i " +
+        serf_discovery_interface +
+        " -j ACCEPT -m comment --comment " +
+        serf_tcp_comment -%}
+
 case $1 in
 start)
     # fail on error
     set -e
 
-    # install necessary iptables to let mdns work
-    # XXX: the interface name should be discovered
-    echo setting up iptables for mdns
-    iptables -I INPUT -p udp --dport 5353 -i {{ serf_discovery_interface }}  -j ACCEPT && \
-    iptables -I INPUT -p udp --sport 5353 -i {{ serf_discovery_interface }}  -j ACCEPT
+    # install necessary iptables to let serf work
+    echo setting up iptables for serf
+    ( /sbin/iptables -L INPUT | grep {{ mdns_sport_comment }} || \
+        /sbin/iptables -I INPUT 1 {{ mdns_sport_rule }} )
+    ( /sbin/iptables -L INPUT | grep {{ mdns_dport_comment }} || \
+        /sbin/iptables -I INPUT 1 {{ mdns_dport_rule }} )
+    ( /sbin/iptables -L INPUT | grep {{ serf_tcp_comment }} || \
+        /sbin/iptables -I INPUT 1 {{ serf_tcp_rule }} )
 
     echo starting serf
-    label=$(hostname)
+    label=$(hostname -s)
     serial=$(lshw -c system | grep serial | awk '{print $2}')
     addr=$(ip addr list dev {{ serf_discovery_interface }} | \
             grep inet | grep {{ serf_discovery_interface }} | \
@@ -29,12 +48,19 @@ start)
     fi
 
     # start serf
-    serf agent -discover mycluster -iface eth1 \
+    /usr/bin/serf agent -node="$label-$serial" -discover mycluster -iface {{ serf_discovery_interface }} \
         -tag NodeLabel=$label \
         -tag NodeSerial=$serial \
         -tag NodeAddr=$addr
     ;;
 
+stop)
+    # cleanup iptables
+    /sbin/iptables -D INPUT {{ mdns_sport_rule }}
+    /sbin/iptables -D INPUT {{ mdns_dport_rule }}
+    /sbin/iptables -D INPUT {{ serf_tcp_rule }}
+    ;;
+
 *)
     echo USAGE: $usage
     exit 1
diff --git a/roles/swarm/defaults/main.yml b/roles/swarm/defaults/main.yml
index eedb394..6db3841 100644
--- a/roles/swarm/defaults/main.yml
+++ b/roles/swarm/defaults/main.yml
@@ -3,3 +3,4 @@
 #
 swarm_api_port: 2375
 swarm_version: "1.1.2"
+swarm_rule_comment: "swarm traffic"
diff --git a/roles/swarm/tasks/cleanup.yml b/roles/swarm/tasks/cleanup.yml
index 85707ce..0d23cd6 100644
--- a/roles/swarm/tasks/cleanup.yml
+++ b/roles/swarm/tasks/cleanup.yml
@@ -3,3 +3,9 @@
 
 - name: stop swarm
   service: name=swarm state=stopped
+
+- name: cleanup iptables for swarm
+  shell: iptables -D INPUT -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ swarm_rule_comment }} ({{ item }})"
+  become: true
+  with_items:
+    - "{{ swarm_api_port }}"
diff --git a/roles/swarm/tasks/main.yml b/roles/swarm/tasks/main.yml
index 5ce5ab2..8b27e33 100644
--- a/roles/swarm/tasks/main.yml
+++ b/roles/swarm/tasks/main.yml
@@ -1,10 +1,25 @@
 ---
 # This role contains tasks for configuring and starting swarm service
+- name: check for swarm image
+  shell: "docker images | grep swarm | grep -q {{ swarm_version }}"
+  ignore_errors: true
+  register: swarm_exists
+  tags:
+    - prebake-for-dev
 
 - name: download swarm container image
   shell: docker pull swarm:{{ swarm_version }}
   tags:
     - prebake-for-dev
+  when: not swarm_exists|success
+
+- name: setup iptables for swarm
+  shell: >
+      ( iptables -L INPUT | grep "{{ swarm_rule_comment }} ({{ item }})" ) || \
+      iptables -I INPUT 1 -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ swarm_rule_comment }} ({{ item }})"
+  become: true
+  with_items:
+    - "{{ swarm_api_port }}"
 
 - name: copy the swarm start/stop script
   template: src=swarm.j2 dest=/usr/bin/swarm.sh mode=u=rwx,g=rx,o=rx
diff --git a/roles/ucarp/files/ucarp/vip_up.sh b/roles/ucarp/files/ucarp/vip_up.sh
index 5385199..46327a6 100644
--- a/roles/ucarp/files/ucarp/vip_up.sh
+++ b/roles/ucarp/files/ucarp/vip_up.sh
@@ -13,7 +13,6 @@ vip=$2
     
 /sbin/ip link add name ${intf}_0 type dummy
 
-# XXX: the subnet needs to be derived from underlying parent interface
-/sbin/ip addr add ${vip}/24 dev ${intf}_0
+/sbin/ip addr add ${vip} dev ${intf}_0
 
 /sbin/ip link set dev ${intf}_0 up
diff --git a/roles/ucp/defaults/main.yml b/roles/ucp/defaults/main.yml
index 80c0c32..6f39317 100644
--- a/roles/ucp/defaults/main.yml
+++ b/roles/ucp/defaults/main.yml
@@ -8,7 +8,14 @@ ucp_instance_id_file: "ucp-instance-id"
 ucp_fingerprint_file: "ucp-fingerprint"
 ucp_fifo_file: "ucp-fifo"
 ucp_bootstrap_node_name: ""
-
 ucp_admin_user: "admin"
 ucp_admin_password: "orca"
 ucp_controller_replica: "--replica"
+ucp_rule_comment: "ucp traffic"
+ucp_port1: "12376"
+ucp_port2: "12379"
+ucp_port3: "12380"
+ucp_port4: "12381"
+ucp_port5: "12382"
+ucp_swarm_port: "2376"
+ucp_controller_port: "443"
diff --git a/roles/ucp/tasks/cleanup.yml b/roles/ucp/tasks/cleanup.yml
index 0cd867c..b7258a9 100644
--- a/roles/ucp/tasks/cleanup.yml
+++ b/roles/ucp/tasks/cleanup.yml
@@ -10,3 +10,15 @@
     - "{{ ucp_fingerprint_file }}"
     - "{{ ucp_instance_id_file }}"
     - "{{ ucp_fifo_file }}"
+
+- name: cleanup iptables for ucp
+  shell: iptables -D INPUT -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ ucp_rule_comment }} ({{ item }})"
+  become: true
+  with_items:
+    - "{{ ucp_port1 }}"
+    - "{{ ucp_port2 }}"
+    - "{{ ucp_port3 }}"
+    - "{{ ucp_port4 }}"
+    - "{{ ucp_port5 }}"
+    - "{{ ucp_swarm_port }}"
+    - "{{ ucp_controller_port }}"
diff --git a/roles/ucp/tasks/main.yml b/roles/ucp/tasks/main.yml
index 0519b18..5cd4dea 100644
--- a/roles/ucp/tasks/main.yml
+++ b/roles/ucp/tasks/main.yml
@@ -9,6 +9,20 @@
         docker/ucp \
         images --image-version={{ ucp_version }}
 
+- name: setup iptables for ucp
+  shell: >
+      ( iptables -L INPUT | grep "{{ ucp_rule_comment }} ({{ item }})" ) || \
+      iptables -I INPUT 1 -p tcp --dport {{ item }} -j ACCEPT -m comment --comment "{{ ucp_rule_comment }} ({{ item }})"
+  become: true
+  with_items:
+    - "{{ ucp_port1 }}"
+    - "{{ ucp_port2 }}"
+    - "{{ ucp_port3 }}"
+    - "{{ ucp_port4 }}"
+    - "{{ ucp_port5 }}"
+    - "{{ ucp_swarm_port }}"
+    - "{{ ucp_controller_port }}"
+
 - name: copy the ucp files to worker nodes
   copy:
     src: "{{ ucp_local_dir }}/{{ item }}"
diff --git a/roles/ucp/templates/ucp.j2 b/roles/ucp/templates/ucp.j2
index bc12b34..128587f 100644
--- a/roles/ucp/templates/ucp.j2
+++ b/roles/ucp/templates/ucp.j2
@@ -16,8 +16,8 @@ start)
     out=$(/usr/bin/docker run --rm -t --name ucp \
         -v /var/run/docker.sock:/var/run/docker.sock \
         docker/ucp install --host-address={{ node_addr }} \
-          --image-version={{ ucp_version }})
-    echo ${out}
+          --swarm-port={{ ucp_swarm_port }} --controller-port={{ ucp_controller_port }} \
+          --image-version={{ ucp_version }} | tee /dev/stdout)
 
     # copy out the instance ID
     instanceId=$(echo ${out} | egrep -o 'UCP instance ID: [a-zA-Z0-9:_]*' | \
@@ -42,6 +42,7 @@ start)
         -v /var/run/docker.sock:/var/run/docker.sock \
         -e UCP_ADMIN_USER={{ ucp_admin_user }} -e UCP_ADMIN_PASSWORD={{ ucp_admin_password }} \
         docker/ucp join --host-address={{ node_addr }} \
+          --swarm-port={{ ucp_swarm_port }} --controller-port={{ ucp_controller_port }} \
           --image-version={{ ucp_version }} \
           --url="https://{{ service_vip }}:443" \
     {% if run_as == "master" -%}
diff --git a/site.yml b/site.yml
index 9d5fad5..b057d64 100644
--- a/site.yml
+++ b/site.yml
@@ -59,11 +59,11 @@
   - { role: ucarp }
   - { role: docker }
   - { role: etcd, run_as: master }
-  #- { role: ceph-mon, mon_group_name: service-master }
-  #- { role: ceph-osd, mon_group_name: service-master, osd_group_name: service-master }
-  - { role: scheduler_stack, run_as: master }
-  - { role: contiv_network, run_as: master }
-  #- { role: contiv_storage, run_as: master }
+#  - { role: ceph-mon, mon_group_name: service-master, when: host_capability|match('.*can-run-user-containers.*') }
+#  - { role: ceph-osd, mon_group_name: service-master, osd_group_name: service-master, when: host_capability|match('.*storage.*') }
+  - { role: scheduler_stack, run_as: master, when: host_capability|match('.*can-run-user-containers.*') }
+  - { role: contiv_network, run_as: master, when: host_capability|match('.*can-run-user-containers.*') }
+#  - { role: contiv_storage, run_as: master }
 
 # service-worker hosts correspond to cluster machines that run the worker/driver
 # logic of the infra services.
@@ -74,10 +74,10 @@
   - { role: base }
   - { role: docker }
   - { role: etcd, run_as: worker }
-  #- { role: ceph-osd, mon_group_name: service-master, osd_group_name: service-worker }
-  - { role: scheduler_stack, run_as: worker }
-  - { role: contiv_network, run_as: worker }
-  #- { role: contiv_storage, run_as: worker }
+#  - { role: ceph-osd, mon_group_name: service-master, osd_group_name: service-worker, when: host_capability|match('.*storage.*') }
+  - { role: scheduler_stack, run_as: worker, when: host_capability|match('.*can-run-user-containers.*') }
+  - { role: contiv_network, run_as: worker, when: host_capability|match('.*can-run-user-containers.*') }
+#  - { role: contiv_storage, run_as: worker }
 
 # netplugin-node hosts set up netmast/netplugin in a cluster
 - hosts: netplugin-node