Common iso base and appliance

This creates a common iso base for the appliance and the linux bootstrap.
This needs to be reconnciled with the bootstrap approach but I needed
something immediate and this sufficed.
There is a quirk for some reason extracting the ISO from docker using
stdout:

$ docker run -it test-iso cat /tmp/iso > binary/appliance.iso
$ md5sum binary/appliance.iso
0cf11aa283f3b390e9760fa1c46683ca  binary/appliance.iso
$ docker run -it test-iso md5sum /tmp/iso
b631e1dbb6e1c6178154cb06ba99a5bd  /tmp/iso

Author: hickeng

.drone.sec

@@ -1 +1 @@
-eyJhbGciOiJSU0EtT0FFUCIsImVuYyI6IkExMjhHQ00ifQ.J9nAelahosD21RS7Qw0pysUSXZbajQkFPgH92SiuiMa1vE-TuRhscZE7z8RBtH67cVPC-LbkbzbojsKKR1q6CsVgqTfEuO6AXvWKedoi_OZM1xAXezJ64EH41QMwb04pboavREFUq30zL8jecjXYdSVAUmnpQHempDK_XtP3cCIyqU0JR51U7mGH4WUZyHydT9KO2OOYRp1iJjuhPIYbjMQb0j3Csi72Nsh65zRUZxGBBPPAefL_hTP81cc67PZMvwB-aHKrTXvq16HhjpyoUh60Z5Q-Rgs-p1vNOKEZ-q0AHf3ev5lDgOv4U5-nRJngSbZ7O27_cm_GmnZPSkE1Pw.AklVRBWFoyNKjE9O.ZlMOcN3UeB4Q_INk3KtCcdqUMRqOSfICqzRHfUvvPj6Y3eG9HSd9_bU8mZWIOqf5N-IfYJVjoEmX1VeyIkkGcBcUkf4bNtksy1y2YkYALcWJ2zhmhB8dSAfAiZFmj5OjBiEblFCub6F1rQgYkWGXrS9p5Hgbs7Mdtzs2vGyBzcYDoVJmqArMV6bAwvOiUH9Q4PnukqtBTsf51kDvSOT2OaJJpFOOEx4aBipz4K1lDE8nCULdEWyv5we3LdvOgOhP2cmoor8Pd0Gv9IxlwZtK6A2vTcRFj7vY.Hip-81Mo6BsINphL1R6c5g
+eyJhbGciOiJSU0EtT0FFUCIsImVuYyI6IkExMjhHQ00ifQ.St23O7dTFvvOhS1L9Ox25cv2GmmCbhDC0tl1AlONuBjGbWubI0ZcUeYybvvhfn_vNypJWMjT4N0vdHWmYS-xhj3bhNz94de_f6tCMz7Rs154snPzUVzrzghorxpo9vgHne0DwaVPg5F_OJxh8sc5HHzkhPQGjXTFl9c9t9CJJ5yJTXx7GPlpv8saqdKJvu-UTcuepL5iqDLXJp1UIhjZjiZ_Gv6MDPwH8cf6ZZbrsK_PrH6T6W3uwPm2ne7EhOMgtRI0IYTPackRq_MjHuBzzz4YeuCnVpVT1RzU0Y0kXrWeGRrbQ_MOwjO-nKffhVYbatzFzHQ2bKCKeHGGbfc1Lw.tqobw_Kn7c5VBX0U.k2MZ1aOCeGhhXvnFul2nb3sYw_rM8hzEsgCk5ZahNt7EMbyaOADohHfXNlkLgEOFUBsdfux5RkMawnodIvZ2oyNJ5YbPkoOD4fK9ZRyPQ-WL6IYOkAIndUPo54NDH5qFiI8uzqqE0razcF1IHu4cIOACVIt-OcHDkfpZpn21QNmbDonxOF3RUBWvwNjurI72QuMzSvzSQ9ncBHoHfK-1bhDud4cMk3noNR9H3taj67w-D_4mLM0YW6DGgu0IckoedXajIbLU3ya1mOoTcwq0lXIbROyCVaX11jVXSp7iQF09AmoPub0mf_YODkUInR1AVhkTOdyT7durPPKAkux7qQwIswXo-XPgpmy3RnUgSDdY-5HGOM33-RefyAqh.oUhZJ0hoLz26PPkChLWEHg

.drone.yml

@@ -6,33 +6,71 @@
 clone:
   path: github.com/vmware/vic
   tags: true
+
 build:
+  components:
+    image: golang:1.6.0
+    pull: true
+    environment:
+      BIN: bin
+      SHELL: /bin/bash
+    commands:
+      - make components
+      - make install
+
+  iso-base:
+    image: golang:1.6.0
+    pull: true
+    environment:
+      BIN: bin
+    commands:
+      - make iso-base
+
+  appliance-staging:
+    image: golang:1.6.0
+    pull: true
+    environment:
+      BIN: bin
+    commands:
+      - make appliance-staging
+
+  appliance:
+    image: golang:1.6.0
+    pull: true
+    environment:
+      BIN: bin
+      INIT: /lib/systemd/systemd
+    commands:
+      - make appliance
+
   test:
     image: golang:1.6.0
     pull: true
     environment:
+      BIN: bin
       GOPATH: /drone
       SHELL: /bin/bash
       DOCKER_API_VERSION: "1.21"
-      VIC_ESX_TEST_URL: $$VIC_ESX_TEST_URL
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
     commands:
       - wget -q https://get.docker.com/builds/Linux/x86_64/docker-latest
       - chmod +x docker-latest
       - mv docker-latest /usr/local/bin/docker
-      - make all
       - make test
       - make integration-tests
+    when:
+      success: true
+
   bundle:
     image: golang:1.6.0
     pull: true
     environment:
+      BIN: bin
       GOPATH: /drone
       SHELL: /bin/bash
     commands:
-      - make all
-      - tar -cvf vic_binaries_$$BUILD_NUMBER.tar.gz binary
+      - tar -cvf $BIN/vic_$$BUILD_NUMBER.tar.gz $BIN/appliance.iso $BIN/install.sh
     when:
       success: true
 
@@ -48,15 +86,21 @@ publish:
     username: $$BINTRAY_USERNAME
     api_key: $$BINTRAY_API_KEY
     artifacts:
-      - file: vic_binaries_$$BUILD_NUMBER.tar.gz
+      - file: vic_$$BUILD_NUMBER.tar.gz
         owner: vmware
         type: executable
         repository: vic-repo
         package: build
         version: 0.1
-        target: vic_binaries_$$BUILD_NUMBER.tar.gz
+        target: vic_$$BUILD_NUMBER.tar.gz
         publish: true
     when:
       repo: vmware/vic
       branch: master
       success: true
+
+cache:
+  mount:
+    - /drone/bin
+    - bin
+

.gitignore

@@ -3,10 +3,14 @@
 binary/
 vic_binaries*.tar.gz
 secrets.yml
+bin/
 
 bootstrap/targets/linux/tether/
 imagec/imagec
 
+tests/helpers/bats-assert
+tests/helpers/bats-support
+
 apiservers/docker/cmd
 apiservers/docker/models
 apiservers/docker/restapi/*.go

Dockerfile.integration-tests

@@ -20,15 +20,15 @@ RUN apt-get update && apt-get upgrade -y
 RUN apt-get install -yq jq
 
 # build imagec and port-layer-server
-ADD ./ /go/src/github.com/vmware/vic
-RUN make -C /go/src/github.com/vmware/vic imagec portlayerapi
+ADD ./ $GOPATH/src/github.com/vmware/vic
+RUN make -C $GOPATH/src/github.com/vmware/vic imagec portlayerapi
 
 # download and install bats and helpers
 RUN git clone https://github.com/sstephenson/bats /tmp/bats && \
-        git clone https://github.com/ztombol/bats-assert /go/src/github.com/vmware/vic/tests/helpers/bats-assert && \
-        git clone https://github.com/ztombol/bats-support /go/src/github.com/vmware/vic/tests/helpers/bats-support && \
+        git clone https://github.com/ztombol/bats-assert $GOPATH/src/github.com/vmware/vic/tests/helpers/bats-assert && \
+        git clone https://github.com/ztombol/bats-support $GOPATH/src/github.com/vmware/vic/tests/helpers/bats-support && \
         /tmp/bats/install.sh /usr/local
 
 # run the actual tests
-WORKDIR /go/src/github.com/vmware/vic/tests/
+WORKDIR $GOPATH/src/github.com/vmware/vic/tests/
 CMD /usr/local/bin/bats -t .

Makefile

@@ -6,35 +6,47 @@ ifeq ($(USER),vagrant)
 	BIN_ARCH := -$(OS)
 endif
 
+BASE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+
+BIN ?= bin
+IGNORE := $(shell mkdir -p $(BIN))
 export GOPATH ?= $(shell echo $(CURDIR) | sed -e 's,/src/.*,,')
 SWAGGER ?= $(GOPATH)/bin/swagger$(BIN_ARCH)
 GOVET ?= $(GOPATH)/bin/vet$(BIN_ARCH)
 GOIMPORTS ?= $(GOPATH)/bin/goimports$(BIN_ARCH)
 GOLINT ?= $(GOPATH)/bin/golint$(BIN_ARCH)
 GVT ?= $(GOPATH)/bin/gvt$(BIN_ARCH)
 
-
 .PHONY: all tools clean test check \
 	goversion govet goimports gvt gopath \
-	bootstrap apiservers  \
-
+	isos tethers apiservers
 
 .DEFAULT_GOAL := all
 
 
 # target aliases - environment variable definition
-docker-engine-api := binary/docker-engine-server
-
-portlayerapi := binary/port-layer-server
+docker-engine-api := $(BIN)/docker-engine-server
+portlayerapi := $(BIN)/port-layer-server
 portlayerapi-client := apiservers/portlayer/client/port_layer_client.go
 portlayerapi-server := apiservers/portlayer/restapi/server.go
 
-imagec := binary/imagec
-vicadmin := binary/vicadmin
-rpctool := binary/rpctool
+imagec := $(BIN)/imagec
+vicadmin := $(BIN)/vicadmin
+rpctool := $(BIN)/rpctool
+
+tether-linux := $(BIN)/tether-linux
+tether-windows := $(BIN)/tether-windows.exe
+
+appliance := $(BIN)/appliance.iso
+appliance-staging := $(BIN)/appliance-staging.tgz
+bootstrap := $(BIN)/bootstrap.iso
+iso-base := $(BIN)/iso-base.tgz
 
-tether-linux := binary/tether-linux
-tether-windows := binary/tether-windows.exe
+install := $(BIN)/install.sh
+
+go-lint := $(BIN)/.golint
+go-vet := $(BIN)/.govet
+go-imports := $(BIN)/.goimports
 
 # target aliases - target mapping
 docker-engine-api: $(docker-engine-api)
@@ -49,25 +61,41 @@ rpctool: $(rpctool)
 tether-linux: $(tether-linux)
 tether-windows: $(tether-windows)
 
+appliance: $(appliance)
+appliance-staging: $(appliance-staging)
+bootstrap: $(bootstrap)
+iso-base: $(iso-base)
+
+install: $(install)
+
 swagger: $(SWAGGER)
 
+golint: $(go-lint)
+govet: $(go-vet)
+goimports: $(go-imports)
+
+
 # convenience targets
-all: check bootstrap apiservers $(imagec) $(vicadmin)
+all: components isos install
 tools: $(GOIMPORTS) $(GOVET) $(GVT) $(GOLINT) $(SWAGGER) goversion
 check: goversion goimports govet golint
 apiservers: $(portlayerapi) $(docker-engine-api)
 bootstrap: $(tether-linux) $(tether-windows) $(rpctool)
+components: check apiservers $(imagec) $(vicadmin) $(rpctool)
+isos: $(appliance) $(bootstrap)
+tethers: $(tether-linux) $(tether-windows)
 
 
+# utility targets
 goversion:
 	@echo checking go version...
 	@( $(GO) version | grep -q $(GOVERSION) ) || ( echo "Please install $(GOVERSION) (found: $$($(GO) version))" && exit 1 )
 
-$(GOIMPORTS):
+$(GOIMPORTS): vendor/manifest
 	@echo building $(GOIMPORTS)...
 	$(GO) build -o $(GOIMPORTS) ./vendor/golang.org/x/tools/cmd/goimports
 
-$(GOVET):
+$(GOVET): vendor/manifest
 	@echo building $(GOVET)...
 	$(GO) build -o $(GOVET) ./vendor/golang.org/x/tools/cmd/vet
 
@@ -79,16 +107,16 @@ $(GOLINT): vendor/manifest
 	@echo building $(GOLINT)...
 	$(GO) build -o $(GOLINT) ./vendor/github.com/golang/lint/golint
 
-$(SWAGGER):
+$(SWAGGER): vendor/manifest
 	@echo building $(SWAGGER)...
 	@$(GO) build -o $(SWAGGER) ./vendor/github.com/go-swagger/go-swagger/cmd/swagger
 
 # exit 1 if golint complains about anything other than comments
 golintf = $(GOLINT) $(1) | sh -c "! grep -v 'should have comment'"
 
-golint: $(GOLINT)
+$(go-lint): $(GOLINT)
 	@echo checking go lint...
-	@#$(call golintf,github.com/vmware/vic/bootstrap/...) # this is commented out due to number of warnings
+	@ #$(call golintf,github.com/vmware/vic/bootstrap/...) # this is commented out due to number of warnings
 	@$(call golintf,github.com/vmware/vic/imagec/...)
 	@$(call golintf,github.com/vmware/vic/vicadmin/...)
 	@$(call golintf,github.com/vmware/vic/pkg/...)
@@ -102,21 +130,24 @@ golint: $(GOLINT)
 gopath:
 	@echo -n $(GOPATH)
 
-goimports: $(GOIMPORTS)
+$(go-imports): $(GOIMPORTS) $(find . -type f -name '*.go' -not -path "./vendor/*")
 	@echo checking go imports...
 	@! $(GOIMPORTS) -d $$(find . -type f -name '*.go' -not -path "./vendor/*") 2>&1 | egrep -v '^$$'
+	@touch $@
 
-govet: $(GOVET)
+$(go-vet): $(GOVET) $(find . -type f -name '*.go' -not -path "./vendor/*")
 	@echo checking go vet...
 	@$(GOVET) -all -shadow $$(find . -type f -name '*.go' -not -path "./vendor/*")
+	@touch $@
 
 vendor: $(GVT)
 	@echo restoring vendor
 	$(GOPATH)/bin/gvt restore
 
-integration-tests:
-	docker build -t imagec_tests -f Dockerfile.integration-tests .
-	docker run --rm imagec_tests
+integration-tests: Dockerfile.integration-tests tests/imagec.bats tests/helpers/helpers.bash components
+	@echo Running integration tests
+	@docker build -t imagec_tests -f Dockerfile.integration-tests .
+	docker run -e BIN=$(BIN) --rm imagec_tests
 
 TEST_DIRS=github.com/vmware/vic/bootstrap/...
 TEST_DIRS+=github.com/vmware/vic/imagec
@@ -159,6 +190,7 @@ $(imagec): imagec/*.go $(portlayerapi-client)
 	@echo building imagec...
 	@CGO_ENABLED=0 $(GO) build -o ./$@ --ldflags '-extldflags "-static"'  ./$(dir $<)
 
+
 $(docker-engine-api): $(portlayerapi-client) apiservers/engine/server/*.go apiservers/engine/backends/*.go
 ifeq ($(OS),linux)
 	@echo Building docker-engine-api server...
@@ -167,26 +199,52 @@ else
 	@echo skipping docker-engine-api server, cannot build on non-linux
 endif
 
+
+
 # Common portlayer dependencies between client and server
 PORTLAYER_DEPS ?= apiservers/portlayer/swagger.yml \
 				  apiservers/portlayer/restapi/configure_port_layer.go \
 				  apiservers/portlayer/restapi/options/*.go apiservers/portlayer/restapi/handlers/*.go
 
 $(portlayerapi-client): $(PORTLAYER_DEPS)  $(SWAGGER)
 	@echo regenerating swagger models and operations for Portlayer API client...
-	@$(SWAGGER) generate client -A PortLayer -t $(dir $<) -f $<
+	@$(SWAGGER) generate client -A PortLayer -t $(realpath $(dir $<)) -f $<
 
 
 $(portlayerapi-server): $(PORTLAYER_DEPS) $(SWAGGER)
 	@echo regenerating swagger models and operations for Portlayer API server...
-	@$(SWAGGER) generate server -A PortLayer -t $(dir $<) -f $<
+	@$(SWAGGER) generate server -A PortLayer -t $(realpath $(dir $<)) -f $<
 
 $(portlayerapi): $(portlayerapi-server) $(shell find apiservers/engine/ -name '*.go')
 	@echo building Portlayer API server...
 	@$(GO) build -o $@ ./apiservers/portlayer/cmd/port-layer-server
 
+
+$(iso-base): isos/base.sh isos/base/*.repo isos/base/isolinux/**
+	@echo building iso-base docker image
+	@ #docker build -t $@ -f $< $(dir $<)
+	@$< -p $@
+
+# appliance staging - allows for caching of package install
+$(appliance-staging): isos/appliance-staging.sh $(iso-base)
+	@echo staging for VCH appliance
+	@$< -p $(iso-base) -o $@
+
+# main appliance target - depends on all top level component targets
+$(appliance): isos/appliance.sh $(rpctool) $(vicadmin) $(imagec) $(portlayerapi) $(docker-engine-api) $(appliance-staging)
+	@echo building VCH appliance ISO
+	@$< -p $(appliance-staging) -b $(BIN)
+
+# main bootstrap target
+$(bootstrap): $(tether-linux) $(rpctool) $(iso-base)
+	@echo "Placeholder target for bootstrap"
+
+$(install): install/install.sh
+	@echo Building installer
+	@cp $< $@
+
 clean:
-	rm -rf ./binary
+	rm -rf $(BIN)
 
 	@echo removing swagger generated files...
 	rm -f ./apiservers/portlayer/restapi/doc.go
@@ -196,3 +254,6 @@ clean:
 	rm -rf ./apiservers/portlayer/cmd/
 	rm -rf ./apiservers/portlayer/models/
 	rm -rf ./apiservers/portlayer/restapi/operations/
+	rm -fr ./tests/helpers/bats-assert/
+	rm -fr ./tests/helpers/bats-support/
+

README.md

@@ -17,7 +17,7 @@ To build the bootstrap binaries, ensure `GOPATH` is set, then issue the followin
 ```
 $ make all
 ```
-This will install required tools, build the bootstrap binaries `tether-windows`, `tether-linux`, `rpctool` and server binaries `docker-server`, `port-layer-server`.  The binaries will be created in the `./binaries` directory.
+This will install required tools, build the bootstrap binaries `tether-windows`, `tether-linux`, `rpctool` and server binaries `docker-engine-server`, `port-layer-server`.  The binaries will be created in the `./binaries` directory.
 
 To run tests after a successfull build, issue the following:
 ```
@@ -81,18 +81,18 @@ Ensure that you have Docker 1.6 or higher installed.
 Install the Drone command line tools.
 From the root directory of the `vic` repository run `drone exec -trusted -e VIC_ESX_TEST_URL=""`
 
-## Starting docker-server
+## Starting docker-engine-server
 
 Generate certificate pair
 
 ```
 go run `go env GOROOT`/src/crypto/tls/generate_cert.go --host localhost
 ```
 
-Start docker-server
+Start docker-engine-server
 
 ```
-binary/docker-server --port=2376 --tls-certificate=cert.pem --tls-key=key.pem
+binary/docker-engine-server --port=2376 --tls-certificate=cert.pem --tls-key=key.pem
 ```
 
 ## Starting port-layer-server