Implement TorchElastic Controller for Kubernetes (#63)

Summary:
Please help review on the TorchElastic operator. This is based on design here #36 (needs update).  API has been changed a little bit, idea is still the same.

Here's the code structure

```
.
├── CONTRIBUTING.md
├── Dockerfile
├── INSTRUCTION.md
├── Makefile
├── PROJECT
├── README.md
├── api
│   └── v1alpha1
│       ├── constants.go
│       ├── elasticjob_types.go               ---> API Definition
│       ├── groupversion_info.go
│       └── zz_generated.deepcopy.go
├── bin
│   └── README.md
├── classy-vision.yaml
├── config.                   ---> K8s manifest, some of them are not being used yet, come from skeleton
│   ├── certmanager
│   ├── crd
│   ├── default
│   ├── manager
│   ├── prometheus
│   ├── rbac
│   ├── samples
│   └── webhook
├── controllers
│   ├── elasticjob_controller.go.   --> controller code
│   ├── expectation.go
│   ├── job.go
│   ├── pod.go
│   ├── service.go
│   ├── suite_test.go
│   └── util.go
├── go.mod
├── go.sum
├── hack
│   └── boilerplate.go.txt
└── main.go
```

A few main files to review.
1. ElasticJob API  - api/v1alpha1/elasticjob_types.go
2. Controller codes - controllers/
3. Docs - CONTRIBUTING.md and INSTRUCTION.md
4. Naming. The api group is `elasticjobs.elastic.pytorch.org`.
5. License - use Apache2 as header. Author is ` The PyTorch Elastic Authors`
Pull Request resolved: #63

Reviewed By: drdarshan

Differential Revision: D20658020

Pulled By: kiukchung

fbshipit-source-id: e8e7372f48351fb1ae60934bc6fc3a4c1ce83984

Author: Jeffwan

.gitignore

@@ -2,3 +2,24 @@ docs/src
 docs/build
 docs/torchelastic.docset
 **/__pycache__
+
+# Mac OS X files
+.DS_Store
+
+# Binaries for programs and plugins
+*.exe
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# IDE
+**/.idea/
+
+# Operator Binary
+kubernetes//bin/manager

kubernetes/DEVELOPMENT.md

@@ -0,0 +1,47 @@
+## Development Guidance
+
+This project uses go modules. We suggest to use golang 1.13.x for development and make sure`GO111MODULE` is enabled.
+
+### Setup development environment
+
+Fork [PyTorch/Elastic](https://github.com/pytorch/elastic)
+
+```shell
+mkdir -p ${GOPATH}/src/github.com/pytorch/
+cd ${GOPATH}/src/github.com/pytorch
+git clone [email protected]:${GITHUB_USER}/elastic.git
+
+# operator codes is under kubernetes directory
+cd elastic/kubernetes
+```
+
+### Download dependencies.
+
+```shell
+go mod download
+```
+
+### Build the binary locally
+
+```shell
+make manager
+```
+
+### Test Binaries locally
+
+```shell
+./bin/manager
+```
+
+### Run Tests
+
+```shell
+go test ./... -coverprofile cover.out
+```
+
+### Build container image
+
+```shell
+# It requires you to build binary locally first.
+docker build -t ${your_dockerhub_username}/torch-elastic-operator:latest .
+```

kubernetes/Dockerfile

@@ -0,0 +1,27 @@
+# Build the manager binary
+FROM golang:1.13 as builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the go source
+COPY main.go main.go
+COPY api/ api/
+COPY controllers/ controllers/
+
+# Build
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go
+
+# Use distroless as minimal base image to package the manager binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM gcr.io/distroless/static:nonroot
+WORKDIR /
+COPY --from=builder /workspace/manager .
+USER nonroot:nonroot
+
+ENTRYPOINT ["/manager"]

kubernetes/Makefile

@@ -0,0 +1,80 @@
+
+# Image URL to use all building/pushing image targets
+IMG ?= controller:latest
+# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
+CRD_OPTIONS ?= "crd:trivialVersions=true"
+
+# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+ifeq (,$(shell go env GOBIN))
+GOBIN=$(shell go env GOPATH)/bin
+else
+GOBIN=$(shell go env GOBIN)
+endif
+
+all: manager
+
+# Run tests
+test: generate fmt vet manifests
+	go test ./... -coverprofile cover.out
+
+# Build manager binary
+manager: generate fmt vet
+	go build -o bin/manager main.go
+
+# Run against the configured Kubernetes cluster in ~/.kube/config
+run: generate fmt vet manifests
+	go run ./main.go
+
+# Install CRDs into a cluster
+install: manifests
+	kustomize build config/crd | kubectl apply -f -
+
+# Uninstall CRDs from a cluster
+uninstall: manifests
+	kustomize build config/crd | kubectl delete -f -
+
+# Deploy controller in the configured Kubernetes cluster in ~/.kube/config
+deploy: manifests
+	cd config/manager && kustomize edit set image controller=${IMG}
+	kustomize build config/default | kubectl apply -f -
+
+# Generate manifests e.g. CRD, RBAC etc.
+manifests: controller-gen
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+
+# Run go fmt against code
+fmt:
+	go fmt ./...
+
+# Run go vet against code
+vet:
+	go vet ./...
+
+# Generate code
+generate: controller-gen
+	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths="./..."
+
+# Build the docker image
+docker-build: test
+	docker build . -t ${IMG}
+
+# Push the docker image
+docker-push:
+	docker push ${IMG}
+
+# find or download controller-gen
+# download controller-gen if necessary
+controller-gen:
+ifeq (, $(shell which controller-gen))
+	@{ \
+	set -e ;\
+	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
+	cd $$CONTROLLER_GEN_TMP_DIR ;\
+	go mod init tmp ;\
+	go get sigs.k8s.io/controller-tools/cmd/[email protected] ;\
+	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
+	}
+CONTROLLER_GEN=$(GOBIN)/controller-gen
+else
+CONTROLLER_GEN=$(shell which controller-gen)
+endif

kubernetes/PROJECT

@@ -0,0 +1,7 @@
+domain: pytorch.org
+repo: github.com/pytorch/elastic/kubernetes
+resources:
+- group: elastic
+  kind: ElasticJob
+  version: v1alpha1
+version: "2"