Adds support for multi-Job CPIs

This commit adds CLI support for deployment manifests that specify a CPI
with multiple Jobs.

A CPI's Job has traditionally been specified in a Director manifest with
the `{cloud_provider: {template: {name: "name", release: "release"}}}`
manifest construction. This commit extends the `cloud_provider`
defintion with a new `templates` key whose value is an array of zero or
more `{name:, release:}` hashes/mappings/dictionaries.

Supporting this change required relaxing some explicit requirements in
sections of the code that expected that a CPI could only ever have one
Job. However, the requirement that exactly one of the CPI's Jobs have a
template that creates a `bin/cpi` executable is retained.

These changes are backwards-compatible, so existing manifests that use
`cloud_provider: template...` syntax will work just fine.

Why have we made these changes? To support upcoming work to permit the
creation of "sidecar" or "plugin" Jobs for CPIs that can be used by the
CPI during both CLI-driven CPI invocations (like `create-env`) and also
Director-driven CPI invocations.

Signed-off-by: Kenneth Lakin <[email protected]>

bin/gen-fakes

@@ -31,6 +31,9 @@ counterfeiter director FileReporter
 counterfeiter director TaskReporter
 counterfeiter director Event
 
+# FakeInstallation is generated in cmd/cmdfakes because that's where it's used
+counterfeiter -o cmd/cmdfakes installation Installation
+
 counterfeiter uaa UAA
 counterfeiter uaa Token
 counterfeiter uaa AccessToken

cloud/factory.go

@@ -31,19 +31,28 @@ func NewFactory(
 }
 
 func (f *factory) NewCloud(installation biinstall.Installation, directorID string, stemcellApiVersion int) (Cloud, error) {
-	cpiJob := installation.Job()
-	target := installation.Target()
-	cpi := CPI{
-		JobPath:     cpiJob.Path,
-		JobsDir:     target.JobsPath(),
-		PackagesDir: target.PackagesPath(),
+	numberCpiBinariesFound := 0
+	foundCPI := CPI{}
+
+	for _, cpiJob := range installation.Jobs() {
+		target := installation.Target()
+		cpi := CPI{
+			JobPath:     cpiJob.Path,
+			JobsDir:     target.JobsPath(),
+			PackagesDir: target.PackagesPath(),
+		}
+
+		cmdPath := cpi.ExecutablePath()
+		if f.fs.FileExists(cmdPath) {
+			numberCpiBinariesFound += 1
+			foundCPI = cpi
+		}
 	}
 
-	cmdPath := cpi.ExecutablePath()
-	if !f.fs.FileExists(cmdPath) {
-		return nil, bosherr.Errorf("Installed CPI job '%s' does not contain the required executable '%s'", cpiJob.Name, cmdPath)
+	if numberCpiBinariesFound != 1 {
+		return nil, bosherr.Errorf("Found %d Jobs with a 'bin/cpi' binary. Expected 1.", numberCpiBinariesFound)
 	}
 
-	cpiCmdRunner := NewCPICmdRunner(f.cmdRunner, cpi, f.logger)
+	cpiCmdRunner := NewCPICmdRunner(f.cmdRunner, foundCPI, f.logger)
 	return NewCloud(cpiCmdRunner, directorID, stemcellApiVersion, f.logger), nil
 }

cmd/create_env_test.go

@@ -251,9 +251,8 @@ var _ = Describe("CreateEnvCmd", func() {
 
 			// parsed CPI deployment manifest
 			installationManifest = biinstallmanifest.Manifest{
-				Template: biinstallmanifest.ReleaseJobRef{
-					Name:    "fake-cpi-release-job-name",
-					Release: "fake-cpi-release-name",
+				Templates: []biinstallmanifest.ReleaseJobRef{
+					{Name: "fake-cpi-release-job-name", Release: "fake-cpi-release-name"},
 				},
 				Mbus: mbusURL,
 			}
@@ -322,7 +321,6 @@ var _ = Describe("CreateEnvCmd", func() {
 				cpiInstaller := bicpirel.CpiInstaller{
 					ReleaseManager:   releaseManager,
 					InstallerFactory: mockInstallerFactory,
-					Validator:        bicpirel.NewValidator(),
 				}
 				releaseFetcher := biinstall.NewReleaseFetcher(tarballProvider, releaseReader, releaseManager)
 				stemcellFetcher := bistemcell.Fetcher{
@@ -430,14 +428,15 @@ var _ = Describe("CreateEnvCmd", func() {
 
 			mockInstallerFactory.EXPECT().NewInstaller(target).Return(mockInstaller).AnyTimes()
 
-			installation := biinstall.NewInstallation(target, installedJob, installationManifest)
+			installation := biinstall.NewInstallation(target, []biinstall.InstalledJob{installedJob},
+				installationManifest)
 
 			expectInstall = mockInstaller.EXPECT().Install(installationManifest, gomock.Any()).Do(func(_ interface{}, stage boshui.Stage) {
 				Expect(fakeStage.SubStages).To(ContainElement(stage))
 			}).Return(installation, nil).AnyTimes()
 			mockInstaller.EXPECT().Cleanup(installation).AnyTimes()
 
-			//mockDeployment := mock_deployment.NewMockDeployment(mockCtrl)
+			// mockDeployment := mock_deployment.NewMockDeployment(mockCtrl)
 
 			expectDeploy = mockDeployer.EXPECT().Deploy(
 				mockCloud,
@@ -748,7 +747,8 @@ var _ = Describe("CreateEnvCmd", func() {
 			It("returns error", func() {
 				err := command.Run(fakeStage, defaultCreateEnvOpts)
 				Expect(err).To(HaveOccurred())
-				Expect(err.Error()).To(Equal("Invalid CPI release 'fake-cpi-release-name': CPI release must contain specified job 'fake-cpi-release-job-name'"))
+				//nolint:gosimple
+				Expect(err.Error()).To(Equal(fmt.Sprintf("Found 0 releases containing a template that renders to target 'bin/cpi'. Expected to find 1. Releases inspected: [fake-cpi-release-name]\nrelease 'fake-cpi-release-name' must contain specified job 'fake-cpi-release-job-name'")))
 			})
 		})
 

cmd/deployment_deleter.go

@@ -126,15 +126,23 @@ func (c *deploymentDeleter) DeleteDeployment(skipDrain bool, stage biui.Stage) (
 			return err
 		}
 
-		cpiReleaseName := installationManifest.Template.Release
-		cpiReleaseRef, found := releaseSetManifest.FindByName(cpiReleaseName)
-		if !found {
-			return bosherr.Errorf("installation release '%s' must refer to a release in releases", cpiReleaseName)
+		errs := []error{}
+		for _, template := range installationManifest.Templates {
+
+			cpiReleaseName := template.Release
+			cpiReleaseRef, found := releaseSetManifest.FindByName(cpiReleaseName)
+			if !found {
+				return bosherr.Errorf("installation release '%s' must refer to a release in releases", cpiReleaseName)
+			}
+
+			err = c.releaseFetcher.DownloadAndExtract(cpiReleaseRef, stage)
+			if err != nil {
+				errs = append(errs, err)
+			}
 		}
 
-		err = c.releaseFetcher.DownloadAndExtract(cpiReleaseRef, stage)
-		if err != nil {
-			return err
+		if len(errs) > 0 {
+			return bosherr.NewMultiError(errs...)
 		}
 
 		err = c.cpiInstaller.ValidateCpiRelease(installationManifest, stage)

cmd/deployment_deleter_test.go

@@ -182,9 +182,8 @@ cloud_provider:
 		var allowCPIToBeInstalled = func() {
 			installationManifest := biinstallmanifest.Manifest{
 				Name: "test-release",
-				Template: biinstallmanifest.ReleaseJobRef{
-					Name:    "fake-cpi-release-job-name",
-					Release: "fake-cpi-release-name",
+				Templates: []biinstallmanifest.ReleaseJobRef{
+					{Name: "fake-cpi-release-job-name", Release: "fake-cpi-release-name"},
 				},
 				Mbus:       mbusURL,
 				Properties: biproperty.Map{},
@@ -216,7 +215,6 @@ cloud_provider:
 			cpiInstaller := bicpirel.CpiInstaller{
 				ReleaseManager:   releaseManager,
 				InstallerFactory: mockInstallerFactory,
-				Validator:        bicpirel.NewValidator(),
 			}
 			releaseFetcher := biinstall.NewReleaseFetcher(tarballProvider, releaseReader, releaseManager)
 			releaseSetAndInstallationManifestParser := cmd.ReleaseSetAndInstallationManifestParser{
@@ -526,9 +524,8 @@ cloud_provider:
 			JustBeforeEach(func() {
 				installationManifest := biinstallmanifest.Manifest{
 					Name: "test-release",
-					Template: biinstallmanifest.ReleaseJobRef{
-						Name:    "fake-cpi-release-job-name",
-						Release: "fake-cpi-release-name",
+					Templates: []biinstallmanifest.ReleaseJobRef{
+						{Name: "fake-cpi-release-job-name", Release: "fake-cpi-release-name"},
 					},
 					Mbus:       mbusURL,
 					Properties: biproperty.Map{},

cmd/env_factory.go

@@ -122,7 +122,6 @@ func NewEnvFactory(
 		f.cpiInstaller = bicpirel.CpiInstaller{
 			ReleaseManager:   f.releaseManager,
 			InstallerFactory: installerFactory,
-			Validator:        bicpirel.NewValidator(),
 		}
 	}