Add top-level products.meta file (#627)

* Add top-level products.meta file

The products.meta file describes which products we can deploy, along
with which packages belong to those products. This PR also adds a
package.meta that can be added for any package in its component
directory. With this file, we can describe things about the file like if
it's a data service, what binlinks it should install, etc.

Author: Jay Mundrawala

.bldr.toml

@@ -227,6 +227,7 @@ paths = [
   "lib/logger/*",
   "lib/platform/*",
   "lib/proc/*",
+  "lib/product/*",
   "lib/proxy/*",
   "lib/stringutils/*",
   "lib/tls/*",
@@ -391,6 +392,7 @@ paths = [
   "lib/io/*",
   "lib/platform/*",
   "lib/proc/*",
+  "lib/product/*",
   "lib/proxy/*",
   "lib/secrets/*",
   "lib/stringutils/*",
@@ -473,6 +475,7 @@ paths = [
   "lib/license/*",
   "lib/platform/*",
   "lib/proc/*",
+  "lib/product/*",
   "lib/proxy/*",
   "lib/secrets/*",
   "lib/stringutils/*",

.expeditor/create-manifest.rb

@@ -11,8 +11,7 @@
 BLDR_API_HOST="bldr.habitat.sh"
 BLDR_API_USER_AGENT="Chef Expeditor"
 
-# Packages that are present in
-# components/automate-deployment/pkg/assets/data/services.json but we wish to
+# Packages that are present in products.meta but we wish to
 # exclude from the manifest (probably because they are not yet published to the
 # depot).
 #
@@ -164,42 +163,25 @@ def get_hab_deps_latest()
 manifest["git_sha"] = out.strip
 
 
-collections = File.open("components/automate-deployment/pkg/assets/data/services.json") do |f|
+products_meta = File.open("products.meta") do |f|
   JSON.parse(f.read)
 end
 
-pkg_paths_by_collection = {}
-
-non_package_data_keys = %w{ collection binlinks }
-
-collections.each do |collection|
-  paths_for_collection = []
-  collection.each do |pkg_type, pkg_list|
-    next if non_package_data_keys.include?(pkg_type)
-    paths_for_collection += pkg_list
-  end
-  collection_name = collection["collection"]
-  pkg_paths_by_collection[collection_name] = paths_for_collection
-end
-
 manifest["packages"] = []
-pkg_paths_by_collection.each do |name, pkg_paths|
-
-  pkg_paths.each do |pkg_path|
-    next if SKIP_PACKAGES.include?(pkg_path)
+products_meta["packages"].each do |pkg_path|
+  next if SKIP_PACKAGES.include?(pkg_path)
 
-    package_ident = pkg_path.split("/")
-    pkg_origin = package_ident[0]
-    pkg_name = package_ident[1]
+  package_ident = pkg_path.split("/")
+  pkg_origin = package_ident[0]
+  pkg_name = package_ident[1]
 
-    latest_release = get_latest(channel_for_origin(pkg_origin), pkg_origin, pkg_name)
+  latest_release = get_latest(channel_for_origin(pkg_origin), pkg_origin, pkg_name)
 
-    pkg_version = latest_release["version"]
-    pkg_release = latest_release["release"]
+  pkg_version = latest_release["version"]
+  pkg_release = latest_release["release"]
 
-    puts "  Adding package #{pkg_origin}/#{pkg_name}/#{pkg_version}/#{pkg_release} from collection #{name}"
-    manifest["packages"] << "#{pkg_origin}/#{pkg_name}/#{pkg_version}/#{pkg_release}"
-  end
+  puts "  Adding package #{pkg_origin}/#{pkg_name}/#{pkg_version}/#{pkg_release}"
+  manifest["packages"] << "#{pkg_origin}/#{pkg_name}/#{pkg_version}/#{pkg_release}"
 end
 
 # Add extra packages to manifest that deployment-service doesn't need to manage

.expeditor/verify_private.pipeline.yml

@@ -514,7 +514,7 @@ steps:
   - label: "[integration] airgap upgrade"
     command:
       - integration/run_test integration/tests/airgap_upgrade.sh
-    timeout_in_minutes: 20
+    timeout_in_minutes: 30
     expeditor:
       executor:
         linux:
@@ -589,7 +589,7 @@ steps:
   - label: "[integration] upgrade dev -> master"
     command:
       - integration/run_test integration/tests/upgrade.sh
-    timeout_in_minutes: 20
+    timeout_in_minutes: 25
     expeditor:
       executor:
         linux:
@@ -598,7 +598,7 @@ steps:
   - label: "[integration] upgrade acceptance -> master"
     command:
       - integration/run_test integration/tests/upgrade_acceptance_master.sh
-    timeout_in_minutes: 20
+    timeout_in_minutes: 25
     expeditor:
       executor:
         linux:
@@ -607,7 +607,7 @@ steps:
   - label: "[integration] upgrade current -> master"
     command:
       - integration/run_test integration/tests/upgrade_current_master.sh
-    timeout_in_minutes: 20
+    timeout_in_minutes: 25
     expeditor:
       executor:
         linux:
@@ -616,7 +616,7 @@ steps:
   - label: "[integration] manual upgrade current -> master"
     command:
       - integration/run_test integration/tests/manual_upgrade.sh
-    timeout_in_minutes: 20
+    timeout_in_minutes: 25
     expeditor:
       executor:
         linux:
@@ -625,7 +625,7 @@ steps:
   - label: "[integration] deep upgrades"
     command:
       - integration/run_test integration/tests/deep_upgrade.sh
-    timeout_in_minutes: 20
+    timeout_in_minutes: 25
     expeditor:
       executor:
         linux:
@@ -634,7 +634,7 @@ steps:
   - label: "[integration] deep migrate upgrade"
     command:
       - integration/run_test integration/tests/migrate_upgrade.sh
-    timeout_in_minutes: 20
+    timeout_in_minutes: 25
     expeditor:
       executor:
         linux:

api/config/deployment/config_request.go

@@ -95,11 +95,9 @@ func (c *ConfigRequest) Validate() error {
 	}
 
 	if desiredProducts := c.V1.Svc.GetProducts(); len(desiredProducts) > 0 {
-		availableProducts := services.ListProducts()
-		for _, desiredProduct := range desiredProducts {
-			if !stringutils.SliceContains(availableProducts, desiredProduct) {
-				err.AddInvalidValue("deployment.v1.svc.products", fmt.Sprintf("Valid products are %s", strings.Join(availableProducts, ", ")))
-			}
+		validationErr := services.ValidateProductDeployment(desiredProducts)
+		if validationErr != nil {
+			err.AddInvalidValue("deployment.v1.svc.products", validationErr.Error())
 		}
 	}
 

components/automate-cli/package.meta

@@ -0,0 +1,4 @@
+{
+  "name": "chef/automate-cli",
+  "binlinks": ["chef-automate"]
+}

components/automate-cs-nginx/package.meta

@@ -0,0 +1,4 @@
+{
+  "name": "chef/automate-cs-nginx",
+  "binlinks": ["knife", "chef-server-ctl"]
+}

components/automate-deployment/docs/how-to-add-a-service.md

@@ -140,7 +140,7 @@ determining what packages to build. Add your package to the
 
 ### Manifest Chicken and Egg Issue
 
-"The manifest" is a JSON file which is created by the script `.expeditor/create-manifest.rb`. The manifest contains exact versions of habitat packages that comprise an Automate release. The package versions are determined by querying the habitat depot/bldr for the versions of every package described in the `services.json` file (more on that below). The reason we must query the depot is because we do not build every package on every build, so instead we build the manifest based on what exists in the depot. During tests we rely on the deployment service preferring packages from local disk to upgrade just those packages with changes.
+"The manifest" is a JSON file which is created by the script `.expeditor/create-manifest.rb`. The manifest contains exact versions of habitat packages that comprise an Automate release. The package versions are determined by querying the habitat depot/bldr for the versions of every package described in the `product.meta` file (more on that below). The reason we must query the depot is because we do not build every package on every build, so instead we build the manifest based on what exists in the depot. During tests we rely on the deployment service preferring packages from local disk to upgrade just those packages with changes.
 
 When introducing a new service, the manifest is a problem because there are initially zero version of your new package in the depot, so a query for the current version of that package will fail and the manifest can't be generated. One way around this is to merge your new service in two pull requests. The first just needs to have a buildable package so that there's something in the depot to query. If you've followed this guide in order, you should be able to submit and merge your first pull request now and then follow up with a second pull request which includes the work described below (along with whatever other test suites, etc. you need).
 
@@ -150,6 +150,6 @@ Note: deployment-team has added some code that makes tests use a locally-generat
 
 You need to run `make update-bindings` from `components/automate-deployment` whenever you add/change/remove bindings. You don't necessarily need to change the bindings when first adding your service, but when you start to wire up your new service to other services this will be needed.
 
-### The Services File
+### The product.meta File
 
-The list of services and other mandatory packages that comprise Automate 2 is maintained in `components/automate-deployment/pkg/assets/data/services.json`. This file is also used to generate the manifest file that represents a Chef Automate 2 release as described above. Edit this file and add your service. In general, you will want to add it to the "automate-full" collection; the other collections are optional additions.
+The list of services and other mandatory packages that comprise Automate 2 and other Chef products is maintained in `product.meta`. This file is also used to generate the manifest file that represents a Chef Automate 2 release as described above. Edit this file and add your service. Services may also define extra metadata for their service in a `package.meta` for their component. See https://godoc.org/github.com/chef/automate/lib/product for more information.

components/automate-deployment/pkg/assets/assets.bindata.go

@@ -1,6 +1,5 @@
 package assets
 
-//go:generate go run ../../tools/sort-services/sort-services.go data/services.json data/binds.txt
 //go:generate go-bindata -pkg $GOPACKAGE -o assets.bindata.go data/...
 //go:generate ../../../../scripts/fix_bindata_header assets.bindata.go
 //go:generate go fmt ./