Support managed updates (#349)

A few notes:
- We handle v2 deployer in this PR. In this updated schema, the images are declared explicitly under `x-google-marketplace.images`. We will use a `Secret` for the parameters, instead of `ConfigMap`. The properties that receive images will be generated by the deployer.
- To __additionally__ indicate that an app supports KALM in a v2 deployer, the following entry in `schema.yaml` v2 is added:

```yaml
x-google-marketplace:
  schemaVersion: v2
  managedUpdates:
    kalmSupported: true
```

Note that it's possible to use v2 deployer without supporting managed updates. In fact, v2 will be the desired implementation for new partners.

Author: huyhg

marketplace/deployer_util/clean_iam_resources.sh

@@ -19,17 +19,11 @@ set -eox pipefail
 [[ -z "$NAME" ]] && echo "NAME must be set" && exit 1
 [[ -z "$NAMESPACE" ]] && echo "NAMESPACE must be set" && exit 1
 
-# Clean up IAM resources.
-kubectl delete --namespace="$NAMESPACE" --filename=- <<EOF
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: "${NAME}-deployer-sa"
-  namespace: "${NAMESPACE}"
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: "${NAME}-deployer-rb"
-  namespace: "${NAMESPACE}"
-EOF
+# Delete the service account and RBAC objects by labels.
+# Note that only resources of a one-shot deployer have this label.
+# Resources of a KALM-managed deployer have
+# app.kubernetes.io/component=kalm.marketplace.cloud.google.com label.
+kubectl delete --namespace="$NAMESPACE" \
+  ServiceAccount,RoleBinding \
+  -l 'app.kubernetes.io/component'='deployer.marketplace.cloud.google.com','app.kubernetes.io/name'="$NAME" \
+  --ignore-not-found

marketplace/deployer_util/config_helper.py

@@ -217,6 +217,9 @@ def __init__(self, dictionary):
         dictionary, 'publishedVersionMetadata', lambda v: SchemaVersionMeta(v),
         'x-google-marketplace.publishedVersionMetadata is required')
 
+    self._managed_updates = SchemaManagedUpdates(
+        dictionary.get('managedUpdates', {}))
+
     images = _must_get(dictionary, 'images',
                        'x-google-marketplace.images is required')
     self._images = {k: SchemaImage(k, v) for k, v in images.iteritems()}
@@ -244,10 +247,25 @@ def published_version_meta(self):
   def images(self):
     return self._images
 
+  @property
+  def managed_updates(self):
+    return self._managed_updates
+
   def is_v2(self):
     return self._schema_version == _SCHEMA_VERSION_2
 
 
+class SchemaManagedUpdates:
+  """Accesses managedUpdates."""
+
+  def __init__(self, dictionary):
+    self._kalm_supported = dictionary.get('kalmSupported', False)
+
+  @property
+  def kalm_supported(self):
+    return self._kalm_supported
+
+
 class SchemaClusterConstraints:
   """Accesses top level clusterConstraints."""
 

marketplace/deployer_util/config_helper_test.py

@@ -648,6 +648,9 @@ def test_v2_fields(self):
             - BUG_FIX
             recommended: true
 
+          managedUpdates:
+            kalmSupported: true
+
           images:
             main:
               properties:
@@ -690,6 +693,9 @@ def test_v2_fields(self):
                      'db.image.tag')
     self.assertEqual(images['db'].properties['db.image.tag'].part_type, 'TAG')
 
+    self.assertEqual(schema.x_google_marketplace.managed_updates.kalm_supported,
+                     True)
+
   def test_k8s_version_constraint(self):
     schema = config_helper.Schema.load_yaml("""
         applicationApiVersion: v1beta1

marketplace/deployer_util/expand_config.py

@@ -32,6 +32,8 @@
 according to their schema.
 """
 
+_IMAGE_REPO_PREFIX_PROPERTY_NAME = '__image_repo_prefix__'
+
 
 class InvalidProperty(Exception):
   pass
@@ -41,6 +43,10 @@ class MissingRequiredProperty(Exception):
   pass
 
 
+class MissingRequiredValue(Exception):
+  pass
+
+
 def main():
   parser = ArgumentParser(description=_PROG_HELP)
   schema_values_common.add_to_argument_parser(parser)
@@ -64,25 +70,32 @@ def expand(values_dict, schema, app_uid=''):
   """Returns the expanded values according to schema."""
   schema.validate()
 
+  valid_property_names = set(schema.properties.keys() +
+                             [_IMAGE_REPO_PREFIX_PROPERTY_NAME])
   for k in values_dict:
-    if k not in schema.properties:
+    if k not in valid_property_names:
       raise InvalidProperty('No such property defined in schema: {}'.format(k))
 
   # Captures the final property name-value mappings.
   # This has both properties directly specified under schema's `properties` and
   # generated properties. See below for details about generated properties.
   result = {}
-  # Captures only the generated properties. These are not directly specified in the schema
-  # under `properties`. Rather, their name are specified in special `generatedProperties` fields
-  # under individual property `x-google-marketplace`.
+  # Captures only the generated properties. These are not directly specified in
+  # the schema under `properties`. Rather, their name are specified in special
+  # `generatedProperties` fields under each property's `x-google-marketplace`.
   # Note that properties with generated values are NOT generated properties.
   generated = {}
 
+  if schema.is_v2():
+    # Handles the images section of the schema.
+    generate_v2_image_properties(schema, values_dict, generated)
+
   # Copy explicitly specified values and generate values into result.
   for k, prop in schema.properties.iteritems():
     v = values_dict.get(k, None)
 
-    # The value is not explicitly specified and thus is eligible for auto-generation.
+    # The value is not explicitly specified and
+    # thus is eligible for auto-generation.
     if v is None:
       if prop.password:
         v = generate_password(prop.password)
@@ -106,7 +119,7 @@ def expand(values_dict, schema, app_uid=''):
         if not isinstance(v, str):
           raise InvalidProperty(
               'Invalid value for IMAGE property {}: {}'.format(k, v))
-        generate_properties_for_image(prop, v, generated)
+        generate_v1_properties_for_image(prop, v, generated)
       elif prop.string:
         if not isinstance(v, str):
           raise InvalidProperty(
@@ -118,13 +131,13 @@ def expand(values_dict, schema, app_uid=''):
               'Invalid value for TLS_CERTIFICATE property {}: {}'.format(k, v))
         generate_properties_for_tls_certificate(prop, v, generated)
 
-    # Copy generated properties into result, validating that there are no collisions.
     if v is not None:
       result[k] = v
 
   validate_value_types(result, schema)
   validate_required_props(result, schema)
 
+  # Copy generated properties into result, validating no collisions.
   for k, v in generated.iteritems():
     if k in result:
       raise InvalidProperty(
@@ -155,7 +168,7 @@ def generate_properties_for_appuid(prop, value, result):
     result[prop.application_uid.application_create] = False if value else True
 
 
-def generate_properties_for_image(prop, value, result):
+def generate_v1_properties_for_image(prop, value, result):
   if prop.image.split_by_colon:
     before_name, after_name = prop.image.split_by_colon
     parts = value.split(':', 1)
@@ -185,6 +198,37 @@ def generate_properties_for_image(prop, value, result):
     result[tag_name] = tag_value
 
 
+def generate_v2_image_properties(schema, values_dict, result):
+  repo_prefix = values_dict.get(_IMAGE_REPO_PREFIX_PROPERTY_NAME, None)
+  if not repo_prefix:
+    raise MissingRequiredValue('A valid value for __image_repo_prefix__ '
+                               'must be specified in values.yaml')
+  tag = schema.x_google_marketplace.published_version
+  for img in schema.x_google_marketplace.images.values():
+    if img.name:
+      # Allows an empty image name for legacy reason.
+      registry_repo = '{}/{}'.format(repo_prefix, img.name)
+    else:
+      registry_repo = repo_prefix
+    registry, repo = registry_repo.split('/', 1)
+    full = '{}:{}'.format(registry_repo, tag)
+    for prop in img.properties.values():
+      if prop.part_type == config_helper.IMAGE_PROJECTION_TYPE_FULL:
+        result[prop.name] = full
+      elif prop.part_type == config_helper.IMAGE_PROJECTION_TYPE_REGISTRY:
+        result[prop.name] = registry
+      elif prop.part_type == config_helper.IMAGE_PROJECTION_TYPE_REGISTRY_REPO:
+        result[prop.name] = registry_repo
+      elif prop.part_type == config_helper.IMAGE_PROJECTION_TYPE_REPO:
+        result[prop.name] = repo
+      elif prop.part_type == config_helper.IMAGE_PROJECTION_TYPE_TAG:
+        result[prop.name] = tag
+      else:
+        raise InvalidProperty(
+            'Invalid type for images.properties.type: {}'.format(
+                prop.part_type))
+
+
 def generate_properties_for_string(prop, value, result):
   if prop.string.base64_encoded:
     result[prop.string.base64_encoded] = base64.b64encode(value)

marketplace/deployer_util/expand_config_test.py

@@ -47,7 +47,7 @@ def test_invalid_value_type(self):
     self.assertRaises(expand_config.InvalidProperty, lambda: expand_config.
                       expand({'p1': 3}, schema))
 
-  def test_generate_properties_for_image_split_by_colon(self):
+  def test_generate_properties_for_v1_image_split_by_colon(self):
     schema = config_helper.Schema.load_yaml("""
         applicationApiVersion: v1beta1
         properties:
@@ -69,7 +69,7 @@ def test_generate_properties_for_image_split_by_colon(self):
             'i1.after': 'bar',
         }, result)
 
-  def test_generate_properties_for_image_split_to_registry_repo_tag(self):
+  def test_generate_properties_for_v1_image_split_to_registry_repo_tag(self):
     schema = config_helper.Schema.load_yaml("""
         applicationApiVersion: v1beta1
         properties:
@@ -93,6 +93,58 @@ def test_generate_properties_for_image_split_to_registry_repo_tag(self):
             'i1.tag': 'baz',
         }, result)
 
+  def test_generate_properties_for_v2_images(self):
+    schema = config_helper.Schema.load_yaml("""
+        x-google-marketplace:
+          schemaVersion: v2
+          applicationApiVersion: v1beta1
+          publishedVersion: '0.1.1'
+          publishedVersionMetadata:
+            releaseNote: Release note for 0.1.1
+          images:
+            "":
+              properties:
+                image.full: {type: FULL}
+                image.registry: {type: REGISTRY}
+                image.registry_repo: {type: REPO_WITH_REGISTRY}
+                image.repo: {type: REPO_WITHOUT_REGISTRY}
+                image.tag: {type: TAG}
+            i1:
+              properties:
+                image.i1.full: {type: FULL}
+                image.i1.registry: {type: REGISTRY}
+                image.i1.registry_repo: {type: REPO_WITH_REGISTRY}
+                image.i1.repo: {type: REPO_WITHOUT_REGISTRY}
+                image.i1.tag: {type: TAG}
+            i2:
+              properties:
+                image.i2.full: {type: FULL}
+                image.i2.registry: {type: REGISTRY}
+                image.i2.registry_repo: {type: REPO_WITH_REGISTRY}
+                image.i2.repo: {type: REPO_WITHOUT_REGISTRY}
+                image.i2.tag: {type: TAG}
+        """)
+    result = expand_config.expand({'__image_repo_prefix__': 'gcr.io/app'},
+                                  schema)
+    self.assertEqual(
+        {
+            'image.full': 'gcr.io/app:0.1.1',
+            'image.tag': '0.1.1',
+            'image.registry': 'gcr.io',
+            'image.registry_repo': 'gcr.io/app',
+            'image.repo': 'app',
+            'image.i1.full': 'gcr.io/app/i1:0.1.1',
+            'image.i1.tag': '0.1.1',
+            'image.i1.registry': 'gcr.io',
+            'image.i1.registry_repo': 'gcr.io/app/i1',
+            'image.i1.repo': 'app/i1',
+            'image.i2.full': 'gcr.io/app/i2:0.1.1',
+            'image.i2.tag': '0.1.1',
+            'image.i2.registry': 'gcr.io',
+            'image.i2.registry_repo': 'gcr.io/app/i2',
+            'image.i2.repo': 'app/i2',
+        }, result)
+
   def test_generate_properties_for_string_base64_encoded(self):
     schema = config_helper.Schema.load_yaml("""
         applicationApiVersion: v1beta1

marketplace/deployer_util/print_version_metadata.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import collections
 import datetime
 import sys
 import yaml
@@ -35,6 +36,10 @@ def main():
       '--deployer_image',
       required=True,
       help='The full deployer image, required')
+  parser.add_argument(
+      '--deployer_image_digest',
+      required=True,
+      help='The digest of the deployer image, required')
   args = parser.parse_args()
 
   schema = schema_values_common.load_schema(args)
@@ -44,20 +49,34 @@ def main():
     raise Exception('schema.yaml must be in v2 version')
 
   x = schema.x_google_marketplace
-  meta = {
-      'releaseNote': x.published_version_meta.release_note,
-      'releaseDate': _utcnow_timestamp(),
-      'url': args.deployer_image,
-  }
+  meta = collections.OrderedDict([
+      ('releaseDate', _utcnow_timestamp()),
+      ('url', args.deployer_image),
+      ('digest', args.deployer_image_digest),
+      ('releaseNote', x.published_version_meta.release_note),
+  ])
   if x.published_version_meta.release_types:
     meta['releaseTypes'] = x.published_version_meta.release_types
   if x.published_version_meta.recommended is not None:
     meta['recommended'] = x.published_version_meta.recommended
 
-  sys.stdout.write(yaml.safe_dump(meta, default_flow_style=False, indent=2))
+  sys.stdout.write(_ordered_dump(meta, default_flow_style=False, indent=2))
   sys.stdout.flush()
 
 
+def _ordered_dump(data, stream=None, dumper=yaml.Dumper, **kwds):
+
+  class OrderedDumper(dumper):
+    pass
+
+  def _dict_representer(dumper, data):
+    return dumper.represent_mapping(
+        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items())
+
+  OrderedDumper.add_representer(collections.OrderedDict, _dict_representer)
+  return yaml.dump(data, stream, OrderedDumper, **kwds)
+
+
 def _utcnow_timestamp():
   # Instructed to output timezone, python would outputs +00:00 instead of Z.
   # So we add that manually here.