Merge pull request #90 from baloise/feat/deploy-array-path

feat(deploy): add support for updating values in YAML arrays

Author: christiansiegel

docs/commands/deploy.md

@@ -13,6 +13,9 @@ frontend:
 backend:
   repository: my-app/backend
   tag: 1.0.0 # <- and this one
+  env:
+  - name: TEST
+    value: foo # <- and even one in a list
 ```
 
 With the following command GitOps CLI will update both values to `1.1.0` on the `master` branch.
@@ -27,14 +30,20 @@ gitopscli deploy \
   --organisation "deployment" \
   --repository-name "myapp-non-prod" \
   --file "example/values.yaml" \
-  --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0}"
+  --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, backend.env.[0].value: bar}"
 ```
 
 ### Number Of Commits
 
 Note that by default GitOps CLI will create a separate commit for every value change:
 
 ```
+commit 0dcaa136b4c5249576bb1f40b942bff6ac718144
+Author: GitOpsCLI <[email protected]>
+Date:   Thu Mar 12 15:30:32 2020 +0100
+
+    changed 'backend.env.[0].value' to 'bar' in example/values.yaml
+
 commit d98913ad8fecf571d5f8c3635f8070b05c43a9ca
 Author: GitOpsCLI <[email protected]>
 Date:   Thu Mar 12 15:30:32 2020 +0100
@@ -59,6 +68,7 @@ Date:   Thu Mar 12 15:30:00 2020 +0100
 
     frontend.tag: '1.1.0'
     backend.tag: '1.1.0'
+    backend.env.[0].value: 'bar'
 ```
 
 ### Create Pull Request
@@ -75,7 +85,7 @@ gitopscli deploy \
   --organisation "deployment" \
   --repository-name "myapp-non-prod" \
   --file "example/values.yaml" \
-  --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0}" \
+  --values "{frontend.tag: 1.1.0, backend.tag: 1.1.0, backend.env.[0].value: bar}" \
   --create-pr \
   --auto-merge
 ```

gitopscli/yaml/yaml_util.py

@@ -1,5 +1,8 @@
+import re
 from ruamel.yaml import YAML
 
+ARRAY_KEY_SEGMENT_PATTERN = re.compile(r"\[(\d+)\]")
+
 
 def yaml_load(doc):
     return YAML().load(doc)
@@ -20,27 +23,34 @@ def get_string(self):
     return stream.get_string()
 
 
-def update_yaml_file(file_path, key, value, create_new=False):
+def update_yaml_file(file_path, key, value):
     yaml = YAML()
     with open(file_path, "r") as stream:
         content = yaml.load(stream)
 
-    keys, obj = key.split("."), content
-    for k in keys[:-1]:
-        if k not in obj or not isinstance(obj[k], dict):
-            if not create_new:
-                raise KeyError(f"Key '{key}' not found in YAML!")
-            obj[k] = dict()
-        obj = obj[k]
-    if keys[-1] in obj and obj[keys[-1]] == value:
-        return False  # nothing to update
-    if not create_new and keys[-1] not in obj:
-        raise KeyError(f"Key '{key}' not found in YAML!")
-    obj[keys[-1]] = value
-
-    with open(file_path, "w+") as stream:
-        yaml.dump(content, stream)
-    return True
+    key_segments = key.split(".")
+    current_key_segments = []
+    parent_item = content
+    for current_key_segment in key_segments:
+        current_key_segments.append(current_key_segment)
+        current_key = ".".join(current_key_segments)
+        is_array = ARRAY_KEY_SEGMENT_PATTERN.match(current_key_segment)
+        if is_array:
+            current_key_segment = int(is_array.group(1))
+            if not isinstance(parent_item, list) or current_key_segment >= len(parent_item):
+                raise KeyError(f"Key '{current_key}' not found in YAML!")
+        else:
+            if not isinstance(parent_item, dict) or current_key_segment not in parent_item:
+                raise KeyError(f"Key '{current_key}' not found in YAML!")
+        if current_key == key:
+            if parent_item[current_key_segment] == value:
+                return False  # nothing to update
+            parent_item[current_key_segment] = value
+            with open(file_path, "w+") as stream:
+                yaml.dump(content, stream)
+            return True
+        parent_item = parent_item[current_key_segment]
+    raise KeyError(f"Empty key!")
 
 
 def merge_yaml_element(file_path, element_path, desired_value, delete_missing_key=False):

tests/yaml/test_yaml_util.py

@@ -47,71 +47,70 @@ def test_yaml_dump(self):
     def test_update_yaml_file(self):
         test_file = self._create_file(
             """\
-a: # comment
-# comment
+a: # comment 1
+# comment 2
   b:
-    d: 1 # comment
-    c: 2 # comment"""
+    d: 1 # comment 3
+    c: 2 # comment 4
+  e:
+  - f: 3 # comment 5
+    g: 4 # comment 6
+  - [hello, world] # comment 7
+  - foo: # comment 8
+      bar # comment 9"""
         )
 
-        updated = update_yaml_file(test_file, "a.b.c", "2")
-        self.assertTrue(updated)
+        self.assertTrue(update_yaml_file(test_file, "a.b.c", "2"))
+        self.assertFalse(update_yaml_file(test_file, "a.b.c", "2"))  # already updated
 
-        updated = update_yaml_file(test_file, "a.b.c", "2")
-        self.assertFalse(updated)  # already updated
+        self.assertTrue(update_yaml_file(test_file, "a.e.[0].g", 42))
+        self.assertFalse(update_yaml_file(test_file, "a.e.[0].g", 42))  # already updated
 
-        expected = """\
-a: # comment
-# comment
-  b:
-    d: 1 # comment
-    c: '2' # comment
-"""
-        actual = self._read_file(test_file)
-        self.assertEqual(expected, actual)
+        self.assertTrue(update_yaml_file(test_file, "a.e.[1].[1]", "tester"))
+        self.assertFalse(update_yaml_file(test_file, "a.e.[1].[1]", "tester"))  # already updated
 
-        with pytest.raises(KeyError):
-            updated = update_yaml_file(test_file, "a.x", "foo")
-        updated = update_yaml_file(test_file, "a.x", "foo", create_new=True)
-        self.assertTrue(updated)
+        self.assertTrue(update_yaml_file(test_file, "a.e.[2]", "replaced object"))
+        self.assertFalse(update_yaml_file(test_file, "a.e.[2]", "replaced object"))  # already updated
 
         expected = """\
-a: # comment
-# comment
+a: # comment 1
+# comment 2
   b:
-    d: 1 # comment
-    c: '2' # comment
-  x: foo
+    d: 1 # comment 3
+    c: '2' # comment 4
+  e:
+  - f: 3 # comment 5
+    g: 42 # comment 6
+  - [hello, tester] # comment 7
+  - replaced object
 """
         actual = self._read_file(test_file)
         self.assertEqual(expected, actual)
 
-        with pytest.raises(KeyError):
-            updated = update_yaml_file(test_file, "a.x.z", "foo_z")
-        updated = update_yaml_file(test_file, "a.x.z", "foo_z", create_new=True)
-        self.assertTrue(updated)
+        with pytest.raises(KeyError) as ex:
+            update_yaml_file(test_file, "x.y", "foo")
+        self.assertEqual("\"Key 'x' not found in YAML!\"", str(ex.value))
 
-        with pytest.raises(KeyError):
-            updated = update_yaml_file(test_file, "a.x.y", "foo_y")
-        updated = update_yaml_file(test_file, "a.x.y", "foo_y", create_new=True)
-        self.assertTrue(updated)
+        with pytest.raises(KeyError) as ex:
+            update_yaml_file(test_file, "[42].y", "foo")
+        self.assertEqual("\"Key '[42]' not found in YAML!\"", str(ex.value))
 
-        with pytest.raises(KeyError):
-            updated = update_yaml_file(test_file, "a.x.y.z", "foo_y_z")
-        updated = update_yaml_file(test_file, "a.x.y.z", "foo_y_z", create_new=True)
-        self.assertTrue(updated)
+        with pytest.raises(KeyError) as ex:
+            update_yaml_file(test_file, "a.x", "foo")
+        self.assertEqual("\"Key 'a.x' not found in YAML!\"", str(ex.value))
+
+        with pytest.raises(KeyError) as ex:
+            update_yaml_file(test_file, "a.[42]", "foo")
+        self.assertEqual("\"Key 'a.[42]' not found in YAML!\"", str(ex.value))
+
+        with pytest.raises(KeyError) as ex:
+            update_yaml_file(test_file, "a.e.[3]", "foo")
+        self.assertEqual("\"Key 'a.e.[3]' not found in YAML!\"", str(ex.value))
+
+        with pytest.raises(KeyError) as ex:
+            update_yaml_file(test_file, "a.e.[2].[2]", "foo")
+        self.assertEqual("\"Key 'a.e.[2].[2]' not found in YAML!\"", str(ex.value))
 
-        expected = """\
-a: # comment
-# comment
-  b:
-    d: 1 # comment
-    c: '2' # comment
-  x:
-    z: foo_z
-    y:
-      z: foo_y_z
-"""
         actual = self._read_file(test_file)
         self.assertEqual(expected, actual)