Merge pull request #1284 from stackhpc/check-image-map

zed: Fix image mapping in kolla-images.py, add tests and CI

.github/path-filters.yml

@@ -1,5 +1,5 @@
 # This file is a list of path filters for the PR workflow in .github/workflows/stackhpc-pull-request.yml.
-aio:
+aio: &aio
   - '.automation'
   - '.automation.conf/config.sh'
   - '.automation.conf/tempest/load-lists/default'
@@ -20,6 +20,11 @@ aio:
   - 'kayobe-env'
   - 'requirements.txt'
   - 'terraform/aio/**'
-check-tags:
+check-tags: &check-tags
   - '.github/workflows/stackhpc-check-tags.yml'
   - 'etc/kayobe/kolla-image-tags.yml'
+  - 'etc/kayobe/pulp.yml'
+  - 'tools/kolla-images.py'
+build-kayobe-image:
+  - *aio
+  - *check-tags

.github/workflows/stackhpc-check-tags.yml

@@ -46,6 +46,15 @@ jobs:
         run: |
           docker image pull $KAYOBE_IMAGE
 
+      - name: Check kolla-images.py image map and tag hierarchy
+        run: |
+          docker run -t --rm \
+            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
+            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
+            $KAYOBE_IMAGE \
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh \
+            '$KAYOBE_CONFIG_PATH/ansible/check-kolla-images-py.yml'
+
       - name: Check container image tags
         run: |
           docker run -t --rm \

.github/workflows/stackhpc-pull-request.yml

@@ -20,6 +20,7 @@ jobs:
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
     outputs:
       aio: ${{ steps.changes.outputs.aio }}
+      build-kayobe-image: ${{ steps.changes.outputs.build-kayobe-image }}
       check-tags: ${{ steps.changes.outputs.check-tags }}
     steps:
       - name: GitHub Checkout
@@ -74,7 +75,7 @@ jobs:
       - check-changes
     uses: ./.github/workflows/stackhpc-build-kayobe-image.yml
     with:
-      if: ${{ needs.check-changes.outputs.aio == 'true' }}
+      if: ${{ needs.check-changes.outputs.build-kayobe-image == 'true' }}
     if: github.repository == 'stackhpc/stackhpc-kayobe-config'
 
   check-tags:

etc/kayobe/ansible/check-kolla-images-py.yml

@@ -0,0 +1,35 @@
+---
+- name: Check kolla-images.py image map and tag hierarchy
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Create a temporary directory
+      ansible.builtin.tempfile:
+        state: directory
+        suffix: kolla-ansible
+      register: tempdir_result
+
+    - name: Clone Kolla Ansible repository
+      ansible.builtin.git:
+        repo: "{{ stackhpc_kolla_ansible_source_url }}"
+        version: "{{ stackhpc_kolla_ansible_source_version }}"
+        dest: "{{ tempdir_result.path }}"
+
+    - name: Check image mapping
+      ansible.builtin.command:
+        cmd: >-
+          {{ kayobe_config_path }}/../../tools/kolla-images.py
+          check-image-map
+          --kolla-ansible-path {{ tempdir_result.path }}
+
+    - name: Check tag hierarchy
+      ansible.builtin.command:
+        cmd: >-
+          {{ kayobe_config_path }}/../../tools/kolla-images.py
+          check-hierarchy
+          --kolla-ansible-path {{ tempdir_result.path }}
+
+    - name: Remove temporary directory
+      ansible.builtin.file:
+        path: "{{ tempdir_result.path }}"
+        state: absent

etc/kayobe/ansible/check-tags.yml

@@ -4,6 +4,7 @@
 
 - name: Check whether tags exist in Pulp container registry
   hosts: localhost
+  gather_facts: false
   tasks:
     - name: Query images and tags
       command:

tools/kolla-images.py

@@ -39,15 +39,20 @@
 
 # Maps a Kolla image to a list of containers that use the image.
 IMAGE_TO_CONTAINERS_EXCEPTIONS: Dict[str, List[str]] = {
+    "dnsmasq": [
+        "ironic_dnsmasq",
+    ],
     "haproxy": [
         "glance_tls_proxy",
+        "haproxy",
         "neutron_tls_proxy",
     ],
     "mariadb-server": [
         "mariadb",
         "mariabackup",
     ],
-    "neutron-eswitchd": [
+    "neutron-mlnx-agent": [
+        "neutron_eswitchd",
         "neutron_mlnx_agent",
     ],
     "neutron-metadata-agent": [
@@ -58,6 +63,15 @@
         "nova_super_conductor",
         "nova_conductor",
     ],
+    "openvswitch-db-server": [
+        "openvswitch_db",
+    ],
+    "ovn-nb-db-server": [
+        "ovn_nb_db",
+    ],
+    "ovn-sb-db-server": [
+        "ovn_sb_db",
+    ],
     "prometheus-v2-server": [
         "prometheus_server",
     ],
@@ -96,6 +110,9 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--base-distros", default=",".join(SUPPORTED_BASE_DISTROS), choices=SUPPORTED_BASE_DISTROS)
     subparsers = parser.add_subparsers(dest="command", required=True)
 
+    subparser = subparsers.add_parser("check-image-map", help="Check image mapping against kolla-ansible")
+    subparser.add_argument("--kolla-ansible-path", required=True, help="Path to kolla-ansible repostory checked out to correct branch")
+
     subparser = subparsers.add_parser("check-hierarchy", help="Check tag variable hierarchy against kolla-ansible")
     subparser.add_argument("--kolla-ansible-path", required=True, help="Path to kolla-ansible repostory checked out to correct branch")
 
@@ -114,7 +131,7 @@ def parse_args() -> argparse.Namespace:
     return parser.parse_args()
 
 
-def get_abs_path(relative_path: str) -> str:
+def get_abs_path(relative_path: str) -> pathlib.Path:
     """Return the absolute path of a file in SKC."""
     script_path = pathlib.Path(inspect.getfile(inspect.currentframe()))
     return script_path.parent.parent / relative_path
@@ -277,6 +294,45 @@ def check_tags(base_distros: List[str], kolla_image_tags: KollaImageTags, regist
         sys.exit(1)
 
 
+def check_image_map(kolla_ansible_path: str):
+    """Check the image mapping against Kolla Ansible variables.
+
+    The *_image variables in Kolla Ansible define the mapping between
+    containers and images. Ensure that the mapping defined in this script
+    matches the one in Kolla Ansible.
+    """
+    supported_images = read_images("etc/kayobe/pulp.yml")
+    assert supported_images
+    # Build a map from container to image name.
+    cmd = """git grep -h '^[a-z0-9_]*_image:' ansible/roles/*/defaults/main.yml"""
+    image_map_str = subprocess.check_output(cmd, shell=True, cwd=os.path.realpath(kolla_ansible_path))
+    image_map = yaml.safe_load(image_map_str)
+    image_var_re = re.compile(r"^([a-z0-9_]+)_image$")
+    image_map = {
+        image_var_re.match(image_var).group(1): image.split("/")[-1]
+        for image_var, image in image_map.items()
+    }
+    # Filter out unsupported images.
+    image_map = {
+        container: image
+        for container, image in image_map.items()
+        if image in supported_images
+    }
+    assert image_map
+    errors = []
+    # Check that our mapping is correct.
+    for container, image in image_map.items():
+        containers = get_containers(image)
+        if container not in containers:
+            errors.append((container, image))
+    if errors:
+        print("Errors:")
+    for tag_var, image in errors:
+        print(f"Expected {tag_var} container to use {image} image")
+    if errors:
+        sys.exit(1)
+
+
 def check_hierarchy(kolla_ansible_path: str):
     """Check the tag variable hierarchy against Kolla Ansible variables."""
     cmd = """git grep -h '^[a-z0-9_]*_tag:' ansible/roles/*/defaults/main.yml"""
@@ -352,7 +408,9 @@ def main():
 
     validate(kolla_image_tags)
 
-    if args.command == "check-hierarchy":
+    if args.command == "check-image-map":
+        check_image_map(args.kolla_ansible_path)
+    elif args.command == "check-hierarchy":
         check_hierarchy(args.kolla_ansible_path)
     elif args.command == "check-tags":
         check_tags(base_distros, kolla_image_tags, args.registry, args.namespace)