diff --git a/src/confcom/.gitignore b/src/confcom/.gitignore index 562e4134172..a80f53ac125 100644 --- a/src/confcom/.gitignore +++ b/src/confcom/.gitignore @@ -36,3 +36,5 @@ azext_confcom/bin/* **/.coverage **/htmlcov + +!lib/ \ No newline at end of file diff --git a/src/confcom/azext_confcom/container.py b/src/confcom/azext_confcom/container.py index fcdc063f33b..0b252a6b3b9 100644 --- a/src/confcom/azext_confcom/container.py +++ b/src/confcom/azext_confcom/container.py @@ -791,12 +791,17 @@ def from_json( ) -> "UserContainerImage": image = super().from_json(container_json) image.__class__ = UserContainerImage + mount_paths = {m["mountPath"] for m in image.get_mounts()} # inject default mounts for user container - if (image.base not in config.BASELINE_SIDECAR_CONTAINERS) and (not is_vn2): - image.get_mounts().extend(_DEFAULT_MOUNTS) + if image.base not in config.BASELINE_SIDECAR_CONTAINERS and not is_vn2: + for mount in _DEFAULT_MOUNTS: + if mount["mountPath"] not in mount_paths: + image.get_mounts().append(mount) if (image.base not in config.BASELINE_SIDECAR_CONTAINERS) and (is_vn2): - image.get_mounts().extend(_DEFAULT_MOUNTS_VN2) + for mount in _DEFAULT_MOUNTS_VN2: + if mount["mountPath"] not in mount_paths: + image.get_mounts().append(mount) # Start with the customer environment rules env_rules = copy.deepcopy(_INJECTED_CUSTOMER_ENV_RULES) diff --git a/src/confcom/azext_confcom/custom.py b/src/confcom/azext_confcom/custom.py index 4405fefcc14..7dabb5f3a89 100644 --- a/src/confcom/azext_confcom/custom.py +++ b/src/confcom/azext_confcom/custom.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +import json import os import sys @@ -17,7 +18,7 @@ from azext_confcom.kata_proxy import KataPolicyGenProxy from azext_confcom.security_policy import OutputType from azext_confcom.template_util import ( - get_image_name, inject_policy_into_template, inject_policy_into_yaml, + extract_confidential_properties, get_image_name, inject_policy_into_template, inject_policy_into_yaml, pretty_print_func, print_existing_policy_from_arm_template, print_existing_policy_from_yaml, print_func, str_to_sha256) from knack.log import get_logger @@ -123,8 +124,7 @@ def acipolicygen_confcom( debug_mode=debug_mode, disable_stdio=disable_stdio, approve_wildcards=approve_wildcards, - diff_mode=diff, - rego_imports=fragments_list, + included_fragments=fragments_list, exclude_default_fragments=exclude_default_fragments, ) elif image_name: @@ -168,7 +168,23 @@ def acipolicygen_confcom( for policy in container_group_policies: policy.set_fragment_contents(fragment_policy_list) - for count, policy in enumerate(container_group_policies): + for idx, policy in enumerate(container_group_policies): + + # We will deprecate diff mode in favour of a separate tool, so we want + # supporting code to be all in one place even if it makes it more nasty + if diff: + if arm_template: + with open(arm_template, 'r') as f: + # pylint: disable=protected-access + policy._existing_cce_policy = extract_confidential_properties( + [r for r in json.load(f)["resources"] if r["type"] in { + "Microsoft.ContainerInstance/containerGroups", + "Microsoft.ContainerInstance/containerGroupProfiles", + }][idx].get("properties", {}))[0] + + elif virtual_node_yaml_path: + ... # diff mode is handled in the load function + # this is where parameters and variables are populated policy.populate_policy_content_for_all_images( individual_image=bool(image_name), tar_mapping=tar_mapping, faster_hashing=faster_hashing @@ -178,7 +194,7 @@ def acipolicygen_confcom( exit_code = validate_sidecar_in_policy(policy, output_type == security_policy.OutputType.PRETTY_PRINT) elif virtual_node_yaml_path and not (print_policy_to_terminal or outraw or outraw_pretty_print or diff): result = inject_policy_into_yaml( - virtual_node_yaml_path, policy.get_serialized_output(omit_id=omit_id), count + virtual_node_yaml_path, policy.get_serialized_output(omit_id=omit_id), idx ) if result: print(str_to_sha256(policy.get_serialized_output(OutputType.RAW, omit_id=omit_id))) @@ -187,7 +203,7 @@ def acipolicygen_confcom( exit_code = get_diff_outputs(policy, output_type == security_policy.OutputType.PRETTY_PRINT) elif arm_template and not (print_policy_to_terminal or outraw or outraw_pretty_print): result = inject_policy_into_template(arm_template, arm_template_parameters, - policy.get_serialized_output(omit_id=omit_id), count) + policy.get_serialized_output(omit_id=omit_id), idx) if result: # this is always going to be the unencoded policy print(str_to_sha256(policy.get_serialized_output(OutputType.RAW, omit_id=omit_id))) diff --git a/src/confcom/azext_confcom/lib/aci_policy_spec.py b/src/confcom/azext_confcom/lib/aci_policy_spec.py new file mode 100644 index 00000000000..52c39a1a979 --- /dev/null +++ b/src/confcom/azext_confcom/lib/aci_policy_spec.py @@ -0,0 +1,84 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from dataclasses import dataclass +from typing import Optional +from typing_extensions import Literal + + +@dataclass +class AciContainerPropertyEnvVariable: + name: str + value: str + strategy: str + required: Optional[bool] = False + + +@dataclass +class AciContainerPropertyExecProcesses: + command: list[str] + signals: Optional[list[str]] = None + allow_stdio_access: bool = True + + +@dataclass +class AciContainerPropertyVolumeMounts: + mountPath: str + name: Optional[str] = None + readonly: bool = False + mountType: Optional[Literal["azureFile", "secret", "configMap", "emptyDir"]] = None + + +@dataclass +class AciContainerPropertySecurityContextCapabilities: + add: list[str] + drop: list[str] + + +@dataclass +class AciContainerPropertySecurityContext: + privileged: Optional[bool] = None + allowPrivilegeEscalation: Optional[bool] = None + runAsUser: Optional[int] = None + runAsGroup: Optional[int] = None + runAsNonRoot: Optional[bool] = None + readOnlyRootFilesystem: Optional[bool] = None + capabilities: Optional[AciContainerPropertySecurityContextCapabilities] = None + seccompProfile: Optional[str] = None + + +@dataclass +class AciContainerProperties(): + image: str + allowStdioAccess: bool = True + environmentVariables: Optional[list[AciContainerPropertyEnvVariable]] = None + execProcesses: Optional[list[AciContainerPropertyExecProcesses]] = None + volumeMounts: Optional[list[AciContainerPropertyVolumeMounts]] = None + securityContext: Optional[AciContainerPropertySecurityContext] = None + command: Optional[list[str]] = None + + +# ------------------------------------------------------------------------------ + + +@dataclass +class AciFragmentSpec: + feed: str + issuer: str + minimum_svn: str + includes: list[Literal["containers", "fragments"]] + path: Optional[str] = None + + +@dataclass +class AciContainerSpec: + name: str + properties: AciContainerProperties + + +@dataclass +class AciPolicySpec: + fragments: Optional[list[AciFragmentSpec]] + containers: Optional[list[AciContainerSpec]] diff --git a/src/confcom/azext_confcom/lib/arm_to_aci_policy_spec.py b/src/confcom/azext_confcom/lib/arm_to_aci_policy_spec.py new file mode 100644 index 00000000000..c2711563f30 --- /dev/null +++ b/src/confcom/azext_confcom/lib/arm_to_aci_policy_spec.py @@ -0,0 +1,245 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from typing import Iterator +import json +import re +from azext_confcom import config +from azext_confcom.template_util import ( + get_probe_exec_processes, + is_sidecar, + process_configmap, + process_env_vars_from_template, + process_mounts +) +from azext_confcom.lib.aci_policy_spec import ( + AciContainerPropertyEnvVariable, + AciContainerPropertyExecProcesses, + AciContainerPropertySecurityContext, + AciContainerPropertySecurityContextCapabilities, + AciContainerPropertyVolumeMounts, + AciContainerSpec, + AciContainerProperties, + AciFragmentSpec, + AciPolicySpec, +) + + +def get_parameters( + arm_template: dict, + arm_template_parameters: dict, +) -> dict: + + return { + parameter_key: ( + arm_template_parameters.get("parameters", {}).get(parameter_key, {}).get("value") + or arm_template.get("parameters", {}).get(parameter_key, {}).get("value") + or arm_template.get("parameters", {}).get(parameter_key, {}).get("defaultValue") + ) + for parameter_key in arm_template.get("parameters", {}).keys() + } + + +def eval_parameters( + arm_template: dict, + arm_template_parameters: dict, +) -> dict: + + parameters = get_parameters(arm_template, arm_template_parameters) + return json.loads(re.compile(r"\[parameters\(\s*'([^']+)'\s*\)\]").sub( + lambda match: json.dumps(parameters.get(match.group(1)) or match.group(0))[1:-1], + json.dumps(arm_template), + )) + + +def eval_variables( + arm_template: dict, + arm_template_parameters: dict, +) -> dict: + + variables = arm_template.get("variables", {}) + return json.loads(re.compile(r"\[variables\(\s*'([^']+)'\s*\)\]").sub( + lambda match: json.dumps(variables.get(match.group(1), match.group(0)))[1:-1], + json.dumps(arm_template), + )) + + +EVAL_FUNCS = [ + eval_parameters, + eval_variables, +] + + +def arm_container_env_to_aci_policy_spec_env( + container_properties: dict, + parameters: dict, + approve_wildcards: bool, +) -> Iterator[AciContainerPropertyEnvVariable]: + + for env_var in [ + *process_env_vars_from_template(parameters, {}, container_properties, approve_wildcards), + ]: + yield AciContainerPropertyEnvVariable( + # At time of writing, we only get env vars from process_env_vars_from_template + # which never specifies "required", however futures sources might so + # we need to handle both in a way the type system can understand + required=bool(env_var.pop("required")) if "required" in env_var else None, + **env_var + ) + + +def arm_container_volumes_to_aci_policy_spec_volumes( + container_properties: dict, + container_group_volumes: list[dict], +) -> Iterator[AciContainerPropertyVolumeMounts]: + + for vol_mount in [ + *process_mounts(container_properties, container_group_volumes), + *process_configmap(container_properties), + *( + config.DEFAULT_MOUNTS_USER + if not is_sidecar(container_properties["image"]) else [] + ) + ]: + yield AciContainerPropertyVolumeMounts( + **{k: v for k, v in vol_mount.items() if v is not None} + ) + + +def arm_container_exec_procs_to_aci_policy_spec_exec_procs( + container_properties: dict, + debug_mode: bool, +) -> Iterator[AciContainerPropertyExecProcesses]: + + for exec_process in [ + *container_properties.get("execProcesses", []), + *get_probe_exec_processes(container_properties), + *(config.DEBUG_MODE_SETTINGS.get("execProcesses", []) if debug_mode else []), + ]: + yield AciContainerPropertyExecProcesses(**exec_process) + + +def arm_container_props_to_aci_policy_spec_props( + container_group: dict, + container_properties: dict, + parameters: dict, + debug_mode: bool, + allow_stdio_access: bool, + approve_wildcards: bool, +) -> AciContainerProperties: + + capabilities = container_properties.get("securityContext", {}).pop("capabilities", None) + + return AciContainerProperties( + image=container_properties["image"], + command=container_properties.get("command", []), + allowStdioAccess=allow_stdio_access, + environmentVariables=list(arm_container_env_to_aci_policy_spec_env( + container_properties=container_properties, + parameters=parameters, + approve_wildcards=approve_wildcards, + )), + volumeMounts=list(arm_container_volumes_to_aci_policy_spec_volumes( + container_properties=container_properties, + container_group_volumes=container_group["properties"].get("volumes", [])), + ), + execProcesses=list(arm_container_exec_procs_to_aci_policy_spec_exec_procs( + container_properties=container_properties, + debug_mode=debug_mode, + )), + securityContext=AciContainerPropertySecurityContext( + capabilities=AciContainerPropertySecurityContextCapabilities( + add=capabilities.get("add", []), + drop=capabilities.get("drop", []), + ) if capabilities else None, + **container_properties["securityContext"] + ) if "securityContext" in container_properties else None, + ) + + +def arm_container_to_aci_policy_spec_container( + container_group: dict, + container: dict, + parameters: dict, + debug_mode: bool, + allow_stdio_access: bool, + approve_wildcards: bool, +) -> AciContainerSpec: + + return AciContainerSpec( + name=container["name"], + properties=arm_container_props_to_aci_policy_spec_props( + container_group=container_group, + container_properties=container["properties"], + parameters=parameters, + debug_mode=debug_mode, + allow_stdio_access=allow_stdio_access, + approve_wildcards=approve_wildcards, + ), + ) + + +def arm_container_group_to_aci_policy_spec_fragments( + container_group: dict, +) -> Iterator[AciFragmentSpec]: + + for fragment in container_group.get("properties", {}).get("standaloneFragments", []): + yield AciFragmentSpec(**fragment) + + +def arm_container_group_to_aci_policy_spec( + container_group: dict, + parameters: dict, + fragments: list[AciFragmentSpec], + debug_mode: bool, + allow_stdio_access: bool, + approve_wildcards: bool, +) -> AciPolicySpec: + + containers = container_group.get("properties", {})["containers"] + assert containers + + return AciPolicySpec( + fragments=[ + *(fragments if not container_group.get("tags", {}).get("Annotate-zero-sidecar") else []), + *arm_container_group_to_aci_policy_spec_fragments(container_group), + ], + containers=[ + arm_container_to_aci_policy_spec_container( + container_group=container_group, + container=c, + parameters=parameters, + debug_mode=debug_mode, + allow_stdio_access=allow_stdio_access, + approve_wildcards=approve_wildcards, + ) + for c in containers + container_group.get("properties", {}).get("initContainers", []) + ] + ) + + +def arm_to_aci_policy_spec( + arm_template: dict, + arm_template_parameters: dict, + fragments: list[AciFragmentSpec], + debug_mode: bool = False, + allow_stdio_access: bool = True, + approve_wildcards: bool = False, +) -> Iterator[AciPolicySpec]: + + for eval_func in EVAL_FUNCS: + arm_template = eval_func(arm_template, arm_template_parameters) + + parameters = arm_template.get("parameters", {}) + + for resource in arm_template.get("resources", []): + parser = { + "Microsoft.ContainerInstance/containerGroups": arm_container_group_to_aci_policy_spec, + "Microsoft.ContainerInstance/containerGroupProfiles": arm_container_group_to_aci_policy_spec, + }.get(resource["type"], (lambda r, p, f, d, io, w: None)) + + spec = parser(resource, parameters, fragments, debug_mode, allow_stdio_access, approve_wildcards) + if spec is not None: + yield spec diff --git a/src/confcom/azext_confcom/lib/image_refs_to_aci_policy_spec.py b/src/confcom/azext_confcom/lib/image_refs_to_aci_policy_spec.py new file mode 100644 index 00000000000..875d8a9ccc7 --- /dev/null +++ b/src/confcom/azext_confcom/lib/image_refs_to_aci_policy_spec.py @@ -0,0 +1,39 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from azext_confcom.lib.aci_policy_spec import ( + AciContainerSpec, + AciContainerProperties, + AciFragmentSpec, + AciPolicySpec, +) + + +def image_ref_to_aci_container_spec( + image_ref: str, +) -> AciContainerSpec: + + return AciContainerSpec( + name=image_ref, + properties=AciContainerProperties( + image=image_ref, + ) + ) + + +def image_refs_to_aci_policy_spec( + image_refs: list[str], + fragments: list[AciFragmentSpec], +) -> AciPolicySpec: + + return AciPolicySpec( + fragments=[ + *fragments, + ], + containers=[ + image_ref_to_aci_container_spec(image_ref) + for image_ref in image_refs + ] + ) diff --git a/src/confcom/azext_confcom/oras_proxy.py b/src/confcom/azext_confcom/oras_proxy.py index a53f69127c0..36c9c091dca 100644 --- a/src/confcom/azext_confcom/oras_proxy.py +++ b/src/confcom/azext_confcom/oras_proxy.py @@ -191,7 +191,7 @@ def pull_all_standalone_fragments(fragment_imports): proxy = CoseSignToolProxy() for fragment in fragment_imports: - if fragment in DEFAULT_REGO_FRAGMENTS: + if any(fragment["feed"] == default_fragment["feed"] for default_fragment in DEFAULT_REGO_FRAGMENTS): continue path = fragment.get("path") feed = fragment.get("feed") diff --git a/src/confcom/azext_confcom/security_policy.py b/src/confcom/azext_confcom/security_policy.py index 8ab29f52032..0ddcb1dfb7a 100644 --- a/src/confcom/azext_confcom/security_policy.py +++ b/src/confcom/azext_confcom/security_policy.py @@ -6,15 +6,17 @@ import copy import json import warnings +import deepdiff from enum import Enum, auto from typing import Any, Dict, List, Tuple, Union +from dataclasses import asdict -import deepdiff -from azext_confcom import config, os_util +from azext_confcom.lib.aci_policy_spec import AciFragmentSpec +from azext_confcom.lib.arm_to_aci_policy_spec import arm_to_aci_policy_spec +from azext_confcom import (config, os_util) from azext_confcom.container import ContainerImage, UserContainerImage from azext_confcom.errors import eprint from azext_confcom.fragment_util import sanitize_fragment_fields -from azext_confcom.oras_proxy import create_list_of_standalone_imports from azext_confcom.rootfs_proxy import SecurityPolicyProxy from azext_confcom.template_util import (case_insensitive_dict_get, compare_env_vars, @@ -22,22 +24,17 @@ convert_to_pod_spec, decompose_confidential_properties, detect_old_format, - extract_confidential_properties, extract_lifecycle_hook, extract_probe, - extract_standalone_fragments, filter_non_pod_resources, get_container_diff, get_diff_size, get_image_info, get_tar_location_from_mapping, - get_values_for_params, get_volume_claim_templates, is_sidecar, pretty_print_func, print_func, process_configmap, process_env_vars_from_config, - process_env_vars_from_template, process_env_vars_from_yaml, process_fragment_imports, - process_mounts, process_mounts_from_config, readable_diff) from knack.log import get_logger @@ -625,191 +622,48 @@ def load_policy_from_arm_template_str( debug_mode: bool = False, disable_stdio: bool = False, approve_wildcards: bool = False, - diff_mode: bool = False, - rego_imports: Any = None, - fragment_contents: Any = None, + included_fragments: list[dict[str, Any]] = None, exclude_default_fragments: bool = False, ) -> List[AciPolicy]: - """Function that converts ARM template string to an ACI Policy""" - input_arm_json = os_util.load_json_from_str(template_data) - - input_parameter_json = {} - if parameter_data: - input_parameter_json = os_util.load_json_from_str(parameter_data) - # find the image names and extract them from the template - arm_resources = case_insensitive_dict_get( - input_arm_json, config.ACI_FIELD_RESOURCES - ) - - if not arm_resources: - eprint(f"Field [{config.ACI_FIELD_RESOURCES}] is empty or cannot be found") + aci_policies = [] - aci_list = [ - item - for item in arm_resources - if item["type"] in config.ACI_FIELD_SUPPORTED_RESOURCES - ] + if included_fragments is None: + included_fragments = [] - if not aci_list: + if not exclude_default_fragments: + for idx, fragment in enumerate(config.DEFAULT_REGO_FRAGMENTS): + if infrastructure_svn: + fragment["minimum_svn"] = infrastructure_svn + included_fragments.insert(idx, fragment) + + try: + for policy_spec in arm_to_aci_policy_spec( + arm_template=json.loads(template_data), + arm_template_parameters=json.loads(parameter_data) if parameter_data else {}, + fragments=[AciFragmentSpec(**fragment) for fragment in included_fragments], + debug_mode=debug_mode, + allow_stdio_access=not disable_stdio, + approve_wildcards=approve_wildcards, + ): + aci_policies.append(load_policy_from_json( + json.dumps(asdict(policy_spec)), + debug_mode, + disable_stdio, + infrastructure_svn, + # Fragments are already parsed + True, + )) + # Catch broad exception since we don't want to assume what errors might occur pylint: disable=W0718 + except Exception as e: + eprint(f"Error processing ARM template: {e}") + + if len(aci_policies) == 0: eprint( - f'Field ["type"] must contain one of {config.ACI_FIELD_SUPPORTED_RESOURCES}' - ) - - # extract variables and parameters in case we need to do substitutions - # while searching for image names - all_params = ( - case_insensitive_dict_get(input_arm_json, config.ACI_FIELD_TEMPLATE_PARAMETERS) - or {} - ) - - get_values_for_params(input_parameter_json, all_params) - - AciPolicy.all_params = all_params - AciPolicy.all_vars = case_insensitive_dict_get(input_arm_json, config.ACI_FIELD_TEMPLATE_VARIABLES) or {} - - container_groups = [] - - for resource in aci_list: - # initialize the list of containers we need to generate policies for - containers = [] - existing_containers = None - fragments = None - exclude_default_fragments = False - - tags = case_insensitive_dict_get(resource, config.ACI_FIELD_TEMPLATE_TAGS) - if tags: - exclude_default_fragments = case_insensitive_dict_get(tags, config.ACI_FIELD_TEMPLATE_ZERO_SIDECAR) - if isinstance(exclude_default_fragments, str): - exclude_default_fragments = exclude_default_fragments.lower() == "true" - - container_group_properties = case_insensitive_dict_get( - resource, config.ACI_FIELD_TEMPLATE_PROPERTIES - ) - container_list = case_insensitive_dict_get( - container_group_properties, config.ACI_FIELD_TEMPLATE_CONTAINERS + f'At least one resource must have ["type"] equalling one of {config.ACI_FIELD_SUPPORTED_RESOURCES}' ) - if not container_list: - eprint( - f'Field ["{config.POLICY_FIELD_CONTAINERS}"] must be a list of {config.POLICY_FIELD_CONTAINERS}' - ) - - init_container_list = case_insensitive_dict_get( - container_group_properties, config.ACI_FIELD_TEMPLATE_INIT_CONTAINERS - ) - # add init containers to the list of other containers since they aren't treated differently - # in the security policy - if init_container_list: - container_list.extend(init_container_list) - - # these are standalone fragments coming from the ARM template itself - standalone_fragments = extract_standalone_fragments(container_group_properties) - if standalone_fragments: - standalone_fragment_imports = create_list_of_standalone_imports(standalone_fragments) - unique_imports = set(rego_imports) - for fragment in standalone_fragment_imports: - if fragment not in unique_imports: - rego_imports.append(fragment) - unique_imports.add(fragment) - - try: - existing_containers, fragments = extract_confidential_properties( - container_group_properties - ) - except ValueError as e: - if diff_mode: - # In diff mode, we raise an error if the base64 policy is malformed - eprint(f"Unable to decode existing policy. Please check the base64 encoding.\n{e}") - else: - # In non-diff mode, we ignore the error and proceed without the policy - existing_containers, fragments = ([], []) - - rego_fragments = copy.deepcopy(config.DEFAULT_REGO_FRAGMENTS) if not exclude_default_fragments else [] - if infrastructure_svn: - # assumes the first DEFAULT_REGO_FRAGMENT is always the - # infrastructure fragment - rego_fragments[0][ - config.POLICY_FIELD_CONTAINERS_ELEMENTS_REGO_FRAGMENTS_MINIMUM_SVN - ] = infrastructure_svn - if rego_imports: - # error check the rego imports for invalid data types - processed_imports = process_fragment_imports(rego_imports) - rego_fragments.extend(processed_imports) - - volumes = ( - case_insensitive_dict_get( - container_group_properties, config.ACI_FIELD_TEMPLATE_VOLUMES - ) - or [] - ) - if volumes and not isinstance(volumes, list): - # parameter definition is in parameter file but not arm template - eprint(f'Parameter ["{config.ACI_FIELD_TEMPLATE_VOLUMES}"] must be a list') - - for container in container_list: - image_properties = case_insensitive_dict_get( - container, config.ACI_FIELD_TEMPLATE_PROPERTIES - ) - image_name = case_insensitive_dict_get( - image_properties, config.ACI_FIELD_TEMPLATE_IMAGE - ) - - # this is guaranteed unique for a valid ARM template - container_name = case_insensitive_dict_get( - container, config.ACI_FIELD_CONTAINERS_NAME - ) - - if not image_name: - eprint( - f'Field ["{config.ACI_FIELD_TEMPLATE_IMAGE}"] is empty or cannot be found' - ) - - exec_processes = [] - extract_probe(exec_processes, image_properties, config.ACI_FIELD_CONTAINERS_READINESS_PROBE) - extract_probe(exec_processes, image_properties, config.ACI_FIELD_CONTAINERS_LIVENESS_PROBE) - - containers.append( - { - config.ACI_FIELD_CONTAINERS_ID: image_name, - config.ACI_FIELD_CONTAINERS_NAME: container_name, - config.ACI_FIELD_CONTAINERS_CONTAINERIMAGE: image_name, - config.ACI_FIELD_CONTAINERS_ENVS: process_env_vars_from_template( - AciPolicy.all_params, AciPolicy.all_vars, image_properties, approve_wildcards), - config.ACI_FIELD_CONTAINERS_COMMAND: case_insensitive_dict_get( - image_properties, config.ACI_FIELD_TEMPLATE_COMMAND - ) - or [], - config.ACI_FIELD_CONTAINERS_MOUNTS: process_mounts(image_properties, volumes) - + process_configmap(image_properties), - config.ACI_FIELD_CONTAINERS_EXEC_PROCESSES: exec_processes - + config.DEBUG_MODE_SETTINGS.get(config.ACI_FIELD_CONTAINERS_EXEC_PROCESSES) - if debug_mode - else exec_processes, - config.ACI_FIELD_CONTAINERS_SIGNAL_CONTAINER_PROCESSES: [], - config.ACI_FIELD_CONTAINERS_ALLOW_STDIO_ACCESS: not disable_stdio, - config.ACI_FIELD_CONTAINERS_SECURITY_CONTEXT: case_insensitive_dict_get( - image_properties, config.ACI_FIELD_TEMPLATE_SECURITY_CONTEXT - ), - } - ) - - container_groups.append( - AciPolicy( - { - config.ACI_FIELD_VERSION: "1.0", - config.ACI_FIELD_CONTAINERS: containers, - config.ACI_FIELD_TEMPLATE_CCE_POLICY: existing_containers, - }, - disable_stdio=disable_stdio, - rego_fragments=rego_fragments, - # fallback to default fragments if the policy is not present - existing_rego_fragments=fragments, - debug_mode=debug_mode, - fragment_contents=fragment_contents, - ) - ) - return container_groups + return aci_policies def load_policy_from_arm_template_file( @@ -819,9 +673,7 @@ def load_policy_from_arm_template_file( debug_mode: bool = False, disable_stdio: bool = False, approve_wildcards: bool = False, - diff_mode: bool = False, - rego_imports: list = None, - fragment_contents: list = None, + included_fragments: list[dict[str, Any]] = None, exclude_default_fragments: bool = False, ) -> List[AciPolicy]: """Utility function: generate policy object from given arm template and parameter file paths""" @@ -836,9 +688,7 @@ def load_policy_from_arm_template_file( debug_mode=debug_mode, disable_stdio=disable_stdio, approve_wildcards=approve_wildcards, - rego_imports=rego_imports, - diff_mode=diff_mode, - fragment_contents=fragment_contents, + included_fragments=included_fragments, exclude_default_fragments=exclude_default_fragments, ) @@ -1009,6 +859,11 @@ def load_policy_from_json( envs += process_env_vars_from_config(container_properties) + if debug_mode: + for exec_process in config.DEBUG_MODE_SETTINGS.get(config.ACI_FIELD_CONTAINERS_EXEC_PROCESSES, []): + if exec_process not in exec_processes: + exec_processes.append(exec_process) + output_containers.append( { config.ACI_FIELD_CONTAINERS_ID: image_name, @@ -1020,10 +875,7 @@ def load_policy_from_json( container_properties, config.ACI_FIELD_TEMPLATE_COMMAND ) or [], config.ACI_FIELD_CONTAINERS_MOUNTS: mounts, - config.ACI_FIELD_CONTAINERS_EXEC_PROCESSES: exec_processes - + config.DEBUG_MODE_SETTINGS.get(config.ACI_FIELD_CONTAINERS_EXEC_PROCESSES) - if debug_mode - else exec_processes, + config.ACI_FIELD_CONTAINERS_EXEC_PROCESSES: exec_processes, config.ACI_FIELD_CONTAINERS_SIGNAL_CONTAINER_PROCESSES: [], config.ACI_FIELD_CONTAINERS_ALLOW_STDIO_ACCESS: not disable_stdio, config.ACI_FIELD_CONTAINERS_SECURITY_CONTEXT: case_insensitive_dict_get( diff --git a/src/confcom/azext_confcom/template_util.py b/src/confcom/azext_confcom/template_util.py index c968695d5f7..3674c498405 100644 --- a/src/confcom/azext_confcom/template_util.py +++ b/src/confcom/azext_confcom/template_util.py @@ -520,7 +520,7 @@ def process_env_vars_from_config(container) -> List[Dict[str, str]]: config.ACI_FIELD_CONTAINERS_ENVS_NAME: name, config.ACI_FIELD_CONTAINERS_ENVS_VALUE: value, config.ACI_FIELD_CONTAINERS_ENVS_STRATEGY: - "re2" if case_insensitive_dict_get(env_var, "regex") else "string", + env_var.get("strategy", "re2" if (case_insensitive_dict_get(env_var, "regex")) else "string"), }) return env_vars @@ -789,6 +789,13 @@ def extract_probe(exec_processes: List[dict], image_properties: dict, probe: str }) +def get_probe_exec_processes(image_properties: dict) -> List[dict]: + exec_processes: List[dict] = [] + extract_probe(exec_processes, image_properties, config.ACI_FIELD_CONTAINERS_READINESS_PROBE) + extract_probe(exec_processes, image_properties, config.ACI_FIELD_CONTAINERS_LIVENESS_PROBE) + return exec_processes + + def extract_lifecycle_hook(exec_processes: List[dict], image_properties: dict, hook: str): lifecycle = case_insensitive_dict_get( image_properties, config.VIRTUAL_NODE_YAML_LIFECYCLE diff --git a/src/confcom/azext_confcom/tests/latest/test_confcom_types.py b/src/confcom/azext_confcom/tests/latest/test_confcom_types.py new file mode 100644 index 00000000000..645c075c64b --- /dev/null +++ b/src/confcom/azext_confcom/tests/latest/test_confcom_types.py @@ -0,0 +1,81 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from pathlib import Path + +import pytest + +try: + from mypy import api as mypy_api +except ImportError: + mypy_api = None + + +@pytest.fixture(scope="module", autouse=True) +def check_mypy(): + if mypy_api is None: + pytest.skip("missing mypy import") + + +AZEXT_ROOT = Path(__file__).resolve().parents[2] +MYPY_ARGS = [ + "--explicit-package-bases", +] + + +def _discover_modules() -> list[tuple[str, Path]]: + modules: list[tuple[str, Path]] = [] + for path in sorted(AZEXT_ROOT.rglob("*.py")): + if not path.is_file(): + continue + rel = path.relative_to(AZEXT_ROOT) + if rel.parts[0] == "tests": + continue + module_name = f"{rel.with_suffix('').as_posix().replace('/', '.')}" + modules.append((module_name, path)) + return modules + + +MODULE_PATHS = _discover_modules() + + +# These files already existed when this test was added and will be fixed incrementally +BAD_MODULES = { + "__init__", + "_params", + "_validators", + "_help", + "config", + "container", + "cose_proxy", + "custom", + "errors", + "fragment_util", + "init_checks", + "kata_proxy", + "oras_proxy", + "os_util", + "rootfs_proxy", + "security_policy", + "template_util", +} + + +@pytest.mark.parametrize( + "module_name, target_path", + [pytest.param(name, path, id=name) for name, path in MODULE_PATHS], +) +def test_mypy(module_name: str, target_path: Path): + + if any(module_name.startswith(bad_module) for bad_module in BAD_MODULES): + pytest.skip(f"Skipping mypy test for {module_name} due to known issues") + + stdout, stderr, exit_status = mypy_api.run([*MYPY_ARGS, str(target_path)]) + assert exit_status == 0, ( + f"mypy reported issues for {module_name}\n" + f"command: mypy {' '.join(MYPY_ARGS + [str(target_path)])}\n" + f"stdout:\n{stdout}\n" + f"stderr:\n{stderr}" + )