Fixes #716 Improve environment-mapping behavior in optional config fields

docs/how_to/config_deployment.md

@@ -239,7 +239,89 @@ constants:
         <env..>: <constant_value..>
 ```
 
-### Sample `config.yml` File
+## Environment-Specific Values
+
+All configuration fields support environment-specific values using a mapping format:
+
+```yaml
+core:
+    workspace_id:
+        dev: "dev-workspace-id"
+        test: "test-workspace-id"
+        prod: "prod-workspace-id"
+```
+
+### Required vs Optional Fields
+
+Fields are categorized as **required** or **optional**, which affects how missing environment values are handled:
+
+| Field                                   | Required | Environment Missing Behavior    |
+| --------------------------------------- | -------- | ------------------------------- |
+| `core.workspace_id` or `core.workspace` | ✅       | Validation error                |
+| `core.repository_directory`             | ✅       | Validation error                |
+| `core.item_types_in_scope`              | ❌       | Warning logged, setting skipped |
+| `core.parameter`                        | ❌       | Warning logged, setting skipped |
+| `publish.exclude_regex`                 | ❌       | Debug logged, setting skipped   |
+| `publish.folder_exclude_regex`          | ❌       | Debug logged, setting skipped   |
+| `publish.shortcut_exclude_regex`        | ❌       | Debug logged, setting skipped   |
+| `publish.items_to_include`              | ❌       | Debug logged, setting skipped   |
+| `publish.skip`                          | ❌       | Defaults to `False`             |
+| `unpublish.exclude_regex`               | ❌       | Debug logged, setting skipped   |
+| `unpublish.items_to_include`            | ❌       | Debug logged, setting skipped   |
+| `unpublish.skip`                        | ❌       | Defaults to `False`             |
+| `features`                              | ❌       | Debug logged, setting skipped   |
+| `constants`                             | ❌       | Debug logged, setting skipped   |
+
+### Selective Environment Configuration
+
+Optional fields allow you to apply settings to specific environments without affecting others. This is useful when you want different behavior per environment:
+
+```yaml
+core:
+    workspace_id:
+        dev: "dev-workspace-id"
+        test: "test-workspace-id"
+        prod: "prod-workspace-id"
+    repository_directory: "./workspace" # Same for all environments
+
+publish:
+    # Only exclude test folders in dev environment
+    folder_exclude_regex:
+        dev: "^test_.*"
+        # test and prod not specified - no folder exclusion applied
+
+    # Skip publish in dev, run in test and prod
+    skip:
+        dev: true
+        # test and prod default to false
+```
+
+In this example:
+
+-   Deploying to `dev`: `folder_exclude_regex` = `"^test_.*"`, `skip` = `true`
+-   Deploying to `test`: No folder exclusion applied, `skip` = `false`
+-   Deploying to `prod`: No folder exclusion applied, `skip` = `false`
+
+### Logging Behavior
+
+When an optional field uses environment mapping and does not include the target environment:
+
+-   **Important optional fields** (`item_types_in_scope`, `parameter`): A **warning** is logged to alert users that the setting is being skipped.
+-   **Other optional fields**: A **debug** message is logged, visible only when debug logging is enabled.
+
+Example log output when deploying to `prod` with the config above:
+
+```
+[Debug] - No value for 'folder_exclude_regex' in environment 'prod'. Available environments: ['dev']. This setting will be skipped.
+```
+
+To enable debug logging:
+
+```python
+change_log_level()
+```
+
+## Sample `config.yml` File
 
 ```yaml
 core:

src/fabric_cicd/_common/_config_utils.py

@@ -27,47 +27,64 @@ def load_config_file(config_file_path: str, environment: str, config_override: O
     return validator.validate_config_file(config_file_path, environment, config_override)
 
 
+def get_config_value(config_section: dict, key: str, environment: str) -> str | list | bool | None:
+    """Extract a value from config, handling both single and environment-specific formats.
+
+    Args:
+        config_section: The config section to extract from
+        key: The key to extract
+        environment: Target environment
+
+    Returns:
+        The extracted value, or None if key doesn't exist or environment not found in dict
+    """
+    if key not in config_section:
+        return None
+
+    value = config_section[key]
+
+    if isinstance(value, dict):
+        return value.get(environment)
+
+    return value
+
+
 def extract_workspace_settings(config: dict, environment: str) -> dict:
     """Extract workspace-specific settings from config for the given environment."""
     environment = environment.strip()
     core = config["core"]
     settings = {}
 
-    # Extract workspace ID or name based on environment
+    # Workspace ID or name - required, validation ensures environment exists
     if "workspace_id" in core:
         if isinstance(core["workspace_id"], dict):
             settings["workspace_id"] = core["workspace_id"][environment]
         else:
             settings["workspace_id"] = core["workspace_id"]
-
         logger.info(f"Using workspace ID '{settings['workspace_id']}'")
 
     elif "workspace" in core:
         if isinstance(core["workspace"], dict):
             settings["workspace_name"] = core["workspace"][environment]
         else:
             settings["workspace_name"] = core["workspace"]
-
         logger.info(f"Using workspace '{settings['workspace_name']}'")
 
-    # Extract other settings
+    # Repository directory - required, validation ensures environment exists
     if "repository_directory" in core:
         if isinstance(core["repository_directory"], dict):
             settings["repository_directory"] = core["repository_directory"][environment]
         else:
             settings["repository_directory"] = core["repository_directory"]
 
-    if "item_types_in_scope" in core:
-        if isinstance(core["item_types_in_scope"], dict):
-            settings["item_types_in_scope"] = core["item_types_in_scope"][environment]
-        else:
-            settings["item_types_in_scope"] = core["item_types_in_scope"]
+    # Optional settings - validation logs warning if environment not found
+    item_types_in_scope = get_config_value(core, "item_types_in_scope", environment)
+    if item_types_in_scope is not None:
+        settings["item_types_in_scope"] = item_types_in_scope
 
-    if "parameter" in core:
-        if isinstance(core["parameter"], dict):
-            settings["parameter_file_path"] = core["parameter"][environment]
-        else:
-            settings["parameter_file_path"] = core["parameter"]
+    parameter_file_path = get_config_value(core, "parameter", environment)
+    if parameter_file_path is not None:
+        settings["parameter_file_path"] = parameter_file_path
 
     return settings
 
@@ -76,38 +93,35 @@ def extract_publish_settings(config: dict, environment: str) -> dict:
     """Extract publish-specific settings from config for the given environment."""
     settings = {}
 
-    if "publish" in config:
-        publish_config = config["publish"]
-
-        if "exclude_regex" in publish_config:
-            if isinstance(publish_config["exclude_regex"], dict):
-                settings["exclude_regex"] = publish_config["exclude_regex"][environment]
-            else:
-                settings["exclude_regex"] = publish_config["exclude_regex"]
-
-        if "folder_exclude_regex" in publish_config:
-            if isinstance(publish_config["folder_exclude_regex"], dict):
-                settings["folder_exclude_regex"] = publish_config["folder_exclude_regex"][environment]
-            else:
-                settings["folder_exclude_regex"] = publish_config["folder_exclude_regex"]
-
-        if "items_to_include" in publish_config:
-            if isinstance(publish_config["items_to_include"], dict):
-                settings["items_to_include"] = publish_config["items_to_include"][environment]
-            else:
-                settings["items_to_include"] = publish_config["items_to_include"]
-
-        if "shortcut_exclude_regex" in publish_config:
-            if isinstance(publish_config["shortcut_exclude_regex"], dict):
-                settings["shortcut_exclude_regex"] = publish_config["shortcut_exclude_regex"][environment]
-            else:
-                settings["shortcut_exclude_regex"] = publish_config["shortcut_exclude_regex"]
-
-        if "skip" in publish_config:
-            if isinstance(publish_config["skip"], dict):
-                settings["skip"] = publish_config["skip"].get(environment, False)
-            else:
-                settings["skip"] = publish_config["skip"]
+    if "publish" not in config:
+        return settings
+
+    publish_config = config["publish"]
+
+    # Optional settings - validation logs debug if environment not found
+    exclude_regex = get_config_value(publish_config, "exclude_regex", environment)
+    if exclude_regex is not None:
+        settings["exclude_regex"] = exclude_regex
+
+    folder_exclude_regex = get_config_value(publish_config, "folder_exclude_regex", environment)
+    if folder_exclude_regex is not None:
+        settings["folder_exclude_regex"] = folder_exclude_regex
+
+    items_to_include = get_config_value(publish_config, "items_to_include", environment)
+    if items_to_include is not None:
+        settings["items_to_include"] = items_to_include
+
+    shortcut_exclude_regex = get_config_value(publish_config, "shortcut_exclude_regex", environment)
+    if shortcut_exclude_regex is not None:
+        settings["shortcut_exclude_regex"] = shortcut_exclude_regex
+
+    # Skip defaults to False if environment not found
+    if "skip" in publish_config:
+        skip_value = publish_config["skip"]
+        if isinstance(skip_value, dict):
+            settings["skip"] = skip_value.get(environment, False)
+        else:
+            settings["skip"] = skip_value
 
     return settings
 
@@ -116,26 +130,27 @@ def extract_unpublish_settings(config: dict, environment: str) -> dict:
     """Extract unpublish-specific settings from config for the given environment."""
     settings = {}
 
-    if "unpublish" in config:
-        unpublish_config = config["unpublish"]
-
-        if "exclude_regex" in unpublish_config:
-            if isinstance(unpublish_config["exclude_regex"], dict):
-                settings["exclude_regex"] = unpublish_config["exclude_regex"][environment]
-            else:
-                settings["exclude_regex"] = unpublish_config["exclude_regex"]
-
-        if "items_to_include" in unpublish_config:
-            if isinstance(unpublish_config["items_to_include"], dict):
-                settings["items_to_include"] = unpublish_config["items_to_include"][environment]
-            else:
-                settings["items_to_include"] = unpublish_config["items_to_include"]
-
-        if "skip" in unpublish_config:
-            if isinstance(unpublish_config["skip"], dict):
-                settings["skip"] = unpublish_config["skip"].get(environment, False)
-            else:
-                settings["skip"] = unpublish_config["skip"]
+    if "unpublish" not in config:
+        return settings
+
+    unpublish_config = config["unpublish"]
+
+    # Optional settings - validation logs debug if environment not found
+    exclude_regex = get_config_value(unpublish_config, "exclude_regex", environment)
+    if exclude_regex is not None:
+        settings["exclude_regex"] = exclude_regex
+
+    items_to_include = get_config_value(unpublish_config, "items_to_include", environment)
+    if items_to_include is not None:
+        settings["items_to_include"] = items_to_include
+
+    # Skip defaults to False if environment not found
+    if "skip" in unpublish_config:
+        skip_value = unpublish_config["skip"]
+        if isinstance(skip_value, dict):
+            settings["skip"] = unpublish_config["skip"].get(environment, False)
+        else:
+            settings["skip"] = skip_value
 
     return settings
 

src/fabric_cicd/_common/_config_validator.py

@@ -342,14 +342,14 @@ def _validate_environment_exists(self) -> None:
             # Handle no target environment case
             if any(
                 field_name in section and isinstance(section[field_name], dict)
-                for section, field_name, _ in _get_config_fields(self.config)
+                for section, field_name, _, _, _ in _get_config_fields(self.config)
                 if not (field_name == "constants" and _is_regular_constants_dict(section.get(field_name, {})))
             ):
                 self.errors.append(constants.CONFIG_VALIDATION_MSGS["environment"]["no_env_with_mappings"])
             return
 
         # Check each field for target environment presence
-        for section, field_name, display_name in _get_config_fields(self.config):
+        for section, field_name, display_name, is_required, log_warning in _get_config_fields(self.config):
             if field_name in section:
                 field_value = section[field_name]
                 # Handle constants special case
@@ -359,12 +359,22 @@ def _validate_environment_exists(self) -> None:
                 # If it's a dict (environment mapping), check if target environment exists
                 if isinstance(field_value, dict) and self.environment not in field_value:
                     available_envs = list(field_value.keys())
-                    self.errors.append(
-                        constants.CONFIG_VALIDATION_MSGS["environment"]["env_not_found"].format(
-                            self.environment, display_name, available_envs
-                        )
+                    msg = (
+                        f"Environment '{self.environment}' not found in '{display_name}'. "
+                        f"Available environments: {available_envs}. This setting will be skipped."
                     )
 
+                    if is_required:
+                        self.errors.append(
+                            constants.CONFIG_VALIDATION_MSGS["environment"]["env_not_found"].format(
+                                self.environment, display_name, available_envs
+                            )
+                        )
+                    elif log_warning:
+                        logger.warning(msg)
+                    else:
+                        logger.debug(msg)
+
     def _validate_environment_mapping(self, field_value: dict, field_name: str, accepted_type: type) -> bool:
         """Validate field with environment mapping."""
         if not field_value:
@@ -954,26 +964,35 @@ def _validate_constants_dict(self, constants_dict: dict, context: str) -> None:
                 )
 
 
-def _get_config_fields(config: dict) -> list[tuple[dict, str, str]]:
-    """Get list of all fields that support environment mappings."""
+def _get_config_fields(config: dict) -> list[tuple[dict, str, str, bool, bool]]:
+    """Get list of all fields that support environment mappings.
+
+    Returns:
+        List of tuples: (section_dict, field_name, display_name, is_required, log_warning)
+        - is_required: If True, missing environment causes error.
+        - log_warning: logging type (e.g., warning (True), debug (False)).
+    """
     return [
-        # Core section fields
-        (config.get("core", {}), "workspace_id", "core.workspace_id"),
-        (config.get("core", {}), "workspace", "core.workspace"),
-        (config.get("core", {}), "repository_directory", "core.repository_directory"),
-        (config.get("core", {}), "item_types_in_scope", "core.item_types_in_scope"),
-        (config.get("core", {}), "parameter", "core.parameter"),
-        # Publish section fields
-        (config.get("publish", {}), "exclude_regex", "publish.exclude_regex"),
-        (config.get("publish", {}), "items_to_include", "publish.items_to_include"),
-        (config.get("publish", {}), "skip", "publish.skip"),
-        # Unpublish section fields
-        (config.get("unpublish", {}), "exclude_regex", "unpublish.exclude_regex"),
-        (config.get("unpublish", {}), "items_to_include", "unpublish.items_to_include"),
-        (config.get("unpublish", {}), "skip", "unpublish.skip"),
-        # Top-level sections
-        (config, "features", "features"),
-        (config, "constants", "constants"),
+        # Core section fields - required
+        (config.get("core", {}), "workspace_id", "core.workspace_id", True, False),
+        (config.get("core", {}), "workspace", "core.workspace", True, False),
+        (config.get("core", {}), "repository_directory", "core.repository_directory", True, False),
+        # Core section fields - optional but important (warn if missing)
+        (config.get("core", {}), "item_types_in_scope", "core.item_types_in_scope", False, True),
+        (config.get("core", {}), "parameter", "core.parameter", False, True),
+        # Publish section fields - optional (debug if missing)
+        (config.get("publish", {}), "exclude_regex", "publish.exclude_regex", False, False),
+        (config.get("publish", {}), "folder_exclude_regex", "publish.folder_exclude_regex", False, False),
+        (config.get("publish", {}), "shortcut_exclude_regex", "publish.shortcut_exclude_regex", False, False),
+        (config.get("publish", {}), "items_to_include", "publish.items_to_include", False, False),
+        (config.get("publish", {}), "skip", "publish.skip", False, False),
+        # Unpublish section fields - optional (debug if missing)
+        (config.get("unpublish", {}), "exclude_regex", "unpublish.exclude_regex", False, False),
+        (config.get("unpublish", {}), "items_to_include", "unpublish.items_to_include", False, False),
+        (config.get("unpublish", {}), "skip", "unpublish.skip", False, False),
+        # Top-level sections - optional (debug if missing)
+        (config, "features", "features", False, False),
+        (config, "constants", "constants", False, False),
     ]