perf: Parallelize all items by central encapsulation into base classes

docs/how_to/optional_feature.md

@@ -24,6 +24,7 @@ For scenarios that aren't supported by default, fabric-cicd offers `feature-flag
 | `enable_config_deploy`                    | Set to enable config file-based deployment                              | ☑️           |
 | `enable_response_collection`              | Set to enable collection of API responses during publish operations     |              |
 | `continue_on_shortcut_failure`            | Set to allow deployment to continue even when shortcuts fail to publish |              |
+| `enable_debug_mode`                       | Set to enable debug mode for detailed logging                           |              |
 
 <span class="md-h3-nonanchor">Example</span>
 

src/fabric_cicd/__init__.py

@@ -9,6 +9,7 @@
 import fabric_cicd.constants as constants
 from fabric_cicd._common._check_utils import check_version
 from fabric_cicd._common._logging import configure_logger, exception_handler
+from fabric_cicd.constants import FeatureFlag, ItemType
 from fabric_cicd.fabric_workspace import FabricWorkspace
 from fabric_cicd.publish import deploy_with_config, publish_all_items, unpublish_all_orphan_items
 
@@ -56,6 +57,8 @@ def change_log_level(level: str = "DEBUG") -> None:
 
 __all__ = [
     "FabricWorkspace",
+    "FeatureFlag",
+    "ItemType",
     "append_feature_flag",
     "change_log_level",
     "deploy_with_config",

src/fabric_cicd/_common/_validate_input.py

@@ -16,6 +16,7 @@
 
 import fabric_cicd.constants as constants
 from fabric_cicd._common._exceptions import InputError
+from fabric_cicd.constants import OperationType
 from fabric_cicd.fabric_workspace import FabricWorkspace
 
 logger = logging.getLogger(__name__)
@@ -156,3 +157,32 @@ def validate_token_credential(input_value: TokenCredential) -> TokenCredential:
     validate_data_type("TokenCredential", "credential", input_value)
 
     return input_value
+
+
+def validate_items_to_include(items_to_include: Optional[list[str]], operation: "OperationType") -> None:
+    """
+    Validate items_to_include parameter and check required feature flags.
+
+    Args:
+        items_to_include: List of items in "item_name.item_type" format, or None.
+        operation: The type of operation being performed (publish or unpublish).
+
+    Raises:
+        InputError: If required feature flags are not enabled.
+    """
+    from fabric_cicd.constants import FeatureFlag
+
+    if items_to_include is None:
+        return
+
+    if (
+        FeatureFlag.ENABLE_EXPERIMENTAL_FEATURES.value not in constants.FEATURE_FLAG
+        or FeatureFlag.ENABLE_ITEMS_TO_INCLUDE.value not in constants.FEATURE_FLAG
+    ):
+        msg = "Feature flags 'enable_experimental_features' and 'enable_items_to_include' must be set."
+        raise InputError(msg, logger)
+
+    logger.warning(f"Selective {operation.value} is enabled.")
+    logger.warning(
+        f"Using items_to_include is risky as it can prevent needed dependencies from being {operation.value}.  Use at your own risk."
+    )

src/fabric_cicd/_items/__init__.py

@@ -1,62 +1,10 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-from fabric_cicd._items._activator import publish_activators
-from fabric_cicd._items._apacheairflowjob import publish_apacheairflowjobs
-from fabric_cicd._items._copyjob import publish_copyjobs
-from fabric_cicd._items._dataagent import publish_dataagents
-from fabric_cicd._items._dataflowgen2 import publish_dataflows
-from fabric_cicd._items._datapipeline import find_referenced_datapipelines, publish_datapipelines
-from fabric_cicd._items._environment import check_environment_publish_state, publish_environments
-from fabric_cicd._items._eventhouse import publish_eventhouses
-from fabric_cicd._items._eventstream import publish_eventstreams
-from fabric_cicd._items._graphqlapi import publish_graphqlapis
-from fabric_cicd._items._kqldashboard import publish_kqldashboard
-from fabric_cicd._items._kqldatabase import publish_kqldatabases
-from fabric_cicd._items._kqlqueryset import publish_kqlquerysets
-from fabric_cicd._items._lakehouse import publish_lakehouses
-from fabric_cicd._items._manage_dependencies import set_unpublish_order
-from fabric_cicd._items._mirroreddatabase import publish_mirroreddatabase
-from fabric_cicd._items._mlexperiment import publish_mlexperiments
-from fabric_cicd._items._mounteddatafactory import publish_mounteddatafactories
-from fabric_cicd._items._notebook import publish_notebooks
-from fabric_cicd._items._orgapp import publish_orgapps
-from fabric_cicd._items._report import publish_reports
-from fabric_cicd._items._semanticmodel import publish_semanticmodels
-from fabric_cicd._items._sparkjobdefinition import publish_sparkjobdefinitions
-from fabric_cicd._items._sqldatabase import publish_sqldatabases
-from fabric_cicd._items._userdatafunction import publish_userdatafunctions
-from fabric_cicd._items._variablelibrary import publish_variablelibraries
-from fabric_cicd._items._warehouse import publish_warehouses
+from fabric_cicd._items._base_publisher import ItemPublisher, ParallelConfig, PublishError
 
 __all__ = [
-    "check_environment_publish_state",
-    "find_referenced_datapipelines",
-    "publish_activators",
-    "publish_apacheairflowjobs",
-    "publish_copyjobs",
-    "publish_dataagents",
-    "publish_dataflows",
-    "publish_datapipelines",
-    "publish_environments",
-    "publish_eventhouses",
-    "publish_eventstreams",
-    "publish_graphqlapis",
-    "publish_kqldashboard",
-    "publish_kqldatabases",
-    "publish_kqlquerysets",
-    "publish_lakehouses",
-    "publish_mirroreddatabase",
-    "publish_mlexperiments",
-    "publish_mounteddatafactories",
-    "publish_notebooks",
-    "publish_orgapps",
-    "publish_reports",
-    "publish_semanticmodels",
-    "publish_sparkjobdefinitions",
-    "publish_sqldatabases",
-    "publish_userdatafunctions",
-    "publish_variablelibraries",
-    "publish_warehouses",
-    "set_unpublish_order",
+    "ItemPublisher",
+    "ParallelConfig",
+    "PublishError",
 ]

src/fabric_cicd/_items/_activator.py

@@ -3,21 +3,11 @@
 
 """Functions to process and deploy Reflex item."""
 
-import logging
+from fabric_cicd._items._base_publisher import ItemPublisher
+from fabric_cicd.constants import ItemType
 
-from fabric_cicd import FabricWorkspace
 
-logger = logging.getLogger(__name__)
+class ActivatorPublisher(ItemPublisher):
+    """Publisher for Reflex AKA Activator items."""
 
-
-def publish_activators(fabric_workspace_obj: FabricWorkspace) -> None:
-    """
-    Publishes all reflex items from the repository.
-
-    Args:
-        fabric_workspace_obj: The FabricWorkspace object containing the items to be published
-    """
-    item_type = "Reflex"
-
-    for item_name in fabric_workspace_obj.repository_items.get(item_type, {}):
-        fabric_workspace_obj._publish_item(item_name=item_name, item_type=item_type)
+    item_type = ItemType.REFLEX.value

src/fabric_cicd/_items/_apacheairflowjob.py

@@ -3,21 +3,11 @@
 
 """Functions to process and deploy Apache Airflow Job item."""
 
-import logging
+from fabric_cicd._items._base_publisher import ItemPublisher
+from fabric_cicd.constants import ItemType
 
-from fabric_cicd import FabricWorkspace
 
-logger = logging.getLogger(__name__)
+class ApacheAirflowJobPublisher(ItemPublisher):
+    """Publisher for Apache Airflow Job items."""
 
-
-def publish_apacheairflowjobs(fabric_workspace_obj: FabricWorkspace) -> None:
-    """
-    Publishes all Apache Airflow job items from the repository.
-
-    Args:
-        fabric_workspace_obj: The FabricWorkspace object containing the items to be published.
-    """
-    item_type = "ApacheAirflowJob"
-
-    for item_name in fabric_workspace_obj.repository_items.get(item_type, {}):
-        fabric_workspace_obj._publish_item(item_name=item_name, item_type=item_type)
+    item_type = ItemType.APACHE_AIRFLOW_JOB.value