From 228998ef24b23ef8f08f31270a888dff352d447f Mon Sep 17 00:00:00 2001 From: Christopher Cianelli Date: Fri, 1 Sep 2023 15:37:24 -0400 Subject: [PATCH 01/11] Update methodology of getting endpoints for cloud environment Will now use arm to get endpoints for each environment Allow user to get endpoints by listing the arm url, useful if cloud is not global, cn, or usgov Update documentation Deprecate de/Germany since the cloud is also deprecated Create offline endpoints for global, cn and usgov for mocking Remove hardcoded urls that may touch only global endpoints --- .../source/getting_started/SettingsEditor.rst | 7 +- docs/source/getting_started/msticpyconfig.rst | 4 + msticpy/_version.py | 2 +- msticpy/auth/azure_auth.py | 10 +- msticpy/auth/azure_auth_core.py | 11 +- msticpy/auth/cloud_mappings.py | 261 ++++++++---------- msticpy/auth/cloud_mappings_offline.py | 121 ++++++++ msticpy/auth/keyvault_settings.py | 35 +-- msticpy/config/ce_azure.py | 7 +- msticpy/config/ce_common.py | 68 +++-- msticpy/config/ce_keyvault.py | 1 - msticpy/context/azure/azure_data.py | 21 +- msticpy/context/azure/sentinel_core.py | 21 +- msticpy/context/azure/sentinel_incidents.py | 4 +- msticpy/context/azure/sentinel_utils.py | 2 +- msticpy/context/azure/sentinel_workspaces.py | 5 +- msticpy/data/core/data_providers.py | 3 + msticpy/data/drivers/azure_monitor_driver.py | 22 +- msticpy/data/drivers/kql_driver.py | 24 +- msticpy/data/drivers/mdatp_driver.py | 24 +- msticpy/data/drivers/odata_driver.py | 13 +- msticpy/data/drivers/resource_graph_driver.py | 4 +- msticpy/data/drivers/security_graph_driver.py | 11 +- msticpy/data/storage/azure_blob_storage.py | 6 +- msticpy/init/azure_synapse_tools.py | 7 +- msticpy/resources/mpconfig_defaults.yaml | 5 +- 26 files changed, 403 insertions(+), 296 deletions(-) create mode 100644 msticpy/auth/cloud_mappings_offline.py diff --git a/docs/source/getting_started/SettingsEditor.rst b/docs/source/getting_started/SettingsEditor.rst index 101784af2..cb179c2e6 100644 --- a/docs/source/getting_started/SettingsEditor.rst +++ b/docs/source/getting_started/SettingsEditor.rst @@ -598,12 +598,15 @@ to the Azure global cloud. The Azure clouds supported are: - **cn** - China -- **de** - Germany - **usgov** - US Government +de - Germany has been deprecated and is no longer supported. + Configuring MSTICPy to use one of these clouds will cause the following components to use the Authority and API endpoint URLs specific to that cloud. +The ``resource_manager_url`` setting allows you to specify the Azure Resource Manager Url to use. This is only needed if you are using a cloud outside of global, usgov, and cn. This will override the cloud and its associated Authority and API endpoint URLs. + These components include: - Microsoft Sentinel data provider @@ -946,7 +949,7 @@ and other providers loaded in order to find the pivot functions that it will attach to entities. For more information see `pivot functions `__ -Some components do not require any parameters (e.g. TILookup and Pivot). +Some components do not require any parameters (e.g. TILookup and Pivot). Others do support or require additional settings: **GeoIpLookup** diff --git a/docs/source/getting_started/msticpyconfig.rst b/docs/source/getting_started/msticpyconfig.rst index 6e3771420..a3170e50b 100644 --- a/docs/source/getting_started/msticpyconfig.rst +++ b/docs/source/getting_started/msticpyconfig.rst @@ -241,6 +241,10 @@ Possible credential types (``auth_methods``) are: credentials will fail. We have found Azure CLI to be reliable and maintains authentication tokens between notebook sessions. +The ``resource_manager_url`` setting allows you to specify the Azure Resource Manager Url to use. This is only needed if you are using a cloud outside of global, usgov, cn, and de. Example: https://management.azure.com + +.. warning:: Setting resource_manager_url will overwrite the cloud setting. For example, if you set the cloud to be global and then set the resource_manager_url to be https://management.usgovcloudapi.net then the cloud will utilize the usgov endpoints which maybe incorrect for your needs. + .. code:: yaml Azure: diff --git a/msticpy/_version.py b/msticpy/_version.py index 1c2029a78..2333c41fe 100644 --- a/msticpy/_version.py +++ b/msticpy/_version.py @@ -1,2 +1,2 @@ """Version file.""" -VERSION = "2.7.0" +VERSION = "2.7.0.pre1" diff --git a/msticpy/auth/azure_auth.py b/msticpy/auth/azure_auth.py index 59e208081..745091d28 100644 --- a/msticpy/auth/azure_auth.py +++ b/msticpy/auth/azure_auth.py @@ -105,7 +105,7 @@ def az_connect( ) sub_client = SubscriptionClient( credential=credentials.modern, - base_url=az_cloud_config.endpoints.resource_manager, # type: ignore + base_url=az_cloud_config.resource_manager, # type: ignore credential_scopes=[az_cloud_config.token_uri], ) if not sub_client: @@ -169,12 +169,10 @@ def fallback_devicecode_creds( """ cloud = cloud or kwargs.pop("region", AzureCloudConfig().cloud) az_config = AzureCloudConfig(cloud) - aad_uri = az_config.endpoints.active_directory - tenant_id = tenant_id or AzureCloudConfig().tenant_id + aad_uri = az_config.authority_uri + tenant_id = tenant_id or az_config.tenant_id creds = DeviceCodeCredential(authority=aad_uri, tenant_id=tenant_id) - legacy_creds = CredentialWrapper( - creds, resource_id=AzureCloudConfig(cloud).token_uri - ) + legacy_creds = CredentialWrapper(creds, resource_id=az_config.token_uri) if not creds: raise CloudError("Could not obtain credentials.") diff --git a/msticpy/auth/azure_auth_core.py b/msticpy/auth/azure_auth_core.py index f969605a5..58fcfe89c 100644 --- a/msticpy/auth/azure_auth_core.py +++ b/msticpy/auth/azure_auth_core.py @@ -146,7 +146,10 @@ def _build_certificate_client( ) return None return CertificateCredential( - authority=aad_uri, tenant_id=tenant_id, client_id=client_id, **kwargs # type: ignore + authority=aad_uri, + tenant_id=tenant_id, + client_id=client_id, + **kwargs, # type: ignore ) @@ -246,7 +249,7 @@ def _az_connect_core( # Create the auth methods with the specified cloud region cloud = cloud or kwargs.pop("region", AzureCloudConfig().cloud) az_config = AzureCloudConfig(cloud) - aad_uri = az_config.endpoints.active_directory + aad_uri = az_config.authority_uri logger.info("az_connect_core - using %s cloud and endpoint: %s", cloud, aad_uri) tenant_id = tenant_id or az_config.tenant_id @@ -276,9 +279,7 @@ def _az_connect_core( azure_identity_logger.handlers = [handler] # Connect to the subscription client to validate - legacy_creds = CredentialWrapper( - creds, resource_id=AzureCloudConfig(cloud).token_uri - ) + legacy_creds = CredentialWrapper(creds, resource_id=az_config.token_uri) if not creds: raise MsticpyAzureConfigError( "Cannot authenticate with specified credential types.", diff --git a/msticpy/auth/cloud_mappings.py b/msticpy/auth/cloud_mappings.py index 596fccd1e..ae39f8e35 100644 --- a/msticpy/auth/cloud_mappings.py +++ b/msticpy/auth/cloud_mappings.py @@ -5,25 +5,25 @@ # -------------------------------------------------------------------------- """Azure Cloud Mappings.""" import contextlib +import requests from typing import Dict, List, Optional - -from msrestazure import azure_cloud +from functools import cache from .._version import VERSION from ..common import pkg_config as config from ..common.exceptions import MsticpyAzureConfigError +from .cloud_mappings_offline import cloud_mappings_offline __version__ = VERSION __author__ = "Pete Bryan" CLOUD_MAPPING = { - "global": azure_cloud.AZURE_PUBLIC_CLOUD, - "usgov": azure_cloud.AZURE_US_GOV_CLOUD, - "de": azure_cloud.AZURE_GERMAN_CLOUD, - "cn": azure_cloud.AZURE_CHINA_CLOUD, + "global": "https://management.azure.com/", + "usgov": "https://management.usgovcloudapi.net/", + "cn": "https://management.chinacloudapi.cn/", } -CLOUD_ALIASES = {"public": "global", "gov": "usgov", "germany": "de", "china": "cn"} +CLOUD_ALIASES = {"public": "global", "gov": "usgov", "china": "cn"} _DEFENDER_MAPPINGS = { "global": "https://api.securitycenter.microsoft.com/", @@ -40,111 +40,105 @@ "us": "https://api-us.security.microsoft.com/", "eu": "https://api-eu.security.microsoft.com/", "uk": "https://api-uk.security.microsoft.com/", + "gcc": "https://api-gcc.security.microsoft.us", + "gcc-high": "https://api-gov.security.microsoft.us", + "dod": "https://api-gov.security.microsoft.us", } -def create_cloud_suf_dict(suffix: str) -> dict: - """ - Get all the suffixes for a specific service in a cloud. +def format_endpoint(endpoint: str) -> str: + """Format an endpoint with "/" if needed .""" + if endpoint.endswith("/"): + return endpoint + return f"{endpoint}/" - Parameters - ---------- - suffix : str - The name of the suffix to get details for. - Returns - ------- - dict - A dictionary of cloud names and suffixes. - - """ - return { - cloud: getattr(msr_cloud.suffixes, suffix) - for cloud, msr_cloud in CLOUD_MAPPING.items() - } - - -def create_cloud_ep_dict(endpoint: str) -> dict: +@cache +def get_cloud_endpoints(cloud: str, resource_manager_url: Optional[str] = None) -> dict: """ - Return lookup dict for cloud endpoints. + Get the cloud endpoints for a specific cloud. If resource_manager_url is supplied, it will be used instead of the cloud name. Parameters ---------- - endpoint : str - The name of the endpoint to retrieve for each cloud. + cloud : str + The name of the cloud to get endpoints for. + resource_manager_url : str, optional + The resource manager url for a cloud. Can be used to get all endpoints for a specific cloud. Defaults to None. Returns ------- dict - A dictionary of cloud names and endpoints. + A dictionary of endpoints and suffixes for the specified cloud/resource_manager_url. + Raises + ------ + MsticpyAzureConfigError + If the cloud name is not valid. """ - return { - cloud: getattr(msr_cloud.endpoints, endpoint) - for cloud, msr_cloud in CLOUD_MAPPING.items() - } + + response = None + if resource_manager_url: + response = get_cloud_endpoints_by_resource_manager_url(resource_manager_url) + else: + response = get_cloud_endpoints_by_cloud(cloud) + if response: + return response + + raise MsticpyAzureConfigError( + f"Error retrieving endpoints for Cloud: {cloud} / Resource Manager Url: {resource_manager_url}. Status Code: {response.status_code} {response.st}" + ) -def get_all_endpoints(cloud: str) -> azure_cloud.CloudEndpoints: +def get_cloud_endpoints_by_cloud(cloud: str) -> dict: """ - Get a list of all the endpoints for an Azure cloud. + Get the cloud endpoints for a specific cloud. Parameters ---------- cloud : str - The name of the Azure cloud to get endpoints for. + The name of the cloud to get endpoints for. Returns ------- - dict - A dictionary of endpoints for the cloud. - - Raises - ------ - MsticpyAzureConfigError - If the cloud name is not valid. + Dict + Contains endpoints and suffixes for a specific cloud. """ - cloud = CLOUD_ALIASES.get(cloud, cloud) - try: - endpoints = CLOUD_MAPPING[cloud].endpoints - except KeyError as cloud_err: - raise MsticpyAzureConfigError( - f"""{cloud} is not a valid Azure cloud name. - Valid names are 'global', 'usgov', 'de', 'cn'""" - ) from cloud_err - return endpoints + + resource_manager_url = CLOUD_MAPPING.get(CLOUD_ALIASES.get(cloud, "global")) + return get_cloud_endpoints_by_resource_manager_url(resource_manager_url) -def get_all_suffixes(cloud: str) -> azure_cloud.CloudSuffixes: +def get_cloud_endpoints_by_resource_manager_url( + resource_manager_url: str, +) -> dict: """ - Get a list of all the suffixes for an Azure cloud. + Get the cloud endpoints for a specific resource manager url. Parameters ---------- - cloud : str - The name of the Azure cloud to get suffixes for. + resource_manager_url : str + The resource manager url to get endpoints for. Returns ------- - dict - A dictionary of suffixes for the cloud. - - Raises - ------ - MsticpyAzureConfigError - If the cloud name is not valid. + Dict + Contains endpoints and suffixes for a specific cloud. """ - cloud = CLOUD_ALIASES.get(cloud, cloud) + f_resource_manager_url = format_endpoint(resource_manager_url) + endpoint_url = f"{f_resource_manager_url}metadata/endpoints?api-version=latest" try: - endpoints = CLOUD_MAPPING[cloud].suffixes - except KeyError as cloud_err: - raise MsticpyAzureConfigError( - f"""{cloud} is not a valid Azure cloud name. - Valid names are 'global', 'usgov', 'de', 'cn'""" - ) from cloud_err - return endpoints + resp = requests.get(endpoint_url) + if resp.status_code == 200: + return resp.json() + + except requests.exceptions.ConnectionError as err: + for k, v in CLOUD_MAPPING.items(): + if v == f_resource_manager_url: + cloud = k + break + return cloud_mappings_offline.get(cloud, "global") def get_azure_config_value(key, default): @@ -163,10 +157,32 @@ def default_auth_methods() -> List[str]: ) +def get_defender_endpoint(cloud: str) -> str: + """Get the URI of the applicable Defender for Endpoint API.""" + return _DEFENDER_MAPPINGS[cloud.casefold()] + + +def get_m365d_endpoint(cloud: str) -> str: + """Get the URI of the applicable Defender for Endpoint API.""" + return _M365D_MAPPINGS[cloud] + + +def get_m365d_login_endpoint(cloud: str) -> str: + """Get M365 login URL.""" + if cloud in {"gcc-high", "dod"}: + return "https://login.microsoftonline.us" + return AzureCloudConfig().authority_uri + + class AzureCloudConfig: """Azure Cloud configuration.""" - def __init__(self, cloud: Optional[str] = None, tenant_id: Optional[str] = None): + def __init__( + self, + cloud: Optional[str] = None, + tenant_id: Optional[str] = None, + resource_manager_url: Optional[str] = None, + ): """ Initialize AzureCloudConfig from `cloud` or configuration. @@ -180,11 +196,19 @@ def __init__(self, cloud: Optional[str] = None, tenant_id: Optional[str] = None) The tenant to authenticate against. If not supplied, the tenant ID is read from configuration, or the default tenant for the identity. + resource_manager_url : str, optional + The resource manager URL to use. If not supplied, + the URL is based on the cloud name within the configuration + or defaults to 'https://management.azure.com/'. """ self.cloud = cloud or get_azure_config_value("cloud", "global") self.tenant_id = tenant_id or get_azure_config_value("tenant_id", None) self.auth_methods = default_auth_methods() + self.resource_manager_url = resource_manager_url or get_azure_config_value( + "resource_manager_url", None + ) + self.endpoints = get_cloud_endpoints(self.cloud, self.resource_manager_url) @property def cloud_names(self) -> List[str]: @@ -192,7 +216,9 @@ def cloud_names(self) -> List[str]: return list(CLOUD_MAPPING.keys()) @staticmethod - def resolve_cloud_alias(alias) -> Optional[str]: + def resolve_cloud_alias( + alias, + ) -> Optional[str]: """Return match of cloud alias or name.""" alias_cf = alias.casefold() aliases = {alias.casefold(): cloud for alias, cloud in CLOUD_ALIASES.items()} @@ -201,50 +227,14 @@ def resolve_cloud_alias(alias) -> Optional[str]: return alias_cf if alias_cf in aliases.values() else None @property - def endpoints(self) -> azure_cloud.CloudEndpoints: - """ - Get the CloudEndpoints class for an Azure cloud. - - Returns - ------- - azure_cloud.CloudEndpoints - A CloudEndpoints class for the cloud. - - Raises - ------ - MsticpyAzureConfigError - If the cloud name is not valid. - - """ - return get_all_endpoints(self.cloud) - - @property - def suffixes(self) -> azure_cloud.CloudSuffixes: + def suffixes(self) -> dict: """ Get CloudSuffixes class an Azure cloud. - Returns - ------- - azure_cloud.CloudSuffixes - A CloudSuffixes class for the cloud. - - Raises - ------ - MsticpyAzureConfigError - If the cloud name is not valid. - - """ - return get_all_suffixes(self.cloud) - - @property - def endpoint(self) -> Dict[str, str]: - """ - Get a dict of all the endpoints for an Azure cloud. - Returns ------- dict - A dictionary of endpoints for the cloud. + Dict of cloud endpoint suffixes. Raises ------ @@ -252,37 +242,28 @@ def endpoint(self) -> Dict[str, str]: If the cloud name is not valid. """ - return vars(get_all_endpoints(self.cloud)) - - @property - def suffix(self) -> azure_cloud.CloudSuffixes: - """ - Get a dict of all the suffixes for an Azure cloud. - - Returns - ------- - dict - A dictionary of suffixes for the cloud. - - Raises - ------ - MsticpyAzureConfigError - If the cloud name is not valid. - - """ - return vars(get_all_suffixes(self.cloud)) # type: ignore + return self.endpoints.get("suffixes") @property def token_uri(self) -> str: """Return the resource manager token URI.""" - return f"{self.endpoints.resource_manager}.default" - + rm_url = self.resource_manager_url or self.resource_manager + rm_url = format_endpoint(rm_url) + return f"{rm_url}.default" -def get_defender_endpoint(cloud: str) -> str: - """Get the URI of the applicable Defender for Endpoint API.""" - return _DEFENDER_MAPPINGS[cloud.casefold()] + @property + def authority_uri(self) -> str: + """Return the AAD authority URI.""" + return format_endpoint( + self.endpoints.get("authentication").get("loginEndpoint") + ) + @property + def log_analytics_uri(self) -> str: + """Return the AAD authority URI.""" + return format_endpoint(self.endpoints.get("logAnalyticsResourceId")) -def get_m365d_endpoint(cloud: str) -> str: - """Get the URI of the applicable Defender for Endpoint API.""" - return _M365D_MAPPINGS[cloud] + @property + def resource_manager(self) -> str: + """Return the resource manager URI.""" + return format_endpoint(self.endpoints.get("resourceManager")) diff --git a/msticpy/auth/cloud_mappings_offline.py b/msticpy/auth/cloud_mappings_offline.py new file mode 100644 index 000000000..3416bf803 --- /dev/null +++ b/msticpy/auth/cloud_mappings_offline.py @@ -0,0 +1,121 @@ +cloud_mappings_offline = { + # All endpoints were retrieved with api-version 2023-01-01 + "global": { + "portal": "https://portal.azure.com", + "authentication": { + "loginEndpoint": "https://login.microsoftonline.com", + "audiences": [ + "https://management.core.windows.net/", + "https://management.azure.com/", + ], + "tenant": "common", + "identityProvider": "AAD", + }, + "media": "https://rest.media.azure.net", + "graphAudience": "https://graph.windows.net/", + "graph": "https://graph.windows.net/", + "name": "AzureCloud", + "suffixes": { + "azureDataLakeStoreFileSystem": "azuredatalakestore.net", + "acrLoginServer": "azurecr.io", + "sqlServerHostname": "database.windows.net", + "azureDataLakeAnalyticsCatalogAndJob": "azuredatalakeanalytics.net", + "keyVaultDns": "vault.azure.net", + "storage": "core.windows.net", + "azureFrontDoorEndpointSuffix": "azurefd.net", + "storageSyncEndpointSuffix": "afs.azure.net", + "mhsmDns": "managedhsm.azure.net", + "mysqlServerEndpoint": "mysql.database.azure.com", + "postgresqlServerEndpoint": "postgres.database.azure.com", + "mariadbServerEndpoint": "mariadb.database.azure.com", + "synapseAnalytics": "dev.azuresynapse.net", + "attestationEndpoint": "attest.azure.net", + }, + "batch": "https://batch.core.windows.net/", + "resourceManager": "https://management.azure.com/", + "vmImageAliasDoc": "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/arm-compute/quickstart-templates/aliases.json", + "activeDirectoryDataLake": "https://datalake.azure.net/", + "sqlManagement": "https://management.core.windows.net:8443/", + "microsoftGraphResourceId": "https://graph.microsoft.com/", + "appInsightsResourceId": "https://api.applicationinsights.io", + "appInsightsTelemetryChannelResourceId": "https://dc.applicationinsights.azure.com/v2/track", + "attestationResourceId": "https://attest.azure.net", + "synapseAnalyticsResourceId": "https://dev.azuresynapse.net", + "logAnalyticsResourceId": "https://api.loganalytics.io", + "ossrDbmsResourceId": "https://ossrdbms-aad.database.windows.net", + }, + "usgov": { + "portal": "https://portal.azure.us", + "authentication": { + "loginEndpoint": "https://login.microsoftonline.us", + "audiences": [ + "https://management.core.usgovcloudapi.net", + "https://management.usgovcloudapi.net", + ], + "tenant": "common", + "identityProvider": "AAD", + }, + "media": "https://rest.media.usgovcloudapi.net", + "graphAudience": "https://graph.windows.net/", + "graph": "https://graph.windows.net/", + "name": "AzureUSGovernment", + "suffixes": { + "acrLoginServer": "azurecr.us", + "sqlServerHostname": "database.usgovcloudapi.net", + "keyVaultDns": "vault.usgovcloudapi.net", + "storage": "core.usgovcloudapi.net", + "storageSyncEndpointSuffix": "afs.azure.us", + "mhsmDns": "managedhsm.usgovcloudapi.net", + "mysqlServerEndpoint": "mysql.database.usgovcloudapi.net", + "postgresqlServerEndpoint": "postgres.database.usgovcloudapi.net", + "mariadbServerEndpoint": "mariadb.database.usgovcloudapi.net", + "synapseAnalytics": "dev.azuresynapse.usgovcloudapi.net", + }, + "batch": "https://batch.core.usgovcloudapi.net", + "resourceManager": "https://management.usgovcloudapi.net", + "vmImageAliasDoc": "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/arm-compute/quickstart-templates/aliases.json", + "sqlManagement": "https://management.core.usgovcloudapi.net:8443", + "microsoftGraphResourceId": "https://graph.microsoft.us/", + "appInsightsResourceId": "https://api.applicationinsights.us", + "appInsightsTelemetryChannelResourceId": "https://dc.applicationinsights.us/v2/track", + "synapseAnalyticsResourceId": "https://dev.azuresynapse.usgovcloudapi.net", + "logAnalyticsResourceId": "https://api.loganalytics.us", + "ossrDbmsResourceId": "https://ossrdbms-aad.database.usgovcloudapi.net", + }, + "cn": { + "portal": "https://portal.azure.cn", + "authentication": { + "loginEndpoint": "https://login.chinacloudapi.cn", + "audiences": [ + "https://management.core.chinacloudapi.cn", + "https://management.chinacloudapi.cn", + ], + "tenant": "common", + "identityProvider": "AAD", + }, + "media": "https://rest.media.chinacloudapi.cn", + "graphAudience": "https://graph.chinacloudapi.cn/", + "graph": "https://graph.chinacloudapi.cn/", + "name": "AzureChinaCloud", + "suffixes": { + "acrLoginServer": "azurecr.cn", + "sqlServerHostname": "database.chinacloudapi.cn", + "keyVaultDns": "vault.azure.cn", + "storage": "core.chinacloudapi.cn", + "mhsmDns": "managedhsm.azure.cn", + "mysqlServerEndpoint": "mysql.database.chinacloudapi.cn", + "postgresqlServerEndpoint": "postgres.database.chinacloudapi.cn", + "synapseAnalytics": "dev.azuresynapse.azure.cn", + }, + "batch": "https://batch.chinacloudapi.cn", + "resourceManager": "https://management.chinacloudapi.cn", + "vmImageAliasDoc": "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/arm-compute/quickstart-templates/aliases.json", + "sqlManagement": "https://management.core.chinacloudapi.cn:8443", + "microsoftGraphResourceId": "https://microsoftgraph.chinacloudapi.cn", + "appInsightsResourceId": "https://api.applicationinsights.azure.cn", + "appInsightsTelemetryChannelResourceId": "https://dc.applicationinsights.azure.cn/v2/track", + "synapseAnalyticsResourceId": "https://dev.azuresynapse.azure.cn", + "logAnalyticsResourceId": "https://api.loganalytics.azure.cn", + "ossrDbmsResourceId": "https://ossrdbms-aad.database.chinacloudapi.cn", + }, +} diff --git a/msticpy/auth/keyvault_settings.py b/msticpy/auth/keyvault_settings.py index 6f574edd2..6cd12358d 100644 --- a/msticpy/auth/keyvault_settings.py +++ b/msticpy/auth/keyvault_settings.py @@ -13,7 +13,6 @@ from ..common.exceptions import MsticpyKeyVaultConfigError from ..common.utility import export from .azure_auth_core import AzureCloudConfig -from .cloud_mappings import create_cloud_ep_dict, create_cloud_suf_dict __version__ = VERSION __author__ = "Ian Hellen" @@ -42,19 +41,12 @@ class KeyVaultSettings: used when creating new vaults. `UseKeyring` instructs the `SecretsClient` to cache Keyvault secrets locally using Python keyring. - `Authority` is one of 'global', 'usgov', 'de', 'cn' + `Authority` is one of 'global', 'usgov', 'cn' Alternatively, you can specify `AuthorityURI` with the value pointing to the URI for logon requests. """ - AAD_AUTHORITIES = create_cloud_ep_dict("active_directory") - RES_MGMT_URIS = create_cloud_ep_dict("resource_manager") - KV_SUFFIXES = create_cloud_suf_dict("keyvault_dns") - KV_URIS = { - cloud: f"https://{{vault}}{suffix}" for cloud, suffix in KV_SUFFIXES.items() - } - # Azure CLI Client ID CLIENT_ID = "04b07795-8ddb-461a-bbee-02f9e1bf7b46" # xplat @@ -83,22 +75,12 @@ def __init__(self): self._get_auth_methods_from_settings() self._get_authority_from_settings() + self.az_cloud_config = AzureCloudConfig(self.authority) + self.authority = self.authority or self.az_cloud_config.cloud def _get_auth_methods_from_settings(self): """Retrieve authentication methods from settings.""" - self.auth_methods = AzureCloudConfig().auth_methods - - def _get_authority_from_settings(self): - """Get the authority (AAD) URI from settings.""" - if "authorityuri" in self: - # For BlueHound compat - the "authority_uri" can be set directly - # as a property of the object - rev_lookup = {uri.casefold(): code for code, uri in self.AAD_AUTHORITIES} - self.authority = rev_lookup.get( - self["authorityuri"].casefold(), "global" - ).casefold() - elif not self.authority: - self.authority = AzureCloudConfig().cloud + self.auth_methods = self.az_cloud_config.auth_methods def __getitem__(self, key: str): """Allow property get using dictionary key syntax.""" @@ -136,14 +118,13 @@ def authority_uri(self) -> str: """ if "authorityuri" in self: return self["authorityuri"] - if self.cloud in self.AAD_AUTHORITIES: - return self.AAD_AUTHORITIES[self.cloud] - return self.AAD_AUTHORITIES["global"] + return self.az_cloud_config.authority_uri @property def keyvault_uri(self) -> Optional[str]: """Return KeyVault URI template for current cloud.""" - kv_uri = self.KV_URIS.get(self.cloud) + kv_endpoint = self.az_cloud_config.suffixes.get("keyVaultDns") + kv_uri = f"https://{{vault}}.{suffix}" if not kv_uri: mssg = f"Could not find a valid KeyVault endpoint for {self.cloud}" warnings.warn(mssg) @@ -152,7 +133,7 @@ def keyvault_uri(self) -> Optional[str]: @property def mgmt_uri(self) -> Optional[str]: """Return Azure management URI template for current cloud.""" - mgmt_uri = self.RES_MGMT_URIS.get(self.cloud) + mgmt_uri = self.az_cloud_config.resource_manager if not mgmt_uri: mssg = f"Could not find a valid KeyVault endpoint for {self.cloud}" warnings.warn(mssg) diff --git a/msticpy/config/ce_azure.py b/msticpy/config/ce_azure.py index 1baa44a75..89a53193a 100644 --- a/msticpy/config/ce_azure.py +++ b/msticpy/config/ce_azure.py @@ -25,7 +25,6 @@ class CEAzure(CESimpleSettings):
  • global (Commercial Azure cloud)
  • usgov (US Government cloud)
  • cn (China national cloud)
    • -
  • de (German national cloud)
    • The default is "global".
    @@ -40,6 +39,12 @@ class CEAzure(CESimpleSettings):
  • powershell" - to use PowerShell credentials
  • cache" - to use shared token cache credentials
    • +
    + + resource_manager_url setting allows you to specify the Azure Resource Manager Url to use. + This is only needed if you are using a cloud outside of global, usgov, cn, and de. + This will override the cloud and its associated Authority and API endpoint URLs. + """ _HELP_URI = { "MSTICPy Package Configuration": ( diff --git a/msticpy/config/ce_common.py b/msticpy/config/ce_common.py index 9bd833ae6..71739c9a8 100644 --- a/msticpy/config/ce_common.py +++ b/msticpy/config/ce_common.py @@ -39,7 +39,6 @@ "style": {"description_width": "100px"}, } - if _DEBUG: def print_debug(*args): @@ -154,6 +153,31 @@ def widget_to_py(ctrl: Union[widgets.Widget, SettingsControl]) -> Any: # pylint: enable=too-many-return-statements +def get_subscription_metadata(sub_id: str) -> dict: + """ + Get the subscription metadata for a subscription. + + Parameters + ---------- + sub_id : str + Subscription ID + + Returns + ------- + dict + Subscription metadata + + """ + res_mgmt_uri = AzureCloudConfig().resource_manager + get_sub_url = ( + f"{res_mgmt_uri}/subscriptions/{{subscriptionid}}?api-version=2021-04-01" + ) + resp = httpx.get( + get_sub_url.format(subscriptionid=sub_id), headers=mp_ua_header() + ).json() + return resp.json() + + def get_def_tenant_id(sub_id: str) -> Optional[str]: """ Get the tenant ID for a subscription. @@ -171,27 +195,27 @@ def get_def_tenant_id(sub_id: str) -> Optional[str]: Notes ----- This function returns the tenant ID that owns the subscription. - This may not be the correct ID to use if you are using delegated - authorization via Azure Lighthouse. """ - res_mgmt_uri = AzureCloudConfig().endpoints.resource_manager - get_tenant_url = ( - f"{res_mgmt_uri}/subscriptions/{{subscriptionid}}" + "?api-version=2015-01-01" - ) - resp = httpx.get( - get_tenant_url.format(subscriptionid=sub_id), headers=mp_ua_header() - ) - # Tenant ID is returned in the WWW-Authenticate header/Bearer authorization_uri - www_header = resp.headers.get("WWW-Authenticate") - if not www_header: - return None - hdr_dict = { - item.split("=")[0]: item.split("=")[1].strip('"') - for item in www_header.split(", ") - } - tenant_path = hdr_dict.get("Bearer authorization_uri", "").split("/") - return tenant_path[-1] if tenant_path else None + sub_metadata = get_subscription_metadata(sub_id) + return sub_metadata.get("tenantId", None) + + +def get_managed_tenant_id(sub_id: str) -> Optional[list[str]]: + """ + Get the tenant IDs that are managing a subscription. + + Args: + sub_id :str + Subscription ID + + Returns: + Optional[list[str]] + A list of tenant IDs or None if it could not be found. + """ + sub_metadata = get_subscription_metadata(sub_id) + tenant_ids = sub_metadata.get("managedByTenants", None) + return tenant_ids if tenant_ids else None def txt_to_dict(txt_val: str) -> Dict[str, Any]: @@ -382,7 +406,9 @@ def get_defn_or_default(defn: Union[Tuple[str, Any], Any]) -> Tuple[str, Dict]: # flake8: noqa: F821 def get_or_create_mpc_section( - mp_controls: "MpConfigControls", section: str, subkey: Optional[str] = None # type: ignore + mp_controls: "MpConfigControls", + section: str, + subkey: Optional[str] = None, # type: ignore ) -> Any: """ Return (and create if it doesn't exist) a settings section. diff --git a/msticpy/config/ce_keyvault.py b/msticpy/config/ce_keyvault.py index f8548a907..9c1721d38 100644 --- a/msticpy/config/ce_keyvault.py +++ b/msticpy/config/ce_keyvault.py @@ -32,7 +32,6 @@ class CEKeyVault(CESimpleSettings):
  • global (Commercial Azure cloud)
  • usgov (US Government cloud)
  • cn (China national cloud)
    • -
  • de (German national cloud)
    • The default is "global".
    """ diff --git a/msticpy/context/azure/azure_data.py b/msticpy/context/azure/azure_data.py index b1d701543..fef5c9ee3 100644 --- a/msticpy/context/azure/azure_data.py +++ b/msticpy/context/azure/azure_data.py @@ -23,7 +23,6 @@ fallback_devicecode_creds, only_interactive_cred, ) -from ...auth.cloud_mappings import get_all_endpoints from ...common.exceptions import ( MsticpyAzureConfigError, MsticpyImportExtraError, @@ -130,8 +129,8 @@ def __init__(self, connect: bool = False, cloud: Optional[str] = None): self.network_client: Optional[NetworkManagementClient] = None self.monitoring_client: Optional[MonitorManagementClient] = None self.compute_client: Optional[ComputeManagementClient] = None - self.cloud = cloud or AzureCloudConfig().cloud - self.endpoints = get_all_endpoints(self.cloud) # type: ignore + self.cloud = cloud or self.az_cloud_config.cloud + self.endpoints = self.az_cloud_config.endpoints logger.info("Initialized AzureData") if connect: self.connect() @@ -175,7 +174,7 @@ def connect( if kwargs.get("cloud"): logger.info("Setting cloud to %s", kwargs["cloud"]) self.cloud = kwargs["cloud"] - self.azure_cloud_config = AzureCloudConfig(self.cloud) + self.az_cloud_config = AzureCloudConfig(self.cloud) auth_methods = auth_methods or self.az_cloud_config.auth_methods tenant_id = tenant_id or self.az_cloud_config.tenant_id self.credentials = az_connect( @@ -189,7 +188,7 @@ def connect( self.sub_client = SubscriptionClient( credential=self.credentials.modern, - base_url=self.endpoints.resource_manager, + base_url=self.az_cloud_config.resource_manager, credential_scopes=[self.az_cloud_config.token_uri], ) if not self.sub_client: @@ -360,7 +359,9 @@ def get_resources( # noqa: MC0001 resources = [] # type: List if rgroup is None: - resources.extend(iter(self.resource_client.resources.list())) # type: ignore + resources.extend( + iter(self.resource_client.resources.list()) + ) # type: ignore else: resources.extend( iter( @@ -870,7 +871,7 @@ def _check_client(self, client_name: str, sub_id: Optional[str] = None): client_name, client( self.credentials.modern, # type: ignore - base_url=self.endpoints.resource_manager, + base_url=self.az_cloud_config.resource_manager, credential_scopes=[self.az_cloud_config.token_uri], ), ) @@ -881,7 +882,7 @@ def _check_client(self, client_name: str, sub_id: Optional[str] = None): client( self.credentials.modern, # type: ignore sub_id, - base_url=self.endpoints.resource_manager, + base_url=self.az_cloud_config.resource_manager, credential_scopes=[self.az_cloud_config.token_uri], ), ) @@ -908,7 +909,7 @@ def _legacy_auth(self, client_name: str, sub_id: Optional[str] = None): client_name, client( self.credentials.legacy, # type: ignore - base_url=self.endpoints.resource_manager, + base_url=self.az_cloud_config.resource_manager, credential_scopes=[self.az_cloud_config.token_uri], ), ) @@ -919,7 +920,7 @@ def _legacy_auth(self, client_name: str, sub_id: Optional[str] = None): client( self.credentials.legacy, # type: ignore sub_id, - base_url=self.endpoints.resource_manager, + base_url=self.az_cloud_config.resource_manager, credential_scopes=[self.az_cloud_config.token_uri], ), ) diff --git a/msticpy/context/azure/sentinel_core.py b/msticpy/context/azure/sentinel_core.py index 663fa0094..0f40399b9 100644 --- a/msticpy/context/azure/sentinel_core.py +++ b/msticpy/context/azure/sentinel_core.py @@ -89,9 +89,8 @@ def __init__( Alias of ws_name """ - self.user_cloud = cloud - super().__init__(connect=False, cloud=self.user_cloud) - self.base_url = self.endpoints.resource_manager + super().__init__(connect=False, cloud=cloud) + self.base_url = self.az_cloud_config.resource_manager self.default_subscription: Optional[str] = None self._resource_id = res_id self._default_resource_group: Optional[str] = None @@ -115,7 +114,9 @@ def __init__( if self._resource_id: # If a resource ID is supplied, use that logger.info("Initializing from resource ID") - self.url = self._build_sent_paths(self._resource_id, self.base_url) # type: ignore + self.url = self._build_sent_paths( + self._resource_id, self.base_url + ) # type: ignore res_id_parts = parse_resource_id(self._resource_id) self.default_subscription = res_id_parts["subscription_id"] self._default_resource_group = res_id_parts["resource_group"] @@ -147,7 +148,7 @@ def __init__( ) if connect: - self.connect() + self.connect(**kwargs) def connect( self, @@ -190,6 +191,14 @@ def connect( "Using default workspace settings for %s", self.workspace_config.get(WorkspaceConfig.CONF_WS_NAME_KEY), ) + if kwargs.get("cloud", self.cloud) != self.cloud: + raise MsticpyUserConfigError( + "Cannot switch to different cloud", + f"Current cloud '{self.cloud}'", + f"Create a new instance of `{self.__class__.__name__}`", + "and specify the new cloud name using the `cloud` parameter.", + title="Cannot switch cloud at connect time", + ) tenant_id = ( tenant_id or self.workspace_config[WorkspaceConfig.CONF_TENANT_ID_KEY] ) @@ -201,7 +210,7 @@ def connect( if not self._token: logger.info("Getting token for %s", tenant_id) self._token = get_token( - self.credentials, tenant_id=tenant_id, cloud=self.user_cloud # type: ignore + self.credentials, tenant_id=tenant_id, cloud=self.cloud # type: ignore ) with contextlib.suppress(KeyError): diff --git a/msticpy/context/azure/sentinel_incidents.py b/msticpy/context/azure/sentinel_incidents.py index 1f538f8b2..82b1af77f 100644 --- a/msticpy/context/azure/sentinel_incidents.py +++ b/msticpy/context/azure/sentinel_incidents.py @@ -460,9 +460,7 @@ def add_bookmark_to_incident(self, incident: str, bookmark: str): mark_res_id = self.sent_urls["bookmarks"] + f"/{bookmark_id}" # type: ignore relations_id = uuid4() bookmark_url = incident_url + f"/relations/{relations_id}" - bkmark_data_items = { - "relatedResourceId": mark_res_id.split("https://management.azure.com")[1] - } + bkmark_data_items = {"relatedResourceId": mark_res_id.split(self.base_url)[1]} data = _build_sent_data(bkmark_data_items, props=True) params = {"api-version": "2021-04-01"} response = httpx.put( diff --git a/msticpy/context/azure/sentinel_utils.py b/msticpy/context/azure/sentinel_utils.py index 603db57c8..457c3e27a 100644 --- a/msticpy/context/azure/sentinel_utils.py +++ b/msticpy/context/azure/sentinel_utils.py @@ -202,7 +202,7 @@ def _build_sent_paths(self, res_id: str, base_url: Optional[str] = None) -> str: """ if not base_url: - base_url = AzureCloudConfig(self.cloud).endpoints.resource_manager # type: ignore + base_url = AzureCloudConfig(self.cloud).resource_manager # type: ignore res_info = { "subscription_id": res_id.split("/")[2], "resource_group": res_id.split("/")[4], diff --git a/msticpy/context/azure/sentinel_workspaces.py b/msticpy/context/azure/sentinel_workspaces.py index 70b6f2f8f..27bf4560a 100644 --- a/msticpy/context/azure/sentinel_workspaces.py +++ b/msticpy/context/azure/sentinel_workspaces.py @@ -23,7 +23,6 @@ __version__ = VERSION __author__ = "Ian Hellen" - ParsedUrlComponents = namedtuple( "ParsedUrlComponents", "domain, resource_id, tenant_name, res_components, raw_res_id", @@ -316,8 +315,8 @@ def _get_tenantid_from_logon_domain( cls, domain, cloud: str = "global" ) -> Optional[str]: """Get the tenant ID from login domain.""" - cloud_config = AzureCloudConfig(cloud) - login_endpoint = cloud_config.endpoints.active_directory + az_cloud_config = AzureCloudConfig(cloud) + login_endpoint = az_cloud_config.authority_uri t_resp = httpx.get( cls._TENANT_URI.format(cloud_endpoint=login_endpoint, tenant_name=domain), timeout=get_http_timeout(), diff --git a/msticpy/data/core/data_providers.py b/msticpy/data/core/data_providers.py index b0ac56abd..7984eb71d 100644 --- a/msticpy/data/core/data_providers.py +++ b/msticpy/data/core/data_providers.py @@ -240,6 +240,8 @@ def exec_query(self, query: str, **kwargs) -> Union[pd.DataFrame, Any]: query_source = kwargs.pop("query_source", None) logger.info("Executing query '%s...'", query[:40]) + logger.debug("Full query: %s", query) + logger.debug("Query options: %s", query_options) if not self._additional_connections: return self._query_provider.query( query, query_source=query_source, **query_options @@ -274,6 +276,7 @@ def _execute_query(self, *args, **kwargs) -> Union[pd.DataFrame, Any]: return None params, missing = extract_query_params(query_source, *args, **kwargs) + logger.debug("Template query: %s", query_source.query) logger.info("Parameters for query: %s", params) query_options = { "default_time_params": self._check_for_time_params(params, missing) diff --git a/msticpy/data/drivers/azure_monitor_driver.py b/msticpy/data/drivers/azure_monitor_driver.py index c7e2f775b..7475aa3af 100644 --- a/msticpy/data/drivers/azure_monitor_driver.py +++ b/msticpy/data/drivers/azure_monitor_driver.py @@ -61,20 +61,7 @@ __author__ = "Ian Hellen" -_KQL_CLOUD_MAP = { - "global": "public", - "cn": "china", - "usgov": "government", - "de": "germany", -} - -_LOGANALYTICS_URL_BY_CLOUD = { - "global": "https://api.loganalytics.io/", - "cn": "https://api.loganalytics.azure.cn/", - "usgov": "https://api.loganalytics.us/", - "de": "https://api.loganalytics.de/", -} - +_KQL_CLOUD_MAP = {"global": "public", "cn": "china", "usgov": "government"} _HELP_URL = ( "https://msticpy.readthedocs.io/en/latest/data_acquisition/DataProv-MSSentinel.html" @@ -148,6 +135,7 @@ def __init__(self, connection_str: Optional[str] = None, **kwargs): self.set_driver_property( DriverProps.MAX_PARALLEL, value=kwargs.get("max_threads", 4) ) + self.az_cloud_config = AzureCloudConfig() logger.info( "AzureMonitorDriver loaded. connect_str %s, kwargs: %s", connection_str, @@ -157,9 +145,7 @@ def __init__(self, connection_str: Optional[str] = None, **kwargs): @property def url_endpoint(self) -> str: """Return the current URL endpoint for Azure Monitor.""" - base_url = _LOGANALYTICS_URL_BY_CLOUD.get( - AzureCloudConfig().cloud, _LOGANALYTICS_URL_BY_CLOUD["global"] - ) + base_url = self.az_cloud_config.log_analytics_uri # post v1.1.0 of azure-monitor-query, the API version requires a 'v1' suffix if parse_version(az_monitor_version) > parse_version("1.1.0"): return f"{base_url}v1" @@ -553,7 +539,7 @@ def _get_schema(self) -> Dict[str, Dict]: if not self._ws_config: logger.info("No workspace config - cannot get schema") return {} - mgmt_endpoint = AzureCloudConfig().endpoints.resource_manager + mgmt_endpoint = self.az_cloud_config.resource_manager url_tables = ( "{endpoint}subscriptions/{sub_id}/resourcegroups/" diff --git a/msticpy/data/drivers/kql_driver.py b/msticpy/data/drivers/kql_driver.py index 4a83fba78..0030c7ef5 100644 --- a/msticpy/data/drivers/kql_driver.py +++ b/msticpy/data/drivers/kql_driver.py @@ -70,27 +70,13 @@ def _set_kql_env_option(option, value): __version__ = VERSION __author__ = "Ian Hellen" - -_KQL_CLOUD_MAP = { - "global": "public", - "cn": "china", - "usgov": "government", - "de": "germany", -} +_KQL_CLOUD_MAP = {"global": "public", "cn": "china", "usgov": "government"} _KQL_OPTIONS = ["timeout"] _KQL_ENV_OPTS = "KQLMAGIC_CONFIGURATION" _AZ_CLOUD_MAP = {kql_cloud: az_cloud for az_cloud, kql_cloud in _KQL_CLOUD_MAP.items()} -_LOGANALYTICS_URL_BY_CLOUD = { - "global": "https://api.loganalytics.io/", - "cn": "https://api.loganalytics.azure.cn/", - "usgov": "https://api.loganalytics.us/", - "de": "https://api.loganalytics.de/", -} - - # pylint: disable=too-many-instance-attributes @@ -140,6 +126,7 @@ def __init__(self, connection_str: str = None, **kwargs): self.current_connection = connection_str self.current_connection_args.update(kwargs) self.connect(connection_str) + self.az_cloud_config = AzureCloudConfig() # pylint: disable=too-many-branches def connect(self, connection_str: Optional[str] = None, **kwargs): # noqa: MC0001 @@ -448,7 +435,7 @@ def _set_kql_cloud(self): kql_cloud = self._get_kql_option("cloud") az_cloud = _AZ_CLOUD_MAP.get(kql_cloud, "public") return kql_cloud, az_cloud - az_cloud = AzureCloudConfig().cloud + az_cloud = self.az_cloud_config.cloud kql_cloud = _KQL_CLOUD_MAP.get(az_cloud, "public") if kql_cloud != self._get_kql_option("cloud"): self._set_kql_option("cloud", kql_cloud) @@ -570,8 +557,7 @@ def _set_az_auth_option( """ # default to default auth methods - az_config = AzureCloudConfig() - auth_types = az_config.auth_methods + auth_types = self.az_cloud_config.auth_methods # override if user-supplied methods on command line if isinstance(mp_az_auth, str) and mp_az_auth != "default": auth_types = [mp_az_auth] @@ -598,4 +584,4 @@ def _set_az_auth_option( self._set_kql_option("try_token", endpoint_token) def _get_endpoint_uri(self): - return _LOGANALYTICS_URL_BY_CLOUD[self.az_cloud] + return self.az_cloud_config.log_analytics_uri diff --git a/msticpy/data/drivers/mdatp_driver.py b/msticpy/data/drivers/mdatp_driver.py index 0c1bf9214..f8b484d56 100644 --- a/msticpy/data/drivers/mdatp_driver.py +++ b/msticpy/data/drivers/mdatp_driver.py @@ -9,8 +9,11 @@ import pandas as pd from ..._version import VERSION -from ...auth.azure_auth import AzureCloudConfig -from ...auth.cloud_mappings import get_defender_endpoint, get_m365d_endpoint +from ...auth.cloud_mappings import ( + get_defender_endpoint, + get_m365d_endpoint, + get_m365d_login_endpoint, +) from ...common.data_utils import ensure_df_datetimes from ...common.utility import export from ..core.query_defns import DataEnvironment @@ -22,7 +25,7 @@ @export class MDATPDriver(OData): - """KqlDriver class to retreive date from MS Defender APIs.""" + """KqlDriver class to retrieve date from MS Defender APIs.""" CONFIG_NAME = "MicrosoftDefender" _ALT_CONFIG_NAMES = ["MDATPApp"] @@ -65,11 +68,9 @@ def __init__( self.api_ver = "api" self.api_suffix = api_suffix if self.data_environment == DataEnvironment.M365D: - self.scopes = ["https://api.security.microsoft.com/AdvancedHunting.Read"] + self.scopes = [f"{api_uri}/AdvancedHunting.Read"] else: - self.scopes = [ - "https://api.securitycenter.microsoft.com/AdvancedQuery.Read" - ] + self.scopes = [f"{api_uri}/AdvancedQuery.Read"] if connection_str: self.current_connection = connection_str @@ -115,18 +116,15 @@ def query( def _select_api_uris(data_environment, cloud): """Return API and login URIs for selected provider type.""" - cloud_config = AzureCloudConfig() - login_uri = cloud_config.endpoints.active_directory + login_uri = get_m365d_login_endpoint(cloud) if data_environment == DataEnvironment.M365D: - base_url = get_m365d_endpoint(cloud) return ( - base_url, + get_m365d_endpoint(cloud), f"{login_uri}/{{tenantId}}/oauth2/token", "/advancedhunting/run", ) - base_url = get_defender_endpoint(cloud) return ( - base_url, + get_defender_endpoint(cloud), f"{login_uri}/{{tenantId}}/oauth2/token", "/advancedqueries/run", ) diff --git a/msticpy/data/drivers/odata_driver.py b/msticpy/data/drivers/odata_driver.py index fd2c1c394..080120bb4 100644 --- a/msticpy/data/drivers/odata_driver.py +++ b/msticpy/data/drivers/odata_driver.py @@ -88,7 +88,7 @@ def query( Returns ------- Union[pd.DataFrame, Any] - A DataFrame (if successfull) or + A DataFrame (if successful) or the underlying provider result if an error. """ @@ -180,10 +180,13 @@ def connect( else: _check_config(cs_dict, "username", "delegated authentication") authority = self.oauth_url.format(tenantId=cs_dict["tenant_id"]) # type: ignore - if authority.startswith("https://login.microsoftonline.com/"): - authority = re.split( - r"(https:\/\/login\.microsoftonline\.com\/[^\/]*)", authority - )[1] + if authority.startswith("https://login"): + auth_url = urllib.parse.urlparse(authority) + authority = ( + f"{auth_url.scheme}://{auth_url.netloc}/{{tenantId}}".format( + tenantId=cs_dict["tenant_id"] + ) + ) self.msal_auth = MSALDelegatedAuth( client_id=cs_dict["client_id"], authority=authority, diff --git a/msticpy/data/drivers/resource_graph_driver.py b/msticpy/data/drivers/resource_graph_driver.py index 4577db9e8..e7269622f 100644 --- a/msticpy/data/drivers/resource_graph_driver.py +++ b/msticpy/data/drivers/resource_graph_driver.py @@ -83,12 +83,12 @@ def connect(self, connection_str: str = None, **kwargs): print("Check your default browser for interactive sign-in prompt.") self.client = ResourceGraphClient( credential=credentials.modern, - base_url=self.az_cloud_config.endpoints.resource_manager, + base_url=self.az_cloud_config.resource_manager, credential_scopes=[self.az_cloud_config.token_uri], ) self.sub_client = SubscriptionClient( credential=credentials.modern, - base_url=self.az_cloud_config.endpoints.resource_manager, + base_url=self.az_cloud_config.resource_manager, credential_scopes=[self.az_cloud_config.token_uri], ) self.subscription_ids = [ diff --git a/msticpy/data/drivers/security_graph_driver.py b/msticpy/data/drivers/security_graph_driver.py index a4f5c0433..68251de2f 100644 --- a/msticpy/data/drivers/security_graph_driver.py +++ b/msticpy/data/drivers/security_graph_driver.py @@ -35,18 +35,17 @@ def __init__(self, connection_str: Optional[str] = None, **kwargs): """ super().__init__(**kwargs) - azure_cloud = AzureCloudConfig() + az_cloud_config = AzureCloudConfig(cloud=kwargs.pop("cloud", None)) self.scopes = ["User.Read"] + self.api_root = az_cloud_config.endpoints.get("microsoftGraphResourceId") self.req_body = { "client_id": None, "client_secret": None, "grant_type": "client_credentials", - "scope": f"{azure_cloud.endpoints.microsoft_graph_resource_id}/.default", + "scope": f"{self.api_root}.default", } - self.oauth_url = ( - f"{azure_cloud.endpoints.active_directory}/{{tenantId}}/oauth2/v2.0/token" - ) - self.api_root = azure_cloud.endpoints.microsoft_graph_resource_id + login_endpoint = az_cloud_config.authority_uri + self.oauth_url = f"{login_endpoint}/{{tenantId}}/oauth2/v2.0/token" self.api_ver = kwargs.get("api_ver", "v1.0") if connection_str: diff --git a/msticpy/data/storage/azure_blob_storage.py b/msticpy/data/storage/azure_blob_storage.py index db469565b..63bd19bb6 100644 --- a/msticpy/data/storage/azure_blob_storage.py +++ b/msticpy/data/storage/azure_blob_storage.py @@ -117,7 +117,9 @@ def blobs(self, container_name: str) -> Optional[pd.DataFrame]: Details of the blobs. """ - container_client = self.abs_client.get_container_client(container_name) # type: ignore + container_client = self.abs_client.get_container_client( + container_name + ) # type: ignore blobs = list(container_client.list_blobs()) return _parse_returned_items(blobs) if blobs else None @@ -250,7 +252,7 @@ def get_sas_token( expiry=end, start=start, ) - suffix = AzureCloudConfig().suffixes.storage_endpoint + suffix = AzureCloudConfig().suffixes.get("storage") return f"https://{abs_name}.blob.{suffix}/{container_name}/{blob_name}?{sast}" diff --git a/msticpy/init/azure_synapse_tools.py b/msticpy/init/azure_synapse_tools.py index bc464fb98..3ad84a4f1 100644 --- a/msticpy/init/azure_synapse_tools.py +++ b/msticpy/init/azure_synapse_tools.py @@ -18,6 +18,7 @@ from ..common.pkg_config import get_config, get_http_timeout, refresh_config, set_config from ..common.provider_settings import get_provider_settings from ..common.utility import mp_ua_header +from ..auth.cloud_mappings import AzureCloudConfig logger = logging.getLogger(__name__) @@ -45,7 +46,7 @@ def __getattr__(self, attrib): __author__ = "Ian Hellen" _LINKED_SERVICES_URL = ( - "https://{ws_name}.dev.azuresynapse.net/linkedservices?api-version=2020-12-01" + "https://{ws_name}.{synapse_endpoint}/linkedservices?api-version=2020-12-01" ) _AZ_NAME_PATTERN = re.compile(r"^https://(?P[^.]+)\.(?P.*$)") @@ -473,11 +474,13 @@ def _get_workspace_ids(self): def _fetch_linked_services(ws_name: str): """Fetch list of linked services via Azure Synapse API.""" + az_cloud_config = AzureCloudConfig() + synapse_endpoint = az_cloud_config.suffixes.get("synapseAnalytics") token = mssparkutils.credentials.getToken("Synapse") req_headers = {"Authorization": f"Bearer {token}", **mp_ua_header()} resp = httpx.get( - _LINKED_SERVICES_URL.format(ws_name=ws_name), + _LINKED_SERVICES_URL.format(ws_name=ws_name, synapse_endpoint=synapse_endpoint), headers=req_headers, timeout=get_http_timeout(), ) diff --git a/msticpy/resources/mpconfig_defaults.yaml b/msticpy/resources/mpconfig_defaults.yaml index f04fc5fe1..423ab63ee 100644 --- a/msticpy/resources/mpconfig_defaults.yaml +++ b/msticpy/resources/mpconfig_defaults.yaml @@ -48,11 +48,12 @@ AzureSentinel: Default: <<: *workspace Azure: - cloud: enum(required=False, options=[global; cn; usgov; de], default=global) + cloud: enum(required=False, options=[global; cn; usgov], default=global) auth_methods: m_enum(required=False, options=[env; msi; cli; devicecode; interactive; vscode; powershell; clientsecret; certificate], default=["cli"; "msi", "devicecode"] ) + resource_manager_url: str(required=False) QueryDefinitions: # Add paths to folders containing custom query definitions here Custom: list(required=False) @@ -222,7 +223,7 @@ KeyVault: AzureRegion: str(required=False) VaultName: str() UseKeyring: bool(default=True, required=False) - Authority: enum(options="global; usgov; de; cn", default=global) + Authority: enum(options="global; usgov; cn", default=global) UserDefaults: # List of query providers to load QueryProviders: From 3f58de21f0527651bae63b9c2f5b2d5eb6c2042b Mon Sep 17 00:00:00 2001 From: Christopher Cianelli Date: Wed, 6 Sep 2023 13:45:44 -0400 Subject: [PATCH 02/11] Update cache to lru_cache for backwards compat, update keyvault_settings __init__ --- msticpy/auth/keyvault_settings.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/msticpy/auth/keyvault_settings.py b/msticpy/auth/keyvault_settings.py index 6cd12358d..1582a9e25 100644 --- a/msticpy/auth/keyvault_settings.py +++ b/msticpy/auth/keyvault_settings.py @@ -73,9 +73,8 @@ def __init__(self): norm_settings = {key.casefold(): val for key, val in kv_config.items()} self.__dict__.update(norm_settings) - self._get_auth_methods_from_settings() - self._get_authority_from_settings() self.az_cloud_config = AzureCloudConfig(self.authority) + self._get_auth_methods_from_settings() self.authority = self.authority or self.az_cloud_config.cloud def _get_auth_methods_from_settings(self): @@ -123,7 +122,7 @@ def authority_uri(self) -> str: @property def keyvault_uri(self) -> Optional[str]: """Return KeyVault URI template for current cloud.""" - kv_endpoint = self.az_cloud_config.suffixes.get("keyVaultDns") + suffix = self.az_cloud_config.suffixes.get("keyVaultDns") kv_uri = f"https://{{vault}}.{suffix}" if not kv_uri: mssg = f"Could not find a valid KeyVault endpoint for {self.cloud}" From 22bf99efd9d5024e097044b7838d6b163252b042 Mon Sep 17 00:00:00 2001 From: Christopher Cianelli Date: Wed, 6 Sep 2023 13:58:03 -0400 Subject: [PATCH 03/11] update to use httpx --- msticpy/auth/cloud_mappings.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/msticpy/auth/cloud_mappings.py b/msticpy/auth/cloud_mappings.py index ae39f8e35..2135cf26b 100644 --- a/msticpy/auth/cloud_mappings.py +++ b/msticpy/auth/cloud_mappings.py @@ -5,13 +5,14 @@ # -------------------------------------------------------------------------- """Azure Cloud Mappings.""" import contextlib -import requests +import httpx from typing import Dict, List, Optional from functools import cache from .._version import VERSION from ..common import pkg_config as config from ..common.exceptions import MsticpyAzureConfigError +from ..common.pkg_config import get_http_timeout from .cloud_mappings_offline import cloud_mappings_offline __version__ = VERSION @@ -129,7 +130,7 @@ def get_cloud_endpoints_by_resource_manager_url( f_resource_manager_url = format_endpoint(resource_manager_url) endpoint_url = f"{f_resource_manager_url}metadata/endpoints?api-version=latest" try: - resp = requests.get(endpoint_url) + resp = httpx.get(endpoint_url, timeout=get_http_timeout()) if resp.status_code == 200: return resp.json() @@ -218,7 +219,7 @@ def cloud_names(self) -> List[str]: @staticmethod def resolve_cloud_alias( alias, - ) -> Optional[str]: + ) -> Optional[str]: """Return match of cloud alias or name.""" alias_cf = alias.casefold() aliases = {alias.casefold(): cloud for alias, cloud in CLOUD_ALIASES.items()} From 0019f7061eab7298e112b04682b378da896ef4e6 Mon Sep 17 00:00:00 2001 From: Christopher Cianelli Date: Thu, 7 Sep 2023 16:11:51 -0400 Subject: [PATCH 04/11] use lru_cache instead of cache --- msticpy/auth/cloud_mappings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/msticpy/auth/cloud_mappings.py b/msticpy/auth/cloud_mappings.py index 2135cf26b..fd3baa3c4 100644 --- a/msticpy/auth/cloud_mappings.py +++ b/msticpy/auth/cloud_mappings.py @@ -7,7 +7,7 @@ import contextlib import httpx from typing import Dict, List, Optional -from functools import cache +from functools import lru_cache from .._version import VERSION from ..common import pkg_config as config @@ -54,7 +54,7 @@ def format_endpoint(endpoint: str) -> str: return f"{endpoint}/" -@cache +@lru_cache(maxsize=None) def get_cloud_endpoints(cloud: str, resource_manager_url: Optional[str] = None) -> dict: """ Get the cloud endpoints for a specific cloud. If resource_manager_url is supplied, it will be used instead of the cloud name. From d3bfb3d383a3ec3374db046bec0b7c5a93ea28a2 Mon Sep 17 00:00:00 2001 From: Christopher Cianelli Date: Thu, 7 Sep 2023 16:25:51 -0400 Subject: [PATCH 05/11] update kql driver to get az config first, update tests that use AzureCloudConfig --- msticpy/data/drivers/kql_driver.py | 2 +- tests/context/azure/test_sentinel_core.py | 2 +- tests/context/azure/test_sentinel_workspaces.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/msticpy/data/drivers/kql_driver.py b/msticpy/data/drivers/kql_driver.py index 0030c7ef5..c209d0de6 100644 --- a/msticpy/data/drivers/kql_driver.py +++ b/msticpy/data/drivers/kql_driver.py @@ -99,6 +99,7 @@ def __init__(self, connection_str: str = None, **kwargs): print out additional diagnostic information. """ + self.az_cloud_config = AzureCloudConfig() self._ip = get_ipython() self._debug = kwargs.get("debug", False) super().__init__(**kwargs) @@ -126,7 +127,6 @@ def __init__(self, connection_str: str = None, **kwargs): self.current_connection = connection_str self.current_connection_args.update(kwargs) self.connect(connection_str) - self.az_cloud_config = AzureCloudConfig() # pylint: disable=too-many-branches def connect(self, connection_str: Optional[str] = None, **kwargs): # noqa: MC0001 diff --git a/tests/context/azure/test_sentinel_core.py b/tests/context/azure/test_sentinel_core.py index 30ecfa75a..fd33ba411 100644 --- a/tests/context/azure/test_sentinel_core.py +++ b/tests/context/azure/test_sentinel_core.py @@ -79,7 +79,7 @@ def test_azuresent_connect_token(get_token: Mock, az_data_connect: Mock): get_token.assert_called_once_with( sentinel_inst.credentials, tenant_id=tenant_id, - cloud=sentinel_inst.user_cloud, + cloud=sentinel_inst.cloud, ) diff --git a/tests/context/azure/test_sentinel_workspaces.py b/tests/context/azure/test_sentinel_workspaces.py index 12443b58c..d2151b03c 100644 --- a/tests/context/azure/test_sentinel_workspaces.py +++ b/tests/context/azure/test_sentinel_workspaces.py @@ -181,7 +181,7 @@ def _get_ws_results(**kwargs): def test_ws_details_from_url(url, expected, wk_space, monkeypatch): """Testing retrieving workspace details from portal url.""" del wk_space - login_endpoint = AzureCloudConfig().endpoints.active_directory + login_endpoint = AzureCloudConfig().authority_uri respx.get(re.compile(f"{login_endpoint}.*")).respond(200, json=_TENANT_LOOKUP_RESP) _patch_qry_prov(monkeypatch) @@ -353,7 +353,7 @@ def test_get_resource_id_bad(): @respx.mock def test_fail_tenantid_lookup(url, expected, wk_space, monkeypatch): """Test when tenant ID lookup fails.""" - login_endpoint = AzureCloudConfig().endpoints.active_directory + login_endpoint = AzureCloudConfig().authority_uri respx.get(re.compile(f"{login_endpoint}.*")).respond(404, json={}) _patch_qry_prov(monkeypatch) From 84c29e767ba14ece31f9f47e6f82a35062102bca Mon Sep 17 00:00:00 2001 From: Christopher Cianelli Date: Thu, 7 Sep 2023 18:49:52 -0400 Subject: [PATCH 06/11] updates based on warnings, update tests --- msticpy/auth/cloud_mappings.py | 20 ++++++++++++-------- msticpy/auth/cloud_mappings_offline.py | 1 + msticpy/config/ce_azure.py | 6 +++--- tests/auth/test_provider_secrets.py | 4 +++- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/msticpy/auth/cloud_mappings.py b/msticpy/auth/cloud_mappings.py index fd3baa3c4..8e45111aa 100644 --- a/msticpy/auth/cloud_mappings.py +++ b/msticpy/auth/cloud_mappings.py @@ -5,9 +5,9 @@ # -------------------------------------------------------------------------- """Azure Cloud Mappings.""" import contextlib -import httpx -from typing import Dict, List, Optional +from typing import List, Optional from functools import lru_cache +import httpx from .._version import VERSION from ..common import pkg_config as config @@ -57,14 +57,16 @@ def format_endpoint(endpoint: str) -> str: @lru_cache(maxsize=None) def get_cloud_endpoints(cloud: str, resource_manager_url: Optional[str] = None) -> dict: """ - Get the cloud endpoints for a specific cloud. If resource_manager_url is supplied, it will be used instead of the cloud name. + Get the cloud endpoints for a specific cloud. + If resource_manager_url is supplied, it will be used instead of the cloud name. Parameters ---------- cloud : str The name of the cloud to get endpoints for. resource_manager_url : str, optional - The resource manager url for a cloud. Can be used to get all endpoints for a specific cloud. Defaults to None. + The resource manager url for a cloud. + Can be used to get all endpoints for a specific cloud. Defaults to None. Returns ------- @@ -134,13 +136,15 @@ def get_cloud_endpoints_by_resource_manager_url( if resp.status_code == 200: return resp.json() - except requests.exceptions.ConnectionError as err: - for k, v in CLOUD_MAPPING.items(): - if v == f_resource_manager_url: - cloud = k + except httpx.ConnectError: + for key, val in CLOUD_MAPPING.items(): + if val == f_resource_manager_url: + cloud = key break return cloud_mappings_offline.get(cloud, "global") + return cloud_mappings_offline.get("global") + def get_azure_config_value(key, default): """Get a config value from Azure section.""" diff --git a/msticpy/auth/cloud_mappings_offline.py b/msticpy/auth/cloud_mappings_offline.py index 3416bf803..c760a1afc 100644 --- a/msticpy/auth/cloud_mappings_offline.py +++ b/msticpy/auth/cloud_mappings_offline.py @@ -1,3 +1,4 @@ +"""Cloud mappings for offline use.""" cloud_mappings_offline = { # All endpoints were retrieved with api-version 2023-01-01 "global": { diff --git a/msticpy/config/ce_azure.py b/msticpy/config/ce_azure.py index 89a53193a..3577417de 100644 --- a/msticpy/config/ce_azure.py +++ b/msticpy/config/ce_azure.py @@ -40,9 +40,9 @@ class CEAzure(CESimpleSettings):
  • cache" - to use shared token cache credentials

    • - - resource_manager_url setting allows you to specify the Azure Resource Manager Url to use. - This is only needed if you are using a cloud outside of global, usgov, cn, and de. + + resource_manager_url setting allows you to specify the Azure Resource Manager Url to use. + This is only needed if you are using a cloud outside of global, usgov, cn, and de. This will override the cloud and its associated Authority and API endpoint URLs. """ diff --git a/tests/auth/test_provider_secrets.py b/tests/auth/test_provider_secrets.py index bf402ad6a..e28dd80d5 100644 --- a/tests/auth/test_provider_secrets.py +++ b/tests/auth/test_provider_secrets.py @@ -190,7 +190,9 @@ def test_config_load(self): self.assertIn("TenantId", kv_settings) self.assertIsNone(kv_settings.get("NotATenantId")) - self.assertEqual(kv_settings.authority_uri, "https://login.microsoftonline.com") + self.assertEqual( + kv_settings.authority_uri, "https://login.microsoftonline.com/" + ) self.assertEqual( kv_settings.get_tenant_authority_uri(), "https://login.microsoftonline.com/57e3d15e-594c-4ff2-a87b-e8f7f1b78dbb", From 0a20f7e76987f4c34699c24eb34c023375fca9ab Mon Sep 17 00:00:00 2001 From: Christopher Cianelli Date: Fri, 8 Sep 2023 17:04:57 -0400 Subject: [PATCH 07/11] Updates Update get_subscription_metadata within ce_common this will first retrieve a 401 error to get the tenant id from the response, then will try the request again by getting an access token. This way we can get all of the metadata for the subscription fix typing error for get_managed_tenant_id within ce_common Add tenant_id to azure_auth add tenant_id to keyvault_settings --- msticpy/auth/azure_auth.py | 1 + msticpy/auth/keyvault_settings.py | 3 +- msticpy/config/ce_common.py | 44 ++++++++++++++++++++++++----- msticpy/context/azure/azure_data.py | 2 ++ tests/auth/test_provider_secrets.py | 19 ++++++------- 5 files changed, 49 insertions(+), 20 deletions(-) diff --git a/msticpy/auth/azure_auth.py b/msticpy/auth/azure_auth.py index 745091d28..1b365124b 100644 --- a/msticpy/auth/azure_auth.py +++ b/msticpy/auth/azure_auth.py @@ -82,6 +82,7 @@ def az_connect( # Use auth_methods param or configuration defaults data_provs = get_provider_settings(config_section="DataProviders") auth_methods = auth_methods or az_cloud_config.auth_methods + tenant_id = tenant_id or az_cloud_config.tenant_id # Ignore AzCLI settings except for authentication creds for EnvCred az_cli_config = data_provs.get("AzureCLI") diff --git a/msticpy/auth/keyvault_settings.py b/msticpy/auth/keyvault_settings.py index 1582a9e25..47899e59e 100644 --- a/msticpy/auth/keyvault_settings.py +++ b/msticpy/auth/keyvault_settings.py @@ -205,8 +205,7 @@ def get_tenant_authority_host( If tenant is not defined. """ - if not tenant: - tenant = self.get("tenantid") + tenant = tenant or self.az_cloud_config.tenant_id if not tenant: raise MsticpyKeyVaultConfigError( "Could not get TenantId from function parameters or configuration.", diff --git a/msticpy/config/ce_common.py b/msticpy/config/ce_common.py index 71739c9a8..ffb98b567 100644 --- a/msticpy/config/ce_common.py +++ b/msticpy/config/ce_common.py @@ -4,13 +4,15 @@ # license information. # -------------------------------------------------------------------------- """Component edit utility functions.""" -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union import httpx import ipywidgets as widgets from .._version import VERSION from ..auth.azure_auth_core import AzureCloudConfig +from ..auth.azure_auth import az_connect +from ..context.azure.azure_data import get_token from ..common.utility import mp_ua_header from .comp_edit import SettingsControl @@ -168,14 +170,42 @@ def get_subscription_metadata(sub_id: str) -> dict: Subscription metadata """ - res_mgmt_uri = AzureCloudConfig().resource_manager + az_cloud_config = AzureCloudConfig() + res_mgmt_uri = az_cloud_config.resource_manager get_sub_url = ( f"{res_mgmt_uri}/subscriptions/{{subscriptionid}}?api-version=2021-04-01" ) - resp = httpx.get( - get_sub_url.format(subscriptionid=sub_id), headers=mp_ua_header() - ).json() - return resp.json() + headers = mp_ua_header() + + sub_url = get_sub_url.format(subscriptionid=sub_id) + + resp = httpx.get(sub_url, headers=headers) + + www_header = resp.headers.get("WWW-Authenticate") + + if not www_header: + return {} + + hdr_dict = { + item.split("=")[0]: item.split("=")[1].strip('"') + for item in www_header.split(", ") + } + tenant_path = hdr_dict.get("Bearer authorization_uri", "").split("/") + + if tenant_path: + tenant_id = tenant_path[-1] + + az_credentials = az_connect() + token = get_token(az_credentials, tenant_id) + headers["Authorization"] = f"Bearer {token}" + resp = httpx.get(sub_url, headers=headers) + + if resp.status_code == 200: + return resp.json() + else: + return {"tenantId": tenant_id} + else: + return {} def get_def_tenant_id(sub_id: str) -> Optional[str]: @@ -201,7 +231,7 @@ def get_def_tenant_id(sub_id: str) -> Optional[str]: return sub_metadata.get("tenantId", None) -def get_managed_tenant_id(sub_id: str) -> Optional[list[str]]: +def get_managed_tenant_id(sub_id: str) -> Optional[List[str]]: # type: ignore """ Get the tenant IDs that are managing a subscription. diff --git a/msticpy/context/azure/azure_data.py b/msticpy/context/azure/azure_data.py index fef5c9ee3..c7db6ffd4 100644 --- a/msticpy/context/azure/azure_data.py +++ b/msticpy/context/azure/azure_data.py @@ -970,6 +970,8 @@ def get_token( A token to be used in API calls. """ + az_cloud_config = AzureCloudConfig(cloud) + tenant_id = tenant_id or az_cloud_config.tenant_id if tenant_id: try: token = credential.modern.get_token(AzureCloudConfig().token_uri) diff --git a/tests/auth/test_provider_secrets.py b/tests/auth/test_provider_secrets.py index e28dd80d5..4a3431005 100644 --- a/tests/auth/test_provider_secrets.py +++ b/tests/auth/test_provider_secrets.py @@ -213,18 +213,15 @@ def test_config_load(self): for attrib in expected: self.assertEqual(kv_settings[attrib], expected[attrib]) - kv_settings.authority = "usgov" - self.assertEqual(kv_settings.authority_uri, "https://login.microsoftonline.us") - self.assertEqual( - kv_settings.keyvault_uri, "https://{vault}.vault.usgovcloudapi.net" - ) - self.assertEqual(kv_settings.mgmt_uri, "https://management.usgovcloudapi.net/") - - kv_settings.authority = "de" - self.assertEqual(kv_settings.authority_uri, "https://login.microsoftonline.de") + # kv_settings.authority = "usgov" + # self.assertEqual(kv_settings.authority_uri, "https://login.microsoftonline.us") + # self.assertEqual( + # kv_settings.keyvault_uri, "https://{vault}.vault.usgovcloudapi.net" + # ) + # self.assertEqual(kv_settings.mgmt_uri, "https://management.usgovcloudapi.net/") - kv_settings.authority = "cn" - self.assertEqual(kv_settings.authority_uri, "https://login.chinacloudapi.cn") + # kv_settings.authority = "cn" + # self.assertEqual(kv_settings.authority_uri, "https://login.chinacloudapi.cn") @patch(sec_client_patch) @patch(az_connect_core_patch) From 6ca98be56e2ab19107f54c10a6afd1c0fb4a3152 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Mon, 25 Sep 2023 18:41:49 -0700 Subject: [PATCH 08/11] Test and linting fixes --- msticpy/auth/azure_auth_core.py | 2 +- msticpy/auth/cloud_mappings.py | 51 +++++++---- msticpy/auth/cloud_mappings_offline.py | 3 + msticpy/auth/keyvault_settings.py | 5 +- msticpy/config/ce_common.py | 42 ++++----- msticpy/data/drivers/elastic_driver.py | 2 +- .../data/drivers/local_velociraptor_driver.py | 1 + msticpy/data/drivers/mdatp_driver.py | 4 +- msticpy/data/drivers/odata_driver.py | 91 ++++++++++--------- msticpy/data/drivers/security_graph_driver.py | 2 +- msticpy/init/azure_synapse_tools.py | 2 +- tests/config/test_item_editors.py | 8 ++ tests/testdata/az_global_cloud_endpoints.json | 44 +++++++++ 13 files changed, 162 insertions(+), 95 deletions(-) create mode 100644 tests/testdata/az_global_cloud_endpoints.json diff --git a/msticpy/auth/azure_auth_core.py b/msticpy/auth/azure_auth_core.py index 58fcfe89c..dba565b00 100644 --- a/msticpy/auth/azure_auth_core.py +++ b/msticpy/auth/azure_auth_core.py @@ -147,7 +147,7 @@ def _build_certificate_client( return None return CertificateCredential( authority=aad_uri, - tenant_id=tenant_id, + tenant_id=tenant_id, # type: ignore client_id=client_id, **kwargs, # type: ignore ) diff --git a/msticpy/auth/cloud_mappings.py b/msticpy/auth/cloud_mappings.py index 8e45111aa..5f1d4eb30 100644 --- a/msticpy/auth/cloud_mappings.py +++ b/msticpy/auth/cloud_mappings.py @@ -5,8 +5,9 @@ # -------------------------------------------------------------------------- """Azure Cloud Mappings.""" import contextlib -from typing import List, Optional from functools import lru_cache +from typing import Any, Dict, List, Optional + import httpx from .._version import VERSION @@ -24,7 +25,14 @@ "cn": "https://management.chinacloudapi.cn/", } -CLOUD_ALIASES = {"public": "global", "gov": "usgov", "china": "cn"} +CLOUD_ALIASES = { + "public": "global", + "global": "global", + "gov": "usgov", + "usgov": "usgov", + "china": "cn", + "cn": "cn", +} _DEFENDER_MAPPINGS = { "global": "https://api.securitycenter.microsoft.com/", @@ -49,15 +57,16 @@ def format_endpoint(endpoint: str) -> str: """Format an endpoint with "/" if needed .""" - if endpoint.endswith("/"): - return endpoint - return f"{endpoint}/" + return endpoint if endpoint.endswith("/") else f"{endpoint}/" @lru_cache(maxsize=None) -def get_cloud_endpoints(cloud: str, resource_manager_url: Optional[str] = None) -> dict: +def get_cloud_endpoints( + cloud: str, resource_manager_url: Optional[str] = None +) -> Dict[str, Any]: """ Get the cloud endpoints for a specific cloud. + If resource_manager_url is supplied, it will be used instead of the cloud name. Parameters @@ -77,8 +86,8 @@ def get_cloud_endpoints(cloud: str, resource_manager_url: Optional[str] = None) ------ MsticpyAzureConfigError If the cloud name is not valid. - """ + """ response = None if resource_manager_url: response = get_cloud_endpoints_by_resource_manager_url(resource_manager_url) @@ -88,11 +97,12 @@ def get_cloud_endpoints(cloud: str, resource_manager_url: Optional[str] = None) return response raise MsticpyAzureConfigError( - f"Error retrieving endpoints for Cloud: {cloud} / Resource Manager Url: {resource_manager_url}. Status Code: {response.status_code} {response.st}" + f"Error retrieving endpoints for Cloud: {cloud}", + f"Resource Manager Url: {resource_manager_url}.", ) -def get_cloud_endpoints_by_cloud(cloud: str) -> dict: +def get_cloud_endpoints_by_cloud(cloud: str) -> Dict[str, Any]: """ Get the cloud endpoints for a specific cloud. @@ -107,14 +117,13 @@ def get_cloud_endpoints_by_cloud(cloud: str) -> dict: Contains endpoints and suffixes for a specific cloud. """ - resource_manager_url = CLOUD_MAPPING.get(CLOUD_ALIASES.get(cloud, "global")) - return get_cloud_endpoints_by_resource_manager_url(resource_manager_url) + return get_cloud_endpoints_by_resource_manager_url(resource_manager_url) # type: ignore def get_cloud_endpoints_by_resource_manager_url( resource_manager_url: str, -) -> dict: +) -> Dict[str, Any]: """ Get the cloud endpoints for a specific resource manager url. @@ -141,9 +150,9 @@ def get_cloud_endpoints_by_resource_manager_url( if val == f_resource_manager_url: cloud = key break - return cloud_mappings_offline.get(cloud, "global") + return cloud_mappings_offline[cloud or "global"] - return cloud_mappings_offline.get("global") + return cloud_mappings_offline["global"] def get_azure_config_value(key, default): @@ -175,7 +184,7 @@ def get_m365d_endpoint(cloud: str) -> str: def get_m365d_login_endpoint(cloud: str) -> str: """Get M365 login URL.""" if cloud in {"gcc-high", "dod"}: - return "https://login.microsoftonline.us" + return "https://login.microsoftonline.us/" return AzureCloudConfig().authority_uri @@ -232,7 +241,7 @@ def resolve_cloud_alias( return alias_cf if alias_cf in aliases.values() else None @property - def suffixes(self) -> dict: + def suffixes(self) -> Dict[str, str]: """ Get CloudSuffixes class an Azure cloud. @@ -247,7 +256,7 @@ def suffixes(self) -> dict: If the cloud name is not valid. """ - return self.endpoints.get("suffixes") + return self.endpoints.get("suffixes", {}) @property def token_uri(self) -> str: @@ -260,15 +269,17 @@ def token_uri(self) -> str: def authority_uri(self) -> str: """Return the AAD authority URI.""" return format_endpoint( - self.endpoints.get("authentication").get("loginEndpoint") + self.endpoints.get("authentication", {}).get("loginEndpoint") ) @property def log_analytics_uri(self) -> str: """Return the AAD authority URI.""" - return format_endpoint(self.endpoints.get("logAnalyticsResourceId")) + return format_endpoint( + self.endpoints.get("logAnalyticsResourceId") # type: ignore + ) @property def resource_manager(self) -> str: """Return the resource manager URI.""" - return format_endpoint(self.endpoints.get("resourceManager")) + return format_endpoint(self.endpoints.get("resourceManager")) # type: ignore diff --git a/msticpy/auth/cloud_mappings_offline.py b/msticpy/auth/cloud_mappings_offline.py index c760a1afc..b365ed11e 100644 --- a/msticpy/auth/cloud_mappings_offline.py +++ b/msticpy/auth/cloud_mappings_offline.py @@ -1,4 +1,7 @@ """Cloud mappings for offline use.""" + +# pylint: disable=line-too-long + cloud_mappings_offline = { # All endpoints were retrieved with api-version 2023-01-01 "global": { diff --git a/msticpy/auth/keyvault_settings.py b/msticpy/auth/keyvault_settings.py index 47899e59e..e00a81318 100644 --- a/msticpy/auth/keyvault_settings.py +++ b/msticpy/auth/keyvault_settings.py @@ -166,8 +166,7 @@ def get_tenant_authority_uri( """ auth = authority_uri or self.authority_uri.strip() - if not tenant: - tenant = self.get("tenantid") + tenant = tenant or self.get("tenantid") or self.az_cloud_config.tenant_id if not tenant: raise MsticpyKeyVaultConfigError( "Could not get TenantId from function parameters or configuration.", @@ -205,7 +204,7 @@ def get_tenant_authority_host( If tenant is not defined. """ - tenant = tenant or self.az_cloud_config.tenant_id + tenant = tenant or self.get("tenantid") or self.az_cloud_config.tenant_id if not tenant: raise MsticpyKeyVaultConfigError( "Could not get TenantId from function parameters or configuration.", diff --git a/msticpy/config/ce_common.py b/msticpy/config/ce_common.py index ffb98b567..61ff58455 100644 --- a/msticpy/config/ce_common.py +++ b/msticpy/config/ce_common.py @@ -10,10 +10,10 @@ import ipywidgets as widgets from .._version import VERSION -from ..auth.azure_auth_core import AzureCloudConfig from ..auth.azure_auth import az_connect -from ..context.azure.azure_data import get_token +from ..auth.azure_auth_core import AzureCloudConfig from ..common.utility import mp_ua_header +from ..context.azure.azure_data import get_token from .comp_edit import SettingsControl __version__ = VERSION @@ -176,11 +176,8 @@ def get_subscription_metadata(sub_id: str) -> dict: f"{res_mgmt_uri}/subscriptions/{{subscriptionid}}?api-version=2021-04-01" ) headers = mp_ua_header() - sub_url = get_sub_url.format(subscriptionid=sub_id) - resp = httpx.get(sub_url, headers=headers) - www_header = resp.headers.get("WWW-Authenticate") if not www_header: @@ -192,20 +189,16 @@ def get_subscription_metadata(sub_id: str) -> dict: } tenant_path = hdr_dict.get("Bearer authorization_uri", "").split("/") - if tenant_path: - tenant_id = tenant_path[-1] + if not tenant_path: + return {} - az_credentials = az_connect() - token = get_token(az_credentials, tenant_id) - headers["Authorization"] = f"Bearer {token}" - resp = httpx.get(sub_url, headers=headers) + tenant_id = tenant_path[-1] + az_credentials = az_connect() + token = get_token(az_credentials, tenant_id) + headers["Authorization"] = f"Bearer {token}" + resp = httpx.get(sub_url, headers=headers) - if resp.status_code == 200: - return resp.json() - else: - return {"tenantId": tenant_id} - else: - return {} + return resp.json() if resp.status_code == 200 else {"tenantId": tenant_id} def get_def_tenant_id(sub_id: str) -> Optional[str]: @@ -235,13 +228,16 @@ def get_managed_tenant_id(sub_id: str) -> Optional[List[str]]: # type: ignore """ Get the tenant IDs that are managing a subscription. - Args: - sub_id :str - Subscription ID + Parameters + ---------- + sub_id :str + Subscription ID + + Returns + ------- + Optional[list[str]] + A list of tenant IDs or None if it could not be found. - Returns: - Optional[list[str]] - A list of tenant IDs or None if it could not be found. """ sub_metadata = get_subscription_metadata(sub_id) tenant_ids = sub_metadata.get("managedByTenants", None) diff --git a/msticpy/data/drivers/elastic_driver.py b/msticpy/data/drivers/elastic_driver.py index 6d72042d4..d0ba72df3 100644 --- a/msticpy/data/drivers/elastic_driver.py +++ b/msticpy/data/drivers/elastic_driver.py @@ -142,7 +142,7 @@ def query( # TBD # Run query and return results - return pd.DateFrame() + return pd.DataFrame() def query_with_results(self, query: str, **kwargs) -> Tuple[pd.DataFrame, Any]: """ diff --git a/msticpy/data/drivers/local_velociraptor_driver.py b/msticpy/data/drivers/local_velociraptor_driver.py index a8e8af90f..7d91dc213 100644 --- a/msticpy/data/drivers/local_velociraptor_driver.py +++ b/msticpy/data/drivers/local_velociraptor_driver.py @@ -114,6 +114,7 @@ def schema(self) -> Dict[str, Dict]: if not files: continue sample_df = pd.read_json(files[0], lines=True, nrows=1) + # pylint: disable=no-member self._schema[table] = { col: dtype.name for col, dtype in sample_df.dtypes.to_dict().items() } diff --git a/msticpy/data/drivers/mdatp_driver.py b/msticpy/data/drivers/mdatp_driver.py index f8b484d56..74201b184 100644 --- a/msticpy/data/drivers/mdatp_driver.py +++ b/msticpy/data/drivers/mdatp_driver.py @@ -120,11 +120,11 @@ def _select_api_uris(data_environment, cloud): if data_environment == DataEnvironment.M365D: return ( get_m365d_endpoint(cloud), - f"{login_uri}/{{tenantId}}/oauth2/token", + f"{login_uri}{{tenantId}}/oauth2/token", "/advancedhunting/run", ) return ( get_defender_endpoint(cloud), - f"{login_uri}/{{tenantId}}/oauth2/token", + f"{login_uri}{{tenantId}}/oauth2/token", "/advancedqueries/run", ) diff --git a/msticpy/data/drivers/odata_driver.py b/msticpy/data/drivers/odata_driver.py index 080120bb4..80fd67a6c 100644 --- a/msticpy/data/drivers/odata_driver.py +++ b/msticpy/data/drivers/odata_driver.py @@ -155,50 +155,9 @@ def connect( # Default to using application based authentication if not delegated_auth: - _check_config(cs_dict, "client_secret", "application authentication") - # self.oauth_url and self.req_body are correctly set in concrete - # instances __init__ - req_url = self.oauth_url.format(tenantId=cs_dict["tenant_id"]) # type: ignore - req_body = dict(self.req_body) # type: ignore - req_body["client_id"] = cs_dict["client_id"] - req_body["client_secret"] = cs_dict["client_secret"] - - # Authenticate and obtain AAD Token for future calls - data = urllib.parse.urlencode(req_body).encode("utf-8") - response = httpx.post( - url=req_url, - content=data, - timeout=self.get_http_timeout(**kwargs), - headers=mp_ua_header(), - ) - json_response = response.json() - self.aad_token = json_response.get("access_token", None) - if not self.aad_token: - raise MsticpyConnectionError( - f"Could not obtain access token - {json_response['error_description']}" - ) + json_response = self._get_token_standard_auth(kwargs, cs_dict) else: - _check_config(cs_dict, "username", "delegated authentication") - authority = self.oauth_url.format(tenantId=cs_dict["tenant_id"]) # type: ignore - if authority.startswith("https://login"): - auth_url = urllib.parse.urlparse(authority) - authority = ( - f"{auth_url.scheme}://{auth_url.netloc}/{{tenantId}}".format( - tenantId=cs_dict["tenant_id"] - ) - ) - self.msal_auth = MSALDelegatedAuth( - client_id=cs_dict["client_id"], - authority=authority, - username=cs_dict["username"], - scopes=self.scopes, - auth_type=kwargs.get("auth_type", "device"), - location=cs_dict.get("location", "token_cache.bin"), - connect=True, - ) - self.aad_token = self.msal_auth.token - json_response = {} - self.token_type = "MSAL" # nosec + json_response = self._get_token_delegate_auth(kwargs, cs_dict) self.req_headers["Authorization"] = f"Bearer {self.aad_token}" self.api_root = cs_dict.get("apiRoot", self.api_root) @@ -215,6 +174,52 @@ def connect( json_response["access_token"] = None return json_response + def _get_token_standard_auth(self, kwargs, cs_dict) -> Dict[str, Any]: + _check_config(cs_dict, "client_secret", "application authentication") + # self.oauth_url and self.req_body are correctly set in concrete + # instances __init__ + req_url = self.oauth_url.format(tenantId=cs_dict["tenant_id"]) # type: ignore + req_body = dict(self.req_body) # type: ignore + req_body["client_id"] = cs_dict["client_id"] + req_body["client_secret"] = cs_dict["client_secret"] + + # Authenticate and obtain AAD Token for future calls + data = urllib.parse.urlencode(req_body).encode("utf-8") + response = httpx.post( + url=req_url, + content=data, + timeout=self.get_http_timeout(**kwargs), + headers=mp_ua_header(), + ) + json_response = response.json() + self.aad_token = json_response.get("access_token", None) + if not self.aad_token: + raise MsticpyConnectionError( + f"Could not obtain access token - {json_response['error_description']}" + ) + return json_response + + def _get_token_delegate_auth(self, kwargs, cs_dict) -> Dict[str, Any]: + _check_config(cs_dict, "username", "delegated authentication") + authority = self.oauth_url.format(tenantId=cs_dict["tenant_id"]) # type: ignore + if authority.startswith("https://login"): + auth_url = urllib.parse.urlparse(authority) + authority = f"{auth_url.scheme}://{auth_url.netloc}/{{tenantId}}".format( + tenantId=cs_dict["tenant_id"] + ) + self.msal_auth = MSALDelegatedAuth( + client_id=cs_dict["client_id"], + authority=authority, + username=cs_dict["username"], + scopes=self.scopes, + auth_type=kwargs.get("auth_type", "device"), + location=cs_dict.get("location", "token_cache.bin"), + connect=True, + ) + self.aad_token = self.msal_auth.token + self.token_type = "MSAL" # nosec + return {} + # pylint: disable=too-many-branches def query_with_results(self, query: str, **kwargs) -> Tuple[pd.DataFrame, Any]: """ diff --git a/msticpy/data/drivers/security_graph_driver.py b/msticpy/data/drivers/security_graph_driver.py index 68251de2f..07c0196b9 100644 --- a/msticpy/data/drivers/security_graph_driver.py +++ b/msticpy/data/drivers/security_graph_driver.py @@ -45,7 +45,7 @@ def __init__(self, connection_str: Optional[str] = None, **kwargs): "scope": f"{self.api_root}.default", } login_endpoint = az_cloud_config.authority_uri - self.oauth_url = f"{login_endpoint}/{{tenantId}}/oauth2/v2.0/token" + self.oauth_url = f"{login_endpoint}{{tenantId}}/oauth2/v2.0/token" self.api_ver = kwargs.get("api_ver", "v1.0") if connection_str: diff --git a/msticpy/init/azure_synapse_tools.py b/msticpy/init/azure_synapse_tools.py index 3ad84a4f1..ba760d826 100644 --- a/msticpy/init/azure_synapse_tools.py +++ b/msticpy/init/azure_synapse_tools.py @@ -15,10 +15,10 @@ from .._version import VERSION from ..auth.azure_auth import AzureCredEnvNames, az_connect +from ..auth.cloud_mappings import AzureCloudConfig from ..common.pkg_config import get_config, get_http_timeout, refresh_config, set_config from ..common.provider_settings import get_provider_settings from ..common.utility import mp_ua_header -from ..auth.cloud_mappings import AzureCloudConfig logger = logging.getLogger(__name__) diff --git a/tests/config/test_item_editors.py b/tests/config/test_item_editors.py index eee2721c9..26b1289a7 100644 --- a/tests/config/test_item_editors.py +++ b/tests/config/test_item_editors.py @@ -4,6 +4,7 @@ # license information. # -------------------------------------------------------------------------- """Config settings Items editors.""" +import json import os import re from pathlib import Path @@ -291,6 +292,12 @@ def test_tiproviders_editor(kv_sec, mp_conf_ctrl): @respx.mock def test_get_tenant_id(): """Test get tenantID function.""" + endpoints_uri = "https://management.azure.com/metadata/endpoints.*" + endpoints_resp = json.loads( + Path(TEST_DATA_PATH) + .joinpath("az_global_cloud_endpoints.json") + .read_text(encoding="utf-8") + ) subs_uri = ( r"https://management\.azure\.com//subscriptions/" r"40dcc8bf-0478-4f3b-b275-ed0a94f2c013.*" @@ -314,6 +321,7 @@ def test_get_tenant_id(): "content-length": "115", } respx.get(re.compile(subs_uri)).respond(401, json=subs_json, headers=subs_headers) + respx.get(re.compile(endpoints_uri)).respond(200, json=endpoints_resp) tenantid = get_def_tenant_id("40dcc8bf-0478-4f3b-b275-ed0a94f2c013") check.equal(tenantid.casefold(), "72f988bf-86f1-41af-91ab-2d7cd011db47".casefold()) diff --git a/tests/testdata/az_global_cloud_endpoints.json b/tests/testdata/az_global_cloud_endpoints.json new file mode 100644 index 000000000..fdfee3a97 --- /dev/null +++ b/tests/testdata/az_global_cloud_endpoints.json @@ -0,0 +1,44 @@ +{ + "portal": "https://portal.azure.com", + "authentication": { + "loginEndpoint": "https://login.microsoftonline.com", + "audiences": [ + "https://management.core.windows.net/", + "https://management.azure.com/" + ], + "tenant": "common", + "identityProvider": "AAD" + }, + "media": "https://rest.media.azure.net", + "graphAudience": "https://graph.windows.net/", + "graph": "https://graph.windows.net/", + "name": "AzureCloud", + "suffixes": { + "azureDataLakeStoreFileSystem": "azuredatalakestore.net", + "acrLoginServer": "azurecr.io", + "sqlServerHostname": "database.windows.net", + "azureDataLakeAnalyticsCatalogAndJob": "azuredatalakeanalytics.net", + "keyVaultDns": "vault.azure.net", + "storage": "core.windows.net", + "azureFrontDoorEndpointSuffix": "azurefd.net", + "storageSyncEndpointSuffix": "afs.azure.net", + "mhsmDns": "managedhsm.azure.net", + "mysqlServerEndpoint": "mysql.database.azure.com", + "postgresqlServerEndpoint": "postgres.database.azure.com", + "mariadbServerEndpoint": "mariadb.database.azure.com", + "synapseAnalytics": "dev.azuresynapse.net", + "attestationEndpoint": "attest.azure.net" + }, + "batch": "https://batch.core.windows.net/", + "resourceManager": "https://management.azure.com/", + "vmImageAliasDoc": "https://raw.githubusercontent.com/Azure/azure-rest-api-specs/master/arm-compute/quickstart-templates/aliases.json", + "activeDirectoryDataLake": "https://datalake.azure.net/", + "sqlManagement": "https://management.core.windows.net:8443/", + "microsoftGraphResourceId": "https://graph.microsoft.com/", + "appInsightsResourceId": "https://api.applicationinsights.io", + "appInsightsTelemetryChannelResourceId": "https://dc.applicationinsights.azure.com/v2/track", + "attestationResourceId": "https://attest.azure.net", + "synapseAnalyticsResourceId": "https://dev.azuresynapse.net", + "logAnalyticsResourceId": "https://api.loganalytics.io", + "ossrDbmsResourceId": "https://ossrdbms-aad.database.windows.net" + } From 8a344516fbb831adb1bde841f60ecd1f267f1630 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Tue, 26 Sep 2023 07:56:43 -0700 Subject: [PATCH 09/11] Force cloud mapper offline for unit-tests --- msticpy/auth/cloud_mappings.py | 19 +++++++++++++------ msticpy/context/domain_utils.py | 4 ++-- tests/config/test_item_editors.py | 7 ------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/msticpy/auth/cloud_mappings.py b/msticpy/auth/cloud_mappings.py index 5f1d4eb30..83ed94119 100644 --- a/msticpy/auth/cloud_mappings.py +++ b/msticpy/auth/cloud_mappings.py @@ -14,6 +14,7 @@ from ..common import pkg_config as config from ..common.exceptions import MsticpyAzureConfigError from ..common.pkg_config import get_http_timeout +from ..common.utility.package import unit_testing from .cloud_mappings_offline import cloud_mappings_offline __version__ = VERSION @@ -138,6 +139,8 @@ def get_cloud_endpoints_by_resource_manager_url( Contains endpoints and suffixes for a specific cloud. """ + if unit_testing(): + return cloud_mappings_offline["global"] f_resource_manager_url = format_endpoint(resource_manager_url) endpoint_url = f"{f_resource_manager_url}metadata/endpoints?api-version=latest" try: @@ -145,12 +148,16 @@ def get_cloud_endpoints_by_resource_manager_url( if resp.status_code == 200: return resp.json() - except httpx.ConnectError: - for key, val in CLOUD_MAPPING.items(): - if val == f_resource_manager_url: - cloud = key - break - return cloud_mappings_offline[cloud or "global"] + except httpx.RequestError: + cloud = next( + ( + key + for key, val in CLOUD_MAPPING.items() + if val == f_resource_manager_url + ), + "global", + ) + return cloud_mappings_offline[cloud] return cloud_mappings_offline["global"] diff --git a/msticpy/context/domain_utils.py b/msticpy/context/domain_utils.py index 2d0e56cac..29fb7d378 100644 --- a/msticpy/context/domain_utils.py +++ b/msticpy/context/domain_utils.py @@ -172,8 +172,8 @@ def validate_tld(url_domain: str) -> bool: True if valid public TLD, False if not. """ - _, _, tld = tldextract.extract(url_domain.lower()) - return bool(tld) + extract_result = tldextract.extract(url_domain.lower()) + return bool(extract_result.suffix) @staticmethod def is_resolvable(url_domain: str) -> bool: diff --git a/tests/config/test_item_editors.py b/tests/config/test_item_editors.py index 26b1289a7..7a318ed41 100644 --- a/tests/config/test_item_editors.py +++ b/tests/config/test_item_editors.py @@ -292,12 +292,6 @@ def test_tiproviders_editor(kv_sec, mp_conf_ctrl): @respx.mock def test_get_tenant_id(): """Test get tenantID function.""" - endpoints_uri = "https://management.azure.com/metadata/endpoints.*" - endpoints_resp = json.loads( - Path(TEST_DATA_PATH) - .joinpath("az_global_cloud_endpoints.json") - .read_text(encoding="utf-8") - ) subs_uri = ( r"https://management\.azure\.com//subscriptions/" r"40dcc8bf-0478-4f3b-b275-ed0a94f2c013.*" @@ -321,7 +315,6 @@ def test_get_tenant_id(): "content-length": "115", } respx.get(re.compile(subs_uri)).respond(401, json=subs_json, headers=subs_headers) - respx.get(re.compile(endpoints_uri)).respond(200, json=endpoints_resp) tenantid = get_def_tenant_id("40dcc8bf-0478-4f3b-b275-ed0a94f2c013") check.equal(tenantid.casefold(), "72f988bf-86f1-41af-91ab-2d7cd011db47".casefold()) From 0c7eb17c1bb72a3242a5818b2466cdf983ac580d Mon Sep 17 00:00:00 2001 From: Christopher Cianelli Date: Tue, 26 Sep 2023 12:26:03 -0400 Subject: [PATCH 10/11] add try except for get_subscription_metadata --- msticpy/config/ce_common.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/msticpy/config/ce_common.py b/msticpy/config/ce_common.py index 61ff58455..ee9eab745 100644 --- a/msticpy/config/ce_common.py +++ b/msticpy/config/ce_common.py @@ -193,12 +193,14 @@ def get_subscription_metadata(sub_id: str) -> dict: return {} tenant_id = tenant_path[-1] - az_credentials = az_connect() - token = get_token(az_credentials, tenant_id) - headers["Authorization"] = f"Bearer {token}" - resp = httpx.get(sub_url, headers=headers) - - return resp.json() if resp.status_code == 200 else {"tenantId": tenant_id} + try: + az_credentials = az_connect() + token = get_token(az_credentials, tenant_id) + headers["Authorization"] = f"Bearer {token}" + resp = httpx.get(sub_url, headers=headers) + return resp.json() if resp.status_code == 200 else {"tenantId": tenant_id} + except: + return {"tenantId": tenant_id} def get_def_tenant_id(sub_id: str) -> Optional[str]: From 0dc9e76ec6843f238541c1fa6b7c416789f4f285 Mon Sep 17 00:00:00 2001 From: Ian Hellen Date: Tue, 26 Sep 2023 09:39:54 -0700 Subject: [PATCH 11/11] Fix test and mypy failures --- msticpy/config/ce_common.py | 2 +- msticpy/context/azure/azure_data.py | 4 ++-- msticpy/context/azure/sentinel_incidents.py | 4 +++- msticpy/data/storage/azure_blob_storage.py | 2 +- tests/config/test_item_editors.py | 5 +++-- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/msticpy/config/ce_common.py b/msticpy/config/ce_common.py index 61ff58455..371def185 100644 --- a/msticpy/config/ce_common.py +++ b/msticpy/config/ce_common.py @@ -432,7 +432,7 @@ def get_defn_or_default(defn: Union[Tuple[str, Any], Any]) -> Tuple[str, Dict]: # flake8: noqa: F821 def get_or_create_mpc_section( - mp_controls: "MpConfigControls", + mp_controls: "MpConfigControls", # type: ignore[name-defined] section: str, subkey: Optional[str] = None, # type: ignore ) -> Any: diff --git a/msticpy/context/azure/azure_data.py b/msticpy/context/azure/azure_data.py index c7db6ffd4..94a97fb73 100644 --- a/msticpy/context/azure/azure_data.py +++ b/msticpy/context/azure/azure_data.py @@ -360,8 +360,8 @@ def get_resources( # noqa: MC0001 resources = [] # type: List if rgroup is None: resources.extend( - iter(self.resource_client.resources.list()) - ) # type: ignore + iter(self.resource_client.resources.list()) # type: ignore + ) else: resources.extend( iter( diff --git a/msticpy/context/azure/sentinel_incidents.py b/msticpy/context/azure/sentinel_incidents.py index 82b1af77f..1c1c632e0 100644 --- a/msticpy/context/azure/sentinel_incidents.py +++ b/msticpy/context/azure/sentinel_incidents.py @@ -460,7 +460,9 @@ def add_bookmark_to_incident(self, incident: str, bookmark: str): mark_res_id = self.sent_urls["bookmarks"] + f"/{bookmark_id}" # type: ignore relations_id = uuid4() bookmark_url = incident_url + f"/relations/{relations_id}" - bkmark_data_items = {"relatedResourceId": mark_res_id.split(self.base_url)[1]} + bkmark_data_items = { + "relatedResourceId": mark_res_id.split(self.base_url)[1] # type: ignore + } data = _build_sent_data(bkmark_data_items, props=True) params = {"api-version": "2021-04-01"} response = httpx.put( diff --git a/msticpy/data/storage/azure_blob_storage.py b/msticpy/data/storage/azure_blob_storage.py index 63bd19bb6..4f16c46ce 100644 --- a/msticpy/data/storage/azure_blob_storage.py +++ b/msticpy/data/storage/azure_blob_storage.py @@ -117,7 +117,7 @@ def blobs(self, container_name: str) -> Optional[pd.DataFrame]: Details of the blobs. """ - container_client = self.abs_client.get_container_client( + container_client = self.abs_client.get_container_client( # type: ignore[union-attr] container_name ) # type: ignore blobs = list(container_client.list_blobs()) diff --git a/tests/config/test_item_editors.py b/tests/config/test_item_editors.py index 7a318ed41..69eb6087f 100644 --- a/tests/config/test_item_editors.py +++ b/tests/config/test_item_editors.py @@ -4,7 +4,6 @@ # license information. # -------------------------------------------------------------------------- """Config settings Items editors.""" -import json import os import re from pathlib import Path @@ -290,8 +289,10 @@ def test_tiproviders_editor(kv_sec, mp_conf_ctrl): @respx.mock -def test_get_tenant_id(): +@patch("msticpy.config.ce_common.get_token") +def test_get_tenant_id(get_token): """Test get tenantID function.""" + get_token.return_value = "[PLACEHOLDER]" subs_uri = ( r"https://management\.azure\.com//subscriptions/" r"40dcc8bf-0478-4f3b-b275-ed0a94f2c013.*"