From 39fd26f07fd9f03f818d26b66d426b562ceeaeec Mon Sep 17 00:00:00 2001 From: Ryan Cobb Date: Tue, 25 Jun 2024 13:47:45 -0400 Subject: [PATCH 1/5] Add driver and docs --- .../data_acquisition/DataProv-MSDefender.rst | 38 ++++- msticpy/data/drivers/mdatp_driver.py | 143 ++++++++++++------ 2 files changed, 129 insertions(+), 52 deletions(-) diff --git a/docs/source/data_acquisition/DataProv-MSDefender.rst b/docs/source/data_acquisition/DataProv-MSDefender.rst index c55413fc3..db1d1d1df 100644 --- a/docs/source/data_acquisition/DataProv-MSDefender.rst +++ b/docs/source/data_acquisition/DataProv-MSDefender.rst @@ -16,10 +16,37 @@ M365 Defender Configuration Creating a Client App for M365 Defender ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Details on registering an Azure AD application for MS 365 Defender can be found -`here `__. +Microsoft 365 Defender APIs can be accessed in both `application ` +and `delegated user contexts `. +Accessing Microsoft 365 Defender APIs as an application requires +either a client secret or certificate, while delegated user auth requires +an interactive signin through a browser or via device code. + +As such, the details on registering an Azure AD application for MS 365 Defender +are different for application and delegated user auth scenarios. Please +see the above links for more information. Notably, delegated user auth +scenarios do not require a application credential and thus is preferrable. + +For delegated user auth scenarios, ensure that the application has a +"Mobile or Desktop Application" redirect URI configured as `http://localhost`. +A redirect URI is not required for applications with their own credentials. + +API permissions for the client application will require tenant admin consent. +Ensure that the consented permissions are correct for the chosen data environment +and auth scenario (application or delegated user): + ++-----------------------------+------------------------+------------------+ +| API Name | Permission | Data Environment | ++=============================+========================+==================+ +| WindowsDefenderATP | AdvancedQuery.Read | MDE, MDATP | ++-----------------------------+------------------------+------------------+ +| Microsoft Threat Protection | AdvancedHunting.Read | M365D | ++-----------------------------+------------------------+------------------+ +| Microsoft Graph | ThreatHunting.Read.All | M365DGraph | ++-----------------------------+------------------------+------------------+ + Once you have registered the application, you can use it to connect to -the MS Defender API. +the MS Defender API using the chosen data environment. M365 Defender Configuration in MSTICPy ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -39,13 +66,13 @@ The settings in the file should look like the following: MicrosoftDefender: Args: ClientId: "CLIENT ID" - ClientSecret: "CLIENT SECRET" TenantId: "TENANT ID" UserName: "User Name" Cloud: "global" -We strongly recommend storing the client secret value +If connecting to the MS Defender 365 API using application auth, +we strongly recommend storing the client secret value in Azure Key Vault. You can replace the text value with a referenced to a Key Vault secret using the MSTICPy configuration editor. See :doc:`msticpy Settings Editor <../getting_started/SettingsEditor>`) @@ -166,6 +193,7 @@ the required parameters are: * client_secret -- The secret used for by the application. * username -- If using delegated auth for your application. +The client_secret and username parameters are mutually exclusive. .. code:: ipython3 diff --git a/msticpy/data/drivers/mdatp_driver.py b/msticpy/data/drivers/mdatp_driver.py index e52b44f87..c46b0126a 100644 --- a/msticpy/data/drivers/mdatp_driver.py +++ b/msticpy/data/drivers/mdatp_driver.py @@ -4,7 +4,8 @@ # license information. # -------------------------------------------------------------------------- """MDATP OData Driver class.""" -from typing import Any, Optional, Union +from dataclasses import dataclass, field +from typing import Any, List, Optional, Union import pandas as pd @@ -24,6 +25,33 @@ __author__ = "Pete Bryan" +@dataclass +class M365DConfiguration: + """A container for M365D API settings. + + This is based on the data environment of the query provider. + """ + + login_uri: str + resource_uri: str + api_version: str + api_endpoint: str + api_uri: str + scopes: List[str] + oauth_v2: bool = field(init=False) + + def __post_init__(self): + """Determine if the selected API supports Entra ID OAuth v2.0. + + This is important because the fields in the request body + are different between the two versions. + """ + if "/oauth2/v2.0" in self.login_uri: + self.oauth_v2 = True + else: + self.oauth_v2 = False + + @export class MDATPDriver(OData): """KqlDriver class to retrieve date from MS Defender APIs.""" @@ -46,6 +74,7 @@ def __init__( """ super().__init__(**kwargs) + cs_dict = _get_driver_settings( self.CONFIG_NAME, self._ALT_CONFIG_NAMES, instance ) @@ -54,40 +83,37 @@ def __init__( if "cloud" in kwargs and kwargs["cloud"]: self.cloud = kwargs["cloud"] - api_uri, oauth_uri, api_suffix = _select_api_uris( - self.data_environment, self.cloud - ) + m365d_params = _select_api(self.data_environment, self.cloud) + self._m365d_params: M365DConfiguration = m365d_params + self.oauth_url = m365d_params.login_uri + self.api_root = m365d_params.resource_uri + self.api_ver = m365d_params.api_version + self.api_suffix = m365d_params.api_endpoint + self.scopes = m365d_params.scopes + self.add_query_filter( "data_environments", ("MDE", "M365D", "MDATP", "GraphHunting") ) - self.req_body = { - "client_id": None, - "client_secret": None, - "grant_type": "client_credentials", - "resource": api_uri, - } - self.oauth_url = oauth_uri - self.api_root = api_uri - self.api_ver = "api" - self.api_suffix = api_suffix - if self.data_environment == DataEnvironment.M365D: - self.scopes = [f"{api_uri}/AdvancedHunting.Read"] - elif self.data_environment == DataEnvironment.M365DGraph: - self.api_ver = kwargs.get("api_ver", "v1.0") - self.req_body = { - "client_id": None, - "client_secret": None, - "grant_type": "client_credentials", - "scope": f"{self.api_root}.default", - } - self.scopes = [f"{api_uri}/ThreatHunting.Read.All"] + self.req_body = {} + if "username" in cs_dict: + delegated_auth = True + else: - self.scopes = [f"{api_uri}/AdvancedQuery.Read"] + delegated_auth = False + self.req_body["grant_type"] = "client_credentials" + + if not m365d_params.oauth_v2: + self.req_body["resource"] = self.scopes if connection_str: self.current_connection = connection_str - self.connect(connection_str) + self.connect( + connection_str, + delegated_auth=delegated_auth, + auth_type=kwargs.get("auth_type", "interactive"), + location=cs_dict.get("location", "token_cache.bin"), + ) def query( self, query: str, query_source: Optional[QuerySource] = None, **kwargs @@ -135,26 +161,49 @@ def query( return response -def _select_api_uris(data_environment, cloud): - """Return API and login URIs for selected provider type.""" - login_uri = get_m365d_login_endpoint(cloud) - if data_environment == DataEnvironment.M365D: - return ( - get_m365d_endpoint(cloud), - f"{login_uri}{{tenantId}}/oauth2/token", - "/advancedhunting/run", - ) +def _select_api(data_environment, cloud) -> M365DConfiguration: + # pylint: disable=line-too-long + """Return API and login URIs for selected provider type. + + Note that the Microsoft Graph is the preferred API. + + | API Name | Resource ID | Scopes Requested | API URI (global cloud) | API Endpoint | Login URI | MSTICpy Data Environment | + | -------- | ----------- | ---------------- | ---------------------- | ------------ | --------- | ------------------------ | + | WindowsDefenderATP | fc780465-2017-40d4-a0c5-307022471b92 | `AdvancedQuery.Read` | `https://api.securitycenter.microsoft.com` | `/advancedqueries/run` | `https://login.microsoftonline.com//oauth2/token` | `MDE`, `MDATP` | + | Microsoft Threat Protection | 8ee8fdad-f234-4243-8f3b-15c294843740 | `AdvancedHunting.Read` | `https://api.security.microsoft.com` | `/advancedhunting/run` | `https://login.microsoftonline.com//oauth2/token` | `M365D` | + | Microsoft Graph | 00000003-0000-0000-c000-000000000000 | `ThreatHunting.Read.All` | `https://graph.microsoft.com//` | `/security/runHuntingQuery` | `https://login.microsoftonline.com//oauth2/v2.0/token` | `M365DGraph` | + + """ + # pylint: enable=line-too-long if data_environment == DataEnvironment.M365DGraph: az_cloud_config = AzureCloudConfig(cloud=cloud) - api_uri = az_cloud_config.endpoints.get("microsoftGraphResourceId") - graph_login = az_cloud_config.authority_uri - return ( - api_uri, - f"{graph_login}{{tenantId}}/oauth2/v2.0/token", - "/security/runHuntingQuery", - ) - return ( - get_defender_endpoint(cloud), - f"{login_uri}{{tenantId}}/oauth2/token", - "/advancedqueries/run", + login_uri = f"{az_cloud_config.authority_uri}{{tenantId}}/oauth2/v2.0/token" + resource_uri = az_cloud_config.endpoints.get("microsoftGraphResourceId") + api_version = "v1.0" + api_endpoint = "/security/runHuntingQuery" + scopes = [f"{resource_uri}ThreatHunting.Read.All"] + + elif data_environment == DataEnvironment.M365D: + login_uri = f"{get_m365d_login_endpoint(cloud)}{{tenantId}}/oauth2/token" + resource_uri = get_m365d_endpoint(cloud) + api_version = "api" + api_endpoint = "/advancedhunting/run" + scopes = [f"{resource_uri}AdvancedHunting.Read"] + + else: + login_uri = f"{get_m365d_login_endpoint(cloud)}{{tenantId}}/oauth2/token" + resource_uri = get_defender_endpoint(cloud) + api_version = "api" + api_endpoint = "/advancedqueries/run" + scopes = [f"{resource_uri}AdvancedQuery.Read"] + + api_uri = f"{resource_uri}{api_version}{api_endpoint}" + + return M365DConfiguration( + login_uri=login_uri, + resource_uri=resource_uri, + api_version=api_version, + api_endpoint=api_endpoint, + api_uri=api_uri, + scopes=scopes, ) From 7ff65f53e67b0f96b32d850387d132f28355f45c Mon Sep 17 00:00:00 2001 From: Ryan Cobb Date: Tue, 25 Jun 2024 17:26:43 -0400 Subject: [PATCH 2/5] Add M365DGraph data environment to compatible driver list --- msticpy/data/core/data_providers.py | 1 + msticpy/data/drivers/mdatp_driver.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/msticpy/data/core/data_providers.py b/msticpy/data/core/data_providers.py index c11f0aa9c..d781c1979 100644 --- a/msticpy/data/core/data_providers.py +++ b/msticpy/data/core/data_providers.py @@ -34,6 +34,7 @@ "mde": ["m365d"], "mssentinel_new": ["mssentinel", "m365d"], "kusto_new": ["kusto"], + "m365dgraph": ["mde", "m365d"], } logger = logging.getLogger(__name__) diff --git a/msticpy/data/drivers/mdatp_driver.py b/msticpy/data/drivers/mdatp_driver.py index c46b0126a..dc5559636 100644 --- a/msticpy/data/drivers/mdatp_driver.py +++ b/msticpy/data/drivers/mdatp_driver.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- """MDATP OData Driver class.""" from dataclasses import dataclass, field -from typing import Any, List, Optional, Union +from typing import Any, Dict, List, Optional, Union import pandas as pd @@ -92,10 +92,10 @@ def __init__( self.scopes = m365d_params.scopes self.add_query_filter( - "data_environments", ("MDE", "M365D", "MDATP", "GraphHunting") + "data_environments", ("MDE", "M365D", "MDATP", "M365DGraph", "GraphHunting") ) - self.req_body = {} + self.req_body: Dict[str, Any] = {} if "username" in cs_dict: delegated_auth = True From fe1327a565c92c466fc11148afbfae4069c9f173 Mon Sep 17 00:00:00 2001 From: Ryan Cobb Date: Tue, 25 Jun 2024 17:37:29 -0400 Subject: [PATCH 3/5] Remove warnings import --- msticpy/data/drivers/mdatp_driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msticpy/data/drivers/mdatp_driver.py b/msticpy/data/drivers/mdatp_driver.py index dc5559636..051a119bc 100644 --- a/msticpy/data/drivers/mdatp_driver.py +++ b/msticpy/data/drivers/mdatp_driver.py @@ -178,7 +178,7 @@ def _select_api(data_environment, cloud) -> M365DConfiguration: if data_environment == DataEnvironment.M365DGraph: az_cloud_config = AzureCloudConfig(cloud=cloud) login_uri = f"{az_cloud_config.authority_uri}{{tenantId}}/oauth2/v2.0/token" - resource_uri = az_cloud_config.endpoints.get("microsoftGraphResourceId") + resource_uri = az_cloud_config.endpoints["microsoftGraphResourceId"] api_version = "v1.0" api_endpoint = "/security/runHuntingQuery" scopes = [f"{resource_uri}ThreatHunting.Read.All"] From af09007a0d2f15b09bc0d1d244f11c500eff268a Mon Sep 17 00:00:00 2001 From: Ryan Cobb Date: Tue, 25 Jun 2024 17:49:15 -0400 Subject: [PATCH 4/5] Fix numpy 2.0 NaN now being nan test failure --- msticpy/vis/entity_graph_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msticpy/vis/entity_graph_tools.py b/msticpy/vis/entity_graph_tools.py index 6d95f63b0..d6d776b94 100644 --- a/msticpy/vis/entity_graph_tools.py +++ b/msticpy/vis/entity_graph_tools.py @@ -335,7 +335,7 @@ def to_df(self) -> pd.DataFrame: } for node in self.alertentity_graph.nodes.values() ] - return pd.DataFrame(node_list).replace("None", np.NaN) + return pd.DataFrame(node_list).replace("None", np.nan) def _add_incident_or_alert_node(self, incident: Union[Incident, Alert, None]): """Check what type of entity is passed in and creates relevant graph.""" From 61f74ce5c6c9215345834164da1cf4b8d761e9cf Mon Sep 17 00:00:00 2001 From: Ryan Cobb Date: Wed, 26 Jun 2024 08:36:34 -0400 Subject: [PATCH 5/5] Default to delegated auth when username is present in config or cs --- msticpy/data/drivers/odata_driver.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/msticpy/data/drivers/odata_driver.py b/msticpy/data/drivers/odata_driver.py index 7696077f1..fe2fb1317 100644 --- a/msticpy/data/drivers/odata_driver.py +++ b/msticpy/data/drivers/odata_driver.py @@ -153,11 +153,14 @@ def connect( help_uri=("Connecting to OData sources.", _HELP_URI), ) - # Default to using application based authentication - if not delegated_auth: - json_response = self._get_token_standard_auth(kwargs, cs_dict) - else: + # Default to using delegated auth if username is present + if "username" in cs_dict: + delegated_auth = True + + if delegated_auth: json_response = self._get_token_delegate_auth(kwargs, cs_dict) + else: + json_response = self._get_token_standard_auth(kwargs, cs_dict) self.req_headers["Authorization"] = f"Bearer {self.aad_token}" self.api_root = cs_dict.get("apiRoot", self.api_root)