From b10d676abc27cdf54dd45119ab56f1a286c39f94 Mon Sep 17 00:00:00 2001 From: Aviv Ron Date: Tue, 4 Apr 2023 11:30:25 +0300 Subject: [PATCH 1/4] introduce changes from 5 --- .../msatp/configuration/config.json | 15 +- .../msatp/configuration/lang_en.json | 18 +- stix_shifter_modules/msatp/readme.md | 419 +++ .../stix_translation/json/from_stix_map.json | 525 ++-- .../stix_translation/json/to_stix_map.json | 2475 +++++++++++++++-- .../stix_translation/query_constructor.py | 20 +- .../stix_translation/results_translator.py | 202 +- .../msatp/stix_translation/transformers.py | 134 +- .../msatp/stix_transmission/api_client.py | 14 +- .../msatp/stix_transmission/connector.py | 60 +- .../connector_post_processing.py | 240 ++ .../test_msatp_json_to_stix.py | 917 +++--- .../test_msatp_stix_to_query.py | 539 ++-- .../test_msatp_transformers.py | 71 + .../test_connector_post_processing.py | 420 +++ .../tests/stix_transmission/test_msatp.py | 118 +- .../msatp/tests/test_utils.py | 77 + 17 files changed, 5184 insertions(+), 1080 deletions(-) create mode 100644 stix_shifter_modules/msatp/readme.md create mode 100644 stix_shifter_modules/msatp/stix_transmission/connector_post_processing.py create mode 100644 stix_shifter_modules/msatp/tests/stix_translation/test_msatp_transformers.py create mode 100644 stix_shifter_modules/msatp/tests/stix_transmission/test_connector_post_processing.py create mode 100644 stix_shifter_modules/msatp/tests/test_utils.py diff --git a/stix_shifter_modules/msatp/configuration/config.json b/stix_shifter_modules/msatp/configuration/config.json index 2229c4cb5..f0a66865c 100644 --- a/stix_shifter_modules/msatp/configuration/config.json +++ b/stix_shifter_modules/msatp/configuration/config.json @@ -1,7 +1,8 @@ { "connection": { "type": { - "displayName": "Microsoft Defender for Endpoint" + "displayName": "Microsoft Defender for Endpoint", + "group": "azure" }, "host": { "type": "text", @@ -38,6 +39,18 @@ "clientSecret": { "type": "password" } + }, + "includeAlerts":{ + "type": "boolean", + "default": false + }, + "includeHostOs": { + "type": "boolean", + "default": false + }, + "includeNetworkInfo": { + "type": "boolean", + "default": false } } } \ No newline at end of file diff --git a/stix_shifter_modules/msatp/configuration/lang_en.json b/stix_shifter_modules/msatp/configuration/lang_en.json index a2d764a17..102d4d51b 100644 --- a/stix_shifter_modules/msatp/configuration/lang_en.json +++ b/stix_shifter_modules/msatp/configuration/lang_en.json @@ -1,13 +1,13 @@ { "connection": { "host": { - "label": "Management IP address or Hostname", + "label": "Management IP address or hostname", "placeholder": "192.168.1.10", - "description": "Specify the IP address or hostname of the data source so that IBM Cloud Pak for Security can communicate with it" + "description": "Specify the IP address or hostname of the data source" }, "port": { "label": "Host Port", - "description": "Set the port number that is associated with the Host name or IP" + "description": "Set the port number that is associated with the hostname or IP address" }, "help": { "label": "Need additional help?", @@ -37,6 +37,18 @@ "label": "Client Secret", "description": "Client Secret of Azure Active directory Application with access to the Microsoft Graph API" } + }, + "includeAlerts": { + "label": "Join events with Alerts?", + "description": "With this feature enabled every event will be searched for correlated alerts in the DeviceAlertsEvent table and the alert data will be included in the results. Keep in mind that this feature will add API CPU consumption." + }, + "includeHostOs": { + "label": "Join events with Host OS info?", + "description": "With this feature enabled every event will include also the device OS details from the DeviceInfo table. Keep in mind that this feature will add API CPU consumption." + }, + "includeNetworkInfo": { + "label": "Join events with Device Network info?", + "description": "With this feature enabled every event will include also the active network adapters IP and MAC addresses of the device at the time of the event from the DeviceNetworkInfo table. Keep in mind that this feature will add API CPU consumption." } } } \ No newline at end of file diff --git a/stix_shifter_modules/msatp/readme.md b/stix_shifter_modules/msatp/readme.md new file mode 100644 index 000000000..66ba3699e --- /dev/null +++ b/stix_shifter_modules/msatp/readme.md @@ -0,0 +1,419 @@ +# MSATP - microsoft defender for endpoint module +This module uses KQL to query the EDR API. + +Queries are constructed as unions for all the relevant event tables based on the queried fields. +The potential tables are `DeviceProcessEvents`, `DeviceNetworkEvents`, +`DeviceRegistryEvents`, `DeviceFileEvents`, `DeviceImageLoadEvents` and `DeviceEvents` + +Every query is joined with `DeviceNetworkInfo` in order to get the active network adapters from the relevant time of\ +the event (IP and mac addresses). + +Every query is also joined with `DeviceInfo` to get the host operating system. + +Queries are joined with `DeviceAlertEvents` to look for relevant alerts. An alert is joined based on +the `Timestamp` of the event, `DeviceId` and `ReportId`. Relevant alerts will be mapped to `x-ibm-finding` +with their relevant TTP's. +Searching for an alert field (for example `x-ibm-finding:alert_id`) will search the alerts table first +and then join it with the relevant events that are correlated to it. + +## Tips for successful hunting +Be concise with what you are looking for. For example searching for a file property will be +very wide as it will look at relevant file events but also at all the binary files of +processes in the initiating process of each event and its parent process. +If you wish to look for a specific process don't query for a file, instead query the process. +For example the query `[process:pid=123]` will only look at the initiating process id. +If you want to search for a process which its parent is called `foo`look for +`[process:parent_ref.name='foo']`. This will bring all the relevant events from processes which +their parent process is `foo` only. + +If you wish to search for a file from the device file events +(a file that has been created, modified etc.) but not search the process binary fields from all +other event tables query for: +`[x-oca-event:file_ref.name='foo']` + +You can query for specific event types by querying the `action` field of `x-oca-event`. +For example if I wish to search for all `RegistryValueSet` events I will query: +`[x-oca-event:action = 'RegistryValueSet' ]` + +If you want to query events on a specific host, provide its hostname as such: +`[x-oca-asset:hostname = 'host.example.com' ]` + +Searching for an IP address `[ipv4-addr:value = '9.9.9.9' ]` is more general as it will search both source and +destination ip fields. If you know the IP usage it is better to narrow the query. +For example for a remote IP use: `[network-traffic:dst_ref.value = '9.9.9.9' ]`. +For a local IP use `[network-traffic:src_ref.value = '9.9.9.1' ]` + +## Examples +Here are a few examples for results from different tables and how to interpret them: + +### DeviceProcessEvents +The main starting point to look at will be `x-oca-event:action` which tells us the +event is a process create event. + +the `process_ref` will lead us to the process that was created and its `parent_ref` +to the process which created it. + +`host_ref` leads us to `x-oca-asset` which shows information on the host on which this +event occurred: its name, unique device id, and the ip addresses and mac addresses +that were observed on the device. + +`created` shows us the timestamp when this event was observed. + +`user_ref` shows us the user account that was performing the event. `account_login` is mapped +to the user UPN which is resolved by msatp from the active directory. `user_id` is mapped +to the account name. + +`external_ref` is a link to the msatp web console that will show us the timeline of this device +one second before and after the event occurred. + +`original_ref` is the original json output from the msatp api response encoded in base64. +If there are fields you are interested to check but are not mapped to stix you can find them +there. +```json +{ + "0": + { + "type": "x-msatp", + "ReportId": 1234 + }, + "1": + { + "type": "x-oca-asset", + "hostname": "host.test.com", + "device_id": "deviceid", + "mac_refs": ["13"], + "ip_refs": ["14"], + "architecture": "64-bit", + "os_name": "Windows10", + "os_version": "10.0" + }, + "2": + { + "type": "x-oca-event", + "host_ref": "1", + "created": "2023-03-17T20:23:03.7116107Z", + "action": "ProcessCreated", + "process_ref": "4", + "user_ref": "7", + "provider": "Microsoft Defender for Endpoint", + "external_ref": "15", + "original_ref": "16" + }, + "3": + { + "type": "file", + "name": "msedge.exe", + "parent_directory_ref": "6", + "hashes": + { + "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed", + "SHA-256": "470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75", + "MD5": "e180c9a532c45eba99eefd01601f5c41" + } + }, + "4": + { + "type": "process", + "name": "msedge.exe", + "binary_ref": "3", + "pid": 37384, + "command_line": "\"msedge.exe\" --type=gpu-process", + "created": "2023-03-17T20:23:03.702Z", + "creator_user_ref": "7", + "parent_ref": "5" + }, + "5": + { + "type": "process", + "child_refs": ["4"], + "creator_user_ref": "8", + "binary_ref": "9", + "name": "msedge.exe", + "pid": 400, + "command_line": "\"msedge.exe\" -- \"https://test.com/login/login.asp\"", + "created": "2023-03-17T20:23:03.441Z", + "parent_ref": "11" + }, + "6": + { + "type": "directory", + "path": "C:\\Program Files (x86)\\Microsoft\\Edge\\Application" + }, + "7": + { + "type": "user-account", + "user_id": "username", + "account_login": "username@test.com" + }, + "8": + { + "type": "user-account", + "user_id": "username", + "account_login": "username@test.com" + }, + "9": + { + "type": "file", + "hashes": + { + "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed", + "SHA-256": "470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75", + "MD5": "e180c9a532c45eba99eefd01601f5c41" + }, + "name": "msedge.exe", + "parent_directory_ref": "10" + }, + "10": + { + "type": "directory", + "path": "c:\\program files (x86)\\microsoft\\edge\\application" + }, + "11": + { + "type": "process", + "pid": 30972, + "name": "iexplore.exe", + "binary_ref": "12", + "created": "2023-03-17T20:23:03.169Z" + }, + "12": + { + "type": "file", + "name": "iexplore.exe" + }, + "13": + { + "type": "mac-addr", + "value": "11:22:33:44:55:66" + }, + "14": + { + "type": "ipv4-addr", + "value": "9.9.9.1" + }, + "15": + { + "type": "external-reference", + "url": "https://security.microsoft.com/machines/deviceid/timeline?from=2023-03-17T20:23:02.000Z&to=2023-03-17T20:23:04.000Z" + }, + "16": + { + "type": "artifact", + "payload_bin": "base64 encoded json output from msatp api response before translation" + } +} +``` + +### DeviceNetworkEvents with associated alert + +An example including a network event to an url. +Query example: `[domain-name:value='malicious.com']` + +`url_ref` points us to the url visited. +`domain_ref` to the domain parsed from the url. +It is possible that only one of these will be populated depending on the data provided. + +`network_ref` will show us the source and destination ip addresses and ports. Mind you that +the destination ip address is the one resolved from the domain name +referenced by `domain_ref` in the `x-oca-event`. + +`finding_refs` indicates that there are associated alerts with this event. These will be +of type `x-ibm-finding`. The event is a connection success event, and the associated alert +title is `Connection to adversary-in-the-middle (AiTM) phishing site`. + +`severity` is translated from `high` (99) `medium` (66) and `low` (33). + +`alert_id` is the unique alert identifier from msatp. + +`ttp_tagging_refs` shows us the relevant TTP's. These will be of type `x-ibm-ttp-tagging`. +Tactics will be mapped to `kill_chain_phases` which might be mapped to the +mitre attack framework or a proprietary Microsoft phase - not all phases are mapped to +mitre attack. +Techniques will be mapped to the `mitre-attack-ext` extension and will describe the name and +id of the mitre attack technique. + +```json +{ + "0": + { + "type": "x-oca-event", + "created": "2023-03-17T20:19:46.6337905Z", + "host_ref": "1", + "process_ref": "4", + "action": "ConnectionSuccess", + "network_ref": "7", + "domain_ref": "9", + "url_ref": "8", + "provider": "Microsoft Defender for Endpoint", + "external_ref": "17", + "finding_refs": ["18"], + "original_ref": "20", + "file_ref": "2" + }, + "1": + { + "type": "x-oca-asset", + "device_id": "deviceid", + "hostname": "host.example.com", + "ip_refs": ["10"], + "mac_refs": ["16"], + "architecture": "64-bit", + "os_name": "Windows10", + "os_version": "10.0" + }, + "2": + { + "type": "file", + "name": "msedge.exe", + "hashes": + { + "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed" + } + }, + "4": + { + "type": "process", + "child_refs": ["3"], + "binary_ref": "11", + "name": "msedge.exe", + "pid": 3052, + "command_line": "\"msedge.exe\" --type=utility", + "created": "2023-03-13T14:22:44.860Z", + "parent_ref": "14", + "creator_user_ref": "15" + }, + "5": + { + "type": "x-msatp", + "ReportId": 1234, + "Table": "DeviceEvents" + }, + "6": + { + "type": "ipv4-addr", + "value": "9.9.9.9" + }, + "7": + { + "type": "network-traffic", + "dst_ref": "6", + "dst_port": 443, + "src_ref": "10", + "src_port": 58993 + }, + "8": + { + "type": "url", + "value": "https://malicious.com" + }, + "9": + { + "type": "domain-name", + "value": "malicious.com" + }, + "10": + { + "type": "ipv4-addr", + "value": "9.9.9.1" + }, + "11": + { + "type": "file", + "hashes": + { + "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed", + "SHA-256": "470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75", + "MD5": "e180c9a532c45eba99eefd01601f5c41" + }, + "name": "msedge.exe", + "parent_directory_ref": "12" + }, + "12": + { + "type": "directory", + "path": "c:\\program files (x86)\\microsoft\\edge\\application" + }, + "13": + { + "type": "file", + "name": "msedge.exe" + }, + "14": + { + "type": "process", + "name": "msedge.exe", + "binary_ref": "13", + "pid": 9952, + "created": "2023-03-13T14:22:44.508Z" + }, + "15": + { + "type": "user-account", + "user_id": "username", + "account_login": "username@test.com" + }, + "16": + { + "type": "mac-addr", + "value": "11:22:33:44:55:66" + }, + "17": + { + "type": "external-reference", + "url": "https://security.microsoft.com/machines/deviceid/timeline?from=2023-03-17T20:19:45.000Z&to=2023-03-17T20:19:47.000Z" + }, + "18": + { + "type": "x-ibm-finding", + "alert_id": "1234567890-1234-1234-1234-123456789012_1", + "severity": 99, + "name": "Connection to adversary-in-the-middle (AiTM) phishing site", + "ttp_tagging_refs": ["19","21"] + }, + "19": + { + "type": "x-ibm-ttp-tagging", + "kill_chain_phases": [ + { + "phase_name": "Credential Access", + "kill_chain_name": "mitre-attack" + }] + }, + "20": + { + "type": "artifact", + "payload_bin": "base64 encoded json from the msatp api before translation" + }, + "21": + { + "type": "x-ibm-ttp-tagging", + "extensions": + { + "mitre-attack-ext": + { + "technique_name": "Spearphishing Link", + "technique_id": "T1566.002" + } + } + } +} +``` + +### March 2023 changes +Changes made from previous version: +- `DeviceEvents` and `DeviceImageLoadEvents` tables where added to the search query. +Prior they were not being searched. +- `DeviceAlertEvents` is being queried in correlation with the events. +- `DeviceNetworkInfo` is being queried to extract the device network addresses at the time of the event +- `DeviceInfo` is queried to extract the device OS details +- `domain-name` was mapped to `DeviceName` causing a mismatch between looking for events on a specific host and looking for domains in network events. The mapping to device name was removed. use `x-oca-asset:hostname` instead to search for a specific device. +- `process` fields such as `name` and `pid` were looking at both the process and its parent. This causes an overload of results. Say I searched for all events by process pid 123 - I would receive also all events of its child processes. This was removed. now searching a process property will not look at the parent process fields. To search for events where the parent process is X search for `process:parent_ref.X` +- `user-account:account_login` was mapped to `AccountName` which is the local account name - which is now mapped to `user-account:user_id`. The `AccountUpn` which is the more meaningful field since it is resolved from active directory and usually contains for users full email is now mapped to `account_login`. The `AccountSid` (in the form of S-1-5- etc.) which was mapped to `user_id` is no longer mapped to any stix field. +- `x-msatp` `computer_name` and `machine_id` are removed as they map to `x-oca-asset` now. +- `x-oca-asset:ip` was switched to `x-oca-asset:ip_refs` array as per the official spec and now support multiple ip addresses for one host. +- `DeviceFileEvents.FileOriginUrl` and `DeviceFileEvents.FileOriginReferrerUrl` were added to the mappings - they were not mapped before. +- `x-oca-event:provider` will always state 'Microsoft Defender for Endpoint' to indicate the source of the event +- `x-oca-event:external_ref` was added and provides a link to the msatp web console filtered one second before and after the event. This is handy in case the analyst would like to look at the source data directly in the msatp console. +- event name was mapped to `x-oca-event:action` +- `x-oca-event:original_ref` was added and includes the raw json output of the msatp api encoded in base64. This is handy to find details that are not mapped to stix. +- `x-msatp:AdditionalFields` was added as this field usually contains important information. diff --git a/stix_shifter_modules/msatp/stix_translation/json/from_stix_map.json b/stix_shifter_modules/msatp/stix_translation/json/from_stix_map.json index 39e5bf500..276cad7af 100644 --- a/stix_shifter_modules/msatp/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/msatp/stix_translation/json/from_stix_map.json @@ -3,7 +3,9 @@ "fields": { "value": [ "DeviceNetworkEvents.LocalIP", - "DeviceNetworkEvents.RemoteIP" + "DeviceNetworkEvents.RemoteIP", + "DeviceEvents.RemoteIP", + "DeviceEvents.LocalIP" ] } }, @@ -11,50 +13,51 @@ "fields": { "value": [ "DeviceNetworkEvents.LocalIP", - "DeviceNetworkEvents.RemoteIP" + "DeviceNetworkEvents.RemoteIP", + "DeviceEvents.RemoteIP", + "DeviceEvents.LocalIP" ] } }, "network-traffic": { "fields": { "src_port": [ - "DeviceNetworkEvents.LocalPort" + "DeviceNetworkEvents.LocalPort", + "DeviceEvents.LocalPort" ], "dst_port": [ - "DeviceNetworkEvents.RemotePort" + "DeviceNetworkEvents.RemotePort", + "DeviceEvents.RemotePort" ], "protocols[*]": [ "DeviceNetworkEvents.Protocol" ], "src_ref.value": [ "DeviceNetworkEvents.LocalIP", - "DeviceNetworkInfo.MacAddress" + "DeviceNetworkInfo.MacAddress", + "DeviceEvents.LocalIP" ], "dst_ref.value": [ - "DeviceNetworkEvents.RemoteIP" + "DeviceNetworkEvents.RemoteIP", + "DeviceEvents.RemoteIP" ] } }, "url": { "fields": { "value": [ - "DeviceNetworkEvents.RemoteUrl" + "DeviceNetworkEvents.RemoteUrl", + "DeviceEvents.RemoteUrl", + "DeviceFileEvents.FileOriginUrl", + "DeviceFileEvents.FileOriginReferrerUrl" ] } }, "domain-name": { "fields": { "value": [ - "DeviceFileEvents.DeviceName", - "DeviceProcessEvents.DeviceName", - "DeviceNetworkEvents.DeviceName", - "DeviceRegistryEvents.DeviceName", - "DeviceEvents.DeviceName", - "DeviceInfo.DeviceName", - "DeviceAlertEvents.DeviceName", - "DeviceImageLoadEvents.DeviceName", - "DeviceLogonEvents.DeviceName", - "DeviceNetworkInfo.DeviceName" + "DeviceNetworkEvents.RemoteUrl", + "DeviceEvents.RemoteUrl" ] } }, @@ -70,7 +73,13 @@ "DeviceNetworkEvents.InitiatingProcessFileName", "DeviceNetworkEvents.InitiatingProcessParentFileName", "DeviceRegistryEvents.InitiatingProcessFileName", - "DeviceRegistryEvents.InitiatingProcessParentFileName" + "DeviceRegistryEvents.InitiatingProcessParentFileName", + "DeviceEvents.FileName", + "DeviceEvents.InitiatingProcessFileName", + "DeviceEvents.InitiatingProcessParentFileName", + "DeviceImageLoadEvents.FileName", + "DeviceImageLoadEvents.InitiatingProcessFileName", + "DeviceImageLoadEvents.InitiatingProcessParentFileName" ], "hashes.'SHA-1'": [ "DeviceFileEvents.SHA1", @@ -78,7 +87,11 @@ "DeviceProcessEvents.SHA1", "DeviceProcessEvents.InitiatingProcessSHA1", "DeviceNetworkEvents.InitiatingProcessSHA1", - "DeviceRegistryEvents.InitiatingProcessSHA1" + "DeviceRegistryEvents.InitiatingProcessSHA1", + "DeviceEvents.SHA1", + "DeviceEvents.InitiatingProcessSHA1", + "DeviceImageLoadEvents.SHA1", + "DeviceImageLoadEvents.InitiatingProcessSHA1" ], "hashes.'SHA-256'": [ "DeviceFileEvents.SHA256", @@ -86,7 +99,11 @@ "DeviceProcessEvents.SHA256", "DeviceProcessEvents.InitiatingProcessSHA256", "DeviceNetworkEvents.InitiatingProcessSHA256", - "DeviceRegistryEvents.InitiatingProcessSHA256" + "DeviceRegistryEvents.InitiatingProcessSHA256", + "DeviceEvents.SHA256", + "DeviceEvents.InitiatingProcessSHA256", + "DeviceImageLoadEvents.SHA256", + "DeviceImageLoadEvents.InitiatingProcessSHA256" ], "hashes.MD5": [ "DeviceFileEvents.MD5", @@ -94,7 +111,11 @@ "DeviceProcessEvents.MD5", "DeviceProcessEvents.InitiatingProcessMD5", "DeviceNetworkEvents.InitiatingProcessMD5", - "DeviceRegistryEvents.InitiatingProcessMD5" + "DeviceRegistryEvents.InitiatingProcessMD5", + "DeviceEvents.MD5", + "DeviceEvents.InitiatingProcessMD5", + "DeviceImageLoadEvents.MD5", + "DeviceImageLoadEvents.InitiatingProcessMD5" ], "parent_directory_ref.path": [ "DeviceFileEvents.FolderPath", @@ -102,7 +123,11 @@ "DeviceProcessEvents.FolderPath", "DeviceProcessEvents.InitiatingProcessFolderPath", "DeviceNetworkEvents.InitiatingProcessFolderPath", - "DeviceRegistryEvents.InitiatingProcessFolderPath" + "DeviceRegistryEvents.InitiatingProcessFolderPath", + "DeviceEvents.FolderPath", + "DeviceEvents.InitiatingProcessFolderPath", + "DeviceImageLoadEvents.FolderPath", + "DeviceImageLoadEvents.InitiatingProcessFolderPath" ] } }, @@ -110,78 +135,150 @@ "fields": { "name": [ "DeviceProcessEvents.FileName", + "DeviceEvents.FileName", "DeviceProcessEvents.InitiatingProcessFileName", - "DeviceProcessEvents.InitiatingProcessParentFileName", - "DeviceNetworkEvents.InitiatingProcessFileName", - "DeviceNetworkEvents.InitiatingProcessParentFileName", - "DeviceRegistryEvents.InitiatingProcessFileName", - "DeviceRegistryEvents.InitiatingProcessParentFileName" - ], - "parent_ref.name": [ - "DeviceProcessEvents.InitiatingProcessFileName", - "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceEvents.InitiatingProcessFileName", + "DeviceFileEvents.InitiatingProcessFileName", "DeviceNetworkEvents.InitiatingProcessFileName", "DeviceRegistryEvents.InitiatingProcessFileName", - "DeviceProcessEvents.InitiatingProcessParentFileName", - "DeviceProcessEvents.InitiatingProcessParentFileName", - "DeviceNetworkEvents.InitiatingProcessParentFileName", - "DeviceRegistryEvents.InitiatingProcessParentFileName" + "DeviceImageLoadEvents.InitiatingProcessFileName" ], "command_line": [ "DeviceProcessEvents.ProcessCommandLine", "DeviceProcessEvents.InitiatingProcessCommandLine", + "DeviceEvents.ProcessCommandLine", + "DeviceEvents.InitiatingProcessCommandLine", + "DeviceFileEvents.InitiatingProcessCommandLine", "DeviceNetworkEvents.InitiatingProcessCommandLine", - "DeviceRegistryEvents.InitiatingProcessCommandLine" + "DeviceRegistryEvents.InitiatingProcessCommandLine", + "DeviceImageLoadEvents.InitiatingProcessCommandLine" ], "pid": [ "DeviceProcessEvents.ProcessId", + "DeviceEvents.ProcessId", + "DeviceProcessEvents.InitiatingProcessId", + "DeviceEvents.InitiatingProcessId", "DeviceProcessEvents.InitiatingProcessId", - "DeviceProcessEvents.InitiatingProcessParentId", "DeviceNetworkEvents.InitiatingProcessId", - "DeviceNetworkEvents.InitiatingProcessParentId", "DeviceRegistryEvents.InitiatingProcessId", - "DeviceRegistryEvents.InitiatingProcessParentId", "DeviceFileEvents.InitiatingProcessId", - "DeviceFileEvents.InitiatingProcessParentId" + "DeviceImageLoadEvents.InitiatingProcessId" + ], + "created": [ + "DeviceProcessEvents.ProcessCreationTime", + "DeviceEvents.ProcessCreationTime", + "DeviceNetworkEvents.InitiatingProcessCreationTime", + "DeviceRegistryEvents.InitiatingProcessCreationTime", + "DeviceFileEvents.InitiatingProcessCreationTime", + "DeviceImageLoadEvents.InitiatingProcessCreationTime" + ], + "parent_ref.name": [ + "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceEvents.InitiatingProcessFileName", + "DeviceFileEvents.InitiatingProcessParentFileName", + "DeviceNetworkEvents.InitiatingProcessParentFileName", + "DeviceRegistryEvents.InitiatingProcessParentFileName", + "DeviceImageLoadEvents.InitiatingProcessParentFileName" ], "parent_ref.pid": [ "DeviceProcessEvents.InitiatingProcessId", "DeviceNetworkEvents.InitiatingProcessId", - "DeviceRegistryEvents.InitiatingProcessId", - "DeviceFileEvents.InitiatingProcessId", - "DeviceProcessEvents.InitiatingProcessParentId", - "DeviceNetworkEvents.InitiatingProcessParentId", "DeviceRegistryEvents.InitiatingProcessParentId", - "DeviceFileEvents.InitiatingProcessParentId" + "DeviceFileEvents.InitiatingProcessParentId", + "DeviceEvents.InitiatingProcessParentId", + "DeviceImageLoadEvents.InitiatingProcessParentId" ], - "child_refs.pid": [ - "DeviceProcessEvents.ProcessId", - "DeviceProcessEvents.InitiatingProcessId", - "DeviceNetworkEvents.InitiatingProcessId", - "DeviceRegistryEvents.InitiatingProcessId", - "DeviceFileEvents.InitiatingProcessId" + "parent_ref.created": [ + "DeviceProcessEvents.InitiatingProcessCreationTime", + "DeviceEvents.InitiatingProcessCreationTime", + "DeviceNetworkEvents.InitiatingProcessParentCreationTime", + "DeviceRegistryEvents.InitiatingProcessParentCreationTime", + "DeviceFileEvents.InitiatingProcessParentCreationTime", + "DeviceImageLoadEvents.InitiatingProcessParentCreationTime" + ], + "parent_ref.parent_ref.name": [ + "DeviceProcessEvents.InitiatingProcessParentFileName", + "DeviceEvents.InitiatingProcessParentFileName" + ], + "parent_ref.parent_ref.pid": [ + "DeviceProcessEvents.InitiatingProcessParentId", + "DeviceNetworkEvents.InitiatingProcessParentId" + ], + "parent_ref.parent_ref.created": [ + "DeviceProcessEvents.InitiatingProcessParentCreationTime", + "DeviceEvents.InitiatingProcessParentCreationTime" ], "creator_user_ref.user_id": [ - "DeviceProcessEvents.AccountSid", - "DeviceProcessEvents.InitiatingProcessAccountSid" + "DeviceProcessEvents.AccountName", + "DeviceEvents.AccountName", + "DeviceNetworkEvents.InitiatingProcessAccountName", + "DeviceRegistryEvents.InitiatingProcessAccountName", + "DeviceFileEvents.InitiatingProcessAccountName", + "DeviceImageLoadEvents.InitiatingProcessAccountName" + ], + "creator_user_ref.account_login": [ + "DeviceProcessEvents.AccountUpn", + "DeviceEvents.AccountUpn", + "DeviceNetworkEvents.InitiatingProcessAccountUpn", + "DeviceRegistryEvents.InitiatingProcessAccountUpn", + "DeviceFileEvents.InitiatingProcessAccountUpn", + "DeviceImageLoadEvents.InitiatingProcessAccountUpn" + ], + "parent_ref.creator_user_ref.user_id": [ + "DeviceProcessEvents.InitiatingProcessAccountName", + "DeviceEvents.InitiatingProcessAccountName" + ], + "parent_ref.creator_user_ref.account_login": [ + "DeviceProcessEvents.InitiatingProcessAccountUpn", + "DeviceEvents.InitiatingProcessAccountUpn" ], - "child_refs.creator_user_ref.account_login": ["DeviceProcessEvents.AccountName"], - "creator_user_ref.account_login": ["DeviceProcessEvents.InitiatingProcessAccountName"], "binary_ref.hashes.'SHA-1'": [ - "DeviceFileEvents.SHA1", - "DeviceFileEvents.InitiatingProcessSHA1", "DeviceProcessEvents.SHA1", - "DeviceProcessEvents.InitiatingProcessSHA1", + "DeviceEvents.SHA1", + "DeviceFileEvents.InitiatingProcessSHA1", "DeviceNetworkEvents.InitiatingProcessSHA1", - "DeviceRegistryEvents.InitiatingProcessSHA1" + "DeviceRegistryEvents.InitiatingProcessSHA1", + "DeviceImageLoadEvents.InitiatingProcessSHA1" ], "binary_ref.hashes.'SHA-256'": [ - "DeviceFileEvents.SHA256", - "DeviceFileEvents.InitiatingProcessSHA256", "DeviceProcessEvents.SHA256", - "DeviceProcessEvents.InitiatingProcessSHA256", + "DeviceEvents.SHA256", + "DeviceFileEvents.InitiatingProcessSHA256", "DeviceNetworkEvents.InitiatingProcessSHA256", - "DeviceRegistryEvents.InitiatingProcessSHA256" + "DeviceRegistryEvents.InitiatingProcessSHA256", + "DeviceImageLoadEvents.InitiatingProcessSHA256" + ], + "binary_ref.hashes.MD5": [ + "DeviceProcessEvents.MD5", + "DeviceEvents.MD5", + "DeviceFileEvents.InitiatingProcessMD5", + "DeviceNetworkEvents.InitiatingProcessMD5", + "DeviceRegistryEvents.InitiatingProcessMD5", + "DeviceImageLoadEvents.InitiatingProcessMD5" + ], + "binary_ref.parent_directory_ref.path": [ + "DeviceProcessEvents.FolderPath", + "DeviceEvents.FolderPath", + "DeviceNetworkEvents.InitiatingProcessFolderPath", + "DeviceRegistryEvents.InitiatingProcessFolderPath", + "DeviceFileEvents.InitiatingProcessFolderPath", + "DeviceImageLoadEvents.InitiatingProcessFolderPath" + ], + "parent_ref.binary_ref.hashes.'SHA-1'": [ + "DeviceProcessEvents.InitiatingProcessSHA1", + "DeviceEvents.InitiatingProcessSHA1" + ], + "parent_ref.binary_ref.hashes.'SHA-256'": [ + "DeviceProcessEvents.InitiatingProcessSHA256", + "DeviceEvents.InitiatingProcessSHA256" + ], + "parent_ref.binary_ref.hashes.MD5": [ + "DeviceProcessEvents.InitiatingProcessMD5", + "DeviceEvents.InitiatingProcessMD5" + ], + "parent_ref.binary_ref.parent_directory_ref.path": [ + "DeviceProcessEvents.InitiatingProcessFolderPath", + "DeviceEvents.InitiatingProcessFolderPath" ], "child_refs.binary_ref.hashes.MD5": [ "DeviceProcessEvents.MD5" @@ -192,62 +289,51 @@ "child_refs.binary_ref.hashes.'SHA-1'": [ "DeviceProcessEvents.SHA1" ], - "binary_ref.hashes.MD5": [ - "DeviceFileEvents.MD5", - "DeviceFileEvents.InitiatingProcessMD5", - "DeviceProcessEvents.MD5", - "DeviceProcessEvents.InitiatingProcessMD5", - "DeviceNetworkEvents.InitiatingProcessMD5", - "DeviceRegistryEvents.InitiatingProcessMD5" - ], - "created": [ - "DeviceProcessEvents.ProcessCreationTime", - "DeviceProcessEvents.InitiatingProcessParentCreationTime", - "DeviceNetworkEvents.InitiatingProcessCreationTime", - "DeviceNetworkEvents.InitiatingProcessParentCreationTime", - "DeviceRegistryEvents.InitiatingProcessCreationTime", - "DeviceRegistryEvents.InitiatingProcessParentCreationTime", - "DeviceFileEvents.InitiatingProcessCreationTime", - "DeviceFileEvents.InitiatingProcessParentCreationTime" - ], - "parent_ref.created": [ - "DeviceProcessEvents.InitiatingProcessParentCreationTime", - "DeviceNetworkEvents.InitiatingProcessParentCreationTime", - "DeviceRegistryEvents.InitiatingProcessParentCreationTime", - "DeviceFileEvents.InitiatingProcessParentCreationTime" - ], - "binary_ref.parent_directory_ref.path": [ - "DeviceProcessEvents.InitiatingProcessFolderPath", - "DeviceNetworkEvents.InitiatingProcessFolderPath", - "DeviceRegistryEvents.InitiatingProcessFolderPath", - "DeviceFileEvents.InitiatingProcessFolderPath" - ], "child_refs.binary_ref.parent_directory_ref.path": [ "DeviceProcessEvents.FolderPath" + ], + "child_refs.creator_user_ref.account_login": [ + "DeviceProcessEvents.AccountName" + ], + "child_refs.pid": [ + "DeviceProcessEvents.ProcessId" ] } }, "user-account": { "fields": { "user_id": [ - "DeviceProcessEvents.AccountSid", - "DeviceNetworkEvents.InitiatingProcessAccountSid", - "DeviceRegistryEvents.InitiatingProcessAccountSid" - ], - "account_login": [ "DeviceProcessEvents.AccountName", + "DeviceFileEvents.RequestAccountName", + "DeviceEvents.AccountName", + "DeviceProcessEvents.InitiatingProcessAccountName", "DeviceNetworkEvents.InitiatingProcessAccountName", - "DeviceRegistryEvents.InitiatingProcessAccountName" + "DeviceRegistryEvents.InitiatingProcessAccountName", + "DeviceFileEvents.InitiatingProcessAccountName", + "DeviceEvents.InitiatingProcessAccountName", + "DeviceImageLoadEvents.InitiatingProcessAccountName" + ], + "account_login": [ + "DeviceProcessEvents.AccountUpn", + "DeviceEvents.AccountUpn", + "DeviceProcessEvents.InitiatingProcessAccountUpn", + "DeviceNetworkEvents.InitiatingProcessAccountUpn", + "DeviceRegistryEvents.InitiatingProcessAccountUpn", + "DeviceFileEvents.InitiatingProcessAccountUpn", + "DeviceEvents.InitiatingProcessAccountUpn", + "DeviceImageLoadEvents.InitiatingProcessAccountUpn" ] } }, "windows-registry-key": { "fields": { "key": [ - "DeviceRegistryEvents.RegistryKey" + "DeviceRegistryEvents.RegistryKey", + "DeviceEvents.RegistryKey" ], "values[*]": [ - "DeviceRegistryEvents.RegistryValueName" + "DeviceRegistryEvents.RegistryValueName", + "DeviceEvents.RegistryValueName" ] } }, @@ -258,22 +344,6 @@ ] } }, - "x-msatp": { - "fields": { - "computer_name": [ - "DeviceProcessEvents.DeviceName", - "DeviceNetworkEvents.DeviceName", - "DeviceRegistryEvents.DeviceName", - "DeviceFileEvents.DeviceName" - ], - "machine_id": [ - "DeviceProcessEvents.DeviceId", - "DeviceNetworkEvents.DeviceId", - "DeviceRegistryEvents.DeviceId", - "DeviceFileEvents.DeviceId" - ] - } - }, "directory": { "fields": { "path": [ @@ -281,24 +351,28 @@ "DeviceFileEvents.InitiatingProcessFolderPath", "DeviceProcessEvents.FolderPath", "DeviceProcessEvents.InitiatingProcessFolderPath", + "DeviceEvents.FolderPath", + "DeviceEvents.InitiatingProcessFolderPath", "DeviceNetworkEvents.InitiatingProcessFolderPath", - "DeviceRegistryEvents.InitiatingProcessFolderPath" + "DeviceRegistryEvents.InitiatingProcessFolderPath", + "DeviceImageLoadEvents.FolderPath", + "DeviceImageLoadEvents.InitiatingProcessFolderPath" ] } }, "x-oca-asset": { "fields": { - "domain": [ - "DeviceFileEvents.DeviceName", - "DeviceProcessEvents.DeviceName", - "DeviceNetworkEvents.DeviceName", - "DeviceRegistryEvents.DeviceName", - "DeviceEvents.DeviceName", - "DeviceInfo.DeviceName", - "DeviceAlertEvents.DeviceName", - "DeviceImageLoadEvents.DeviceName", - "DeviceLogonEvents.DeviceName", - "DeviceNetworkInfo.DeviceName" + "device_id": [ + "DeviceFileEvents.DeviceId", + "DeviceProcessEvents.DeviceId", + "DeviceNetworkEvents.DeviceId", + "DeviceRegistryEvents.DeviceId", + "DeviceEvents.DeviceId", + "DeviceInfo.DeviceId", + "DeviceAlertEvents.DeviceId", + "DeviceImageLoadEvents.DeviceId", + "DeviceLogonEvents.DeviceId", + "DeviceNetworkInfo.DeviceId" ], "hostname": [ "DeviceFileEvents.DeviceName", @@ -312,19 +386,9 @@ "DeviceLogonEvents.DeviceName", "DeviceNetworkInfo.DeviceName" ], - "name": [ - "DeviceFileEvents.DeviceName", - "DeviceProcessEvents.DeviceName", - "DeviceNetworkEvents.DeviceName", - "DeviceRegistryEvents.DeviceName", - "DeviceEvents.DeviceName", - "DeviceInfo.DeviceName", - "DeviceAlertEvents.DeviceName", - "DeviceImageLoadEvents.DeviceName", - "DeviceLogonEvents.DeviceName", - "DeviceNetworkInfo.DeviceName" + "ip_refs[*].value": [ + "DeviceNetworkEvents.LocalIP" ], - "ip": ["DeviceNetworkEvents.LocalIP"], "os.name": [ "DeviceInfo.OSPlatform" ], @@ -335,100 +399,136 @@ }, "x-oca-event": { "fields": { + "action": [ + "DeviceProcessEvents.ActionType", + "DeviceEvents.ActionType", + "DeviceNetworkEvents.ActionType", + "DeviceRegistryEvents.ActionType", + "DeviceFileEvents.ActionType", + "DeviceImageLoadEvents.ActionType" + ], "process_ref.pid": [ - "DeviceProcessEvents.InitiatingProcessId", + "DeviceProcessEvents.ProcessId", + "DeviceEvents.ProcessId", "DeviceNetworkEvents.InitiatingProcessId", "DeviceRegistryEvents.InitiatingProcessId", - "DeviceFileEvents.InitiatingProcessId" - ], - "process_ref.child_refs.pid": [ - "DeviceProcessEvents.ProcessId" + "DeviceFileEvents.InitiatingProcessId", + "DeviceImageLoadEvents.InitiatingProcessId" ], "process_ref.name": [ - "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceProcessEvents.FileName", + "DeviceEvents.FileName", "DeviceNetworkEvents.InitiatingProcessFileName", "DeviceRegistryEvents.InitiatingProcessFileName", - "DeviceFileEvents.InitiatingProcessFileName" - ], - "process_ref.child_refs.name": [ - "DeviceProcessEvents.FileName" + "DeviceFileEvents.InitiatingProcessFileName", + "DeviceImageLoadEvents.InitiatingProcessFileName" ], "process_ref.binary_ref.name": [ - "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceProcessEvents.FileName", + "DeviceEvents.FileName", "DeviceNetworkEvents.InitiatingProcessFileName", "DeviceRegistryEvents.InitiatingProcessFileName", - "DeviceFileEvents.InitiatingProcessFileName" + "DeviceFileEvents.InitiatingProcessFileName", + "DeviceImageLoadEvents.InitiatingProcessFileName" ], "process_ref.creator_user_ref.account_login": [ - "DeviceProcessEvents.AccountName" + "DeviceProcessEvents.AccountUpn", + "DeviceEvents.AccountUpn", + "DeviceNetworkEvents.InitiatingProcessAccountUpn", + "DeviceRegistryEvents.InitiatingProcessAccountUpn", + "DeviceFileEvents.InitiatingProcessAccountUpn", + "DeviceImageLoadEvents.InitiatingProcessAccountUpn" + ], + "process_ref.creator_user_ref.user_id": [ + "DeviceProcessEvents.AccountName", + "DeviceEvents.AccountName", + "DeviceNetworkEvents.InitiatingProcessAccountName", + "DeviceRegistryEvents.InitiatingProcessAccountName", + "DeviceFileEvents.InitiatingProcessAccountName", + "DeviceImageLoadEvents.InitiatingProcessAccountName" + ], + "process_ref.command_line": [ + "DeviceProcessEvents.ProcessCommandLine", + "DeviceEvents.ProcessCommandLine", + "DeviceNetworkEvents.InitiatingProcessCommandLine", + "DeviceRegistryEvents.InitiatingProcessCommandLine", + "DeviceFileEvents.InitiatingProcessCommandLine", + "DeviceImageLoadEvents.InitiatingProcessCommandLine" ], "process_ref.parent_ref.name": [ - "DeviceProcessEvents.InitiatingProcessParentFileName", + "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceEvents.InitiatingProcessFileName", "DeviceNetworkEvents.InitiatingProcessParentFileName", "DeviceRegistryEvents.InitiatingProcessParentFileName", - "DeviceFileEvents.InitiatingProcessParentFileName" + "DeviceFileEvents.InitiatingProcessParentFileName", + "DeviceImageLoadEvents.InitiatingProcessParentFileName" ], "process_ref.parent_ref.pid": [ - "DeviceProcessEvents.InitiatingProcessParentId", + "DeviceProcessEvents.InitiatingProcessId", + "DeviceEvents.InitiatingProcessId", "DeviceNetworkEvents.InitiatingProcessParentId", "DeviceRegistryEvents.InitiatingProcessParentId", - "DeviceFileEvents.InitiatingProcessParentId" + "DeviceFileEvents.InitiatingProcessParentId", + "DeviceImageLoadEvents.InitiatingProcessParentId" ], - "process_ref.command_line": [ + "process_ref.parent_ref.command_line": [ "DeviceProcessEvents.InitiatingProcessCommandLine", - "DeviceNetworkEvents.InitiatingProcessCommandLine", - "DeviceRegistryEvents.InitiatingProcessCommandLine", - "DeviceFileEvents.InitiatingProcessCommandLine" + "DeviceEvents.InitiatingProcessCommandLine" ], "process_ref.binary_ref.hashes.'SHA-256'": [ - "DeviceProcessEvents.InitiatingProcessSHA256", + "DeviceProcessEvents.SHA256", + "DeviceEvents.SHA256", "DeviceNetworkEvents.InitiatingProcessSHA256", "DeviceRegistryEvents.InitiatingProcessSHA256", - "DeviceFileEvents.InitiatingProcessSHA256" + "DeviceFileEvents.InitiatingProcessSHA256", + "DeviceImageLoadEvents.InitiatingProcessSHA256" ], - "process_ref.process_ref.creator_user_ref.account_login": [ - "DeviceProcessEvents.AccountName" + "process_ref.binary_ref.hashes.MD5": [ + "DeviceProcessEvents.MD5", + "DeviceEvents.MD5", + "DeviceNetworkEvents.InitiatingProcessMD5", + "DeviceRegistryEvents.InitiatingProcessMD5", + "DeviceFileEvents.InitiatingProcessMD5", + "DeviceImageLoadEvents.InitiatingProcessMD5" + ], + "process_ref.binary_ref.hashes.'SHA-1'": [ + "DeviceProcessEvents.SHA1", + "DeviceEvents.SHA1", + "DeviceNetworkEvents.InitiatingProcessSHA1", + "DeviceRegistryEvents.InitiatingProcessSHA1", + "DeviceFileEvents.InitiatingProcessSHA1", + "DeviceImageLoadEvents.InitiatingProcessSHA1" ], "parent_process_ref.name": [ - "DeviceProcessEvents.InitiatingProcessParentFileName", + "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceEvents.InitiatingProcessFileName", "DeviceNetworkEvents.InitiatingProcessParentFileName", "DeviceRegistryEvents.InitiatingProcessParentFileName", - "DeviceFileEvents.InitiatingProcessParentFileName" + "DeviceFileEvents.InitiatingProcessParentFileName", + "DeviceImageLoadEvents.InitiatingProcessParentFileName" ], "parent_process_ref.pid": [ - "DeviceProcessEvents.InitiatingProcessParentId", + "DeviceProcessEvents.InitiatingProcessId", + "DeviceEvents.InitiatingProcessId", "DeviceNetworkEvents.InitiatingProcessParentId", "DeviceRegistryEvents.InitiatingProcessParentId", - "DeviceFileEvents.InitiatingProcessParentId" - ], - "process_ref.binary_ref.hashes.MD5": [ - "DeviceProcessEvents.InitiatingProcessMD5", - "DeviceNetworkEvents.InitiatingProcessMD5", - "DeviceRegistryEvents.InitiatingProcessMD5", - "DeviceFileEvents.InitiatingProcessMD5" - ], - "process_ref.binary_ref.hashes.'SHA-256'": [ - "DeviceProcessEvents.InitiatingProcessSHA256", - "DeviceNetworkEvents.InitiatingProcessSHA256", - "DeviceRegistryEvents.InitiatingProcessSHA256", - "DeviceFileEvents.InitiatingProcessSHA256" - ], - "process_ref.creator_user_ref.user_id": [ - "DeviceProcessEvents.InitiatingProcessAccountSid", - "DeviceNetworkEvents.InitiatingProcessAccountSid", - "DeviceRegistryEvents.InitiatingProcessAccountSid", - "DeviceFileEvents.InitiatingProcessAccountSid" + "DeviceFileEvents.InitiatingProcessParentId", + "DeviceImageLoadEvents.InitiatingProcessParentId" ], "domain_ref.value": [ - "DeviceProcessEvents.DeviceName", - "DeviceNetworkEvents.DeviceName", - "DeviceRegistryEvents.DeviceName", - "DeviceFileEvents.DeviceName" + "DeviceNetworkEvents.RemoteUrl", + "DeviceEvents.RemoteUrl" + ], + "url_ref.value": [ + "DeviceNetworkEvents.RemoteUrl", + "DeviceEvents.RemoteUrl", + "DeviceEvents.FileOriginUrl", + "DeviceFileEvents.FileOriginUrl", + "DeviceFileEvents.FileOriginReferrerUrl" ], "file_ref.name": [ - "DeviceProcessEvents.InitiatingProcessFileName", - "DeviceNetworkEvents.InitiatingProcessFileName", - "DeviceRegistryEvents.InitiatingProcessFileName" + "DeviceFileEvents.FileName", + "DeviceImageLoadEvents.FileName" ], "registry_ref.key": [ "DeviceRegistryEvents.RegistryKey" @@ -445,17 +545,30 @@ "DeviceLogonEvents.DeviceName", "DeviceNetworkInfo.DeviceName" ], - "host_ref.name": [ - "DeviceFileEvents.DeviceName", - "DeviceProcessEvents.DeviceName", - "DeviceNetworkEvents.DeviceName", - "DeviceRegistryEvents.DeviceName", - "DeviceEvents.DeviceName", - "DeviceInfo.DeviceName", - "DeviceAlertEvents.DeviceName", - "DeviceImageLoadEvents.DeviceName", - "DeviceLogonEvents.DeviceName", - "DeviceNetworkInfo.DeviceName" + "host_ref.device_id": [ + "DeviceFileEvents.DeviceId", + "DeviceProcessEvents.DeviceId", + "DeviceNetworkEvents.DeviceId", + "DeviceRegistryEvents.DeviceId", + "DeviceEvents.DeviceId", + "DeviceInfo.DeviceId", + "DeviceAlertEvents.DeviceId", + "DeviceImageLoadEvents.DeviceId", + "DeviceLogonEvents.DeviceId", + "DeviceNetworkInfo.DeviceId" + ] + } + }, + "x-ibm-finding": { + "fields": { + "alert_id": [ + "DeviceAlertEvents.AlertId" + ], + "name": [ + "DeviceAlertEvents.Title" + ], + "time_observed": [ + "DeviceAlertEvents.Timestamp" ] } } diff --git a/stix_shifter_modules/msatp/stix_translation/json/to_stix_map.json b/stix_shifter_modules/msatp/stix_translation/json/to_stix_map.json index 4ed99a0f0..0f44ea99d 100644 --- a/stix_shifter_modules/msatp/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/msatp/stix_translation/json/to_stix_map.json @@ -1,6 +1,269 @@ { + "DeviceAlertEvents": { + "AlertId": [ + { + "key": "x-ibm-finding.alert_id", + "object": "finding" + }, + { + "key": "x-oca-event.finding_refs", + "object": "event", + "references": [ + "finding" + ], + "group": true + } + ], + "Severity": { + "key": "x-ibm-finding.severity", + "object": "finding", + "transformer": "SeverityToNumericVal" + }, + "AttackTechniques": [ + { + "key": "x-ibm-ttp-tagging.extensions.mitre-attack-ext.technique_name", + "object": "ttp_tagging", + "unwrap": true + }, + { + "key": "x-ibm-finding.ttp_tagging_refs", + "object": "finding", + "references": [ + "ttp_tagging" + ], + "unwrap": true, + "group": true + } + ], + "Category": [ + { + "key": "x-ibm-ttp-tagging.kill_chain_phases.phase_name", + "object": "ttp_tagging_phase" + }, + { + "key": "x-ibm-finding.ttp_tagging_refs", + "object": "finding", + "references": [ + "ttp_tagging_phase" + ], + "unwrap": true, + "group": true + } + ], + "Title": [ + { + "key": "x-ibm-finding.name", + "object": "finding" + }, + { + "key": "x-oca-event.action", + "object": "event" + }, + { + "key": "x-oca-event.category", + "object": "event", + "transformer": "Alert" + }, + { + "key": "x-ibm-finding.finding_type", + "object": "finding", + "transformer": "Alert" + } + ], + "RemoteUrl": [ + { + "key": "url.value", + "object": "url", + "transformer": "IfValidUrl" + }, + { + "key": "domain-name.value", + "object": "domain-name", + "transformer": "GetDomainName" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain-name" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" + }, + { + "key": "x-ibm-finding.ioc_refs", + "object": "finding", + "references": [ + "url", + "domain-name" + ], + "group": true + } + ], + "RemoteIP": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + }, + { + "key": "x-oca-event.ip_refs", + "object": "event", + "references": [ + "dst_ip" + ], + "group": true + }, + { + "key": "x-ibm-finding.ioc_refs", + "object": "finding", + "references": [ + "dst_ip" + ], + "group": true + } + ], + "Timestamp": [ + { + "key": "x-oca-event.created", + "object": "event" + }, + { + "key": "x-ibm-finding.time_observed", + "object": "finding" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + }, + { + "key": "last_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + } + ], + "FileName": [ + { + "key": "file.name", + "object": "file", + "transformer": "ToFileName" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "file" + }, + { + "key": "directory.path", + "object": "file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "file_dir" + } + ], + "SHA1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "SHA256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "MD5": { + "key": "file.hashes.MD5", + "object": "file" + }, + "DeviceName": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "DeviceId": [ + { + "key": "x-oca-asset.device_id", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "original_ref": [ + { + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + } + ], + "event_link": [ + { + "key": "external-reference.url", + "object": "device_external_ref" + }, + { + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + } + }, "DeviceNetworkEvents": { + "Alerts": { + "key": "x-json-alert.data" + }, "Timestamp": [ + { + "key": "x-oca-event.created", + "object": "event" + }, { "key": "first_observed", "cybox": false, @@ -25,6 +288,19 @@ "key": "network-traffic.src_ref", "object": "nt", "references": "src_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "src_ip" + ], + "group": true + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" } ], "RemoteIP": [ @@ -40,6 +316,33 @@ "key": "network-traffic.dst_ref", "object": "nt", "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "RemoteUrl": [ + { + "key": "url.value", + "object": "url", + "transformer": "IfValidUrl" + }, + { + "key": "domain-name.value", + "object": "domain-name", + "transformer": "GetDomainName" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain-name" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" } ], "LocalPort": [ @@ -47,6 +350,11 @@ "key": "network-traffic.src_port", "object": "nt", "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" } ], "RemotePort": [ @@ -54,6 +362,11 @@ "key": "network-traffic.dst_port", "object": "nt", "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" } ], "Protocol": { @@ -61,9 +374,6 @@ "object": "nt", "transformer": "FormatTCPProtocol" }, - "RemoteUrl": { - "key": "url.value" - }, "InitiatingProcessSHA1": [ { "key": "file.hashes.SHA-1", @@ -73,6 +383,11 @@ "key": "process.binary_ref", "object": "nc_process", "references": "nc_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "nc_process" } ], "InitiatingProcessSHA256": [ @@ -84,6 +399,11 @@ "key": "process.binary_ref", "object": "nc_process", "references": "nc_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "nc_process" } ], "InitiatingProcessMD5": [ @@ -95,31 +415,55 @@ "key": "process.binary_ref", "object": "nc_process", "references": "nc_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "nc_process" } ], "InitiatingProcessFileName": [ { "key": "file.name", - "object": "nc_process_file" + "object": "nc_process_file", + "transformer": "ToFileName" }, { "key": "process.name", - "object": "nc_process" + "object": "nc_process", + "transformer": "ToFileName" }, { "key": "process.binary_ref", "object": "nc_process", "references": "nc_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "nc_process" + }, + { + "key": "directory.path", + "object": "nc_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "nc_process_file", + "references": "nc_process_file_dir" } ], "InitiatingProcessParentFileName": [ { "key": "file.name", - "object": "nc_parent_process_file" + "object": "nc_parent_process_file", + "transformer": "ToFileName" }, { "key": "process.name", - "object": "nc_parent_process" + "object": "nc_parent_process", + "transformer": "ToFileName" }, { "key": "process.parent_ref", @@ -130,13 +474,30 @@ "key": "process.binary_ref", "object": "nc_parent_process", "references": "nc_parent_process_file" + }, + { + "key": "directory.path", + "object": "nc_parent_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "nc_parent_process_file", + "references": "nc_parent_process_file_dir" + } + ], + "InitiatingProcessId": [ + { + "key": "process.pid", + "object": "nc_process", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "nc_process" } ], - "InitiatingProcessId": { - "key": "process.pid", - "object": "nc_process", - "transformer": "ToInteger" - }, "InitiatingProcessParentId": [ { "key": "process.pid", @@ -149,10 +510,17 @@ "references": "nc_parent_process" } ], - "InitiatingProcessCommandLine": { - "key": "process.command_line", - "object": "nc_process" - }, + "InitiatingProcessCommandLine": [ + { + "key": "process.command_line", + "object": "nc_process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "nc_process" + } + ], "InitiatingProcessCreationTime": { "key": "process.created", "object": "nc_process", @@ -170,7 +538,7 @@ "references": "nc_parent_process" } ], - "InitiatingProcessAccountSid": [ + "InitiatingProcessAccountName": [ { "key": "user-account.user_id", "object": "nc_process_creator" @@ -181,7 +549,7 @@ "references": "nc_process_creator" } ], - "InitiatingProcessAccountName": [ + "InitiatingProcessAccountUpn": [ { "key": "user-account.account_login", "object": "nc_process_creator" @@ -196,7 +564,7 @@ { "key": "directory.path", "object": "nc_process_file_dir", - "transformer": "ToDirectoryPath" + "transformer": "ToMSATPDirectoryPath" }, { "key": "file.parent_directory_ref", @@ -218,7 +586,9 @@ { "key": "ipv4-addr.resolves_to_refs", "object": "src_ip", - "references": ["mac"] + "references": [ + "mac" + ] }, { "key": "network-traffic.src_ref", @@ -231,19 +601,150 @@ "references": "nt" } ], - "DeviceName": { - "key": "x-msatp.computer_name" + "DeviceName": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "DeviceId": [ + { + "key": "x-oca-asset.device_id", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "PublicIP": [ + { + "key": "ipv4-addr.value", + "object": "public-ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "public-ip" + ], + "group": true + } + ], + "OSPlatform": { + "key": "x-oca-asset.os_name", + "object": "host" + }, + "OSArchitecture": { + "key": "x-oca-asset.architecture", + "object": "host" }, - "DeviceId": { - "key": "x-msatp.machine_id" + "OSVersion": { + "key": "x-oca-asset.os_version", + "object": "host" }, "event_count": { "key": "number_observed", "cybox": false, "transformer": "ToInteger" + }, + "original_ref": [ + { + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + } + ], + "ActionType": { + "key": "x-oca-event.action", + "object": "event" + }, + "event_link": [ + { + "key": "external-reference.url", + "object": "device_external_ref" + }, + { + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" + } + ], + "MacAddressSet": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacList", + "unwrap": true + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac" + ], + "unwrap": true, + "group": true + } + ], + "IPAddresses": [ + { + "key": "ipv4-addr.value", + "object": "host_ip4", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "host_ip6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip4", + "host_ip6" + ], + "unwrap": true, + "group": true + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + }, + "AdditionalFields": { + "key": "x-msatp.AdditionalFields", + "object": "x-msatp", + "transformer": "JsonToString" } }, "DeviceProcessEvents": { + "Alerts": { + "key": "x-json-alert.data" + }, "Timestamp": [ { "key": "x-oca-event.created", @@ -269,16 +770,18 @@ "FileName": [ { "key": "file.name", - "object": "child_file" + "object": "child_process_file", + "transformer": "ToFileName" }, { "key": "process.name", - "object": "child_process" + "object": "child_process", + "transformer": "ToFileName" }, { "key": "process.binary_ref", "object": "child_process", - "references": "child_file" + "references": "child_process_file" }, { "key": "process.child_refs", @@ -290,32 +793,42 @@ { "key": "x-oca-event.process_ref", "object": "event", - "references": "process" + "references": "child_process" + }, + { + "key": "directory.path", + "object": "child_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "child_process_file", + "references": "child_process_file_dir" } ], "FolderPath": [ { "key": "directory.path", - "object": "child_file_dir", - "transformer": "ToDirectoryPath" + "object": "child_process_file_dir", + "transformer": "ToMSATPDirectoryPath" }, { "key": "file.parent_directory_ref", - "object": "child_file", - "references": "child_file_dir" + "object": "child_process_file", + "references": "child_process_file_dir" } ], "SHA1": { "key": "file.hashes.SHA-1", - "object": "child_file" + "object": "child_process_file" }, "SHA256": { "key": "file.hashes.SHA-256", - "object": "child_file" + "object": "child_process_file" }, "MD5": { "key": "file.hashes.MD5", - "object": "child_file" + "object": "child_process_file" }, "ProcessId": [ { @@ -333,7 +846,7 @@ { "key": "x-oca-event.process_ref", "object": "event", - "references": "process" + "references": "child_process" } ], "ProcessCommandLine": [ @@ -349,20 +862,11 @@ ] } ], - "ProcessCreationTime": [ - { - "key": "process.created", - "object": "child_process", - "transformer": "MsatpToTimestamp" - }, - { - "key": "process.child_refs", - "object": "process", - "references": [ - "child_process" - ] - } - ], + "ProcessCreationTime": { + "key": "process.created", + "object": "child_process", + "transformer": "MsatpToTimestamp" + }, "InitiatingProcessSHA1": [ { "key": "file.hashes.SHA-1", @@ -399,53 +903,45 @@ "InitiatingProcessFileName": [ { "key": "file.name", - "object": "process_file" + "object": "process_file", + "transformer": "ToFileName" }, { "key": "process.name", - "object": "process" + "object": "process", + "transformer": "ToFileName" }, { "key": "process.binary_ref", "object": "process", "references": "process_file" - } - ], - "InitiatingProcessParentFileName": [ - { - "key": "file.name", - "object": "parent_process_file" }, { - "key": "process.name", - "object": "parent_process" + "key": "process.parent_ref", + "object": "child_process", + "references": "process" }, { - "key": "process.parent_ref", - "object": "process", - "references": "parent_process" + "key": "directory.path", + "object": "process_file_dir", + "transformer": "ToMSATPDirectoryPath" }, { - "key": "process.binary_ref", - "object": "parent_process", - "references": "parent_process_file" + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "process_file_dir" } ], - "InitiatingProcessId": { - "key": "process.pid", - "object": "process", - "transformer": "ToInteger" - }, - "InitiatingProcessParentId": [ + "InitiatingProcessId": [ { "key": "process.pid", - "object": "parent_process", + "object": "process", "transformer": "ToInteger" }, { "key": "process.parent_ref", - "object": "process", - "references": "parent_process" + "object": "child_process", + "references": "process" } ], "InitiatingProcessCommandLine": { @@ -457,6 +953,60 @@ "object": "process", "transformer": "MsatpToTimestamp" }, + "InitiatingProcessFolderPath": [ + { + "key": "directory.path", + "object": "process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "process_file_dir" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + } + ], + "InitiatingProcessParentId": { + "key": "process.pid", + "object": "parent_process", + "transformer": "ToInteger" + }, + "InitiatingProcessParentFileName": [ + { + "key": "file.name", + "object": "parent_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "parent_process", + "transformer": "ToFileName" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "process.binary_ref", + "object": "parent_process", + "references": "parent_process_file" + }, + { + "key": "directory.path", + "object": "parent_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "parent_process_file", + "references": "parent_process_file_dir" + } + ], "InitiatingProcessParentCreationTime": [ { "key": "process.created", @@ -469,7 +1019,7 @@ "references": "parent_process" } ], - "AccountSid": [ + "AccountName": [ { "key": "user-account.user_id", "object": "child_process_creator" @@ -478,9 +1028,14 @@ "key": "process.creator_user_ref", "object": "child_process", "references": "child_process_creator" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "child_process_creator" } ], - "InitiatingProcessAccountSid": [ + "InitiatingProcessAccountName": [ { "key": "user-account.user_id", "object": "process_creator" @@ -491,7 +1046,7 @@ "references": "process_creator" } ], - "AccountName": [ + "AccountUpn": [ { "key": "user-account.account_login", "object": "child_process_creator" @@ -500,9 +1055,14 @@ "key": "process.creator_user_ref", "object": "child_process", "references": "child_process_creator" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "child_process_creator" } ], - "InitiatingProcessAccountName": [ + "InitiatingProcessAccountUpn": [ { "key": "user-account.account_login", "object": "process_creator" @@ -513,36 +1073,154 @@ "references": "process_creator" } ], - "InitiatingProcessFolderPath": [ + "DeviceName": [ { - "key": "directory.path", - "object": "process_file_dir", - "transformer": "ToDirectoryPath" + "key": "x-oca-asset.hostname", + "object": "host" }, { - "key": "file.parent_directory_ref", - "object": "process_file", - "references": "process_file_dir" + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "DeviceId": [ + { + "key": "x-oca-asset.device_id", + "object": "host" }, { - "key": "process.binary_ref", - "object": "process", - "references": "process_file" + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" } ], - "DeviceName": { - "key": "x-msatp.computer_name" - }, - "DeviceId": { - "key": "x-msatp.machine_id" - }, - "event_count": { - "key": "number_observed", + "PublicIP": [ + { + "key": "ipv4-addr.value", + "object": "public-ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "public-ip" + ], + "group": true + } + ], + "OSPlatform": { + "key": "x-oca-asset.os_name", + "object": "host" + }, + "OSArchitecture": { + "key": "x-oca-asset.architecture", + "object": "host" + }, + "OSVersion": { + "key": "x-oca-asset.os_version", + "object": "host" + }, + "original_ref": [ + { + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + }, + { + "key": "x-oca-event.finding_refs", + "object": "event", + "references": [ + "finding" + ], + "group": true + } + ], + "event_link": [ + { + "key": "external-reference.url", + "object": "device_external_ref" + }, + { + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" + } + ], + "MacAddressSet": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacList", + "unwrap": true + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac" + ], + "unwrap": true, + "group": true + } + ], + "IPAddresses": [ + { + "key": "ipv4-addr.value", + "object": "host_ip4", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "host_ip6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip4", + "host_ip6" + ], + "unwrap": true, + "group": true + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "event_count": { + "key": "number_observed", "cybox": false, "transformer": "ToInteger" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + }, + "AdditionalFields": { + "key": "x-msatp.AdditionalFields", + "object": "x-msatp", + "transformer": "JsonToString" } }, "DeviceFileEvents": { + "Alerts": { + "key": "x-json-alert.data" + }, "Timestamp": [ { "key": "x-oca-event.created", @@ -568,12 +1246,23 @@ "FileName": [ { "key": "file.name", - "object": "file" + "object": "file", + "transformer": "ToFileName" }, { "key": "x-oca-event.file_ref", "object": "event", "references": "file" + }, + { + "key": "directory.path", + "object": "file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "file_dir" } ], "SHA1": { @@ -588,6 +1277,56 @@ "key": "file.hashes.MD5", "object": "file" }, + "FileOriginReferrerUrl": [ + { + "key": "url.value", + "object": "url_ref", + "transformer": "IfValidUrl" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url_ref" + }, + { + "key": "domain-name.value", + "object": "file-origin-ref-domain", + "transformer": "GetDomainName" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "file-origin-ref-domain" + } + ], + "FileOriginIP": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + }, + { + "key": "x-oca-event.ips_ref", + "object": "event", + "references": [ + "dst_ip" + ], + "group": true + } + ], "InitiatingProcessSHA1": [ { "key": "file.hashes.SHA-1", @@ -624,26 +1363,45 @@ "InitiatingProcessFileName": [ { "key": "file.name", - "object": "fc_process_file" + "object": "fc_process_file", + "transformer": "ToFileName" }, { "key": "process.name", - "object": "fc_process" + "object": "fc_process", + "transformer": "ToFileName" }, { "key": "process.binary_ref", "object": "fc_process", "references": "fc_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "fc_process" + }, + { + "key": "directory.path", + "object": "fc_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "fc_process_file", + "references": "fc_process_file_dir" } ], "InitiatingProcessParentFileName": [ { "key": "file.name", - "object": "fc_parent_process_file" + "object": "fc_parent_process_file", + "transformer": "ToFileName" }, { "key": "process.name", - "object": "fc_parent_process" + "object": "fc_parent_process", + "transformer": "ToFileName" }, { "key": "process.parent_ref", @@ -654,6 +1412,16 @@ "key": "process.binary_ref", "object": "fc_parent_process", "references": "fc_parent_process_file" + }, + { + "key": "directory.path", + "object": "fc_parent_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "fc_parent_process_file", + "references": "fc_parent_process_file_dir" } ], "InitiatingProcessId": [ @@ -706,7 +1474,7 @@ "references": "fc_parent_process" } ], - "InitiatingProcessAccountSid": [ + "InitiatingProcessAccountName": [ { "key": "user-account.user_id", "object": "fc_process_creator" @@ -722,7 +1490,7 @@ "references": "fc_process_creator" } ], - "InitiatingProcessAccountName": [ + "InitiatingProcessAccountUpn": [ { "key": "user-account.account_login", "object": "fc_process_creator" @@ -737,7 +1505,7 @@ { "key": "directory.path", "object": "file_dir", - "transformer": "ToDirectoryPath" + "transformer": "ToMSATPDirectoryPath" }, { "key": "file.parent_directory_ref", @@ -749,7 +1517,7 @@ { "key": "directory.path", "object": "fc_process_file_dir", - "transformer": "ToDirectoryPath" + "transformer": "ToMSATPDirectoryPath" }, { "key": "file.parent_directory_ref", @@ -762,10 +1530,6 @@ "references": "fc_process_file" } ], - "DeviceId": { - "key": "x-oca-asset.device_id", - "object": "host" - }, "DeviceName": [ { "key": "x-oca-asset.hostname", @@ -777,205 +1541,402 @@ "references": "host" } ], - "event_count": { - "key": "number_observed", - "cybox": false, - "transformer": "ToInteger" - } - }, - "DeviceRegistryEvents": { - "Timestamp": [ + "DeviceId": [ { - "key": "x-oca-event.created", - "object": "event" - }, - { - "key": "first_observed", - "cybox": false, - "transformer": "MsatpToTimestamp" + "key": "x-oca-asset.device_id", + "object": "host" }, { - "key": "last_observed", - "cybox": false, - "transformer": "MsatpToTimestamp" + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" } ], - "ActionType": [ + "PublicIP": [ { - "key": "x-oca-event.action", - "object": "event" + "key": "ipv4-addr.value", + "object": "public-ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "public-ip" + ], + "group": true } ], - "RegistryKey": { - "key": "windows-registry-key.key", - "object": "re_key" + "OSPlatform": { + "key": "x-oca-asset.os_name", + "object": "host" }, - "RegistryValues": { - "key": "windows-registry-key.values", - "object": "re_key", - "transformer": "MsatpToRegistryValue" + "OSArchitecture": { + "key": "x-oca-asset.architecture", + "object": "host" }, - "InitiatingProcessSHA1": [ + "OSVersion": { + "key": "x-oca-asset.os_version", + "object": "host" + }, + "original_ref": [ { - "key": "file.hashes.SHA-1", - "object": "re_process_file" + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" }, { - "key": "process.binary_ref", - "object": "re_process", - "references": "re_process_file" + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + }, + { + "key": "x-oca-event.finding_refs", + "object": "event", + "references": [ + "finding" + ], + "group": true } ], - "InitiatingProcessSHA256": [ + "event_link": [ { - "key": "file.hashes.SHA-256", - "object": "re_process_file" + "key": "external-reference.url", + "object": "device_external_ref" }, { - "key": "process.binary_ref", - "object": "re_process", - "references": "re_process_file" + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" } ], - "InitiatingProcessMD5": [ + "MacAddressSet": [ { - "key": "file.hashes.MD5", - "object": "re_process_file" + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacList", + "unwrap": true }, { - "key": "process.binary_ref", - "object": "re_process", - "references": "re_process_file" + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac" + ], + "unwrap": true, + "group": true } ], - "InitiatingProcessFileName": [ + "IPAddresses": [ { - "key": "file.name", - "object": "re_process_file" + "key": "ipv4-addr.value", + "object": "host_ip4", + "unwrap": true, + "transformer": "FilterIPv4List" }, { - "key": "process.name", - "object": "re_process" + "key": "ipv6-addr.value", + "object": "host_ip6", + "unwrap": true, + "transformer": "FilterIPv6List" }, { - "key": "process.binary_ref", - "object": "re_process", - "references": "re_process_file" + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip4", + "host_ip6" + ], + "unwrap": true, + "group": true } ], - "InitiatingProcessParentFileName": [ + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + }, + "AdditionalFields": { + "key": "x-msatp.AdditionalFields", + "object": "x-msatp", + "transformer": "JsonToString" + } + }, + "DeviceImageLoadEvents": { + "Alerts": { + "key": "x-json-alert.data" + }, + "Timestamp": [ { - "key": "file.name", - "object": "re_parent_process_file" + "key": "x-oca-event.created", + "object": "event" }, { - "key": "process.name", - "object": "re_parent_process" + "key": "first_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" }, { - "key": "process.parent_ref", - "object": "re_process", - "references": "re_parent_process" + "key": "last_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + } + ], + "ActionType": [ + { + "key": "x-oca-event.action", + "object": "event" + } + ], + "FileName": [ + { + "key": "file.name", + "object": "file", + "transformer": "ToFileName" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "file" + }, + { + "key": "directory.path", + "object": "file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "file_dir" + } + ], + "SHA1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "SHA256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "MD5": { + "key": "file.hashes.MD5", + "object": "file" + }, + "InitiatingProcessSHA1": [ + { + "key": "file.hashes.SHA-1", + "object": "fc_process_file" }, { "key": "process.binary_ref", - "object": "re_parent_process", - "references": "re_parent_process_file" + "object": "fc_process", + "references": "fc_process_file" + } + ], + "InitiatingProcessSHA256": [ + { + "key": "file.hashes.SHA-256", + "object": "fc_process_file" + }, + { + "key": "process.binary_ref", + "object": "fc_process", + "references": "fc_process_file" + } + ], + "InitiatingProcessMD5": [ + { + "key": "file.hashes.MD5", + "object": "fc_process_file" + }, + { + "key": "process.binary_ref", + "object": "fc_process", + "references": "fc_process_file" + } + ], + "InitiatingProcessFileName": [ + { + "key": "file.name", + "object": "fc_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "fc_process", + "transformer": "ToFileName" + }, + { + "key": "process.binary_ref", + "object": "fc_process", + "references": "fc_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "fc_process" + }, + { + "key": "directory.path", + "object": "fc_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "fc_process_file", + "references": "fc_process_file_dir" + } + ], + "InitiatingProcessParentFileName": [ + { + "key": "file.name", + "object": "fc_parent_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "fc_parent_process", + "transformer": "ToFileName" + }, + { + "key": "process.parent_ref", + "object": "fc_process", + "references": "fc_parent_process" + }, + { + "key": "process.binary_ref", + "object": "fc_parent_process", + "references": "fc_parent_process_file" + }, + { + "key": "directory.path", + "object": "fc_parent_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "fc_parent_process_file", + "references": "fc_parent_process_file_dir" } ], "InitiatingProcessId": [ { "key": "process.pid", - "object": "re_process", + "object": "fc_process", "transformer": "ToInteger" }, { "key": "x-oca-event.process_ref", "object": "event", - "references": "re_process" + "references": "fc_process" } ], "InitiatingProcessParentId": [ { "key": "process.pid", - "object": "re_parent_process", + "object": "fc_parent_process", "transformer": "ToInteger" }, { "key": "process.parent_ref", - "object": "re_process", - "references": "re_parent_process" + "object": "fc_process", + "references": "fc_parent_process" }, { "key": "x-oca-event.parent_process_ref", "object": "event", - "references": "re_parent_process" + "references": "fc_parent_process" } ], "InitiatingProcessCommandLine": { "key": "process.command_line", - "object": "re_process" + "object": "fc_process" }, "InitiatingProcessCreationTime": { "key": "process.created", - "object": "re_process", + "object": "fc_process", "transformer": "MsatpToTimestamp" }, "InitiatingProcessParentCreationTime": [ { "key": "process.created", - "object": "re_parent_process", + "object": "fc_parent_process", "transformer": "MsatpToTimestamp" }, { "key": "process.parent_ref", - "object": "re_process", - "references": "re_parent_process" + "object": "fc_process", + "references": "fc_parent_process" } ], - "InitiatingProcessAccountSid": [ + "InitiatingProcessAccountName": [ { "key": "user-account.user_id", - "object": "re_process_creator" + "object": "fc_process_creator" }, { "key": "process.creator_user_ref", - "object": "re_process", - "references": "re_process_creator" + "object": "fc_process", + "references": "fc_process_creator" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "fc_process_creator" } ], - "InitiatingProcessAccountName": [ + "InitiatingProcessAccountUpn": [ { "key": "user-account.account_login", - "object": "re_process_creator" + "object": "fc_process_creator" }, { "key": "process.creator_user_ref", - "object": "re_process", - "references": "re_process_creator" + "object": "fc_process", + "references": "fc_process_creator" + } + ], + "FolderPath": [ + { + "key": "directory.path", + "object": "file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "file_dir" } ], "InitiatingProcessFolderPath": [ { "key": "directory.path", - "object": "re_process_file_dir", - "transformer": "ToDirectoryPath" + "object": "fc_process_file_dir", + "transformer": "ToMSATPDirectoryPath" }, { "key": "file.parent_directory_ref", - "object": "re_process_file", - "references": "re_process_file_dir" + "object": "fc_process_file", + "references": "fc_process_file_dir" }, { "key": "process.binary_ref", - "object": "re_process", - "references": "re_process_file" + "object": "fc_process", + "references": "fc_process_file" } ], - "DeviceId": { - "key": "x-oca-asset.device_id", - "object": "host" - }, "DeviceName": [ { "key": "x-oca-asset.hostname", @@ -987,10 +1948,1110 @@ "references": "host" } ], - "event_count": { - "key": "number_observed", - "cybox": false, - "transformer": "ToInteger" + "DeviceId": [ + { + "key": "x-oca-asset.device_id", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "PublicIP": [ + { + "key": "ipv4-addr.value", + "object": "public-ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "public-ip" + ], + "group": true + } + ], + "OSPlatform": { + "key": "x-oca-asset.os_name", + "object": "host" + }, + "OSArchitecture": { + "key": "x-oca-asset.architecture", + "object": "host" + }, + "OSVersion": { + "key": "x-oca-asset.os_version", + "object": "host" + }, + "original_ref": [ + { + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + }, + { + "key": "x-oca-event.finding_refs", + "object": "event", + "references": [ + "finding" + ], + "group": true + } + ], + "event_link": [ + { + "key": "external-reference.url", + "object": "device_external_ref" + }, + { + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" + } + ], + "MacAddressSet": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacList", + "unwrap": true + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac" + ], + "unwrap": true, + "group": true + } + ], + "IPAddresses": [ + { + "key": "ipv4-addr.value", + "object": "host_ip4", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "host_ip6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip4", + "host_ip6" + ], + "unwrap": true, + "group": true + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + }, + "AdditionalFields": { + "key": "x-msatp.AdditionalFields", + "object": "x-msatp", + "transformer": "JsonToString" + } + }, + "DeviceRegistryEvents": { + "Alerts": { + "key": "x-json-alert.data" + }, + "Timestamp": [ + { + "key": "x-oca-event.created", + "object": "event" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + }, + { + "key": "last_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + } + ], + "ActionType": [ + { + "key": "x-oca-event.action", + "object": "event" + } + ], + "RegistryKey": [ + { + "key": "windows-registry-key.key", + "object": "re_key" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "re_key" + } + ], + "RegistryValues": { + "key": "windows-registry-key.values", + "object": "re_key", + "transformer": "MsatpToRegistryValue" + }, + "InitiatingProcessSHA1": [ + { + "key": "file.hashes.SHA-1", + "object": "re_process_file" + }, + { + "key": "process.binary_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "InitiatingProcessSHA256": [ + { + "key": "file.hashes.SHA-256", + "object": "re_process_file" + }, + { + "key": "process.binary_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "InitiatingProcessMD5": [ + { + "key": "file.hashes.MD5", + "object": "re_process_file" + }, + { + "key": "process.binary_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "InitiatingProcessFileName": [ + { + "key": "file.name", + "object": "re_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "re_process", + "transformer": "ToFileName" + }, + { + "key": "process.binary_ref", + "object": "re_process", + "references": "re_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "re_process" + }, + { + "key": "directory.path", + "object": "re_process_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "re_process_file", + "references": "re_process_file_dir" + } + ], + "InitiatingProcessParentFileName": [ + { + "key": "file.name", + "object": "re_parent_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "re_parent_process", + "transformer": "ToFileName" + }, + { + "key": "process.parent_ref", + "object": "re_process", + "references": "re_parent_process" + }, + { + "key": "process.binary_ref", + "object": "re_parent_process", + "references": "re_parent_process_file" + }, + { + "key": "directory.path", + "object": "re_parent_process_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "re_parent_process_file", + "references": "re_parent_process_file_dir" + } + ], + "InitiatingProcessId": [ + { + "key": "process.pid", + "object": "re_process", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "re_process" + } + ], + "InitiatingProcessParentId": [ + { + "key": "process.pid", + "object": "re_parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "re_process", + "references": "re_parent_process" + } + ], + "InitiatingProcessCommandLine": { + "key": "process.command_line", + "object": "re_process" + }, + "InitiatingProcessCreationTime": { + "key": "process.created", + "object": "re_process", + "transformer": "MsatpToTimestamp" + }, + "InitiatingProcessParentCreationTime": [ + { + "key": "process.created", + "object": "re_parent_process", + "transformer": "MsatpToTimestamp" + }, + { + "key": "process.parent_ref", + "object": "re_process", + "references": "re_parent_process" + } + ], + "InitiatingProcessAccountName": [ + { + "key": "user-account.user_id", + "object": "re_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "re_process", + "references": "re_process_creator" + } + ], + "InitiatingProcessAccountUpn": [ + { + "key": "user-account.account_login", + "object": "re_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "re_process", + "references": "re_process_creator" + } + ], + "InitiatingProcessFolderPath": [ + { + "key": "directory.path", + "object": "re_process_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "re_process_file", + "references": "re_process_file_dir" + }, + { + "key": "process.binary_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "DeviceName": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "DeviceId": [ + { + "key": "x-oca-asset.device_id", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "PublicIP": [ + { + "key": "ipv4-addr.value", + "object": "public-ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "public-ip" + ], + "group": true + } + ], + "OSPlatform": { + "key": "x-oca-asset.os_name", + "object": "host" + }, + "OSArchitecture": { + "key": "x-oca-asset.architecture", + "object": "host" + }, + "OSVersion": { + "key": "x-oca-asset.os_version", + "object": "host" + }, + "original_ref": [ + { + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + }, + { + "key": "x-oca-event.finding_refs", + "object": "event", + "references": [ + "finding" + ], + "group": true + } + ], + "event_link": [ + { + "key": "external-reference.url", + "object": "device_external_ref" + }, + { + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" + } + ], + "MacAddressSet": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacList", + "unwrap": true + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac" + ], + "unwrap": true, + "group": true + } + ], + "IPAddresses": [ + { + "key": "ipv4-addr.value", + "object": "host_ip4", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "host_ip6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip4", + "host_ip6" + ], + "unwrap": true, + "group": true + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + }, + "AdditionalFields": { + "key": "x-msatp.AdditionalFields", + "object": "x-msatp", + "transformer": "JsonToString" + } + }, + "DeviceEvents": { + "Alerts": { + "key": "x-json-alert.data" + }, + "Timestamp": [ + { + "key": "x-oca-event.created", + "object": "event" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + }, + { + "key": "last_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + } + ], + "ActionType": [ + { + "key": "x-oca-event.action", + "object": "event" + } + ], + "LocalIP": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "src_ip" + ], + "group": true + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "RemoteIP": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "RemoteUrl": [ + { + "key": "url.value", + "object": "url", + "transformer": "IfValidUrl" + }, + { + "key": "domain-name.value", + "object": "domain-name", + "transformer": "GetDomainName" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain-name" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" + } + ], + "LocalPort": [ + { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "RemotePort": [ + { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "FileOriginUrl": [ + { + "key": "url.value", + "object": "url_ref", + "transformer": "IfValidUrl" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url_ref" + }, + { + "key": "domain-name.value", + "object": "file-origin-domain", + "transformer": "GetDomainName" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "file-origin-domain" + } + ], + "FileOriginIP": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "FileName": [ + { + "key": "file.name", + "object": "child_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "child_process", + "transformer": "ToFileName" + }, + { + "key": "process.binary_ref", + "object": "child_process", + "references": "child_process_file" + }, + { + "key": "process.child_refs", + "object": "process", + "references": [ + "child_process" + ] + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "child_process" + }, + { + "key": "directory.path", + "object": "child_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "child_process_file", + "references": "child_process_file_dir" + } + ], + "FolderPath": [ + { + "key": "directory.path", + "object": "child_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "child_process_file", + "references": "child_process_file_dir" + } + ], + "SHA1": { + "key": "file.hashes.SHA-1", + "object": "child_process_file" + }, + "SHA256": { + "key": "file.hashes.SHA-256", + "object": "child_process_file" + }, + "MD5": { + "key": "file.hashes.MD5", + "object": "child_process_file" + }, + "ProcessId": [ + { + "key": "process.pid", + "object": "child_process", + "transformer": "ToInteger" + }, + { + "key": "process.child_refs", + "object": "process", + "references": [ + "child_process" + ] + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "child_process" + } + ], + "ProcessCommandLine": [ + { + "key": "process.command_line", + "object": "child_process" + }, + { + "key": "process.child_refs", + "object": "process", + "references": [ + "child_process" + ] + } + ], + "ProcessCreationTime": { + "key": "process.created", + "object": "child_process", + "transformer": "MsatpToTimestamp" + }, + "InitiatingProcessSHA1": [ + { + "key": "file.hashes.SHA-1", + "object": "process_file" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + } + ], + "InitiatingProcessSHA256": [ + { + "key": "file.hashes.SHA-256", + "object": "process_file" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + } + ], + "InitiatingProcessMD5": [ + { + "key": "file.hashes.MD5", + "object": "process_file" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + } + ], + "InitiatingProcessFileName": [ + { + "key": "file.name", + "object": "process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "process", + "transformer": "ToFileName" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + }, + { + "key": "process.parent_ref", + "object": "child_process", + "references": "process" + }, + { + "key": "directory.path", + "object": "process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "process_file_dir" + } + ], + "InitiatingProcessId": [ + { + "key": "process.pid", + "object": "process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "child_process", + "references": "process" + } + ], + "InitiatingProcessCommandLine": { + "key": "process.command_line", + "object": "process" + }, + "InitiatingProcessCreationTime": { + "key": "process.created", + "object": "process", + "transformer": "MsatpToTimestamp" + }, + "InitiatingProcessFolderPath": [ + { + "key": "directory.path", + "object": "process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "process_file_dir" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + } + ], + "missingChildShouldMapInitiatingPid": { + "key": "x-oca-event.missingChildShouldMapInitiatingPid", + "object": "event" + }, + "InitiatingProcessParentId": { + "key": "process.pid", + "object": "parent_process", + "transformer": "ToInteger" + }, + "InitiatingProcessParentFileName": [ + { + "key": "file.name", + "object": "parent_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "parent_process", + "transformer": "ToFileName" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "process.binary_ref", + "object": "parent_process", + "references": "parent_process_file" + }, + { + "key": "directory.path", + "object": "parent_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "parent_process_file", + "references": "parent_process_file_dir" + } + ], + "InitiatingProcessParentCreationTime": [ + { + "key": "process.created", + "object": "parent_process", + "transformer": "MsatpToTimestamp" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "AccountName": [ + { + "key": "user-account.user_id", + "object": "child_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "child_process", + "references": "child_process_creator" + } + ], + "InitiatingProcessAccountName": [ + { + "key": "user-account.user_id", + "object": "process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "process_creator" + } + ], + "AccountUpn": [ + { + "key": "user-account.account_login", + "object": "child_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "child_process", + "references": "child_process_creator" + } + ], + "InitiatingProcessAccountUpn": [ + { + "key": "user-account.account_login", + "object": "process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "process_creator" + } + ], + "DeviceName": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "DeviceId": [ + { + "key": "x-oca-asset.device_id", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "PublicIP": [ + { + "key": "ipv4-addr.value", + "object": "public-ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "public-ip" + ], + "group": true + } + ], + "OSPlatform": { + "key": "x-oca-asset.os_name", + "object": "host" + }, + "OSArchitecture": { + "key": "x-oca-asset.architecture", + "object": "host" + }, + "OSVersion": { + "key": "x-oca-asset.os_version", + "object": "host" + }, + "original_ref": [ + { + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + }, + { + "key": "x-oca-event.finding_refs", + "object": "event", + "references": [ + "finding" + ], + "group": true + } + ], + "event_link": [ + { + "key": "external-reference.url", + "object": "device_external_ref" + }, + { + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" + } + ], + "MacAddressSet": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacList", + "unwrap": true + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac" + ], + "unwrap": true, + "group": true + } + ], + "IPAddresses": [ + { + "key": "ipv4-addr.value", + "object": "host_ip4", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "host_ip6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip4", + "host_ip6" + ], + "unwrap": true, + "group": true + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + }, + "AdditionalFields": { + "key": "x-msatp.AdditionalFields", + "object": "x-msatp", + "transformer": "JsonToString" } }, "DeviceNetworkInfo": { diff --git a/stix_shifter_modules/msatp/stix_translation/query_constructor.py b/stix_shifter_modules/msatp/stix_translation/query_constructor.py index 3bd9a3093..bef1a3266 100644 --- a/stix_shifter_modules/msatp/stix_translation/query_constructor.py +++ b/stix_shifter_modules/msatp/stix_translation/query_constructor.py @@ -16,11 +16,6 @@ class QueryStringPatternTranslator: """ Stix to kusto query translation """ - # Join query to get MAC address value from DeviceNetworkInfo - join_query = ' | join kind= inner (DeviceNetworkInfo {qualifier_string}{floor_time}| mvexpand parse_json(' \ - 'IPAddresses) | extend IP = IPAddresses.IPAddress | project Timestamp ,DeviceId , MacAddress, IP, ' \ - 'FormattedTimeKey) on DeviceId, $left.FormattedTimeKey ' \ - '== $right.FormattedTimeKey | where LocalIP == IP | where {mac_query} | order by Timestamp desc' def __init__(self, pattern: Pattern, data_model_mapper, time_range): self.dmm = data_model_mapper @@ -32,6 +27,12 @@ def __init__(self, pattern: Pattern, data_model_mapper, time_range): self._is_mac = None self.parse_expression(pattern) + @staticmethod + def _escape_value(value): + if isinstance(value, str): + return value.replace("\\", "\\\\") + return value + @staticmethod def _format_equality(value, comparator) -> str: """ @@ -44,7 +45,7 @@ def _format_equality(value, comparator) -> str: comparator = '=~' elif comparator == '!=': comparator = '!~' - return '"{}"'.format(value), comparator + return '"{}"'.format(QueryStringPatternTranslator._escape_value(value)), comparator @staticmethod def _format_set(value) -> str: @@ -55,7 +56,7 @@ def _format_set(value) -> str: """ final_value = [] for val in value.values: - final_value.append('"{}"'.format(val)) + final_value.append('"{}"'.format(QueryStringPatternTranslator._escape_value(val))) return '{}'.format(str(final_value).replace('[', '(').replace(']', ')').replace("'", "")) @staticmethod @@ -66,6 +67,7 @@ def _format_like(value, comparator) -> str: :return: str """ # Replacing value with % to .* and _ to . for supporting Like comparator + value = QueryStringPatternTranslator._escape_value(value) compile_regex = re.compile(r'.*(\%|\_).*') if compile_regex.match(value): if comparator == 'contains': @@ -80,7 +82,7 @@ def _format_matches(value) -> str: :param value: str :return: str """ - return 'regex"({})"'.format(value) + return 'regex"({})"'.format(QueryStringPatternTranslator._escape_value(value)) @staticmethod def _format_datetime(value, expression) -> str: @@ -177,6 +179,8 @@ def _parse_mapped_fields(expression, value, comparator, mapped_fields_array): ComparisonComparators.Equal, ComparisonComparators.NotEqual]: if is_int_field or is_date_field: mapped_field = 'tostring({mapped_field})'.format(mapped_field=mapped_field) + elif expression.object_path == "domain-name:value": + comparator = 'contains' elif expression.comparator in [ComparisonComparators.GreaterThan, ComparisonComparators.GreaterThanOrEqual, ComparisonComparators.LessThan, ComparisonComparators.LessThanOrEqual]: diff --git a/stix_shifter_modules/msatp/stix_translation/results_translator.py b/stix_shifter_modules/msatp/stix_translation/results_translator.py index 366ba2971..fde49d412 100644 --- a/stix_shifter_modules/msatp/stix_translation/results_translator.py +++ b/stix_shifter_modules/msatp/stix_translation/results_translator.py @@ -1,4 +1,204 @@ +import json +import re + +from stix_shifter_modules.msatp.stix_translation.transformers import SeverityToNumericVal from stix_shifter_utils.stix_translation.src.json_to_stix.json_to_stix import JSONToStix +TACTICS = [ + "Reconnaissance", + "Resource Development", + "Initial Access", + "Execution", + "Persistence", + "Privilege Escalation", + "Defense Evasion", + "Credential Access", + "Discovery", + "Lateral Movement", + "Collection", + "Command and Control", + "Exfiltration", + "Impact" +] + + +def parse_camel_case(name): + return "".join([" " + char if char.isupper() and i > 0 else char for i, char in enumerate(name)]) + + +def get_objects_by_type(objects, type_name): + """receives a stix objects dictionary and returns indexes of all objets of the given type""" + return [key for (key, obj) in objects.items() if obj["type"] == type_name] + + +def get_first_object_by_type(objects, type_name): + """receives a stix objects dictionary and returns the first index and value of the object of the given type""" + for k, v in objects.items(): + if v.get("type") == type_name: + return k, v + return None, None + + +def get_next_index(objects): + """returns the next available index in the objects dictionary""" + i = 0 + while str(i) in objects: + i += 1 + next_ref = str(i) + for ref, sco in objects.items(): + remove = [] + for key, value in sco.items(): + if key.endswith("_ref"): + if value == next_ref: + remove.append(key) + elif key.endswith("_refs"): + for r in value: + if r == next_ref: + sco[key] = [item for item in value if item != next_ref] + if len(sco[key]) == 0: + remove.append(key) + for i in remove: + sco.pop(i) + return next_ref + + +def parse_technique(technique): + match = re.match(r'^(.+) \((T.+)\)$', technique) + return { + "technique_name": match.group(1), + "technique_id": match.group(2) + } + + +def create_ttp_from_category(category): + phase_name = parse_camel_case(category) + kill_chain = "microsoft" + if phase_name in TACTICS: + kill_chain = "mitre-attack" + return { + 'type': 'x-ibm-ttp-tagging', + 'kill_chain_phases': [ + { + "phase_name": phase_name, + "kill_chain_name": kill_chain + } + ] + } + + +def create_ttps_from_technique(technique): + t = parse_technique(technique) + return { + 'type': 'x-ibm-ttp-tagging', + 'extensions': { + 'mitre-attack-ext': t + } + } + + +def fix_alerts(observed): + objects = observed["objects"] + json_alert_ref, json_alert = get_first_object_by_type(objects, "x-json-alert") + if json_alert_ref is not None: + objects.pop(json_alert_ref) + event_ref, event = get_first_object_by_type(objects, "x-oca-event") + alerts = json.loads(json_alert.get('data')) + ttps = {} + for alert in alerts: + finding = { + 'type': 'x-ibm-finding', + 'name': alert.get("Title"), + 'alert_id': alert.get("AlertId"), + 'severity': SeverityToNumericVal.transform(alert.get("Severity")), + 'ttp_tagging_refs': [] + } + finding_ref = get_next_index(objects) + objects[finding_ref] = finding + if 'finding_refs' not in event: + event['finding_refs'] = [] + event['finding_refs'].append(finding_ref) + if 'Category' in alert: + cat = alert['Category'] + if cat not in ttps: + cat_ttp = create_ttp_from_category(cat) + cat_ref = get_next_index(objects) + objects[cat_ref] = cat_ttp + ttps[cat] = cat_ref + finding['ttp_tagging_refs'].append(ttps[cat]) + if 'AttackTechniques' in alert: + for technique in alert['AttackTechniques']: + if technique not in ttps: + ttp = create_ttps_from_technique(technique) + ttp_ref = get_next_index(objects) + objects[ttp_ref] = ttp + ttps[technique] = ttp_ref + finding['ttp_tagging_refs'].append(ttps[technique]) + if len(finding['ttp_tagging_refs']) == 0: + finding.pop('ttp_tagging_refs') + + +def get_reference(objects, source, ref_prop, ref_type): + if ref_prop not in source: + return None + ref_index = source[ref_prop] + if ref_index not in objects: + return None + ref = objects[ref_index] + if not 'type' in ref or ref['type'] != ref_type: + return None + return ref + + +def extract_pipe_name(x_msatp, event): + if 'AdditionalFields' in x_msatp: + pattern = r'PipeName:\s*(.+?),' + match = re.search(pattern, x_msatp['AdditionalFields']) + if match: + event['pipe_name'] = match.group(1) + + +def fix_device_event_refs(observed): + objects = observed["objects"] + x_msatp_refs = get_objects_by_type(objects, "x-msatp") + if len(x_msatp_refs) > 0: + x_msatp = objects[x_msatp_refs[0]] + events = get_objects_by_type(objects, "x-oca-event") + if len(events) > 0: + event = objects[events[0]] + if event['action'] == "NamedPipeEvent": + ## if named pipe event add the pip_name to event: + extract_pipe_name(x_msatp, event) + if 'Table' in x_msatp and x_msatp['Table'] == 'DeviceEvents': + ## if event from DeviceEvent and there is only file name without process pid should be a file_ref not process_ref: + validate_process_ref_in_event(event, objects) + + +def validate_process_ref_in_event(event, objects): + if 'missingChildShouldMapInitiatingPid' in event: + pid = event['missingChildShouldMapInitiatingPid'] + if 'process_ref' in event: + proc_ref = event['process_ref'] + proc = get_reference(objects, event, 'process_ref', 'process') + event['process_ref'] = proc['parent_ref'] + ref = proc['binary_ref'] + event['file_ref'] = ref + del objects[proc_ref] + if pid is not None and pid != "": + init_proc = [key for key, value in objects.items() + if value.get("type") == "process" + and value.get("pid") == event['missingChildShouldMapInitiatingPid']] + if len(init_proc) == 1: + event['process_ref'] = init_proc[0] + del event['missingChildShouldMapInitiatingPid'] + + class ResultsTranslator(JSONToStix): - pass \ No newline at end of file + + def translate_results(self, data_source, data): + result = super().translate_results(data_source, data) + for observed in result["objects"]: + if observed["type"] == "observed-data" and "objects" in observed: + fix_alerts(observed) + fix_device_event_refs(observed) + observed['objects'] = {k: observed['objects'][k] for k in sorted(observed['objects'], key=lambda x: int(x))} + return result diff --git a/stix_shifter_modules/msatp/stix_translation/transformers.py b/stix_shifter_modules/msatp/stix_translation/transformers.py index 18332fad3..c6f2abed3 100644 --- a/stix_shifter_modules/msatp/stix_translation/transformers.py +++ b/stix_shifter_modules/msatp/stix_translation/transformers.py @@ -1,5 +1,17 @@ +import json +import ntpath +import re +import urllib +from stix_shifter_utils.utils import logger from stix_shifter_utils.stix_translation.src.utils.transformers import ValueTransformer +HIGH_SEVERITY = 99 +MEDIUM_SEVERITY = 66 +LOW_SEVERITY = 33 +KILL_CHAIN_NAME = 'mitre-attack' + +LOGGER = logger.set_logger(__name__) + class MsatpToTimestamp(ValueTransformer): """A value transformer to truncate milliseconds""" @@ -11,7 +23,6 @@ def transform(msatptime): return converted_time - class MsatpToRegistryValue(ValueTransformer): """A value transformer to convert MSATP Registry value protocol to windows-registry-value-type STIX""" @@ -36,3 +47,124 @@ def transform(registryvalues): converted_value.append(registryvalue_dict) return converted_value + +class FormatMacList(ValueTransformer): + """A value transformer to convert Mac address to STIX Mac address format""" + + @staticmethod + def transform(mac_lst): + addresses = [] + for mac in mac_lst: + mac = mac.replace("-", "").replace(":", "") + val = ':'.join([mac[i:i + 2] for i in range(0, len(mac), 2)]).lower() + addresses.append(val) + return addresses + + +class IfValidUrl(ValueTransformer): + """returns a url if its valid, empty string otherwise""" + + @staticmethod + def transform(value): + parsed_url = urllib.parse.urlparse(value) + if parsed_url.scheme != "": + return value + else: + return "" + + +class GetDomainName(ValueTransformer): + """A value transformer to extract domain name from url""" + + @staticmethod + def is_valid_domain_name(value): + """test if value is a vaild domain name""" + try: + return (not re.search(r"\s", value)) and (re.compile( + r"^(?:[a-zA-Z0-9](?:[a-zA-Z0-9-_]{0,61}[A-Za-z0-9])?\.)+[A-Za-z0-9][A-Za-z0-9-_]{0,61}[A-Za-z]$" + ).match(value.encode("idna").decode("ascii")) is not None) + except UnicodeError: + return False + + @staticmethod + def transform(value): + # if it is already a domain name return it + if GetDomainName.is_valid_domain_name(value): + return value + # it might be a url, parse the domain from the url + parsed_url = urllib.parse.urlparse(value) + return parsed_url.netloc + + +class ToFileName(ValueTransformer): + + @staticmethod + def transform(value): + try: + _, file_name = ntpath.split(value) + return file_name + except ValueError: + LOGGER.error("Cannot convert input to file name string") + + +class ToDirectory(ValueTransformer): + + @staticmethod + def transform(value): + try: + file_path, _ = ntpath.split(value) + return file_path + except ValueError: + LOGGER.error("Cannot convert input to file path string") + + +class ToMSATPDirectoryPath(ValueTransformer): + """A value transformer for expected directory path. Take care of the inconsistency, sometimes shows up as a + folder path and somtimes as a file name """ + + @staticmethod + def transform(value): + try: + file_path, file_name = ntpath.split(value) + if '.' not in file_name: + file_path = value + return file_path + except ValueError: + LOGGER.error("Cannot convert input to directory path string") + + +class SeverityToNumericVal(ValueTransformer): + """A value transformer to convert MSATP Severity value (high/medium/low) to numeric value""" + + @staticmethod + def transform(severity): + severity = severity.lower() + if severity == 'high': + return HIGH_SEVERITY + elif severity == 'medium': + return MEDIUM_SEVERITY + else: + return LOW_SEVERITY + + +class Alert(ValueTransformer): + """A value transformer to convert MSATP Severity value (high/medium/low) to numeric value""" + + @staticmethod + def transform(val): + return "alert" + + +class JsonToString(ValueTransformer): + """A value transformer to convert a string representing a json object to key: value string""" + + @staticmethod + def transform(val): + try: + data = json.loads(val) + except json.JSONDecodeError: + return val + pairs = [] + for key, value in data.items(): + pairs.append(f"{key}: {value}") + return ", ".join(pairs) diff --git a/stix_shifter_modules/msatp/stix_transmission/api_client.py b/stix_shifter_modules/msatp/stix_transmission/api_client.py index 25a150f81..3e5374bdc 100644 --- a/stix_shifter_modules/msatp/stix_transmission/api_client.py +++ b/stix_shifter_modules/msatp/stix_transmission/api_client.py @@ -1,6 +1,6 @@ """Apiclient for MSATP""" import json -from stix_shifter_utils.stix_transmission.utils.RestApiClient import RestApiClient +from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync DEFAULT_LIMIT = 10000 DEFAULT_OFFSET = 0 @@ -21,10 +21,10 @@ def __init__(self, connection, configuration): self.host = connection.get('host') if auth: - if 'access_token' in auth: + if 'access_token' in auth and auth['access_token']: headers['Authorization'] = "Bearer " + auth['access_token'] - self.client = RestApiClient(connection.get('host'), + self.client = RestApiClientAsync(connection.get('host'), connection.get('port', None), headers, url_modifier_function=url_modifier_function, @@ -33,12 +33,12 @@ def __init__(self, connection, configuration): ) self.timeout = connection['options'].get('timeout') - def ping_box(self): + async def ping_box(self): """Ping the endpoint.""" endpoint = '/api' - return self.client.call_api(endpoint, 'GET', timeout=self.timeout) + return await self.client.call_api(endpoint, 'GET', timeout=self.timeout) - def run_search(self, query_expression, offset=DEFAULT_OFFSET, length=DEFAULT_LIMIT): + async def run_search(self, query_expression, offset=DEFAULT_OFFSET, length=DEFAULT_LIMIT): """get the response from MSatp endpoints :param query_expression: str, search_id :param offset: int,offset value @@ -51,4 +51,4 @@ def run_search(self, query_expression, offset=DEFAULT_OFFSET, length=DEFAULT_LIM endpoint = self.endpoint query_expression = query_expression + serialize.format(offset=offset, length=length) query_expression = json.dumps({'Query': query_expression}).encode("utf-8") - return self.client.call_api(endpoint, 'POST', headers=headers, data=query_expression, timeout=self.timeout) + return await self.client.call_api(endpoint, 'POST', headers=headers, data=query_expression, timeout=self.timeout) diff --git a/stix_shifter_modules/msatp/stix_transmission/connector.py b/stix_shifter_modules/msatp/stix_transmission/connector.py index 966372c95..5f2dcbfec 100644 --- a/stix_shifter_modules/msatp/stix_transmission/connector.py +++ b/stix_shifter_modules/msatp/stix_transmission/connector.py @@ -1,13 +1,15 @@ import json + import adal -from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector from .api_client import APIClient from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger -import copy + +from .connector_post_processing import ConnectorPostProcessing -class Connector(BaseSyncConnector): +class Connector(BaseJsonSyncConnector): init_error = None logger = logger.set_logger(__name__) @@ -16,7 +18,7 @@ def __init__(self, connection, configuration): :param connection: dict, connection dict :param configuration: dict,config dict""" self.connector = __name__.split('.')[1] - + self.configuration = configuration self.adal_response = Connector.generate_token(self, connection, configuration) if self.adal_response['success']: configuration['auth']['access_token'] = self.adal_response['access_token'] @@ -70,42 +72,23 @@ def create_results_connection(self, query, offset, length): :param offset: int,offset value :param length: int,length value""" + util = ConnectorPostProcessing(self.configuration, False) response_txt = None - return_obj = dict() + return_obj = { + 'success': True, + 'data': [] + } try: if self.init_error: return self.adal_response - response = self.api_client.run_search(query, offset, length) - return_obj = self._handle_errors(response, return_obj) - response_json = json.loads(return_obj["data"]) - return_obj['data'] = response_json['Results'] - # Customizing the output json, - # Get 'TableName' attribute from each row of event data - # Create a dictionary with 'TableName' as key and other attributes in an event data as value - # Filter the "None" and empty values except for RegistryValueName, which support empty string - # Customizing of Registryvalues json - table_event_data = [] - for event_data in return_obj['data']: - lookup_table = event_data['TableName'] - event_data.pop('TableName') - build_data = dict() - build_data[lookup_table] = {k: v for k, v in event_data.items() if v or k == "RegistryValueName"} - if lookup_table == "DeviceRegistryEvents": - registry_build_data = copy.deepcopy(build_data) - registry_build_data[lookup_table]["RegistryValues"] = [] - registry_value_dict = {} - for k, v in build_data[lookup_table].items(): - if k in ["RegistryValueData", "RegistryValueName", "RegistryValueType"]: - registry_value_dict.update({k: v}) - registry_build_data[lookup_table].pop(k) - registry_build_data[lookup_table]["RegistryValues"].append(registry_value_dict) - - build_data[lookup_table] = registry_build_data[lookup_table] - build_data[lookup_table]['event_count'] = '1' - table_event_data.append(build_data) - return_obj['data'] = table_event_data - return return_obj + joined_query = util.join_query_with_other_tables(query) + response_data = self.api_client_run_search(joined_query, length, offset) + + def api_run(q): + return self.api_client_run_search(q, length, offset) + + return util.post_process(response_data, return_obj, api_run) except Exception as ex: if response_txt is not None: @@ -114,6 +97,13 @@ def create_results_connection(self, query, offset, length): else: raise ex + def api_client_run_search(self, joined_query, length, offset): + temp_return_obj = dict() + response = self.api_client.run_search(joined_query, offset, length) + temp_return_obj = self._handle_errors(response, temp_return_obj) + response_data = json.loads(temp_return_obj["data"]).get("Results") + return response_data + def generate_token(self, connection, configuration): """To generate the Token :param connection: dict, connection dict diff --git a/stix_shifter_modules/msatp/stix_transmission/connector_post_processing.py b/stix_shifter_modules/msatp/stix_transmission/connector_post_processing.py new file mode 100644 index 000000000..8f6ff3f27 --- /dev/null +++ b/stix_shifter_modules/msatp/stix_transmission/connector_post_processing.py @@ -0,0 +1,240 @@ +import copy +from datetime import datetime, timedelta +import json +import re + +from stix_shifter_utils.utils import logger + + +def merge_alerts(data): + """ + msatp has a weird behaviour for some alerts - it returns multiple items of the same alert. all properties + are identical except: 'FileName', 'SHA1', 'RemoteUrl','RemoteIP' + this causes one event to be duplicated multiple times with the same alert with hardly any difference + also - the presence of FileName and SHA1 creates a redundant confusing process object + to eliminate this - we merge all the alerts that have the same timestamp, device, report title etc + """ + keys = ['TableName', 'AlertId', 'Timestamp', 'DeviceId', 'DeviceName', 'Severity', 'Category', 'Title', + 'AttackTechniques', 'ReportId'] + alerts = filter(lambda x: x["TableName"] == "DeviceAlertEvents", data) + non_alerts = filter(lambda x: x["TableName"] != "DeviceAlertEvents", data) + seen_alerts = set() + merged_alerts = [] + for alert in alerts: + key = tuple(alert[k] for k in keys) + if key not in seen_alerts: + merged_alerts.append(alert) + seen_alerts.add(key) + return merged_alerts + list(non_alerts) + + +def remove_duplicate_and_empty_fields(event_data): + unique_values = set() + # remove timestamps from joins of device network info and device info + event_data.pop("DNI_TS", None) + event_data.pop("DI_TS", None) + copied = copy.deepcopy(event_data) + # remove None's empty strings and duplicates from joins such as DeviceId2, DeviceId3 etc + for key, value in copied.items(): + if key in unique_values or value is None or value == '' \ + or (key[-1:].isdigit() and 'SHA' not in key and 'MD' not in key): + event_data.pop(key) + else: + unique_values.add(key) + + +def get_table_name(q): + regex = r"find withsource = TableName in\s*\(([A-Za-z]+)\s*\)" + return re.search(regex, q).group(1) + + +def organize_registry_data(device_registry_events_data): + registry_value_dict = {} + for k in ["RegistryValueData", "RegistryValueName", "RegistryValueType"]: + if k in device_registry_events_data: + registry_value_dict[k] = device_registry_events_data[k] + device_registry_events_data.pop(k) + else: + registry_value_dict[k] = '' + device_registry_events_data["RegistryValues"] = [registry_value_dict] + + +def organize_ips(data): + ips_comp_lst = data.pop('IPAddressesSet') + flat_lst = [ip_obj['IPAddress'] for ips in ips_comp_lst for ip_obj in json.loads(ips) if 'IPAddress' in ip_obj] + data['IPAddresses'] = flat_lst + + +def create_event_link(data, timestamp): + try: + if 'DeviceId' in data: + # parse timestamp to date object striping nanoseconds + timestamp_dt = datetime.strptime(timestamp[:-9], "%Y-%m-%dT%H:%M:%S") + timeline_start = (timestamp_dt - timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%S") + ".000Z" + timeline_end = (timestamp_dt + timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%S") + ".000Z" + event_link = 'https://%s/machines/%s/timeline?from=%s&to=%s' % ( + 'security.microsoft.com', data.get('DeviceId'), timeline_start, timeline_end) + data['event_link'] = event_link + except Exception as ex: + data['event_link'] = '' + + +def remove_duplicate_ips(event_data): + ## remove duplicate ips between LocalIP, PublicIP and IPAddresses: + if 'PublicIP' in event_data and 'LocalIP' in event_data and \ + event_data['PublicIP'] == event_data['LocalIP']: + event_data.pop('PublicIP') + remove_duplicate_ips_from(event_data, 'LocalIP') + remove_duplicate_ips_from(event_data, 'PublicIP') + + +def remove_duplicate_ips_from(event_data, prop_name): + if 'IPAddresses' in event_data and prop_name in event_data: + filtered = [x for x in event_data['IPAddresses'] if + not x == event_data[prop_name]] + if len(filtered) > 0: + event_data['IPAddresses'] = filtered + else: + event_data.pop('IPAddresses') + + +def unify_alert_fields(event_data): + techniques_lists = [] + # attack techniques is a string due to the make_list function in the kql + # need to convert it back to dict + if 'AttackTechniques' in event_data: + for techniques_lst in event_data['AttackTechniques']: + try: + attack_techniques = json.loads(techniques_lst) + except json.decoder.JSONDecodeError: + attack_techniques = '' + finally: + techniques_lists.append(attack_techniques) + event_data['AttackTechniques'] = techniques_lists + + alerts = [] + alerts_count = len(event_data['AlertId']) + for i in range(alerts_count): + alert_dct = {k: (event_data[k][i] if len(event_data[k]) > i else '') + for k in ConnectorPostProcessing.ALERT_FIELDS if k in event_data} + if alert_dct['AlertId'] not in [alert['AlertId'] for alert in alerts]: + alerts.append(alert_dct) + event_data['Alerts'] = json.dumps(alerts) + for f in ConnectorPostProcessing.ALERT_FIELDS: + event_data.pop(f, None) + + +class ConnectorPostProcessing: + logger = logger.set_logger(__name__) + ALERT_FIELDS = ['AlertId', 'Severity', 'Title', 'Category', 'AttackTechniques'] + ALERTS_QUERY = ('{} | join kind=leftouter ' + '(DeviceAlertEvents | summarize AlertId=make_list(AlertId), Severity=make_list(Severity), ' + 'Title=make_list(Title), Category=make_list(Category), ' + 'AttackTechniques=make_list(AttackTechniques) by DeviceName, ReportId, Timestamp)' + ' on ReportId, DeviceName, Timestamp ') + DEVICE_INFO_QUERY = ('{} | join kind=leftouter ' + '(DeviceInfo | project DI_TS = Timestamp, DeviceId, PublicIP, OSArchitecture, OSPlatform, OSVersion) ' + 'on DeviceId | where DI_TS < Timestamp ' + '| summarize arg_max(DI_TS, *) by ReportId, DeviceName, Timestamp ') + DEVICE_NETWORK_QUERY = ('{} | join kind=leftouter ' + '(DeviceNetworkInfo | where NetworkAdapterStatus == "Up" ' + '| project DNI_TS = Timestamp, DeviceId, MacAddress, IPAddresses ' + '| summarize IPAddressesSet=make_set(IPAddresses), MacAddressSet=make_set(MacAddress) ' + 'by DeviceId, DNI_TS) on DeviceId ' + '| where DNI_TS < Timestamp | summarize arg_max(DNI_TS, *) ' + 'by ReportId, DeviceName, Timestamp ' + ) + + EVENTS_QUERY = ('(find withsource = TableName in ({}) where (Timestamp == datetime({})) ' + 'and (DeviceName == "{}") and (ReportId == {}))') + + EVENTS_TABLES = ['DeviceNetworkEvents', 'DeviceProcessEvents', 'DeviceFileEvents', 'DeviceRegistryEvents', + 'DeviceEvents', 'DeviceImageLoadEvents'] + + def __init__(self, configuration, alert_mode): + """Initialization. + :param configuration: dict,config dict""" + self.alert_mode = alert_mode + self.should_include_alerts = configuration.get("includeAlerts") + self.should_include_network_info = configuration.get("includeNetworkInfo") + self.should_include_host_os = configuration.get("includeHostOs") + + def join_alert_with_events(self, timestamp, device_name, report_id): + events_query = "union {}".format(','.join( + [ConnectorPostProcessing.EVENTS_QUERY.format(q, timestamp, device_name, report_id) for q in + ConnectorPostProcessing.EVENTS_TABLES])) + return self.join_query_with_other_tables(events_query, include_alerts=False) + + def join_query_with_other_tables(self, query, include_alerts=True): + table = get_table_name(query) + query = f"({query})" + if 'Alert' in table: + self.alert_mode = True + if self.should_include_alerts and not self.alert_mode and include_alerts: + query = ConnectorPostProcessing.ALERTS_QUERY.format(query) + if self.should_include_host_os: + query = ConnectorPostProcessing.DEVICE_INFO_QUERY.format(query) + if self.should_include_network_info: + query = ConnectorPostProcessing.DEVICE_NETWORK_QUERY.format(query) + return query + + def post_process(self, response_data, return_obj, api_client_run_search): + response_data = merge_alerts(response_data) + for event_data in response_data: + table = event_data.get('TableName') + # values for query + device_name = event_data.get('DeviceName') + report_id = event_data.get('ReportId') + timestamp = event_data.get('Timestamp') + if self.alert_mode and all([device_name, report_id, timestamp]): + # query events table according to alert fields + joined_query = self.join_alert_with_events(timestamp, device_name, report_id) + print("joining alert with events: ", joined_query) + events_data = api_client_run_search(joined_query) + if len(events_data) == 0: + # if only alert - assign the alert title to x-oca-event + event_data['ActionType'] = event_data.get("Title") + return_obj['data'].append({ + table: event_data + }) + else: + # correlated events where found to the alert + self.alert_mode = False + alert_data = copy.deepcopy(event_data) + if 'AttackTechniques' in alert_data: + if alert_data['AttackTechniques'] == '': + alert_data['AttackTechniques'] = '[]' + alert_data = {k: ([v] if k in self.ALERT_FIELDS else v) for (k, v) in alert_data.items()} + for event_data in events_data: + table = event_data.get("TableName") + event_data = {**alert_data, **event_data} + return_obj['data'].append({ + table: event_data + }) + else: + return_obj['data'].append({ + table: event_data + }) + for event in return_obj['data']: + table = next(iter(event)) + event_data = event[table] + timestamp = event_data.get('Timestamp') + event_data['category'] = '1' + event_data['provider'] = '1' + # link the event to ms atp console device timeline with one second before and after the event https://security.microsoft.com/machines//timeline?from=&to= + create_event_link(event_data, timestamp) + if event_data.get('AlertId') is not None and not self.alert_mode: + unify_alert_fields(event_data) + if 'IPAddressesSet' in event_data: + organize_ips(event_data) + if table == "DeviceRegistryEvents": + organize_registry_data(event_data) + if table == "DeviceEvents": + if 'ProcessId' not in event_data or event_data['ProcessId'] is None or \ + event_data['ProcessId'] == "": + event_data["missingChildShouldMapInitiatingPid"] = event_data.get("InitiatingProcessId") + event_data['event_count'] = '1' + remove_duplicate_ips(event_data) + remove_duplicate_and_empty_fields(event_data) + event_data['original_ref'] = json.dumps(event_data) + return return_obj diff --git a/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_json_to_stix.py b/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_json_to_stix.py index 4f8e8b6af..ff522aa55 100644 --- a/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_json_to_stix.py +++ b/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_json_to_stix.py @@ -1,8 +1,14 @@ import unittest import json + +from stix_shifter_modules.msatp.tests.test_utils import resolve_ref, all_keys_in_object, hashes_are_correct, \ + resolve_refs from stix_shifter_utils.stix_translation.src.json_to_stix import json_to_stix_translator from stix_shifter_modules.msatp.entry_point import EntryPoint from stix_shifter_utils.stix_translation.src.utils.transformer_utils import get_module_transformers +from stix_shifter.stix_translation import stix_translation + +HASHES = {'SHA-1', 'SHA-256', 'MD5'} MODULE = "msatp" entry_point = EntryPoint() @@ -14,6 +20,326 @@ "identity_class": "events" } options = {} +device_registry_event = { + 'DeviceRegistryEvents': + { + 'ReportId': 123, + 'DeviceName': 'host.test.com', + 'Timestamp': '2023-03-20T17:12:54.5122634Z', + 'Timestamp3': '2023-03-20T16:42:53.4896508Z', + 'Timestamp2': '2023-03-20T16:42:53.4896508Z', + 'DeviceId': 'deviceid', + 'ActionType': 'RegistryValueSet', + 'RegistryKey': 'HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows Advanced Threat Protection', + 'PreviousRegistryValueName': 'CrashHeartbeat', + 'PreviousRegistryValueData': '12344', + 'InitiatingProcessAccountDomain': 'nt authority', + 'InitiatingProcessAccountName': 'system', + 'InitiatingProcessAccountSid': 'S-1-5-18', + 'InitiatingProcessSHA1': '186d3710b3e909b23e6254480520f247564b4005', + 'InitiatingProcessSHA256': 'a9b445863dd123b4b6cb9749228d9ce19448edd1f610347d8e4011f9fdf584de', + 'InitiatingProcessMD5': '5f278fa24e89535896acb13d42a8f764', + 'InitiatingProcessFileName': 'mssense.exe', + 'InitiatingProcessFileSize': 224184, + 'InitiatingProcessVersionInfoCompanyName': 'Microsoft Corporation', + 'InitiatingProcessVersionInfoProductName': 'Microsoft® Windows® Operating System', + 'InitiatingProcessVersionInfoInternalFileName': 'MsSense.exe', + 'InitiatingProcessVersionInfoOriginalFileName': 'MsSense.exe', + 'InitiatingProcessVersionInfoFileDescription': 'Windows Defender Advanced Threat Protection Service Executable', + 'InitiatingProcessId': 5380, + 'InitiatingProcessCommandLine': '"MsSense.exe"', + 'InitiatingProcessCreationTime': '2023-03-20T11:12:48.6790505Z', + 'InitiatingProcessFolderPath': 'c:\\program files\\windows defender advanced threat protection\\mssense.exe', + 'InitiatingProcessParentId': 1048, + 'InitiatingProcessParentFileName': 'services.exe', + 'InitiatingProcessParentCreationTime': '2023-03-20T11:12:47.7246449Z', + 'InitiatingProcessIntegrityLevel': 'System', + 'InitiatingProcessTokenElevation': 'TokenElevationTypeDefault', + 'DeviceId1': 'deviceid', + 'MacAddressSet': ['AA-BB-11-22-CC-AA'], + 'DeviceId2': 'deviceid', + 'PublicIP': '9.9.9.1', + 'OSArchitecture': '64-bit', + 'OSPlatform': 'Windows10', + 'OSVersion': '10.0', + 'rn': 1, + 'category': '', + 'provider': '', + 'event_link': 'https://security.microsoft.com/machines/deviceid/timeline?from=2023-03-20T17:12:53.000Z&to=2023-03-20T17:12:55.000Z', + 'IPAddresses': ['9.9.9.1'], + 'RegistryValues': [ + { + 'RegistryValueType': 'Qword', + 'RegistryValueName': 'CrashHeartbeat', + 'RegistryValueData': '12345' + }], + 'event_count': '1' + } +} + +device_network_event = { + 'DeviceNetworkEvents': + { + 'ReportId': 1234, + 'DeviceName': 'host.test.com', + 'Timestamp': '2023-03-13T22:08:58.8407802Z', + 'Timestamp3': '2023-03-13T21:42:09.8518066Z', + 'Timestamp2': '2023-03-13T21:42:09.8518066Z', + 'DeviceId': 'deviceid', + 'ActionType': 'ConnectionSuccess', + 'RemoteIP': '9.9.9.9', + 'RemotePort': 443, + 'RemoteUrl': 'quad9.net', + 'LocalIP': '9.9.9.1', + 'LocalPort': 60773, + 'Protocol': 'Tcp', + 'LocalIPType': 'Private', + 'RemoteIPType': 'Public', + 'InitiatingProcessSHA1': '4a65b267d5fc37527f567f0300e1624845600be1', + 'InitiatingProcessSHA256': 'b84257d238582d3768799e08df03f0b3378a7f8d7342b8c8ffcc453cf6a7b867', + 'InitiatingProcessMD5': '58f918b86a4798177032abcb12c9c605', + 'InitiatingProcessFileName': 'OUTLOOK.EXE', + 'InitiatingProcessFileSize': 42954600, + 'InitiatingProcessVersionInfoCompanyName': 'Microsoft Corporation', + 'InitiatingProcessVersionInfoProductName': 'Microsoft Outlook', + 'InitiatingProcessVersionInfoProductVersion': '16.0.15601.20538', + 'InitiatingProcessVersionInfoInternalFileName': 'Outlook', + 'InitiatingProcessVersionInfoOriginalFileName': 'Outlook.exe', + 'InitiatingProcessVersionInfoFileDescription': 'Microsoft Outlook', + 'InitiatingProcessId': 13748, + 'InitiatingProcessCommandLine': '"OUTLOOK.EXE" ', + 'InitiatingProcessCreationTime': '2023-03-13T14:36:08.6982223Z', + 'InitiatingProcessFolderPath': 'c:\\program files\\microsoft office\\root\\office16\\outlook.exe', + 'InitiatingProcessParentFileName': 'explorer.exe', + 'InitiatingProcessParentId': 18936, + 'InitiatingProcessParentCreationTime': '2023-03-13T14:31:51.6553402Z', + 'InitiatingProcessAccountDomain': 'asd', + 'InitiatingProcessAccountName': 'username', + 'InitiatingProcessAccountSid': 'S-1-5-21', + 'InitiatingProcessAccountUpn': 'username@test.com', + 'InitiatingProcessAccountObjectId': 'asdasd', + 'InitiatingProcessIntegrityLevel': 'Medium', + 'InitiatingProcessTokenElevation': 'TokenElevationTypeLimited', + 'RegistryValueName': '', + 'DeviceId1': 'deviceid', + 'MacAddressSet': ['00-01-02-03-04-AA'], + 'DeviceId2': 'deviceid', + 'PublicIP': '9.9.9.1', + 'OSArchitecture': '64-bit', + 'OSPlatform': 'Windows10', + 'OSVersion': '10.0', + 'rn': 1, + 'category': '', + 'provider': '', + 'event_link': 'https://{domain}/machines/{device}/timeline?from=2023-03-13T22:08:57.000Z&' + 'to=2023-03-13T22:08:59.000Z', + 'IPAddresses': ['192.168.86.46'], + 'event_count': '1', + 'original_ref': 'reducted' + } +} + +device_process_event = { + 'DeviceProcessEvents': + { + 'ReportId': 1234, + 'DeviceName': 'host.test.com', + 'Timestamp': '2023-03-17T20:23:03.7116107Z', + 'Timestamp3': '2023-03-17T19:40:08.8911345Z', + 'Timestamp2': '2023-03-17T19:40:08.8911345Z', + 'DeviceId': 'deviceid', + 'ActionType': 'ProcessCreated', + 'FileName': 'msedge.exe', + 'FolderPath': 'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe', + 'SHA1': 'c737742b81292c764ac2a7e419a37ed7fdf4a1ed', + 'SHA256': '470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75', + 'MD5': 'e180c9a532c45eba99eefd01601f5c41', + 'FileSize': 4243360, + 'ProcessVersionInfoCompanyName': 'Microsoft Corporation', + 'ProcessVersionInfoProductName': 'Microsoft Edge', + 'ProcessVersionInfoProductVersion': '110.0.1587.50', + 'ProcessVersionInfoInternalFileName': 'msedge_exe', + 'ProcessVersionInfoOriginalFileName': 'msedge.exe', + 'ProcessVersionInfoFileDescription': 'Microsoft Edge', + 'ProcessId': 37384, + 'ProcessCommandLine': '"msedge.exe" --type=gpu-process', + 'ProcessIntegrityLevel': 'Low', + 'ProcessTokenElevation': 'TokenElevationTypeDefault', + 'ProcessCreationTime': '2023-03-17T20:23:03.7021445Z', + 'AccountDomain': 'asd', + 'AccountName': 'username', + 'AccountSid': 'S-1-5-21', + 'AccountObjectId': 'aaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', + 'AccountUpn': 'username@test.com', + 'LogonId': 111111111, + 'InitiatingProcessAccountDomain': 'asd', + 'InitiatingProcessAccountName': 'username', + 'InitiatingProcessAccountSid': 'S-1-5-21', + 'InitiatingProcessAccountUpn': 'username@test.com', + 'InitiatingProcessAccountObjectId': 'aaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', + 'InitiatingProcessLogonId': 111111111, + 'InitiatingProcessIntegrityLevel': 'Medium', + 'InitiatingProcessTokenElevation': 'TokenElevationTypeDefault', + 'InitiatingProcessSHA1': 'c737742b81292c764ac2a7e419a37ed7fdf4a1ed', + 'InitiatingProcessSHA256': '470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75', + 'InitiatingProcessMD5': 'e180c9a532c45eba99eefd01601f5c41', + 'InitiatingProcessFileName': 'msedge.exe', + 'InitiatingProcessFileSize': 4243360, + 'InitiatingProcessVersionInfoCompanyName': 'Microsoft Corporation', + 'InitiatingProcessVersionInfoProductName': 'Microsoft Edge', + 'InitiatingProcessVersionInfoProductVersion': '110.0.1587.50', + 'InitiatingProcessVersionInfoInternalFileName': 'msedge_exe', + 'InitiatingProcessVersionInfoOriginalFileName': 'msedge.exe', + 'InitiatingProcessVersionInfoFileDescription': 'Microsoft Edge', + 'InitiatingProcessId': 400, + 'InitiatingProcessCommandLine': '"msedge.exe" -- "https://test.com/login/login.asp"', + 'InitiatingProcessCreationTime': '2023-03-17T20:23:03.441179Z', + 'InitiatingProcessFolderPath': 'c:\\program files (x86)\\microsoft\\edge\\application\\msedge.exe', + 'InitiatingProcessParentId': 30972, + 'InitiatingProcessParentFileName': 'iexplore.exe', + 'InitiatingProcessParentCreationTime': '2023-03-17T20:23:03.1696537Z', + 'InitiatingProcessSignerType': 'OsVendorApplication', + 'InitiatingProcessSignatureStatus': 'Valid', + 'RegistryValueName': '', + 'DeviceId1': 'deviceid', + 'MacAddressSet': ['11-22-33-44-55-66'], + 'DeviceId2': 'deviceid', + 'PublicIP': '9.9.9.1', + 'OSArchitecture': '64-bit', + 'OSPlatform': 'Windows10', + 'OSVersion': '10.0', + 'rn': 1, + 'category': '', + 'provider': '', + 'event_link': 'https://security.microsoft.com/machines/deviceid/timeline?from=2023-03-17T20:23:02.000Z&to=2023-03-17T20:23:04.000Z', + 'event_count': '1' + } +} + +device_file_event = { + 'DeviceFileEvents': + { + 'ReportId': 1234, + 'DeviceName': 'host.test.com', + 'Timestamp': '2023-03-17T20:19:41.7007151Z', + 'Timestamp3': '2023-03-17T19:40:08.8911345Z', + 'Timestamp2': '2023-03-17T19:40:08.8911345Z', + 'DeviceId': 'deviceid', + 'ActionType': 'FileModified', + 'FileName': 'asdasdasd.html', + 'FolderPath': 'C:\\Users\\username\\Downloads\\asdasdasd.html', + 'SHA1': '3ee189ca7db084de9d630cd6091125d99b3af1e1', + 'SHA256': '92200a5da4433f86af6009486817fc068714ac49050d7f5c6f1f393f17e72411', + 'MD5': '8c541a9caed9f9b52be730cc16df4dc1', + 'FileSize': 33785, + 'InitiatingProcessAccountDomain': 'nt authority', + 'InitiatingProcessAccountName': 'system', + 'InitiatingProcessAccountSid': 'S-1-5-18', + 'InitiatingProcessIntegrityLevel': 'System', + 'InitiatingProcessTokenElevation': 'TokenElevationTypeDefault', + 'InitiatingProcessSHA1': 'e31d842f36952d41d6cc39b0baafeb59c0cbac42', + 'InitiatingProcessSHA256': 'a3e2ef7e6f46566c7f0b56c6a2ac4d07b7ed3c927d9232bbd28483a1100a0e82', + 'InitiatingProcessMD5': 'c977a7757d71bf51d42703ba1799a191', + 'InitiatingProcessFileName': 'AsdService.exe', + 'InitiatingProcessFileSize': 4958832, + 'InitiatingProcessVersionInfoCompanyName': 'asd Corp.', + 'InitiatingProcessVersionInfoProductName': 'asd Privilege Management', + 'InitiatingProcessVersionInfoProductVersion': '21.6.153.0', + 'InitiatingProcessVersionInfoInternalFileName': 'AsdService.exe', + 'InitiatingProcessVersionInfoOriginalFileName': 'AsdService.exe', + 'InitiatingProcessVersionInfoFileDescription': 'asd Privilege Management Service', + 'InitiatingProcessId': 3892, + 'InitiatingProcessCommandLine': '"AsdService.exe"', + 'InitiatingProcessCreationTime': '2023-02-23T03:48:44.4729635Z', + 'InitiatingProcessFolderPath': 'c:\\program files\\asd\\guard client\\asdservice.exe', + 'InitiatingProcessParentId': 1016, + 'InitiatingProcessParentFileName': 'services.exe', + 'InitiatingProcessParentCreationTime': '2023-02-23T03:48:43.6251262Z', + 'RegistryValueName': '', + 'RequestProtocol': 'Local', + 'RequestAccountName': 'username', + 'RequestAccountDomain': 'ASD', + 'RequestAccountSid': 'S-1-5-21-1111111111-111111111-1111111111-1111111', + 'DeviceId1': 'deviceid', + 'MacAddressSet': ['11-22-33-44-55-66'], + 'DeviceId2': 'deviceid', + 'PublicIP': '9.9.9.1', + 'OSArchitecture': '64-bit', + 'OSPlatform': 'Windows10', + 'OSVersion': '10.0', + 'rn': 1, + 'category': '', + 'provider': '', + 'event_link': 'https://security.microsoft.com/machines/deviceid/timeline?from=2023-03-17T20:19:40.000Z&to=2023-03-17T20:19:42.000Z', + 'event_count': '1' + } +} + +device_event_with_alert = { + 'DeviceEvents': + { + 'ReportId': 1234, + 'DeviceName': 'host.test.com', + 'Timestamp': '2023-03-27T08:32:10.8787764Z', + 'TableName': 'DeviceEvents', + 'DeviceId': 'deviceid', + 'RemoteUrl': 'https://malicious.com', + 'RemoteIP': '9.9.9.9', + 'RemotePort': 443, + 'LocalIP': '9.9.9.1', + 'LocalPort': 58993, + 'ActionType': 'OtherAlertRelatedActivity', + 'FileName': 'rundll32.exe', + 'FolderPath': 'C:\\Windows\\System32\\rundll32.exe', + 'SHA1': 'dd399ae46303343f9f0da189aee11c67bd868222', + 'SHA256': 'b53f3c0cd32d7f20849850768da6431e5f876b7bfa61db0aa0700b02873393fa', + 'MD5': 'ef3179d498793bf4234f708d3be28633', + 'AccountDomain': 'ASD', + 'AccountName': 'username', + 'InitiatingProcessAccountDomain': 'ASD', + 'InitiatingProcessAccountName': 'username', + 'InitiatingProcessAccountSid': 'S-1-5-21-1111111111-111111111-1111111111-1111111', + 'InitiatingProcessSHA1': '75cf2cc2e76eb99c4de584a3f2917c5cce3bc1bd', + 'InitiatingProcessSHA256': '0f4e87fab2e5455b89c7263456c461cef80c4ef4b05c893a455d70e2ccd4c387', + 'InitiatingProcessMD5': '1cc2723f23e452686f2426024bf666e5', + 'InitiatingProcessFileName': 'WcInstaller.exe', + 'InitiatingProcessId': 18808, + 'InitiatingProcessCommandLine': '"WcInstaller.exe" --silent', + 'InitiatingProcessCreationTime': '2023-03-27T08:31:19.2589344Z', + 'InitiatingProcessFolderPath': 'C:\\Users\\username\\AppData\\Local\\Temp\\WcInstaller.exe', + 'InitiatingProcessParentId': 18428, + 'InitiatingProcessParentFileName': 'setup_win64.exe', + 'InitiatingProcessParentCreationTime': '2023-03-27T08:31:08.2841083Z', + 'AdditionalFields': '{"Description":"File WcInstaller.exe observed on machine host.test.com"}', + 'PublicIP': '9.9.9.1', + 'OSArchitecture': '64-bit', + 'OSPlatform': 'Windows10', + 'OSVersion': '10.0', + 'MacAddressSet': ['11-22-33-44-55-66'], + 'rn': 1, + 'category': '1', + 'provider': '1', + 'event_link': 'https://security.microsoft.com/machines/deviceid/timeline?from=2023-03-27T08:32:09.000Z&to=2023-03-27T08:32:11.000Z', + 'Alerts': '[{"AlertId": "1234567890-1234-1234-1234-123456789012_1", "Severity": "Medium", "Title": "A suspicious file was observed", "Category": "Malware", "AttackTechniques": ["Obfuscated Files or Information (T1027)", "Malicious File (T1204.002)"]}]', + 'IPAddresses': ['9.9.9.1'], + 'missingChildShouldMapInitiatingPid': 18808, + 'event_count': '1', + 'original_ref': 'base64' + } +} + + +def translate_to_objects(data): + translation = stix_translation.StixTranslation() + result_bundle = translation.translate(module='msatp', translate_type='results', data_source=data_source, + data=[data]) + result_bundle_objects = result_bundle['objects'] + result_bundle_identity = result_bundle_objects[0] + assert result_bundle_identity['type'] == data_source['type'] + observed_data = result_bundle_objects[1] + assert 'objects' in observed_data + objects = observed_data['objects'] + return objects class TestMsatpResultsToStix(unittest.TestCase): @@ -42,34 +368,7 @@ def test_common_prop(self): """ to test the common stix object properties """ - data = {'DeviceProcessEvents': {'Timestamp': '2019-09-20T06:57:11.8218304Z', - 'DeviceId': '8330ed311f1b21b861d63448984eb2632cc9c07c', - 'DeviceName': 'desktop-536bt46', 'ActionType': 'ProcessCreated', - 'FileName': 'consent.exe', 'FolderPath': 'C:\\Windows\\System32\\consent.exe', - 'SHA1': '9329b2362078de27242dd4534f588af3264bf0bf', - 'SHA256': '8f112431143a22baaafb448eefd63bf90e7691c890ac69a296574fd07ba03ec6', - 'MD5': '27992d7ebe51aec655a088de88bad5c9', 'ProcessId': 20948, - 'ProcessCommandLine': 'consent.exe 10088 288 000001CB3AA92A80', - 'ProcessIntegrityLevel': 'System', - 'ProcessTokenElevation': 'TokenElevationTypeDefault', - 'ProcessCreationTime': '2019-09-20T06:57:11.8212034Z', - 'AccountDomain': 'nt authority', 'AccountName': 'system', - 'AccountSid': 'S-1-5-18', 'LogonId': 999, - 'InitiatingProcessAccountDomain': 'nt authority', - 'InitiatingProcessAccountName': 'system', - 'InitiatingProcessAccountSid': 'S-1-5-18', 'InitiatingProcessLogonId': 999, - 'InitiatingProcessIntegrityLevel': 'System', - 'InitiatingProcessTokenElevation': 'TokenElevationTypeDefault', - 'InitiatingProcessSHA1': 'a1385ce20ad79f55df235effd9780c31442aa234', - 'InitiatingProcessMD5': '8a0a29438052faed8a2532da50455756', - 'InitiatingProcessFileName': 'svchost.exe', 'InitiatingProcessId': 10088, - 'InitiatingProcessCommandLine': 'svchost.exe -k netsvcs -p -s Appinfo', - 'InitiatingProcessCreationTime': '2019-09-18T05:56:15.268893Z', - 'InitiatingProcessFolderPath': 'c:\\windows\\system32\\svchost.exe', - 'InitiatingProcessParentId': 856, - 'InitiatingProcessParentFileName': 'services.exe', - 'InitiatingProcessParentCreationTime': '2019-09-17T14:54:59.5778638Z', - 'ReportId': 12048, 'rn': 1, 'event_count': '1'}} + data = device_process_event result_bundle = json_to_stix_translator.convert_to_stix( data_source, map_data, [data], get_module_transformers(MODULE), options) assert result_bundle['type'] == 'bundle' @@ -89,353 +388,277 @@ def test_common_prop(self): assert observed_data['last_observed'] is not None assert observed_data['number_observed'] is not None - - def test_custom_property(self): - """ - to test the custom stix object properties - """ - data = {'DeviceProcessEvents': {'Timestamp': '2019-09-20T06:57:11.8218304Z', - 'DeviceId': '8330ed311f1b21b861d63448984eb2632cc9c07c', - 'DeviceName': 'desktop-536bt46', 'ActionType': 'ProcessCreated', - 'FileName': 'consent.exe', 'FolderPath': 'C:\\Windows\\System32\\consent.exe', - 'SHA1': '9329b2362078de27242dd4534f588af3264bf0bf', - 'SHA256': '8f112431143a22baaafb448eefd63bf90e7691c890ac69a296574fd07ba03ec6', - 'MD5': '27992d7ebe51aec655a088de88bad5c9', 'ProcessId': 20948, - 'ProcessCommandLine': 'consent.exe 10088 288 000001CB3AA92A80', - 'ProcessIntegrityLevel': 'System', - 'ProcessTokenElevation': 'TokenElevationTypeDefault', - 'ProcessCreationTime': '2019-09-20T06:57:11.8212034Z', - 'AccountDomain': 'nt authority', 'AccountName': 'system', - 'AccountSid': 'S-1-5-18', 'LogonId': 999, - 'InitiatingProcessAccountDomain': 'nt authority', - 'InitiatingProcessAccountName': 'system', - 'InitiatingProcessAccountSid': 'S-1-5-18', 'InitiatingProcessLogonId': 999, - 'InitiatingProcessIntegrityLevel': 'System', - 'InitiatingProcessTokenElevation': 'TokenElevationTypeDefault', - 'InitiatingProcessSHA1': 'a1385ce20ad79f55df235effd9780c31442aa234', - 'InitiatingProcessMD5': '8a0a29438052faed8a2532da50455756', - 'InitiatingProcessFileName': 'svchost.exe', 'InitiatingProcessId': 10088, - 'InitiatingProcessCommandLine': 'svchost.exe -k netsvcs -p -s Appinfo', - 'InitiatingProcessCreationTime': '2019-09-18T05:56:15.268893Z', - 'InitiatingProcessFolderPath': 'c:\\windows\\system32\\svchost.exe', - 'InitiatingProcessParentId': 856, - 'InitiatingProcessParentFileName': 'services.exe', - 'InitiatingProcessParentCreationTime': '2019-09-17T14:54:59.5778638Z', - 'ReportId': 12048, 'rn': 1, 'event_count': '1'}} - result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [data], get_module_transformers(MODULE), options) - assert result_bundle['type'] == 'bundle' - result_bundle_objects = result_bundle['objects'] - - observed_data = result_bundle_objects[1] - - def test_file_json_to_stix(self): """ to test file stix object properties """ - data = {'DeviceFileEvents': {'Timestamp': '2019-09-20T06:50:17.3764965Z', - 'DeviceId': '8330ed311f1b21b861d63448984eb2632cc9c07c', - 'DeviceName': 'desktop-536bt46', 'ActionType': 'FileCreated', - 'FileName': 'updater.exe', - 'FolderPath': '', - 'SHA1': 'cf864398950658185fad8207957b46c12f133ea5', - 'MD5': '64c52647783e6b3c0964e41aa38fa5c1', - 'InitiatingProcessAccountDomain': 'nt authority', - 'InitiatingProcessAccountName': 'system', - 'InitiatingProcessAccountSid': 'S-1-5-18', - 'InitiatingProcessMD5': '620f00789f37c453710ebf758bf1772e', - 'InitiatingProcessSHA1': '8bd812436b301dd30d55f76ae418a0e85f7dd020', - 'InitiatingProcessFolderPath': 'c:\\program files (x86)\\mozilla maintenance ' - 'service\\update\\updater.exe', - 'InitiatingProcessFileName': 'updater.exe', 'InitiatingProcessId': 13980, - 'InitiatingProcessCommandLine': '"updater.exe" ' - 'C:\\ProgramData\\Mozilla\\updates\\3080' - '46B0AF4A39CB\\updates\\0 "" "" -1', - 'InitiatingProcessCreationTime': '2019-09-20T06:50:08.1793244Z', - 'InitiatingProcessIntegrityLevel': 'System', - 'InitiatingProcessTokenElevation': 'TokenElevationTypeDefault', - 'InitiatingProcessParentId': 17088, - 'InitiatingProcessParentFileName': 'maintenanceservice.exe', - 'InitiatingProcessParentCreationTime': '2019-09-20T06:50:07.6324849Z', - 'RequestProtocol': 'Unknown', 'ReportId': 11844, 'rn': 1, 'event_count': '1'}} - - result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [data], get_module_transformers(MODULE), options) - - result_bundle_objects = result_bundle['objects'] - - result_bundle_identity = result_bundle_objects[0] - assert result_bundle_identity['type'] == data_source['type'] - - observed_data = result_bundle_objects[1] + objects = translate_to_objects(device_file_event) - assert 'objects' in observed_data - objects = observed_data['objects'] - - file_obj = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'file') - assert file_obj is not None, 'file object type not found' - assert file_obj.keys() == {'type', 'hashes', 'parent_directory_ref', 'name'} + event_obj = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'x-oca-event') + assert event_obj is not None, 'event object type not found' + assert 'file_ref' in event_obj, 'file_ref missing from event' + file_obj = resolve_ref(objects, event_obj, 'file_ref', 'file', 'file object missing') + assert all_keys_in_object({'type', 'hashes', 'parent_directory_ref', 'name'}, file_obj) assert file_obj['type'] == 'file' - assert file_obj['name'] == 'updater.exe' - assert file_obj['hashes'] == {'SHA-1': 'cf864398950658185fad8207957b46c12f133ea5', - 'MD5': '64c52647783e6b3c0964e41aa38fa5c1'} - assert file_obj['parent_directory_ref'] == '3' - directory_object = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'directory') - file_path = get_module_transformers(MODULE)['ToDirectoryPath'].transform(data['DeviceFileEvents']['FolderPath']) - assert directory_object.get('path') == file_path + assert file_obj['name'] == 'asdasdasd.html' + assert all_keys_in_object(HASHES, file_obj['hashes']) + hashes_are_correct(file_obj, { + 'MD5': '8c541a9caed9f9b52be730cc16df4dc1', + 'SHA1': '3ee189ca7db084de9d630cd6091125d99b3af1e1', + 'SHA256': '92200a5da4433f86af6009486817fc068714ac49050d7f5c6f1f393f17e72411' + }) + parent_dir = resolve_ref(objects, file_obj, 'parent_directory_ref', 'directory', 'parent dir missing') + assert parent_dir.get('path') == "C:\\Users\\username\\Downloads" def test_process_json_to_stix(self): """ to test process stix object properties """ - data = {'DeviceProcessEvents': {'Timestamp': '2019-09-20T06:57:11.8218304Z', - 'DeviceId': '8330ed311f1b21b861d63448984eb2632cc9c07c', - 'DeviceName': 'desktop-536bt46', 'ActionType': 'ProcessCreated', - 'FileName': 'consent.exe', 'FolderPath': 'C:\\Windows\\System32\\consent.exe', - 'SHA1': '9329b2362078de27242dd4534f588af3264bf0bf', - 'SHA256': '8f112431143a22baaafb448eefd63bf90e7691c890ac69a296574fd07ba03ec6', - 'MD5': '27992d7ebe51aec655a088de88bad5c9', 'ProcessId': 20948, - 'ProcessCommandLine': 'consent.exe 10088 288 000001CB3AA92A80', - 'ProcessIntegrityLevel': 'System', - 'ProcessTokenElevation': 'TokenElevationTypeDefault', - 'ProcessCreationTime': '2019-09-20T06:57:11.8212034Z', - 'AccountDomain': 'nt authority', 'AccountName': 'system', - 'AccountSid': 'S-1-5-18', 'LogonId': 999, - 'InitiatingProcessAccountDomain': 'nt authority', - 'InitiatingProcessAccountName': 'system', - 'InitiatingProcessAccountSid': 'S-1-5-18', 'InitiatingProcessLogonId': 999, - 'InitiatingProcessIntegrityLevel': 'System', - 'InitiatingProcessTokenElevation': 'TokenElevationTypeDefault', - 'InitiatingProcessSHA1': 'a1385ce20ad79f55df235effd9780c31442aa234', - 'InitiatingProcessMD5': '8a0a29438052faed8a2532da50455756', - 'InitiatingProcessFileName': 'svchost.exe', 'InitiatingProcessId': 10088, - 'InitiatingProcessCommandLine': 'svchost.exe -k netsvcs -p -s Appinfo', - 'InitiatingProcessCreationTime': '2019-09-18T05:56:15.268893Z', - 'InitiatingProcessFolderPath': 'c:\\windows\\system32\\svchost.exe', - 'InitiatingProcessParentId': 856, - 'InitiatingProcessParentFileName': 'services.exe', - 'InitiatingProcessParentCreationTime': '2019-09-17T14:54:59.5778638Z', - 'ReportId': 12048, 'rn': 1, 'Event_count': '1'}} - result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [data], get_module_transformers(MODULE), options) - result_bundle_objects = result_bundle['objects'] - - result_bundle_identity = result_bundle_objects[0] - assert result_bundle_identity['type'] == data_source['type'] - - observed_data = result_bundle_objects[1] - - assert 'objects' in observed_data - objects = observed_data['objects'] - - process_obj = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'process') - assert process_obj is not None, 'process object type not found' - assert process_obj.keys() == {'type', 'name', 'binary_ref', 'pid', 'command_line', 'created', - 'creator_user_ref'} - assert process_obj['type'] == 'process' - assert process_obj['name'] == 'consent.exe' - assert process_obj['binary_ref'] == '2' - assert process_obj['pid'] == 20948 - assert process_obj['command_line'] == 'consent.exe 10088 288 000001CB3AA92A80' - assert process_obj['created'] == '2019-09-20T06:57:11.821Z' - assert process_obj['creator_user_ref'] == '6' - + objects = translate_to_objects(device_process_event) + + event_obj = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'x-oca-event') + assert event_obj is not None, 'event object type not found' + assert 'process_ref' in event_obj, 'process_ref missing from event' + process_obj = resolve_ref(objects, event_obj, 'process_ref', 'process', 'process object missing') + assert all_keys_in_object({'type', 'name', 'binary_ref', 'pid', 'command_line', 'created', 'creator_user_ref'}, + process_obj) + assert process_obj['name'] == 'msedge.exe' + file_obj = resolve_ref(objects, process_obj, 'binary_ref', "file", "binary ref missing from process") + assert file_obj['name'] == 'msedge.exe' + assert all_keys_in_object(HASHES, file_obj['hashes']) + hashes_are_correct(file_obj, {'SHA1': 'c737742b81292c764ac2a7e419a37ed7fdf4a1ed', + 'SHA256': '470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75', + 'MD5': 'e180c9a532c45eba99eefd01601f5c41'}) + dir_obj = resolve_ref(objects, file_obj, 'parent_directory_ref', 'directory', + 'parent dir missing from file binary ref') + assert dir_obj.get("path") == "C:\\Program Files (x86)\\Microsoft\\Edge\\Application" + assert process_obj['pid'] == 37384 + assert process_obj['command_line'] == '"msedge.exe" --type=gpu-process' + assert process_obj['created'] == '2023-03-17T20:23:03.702Z' + user_obj = resolve_ref(objects, process_obj, 'creator_user_ref', "user-account", "missing creator user ref") + assert user_obj['user_id'] == "username" + assert user_obj['account_login'] == "username@test.com" + parent_obj = resolve_ref(objects, process_obj, 'parent_ref', "process", "parent process missing") + assert parent_obj['name'] == "msedge.exe" + assert parent_obj['pid'] == 400 + assert parent_obj['created'] == '2023-03-17T20:23:03.441Z' + assert parent_obj['command_line'] == '"msedge.exe" -- "https://test.com/login/login.asp"' + parent_parent_obj = resolve_ref(objects, parent_obj, 'parent_ref', "process", 'parent parent process missing') + assert parent_parent_obj['name'] == 'iexplore.exe' + assert parent_parent_obj['pid'] == 30972 + assert parent_parent_obj['created'] == '2023-03-17T20:23:03.169Z' def test_network_json_to_stix(self): """to test network stix object properties""" - data = {'DeviceNetworkEvents': {'Timestamp': '2019-09-26T09:47:52.7091342Z', - 'DeviceId': '8330ed311f1b21b861d63448984eb2632cc9c07c', - 'DeviceName': 'desktop-536bt46', 'ActionType': 'ConnectionSuccess', - 'RemoteIP': '168.159.213.203', 'RemotePort': 80, - 'LocalIP': '172.16.2.22', 'LocalPort': 52240, 'Protocol': 'TcP', - 'LocalIPType': 'Private', 'RemoteIPType': 'Public', - 'InitiatingProcessSHA1': 'c12506914be39ee4f152369b6a6692733b1b70e9', - 'InitiatingProcessMD5': 'e407c42454e8520daca3eea0353967fb', - 'InitiatingProcessFileName': 'Microsoft.Photos.exe', - 'InitiatingProcessId': 10756, - 'InitiatingProcessCommandLine': '"Microsoft.Photos.exe" ' - '-ServerName:App.AppXzst44mncqdg8' - '4v7sv6p7yznqwssy6f7f.mca', - 'InitiatingProcessCreationTime': '2019-09-26T09:32:10.8711434Z', - 'InitiatingProcessFolderPath': 'c:\\program ' - 'files\\windowsapps\\microsoft.windows' - '.photos_2019.19071.17920.0_x64__8wekyb' - '3d8bbwe\\microsoft.photos.exe', - 'InitiatingProcessParentFileName': 'svchost.exe', - 'InitiatingProcessParentId': 1020, - 'InitiatingProcessParentCreationTime': '2019-09-17T14:55:00.5337848Z', - 'InitiatingProcessAccountDomain': 'desktop-536bt46', - 'InitiatingProcessAccountName': 'admin', - 'InitiatingProcessAccountSid': - 'S-1-5-21-2603683697-4187888953-3873858-1001', - 'InitiatingProcessIntegrityLevel': 'Low', - 'InitiatingProcessTokenElevation': 'TokenElevationTypeLimited', - 'ReportId': 24239, 'rn': 1, 'event_count': '1'}} - result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [data], get_module_transformers(MODULE), options) - result_bundle_objects = result_bundle['objects'] - - result_bundle_identity = result_bundle_objects[0] - assert result_bundle_identity['type'] == data_source['type'] - - observed_data = result_bundle_objects[1] - - assert 'objects' in observed_data - objects = observed_data['objects'] + objects = translate_to_objects(device_network_event) network_obj = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'network-traffic') assert network_obj is not None, 'network-traffic object type not found' assert network_obj.keys() == {'type', 'dst_ref', 'dst_port', 'src_ref', 'src_port', 'protocols'} assert network_obj['type'] == 'network-traffic' - assert network_obj['dst_ref'] == '1' - assert network_obj['dst_port'] == 80 - assert network_obj['src_ref'] == '3' - assert network_obj['src_port'] == 52240 + assert objects[network_obj['dst_ref']]['type'] == "ipv4-addr" + assert objects[network_obj['dst_ref']]['value'] == "9.9.9.9" + assert network_obj['dst_port'] == 443 + assert objects[network_obj['src_ref']]['type'] == "ipv4-addr" + assert objects[network_obj['src_ref']]['value'] == "9.9.9.1" + assert network_obj['src_port'] == 60773 assert network_obj['protocols'] == ['tcp'] - def test_network_json_to_stix_negative(self): - """to test negative test case for stix object""" - data = {'DeviceNetworkEvents': {'Timestamp': '2019-09-20T06:24:16.830101Z', - 'DeviceId': '8330ed311f1b21b861d63448984eb2632cc9c07c', - 'DeviceName': 'desktop-536bt46', 'ActionType': 'ConnectionSuccess', - 'RemoteIP': '168.159.213.203', 'RemotePort': 80, - 'RemoteUrl': 'https://play.google.com', 'LocalIP': '172.16.2.22', - 'LocalPort': 63043, 'Protocol': 'TcpV4', - 'InitiatingProcessSHA1': 'f6af6cd298f660ff5bb4f89398d1d3edac020a7d', - 'InitiatingProcessMD5': '94e4f3e52bae1a934889aaeb7238dccc', - 'InitiatingProcessFileName': 'chrome.exe', 'InitiatingProcessId': 10404, - 'InitiatingProcessCommandLine': '"chrome.exe" --type=utility ' - '--field-trial-handle=1632,' - '12328523307506075385,' - '13359799139346648205,' - '131072 --lang=en-US ' - '--service-sandbox-type=network ' - '--service-request-channel-token=' - '12003267709621771016 --mojo-platform' - '-channel-handle=2064 /prefetch:8', - 'InitiatingProcessCreationTime': '2019-09-18T04:54:26.1863029Z', - 'InitiatingProcessFolderPath': 'c:\\program files (' - 'x86)\\google\\chrome\\application\\chrome.exe', - 'InitiatingProcessParentFileName': 'chrome.exe', - 'InitiatingProcessParentId': 9792, - 'InitiatingProcessParentCreationTime': '2019-09-18T04:54:24.3181704Z', - 'InitiatingProcessAccountDomain': 'desktop-536bt46', - 'InitiatingProcessAccountName': 'admin', - 'InitiatingProcessAccountSid': - 'S-1-5-21-2603683697-4187888953-3873858-1001', - 'InitiatingProcessIntegrityLevel': 'Medium', - 'InitiatingProcessTokenElevation': 'TokenElevationTypeLimited', - 'ReportId': 10787, 'rn': 1, 'event_count': '1'}} - result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, data, get_module_transformers(MODULE), options) - result_bundle_objects = result_bundle['objects'] - - result_bundle_identity = result_bundle_objects[0] - assert result_bundle_identity['type'] == data_source['type'] - - observed_data = result_bundle_objects[1] - - assert 'objects' in observed_data - objects = observed_data['objects'] - - network_obj = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'file') - assert network_obj is None - def test_mac_json_to_stix(self): """to test mac stix object properties""" - data = {'DeviceNetworkEvents': {'Timestamp': '2019-09-20T06:24:16.830101Z', - 'DeviceId': '8330ed311f1b21b861d63448984eb2632cc9c07c', - 'DeviceName': 'desktop-536bt46', - 'LocalIP': '172.16.2.22', - 'MacAddress': '484D7E9DBD97', - 'RemoteIP': '168.159.213.203', - 'LocalPort': 63043, - 'RemotePort': 80, - 'Protocol': 'TcpV4', - 'RemoteUrl': 'https://play.google.com', - 'InitiatingProcessSHA1': 'f6af6cd298f660ff5bb4f89398d1d3edac020a7d', - 'InitiatingProcessMD5': '94e4f3e52bae1a934889aaeb7238dccc', - 'InitiatingProcessFileName': 'chrome.exe', - 'InitiatingProcessParentFileName': 'chrome.exe', - 'InitiatingProcessId': 10404, 'InitiatingProcessParentId': 9792, - 'InitiatingProcessCommandLine': '"chrome.exe" --type=utility ' - '--field-trial-handle=1632,' - '12328523307506075385,' - '13359799139346648205,' - '131072 --lang=en-US ' - '--service-sandbox-type=network ' - '--service-request-channel-token' - '=12003267709621771016 --mojo-platform-' - 'channel-handle=2064 /prefetch:8', - 'InitiatingProcessCreationTime': '2019-09-18T04:54:26.1863029Z', - 'InitiatingProcessParentCreationTime': '2019-09-18T04:54:24.3181704Z', - 'InitiatingProcessAccountSid': - 'S-1-5-21-2603683697-4187888953-3873858-1001', - 'InitiatingProcessAccountName': 'admin', - 'InitiatingProcessFolderPath': 'c:\\program files (' - 'x86)\\google\\chrome\\application\\chrome.exe', - 'rn': 1, 'event_count': '1'}} - - result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [data], get_module_transformers(MODULE), options) - result_bundle_objects = result_bundle['objects'] - - result_bundle_identity = result_bundle_objects[0] - assert result_bundle_identity['type'] == data_source['type'] - - observed_data = result_bundle_objects[1] - - assert 'objects' in observed_data - objects = observed_data['objects'] + objects = translate_to_objects(device_network_event) network_obj = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'mac-addr') assert network_obj is not None, 'mac-addr object type not found' assert network_obj.keys() == {'type', 'value'} assert network_obj['type'] == 'mac-addr' - assert network_obj['value'] == '48:4d:7e:9d:bd:97' + assert network_obj['value'] == '00:01:02:03:04:aa' def test_registry_json_to_stix(self): """to test registry stix object properties""" - data = {'DeviceRegistryEvents': {'Timestamp': '2019-10-10T10:41:43.0469296Z', - 'DeviceId': 'db40e68dd7358aa450081343587941ce96ca4777', - 'DeviceName': 'testmachine1', 'ActionType': 'RegistryValueSet', - 'RegistryKey': 'HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows Advanced Threat ' - 'Protection', - 'PreviousRegistryValueName': 'Configuration', - 'InitiatingProcessAccountDomain': 'nt authority', - 'InitiatingProcessAccountName': 'system', 'InitiatingProcessAccountSid': 'S-1-5-18', - 'InitiatingProcessSHA1': '657cd516b52b861ae98670c2ab10dc4a467cfd80', - 'InitiatingProcessMD5': 'b97799c4a1ed64e97c9fa83401e8d67c', - 'InitiatingProcessFileName': 'mssense.exe', 'InitiatingProcessId': 1040, - 'InitiatingProcessCommandLine': '"MsSense.exe"', - 'InitiatingProcessCreationTime': '2019-10-10T10:41:29.2621221Z', - 'InitiatingProcessFolderPath': 'c:\\program files\\windows defender advanced threat ' - 'protection\\mssense.exe', - 'InitiatingProcessParentId': 776, 'InitiatingProcessParentFileName': 'services.exe', - 'InitiatingProcessParentCreationTime': '2019-10-10T10:26:00.1611536Z', - 'InitiatingProcessIntegrityLevel': 'System', - 'InitiatingProcessTokenElevation': 'TokenElevationTypeDefault', 'ReportId': 89, - 'rn': 1, 'RegistryValues': - [{'RegistryValueType': 'Binary', 'RegistryValueName': 'Configuration'}], - 'event_count': '1'}} - result_bundle = json_to_stix_translator.convert_to_stix( - data_source, map_data, [data], get_module_transformers(MODULE), options) - result_bundle_objects = result_bundle['objects'] - - result_bundle_identity = result_bundle_objects[0] - assert result_bundle_identity['type'] == data_source['type'] - - observed_data = result_bundle_objects[1] - - assert 'objects' in observed_data - objects = observed_data['objects'] - - network_obj = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'windows-registry-key') - assert network_obj is not None, 'windows-registry-key object type not found' - assert network_obj.keys() == {'type', 'key', 'values'} - assert network_obj['type'] == 'windows-registry-key' - assert network_obj['key'] == 'HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows Advanced Threat Protection' - assert network_obj['values'] == [{'data_type': 'REG_BINARY', 'name': 'Configuration'}] + objects = translate_to_objects(device_registry_event) + + registry_obj = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'windows-registry-key') + assert registry_obj is not None, 'windows-registry-key object type not found' + assert all_keys_in_object({'type', 'key', 'values'}, registry_obj) + assert registry_obj['type'] == 'windows-registry-key' + assert registry_obj['key'] == 'HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows Advanced Threat Protection' + assert len(registry_obj['values']) == 1 + assert all_keys_in_object({'data', 'data_type', 'name'}, registry_obj['values'][0]) + assert registry_obj['values'][0]['data'] == '12345' + assert registry_obj['values'][0]['data_type'] == 'REG_QWORD' + assert registry_obj['values'][0]['name'] == 'CrashHeartbeat' + + def test_alert_json_to_stix(self): + """ + test device event with alert + """ + objects = translate_to_objects(device_event_with_alert) + + event = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'x-oca-event') + assert event is not None, 'event object type not found' + findings = resolve_refs(objects, event, "finding_refs", "x-ibm-finding", "finding refs missing in event") + assert len(findings) == 1 + finding = findings[0] + assert finding.get("name") == "A suspicious file was observed" + assert finding.get("severity") == 66 + assert finding.get("alert_id") == "1234567890-1234-1234-1234-123456789012_1" + ttps = resolve_refs(objects, finding, "ttp_tagging_refs", "x-ibm-ttp-tagging", + "ttp tagging refs missing from finding") + assert len(ttps) == 3 + ttp = ttps[0] + assert "kill_chain_phases" in ttp + phases = ttp["kill_chain_phases"] + assert len(phases) == 1 + assert phases[0].get("phase_name") == "Malware" + assert phases[0].get("kill_chain_name") == "microsoft" + + def test_alert_non_mitre_tactic(self): + data = json.loads(json.dumps(device_event_with_alert)) + # alerts = json.loads(data['DeviceEvents']['Alerts']) + # alerts[0]['Category'] = "Malware" + # data['DeviceEvents']['Alerts'] = json.dumps(alerts) + objects = translate_to_objects(data) + finding = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'x-ibm-finding') + ttps = resolve_refs(objects, finding, "ttp_tagging_refs", "x-ibm-ttp-tagging", + "ttp tagging refs missing from finding") + assert len(ttps) == 3 + ttp = ttps[0] + assert "kill_chain_phases" in ttp + phases = ttp["kill_chain_phases"] + assert len(phases) == 1 + assert phases[0].get("phase_name") == "Malware" + assert phases[0].get("kill_chain_name") == "microsoft" + + def test_alert_mitre_technique(self): + data = json.loads(json.dumps(device_event_with_alert)) + alerts = json.loads(data['DeviceEvents']['Alerts']) + alerts[0]['AttackTechniques'] = ['Spearphishing Link (T1566.002)'] + alerts[0]['Category'] = 'CredentialAccess' + data['DeviceEvents']['Alerts'] = json.dumps(alerts) + objects = translate_to_objects(data) + finding = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'x-ibm-finding') + ttps = resolve_refs(objects, finding, "ttp_tagging_refs", "x-ibm-ttp-tagging", + "ttp tagging refs missing from finding") + assert len(ttps) == 2 + tactic = ttps[0] + assert "kill_chain_phases" in tactic + phases = tactic["kill_chain_phases"] + assert len(phases) == 1 + assert phases[0].get("phase_name") == "Credential Access" + assert phases[0].get("kill_chain_name") == "mitre-attack" + technique = ttps[1] + assert "extensions" in technique + assert "mitre-attack-ext" in technique["extensions"] + assert technique["extensions"]["mitre-attack-ext"]["technique_name"] == "Spearphishing Link" + assert technique["extensions"]["mitre-attack-ext"]["technique_id"] == "T1566.002" + + def test_event(self): + """ + test x-oca-event + """ + objects = translate_to_objects(device_event_with_alert) + + event = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'x-oca-event') + assert event is not None, 'event object type not found' + assert event['action'] == "OtherAlertRelatedActivity" + assert event.get("provider") == "Microsoft Defender for Endpoint" + assert event.get("created") == "2023-03-27T08:32:10.8787764Z" + external = resolve_ref(objects, event, "external_ref", "external-reference", + "missing external ref link in event") + assert external.get( + "url") == "https://security.microsoft.com/machines/deviceid/timeline?from=2023-03-27T08:32:09.000Z&to=2023-03-27T08:32:11.000Z" + url = resolve_ref(objects, event, 'url_ref', "url", "missing url ref in event") + assert url.get("value") == "https://malicious.com" + domain = resolve_ref(objects, event, 'domain_ref', 'domain-name', 'missing domain ref in event') + assert domain.get("value") == "malicious.com" + resolve_ref(objects, event, "host_ref", "x-oca-asset", "missing host ref in event") + process = resolve_ref(objects, event, "process_ref", "process", "missing process ref in event") + assert process.get("pid") == 18808 + network = resolve_ref(objects, event, "network_ref", "network-traffic", "missing network ref in event") + assert network.get("src_port") == 58993 + assert network.get("dst_port") == 443 + dst = resolve_ref(objects, network, "dst_ref", "ipv4-addr", "missing dst ip in network") + assert dst.get("value") == "9.9.9.9" + src = resolve_ref(objects, network, "src_ref", "ipv4-addr", "missing src ip in network") + assert src.get("value") == "9.9.9.1" + orig = resolve_ref(objects, event, "original_ref", "artifact", "missing original ref in event") + assert orig.get("payload_bin") == "YmFzZTY0" + + def test_asset(self): + """test x-oca-asset""" + objects = translate_to_objects(device_process_event) + + event = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'x-oca-event') + assert event is not None, 'event object type not found' + asset = resolve_ref(objects, event, "host_ref", "x-oca-asset", "host is missing") + assert asset.get("hostname") == "host.test.com" + assert asset.get("device_id") == "deviceid" + assert asset.get("os_name") == "Windows10" + ip_refs = resolve_refs(objects, asset, "ip_refs", "ipv4-addr", "ip refs are missing") + assert len(ip_refs) == 1 + ip = ip_refs[0] + assert ip.get("value") == "9.9.9.1" + mac_refs = resolve_refs(objects, asset, "mac_refs", "mac-addr", "mac refs are missing") + assert len(mac_refs) == 1 + mac = mac_refs[0] + assert mac.get("value") == "11:22:33:44:55:66" + + def test_get_next_index_remove_ref(self): + objects = { + '0': { + 'type': 'x-oca-event', + 'ip_ref': '1', + 'ip_refs': ['2', '3'] + }, + '2': { + 'type': 'ipv4-addr', + 'value': '9.9.9.9' + } + } + from stix_shifter_modules.msatp.stix_translation import results_translator + index = results_translator.get_next_index(objects) + assert index == "1" + assert 'ip_ref' not in objects['0'] + + def test_get_next_index_remove_refs(self): + objects = { + '0': { + 'type': 'x-oca-event', + 'ip_ref': '1', + 'ip_refs': ['2', '3'] + }, + '1': { + 'type': 'ipv4-addr', + 'value': '9.9.9.9' + } + } + from stix_shifter_modules.msatp.stix_translation import results_translator + index = results_translator.get_next_index(objects) + assert index == "2" + assert 'ip_ref' in objects['0'] + assert 'ip_refs' in objects['0'] + assert len(objects['0']['ip_refs']) == 1 + assert objects['0']['ip_refs'][0] == '3' + + def test_get_next_index_pops_refs(self): + objects = { + '0': { + 'type': 'x-oca-event', + 'ip_refs': ['1'] + } + } + from stix_shifter_modules.msatp.stix_translation import results_translator + index = results_translator.get_next_index(objects) + assert index == "1" + assert 'ip_refs' not in objects['0'] + + def test_extract_pipe_name(self): + xmsatp = { + 'AdditionalFields': r'DesiredAccess: 1180063, FileOperation: File opened, NamedPipeEnd: Client, PipeName: \\Device\\NamedPipe\\PrinterProviderProbePipe01, RemoteClientsAccess: AcceptRemote, SessionId: 1, ThreadId: 11660' + } + from stix_shifter_modules.msatp.stix_translation import results_translator + event = {} + results_translator.extract_pipe_name(xmsatp, event) + assert 'pipe_name' in event + assert event['pipe_name'] == r'\\Device\\NamedPipe\\PrinterProviderProbePipe01' diff --git a/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_stix_to_query.py b/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_stix_to_query.py index c07146fe4..178e76974 100644 --- a/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_stix_to_query.py +++ b/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_stix_to_query.py @@ -6,12 +6,11 @@ def _remove_timestamp_from_query(queries): - pattern = r'Timestamp\s*>=\s*datetime\((\d{4})(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z\)\s*and\s*Timestamp\s*<\s*' \ - r'datetime\((\d{4})(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z\)\s*' + pattern = r'datetime\((\d{4})(-\d{2}){2}T\d{2}(:\d{2}){2}(\.\d+)?Z\)' if isinstance(queries, list): - return [re.sub(pattern, "", query) for query in queries] + return [re.sub(pattern, "<>", query) for query in queries] elif isinstance(queries, str): - return re.sub(pattern, "", queries) + return re.sub(pattern, "<>", queries) class TestQueryTranslator(unittest.TestCase): @@ -26,30 +25,37 @@ def _test_query_assertions(self, query, queries): self.assertIsInstance(query, dict) self.assertIsInstance(query['queries'], list) for index, each_query in enumerate(query.get('queries'), start=0): - self.assertEqual(each_query, queries[index]) + self.assertEqual(queries[index], each_query) def test_file_comp_exp(self): stix_pattern = "[file:name = 'updater.exe']" query = translation.translate('msatp', 'query', '{}', stix_pattern) query['queries'] = _remove_timestamp_from_query(query['queries']) - queries = ['union (find withsource = TableName in (DeviceFileEvents) where Timestamp >= datetime(' - '2021-05-03T08:54:32.159Z) and Timestamp < datetime(2021-05-03T08:59:32.159Z) | order by ' - 'Timestamp desc | where (FileName =~ "updater.exe") or (InitiatingProcessFileName =~ ' - '"updater.exe") or (InitiatingProcessParentFileName =~ "updater.exe")),(find withsource = ' - 'TableName in (DeviceProcessEvents) where Timestamp >= datetime(2021-05-03T08:54:32.159Z) and ' - 'Timestamp < datetime(2021-05-03T08:59:32.159Z) | order by Timestamp desc | where (FileName =~ ' - '"updater.exe") or (InitiatingProcessFileName =~ "updater.exe") or (' - 'InitiatingProcessParentFileName =~ "updater.exe")),(find withsource = TableName in (' - 'DeviceNetworkEvents) where Timestamp >= datetime(2021-05-03T08:54:32.159Z) and Timestamp < ' - 'datetime(2021-05-03T08:59:32.159Z) | order by Timestamp desc | where (InitiatingProcessFileName ' - '=~ "updater.exe") or (InitiatingProcessParentFileName =~ "updater.exe")),(find withsource = ' - 'TableName in (DeviceRegistryEvents) where Timestamp >= datetime(2021-05-03T08:54:32.159Z) and ' - 'Timestamp < datetime(2021-05-03T08:59:32.159Z) | order by Timestamp desc | where (' - 'InitiatingProcessFileName =~ "updater.exe") or (InitiatingProcessParentFileName =~ ' - '"updater.exe"))'] - - queries = _remove_timestamp_from_query(queries) + queries = [('union (find withsource = TableName in (DeviceFileEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where (FileName =~ "updater.exe") or (InitiatingProcessFileName =~ ' + '"updater.exe") or (InitiatingProcessParentFileName =~ "updater.exe")),(find ' + 'withsource = TableName in (DeviceProcessEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where (FileName =~ "updater.exe") or (InitiatingProcessFileName =~ ' + '"updater.exe") or (InitiatingProcessParentFileName =~ "updater.exe")),(find ' + 'withsource = TableName in (DeviceNetworkEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where (InitiatingProcessFileName =~ "updater.exe") or ' + '(InitiatingProcessParentFileName =~ "updater.exe")),(find withsource = ' + 'TableName in (DeviceRegistryEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + '(InitiatingProcessFileName =~ "updater.exe") or ' + '(InitiatingProcessParentFileName =~ "updater.exe")),(find withsource = ' + 'TableName in (DeviceEvents) where Timestamp >= <> and Timestamp ' + '< <> | order by Timestamp desc | where (FileName =~ ' + '"updater.exe") or (InitiatingProcessFileName =~ "updater.exe") or ' + '(InitiatingProcessParentFileName =~ "updater.exe")),(find withsource = ' + 'TableName in (DeviceImageLoadEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where (FileName =~ ' + '"updater.exe") or (InitiatingProcessFileName =~ "updater.exe") or ' + '(InitiatingProcessParentFileName =~ "updater.exe"))')] self._test_query_assertions(query, queries) def test_process_comp_exp(self): @@ -58,17 +64,24 @@ def test_process_comp_exp(self): query['queries'] = _remove_timestamp_from_query(query['queries']) queries = [ - 'union (find withsource = TableName in (DeviceProcessEvents) where Timestamp >= datetime(' - '2021-05-03T08:55:09.199Z) and Timestamp < datetime(2021-05-03T09:00:09.199Z) | order by Timestamp desc ' - '| where (FileName =~ "consent.exe") or (InitiatingProcessFileName =~ "consent.exe") or (' - 'InitiatingProcessParentFileName =~ "consent.exe")),(find withsource = TableName in (DeviceNetworkEvents) ' - ' where Timestamp >= datetime(2021-05-03T08:55:09.199Z) and Timestamp < datetime(' - '2021-05-03T09:00:09.199Z) | order by Timestamp desc | where (InitiatingProcessFileName =~ ' - '"consent.exe") or (InitiatingProcessParentFileName =~ "consent.exe")),(find withsource = TableName in (' - 'DeviceRegistryEvents) where Timestamp >= datetime(2021-05-03T08:55:09.199Z) and Timestamp < datetime(' - '2021-05-03T09:00:09.199Z) | order by Timestamp desc | where (InitiatingProcessFileName =~ ' - '"consent.exe") or (InitiatingProcessParentFileName =~ "consent.exe"))'] - queries = _remove_timestamp_from_query(queries) + ('union (find withsource = TableName in (DeviceProcessEvents) where Timestamp ' + '>= <> and Timestamp < <> | order by Timestamp desc | ' + 'where (FileName =~ "consent.exe") or (InitiatingProcessFileName =~ ' + '"consent.exe")),(find withsource = TableName in (DeviceEvents) where ' + 'Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where (FileName =~ "consent.exe") or ' + '(InitiatingProcessFileName =~ "consent.exe")),(find withsource = TableName ' + 'in (DeviceFileEvents) where Timestamp >= <> and Timestamp < ' + '<> | order by Timestamp desc | where InitiatingProcessFileName ' + '=~ "consent.exe"),(find withsource = TableName in (DeviceNetworkEvents) ' + 'where Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where InitiatingProcessFileName =~ "consent.exe"),(find ' + 'withsource = TableName in (DeviceRegistryEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where InitiatingProcessFileName =~ "consent.exe"),(find withsource = ' + 'TableName in (DeviceImageLoadEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + 'InitiatingProcessFileName =~ "consent.exe")')] self._test_query_assertions(query, queries) def test_network_comp_exp(self): @@ -77,10 +90,12 @@ def test_network_comp_exp(self): query = translation.translate('msatp', 'query', '{}', stix_pattern) query['queries'] = _remove_timestamp_from_query(query['queries']) queries = [ - '(find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(' - '2019-09-10T08:43:10.003Z) and Timestamp < datetime(2019-09-23T10:43:10.453Z) | order by Timestamp desc ' - '| where (LocalIP =~ "172.16.2.22") or (RemoteIP =~ "172.16.2.22"))'] - queries = _remove_timestamp_from_query(queries) + ('union (find withsource = TableName in (DeviceNetworkEvents) where Timestamp ' + '>= <> and Timestamp < <> | order by Timestamp desc | ' + 'where (LocalIP =~ "172.16.2.22") or (RemoteIP =~ "172.16.2.22")),(find ' + 'withsource = TableName in (DeviceEvents) where Timestamp >= <> ' + 'and Timestamp < <> | order by Timestamp desc | where (RemoteIP ' + '=~ "172.16.2.22") or (LocalIP =~ "172.16.2.22"))')] self._test_query_assertions(query, queries) def test_mac_comp_exp(self): @@ -103,10 +118,12 @@ def test_registry_comp_exp(self): query['queries'] = _remove_timestamp_from_query(query['queries']) queries = [ - '(find withsource = TableName in (DeviceRegistryEvents) where Timestamp >= datetime(' - '2019-09-01T08:43:10.003Z) and Timestamp < datetime(2019-10-10T10:43:10.003Z) | order by Timestamp desc ' - '| where RegistryValueName in~ ("SD", "Index"))'] - queries = _remove_timestamp_from_query(queries) + ('union (find withsource = TableName in (DeviceRegistryEvents) where ' + 'Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where RegistryValueName in~ ("SD", "Index")),(find ' + 'withsource = TableName in (DeviceEvents) where Timestamp >= <> ' + 'and Timestamp < <> | order by Timestamp desc | where ' + 'RegistryValueName in~ ("SD", "Index"))')] self._test_query_assertions(query, queries) def test_directory_comp_exp(self): @@ -115,19 +132,30 @@ def test_directory_comp_exp(self): query = translation.translate('msatp', 'query', '{}', stix_pattern) query['queries'] = _remove_timestamp_from_query(query['queries']) - queries = ['union (find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(' - '2019-10-01T08:43:10.003Z) and Timestamp < datetime(2019-10-30T10:43:10.003Z) | order by ' - 'Timestamp desc | where (InitiatingProcessFolderPath contains "ProgramData") or ((LocalIP =~ ' - '"fe80::4161:ca84:4dc5:f5fc") or (RemoteIP =~ "fe80::4161:ca84:4dc5:f5fc"))),(find withsource = ' - 'TableName in (DeviceFileEvents) where Timestamp >= datetime(2019-10-01T08:43:10.003Z) and ' - 'Timestamp < datetime(2019-10-30T10:43:10.003Z) | order by Timestamp desc | where ((FolderPath ' - 'contains "ProgramData") or (InitiatingProcessFolderPath contains "ProgramData"))),' - '(find withsource = TableName in (DeviceProcessEvents) where Timestamp >= datetime(' - '2019-10-01T08:43:10.003Z) and Timestamp < datetime(2019-10-30T10:43:10.003Z) | order by ' - 'Timestamp desc | where ((FolderPath contains "ProgramData") or (InitiatingProcessFolderPath ' - 'contains "ProgramData"))),(find withsource = TableName in (DeviceRegistryEvents) where Timestamp ' - '>= datetime(2019-10-01T08:43:10.003Z) and Timestamp < datetime(2019-10-30T10:43:10.003Z) | order ' - 'by Timestamp desc | where (InitiatingProcessFolderPath contains "ProgramData"))'] + queries = [('union (find withsource = TableName in (DeviceNetworkEvents) where Timestamp ' + '>= <> and Timestamp < <> | order by Timestamp desc | ' + 'where (InitiatingProcessFolderPath contains "ProgramData") or ((LocalIP =~ ' + '"fe80::4161:ca84:4dc5:f5fc") or (RemoteIP =~ ' + '"fe80::4161:ca84:4dc5:f5fc"))),(find withsource = TableName in ' + '(DeviceEvents) where Timestamp >= <> and Timestamp < ' + '<> | order by Timestamp desc | where ((FolderPath contains ' + '"ProgramData") or (InitiatingProcessFolderPath contains "ProgramData")) or ' + '((RemoteIP =~ "fe80::4161:ca84:4dc5:f5fc") or (LocalIP =~ ' + '"fe80::4161:ca84:4dc5:f5fc"))),(find withsource = TableName in ' + '(DeviceFileEvents) where Timestamp >= <> and Timestamp < ' + '<> | order by Timestamp desc | where ((FolderPath contains ' + '"ProgramData") or (InitiatingProcessFolderPath contains ' + '"ProgramData"))),(find withsource = TableName in (DeviceProcessEvents) ' + 'where Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where ((FolderPath contains "ProgramData") or ' + '(InitiatingProcessFolderPath contains "ProgramData"))),(find withsource = ' + 'TableName in (DeviceRegistryEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + '(InitiatingProcessFolderPath contains "ProgramData")),(find withsource = ' + 'TableName in (DeviceImageLoadEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ((FolderPath ' + 'contains "ProgramData") or (InitiatingProcessFolderPath contains ' + '"ProgramData")))')] queries = _remove_timestamp_from_query(queries) self._test_query_assertions(query, queries) @@ -137,22 +165,23 @@ def test_gt_eq_datetime_comp_exp(self): query['queries'] = _remove_timestamp_from_query(query['queries']) queries = [ - 'union (find withsource = TableName in (DeviceProcessEvents) where Timestamp >= datetime(' - '2021-05-03T08:57:38.841Z) and Timestamp < datetime(2021-05-03T09:02:38.841Z) | order by Timestamp desc ' - '| where (ProcessCreationTime >= datetime(2019-09-04T09:29:29.0882Z)) or (' - 'InitiatingProcessParentCreationTime >= datetime(2019-09-04T09:29:29.0882Z))),(find withsource = ' - 'TableName in (DeviceNetworkEvents) where Timestamp >= datetime(2021-05-03T08:57:38.841Z) and Timestamp ' - '< datetime(2021-05-03T09:02:38.841Z) | order by Timestamp desc | where (InitiatingProcessCreationTime ' - '>= datetime(2019-09-04T09:29:29.0882Z)) or (InitiatingProcessParentCreationTime >= datetime(' - '2019-09-04T09:29:29.0882Z))),(find withsource = TableName in (DeviceRegistryEvents) where Timestamp >= ' - 'datetime(2021-05-03T08:57:38.841Z) and Timestamp < datetime(2021-05-03T09:02:38.841Z) | order by ' - 'Timestamp desc | where (InitiatingProcessCreationTime >= datetime(2019-09-04T09:29:29.0882Z)) or (' - 'InitiatingProcessParentCreationTime >= datetime(2019-09-04T09:29:29.0882Z))),(find withsource = ' - 'TableName in (DeviceFileEvents) where Timestamp >= datetime(2021-05-03T08:57:38.841Z) and Timestamp < ' - 'datetime(2021-05-03T09:02:38.841Z) | order by Timestamp desc | where (InitiatingProcessCreationTime >= ' - 'datetime(2019-09-04T09:29:29.0882Z)) or (InitiatingProcessParentCreationTime >= datetime(' - '2019-09-04T09:29:29.0882Z)))'] - queries = _remove_timestamp_from_query(queries) + ('union (find withsource = TableName in (DeviceProcessEvents) where Timestamp ' + '>= <> and Timestamp < <> | order by Timestamp desc | ' + 'where ProcessCreationTime >= <>),(find withsource = TableName in ' + '(DeviceEvents) where Timestamp >= <> and Timestamp < ' + '<> | order by Timestamp desc | where ProcessCreationTime >= ' + '<>),(find withsource = TableName in (DeviceNetworkEvents) where ' + 'Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where InitiatingProcessCreationTime >= <>),(find ' + 'withsource = TableName in (DeviceRegistryEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where InitiatingProcessCreationTime >= <>),(find withsource = ' + 'TableName in (DeviceFileEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + 'InitiatingProcessCreationTime >= <>),(find withsource = TableName ' + 'in (DeviceImageLoadEvents) where Timestamp >= <> and Timestamp < ' + '<> | order by Timestamp desc | where ' + 'InitiatingProcessCreationTime >= <>)')] self._test_query_assertions(query, queries) def test_lt_eq_datetime_comp_exp(self): @@ -161,10 +190,11 @@ def test_lt_eq_datetime_comp_exp(self): query['queries'] = _remove_timestamp_from_query(query['queries']) queries = [ - '(find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(' - '2021-05-03T08:58:07.130Z) and Timestamp < datetime(2021-05-03T09:03:07.130Z) | order by Timestamp desc ' - '| where LocalPort < 443)'] - queries = _remove_timestamp_from_query(queries) + ('union (find withsource = TableName in (DeviceNetworkEvents) where Timestamp ' + '>= <> and Timestamp < <> | order by Timestamp desc | ' + 'where LocalPort < 443),(find withsource = TableName in (DeviceEvents) where ' + 'Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where LocalPort < 443)')] self._test_query_assertions(query, queries) def test_noteq_comp_exp(self): @@ -174,17 +204,24 @@ def test_noteq_comp_exp(self): query['queries'] = _remove_timestamp_from_query(query['queries']) queries = [ - 'union (find withsource = TableName in (DeviceProcessEvents) where Timestamp >= datetime(' - '2019-09-10T08:43:10.003Z) and Timestamp < datetime(2019-09-23T10:43:10.453Z) | order by Timestamp desc ' - '| where (FileName !~ "consent.exe") or (InitiatingProcessFileName !~ "consent.exe") or (' - 'InitiatingProcessParentFileName !~ "consent.exe")),(find withsource = TableName in (DeviceNetworkEvents) ' - ' where Timestamp >= datetime(2019-09-10T08:43:10.003Z) and Timestamp < datetime(' - '2019-09-23T10:43:10.453Z) | order by Timestamp desc | where (InitiatingProcessFileName !~ ' - '"consent.exe") or (InitiatingProcessParentFileName !~ "consent.exe")),(find withsource = TableName in (' - 'DeviceRegistryEvents) where Timestamp >= datetime(2019-09-10T08:43:10.003Z) and Timestamp < datetime(' - '2019-09-23T10:43:10.453Z) | order by Timestamp desc | where (InitiatingProcessFileName !~ ' - '"consent.exe") or (InitiatingProcessParentFileName !~ "consent.exe"))'] - queries = _remove_timestamp_from_query(queries) + ('union (find withsource = TableName in (DeviceProcessEvents) where Timestamp ' + '>= <> and Timestamp < <> | order by Timestamp desc | ' + 'where (FileName !~ "consent.exe") or (InitiatingProcessFileName !~ ' + '"consent.exe")),(find withsource = TableName in (DeviceEvents) where ' + 'Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where (FileName !~ "consent.exe") or ' + '(InitiatingProcessFileName !~ "consent.exe")),(find withsource = TableName ' + 'in (DeviceFileEvents) where Timestamp >= <> and Timestamp < ' + '<> | order by Timestamp desc | where InitiatingProcessFileName ' + '!~ "consent.exe"),(find withsource = TableName in (DeviceNetworkEvents) ' + 'where Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where InitiatingProcessFileName !~ "consent.exe"),(find ' + 'withsource = TableName in (DeviceRegistryEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where InitiatingProcessFileName !~ "consent.exe"),(find withsource = ' + 'TableName in (DeviceImageLoadEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + 'InitiatingProcessFileName !~ "consent.exe")')] self._test_query_assertions(query, queries) def test_like_comp_exp(self): @@ -193,20 +230,30 @@ def test_like_comp_exp(self): query['queries'] = _remove_timestamp_from_query(query['queries']) queries = [ - 'union (find withsource = TableName in (DeviceFileEvents) where Timestamp >= datetime(' - '2021-05-03T08:58:56.445Z) and Timestamp < datetime(2021-05-03T09:03:56.445Z) | order by Timestamp desc ' - '| where (FileName contains "upd") or (InitiatingProcessFileName contains "upd") or (' - 'InitiatingProcessParentFileName contains "upd")),(find withsource = TableName in (DeviceProcessEvents) ' - 'where Timestamp >= datetime(2021-05-03T08:58:56.445Z) and Timestamp < datetime(2021-05-03T09:03:56.445Z) ' - ' | order by Timestamp desc | where (FileName contains "upd") or (InitiatingProcessFileName contains ' - '"upd") or (InitiatingProcessParentFileName contains "upd")),(find withsource = TableName in (' - 'DeviceNetworkEvents) where Timestamp >= datetime(2021-05-03T08:58:56.445Z) and Timestamp < datetime(' - '2021-05-03T09:03:56.445Z) | order by Timestamp desc | where (InitiatingProcessFileName contains "upd") ' - 'or (InitiatingProcessParentFileName contains "upd")),(find withsource = TableName in (' - 'DeviceRegistryEvents) where Timestamp >= datetime(2021-05-03T08:58:56.445Z) and Timestamp < datetime(' - '2021-05-03T09:03:56.445Z) | order by Timestamp desc | where (InitiatingProcessFileName contains "upd") ' - 'or (InitiatingProcessParentFileName contains "upd"))'] - queries = _remove_timestamp_from_query(queries) + ('union (find withsource = TableName in (DeviceFileEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where (FileName contains "upd") or (InitiatingProcessFileName contains ' + '"upd") or (InitiatingProcessParentFileName contains "upd")),(find withsource ' + '= TableName in (DeviceProcessEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where (FileName ' + 'contains "upd") or (InitiatingProcessFileName contains "upd") or ' + '(InitiatingProcessParentFileName contains "upd")),(find withsource = ' + 'TableName in (DeviceNetworkEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + '(InitiatingProcessFileName contains "upd") or ' + '(InitiatingProcessParentFileName contains "upd")),(find withsource = ' + 'TableName in (DeviceRegistryEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + '(InitiatingProcessFileName contains "upd") or ' + '(InitiatingProcessParentFileName contains "upd")),(find withsource = ' + 'TableName in (DeviceEvents) where Timestamp >= <> and Timestamp ' + '< <> | order by Timestamp desc | where (FileName contains "upd") ' + 'or (InitiatingProcessFileName contains "upd") or ' + '(InitiatingProcessParentFileName contains "upd")),(find withsource = ' + 'TableName in (DeviceImageLoadEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where (FileName ' + 'contains "upd") or (InitiatingProcessFileName contains "upd") or ' + '(InitiatingProcessParentFileName contains "upd"))')] self._test_query_assertions(query, queries) def test_matches_comp_exp(self): @@ -214,22 +261,31 @@ def test_matches_comp_exp(self): query = translation.translate('msatp', 'query', '{}', stix_pattern) query['queries'] = _remove_timestamp_from_query(query['queries']) - queries = ['union (find withsource = TableName in (DeviceFileEvents) where Timestamp >= datetime(' - '2021-05-03T08:59:41.105Z) and Timestamp < datetime(2021-05-03T09:04:41.105Z) | order by ' - 'Timestamp desc | where (FileName matches regex"(^chr)") or (InitiatingProcessFileName matches ' - 'regex"(^chr)") or (InitiatingProcessParentFileName matches regex"(^chr)")),(find withsource = ' - 'TableName in (DeviceProcessEvents) where Timestamp >= datetime(2021-05-03T08:59:41.105Z) and ' - 'Timestamp < datetime(2021-05-03T09:04:41.105Z) | order by Timestamp desc | where (FileName ' - 'matches regex"(^chr)") or (InitiatingProcessFileName matches regex"(^chr)") or (' - 'InitiatingProcessParentFileName matches regex"(^chr)")),(find withsource = TableName in (' - 'DeviceNetworkEvents) where Timestamp >= datetime(2021-05-03T08:59:41.105Z) and Timestamp < ' - 'datetime(2021-05-03T09:04:41.105Z) | order by Timestamp desc | where (InitiatingProcessFileName ' - 'matches regex"(^chr)") or (InitiatingProcessParentFileName matches regex"(^chr)")),' - '(find withsource = TableName in (DeviceRegistryEvents) where Timestamp >= datetime(' - '2021-05-03T08:59:41.105Z) and Timestamp < datetime(2021-05-03T09:04:41.105Z) | order by ' - 'Timestamp desc | where (InitiatingProcessFileName matches regex"(^chr)") or (' - 'InitiatingProcessParentFileName matches regex"(^chr)"))'] - queries = _remove_timestamp_from_query(queries) + queries = [('union (find withsource = TableName in (DeviceFileEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where (FileName matches regex"(^chr)") or (InitiatingProcessFileName matches ' + 'regex"(^chr)") or (InitiatingProcessParentFileName matches ' + 'regex"(^chr)")),(find withsource = TableName in (DeviceProcessEvents) where ' + 'Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where (FileName matches regex"(^chr)") or ' + '(InitiatingProcessFileName matches regex"(^chr)") or ' + '(InitiatingProcessParentFileName matches regex"(^chr)")),(find withsource = ' + 'TableName in (DeviceNetworkEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + '(InitiatingProcessFileName matches regex"(^chr)") or ' + '(InitiatingProcessParentFileName matches regex"(^chr)")),(find withsource = ' + 'TableName in (DeviceRegistryEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + '(InitiatingProcessFileName matches regex"(^chr)") or ' + '(InitiatingProcessParentFileName matches regex"(^chr)")),(find withsource = ' + 'TableName in (DeviceEvents) where Timestamp >= <> and Timestamp ' + '< <> | order by Timestamp desc | where (FileName matches ' + 'regex"(^chr)") or (InitiatingProcessFileName matches regex"(^chr)") or ' + '(InitiatingProcessParentFileName matches regex"(^chr)")),(find withsource = ' + 'TableName in (DeviceImageLoadEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where (FileName ' + 'matches regex"(^chr)") or (InitiatingProcessFileName matches regex"(^chr)") ' + 'or (InitiatingProcessParentFileName matches regex"(^chr)"))')] self._test_query_assertions(query, queries) def test_in_comp_exp(self): @@ -238,26 +294,25 @@ def test_in_comp_exp(self): query['queries'] = _remove_timestamp_from_query(query['queries']) queries = [ - 'union (find withsource = TableName in (DeviceProcessEvents) where Timestamp >= datetime(' - '2021-05-03T09:00:15.898Z) and Timestamp < datetime(2021-05-03T09:05:15.898Z) | order by Timestamp desc ' - '| where (ProcessCreationTime in~ (datetime(2019-09-04T09:29:29.0882Z), datetime(' - '2019-09-04T09:29:29.0881372Z))) or (InitiatingProcessParentCreationTime in~ (datetime(' - '2019-09-04T09:29:29.0882Z), datetime(2019-09-04T09:29:29.0881372Z)))),(find withsource = TableName in (' - 'DeviceNetworkEvents) where Timestamp >= datetime(2021-05-03T09:00:15.898Z) and Timestamp < datetime(' - '2021-05-03T09:05:15.898Z) | order by Timestamp desc | where (InitiatingProcessCreationTime in~ (' - 'datetime(2019-09-04T09:29:29.0882Z), datetime(2019-09-04T09:29:29.0881372Z))) or (' - 'InitiatingProcessParentCreationTime in~ (datetime(2019-09-04T09:29:29.0882Z), ' - 'datetime(2019-09-04T09:29:29.0881372Z)))),(find withsource = TableName in (DeviceRegistryEvents) where ' - 'Timestamp >= datetime(2021-05-03T09:00:15.898Z) and Timestamp < datetime(2021-05-03T09:05:15.898Z) | ' - 'order by Timestamp desc | where (InitiatingProcessCreationTime in~ (datetime(2019-09-04T09:29:29.0882Z), ' - 'datetime(2019-09-04T09:29:29.0881372Z))) or (InitiatingProcessParentCreationTime in~ (datetime(' - '2019-09-04T09:29:29.0882Z), datetime(2019-09-04T09:29:29.0881372Z)))),(find withsource = TableName in (' - 'DeviceFileEvents) where Timestamp >= datetime(2021-05-03T09:00:15.898Z) and Timestamp < datetime(' - '2021-05-03T09:05:15.898Z) | order by Timestamp desc | where (InitiatingProcessCreationTime in~ (' - 'datetime(2019-09-04T09:29:29.0882Z), datetime(2019-09-04T09:29:29.0881372Z))) or (' - 'InitiatingProcessParentCreationTime in~ (datetime(2019-09-04T09:29:29.0882Z), ' - 'datetime(2019-09-04T09:29:29.0881372Z))))'] - queries = _remove_timestamp_from_query(queries) + ('union (find withsource = TableName in (DeviceProcessEvents) where Timestamp ' + '>= <> and Timestamp < <> | order by Timestamp desc | ' + 'where ProcessCreationTime in~ (<>, <>)),(find ' + 'withsource = TableName in (DeviceEvents) where Timestamp >= <> ' + 'and Timestamp < <> | order by Timestamp desc | where ' + 'ProcessCreationTime in~ (<>, <>)),(find withsource = ' + 'TableName in (DeviceNetworkEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + 'InitiatingProcessCreationTime in~ (<>, <>)),(find ' + 'withsource = TableName in (DeviceRegistryEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where InitiatingProcessCreationTime in~ (<>, ' + '<>)),(find withsource = TableName in (DeviceFileEvents) where ' + 'Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where InitiatingProcessCreationTime in~ (<>, ' + '<>)),(find withsource = TableName in (DeviceImageLoadEvents) ' + 'where Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where InitiatingProcessCreationTime in~ (<>, ' + '<>))')] self._test_query_assertions(query, queries) def test_comb_comparison_exp(self): @@ -265,27 +320,49 @@ def test_comb_comparison_exp(self): "t'2019-09-10T08:43:10.003Z' STOP t'2019-09-23T10:43:10.453Z'" query = translation.translate('msatp', 'query', '{}', stix_pattern) query['queries'] = _remove_timestamp_from_query(query['queries']) + # process name mapping: + # DeviceEvents, DeviceProcessEvents - FileName or InitiatingProcessFileName + # DeviceFileEvents, DeviceNetworkEvents, DeviceRegistryEvents, DeviceImageLoadEvents - InitiatingProcessFileName + # file name mapping: + # DeviceEvents, DeviceProcessEvents, DeviceFileEvents - + # FileName or InitiatingProcessFileName or InitiatingProcessParentFileName + # DeviceNetworkEvents, DeviceRegistryEvents, DeviceImageLoadEvents - + # InitiatingProcessFileName or InitiatingProcessParentFileName queries = [ - 'union (find withsource = TableName in (DeviceFileEvents) where Timestamp >= datetime(' - '2019-09-10T08:43:10.003Z) and Timestamp < datetime(2019-09-23T10:43:10.453Z) | order by Timestamp desc ' - '| where ((FileName =~ "updater.exe") or (InitiatingProcessFileName =~ "updater.exe") or (' - 'InitiatingProcessParentFileName =~ "updater.exe"))),(find withsource = TableName in (' - 'DeviceProcessEvents) where Timestamp >= datetime(2019-09-10T08:43:10.003Z) and Timestamp < datetime(' - '2019-09-23T10:43:10.453Z) | order by Timestamp desc | where ((FileName in~ ("consent.exe", "reg.exe")) ' - 'or (InitiatingProcessFileName in~ ("consent.exe", "reg.exe")) or (InitiatingProcessParentFileName in~ (' - '"consent.exe", "reg.exe"))) or ((FileName =~ "updater.exe") or (InitiatingProcessFileName =~ ' - '"updater.exe") or (InitiatingProcessParentFileName =~ "updater.exe"))),(find withsource = TableName in (' - 'DeviceNetworkEvents) where Timestamp >= datetime(2019-09-10T08:43:10.003Z) and Timestamp < datetime(' - '2019-09-23T10:43:10.453Z) | order by Timestamp desc | where ((InitiatingProcessFileName in~ (' - '"consent.exe", "reg.exe")) or (InitiatingProcessParentFileName in~ ("consent.exe", "reg.exe"))) or ((' - 'InitiatingProcessFileName =~ "updater.exe") or (InitiatingProcessParentFileName =~ "updater.exe"))),' - '(find withsource = TableName in (DeviceRegistryEvents) where Timestamp >= datetime(' - '2019-09-10T08:43:10.003Z) and Timestamp < datetime(2019-09-23T10:43:10.453Z) | order by Timestamp desc ' - '| where ((InitiatingProcessFileName in~ ("consent.exe", "reg.exe")) or (InitiatingProcessParentFileName ' - 'in~ ("consent.exe", "reg.exe"))) or ((InitiatingProcessFileName =~ "updater.exe") or (' - 'InitiatingProcessParentFileName =~ "updater.exe")))'] - queries = _remove_timestamp_from_query(queries) + (('union (find withsource = TableName in (DeviceFileEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where (InitiatingProcessFileName in~ ("consent.exe", "reg.exe")) or ' + '((FileName =~ "updater.exe") or (InitiatingProcessFileName =~ "updater.exe") ' + 'or (InitiatingProcessParentFileName =~ "updater.exe"))),(find withsource = ' + 'TableName in (DeviceProcessEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ((FileName in~ ' + '("consent.exe", "reg.exe")) or (InitiatingProcessFileName in~ ' + '("consent.exe", "reg.exe"))) or ((FileName =~ "updater.exe") or ' + '(InitiatingProcessFileName =~ "updater.exe") or ' + '(InitiatingProcessParentFileName =~ "updater.exe"))),(find withsource = ' + 'TableName in (DeviceNetworkEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + '(InitiatingProcessFileName in~ ("consent.exe", "reg.exe")) or ' + '((InitiatingProcessFileName =~ "updater.exe") or ' + '(InitiatingProcessParentFileName =~ "updater.exe"))),(find withsource = ' + 'TableName in (DeviceRegistryEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + '(InitiatingProcessFileName in~ ("consent.exe", "reg.exe")) or ' + '((InitiatingProcessFileName =~ "updater.exe") or ' + '(InitiatingProcessParentFileName =~ "updater.exe"))),(find withsource = ' + 'TableName in (DeviceEvents) where Timestamp >= <> and Timestamp ' + '< <> | order by Timestamp desc | where ((FileName in~ ' + '("consent.exe", "reg.exe")) or (InitiatingProcessFileName in~ ' + '("consent.exe", "reg.exe"))) or ((FileName =~ "updater.exe") or ' + '(InitiatingProcessFileName =~ "updater.exe") or ' + '(InitiatingProcessParentFileName =~ "updater.exe"))),(find withsource = ' + 'TableName in (DeviceImageLoadEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + '(InitiatingProcessFileName in~ ("consent.exe", "reg.exe")) or ((FileName =~ ' + '"updater.exe") or (InitiatingProcessFileName =~ "updater.exe") or ' + '(InitiatingProcessParentFileName =~ "updater.exe")))')) + ] self._test_query_assertions(query, queries) def test_comb_comparison_exp_1(self): @@ -295,18 +372,26 @@ def test_comb_comparison_exp_1(self): query['queries'] = _remove_timestamp_from_query(query['queries']) queries = [ - 'union (find withsource = TableName in (DeviceProcessEvents) where Timestamp >= datetime(' - '2019-09-10T08:43:10.003Z) and Timestamp < datetime(2019-09-23T10:43:10.453Z) | order by Timestamp desc ' - '| where (not ((FileName =~ "python.exe") or (InitiatingProcessFileName =~ "python.exe") or (' - 'InitiatingProcessParentFileName =~ "python.exe")))),(find withsource = TableName in (' - 'DeviceNetworkEvents) where Timestamp >= datetime(2019-09-10T08:43:10.003Z) and Timestamp < datetime(' - '2019-09-23T10:43:10.453Z) | order by Timestamp desc | where (tostring(LocalPort) =~ "454") or (not ((' - 'InitiatingProcessFileName =~ "python.exe") or (InitiatingProcessParentFileName =~ "python.exe")))),' - '(find withsource = TableName in (DeviceRegistryEvents) where Timestamp >= datetime(' - '2019-09-10T08:43:10.003Z) and Timestamp < datetime(2019-09-23T10:43:10.453Z) | order by Timestamp desc ' - '| where (not ((InitiatingProcessFileName =~ "python.exe") or (InitiatingProcessParentFileName =~ ' - '"python.exe"))))'] - queries = _remove_timestamp_from_query(queries) + (('union (find withsource = TableName in (DeviceProcessEvents) where Timestamp ' + '>= <> and Timestamp < <> | order by Timestamp desc | ' + 'where (not ((FileName =~ "python.exe") or (InitiatingProcessFileName =~ ' + '"python.exe")))),(find withsource = TableName in (DeviceEvents) where ' + 'Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where (tostring(LocalPort) =~ "454") or (not ((FileName =~ ' + '"python.exe") or (InitiatingProcessFileName =~ "python.exe")))),(find ' + 'withsource = TableName in (DeviceFileEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where (not (InitiatingProcessFileName =~ "python.exe"))),(find withsource = ' + 'TableName in (DeviceNetworkEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + '(tostring(LocalPort) =~ "454") or (not (InitiatingProcessFileName =~ ' + '"python.exe"))),(find withsource = TableName in (DeviceRegistryEvents) ' + 'where Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where (not (InitiatingProcessFileName =~ ' + '"python.exe"))),(find withsource = TableName in (DeviceImageLoadEvents) ' + 'where Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where (not (InitiatingProcessFileName =~ "python.exe")))')) + ] self._test_query_assertions(query, queries) # check about partesis @@ -315,30 +400,39 @@ def test_comb_observation_obs(self): query = translation.translate('msatp', 'query', '{}', stix_pattern) query['queries'] = _remove_timestamp_from_query(query['queries']) - queries = [ - 'union (find withsource = TableName in (DeviceProcessEvents) where Timestamp >= datetime(' - '2021-05-03T09:01:44.549Z) and Timestamp < datetime(2021-05-03T09:06:44.549Z) | order by Timestamp desc ' - '| where ((FileName matches regex"(upd.ter.exe$)") or (InitiatingProcessFileName matches regex"(' - 'upd.ter.exe$)") or (InitiatingProcessParentFileName matches regex"(upd.ter.exe$)")) or ((tostring(' - 'ProcessCreationTime) == datetime(2019-09-04T09:29:29.0882Z)) or (tostring(' - 'InitiatingProcessParentCreationTime) == datetime(2019-09-04T09:29:29.0882Z)))),(find withsource = ' - 'TableName in (DeviceNetworkEvents) where Timestamp >= datetime(2021-05-03T09:01:44.549Z) and Timestamp ' - '< datetime(2021-05-03T09:06:44.549Z) | order by Timestamp desc | where ((InitiatingProcessFileName ' - 'matches regex"(upd.ter.exe$)") or (InitiatingProcessParentFileName matches regex"(upd.ter.exe$)")) or ((' - 'tostring(InitiatingProcessCreationTime) == datetime(2019-09-04T09:29:29.0882Z)) or (tostring(' - 'InitiatingProcessParentCreationTime) == datetime(2019-09-04T09:29:29.0882Z)))),(find withsource = ' - 'TableName in (DeviceRegistryEvents) where Timestamp >= datetime(2021-05-03T09:01:44.549Z) and Timestamp ' - '< datetime(2021-05-03T09:06:44.549Z) | order by Timestamp desc | where ((InitiatingProcessFileName ' - 'matches regex"(upd.ter.exe$)") or (InitiatingProcessParentFileName matches regex"(upd.ter.exe$)")) or ((' - 'tostring(InitiatingProcessCreationTime) == datetime(2019-09-04T09:29:29.0882Z)) or (tostring(' - 'InitiatingProcessParentCreationTime) == datetime(2019-09-04T09:29:29.0882Z)))),(find withsource = ' - 'TableName in (DeviceFileEvents) where Timestamp >= datetime(2021-05-03T09:01:44.549Z) and Timestamp < ' - 'datetime(2021-05-03T09:06:44.549Z) | order by Timestamp desc | where ((FileName matches regex"(' - 'upd.ter.exe$)") or (InitiatingProcessFileName matches regex"(upd.ter.exe$)") or (' - 'InitiatingProcessParentFileName matches regex"(upd.ter.exe$)")) or ((tostring(' - 'InitiatingProcessCreationTime) == datetime(2019-09-04T09:29:29.0882Z)) or (tostring(' - 'InitiatingProcessParentCreationTime) == datetime(2019-09-04T09:29:29.0882Z))))'] - queries = _remove_timestamp_from_query(queries) + queries = [('union (find withsource = TableName in (DeviceProcessEvents) where Timestamp ' + '>= <> and Timestamp < <> | order by Timestamp desc | ' + 'where ((FileName matches regex"(upd.ter.exe$)") or ' + '(InitiatingProcessFileName matches regex"(upd.ter.exe$)") or ' + '(InitiatingProcessParentFileName matches regex"(upd.ter.exe$)")) or ' + '(tostring(ProcessCreationTime) == <>)),(find withsource = ' + 'TableName in (DeviceEvents) where Timestamp >= <> and Timestamp ' + '< <> | order by Timestamp desc | where ((FileName matches ' + 'regex"(upd.ter.exe$)") or (InitiatingProcessFileName matches ' + 'regex"(upd.ter.exe$)") or (InitiatingProcessParentFileName matches ' + 'regex"(upd.ter.exe$)")) or (tostring(ProcessCreationTime) == ' + '<>)),(find withsource = TableName in (DeviceNetworkEvents) where ' + 'Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where ((InitiatingProcessFileName matches ' + 'regex"(upd.ter.exe$)") or (InitiatingProcessParentFileName matches ' + 'regex"(upd.ter.exe$)")) or (tostring(InitiatingProcessCreationTime) == ' + '<>)),(find withsource = TableName in (DeviceRegistryEvents) ' + 'where Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where ((InitiatingProcessFileName matches ' + 'regex"(upd.ter.exe$)") or (InitiatingProcessParentFileName matches ' + 'regex"(upd.ter.exe$)")) or (tostring(InitiatingProcessCreationTime) == ' + '<>)),(find withsource = TableName in (DeviceFileEvents) where ' + 'Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where ((FileName matches regex"(upd.ter.exe$)") or ' + '(InitiatingProcessFileName matches regex"(upd.ter.exe$)") or ' + '(InitiatingProcessParentFileName matches regex"(upd.ter.exe$)")) or ' + '(tostring(InitiatingProcessCreationTime) == <>)),(find withsource ' + '= TableName in (DeviceImageLoadEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ((FileName ' + 'matches regex"(upd.ter.exe$)") or (InitiatingProcessFileName matches ' + 'regex"(upd.ter.exe$)") or (InitiatingProcessParentFileName matches ' + 'regex"(upd.ter.exe$)")) or (tostring(InitiatingProcessCreationTime) == ' + '<>))')] self._test_query_assertions(query, queries) def test_and_op_comb_comparison_exp(self): @@ -346,13 +440,68 @@ def test_and_op_comb_comparison_exp(self): query = translation.translate('msatp', 'query', '{}', stix_pattern) query['queries'] = _remove_timestamp_from_query(query['queries']) + # ip can appear in either DeviceNetworkEvents or DeviceEvents. process can appear in any table. + # after AND these two tables are left. + # process name can appear in FileName or InititatingProcessFileName in DeviceEvents (this is the only table in + # which a process may appear as FileName or may not. + # process name can appear in InitiatingProcessFileName in DeviceNetworkEvents queries = [ - '(find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(' - '2021-05-05T12:45:04.124Z) and Timestamp < datetime(2021-05-05T12:50:04.124Z) | order by Timestamp desc ' - '| where ((LocalIP =~ "9.147.31.113") or (RemoteIP =~ "9.147.31.113")) and ((InitiatingProcessFileName =~ ' - '"python3") or (InitiatingProcessParentFileName =~ "python3")))'] - queries = _remove_timestamp_from_query(queries) + (('union (find withsource = TableName in (DeviceEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where ((RemoteIP =~ "9.147.31.113") or (LocalIP =~ "9.147.31.113")) and ' + '((FileName =~ "python3") or (InitiatingProcessFileName =~ "python3"))),(find ' + 'withsource = TableName in (DeviceNetworkEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where ((LocalIP =~ "9.147.31.113") or (RemoteIP =~ "9.147.31.113")) and ' + '(InitiatingProcessFileName =~ "python3"))')) + ] self._test_query_assertions(query, queries) + def test_registry_key(self): + stix_pattern = r"[windows-registry-key:key = 'HKEY_LOCAL_MACHINE\\SOFTWARE\\Microsoft\\Windows Advanced Threat Protection']" + query = translation.translate('msatp', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + # ip can appear in either DeviceNetworkEvents or DeviceEvents. process can appear in any table. + # after AND these two tables are left. + # process name can appear in FileName or InititatingProcessFileName in DeviceEvents (this is the only table in + # which a process may appear as FileName or may not. + # process name can appear in InitiatingProcessFileName in DeviceNetworkEvents + queries = [ + ('union (find withsource = TableName in (DeviceRegistryEvents) where ' + 'Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where RegistryKey =~ ' + '"HKEY_LOCAL_MACHINE\\\\SOFTWARE\\\\Microsoft\\\\Windows Advanced Threat ' + 'Protection"),(find withsource = TableName in (DeviceEvents) where Timestamp ' + '>= <> and Timestamp < <> | order by Timestamp desc | ' + 'where RegistryKey =~ "HKEY_LOCAL_MACHINE\\\\SOFTWARE\\\\Microsoft\\\\Windows ' + 'Advanced Threat Protection")') + ] + self._test_query_assertions(query, queries) + + def test_config_no_info(self): + stix_pattern = "[process:name = 'consent.exe']" + query = translation.translate('msatp', 'query', '{}', stix_pattern) + query['queries'] = _remove_timestamp_from_query(query['queries']) + + queries = [ + ('union (find withsource = TableName in (DeviceProcessEvents) where Timestamp ' + '>= <> and Timestamp < <> | order by Timestamp desc | ' + 'where (FileName =~ "consent.exe") or (InitiatingProcessFileName =~ ' + '"consent.exe")),(find withsource = TableName in (DeviceEvents) where ' + 'Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where (FileName =~ "consent.exe") or ' + '(InitiatingProcessFileName =~ "consent.exe")),(find withsource = TableName ' + 'in (DeviceFileEvents) where Timestamp >= <> and Timestamp < ' + '<> | order by Timestamp desc | where InitiatingProcessFileName ' + '=~ "consent.exe"),(find withsource = TableName in (DeviceNetworkEvents) ' + 'where Timestamp >= <> and Timestamp < <> | order by ' + 'Timestamp desc | where InitiatingProcessFileName =~ "consent.exe"),(find ' + 'withsource = TableName in (DeviceRegistryEvents) where Timestamp >= ' + '<> and Timestamp < <> | order by Timestamp desc | ' + 'where InitiatingProcessFileName =~ "consent.exe"),(find withsource = ' + 'TableName in (DeviceImageLoadEvents) where Timestamp >= <> and ' + 'Timestamp < <> | order by Timestamp desc | where ' + 'InitiatingProcessFileName =~ "consent.exe")')] + self._test_query_assertions(query, queries) \ No newline at end of file diff --git a/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_transformers.py b/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_transformers.py new file mode 100644 index 000000000..ede9517c6 --- /dev/null +++ b/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_transformers.py @@ -0,0 +1,71 @@ +import unittest + +from stix_shifter_modules.msatp.stix_translation.transformers import MsatpToTimestamp, MsatpToRegistryValue, \ + FormatMacList, IfValidUrl, GetDomainName, ToFileName, ToDirectory, ToMSATPDirectoryPath, SeverityToNumericVal, \ + Alert, JsonToString + + +class TestMsatpTransformers(unittest.TestCase): + + def test_transform_msatp_to_timestamp(self): + val = MsatpToTimestamp.transform('2023-03-12T21:56:34.0646516Z') + assert val == '2023-03-12T21:56:34.064Z' + + def test_msatp_to_registry_value(self): + data = [{"RegistryValueType": "Binary", + "RegistryValueName": "FailureActions", + "RegistryValueData": ""}] + values = MsatpToRegistryValue.transform(data) + assert len(values) == 1 + val = values[0] + assert val.get("name") == "FailureActions" + assert val.get("data") == "" + assert val.get("data_type") == "REG_BINARY" + + def test_format_mac_list(self): + data = ['B0-4F-13-0F-E1-7B'] + macs = FormatMacList.transform(data) + assert len(macs) == 1 + assert macs[0] == 'b0:4f:13:0f:e1:7b' + + def test_multiple_format_mac_list(self): + data = ['11-22-33-44-55-66', '11-22-AA-BB-CC-DD'] + macs = FormatMacList.transform(data) + assert len(macs) == 2 + assert macs[0] == '11:22:33:44:55:66' + assert macs[1] == '11:22:aa:bb:cc:dd' + + def test_is_valid_url(self): + assert IfValidUrl.transform("ibm.com") == "" + assert IfValidUrl.transform("http://ibm.com") == "http://ibm.com" + assert IfValidUrl.transform("https://ibm.com") == "https://ibm.com" + assert IfValidUrl.transform("http://ibm.com/a/a/a?a=a") == "http://ibm.com/a/a/a?a=a" + assert IfValidUrl.transform("ibm.com/a/a/a?a=a") == "" + + def test_get_domain_name(self): + assert GetDomainName.transform("ibm.com") == "ibm.com" + assert GetDomainName.transform("www.ibm.com") == "www.ibm.com" + assert GetDomainName.transform("http://ibm.com") == "ibm.com" + assert GetDomainName.transform("https://ibm.com/a/a/a?a=a") == "ibm.com" + assert GetDomainName.transform("http://xn--diseolatinoamericano-66b.com/") == "xn--diseolatinoamericano-66b.com" + + def test_to_file_name(self): + assert ToFileName.transform("c:\\a\\a\\a.exe") == "a.exe" + + def test_to_directory(self): + assert ToDirectory.transform("c:\\a\\a\\a.exe") == "c:\\a\\a" + + def test_to_msatp_directory(self): + assert ToMSATPDirectoryPath.transform("c:\\a\\a\\a.exe") == "c:\\a\\a" + + def test_severity_to_numeric_val(self): + assert SeverityToNumericVal.transform("high") == 99 + assert SeverityToNumericVal.transform("medium") == 66 + assert SeverityToNumericVal.transform("low") == 33 + + def test_alert(self): + assert Alert.transform("") == "alert" + + def test_json_to_string(self): + assert JsonToString.transform("{") == "{" + assert JsonToString.transform('{ "a": 1, "b": 2 }') == "a: 1, b: 2" diff --git a/stix_shifter_modules/msatp/tests/stix_transmission/test_connector_post_processing.py b/stix_shifter_modules/msatp/tests/stix_transmission/test_connector_post_processing.py new file mode 100644 index 000000000..9a63331aa --- /dev/null +++ b/stix_shifter_modules/msatp/tests/stix_transmission/test_connector_post_processing.py @@ -0,0 +1,420 @@ +import json + +from stix_shifter_modules.msatp.stix_transmission.connector_post_processing import merge_alerts, \ + remove_duplicate_and_empty_fields, get_table_name, ConnectorPostProcessing, unify_alert_fields, \ + organize_registry_data, organize_ips, create_event_link, remove_duplicate_ips +from stix_shifter_modules.msatp.tests.test_utils import all_keys_in_object +from unittest.mock import patch +import unittest +from tests.utils.async_utils import get_adal_mock_response + + +@patch('stix_shifter_modules.msatp.stix_transmission.connector.adal.AuthenticationContext') +class TestMSATPConnectorPostProcessing(unittest.TestCase): + def config(self): + return { + "auth": { + "tenant": "bla", + "clientId": "bla", + "clientSecret": "bla" + }, + "includeAlerts": True, + "includeHostOs": True, + "includeNetworkInfo": True + } + + def connection(self): + return { + "host": "hostbla", + "port": 8080, + "selfSignedCert": "cert", + "options": { + + } + } + + def test_merge_alert_events(self, mock_adal_auth): + data = [ + { + 'TableName': 'DeviceAlertEvents', + 'AlertId': 'da123456789012345678_-1111111111', + 'Timestamp': '2023-03-17T16:59:12.3036191Z', + 'DeviceId': '1234567890abcdef1234567890abcdef12345678', + 'DeviceName': 'host.test.com', + 'Severity': 'High', + 'Category': 'InitialAccess', + 'Title': 'Suspicious URL clicked', + 'FileName': '', + 'SHA1': '', + 'RemoteUrl': 'https://malicious.com', + 'RemoteIP': '', + 'AttackTechniques': '["Spearphishing Link (T1566.002)"]', + 'ReportId': 1234, + 'Table': 'DeviceEvents', + 'rn': 1 + }, + { + 'TableName': 'DeviceAlertEvents', + 'AlertId': 'da123456789012345678_-1111111111', + 'Timestamp': '2023-03-17T16:59:12.3036191Z', + 'DeviceId': '1234567890abcdef1234567890abcdef12345678', + 'DeviceName': 'host.test.com', + 'Severity': 'High', + 'Category': 'InitialAccess', + 'Title': 'Suspicious URL clicked', + 'FileName': '', + 'SHA1': '', + 'RemoteUrl': '', + 'RemoteIP': '9.9.9.9', + 'AttackTechniques': '["Spearphishing Link (T1566.002)"]', + 'ReportId': 1234, + 'Table': 'DeviceEvents', + 'rn': 1 + }, + { + 'TableName': 'DeviceAlertEvents', + 'AlertId': 'da123456789012345678_-1111111111', + 'Timestamp': '2023-03-17T16:59:12.3036191Z', + 'DeviceId': '1234567890abcdef1234567890abcdef12345678', + 'DeviceName': 'host.test.com', + 'Severity': 'High', + 'Category': 'InitialAccess', + 'Title': 'Suspicious URL clicked', + 'FileName': 'msedge.exe', + 'SHA1': '', + 'RemoteUrl': '', + 'RemoteIP': '', + 'AttackTechniques': '["Spearphishing Link (T1566.002)"]', + 'ReportId': 1234, + 'Table': 'DeviceEvents', + 'rn': 1 + }, + { + 'TableName': 'DeviceProcessEvents', + 'Timestamp': '2023-03-17T16:59:12.3036191Z', + 'DeviceId': '1234567890abcdef1234567890abcdef12345678', + 'DeviceName': 'host.test.com', + }, + { + 'TableName': 'DeviceProcessEvents', + 'Timestamp': '2023-03-17T16:59:12.3036191Z', + 'DeviceId': '1234567890abcdef1234567890abcdef12345678', + 'DeviceName': 'host2.test.com', + } + ] + merged = merge_alerts(data) + assert len(merged) == 3 + alert = merged[0] + assert alert.get("TableName") == "DeviceAlertEvents" + assert all_keys_in_object( + {'TableName', 'AlertId', 'Timestamp', 'DeviceId', 'DeviceName', 'Severity', 'Category', 'Title', + 'AttackTechniques', 'ReportId'}, alert) + proc_event1 = merged[1] + assert proc_event1.get("TableName") == "DeviceProcessEvents" + assert proc_event1.get("DeviceName") == "host.test.com" + proc_event2 = merged[2] + assert proc_event2.get("TableName") == "DeviceProcessEvents" + assert proc_event2.get("DeviceName") == "host2.test.com" + + def test_remove_duplicate_fields(self, mock_adal_auth): + data = { + 'ReportId': 1234, + 'DeviceName': 'host.example.com', + 'DeviceId': '1234567890abcdef1234567890abcdef12345678', + 'Timestamp': '2023-03-17T16:59:12.3036191Z', + 'Timestamp2': '2023-03-17T16:39:21.7061265Z', + 'Timestamp1': '2023-03-17T16:39:21.7061265Z', + 'TableName': 'DeviceEvents', + 'InitiatingProcessSHA1': '4a65b267d5fc37527f567f0300e1624845600be1', + 'InitiatingProcessSHA256': 'b84257d238582d3768799e08df03f0b3378a7f8d7342b8c8ffcc453cf6a7b867', + 'InitiatingProcessMD5': '58f918b86a4798177032abcb12c9c605', + 'DeviceId1': '1234567890abcdef1234567890abcdef12345678', + 'DeviceId2': '1234567890abcdef1234567890abcdef12345678', + 'DNI_TS': '2023-03-17T16:59:12.3036191Z', + 'DI_TS': '2023-03-17T16:59:12.3036191Z', + 'SHA1': None, + 'MD5': '' + } + remove_duplicate_and_empty_fields(data) + assert all_keys_in_object({'ReportId', 'DeviceName', 'DeviceId', 'Timestamp', 'TableName', + 'InitiatingProcessSHA1', 'InitiatingProcessSHA256', 'InitiatingProcessMD5'}, data) + assert 'DeviceId1' not in data + assert 'DeviceId2' not in data + assert 'Timestamp1' not in data + assert 'Timestamp2' not in data + assert 'DNI_TS' not in data + assert 'DI_TS' not in data + assert 'SHA1' not in data + assert 'MD5' not in data + + def test_get_table_name(self, mock_adal_auth): + query = '(find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z) and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc | where AlertId =~ "123123")' + table = get_table_name(query) + assert table == "DeviceAlertEvents" + query = 'union (find withsource = TableName in (DeviceProcessEvents) where Timestamp >= datetime(2023-03-17T20:19:41.000Z) and Timestamp < datetime(2023-03-17T20:19:42.7016812Z) | order by Timestamp desc | where ((DeviceId =~ "1234567890abcdef1234567890abcdef12345678") and (ActionType =~ "FileModified")) and ((tostring(ProcessId) == "3892") or (tostring(InitiatingProcessId) == "3892") or (tostring(InitiatingProcessId) == "3892"))),(find withsource = TableName in (DeviceEvents) where Timestamp >= datetime(2023-03-17T20:19:41.000Z) and Timestamp < datetime(2023-03-17T20:19:42.7016812Z) | order by Timestamp desc | where ((DeviceId =~ "1234567890abcdef1234567890abcdef12345678") and (ActionType =~ "FileModified")) and ((tostring(ProcessId) == "3892") or (tostring(InitiatingProcessId) == "3892"))),(find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(2023-03-17T20:19:41.000Z) and Timestamp < datetime(2023-03-17T20:19:42.7016812Z) | order by Timestamp desc | where ((DeviceId =~ "1234567890abcdef1234567890abcdef12345678") and (ActionType =~ "FileModified")) and (tostring(InitiatingProcessId) == "3892")),(find withsource = TableName in (DeviceRegistryEvents) where Timestamp >= datetime(2023-03-17T20:19:41.000Z) and Timestamp < datetime(2023-03-17T20:19:42.7016812Z) | order by Timestamp desc | where ((DeviceId =~ "1234567890abcdef1234567890abcdef12345678") and (ActionType =~ "FileModified")) and (tostring(InitiatingProcessId) == "3892")),(find withsource = TableName in (DeviceFileEvents) where Timestamp >= datetime(2023-03-17T20:19:41.000Z) and Timestamp < datetime(2023-03-17T20:19:42.7016812Z) | order by Timestamp desc | where ((DeviceId =~ "1234567890abcdef1234567890abcdef12345678") and (ActionType =~ "FileModified")) and (tostring(InitiatingProcessId) == "3892")),(find withsource = TableName in (DeviceImageLoadEvents) where Timestamp >= datetime(2023-03-17T20:19:41.000Z) and Timestamp < datetime(2023-03-17T20:19:42.7016812Z) | order by Timestamp desc | where ((DeviceId =~ "1234567890abcdef1234567890abcdef12345678") and (ActionType =~ "FileModified")) and (tostring(InitiatingProcessId) == "3892"))' + table = get_table_name(query) + assert table == "DeviceProcessEvents" + + def test_join_alerts_with_events(self, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() + + util = ConnectorPostProcessing(self.config(), False) + joined_query = util.join_alert_with_events('<>', 'devicename', 1234) + assert joined_query == ('(union (find withsource = TableName in (DeviceNetworkEvents) where ' + '(Timestamp == datetime(<>)) and (DeviceName == "devicename") and ' + '(ReportId == 1234)),(find withsource = TableName in (DeviceProcessEvents) ' + 'where (Timestamp == datetime(<>)) and (DeviceName == ' + '"devicename") and (ReportId == 1234)),(find withsource = TableName in ' + '(DeviceFileEvents) where (Timestamp == datetime(<>)) and ' + '(DeviceName == "devicename") and (ReportId == 1234)),(find withsource = ' + 'TableName in (DeviceRegistryEvents) where (Timestamp == ' + 'datetime(<>)) and (DeviceName == "devicename") and (ReportId == ' + '1234)),(find withsource = TableName in (DeviceEvents) where (Timestamp == ' + 'datetime(<>)) and (DeviceName == "devicename") and (ReportId == ' + '1234)),(find withsource = TableName in (DeviceImageLoadEvents) where ' + '(Timestamp == datetime(<>)) and (DeviceName == "devicename") and ' + '(ReportId == 1234))) | join kind=leftouter ' + '(DeviceInfo | project DI_TS = Timestamp, DeviceId, PublicIP, OSArchitecture, ' + 'OSPlatform, OSVersion) on DeviceId | where DI_TS < Timestamp | summarize ' + 'arg_max(DI_TS, *) by ReportId, DeviceName, Timestamp | join kind=leftouter ' + '(DeviceNetworkInfo | where NetworkAdapterStatus == "Up" | project DNI_TS = ' + 'Timestamp, DeviceId, MacAddress, IPAddresses | summarize ' + 'IPAddressesSet=make_set(IPAddresses), MacAddressSet=make_set(MacAddress) by ' + 'DeviceId, DNI_TS) on DeviceId | where DNI_TS < Timestamp | summarize ' + 'arg_max(DNI_TS, *) by ReportId, DeviceName, Timestamp ') + + def test_join_query_with_alerts(self, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() + query = 'union (find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (LocalIP =~ "9.9.9.9") or (RemoteIP =~ "9.9.9.9")),(find withsource = TableName in (DeviceEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (RemoteIP =~ "9.9.9.9") or (LocalIP =~ "9.9.9.9"))' + entry_point = ConnectorPostProcessing(self.config(), False) + joined_query = entry_point.join_query_with_other_tables(query) + assert joined_query == ( + "(union (find withsource = TableName in (DeviceNetworkEvents) " + "where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) " + "| order by Timestamp desc " + "| where (LocalIP =~ \"9.9.9.9\") or (RemoteIP =~ \"9.9.9.9\"))," + "(find withsource = TableName in (DeviceEvents) " + "where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) " + "| order by Timestamp desc | where (RemoteIP =~ \"9.9.9.9\") or (LocalIP =~ \"9.9.9.9\"))) " + "| join kind=leftouter (DeviceAlertEvents | summarize AlertId=make_list(AlertId), " + "Severity=make_list(Severity), Title=make_list(Title), Category=make_list(Category), " + "AttackTechniques=make_list(AttackTechniques) by DeviceName, ReportId, Timestamp) " + "on ReportId, DeviceName, Timestamp " + "| join kind=leftouter (DeviceInfo | project DI_TS = Timestamp, DeviceId, PublicIP, OSArchitecture, " + "OSPlatform, OSVersion) on DeviceId | where DI_TS < Timestamp " + "| summarize arg_max(DI_TS, *) by ReportId, DeviceName, Timestamp " + "| join kind=leftouter (DeviceNetworkInfo | where NetworkAdapterStatus == \"Up\" " + "| project DNI_TS = Timestamp, DeviceId, MacAddress, IPAddresses " + "| summarize IPAddressesSet=make_set(IPAddresses), MacAddressSet=make_set(MacAddress) by DeviceId, DNI_TS) " + "on DeviceId | where DNI_TS < Timestamp | summarize arg_max(DNI_TS, *) by ReportId, DeviceName, Timestamp " + ) + + query = '(find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z) and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc | where AlertId =~ "123123")' + entry_point = ConnectorPostProcessing(self.config(), False) + joined_query = entry_point.join_query_with_other_tables(query) + assert joined_query == ( + '((find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z)' + ' and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc ' + '| where AlertId =~ "123123")) ' + '| join kind=leftouter (DeviceInfo | project DI_TS = Timestamp, DeviceId, PublicIP, OSArchitecture, ' + 'OSPlatform, OSVersion) on DeviceId | where DI_TS < Timestamp | summarize arg_max(DI_TS, *) by ReportId, ' + 'DeviceName, Timestamp ' + '| join kind=leftouter (DeviceNetworkInfo | where NetworkAdapterStatus == "Up" | project DNI_TS = Timestamp,' + ' DeviceId, MacAddress, IPAddresses | summarize IPAddressesSet=make_set(IPAddresses), ' + 'MacAddressSet=make_set(MacAddress) by DeviceId, DNI_TS) on DeviceId | where DNI_TS < Timestamp ' + '| summarize arg_max(DNI_TS, *) by ReportId, DeviceName, Timestamp ' + ) + + def test_join_query_no_info(self, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() + query = 'union (find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (LocalIP =~ "9.9.9.9") or (RemoteIP =~ "9.9.9.9")),(find withsource = TableName in (DeviceEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (RemoteIP =~ "9.9.9.9") or (LocalIP =~ "9.9.9.9"))' + config = json.loads(json.dumps(self.config())) + config['includeHostOs'] = False + config['includeNetworkInfo'] = False + entry_point = ConnectorPostProcessing(config, False) + joined_query = entry_point.join_query_with_other_tables(query) + assert joined_query == ( + "(union (find withsource = TableName in (DeviceNetworkEvents) " + "where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) " + "| order by Timestamp desc " + "| where (LocalIP =~ \"9.9.9.9\") or (RemoteIP =~ \"9.9.9.9\"))," + "(find withsource = TableName in (DeviceEvents) " + "where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) " + "| order by Timestamp desc | where (RemoteIP =~ \"9.9.9.9\") or (LocalIP =~ \"9.9.9.9\"))) " + "| join kind=leftouter (DeviceAlertEvents | summarize AlertId=make_list(AlertId), " + "Severity=make_list(Severity), Title=make_list(Title), Category=make_list(Category), " + "AttackTechniques=make_list(AttackTechniques) by DeviceName, ReportId, Timestamp) " + "on ReportId, DeviceName, Timestamp " + ) + + query = '(find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z) and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc | where AlertId =~ "123123")' + entry_point = ConnectorPostProcessing(config, False) + joined_query = entry_point.join_query_with_other_tables(query) + assert joined_query == ( + '((find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z)' + ' and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc ' + '| where AlertId =~ "123123"))' + ) + + def test_join_query_no_alerts(self, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() + query = 'union (find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (LocalIP =~ "9.9.9.9") or (RemoteIP =~ "9.9.9.9")),(find withsource = TableName in (DeviceEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (RemoteIP =~ "9.9.9.9") or (LocalIP =~ "9.9.9.9"))' + config = json.loads(json.dumps(self.config())) + config['includeAlerts'] = False + entry_point = ConnectorPostProcessing(config, False) + joined_query = entry_point.join_query_with_other_tables(query) + assert joined_query == ( + "(union (find withsource = TableName in (DeviceNetworkEvents) " + "where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) " + "| order by Timestamp desc " + "| where (LocalIP =~ \"9.9.9.9\") or (RemoteIP =~ \"9.9.9.9\"))," + "(find withsource = TableName in (DeviceEvents) " + "where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) " + "| order by Timestamp desc | where (RemoteIP =~ \"9.9.9.9\") or (LocalIP =~ \"9.9.9.9\"))) " + "| join kind=leftouter (DeviceInfo | project DI_TS = Timestamp, DeviceId, PublicIP, OSArchitecture, " + "OSPlatform, OSVersion) on DeviceId | where DI_TS < Timestamp " + "| summarize arg_max(DI_TS, *) by ReportId, DeviceName, Timestamp " + "| join kind=leftouter (DeviceNetworkInfo | where NetworkAdapterStatus == \"Up\" " + "| project DNI_TS = Timestamp, DeviceId, MacAddress, IPAddresses " + "| summarize IPAddressesSet=make_set(IPAddresses), MacAddressSet=make_set(MacAddress) by DeviceId, DNI_TS) " + "on DeviceId | where DNI_TS < Timestamp | summarize arg_max(DNI_TS, *) by ReportId, DeviceName, Timestamp " + ) + + query = '(find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z) and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc | where AlertId =~ "123123")' + entry_point = ConnectorPostProcessing(config, False) + joined_query = entry_point.join_query_with_other_tables(query) + assert joined_query == ( + '((find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z)' + ' and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc ' + '| where AlertId =~ "123123")) ' + '| join kind=leftouter (DeviceInfo | project DI_TS = Timestamp, DeviceId, PublicIP, OSArchitecture, ' + 'OSPlatform, OSVersion) on DeviceId | where DI_TS < Timestamp | summarize arg_max(DI_TS, *) by ReportId, ' + 'DeviceName, Timestamp ' + '| join kind=leftouter (DeviceNetworkInfo | where NetworkAdapterStatus == "Up" | project DNI_TS = Timestamp,' + ' DeviceId, MacAddress, IPAddresses | summarize IPAddressesSet=make_set(IPAddresses), ' + 'MacAddressSet=make_set(MacAddress) by DeviceId, DNI_TS) on DeviceId | where DNI_TS < Timestamp ' + '| summarize arg_max(DNI_TS, *) by ReportId, DeviceName, Timestamp ' + ) + + def test_join_query_only_events(self, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() + query = 'union (find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (LocalIP =~ "9.9.9.9") or (RemoteIP =~ "9.9.9.9")),(find withsource = TableName in (DeviceEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (RemoteIP =~ "9.9.9.9") or (LocalIP =~ "9.9.9.9"))' + config = json.loads(json.dumps(self.config())) + config['includeAlerts'] = False + config['includeHostOs'] = False + config['includeNetworkInfo'] = False + entry_point = ConnectorPostProcessing(config, False) + joined_query = entry_point.join_query_with_other_tables(query) + assert joined_query == ( + "(union (find withsource = TableName in (DeviceNetworkEvents) " + "where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) " + "| order by Timestamp desc " + "| where (LocalIP =~ \"9.9.9.9\") or (RemoteIP =~ \"9.9.9.9\"))," + "(find withsource = TableName in (DeviceEvents) " + "where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) " + "| order by Timestamp desc | where (RemoteIP =~ \"9.9.9.9\") or (LocalIP =~ \"9.9.9.9\")))" + ) + + query = '(find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z) and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc | where AlertId =~ "123123")' + entry_point = ConnectorPostProcessing(config, False) + joined_query = entry_point.join_query_with_other_tables(query) + assert joined_query == ( + '((find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z)' + ' and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc ' + '| where AlertId =~ "123123"))' + ) + + def test_unify_alert_fields(self, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() + data = { + 'AlertId': ['da111111111111111111_-1111111111'], + 'Timestamp': '2023-03-17T16:59:12.3036191Z', + 'Severity': ['High'], + 'Category': ['InitialAccess'], + 'Title': ['Suspicious URL clicked'], + 'AttackTechniques': ['["Spearphishing Link (T1566.002)"]'], + 'ReportId': 1234, + 'Table': 'DeviceEvents' + } + + unify_alert_fields(data) + alerts = data.get("Alerts") + assert alerts is not None + assert type(alerts) is str + alerts = json.loads(alerts) + assert len(alerts) == 1 + alert = alerts[0] + assert alert.get("AlertId") == 'da111111111111111111_-1111111111' + assert alert.get("Title") == 'Suspicious URL clicked' + ttps = alert.get("AttackTechniques") + assert ttps is not None + assert len(ttps) == 1 + ttp = ttps[0] + assert ttp == 'Spearphishing Link (T1566.002)' + + def test_organize_registry_data(self, mock_adal_auth): + data = { + "DeviceRegistryEvents": { + "TableName": "DeviceRegistryEvents", + "Timestamp": "2019-10-10T10:43:07.2363291Z", + "DeviceId": "db40e68dd7358aa450081343587941ce96ca4777", + "DeviceName": "testmachine1", + "ActionType": "RegistryValueSet", + "RegistryKey": "HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\ControlSet001\\\\Services\\\\WindowsAzureGuestAgent", + "RegistryValueType": "Binary", + "RegistryValueName": "FailureActions", + "RegistryValueData": "" + } + } + + organize_registry_data(data["DeviceRegistryEvents"]) + assert "RegistryValues" in data["DeviceRegistryEvents"] + values = data["DeviceRegistryEvents"]["RegistryValues"] + assert len(values) == 1 + val = values[0] + assert val.get("RegistryValueType") == "Binary" + assert val.get("RegistryValueName") == "FailureActions" + assert val.get("RegistryValueData") == "" + + def test_organize_ips(self, mock_adal_auth): + data = { + "DeviceRegistryEvents": { + "IPAddressesSet": ["[{\"IPAddress\":\"9.9.9.9\",\"SubnetPrefix\":24,\"AddressType\":\"Private\"}]"] + } + } + organize_ips(data["DeviceRegistryEvents"]) + assert "IPAddresses" in data["DeviceRegistryEvents"] + values = data["DeviceRegistryEvents"]["IPAddresses"] + assert len(values) == 1 + assert values[0] == "9.9.9.9" + + def test_create_event_link(self, mock_adal_auth): + data = { + "DeviceId": "deviceid" + } + create_event_link(data, "2019-10-10T10:43:07.2363291Z") + assert data.get( + "event_link") == "https://security.microsoft.com/machines/deviceid/timeline?from=2019-10-10T10:43:06.000Z&to=2019-10-10T10:43:08.000Z" + + def test_remove_duplicate_ips(self, mock_adal_auth): + data = { + "PublicIP": "9.9.9.9", + "LocalIP": "9.9.9.9", + "IPAddresses": ["9.9.9.9", "9.9.9.1"] + } + remove_duplicate_ips(data) + assert "PublicIP" not in data + assert data.get("LocalIP") == "9.9.9.9" + assert len(data["IPAddresses"]) == 1 + assert data["IPAddresses"][0] == "9.9.9.1" + + def test_do_not_remove_duplicate_ips(self, mock_adal_auth): + data = { + "PublicIP": "9.9.9.1", + "LocalIP": "9.9.9.2", + "IPAddresses": ["9.9.9.3", "9.9.9.4"] + } + remove_duplicate_ips(data) + assert data.get("PublicIP") == "9.9.9.1" + assert data.get("LocalIP") == "9.9.9.2" + assert len(data["IPAddresses"]) == 2 diff --git a/stix_shifter_modules/msatp/tests/stix_transmission/test_msatp.py b/stix_shifter_modules/msatp/tests/stix_transmission/test_msatp.py index 34495733e..688c570b6 100644 --- a/stix_shifter_modules/msatp/tests/stix_transmission/test_msatp.py +++ b/stix_shifter_modules/msatp/tests/stix_transmission/test_msatp.py @@ -3,26 +3,29 @@ import unittest from stix_shifter.stix_transmission import stix_transmission from stix_shifter_utils.utils.error_response import ErrorCode +from tests.utils.async_utils import get_mock_response, get_adal_mock_response -class MSATPMockResponse: - def __init__(self, response_code, obj): - self.code = response_code - self.object = obj +def mocked_1(): + return get_mock_response(200, "{}", 'byte') - def read(self): - return bytearray(self.object, 'utf-8') -class AdalMockResponse: +def mocked_2(): + mocked_return_value = """{ + "Results": [{ + "TableName": "DeviceFileEvents", + "Timestamp": "2019-09-13T11:34:14.0075314Z", + "DeviceName": "desktop-536bt46", + "FileName": "runcit_tlm_hw.bat", + "SHA1": "93b458752aea37a257a7dd2ed51e98ffffc35be8", + "SHA256": "", + "MD5": "26a2fe38dc6f42386659e611219c563c" + }] + }""" + return get_mock_response(200, mocked_return_value, 'byte') - @staticmethod - def acquire_token_with_client_credentials(resource, client_id, client_secret): - context_response = dict() - context_response['accessToken'] = 'abc12345' - return context_response @patch('stix_shifter_modules.msatp.stix_transmission.connector.adal.AuthenticationContext') -@patch('stix_shifter_modules.msatp.stix_transmission.api_client.APIClient.__init__') class TestMSATPConnection(unittest.TestCase): def config(self): return { @@ -38,39 +41,32 @@ def connection(self): "host": "hostbla", "port": 8080, "selfSignedCert": "cert" - } + } - def test_is_async(self, mock_api_client, mock_generate_token): - mock_api_client.return_value = None - mock_generate_token.return_value = AdalMockResponse + def test_is_async(self, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() entry_point = EntryPoint(self.connection(), self.config()) check_async = entry_point.is_async() assert check_async is False @patch('stix_shifter_modules.msatp.stix_transmission.api_client.APIClient.ping_box') - def test_ping_endpoint(self, mock_ping_response, mock_api_client, mock_generate_token): - - mock_api_client.return_value = None - mock_generate_token.return_value = AdalMockResponse + def test_ping_endpoint(self, mock_ping_response, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() mocked_return_value = '["mock", "placeholder"]' - mock_ping_response.return_value = MSATPMockResponse(200, mocked_return_value) - print(str(self.connection)) - print(str(self.config)) + mock_ping_response.return_value = get_mock_response(200, mocked_return_value) transmission = stix_transmission.StixTransmission('msatp', self.connection(), self.config()) ping_response = transmission.ping() - + assert ping_response is not None assert ping_response['success'] @patch('stix_shifter_modules.msatp.stix_transmission.api_client.APIClient.ping_box') - def test_ping_endpoint_exception(self, mock_ping_response, mock_api_client, mock_generate_token): - mock_api_client.return_value = None - mock_generate_token.return_value = AdalMockResponse + def test_ping_endpoint_exception(self, mock_ping_response, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() mocked_return_value = '["mock", "placeholder"]' - mock_ping_response.return_value = MSATPMockResponse(400, mocked_return_value) - mock_ping_response.side_effect = Exception('exception') + mock_ping_response.return_value = get_mock_response(400, mocked_return_value) transmission = stix_transmission.StixTransmission('msatp', self.connection(), self.config()) ping_response = transmission.ping() @@ -79,10 +75,8 @@ def test_ping_endpoint_exception(self, mock_ping_response, mock_api_client, mock assert ping_response['success'] is False assert ping_response['code'] == ErrorCode.TRANSMISSION_UNKNOWN.value - def test_query_connection(self, mock_api_client, mock_generate_token): - - mock_api_client.return_value = None - mock_generate_token.return_value = AdalMockResponse + def test_query_connection(self, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() query = "(find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(" \ "2019-09-24T16:32:32.993821Z) and Timestamp < datetime(2019-09-24T16:37:32.993821Z) | order by " \ @@ -97,11 +91,8 @@ def test_query_connection(self, mock_api_client, mock_generate_token): @patch('stix_shifter_modules.msatp.stix_transmission.api_client.APIClient.run_search', autospec=True) - def test_results_file_response(self, mock_results_response, mock_api_client, mock_generate_token): - - - mock_api_client.return_value = None - mock_generate_token.return_value = AdalMockResponse + def test_results_file_response(self, mock_results_response, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() mocked_return_value = """{ "Results": [{ "TableName": "DeviceFileEvents", @@ -113,7 +104,7 @@ def test_results_file_response(self, mock_results_response, mock_api_client, moc "MD5": "26a2fe38dc6f42386659e611219c563c" }] }""" - mock_results_response.return_value = MSATPMockResponse(200, mocked_return_value) + mock_results_response.return_value = get_mock_response(200, mocked_return_value, 'byte') query = '(find withsource = TableName in (DeviceFileEvents) where Timestamp >= datetime(' \ '2019-09-01T08:43:10.003Z) and Timestamp < datetime(2019-10-01T10:43:10.003Z) | order by ' \ @@ -129,19 +120,17 @@ def test_results_file_response(self, mock_results_response, mock_api_client, moc assert 'data' in results_response assert results_response['data'] is not None + @patch('stix_shifter_modules.msatp.stix_transmission.api_client.APIClient.run_search', autospec=True) - def test_results_registry_response(self, mock_results_response, mock_api_client, mock_generate_token): - - - mock_api_client.return_value = None - mock_generate_token.return_value = AdalMockResponse + def test_results_registry_response(self, mock_results_response, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() mocked_return_value = """{"Results": [{"TableName": "DeviceRegistryEvents","Timestamp": "2019-10-10T10:43:07.2363291Z","DeviceId": "db40e68dd7358aa450081343587941ce96ca4777","DeviceName": "testmachine1","ActionType": "RegistryValueSet", "RegistryKey": "HKEY_LOCAL_MACHINE\\\\SYSTEM\\\\ControlSet001\\\\Services\\\\WindowsAzureGuestAgent", "RegistryValueType": "Binary","RegistryValueName": "FailureActions","RegistryValueData": ""}]}""" - mock_results_response.return_value = MSATPMockResponse(200, mocked_return_value) + mock_results_response.return_value = get_mock_response(200, mocked_return_value, 'byte') query = '(find withsource = TableName in (DeviceRegistryEvents) where Timestamp >= datetime(' \ '2019-09-01T08:43:10.003Z) and Timestamp < datetime(2019-10-10T10:43:10.003Z) | order by Timestamp ' \ @@ -158,35 +147,29 @@ def test_results_registry_response(self, mock_results_response, mock_api_client, assert 'data' in results_response assert results_response['data'] is not None - ''' @patch('stix_shifter_modules.msatp.stix_transmission.api_client.APIClient.run_search', autospec=True) - def test_results_response_exception(self, mock_results_response, mock_api_client, mock_generate_token): - - - mock_api_client.return_value = None - mock_generate_token.return_value = None + def test_results_response_exception(self, mock_results_response, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() mocked_return_value = """ { } """ - mock_results_response.return_value = MSATPMockResponse(404, mocked_return_value) + mock_results_response.return_value = get_mock_response(404, mocked_return_value) query = "(find withsource = TableName in (DeviceNetworkEvents) where " \ "Timestamp >= datetime('2021-04-25T14:09:15.093Z) and Timestamp < datetime(2021-04-25T14:14:15.093Z) " \ "| order by Timestamp desc | where LocalPort < 443) " offset = 0 length = 1 + transmission = stix_transmission.StixTransmission('msatp', self.connection(), self.config()) results_response = transmission.results(query, offset, length) assert results_response['code'] == 'unknown' assert results_response['success'] is False - ''' @patch('stix_shifter_modules.msatp.stix_transmission.api_client.APIClient.run_search', autospec=True) - def test_query_flow(self, mock_results_response, mock_api_client, mock_generate_token): - - mock_api_client.return_value = None - mock_generate_token.return_value = AdalMockResponse + def test_query_flow(self, mock_results_response, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() results_mock = """{ "Results": [{ "TableName": "DeviceFileEvents", @@ -199,7 +182,7 @@ def test_query_flow(self, mock_results_response, mock_api_client, mock_generate_ }] }""" - mock_results_response.return_value = MSATPMockResponse(200, results_mock) + mock_results_response.return_value = get_mock_response(200, results_mock, 'byte') query = '(find withsource = TableName in (DeviceFileEvents) where Timestamp >= datetime(' \ '2019-09-01T08:43:10.003Z) and Timestamp < datetime(2019-10-01T10:43:10.003Z) | order by Timestamp ' \ @@ -228,9 +211,8 @@ def test_query_flow(self, mock_results_response, mock_api_client, mock_generate_ assert 'data' in results_response assert results_response['data'] is not None - def test_delete_query(self, mock_api_client, mock_generate_token): - mock_api_client.return_value = None - mock_generate_token.return_value = AdalMockResponse + def test_delete_query(self, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() search_id = '(find withsource = TableName in (DeviceFileEvents) where Timestamp >= datetime(' \ '2019-09-01T08:43:10.003Z) and Timestamp < datetime(2019-10-01T10:43:10.003Z) | order by ' \ @@ -238,16 +220,13 @@ def test_delete_query(self, mock_api_client, mock_generate_token): 'or InitiatingProcessParentFileName !~ "updater.exe")' entry_point = EntryPoint(self.connection(), self.config()) - status_response = entry_point.delete_query_connection(search_id) + status_response = run_in_thread(entry_point.delete_query_connection, search_id) assert status_response is not None assert 'success' in status_response assert status_response['success'] is True - def test_status_query(self, mock_api_client, mock_generate_token): - - - mock_api_client.return_value = None - mock_generate_token.return_value = AdalMockResponse + def test_status_query(self, mock_adal_auth): + mock_adal_auth.return_value = get_adal_mock_response() search_id = '(find withsource = TableName in (DeviceFileEvents) where Timestamp >= datetime(' \ '2019-09-01T08:43:10.003Z) and Timestamp < datetime(2019-10-01T10:43:10.003Z) | order by ' \ @@ -255,7 +234,8 @@ def test_status_query(self, mock_api_client, mock_generate_token): 'or InitiatingProcessParentFileName !~ "updater.exe")' entry_point = EntryPoint(self.connection(), self.config()) - status_response = entry_point.create_status_connection(search_id) + status_response = run_in_thread(entry_point.create_status_connection, search_id) assert status_response is not None assert 'success' in status_response assert status_response['success'] is True + diff --git a/stix_shifter_modules/msatp/tests/test_utils.py b/stix_shifter_modules/msatp/tests/test_utils.py new file mode 100644 index 000000000..cc654f7b3 --- /dev/null +++ b/stix_shifter_modules/msatp/tests/test_utils.py @@ -0,0 +1,77 @@ +def all_keys_in_object(keys_to_check, obj): + """checks that all the keys in keys_to_check are present in the object (in no certain order) + + parameters + ---------- + keys_to_check : set + a set of the properties that the object must have + obj : dict + the object to check against + """ + return all(key in obj for key in keys_to_check) + + +def resolve_refs(objects, obj, ref, ref_type, error_msg): + """ + resolves an array of references, checks that each is not none and optionaly its type and returns a list of + the referenced objects + + parameters + ---------- + objects : dict + the objects dictionary + obj : dict + the current object + ref : str + the name of the reference property. for example: host_ref + ref_type : str + the type of the objects being referenced or None if they are not uniform + error_msg : str + the error to show if assertions fail + """ + assert ref in obj + ref_arr = obj[ref] + assert type(ref_arr) is list + arr = [] + for ref_idx in ref_arr: + assert ref_idx in objects + ref_obj = objects[ref_idx] + assert ref_obj is not None, error_msg + assert 'type' in ref_obj, error_msg + if ref_type is not None: + assert ref_obj['type'] == ref_type + arr.append(ref_obj) + return arr + + +def resolve_ref(objects, obj, ref, ref_type, error_msg): + """ + resolves an object from a reference, checks that it is not none and its type and returns the referenced object + + parameters + ---------- + objects : dict + the objects dictionary + obj : dict + the current object + ref : str + the name of the reference property. for example: host_ref + ref_type : str + the type of the object being referenced + error_msg : str + the error to show if assertions fail + """ + assert ref in obj, f"property {ref} not found in object {obj.get('type')}" + ref_idx = obj[ref] + assert ref_idx in objects, f"index {ref_idx} from reference {ref} not found in objects" + ref_obj = objects[ref_idx] + assert ref_obj is not None, error_msg + assert 'type' in ref_obj, "referenced object is missing the type property" + assert ref_obj['type'] == ref_type, f"type of referenced object is not as expected. expected {ref_obj} found {ref_obj['type']}" + return ref_obj + + +def hashes_are_correct(file_obj, hashes): + assert file_obj['hashes']['MD5'] == hashes["MD5"] + assert file_obj['hashes']['SHA-1'] == hashes["SHA1"] + assert file_obj['hashes']['SHA-256'] == hashes["SHA256"] \ No newline at end of file From 34d46846239d54a5419ef7384dddb1b431906d2c Mon Sep 17 00:00:00 2001 From: Aviv Ron Date: Tue, 4 Apr 2023 11:45:43 +0300 Subject: [PATCH 2/4] fixed issues, all tests pass --- .../msatp/stix_transmission/api_client.py | 12 +++--- .../msatp/stix_transmission/connector.py | 4 +- .../test_msatp_json_to_stix.py | 4 +- .../tests/stix_transmission/test_msatp.py | 4 +- tests/utils/async_utils.py | 43 +++++++++++++++++++ 5 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 tests/utils/async_utils.py diff --git a/stix_shifter_modules/msatp/stix_transmission/api_client.py b/stix_shifter_modules/msatp/stix_transmission/api_client.py index 3e5374bdc..d3c5722df 100644 --- a/stix_shifter_modules/msatp/stix_transmission/api_client.py +++ b/stix_shifter_modules/msatp/stix_transmission/api_client.py @@ -1,6 +1,6 @@ """Apiclient for MSATP""" import json -from stix_shifter_utils.stix_transmission.utils.RestApiClientAsync import RestApiClientAsync +from stix_shifter_utils.stix_transmission.utils.RestApiClient import RestApiClient DEFAULT_LIMIT = 10000 DEFAULT_OFFSET = 0 @@ -24,7 +24,7 @@ def __init__(self, connection, configuration): if 'access_token' in auth and auth['access_token']: headers['Authorization'] = "Bearer " + auth['access_token'] - self.client = RestApiClientAsync(connection.get('host'), + self.client = RestApiClient(connection.get('host'), connection.get('port', None), headers, url_modifier_function=url_modifier_function, @@ -33,12 +33,12 @@ def __init__(self, connection, configuration): ) self.timeout = connection['options'].get('timeout') - async def ping_box(self): + def ping_box(self): """Ping the endpoint.""" endpoint = '/api' - return await self.client.call_api(endpoint, 'GET', timeout=self.timeout) + return self.client.call_api(endpoint, 'GET', timeout=self.timeout) - async def run_search(self, query_expression, offset=DEFAULT_OFFSET, length=DEFAULT_LIMIT): + def run_search(self, query_expression, offset=DEFAULT_OFFSET, length=DEFAULT_LIMIT): """get the response from MSatp endpoints :param query_expression: str, search_id :param offset: int,offset value @@ -51,4 +51,4 @@ async def run_search(self, query_expression, offset=DEFAULT_OFFSET, length=DEFAU endpoint = self.endpoint query_expression = query_expression + serialize.format(offset=offset, length=length) query_expression = json.dumps({'Query': query_expression}).encode("utf-8") - return await self.client.call_api(endpoint, 'POST', headers=headers, data=query_expression, timeout=self.timeout) + return self.client.call_api(endpoint, 'POST', headers=headers, data=query_expression, timeout=self.timeout) diff --git a/stix_shifter_modules/msatp/stix_transmission/connector.py b/stix_shifter_modules/msatp/stix_transmission/connector.py index 5f2dcbfec..753e158bf 100644 --- a/stix_shifter_modules/msatp/stix_transmission/connector.py +++ b/stix_shifter_modules/msatp/stix_transmission/connector.py @@ -1,7 +1,7 @@ import json import adal -from stix_shifter_utils.modules.base.stix_transmission.base_json_sync_connector import BaseJsonSyncConnector +from stix_shifter_utils.modules.base.stix_transmission.base_sync_connector import BaseSyncConnector from .api_client import APIClient from stix_shifter_utils.utils.error_response import ErrorResponder from stix_shifter_utils.utils import logger @@ -9,7 +9,7 @@ from .connector_post_processing import ConnectorPostProcessing -class Connector(BaseJsonSyncConnector): +class Connector(BaseSyncConnector): init_error = None logger = logger.set_logger(__name__) diff --git a/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_json_to_stix.py b/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_json_to_stix.py index ff522aa55..48d583a8a 100644 --- a/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_json_to_stix.py +++ b/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_json_to_stix.py @@ -331,8 +331,8 @@ def translate_to_objects(data): translation = stix_translation.StixTranslation() - result_bundle = translation.translate(module='msatp', translate_type='results', data_source=data_source, - data=[data]) + result_bundle = translation.translate(module='msatp', translate_type='results', data_source=json.dumps(data_source), + data=json.dumps([data])) result_bundle_objects = result_bundle['objects'] result_bundle_identity = result_bundle_objects[0] assert result_bundle_identity['type'] == data_source['type'] diff --git a/stix_shifter_modules/msatp/tests/stix_transmission/test_msatp.py b/stix_shifter_modules/msatp/tests/stix_transmission/test_msatp.py index 688c570b6..a45b723d9 100644 --- a/stix_shifter_modules/msatp/tests/stix_transmission/test_msatp.py +++ b/stix_shifter_modules/msatp/tests/stix_transmission/test_msatp.py @@ -220,7 +220,7 @@ def test_delete_query(self, mock_adal_auth): 'or InitiatingProcessParentFileName !~ "updater.exe")' entry_point = EntryPoint(self.connection(), self.config()) - status_response = run_in_thread(entry_point.delete_query_connection, search_id) + status_response = entry_point.delete_query_connection(search_id) assert status_response is not None assert 'success' in status_response assert status_response['success'] is True @@ -234,7 +234,7 @@ def test_status_query(self, mock_adal_auth): 'or InitiatingProcessParentFileName !~ "updater.exe")' entry_point = EntryPoint(self.connection(), self.config()) - status_response = run_in_thread(entry_point.create_status_connection, search_id) + status_response = entry_point.create_status_connection(search_id) assert status_response is not None assert 'success' in status_response assert status_response['success'] is True diff --git a/tests/utils/async_utils.py b/tests/utils/async_utils.py new file mode 100644 index 000000000..590951aaf --- /dev/null +++ b/tests/utils/async_utils.py @@ -0,0 +1,43 @@ + +def get_mock_response(status_code, content=None, return_type='str', response=None): + return RequestMockResponse(status_code, content, return_type, response) + +def get_aws_mock_response(obj): + return AWSComposeMockResponse(obj) + +def get_adal_mock_response(): + return AdalMockResponse() + +class RequestMockResponse: + def __init__(self, status_code, content, return_type='str', response=None): + self.code = status_code + self.content = content + self.response = response + self.object = response + self.return_type = return_type + self.history = [] + + def read(self): + if self.return_type == 'byte': + return bytearray(self.content, 'utf-8') + return self.content + +class AWSComposeMockResponse: + def __init__(self, object): + self.object = object + + def __getitem__(self, prop): + return self.object[prop] + + def __contains__(self, prop): + return prop in self.object + + def get(self, prop, default=None): + return self.object.get(prop, default) + +class AdalMockResponse: + @staticmethod + def acquire_token_with_client_credentials(resource, client_id, client_secret): + context_response = dict() + context_response['accessToken'] = 'abc12345' + return context_response From 86d4bda7c3336c9ce998c9a5e5aebbfb1e190ef4 Mon Sep 17 00:00:00 2001 From: Aviv Ron Date: Tue, 4 Apr 2023 13:43:54 +0300 Subject: [PATCH 3/4] removed redundant finding_refs --- .../stix_translation/json/to_stix_map.json | 40 ------------------- 1 file changed, 40 deletions(-) diff --git a/stix_shifter_modules/msatp/stix_translation/json/to_stix_map.json b/stix_shifter_modules/msatp/stix_translation/json/to_stix_map.json index 0f44ea99d..80ba234e8 100644 --- a/stix_shifter_modules/msatp/stix_translation/json/to_stix_map.json +++ b/stix_shifter_modules/msatp/stix_translation/json/to_stix_map.json @@ -1131,14 +1131,6 @@ "key": "x-oca-event.original_ref", "object": "event", "references": "original_ref" - }, - { - "key": "x-oca-event.finding_refs", - "object": "event", - "references": [ - "finding" - ], - "group": true } ], "event_link": [ @@ -1588,14 +1580,6 @@ "key": "x-oca-event.original_ref", "object": "event", "references": "original_ref" - }, - { - "key": "x-oca-event.finding_refs", - "object": "event", - "references": [ - "finding" - ], - "group": true } ], "event_link": [ @@ -1995,14 +1979,6 @@ "key": "x-oca-event.original_ref", "object": "event", "references": "original_ref" - }, - { - "key": "x-oca-event.finding_refs", - "object": "event", - "references": [ - "finding" - ], - "group": true } ], "event_link": [ @@ -2362,14 +2338,6 @@ "key": "x-oca-event.original_ref", "object": "event", "references": "original_ref" - }, - { - "key": "x-oca-event.finding_refs", - "object": "event", - "references": [ - "finding" - ], - "group": true } ], "event_link": [ @@ -2968,14 +2936,6 @@ "key": "x-oca-event.original_ref", "object": "event", "references": "original_ref" - }, - { - "key": "x-oca-event.finding_refs", - "object": "event", - "references": [ - "finding" - ], - "group": true } ], "event_link": [ From fd940c4ff2422f4eddd899d14ac6f7c4c870e816 Mon Sep 17 00:00:00 2001 From: Aviv Ron Date: Sun, 16 Apr 2023 13:01:11 +0300 Subject: [PATCH 4/4] merge from upstream --- .../msatp/configuration/config.json | 32 +- .../msatp/configuration/lang_en.json | 30 +- stix_shifter_modules/msatp/readme.md | 419 --- .../msatp/stix_translation/README.md | 876 ++++-- .../stix_translation/json/from_stix_map.json | 20 +- .../json/stix_2_1/from_stix_map.json | 540 ++-- .../json/stix_2_1/to_stix_map.json | 2499 +++++++++++++++-- .../stix_translation/query_constructor.py | 2 + .../stix_translation/results_translator.py | 55 +- .../msatp/stix_transmission/connector.py | 4 +- .../connector_post_processing.py | 21 +- .../test_msatp_json_to_stix.py | 158 ++ .../test_connector_post_processing.py | 55 +- 13 files changed, 3511 insertions(+), 1200 deletions(-) delete mode 100644 stix_shifter_modules/msatp/readme.md diff --git a/stix_shifter_modules/msatp/configuration/config.json b/stix_shifter_modules/msatp/configuration/config.json index f0a66865c..ed28a4140 100644 --- a/stix_shifter_modules/msatp/configuration/config.json +++ b/stix_shifter_modules/msatp/configuration/config.json @@ -25,11 +25,29 @@ "selfSignedCert": { "type": "password", "optional": true + }, + "options": { + "includeAlerts": { + "type": "boolean", + "default": false + }, + "includeHostOs": { + "type": "boolean", + "default": false + }, + "includeNetworkInfo": { + "type": "boolean", + "default": false + }, + "retainOriginal": { + "type": "boolean", + "default": false + } } }, "configuration": { "auth": { - "type" : "fields", + "type": "fields", "tenant": { "type": "password" }, @@ -39,18 +57,6 @@ "clientSecret": { "type": "password" } - }, - "includeAlerts":{ - "type": "boolean", - "default": false - }, - "includeHostOs": { - "type": "boolean", - "default": false - }, - "includeNetworkInfo": { - "type": "boolean", - "default": false } } } \ No newline at end of file diff --git a/stix_shifter_modules/msatp/configuration/lang_en.json b/stix_shifter_modules/msatp/configuration/lang_en.json index 102d4d51b..1f95f0b55 100644 --- a/stix_shifter_modules/msatp/configuration/lang_en.json +++ b/stix_shifter_modules/msatp/configuration/lang_en.json @@ -20,6 +20,24 @@ "sni": { "label": "Server Name Indicator", "description": "The Server Name Indicator (SNI) enables a separate hostname to be provided for SSL authentication" + }, + "options": { + "includeAlerts": { + "label": "Join events with Alerts?", + "description": "With this feature enabled every event will be searched for correlated alerts in the DeviceAlertsEvent table and the alert data will be included in the results. Keep in mind that this feature will add API CPU consumption." + }, + "includeHostOs": { + "label": "Join events with Host OS info?", + "description": "With this feature enabled every event will include also the device OS details from the DeviceInfo table. Keep in mind that this feature will add API CPU consumption." + }, + "includeNetworkInfo": { + "label": "Join events with Device Network info?", + "description": "With this feature enabled every event will include also the active network adapters IP and MAC addresses of the device at the time of the event from the DeviceNetworkInfo table. Keep in mind that this feature will add API CPU consumption." + }, + "retainOriginal": { + "label": "Keep the original json payload before translation base64 encoded as original_ref?", + "description": "Setting this to true will encode the original json results from the EDR before they are translated to STIX and put them in an artifact object referenced as original_ref from x-oca-event. This is helpful if critical data is missing in the final translation." + } } }, "configuration": { @@ -37,18 +55,6 @@ "label": "Client Secret", "description": "Client Secret of Azure Active directory Application with access to the Microsoft Graph API" } - }, - "includeAlerts": { - "label": "Join events with Alerts?", - "description": "With this feature enabled every event will be searched for correlated alerts in the DeviceAlertsEvent table and the alert data will be included in the results. Keep in mind that this feature will add API CPU consumption." - }, - "includeHostOs": { - "label": "Join events with Host OS info?", - "description": "With this feature enabled every event will include also the device OS details from the DeviceInfo table. Keep in mind that this feature will add API CPU consumption." - }, - "includeNetworkInfo": { - "label": "Join events with Device Network info?", - "description": "With this feature enabled every event will include also the active network adapters IP and MAC addresses of the device at the time of the event from the DeviceNetworkInfo table. Keep in mind that this feature will add API CPU consumption." } } } \ No newline at end of file diff --git a/stix_shifter_modules/msatp/readme.md b/stix_shifter_modules/msatp/readme.md deleted file mode 100644 index 66ba3699e..000000000 --- a/stix_shifter_modules/msatp/readme.md +++ /dev/null @@ -1,419 +0,0 @@ -# MSATP - microsoft defender for endpoint module -This module uses KQL to query the EDR API. - -Queries are constructed as unions for all the relevant event tables based on the queried fields. -The potential tables are `DeviceProcessEvents`, `DeviceNetworkEvents`, -`DeviceRegistryEvents`, `DeviceFileEvents`, `DeviceImageLoadEvents` and `DeviceEvents` - -Every query is joined with `DeviceNetworkInfo` in order to get the active network adapters from the relevant time of\ -the event (IP and mac addresses). - -Every query is also joined with `DeviceInfo` to get the host operating system. - -Queries are joined with `DeviceAlertEvents` to look for relevant alerts. An alert is joined based on -the `Timestamp` of the event, `DeviceId` and `ReportId`. Relevant alerts will be mapped to `x-ibm-finding` -with their relevant TTP's. -Searching for an alert field (for example `x-ibm-finding:alert_id`) will search the alerts table first -and then join it with the relevant events that are correlated to it. - -## Tips for successful hunting -Be concise with what you are looking for. For example searching for a file property will be -very wide as it will look at relevant file events but also at all the binary files of -processes in the initiating process of each event and its parent process. -If you wish to look for a specific process don't query for a file, instead query the process. -For example the query `[process:pid=123]` will only look at the initiating process id. -If you want to search for a process which its parent is called `foo`look for -`[process:parent_ref.name='foo']`. This will bring all the relevant events from processes which -their parent process is `foo` only. - -If you wish to search for a file from the device file events -(a file that has been created, modified etc.) but not search the process binary fields from all -other event tables query for: -`[x-oca-event:file_ref.name='foo']` - -You can query for specific event types by querying the `action` field of `x-oca-event`. -For example if I wish to search for all `RegistryValueSet` events I will query: -`[x-oca-event:action = 'RegistryValueSet' ]` - -If you want to query events on a specific host, provide its hostname as such: -`[x-oca-asset:hostname = 'host.example.com' ]` - -Searching for an IP address `[ipv4-addr:value = '9.9.9.9' ]` is more general as it will search both source and -destination ip fields. If you know the IP usage it is better to narrow the query. -For example for a remote IP use: `[network-traffic:dst_ref.value = '9.9.9.9' ]`. -For a local IP use `[network-traffic:src_ref.value = '9.9.9.1' ]` - -## Examples -Here are a few examples for results from different tables and how to interpret them: - -### DeviceProcessEvents -The main starting point to look at will be `x-oca-event:action` which tells us the -event is a process create event. - -the `process_ref` will lead us to the process that was created and its `parent_ref` -to the process which created it. - -`host_ref` leads us to `x-oca-asset` which shows information on the host on which this -event occurred: its name, unique device id, and the ip addresses and mac addresses -that were observed on the device. - -`created` shows us the timestamp when this event was observed. - -`user_ref` shows us the user account that was performing the event. `account_login` is mapped -to the user UPN which is resolved by msatp from the active directory. `user_id` is mapped -to the account name. - -`external_ref` is a link to the msatp web console that will show us the timeline of this device -one second before and after the event occurred. - -`original_ref` is the original json output from the msatp api response encoded in base64. -If there are fields you are interested to check but are not mapped to stix you can find them -there. -```json -{ - "0": - { - "type": "x-msatp", - "ReportId": 1234 - }, - "1": - { - "type": "x-oca-asset", - "hostname": "host.test.com", - "device_id": "deviceid", - "mac_refs": ["13"], - "ip_refs": ["14"], - "architecture": "64-bit", - "os_name": "Windows10", - "os_version": "10.0" - }, - "2": - { - "type": "x-oca-event", - "host_ref": "1", - "created": "2023-03-17T20:23:03.7116107Z", - "action": "ProcessCreated", - "process_ref": "4", - "user_ref": "7", - "provider": "Microsoft Defender for Endpoint", - "external_ref": "15", - "original_ref": "16" - }, - "3": - { - "type": "file", - "name": "msedge.exe", - "parent_directory_ref": "6", - "hashes": - { - "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed", - "SHA-256": "470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75", - "MD5": "e180c9a532c45eba99eefd01601f5c41" - } - }, - "4": - { - "type": "process", - "name": "msedge.exe", - "binary_ref": "3", - "pid": 37384, - "command_line": "\"msedge.exe\" --type=gpu-process", - "created": "2023-03-17T20:23:03.702Z", - "creator_user_ref": "7", - "parent_ref": "5" - }, - "5": - { - "type": "process", - "child_refs": ["4"], - "creator_user_ref": "8", - "binary_ref": "9", - "name": "msedge.exe", - "pid": 400, - "command_line": "\"msedge.exe\" -- \"https://test.com/login/login.asp\"", - "created": "2023-03-17T20:23:03.441Z", - "parent_ref": "11" - }, - "6": - { - "type": "directory", - "path": "C:\\Program Files (x86)\\Microsoft\\Edge\\Application" - }, - "7": - { - "type": "user-account", - "user_id": "username", - "account_login": "username@test.com" - }, - "8": - { - "type": "user-account", - "user_id": "username", - "account_login": "username@test.com" - }, - "9": - { - "type": "file", - "hashes": - { - "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed", - "SHA-256": "470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75", - "MD5": "e180c9a532c45eba99eefd01601f5c41" - }, - "name": "msedge.exe", - "parent_directory_ref": "10" - }, - "10": - { - "type": "directory", - "path": "c:\\program files (x86)\\microsoft\\edge\\application" - }, - "11": - { - "type": "process", - "pid": 30972, - "name": "iexplore.exe", - "binary_ref": "12", - "created": "2023-03-17T20:23:03.169Z" - }, - "12": - { - "type": "file", - "name": "iexplore.exe" - }, - "13": - { - "type": "mac-addr", - "value": "11:22:33:44:55:66" - }, - "14": - { - "type": "ipv4-addr", - "value": "9.9.9.1" - }, - "15": - { - "type": "external-reference", - "url": "https://security.microsoft.com/machines/deviceid/timeline?from=2023-03-17T20:23:02.000Z&to=2023-03-17T20:23:04.000Z" - }, - "16": - { - "type": "artifact", - "payload_bin": "base64 encoded json output from msatp api response before translation" - } -} -``` - -### DeviceNetworkEvents with associated alert - -An example including a network event to an url. -Query example: `[domain-name:value='malicious.com']` - -`url_ref` points us to the url visited. -`domain_ref` to the domain parsed from the url. -It is possible that only one of these will be populated depending on the data provided. - -`network_ref` will show us the source and destination ip addresses and ports. Mind you that -the destination ip address is the one resolved from the domain name -referenced by `domain_ref` in the `x-oca-event`. - -`finding_refs` indicates that there are associated alerts with this event. These will be -of type `x-ibm-finding`. The event is a connection success event, and the associated alert -title is `Connection to adversary-in-the-middle (AiTM) phishing site`. - -`severity` is translated from `high` (99) `medium` (66) and `low` (33). - -`alert_id` is the unique alert identifier from msatp. - -`ttp_tagging_refs` shows us the relevant TTP's. These will be of type `x-ibm-ttp-tagging`. -Tactics will be mapped to `kill_chain_phases` which might be mapped to the -mitre attack framework or a proprietary Microsoft phase - not all phases are mapped to -mitre attack. -Techniques will be mapped to the `mitre-attack-ext` extension and will describe the name and -id of the mitre attack technique. - -```json -{ - "0": - { - "type": "x-oca-event", - "created": "2023-03-17T20:19:46.6337905Z", - "host_ref": "1", - "process_ref": "4", - "action": "ConnectionSuccess", - "network_ref": "7", - "domain_ref": "9", - "url_ref": "8", - "provider": "Microsoft Defender for Endpoint", - "external_ref": "17", - "finding_refs": ["18"], - "original_ref": "20", - "file_ref": "2" - }, - "1": - { - "type": "x-oca-asset", - "device_id": "deviceid", - "hostname": "host.example.com", - "ip_refs": ["10"], - "mac_refs": ["16"], - "architecture": "64-bit", - "os_name": "Windows10", - "os_version": "10.0" - }, - "2": - { - "type": "file", - "name": "msedge.exe", - "hashes": - { - "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed" - } - }, - "4": - { - "type": "process", - "child_refs": ["3"], - "binary_ref": "11", - "name": "msedge.exe", - "pid": 3052, - "command_line": "\"msedge.exe\" --type=utility", - "created": "2023-03-13T14:22:44.860Z", - "parent_ref": "14", - "creator_user_ref": "15" - }, - "5": - { - "type": "x-msatp", - "ReportId": 1234, - "Table": "DeviceEvents" - }, - "6": - { - "type": "ipv4-addr", - "value": "9.9.9.9" - }, - "7": - { - "type": "network-traffic", - "dst_ref": "6", - "dst_port": 443, - "src_ref": "10", - "src_port": 58993 - }, - "8": - { - "type": "url", - "value": "https://malicious.com" - }, - "9": - { - "type": "domain-name", - "value": "malicious.com" - }, - "10": - { - "type": "ipv4-addr", - "value": "9.9.9.1" - }, - "11": - { - "type": "file", - "hashes": - { - "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed", - "SHA-256": "470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75", - "MD5": "e180c9a532c45eba99eefd01601f5c41" - }, - "name": "msedge.exe", - "parent_directory_ref": "12" - }, - "12": - { - "type": "directory", - "path": "c:\\program files (x86)\\microsoft\\edge\\application" - }, - "13": - { - "type": "file", - "name": "msedge.exe" - }, - "14": - { - "type": "process", - "name": "msedge.exe", - "binary_ref": "13", - "pid": 9952, - "created": "2023-03-13T14:22:44.508Z" - }, - "15": - { - "type": "user-account", - "user_id": "username", - "account_login": "username@test.com" - }, - "16": - { - "type": "mac-addr", - "value": "11:22:33:44:55:66" - }, - "17": - { - "type": "external-reference", - "url": "https://security.microsoft.com/machines/deviceid/timeline?from=2023-03-17T20:19:45.000Z&to=2023-03-17T20:19:47.000Z" - }, - "18": - { - "type": "x-ibm-finding", - "alert_id": "1234567890-1234-1234-1234-123456789012_1", - "severity": 99, - "name": "Connection to adversary-in-the-middle (AiTM) phishing site", - "ttp_tagging_refs": ["19","21"] - }, - "19": - { - "type": "x-ibm-ttp-tagging", - "kill_chain_phases": [ - { - "phase_name": "Credential Access", - "kill_chain_name": "mitre-attack" - }] - }, - "20": - { - "type": "artifact", - "payload_bin": "base64 encoded json from the msatp api before translation" - }, - "21": - { - "type": "x-ibm-ttp-tagging", - "extensions": - { - "mitre-attack-ext": - { - "technique_name": "Spearphishing Link", - "technique_id": "T1566.002" - } - } - } -} -``` - -### March 2023 changes -Changes made from previous version: -- `DeviceEvents` and `DeviceImageLoadEvents` tables where added to the search query. -Prior they were not being searched. -- `DeviceAlertEvents` is being queried in correlation with the events. -- `DeviceNetworkInfo` is being queried to extract the device network addresses at the time of the event -- `DeviceInfo` is queried to extract the device OS details -- `domain-name` was mapped to `DeviceName` causing a mismatch between looking for events on a specific host and looking for domains in network events. The mapping to device name was removed. use `x-oca-asset:hostname` instead to search for a specific device. -- `process` fields such as `name` and `pid` were looking at both the process and its parent. This causes an overload of results. Say I searched for all events by process pid 123 - I would receive also all events of its child processes. This was removed. now searching a process property will not look at the parent process fields. To search for events where the parent process is X search for `process:parent_ref.X` -- `user-account:account_login` was mapped to `AccountName` which is the local account name - which is now mapped to `user-account:user_id`. The `AccountUpn` which is the more meaningful field since it is resolved from active directory and usually contains for users full email is now mapped to `account_login`. The `AccountSid` (in the form of S-1-5- etc.) which was mapped to `user_id` is no longer mapped to any stix field. -- `x-msatp` `computer_name` and `machine_id` are removed as they map to `x-oca-asset` now. -- `x-oca-asset:ip` was switched to `x-oca-asset:ip_refs` array as per the official spec and now support multiple ip addresses for one host. -- `DeviceFileEvents.FileOriginUrl` and `DeviceFileEvents.FileOriginReferrerUrl` were added to the mappings - they were not mapped before. -- `x-oca-event:provider` will always state 'Microsoft Defender for Endpoint' to indicate the source of the event -- `x-oca-event:external_ref` was added and provides a link to the msatp web console filtered one second before and after the event. This is handy in case the analyst would like to look at the source data directly in the msatp console. -- event name was mapped to `x-oca-event:action` -- `x-oca-event:original_ref` was added and includes the raw json output of the msatp api encoded in base64. This is handy to find details that are not mapped to stix. -- `x-msatp:AdditionalFields` was added as this field usually contains important information. diff --git a/stix_shifter_modules/msatp/stix_translation/README.md b/stix_shifter_modules/msatp/stix_translation/README.md index 93defd70c..793781086 100644 --- a/stix_shifter_modules/msatp/stix_translation/README.md +++ b/stix_shifter_modules/msatp/stix_translation/README.md @@ -1,5 +1,557 @@ # Microsoft Defender for Endpoint (previously Microsoft Defender ATP) UDS Connector +# MSATP - microsoft defender for endpoint module +This module uses KQL to query the EDR API. + +Queries are constructed as unions for all the relevant event tables based on the queried fields. +The potential tables are `DeviceProcessEvents`, `DeviceNetworkEvents`, +`DeviceRegistryEvents`, `DeviceFileEvents`, `DeviceImageLoadEvents` and `DeviceEvents` + +## Connection options: + +There are 4 connection options (default to `false`): +### `includeNetworkInfo`: +If true, every query is joined with `DeviceNetworkInfo` table in order to get the active network adapters from the relevant time of\ +the event (IP and mac addresses). + +### `includeHostOs`: +If true, every query is also joined with `DeviceInfo` table to get the host operating system. + +### `includeAlerts`: +If true, queries are joined with `DeviceAlertEvents` to look for relevant alerts. An alert is joined based on +the `Timestamp` of the event, `DeviceId` and `ReportId`. Relevant alerts will be mapped to `x-ibm-finding` +with their relevant TTP's. +Searching for an alert field (for example `x-ibm-finding:alert_id`) will search the alerts table first +and then join it with the relevant events that are correlated to it. + +### `retain_original`: +If true, the original json result from the EDR is encoded in base64 and is included as `artifact` in the observed data +and referenced from `x-oca-event` as `original_ref`. +This is useful if some of the fields in the original output are missing and the analyst would like to see the raw +output before the translation. + +## Tips for successful hunting +Be concise with what you are looking for. For example searching for a file property will be +very wide as it will look at relevant file events but also at all the binary files of +processes in the initiating process of each event and its parent process. +If you wish to look for a specific process don't query for a file, instead query the process. +For example the query `[process:pid=123]` will only look at the initiating process id. +If you want to search for a process which its parent is called `foo`look for +`[process:parent_ref.name='foo']`. This will bring all the relevant events from processes which +their parent process is `foo` only. + +If you wish to search for a file from the device file events +(a file that has been created, modified etc.) but not search the process binary fields from all +other event tables query for: +`[x-oca-event:file_ref.name='foo']` + +You can query for specific event types by querying the `action` field of `x-oca-event`. +For example if I wish to search for all `RegistryValueSet` events I will query: +`[x-oca-event:action = 'RegistryValueSet' ]` + +If you want to query events on a specific host, provide its hostname as such: +`[x-oca-asset:hostname = 'host.example.com' ]` + +Searching for an IP address `[ipv4-addr:value = '9.9.9.9' ]` is more general as it will search both source and +destination ip fields. If you know the IP usage it is better to narrow the query. +For example for a remote IP use: `[network-traffic:dst_ref.value = '9.9.9.9' ]`. +For a local IP use `[network-traffic:src_ref.value = '9.9.9.1' ]` + +## Examples +Here are a few examples for results from different tables and how to interpret them: + +### DeviceProcessEvents + +In this example all connection options are set to `true`. +The main starting point to look at will be `x-oca-event:action` which tells us the +event is a process create event. + +the `process_ref` will lead us to the process that was created and its `parent_ref` +to the process which created it. + +`host_ref` leads us to `x-oca-asset` which shows information on the host on which this +event occurred: its name, unique device id, and the ip addresses and mac addresses +that were observed on the device. + +`created` shows us the timestamp when this event was observed. + +`user_ref` shows us the user account that was performing the event. `account_login` is mapped +to the user UPN which is resolved by msatp from the active directory. `user_id` is mapped +to the account name. + +`external_ref` is a link to the msatp web console that will show us the timeline of this device +one second before and after the event occurred. + +`original_ref` is the original json output from the msatp api response encoded in base64. +If there are fields you are interested to check but are not mapped to stix you can find them +there. +```json +{ + "0": + { + "type": "x-msatp", + "ReportId": 1234 + }, + "1": + { + "type": "x-oca-asset", + "hostname": "host.test.com", + "device_id": "deviceid", + "mac_refs": ["13"], + "ip_refs": ["14"], + "architecture": "64-bit", + "os_name": "Windows10", + "os_version": "10.0" + }, + "2": + { + "type": "x-oca-event", + "host_ref": "1", + "created": "2023-03-17T20:23:03.7116107Z", + "action": "ProcessCreated", + "process_ref": "4", + "user_ref": "7", + "provider": "Microsoft Defender for Endpoint", + "external_ref": "15", + "original_ref": "16" + }, + "3": + { + "type": "file", + "name": "msedge.exe", + "parent_directory_ref": "6", + "hashes": + { + "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed", + "SHA-256": "470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75", + "MD5": "e180c9a532c45eba99eefd01601f5c41" + } + }, + "4": + { + "type": "process", + "name": "msedge.exe", + "binary_ref": "3", + "pid": 37384, + "command_line": "\"msedge.exe\" --type=gpu-process", + "created": "2023-03-17T20:23:03.702Z", + "creator_user_ref": "7", + "parent_ref": "5" + }, + "5": + { + "type": "process", + "child_refs": ["4"], + "creator_user_ref": "8", + "binary_ref": "9", + "name": "msedge.exe", + "pid": 400, + "command_line": "\"msedge.exe\" -- \"https://test.com/login/login.asp\"", + "created": "2023-03-17T20:23:03.441Z", + "parent_ref": "11" + }, + "6": + { + "type": "directory", + "path": "C:\\Program Files (x86)\\Microsoft\\Edge\\Application" + }, + "7": + { + "type": "user-account", + "user_id": "username", + "account_login": "username@test.com" + }, + "8": + { + "type": "user-account", + "user_id": "username", + "account_login": "username@test.com" + }, + "9": + { + "type": "file", + "hashes": + { + "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed", + "SHA-256": "470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75", + "MD5": "e180c9a532c45eba99eefd01601f5c41" + }, + "name": "msedge.exe", + "parent_directory_ref": "10" + }, + "10": + { + "type": "directory", + "path": "c:\\program files (x86)\\microsoft\\edge\\application" + }, + "11": + { + "type": "process", + "pid": 30972, + "name": "iexplore.exe", + "binary_ref": "12", + "created": "2023-03-17T20:23:03.169Z" + }, + "12": + { + "type": "file", + "name": "iexplore.exe" + }, + "13": + { + "type": "mac-addr", + "value": "11:22:33:44:55:66" + }, + "14": + { + "type": "ipv4-addr", + "value": "9.9.9.1" + }, + "15": + { + "type": "external-reference", + "url": "https://security.microsoft.com/machines/deviceid/timeline?from=2023-03-17T20:23:02.000Z&to=2023-03-17T20:23:04.000Z" + }, + "16": + { + "type": "artifact", + "payload_bin": "base64 encoded json output from msatp api response before translation" + } +} +``` + +This same event with all connection options set to `false` will be missing the host ip and mac addresses, the host OS details and the `original_ref` in `x-oca-event`: +```json +{ + "0": + { + "type": "x-msatp", + "ReportId": 1234 + }, + "1": + { + "type": "x-oca-asset", + "hostname": "host.test.com", + "device_id": "deviceid" + }, + "2": + { + "type": "x-oca-event", + "host_ref": "1", + "created": "2023-03-17T20:23:03.7116107Z", + "action": "ProcessCreated", + "process_ref": "4", + "user_ref": "7", + "provider": "Microsoft Defender for Endpoint", + "external_ref": "13" + }, + "3": + { + "type": "file", + "name": "msedge.exe", + "parent_directory_ref": "6", + "hashes": + { + "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed", + "SHA-256": "470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75", + "MD5": "e180c9a532c45eba99eefd01601f5c41" + } + }, + "4": + { + "type": "process", + "name": "msedge.exe", + "binary_ref": "3", + "pid": 37384, + "command_line": "\"msedge.exe\" --type=gpu-process", + "created": "2023-03-17T20:23:03.702Z", + "creator_user_ref": "7", + "parent_ref": "5" + }, + "5": + { + "type": "process", + "child_refs": ["4"], + "creator_user_ref": "8", + "binary_ref": "9", + "name": "msedge.exe", + "pid": 400, + "command_line": "\"msedge.exe\" -- \"https://test.com/login/login.asp\"", + "created": "2023-03-17T20:23:03.441Z", + "parent_ref": "11" + }, + "6": + { + "type": "directory", + "path": "C:\\Program Files (x86)\\Microsoft\\Edge\\Application" + }, + "7": + { + "type": "user-account", + "user_id": "username", + "account_login": "username@test.com" + }, + "8": + { + "type": "user-account", + "user_id": "username", + "account_login": "username@test.com" + }, + "9": + { + "type": "file", + "hashes": + { + "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed", + "SHA-256": "470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75", + "MD5": "e180c9a532c45eba99eefd01601f5c41" + }, + "name": "msedge.exe", + "parent_directory_ref": "10" + }, + "10": + { + "type": "directory", + "path": "c:\\program files (x86)\\microsoft\\edge\\application" + }, + "11": + { + "type": "process", + "pid": 30972, + "name": "iexplore.exe", + "binary_ref": "12", + "created": "2023-03-17T20:23:03.169Z" + }, + "12": + { + "type": "file", + "name": "iexplore.exe" + }, + "13": + { + "type": "external-reference", + "url": "https://security.microsoft.com/machines/deviceid/timeline?from=2023-03-17T20:23:02.000Z&to=2023-03-17T20:23:04.000Z" + } +} +``` + + +### DeviceNetworkEvents with associated alert + +In this example all connection options are set to `true`. +An example including a network event to an url. +Query example: `[domain-name:value='malicious.com']` + +`url_ref` points us to the url visited. +`domain_ref` to the domain parsed from the url. +It is possible that only one of these will be populated depending on the data provided. + +`network_ref` will show us the source and destination ip addresses and ports. Mind you that +the destination ip address is the one resolved from the domain name +referenced by `domain_ref` in the `x-oca-event`. + +`finding_refs` indicates that there are associated alerts with this event. These will be +of type `x-ibm-finding`. The event is a connection success event, and the associated alert +title is `Connection to adversary-in-the-middle (AiTM) phishing site`. + +`severity` is translated from `high` (99) `medium` (66) and `low` (33). + +`alert_id` is the unique alert identifier from msatp. + +`ttp_tagging_refs` shows us the relevant TTP's. These will be of type `x-ibm-ttp-tagging`. +Tactics will be mapped to `kill_chain_phases` which might be mapped to the +mitre attack framework or a proprietary Microsoft phase - not all phases are mapped to +mitre attack. +Techniques will be mapped to the `mitre-attack-ext` extension and will describe the name and +id of the mitre attack technique. + +```json +{ + "0": + { + "type": "x-oca-event", + "created": "2023-03-17T20:19:46.6337905Z", + "host_ref": "1", + "process_ref": "4", + "action": "ConnectionSuccess", + "network_ref": "7", + "domain_ref": "9", + "url_ref": "8", + "provider": "Microsoft Defender for Endpoint", + "external_ref": "17", + "finding_refs": ["18"], + "original_ref": "20", + "file_ref": "2" + }, + "1": + { + "type": "x-oca-asset", + "device_id": "deviceid", + "hostname": "host.example.com", + "ip_refs": ["10"], + "mac_refs": ["16"], + "architecture": "64-bit", + "os_name": "Windows10", + "os_version": "10.0" + }, + "2": + { + "type": "file", + "name": "msedge.exe", + "hashes": + { + "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed" + } + }, + "4": + { + "type": "process", + "child_refs": ["3"], + "binary_ref": "11", + "name": "msedge.exe", + "pid": 3052, + "command_line": "\"msedge.exe\" --type=utility", + "created": "2023-03-13T14:22:44.860Z", + "parent_ref": "14", + "creator_user_ref": "15" + }, + "5": + { + "type": "x-msatp", + "ReportId": 1234, + "Table": "DeviceEvents" + }, + "6": + { + "type": "ipv4-addr", + "value": "9.9.9.9" + }, + "7": + { + "type": "network-traffic", + "dst_ref": "6", + "dst_port": 443, + "src_ref": "10", + "src_port": 58993 + }, + "8": + { + "type": "url", + "value": "https://malicious.com" + }, + "9": + { + "type": "domain-name", + "value": "malicious.com" + }, + "10": + { + "type": "ipv4-addr", + "value": "9.9.9.1" + }, + "11": + { + "type": "file", + "hashes": + { + "SHA-1": "c737742b81292c764ac2a7e419a37ed7fdf4a1ed", + "SHA-256": "470ea019c1ea8882b258dea27e77261dd297eb225fd08edbe591c82796189d75", + "MD5": "e180c9a532c45eba99eefd01601f5c41" + }, + "name": "msedge.exe", + "parent_directory_ref": "12" + }, + "12": + { + "type": "directory", + "path": "c:\\program files (x86)\\microsoft\\edge\\application" + }, + "13": + { + "type": "file", + "name": "msedge.exe" + }, + "14": + { + "type": "process", + "name": "msedge.exe", + "binary_ref": "13", + "pid": 9952, + "created": "2023-03-13T14:22:44.508Z" + }, + "15": + { + "type": "user-account", + "user_id": "username", + "account_login": "username@test.com" + }, + "16": + { + "type": "mac-addr", + "value": "11:22:33:44:55:66" + }, + "17": + { + "type": "external-reference", + "url": "https://security.microsoft.com/machines/deviceid/timeline?from=2023-03-17T20:19:45.000Z&to=2023-03-17T20:19:47.000Z" + }, + "18": + { + "type": "x-ibm-finding", + "alert_id": "1234567890-1234-1234-1234-123456789012_1", + "severity": 99, + "name": "Connection to adversary-in-the-middle (AiTM) phishing site", + "ttp_tagging_refs": ["19","21"] + }, + "19": + { + "type": "x-ibm-ttp-tagging", + "kill_chain_phases": [ + { + "phase_name": "Credential Access", + "kill_chain_name": "mitre-attack" + }] + }, + "20": + { + "type": "artifact", + "payload_bin": "base64 encoded json from the msatp api before translation" + }, + "21": + { + "type": "x-ibm-ttp-tagging", + "extensions": + { + "mitre-attack-ext": + { + "technique_name": "Spearphishing Link", + "technique_id": "T1566.002" + } + } + } +} +``` + +### March 2023 changes +Changes made from previous version: +- `DeviceEvents` and `DeviceImageLoadEvents` tables where added to the search query. +Prior they were not being searched. +- `DeviceAlertEvents` is being queried in correlation with the events. +- `DeviceNetworkInfo` is being queried to extract the device network addresses at the time of the event +- `DeviceInfo` is queried to extract the device OS details +- `domain-name` was mapped to `DeviceName` causing a mismatch between looking for events on a specific host and looking for domains in network events. The mapping to device name was removed. use `x-oca-asset:hostname` instead to search for a specific device. +- `process` fields such as `name` and `pid` were looking at both the process and its parent. This causes an overload of results. Say I searched for all events by process pid 123 - I would receive also all events of its child processes. This was removed. now searching a process property will not look at the parent process fields. To search for events where the parent process is X search for `process:parent_ref.X` +- `user-account:account_login` was mapped to `AccountName` which is the local account name - which is now mapped to `user-account:user_id`. The `AccountUpn` which is the more meaningful field since it is resolved from active directory and usually contains for users full email is now mapped to `account_login`. The `AccountSid` (in the form of S-1-5- etc.) which was mapped to `user_id` is no longer mapped to any stix field. +- `x-msatp` `computer_name` and `machine_id` are removed as they map to `x-oca-asset` now. +- `x-oca-asset:ip` was switched to `x-oca-asset:ip_refs` array as per the official spec and now support multiple ip addresses for one host. +- `DeviceFileEvents.FileOriginUrl` and `DeviceFileEvents.FileOriginReferrerUrl` were added to the mappings - they were not mapped before. +- `x-oca-event:provider` will always state 'Microsoft Defender for Endpoint' to indicate the source of the event +- `x-oca-event:external_ref` was added and provides a link to the msatp web console filtered one second before and after the event. This is handy in case the analyst would like to look at the source data directly in the msatp console. +- event name was mapped to `x-oca-event:action` +- `x-oca-event:original_ref` was added and includes the raw json output of the msatp api encoded in base64. This is handy to find details that are not mapped to stix. +- `x-msatp:AdditionalFields` was added as this field usually contains important information. + + ## Example STIX pattern for file query: #### STIX patterns: @@ -64,15 +616,15 @@ 1. `(find withsource = TableName in (DeviceProcessEvents) where Timestamp >= datetime(2019-10-01T08:43:10.003Z) and Timestamp < datetime(2019-10-30T10:43:10.003Z) | order by Timestamp desc | where (FileName =~ "conhost.exe") or (FolderPath contains "C:\\ProgramData\\Symantec" or InitiatingProcessFolderPath contains "C:\\ProgramData\\Symantec"))` -## Example STIX pattern for Custom Attribute(x-msatp) query: +## Example STIX pattern for device and event query: #### STIX patterns: - 1. `[x-msatp:computer_name = 'ds-win10' OR process:name = 'conhost.exe'] START t'2019-10-01T08:43:10.003Z' STOP t'2019-10-30T10:43:10.003Z'` + 1. `[x-oca-asset:hostname = 'ds-win10' OR x-oca-event:action = 'ConnectionSuccess'] START t'2019-10-01T08:43:10.003Z' STOP t'2019-10-30T10:43:10.003Z'` #### Translated query: - 1. `(find withsource = TableName in (DeviceProcessEvents) where Timestamp >= datetime(2019-10-01T08:43:10.003Z) and Timestamp < datetime(2019-10-30T10:43:10.003Z) | order by Timestamp desc | where (FileName =~ "conhost.exe") or (DeviceName =~ "ds-win10"))` + 1. `'union (find withsource = TableName in (DeviceProcessEvents) where Timestamp >= datetime(2019-10-01T08:43:10.003Z) and Timestamp < datetime(2019-10-30T10:43:10.003Z) | order by Timestamp desc | where (DeviceName =~ "ds-win10") or (ActionType =~ "ConnectionSuccess")),(find withsource = TableName in (DeviceEvents) where Timestamp >= datetime(2019-10-01T08:43:10.003Z) and Timestamp < datetime(2019-10-30T10:43:10.003Z) | order by Timestamp desc | where (DeviceName =~ "ds-win10") or (ActionType =~ "ConnectionSuccess")),(find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(2019-10-01T08:43:10.003Z) and Timestamp < datetime(2019-10-30T10:43:10.003Z) | order by Timestamp desc | where (DeviceName =~ "ds-win10") or (ActionType =~ "ConnectionSuccess")),(find withsource = TableName in (DeviceRegistryEvents) where Timestamp >= datetime(2019-10-01T08:43:10.003Z) and Timestamp < datetime(2019-10-30T10:43:10.003Z) | order by Timestamp desc | where (DeviceName =~ "ds-win10") or (ActionType =~ "ConnectionSuccess")),(find withsource = TableName in (DeviceFileEvents) where Timestamp >= datetime(2019-10-01T08:43:10.003Z) and Timestamp < datetime(2019-10-30T10:43:10.003Z) | order by Timestamp desc | where (DeviceName =~ "ds-win10") or (ActionType =~ "ConnectionSuccess")),(find withsource = TableName in (DeviceImageLoadEvents) where Timestamp >= datetime(2019-10-01T08:43:10.003Z) and Timestamp < datetime(2019-10-30T10:43:10.003Z) | order by Timestamp desc | where (DeviceName =~ "ds-win10") or (ActionType =~ "ConnectionSuccess")),(find withsource = TableName in (DeviceLogonEvents) where Timestamp >= datetime(2019-10-01T08:43:10.003Z) and Timestamp < datetime(2019-10-30T10:43:10.003Z) | order by Timestamp desc | where (DeviceName =~ "ds-win10"))'` ## Example STIX pattern for Combined Observation: @@ -113,7 +665,41 @@ results #### MSATP query result (Result is formatted by STIX transmission result module): ``` -[{'DeviceFileEvents': {'Timestamp': '2019-10-30T08:44:56.8059397Z', 'DeviceId': 'babec47c12ad732b595e803c3320cc32aa26af33', 'DeviceName': 'lp-5cd84714zn.hclt.corp.hcl.in', 'ActionType': 'FileCreated', 'FileName': 'DismHost.exe', 'FolderPath': 'C:\\Users\\annishprashan.stevi\\AppData\\Local\\Temp\\44556507-AAE8-4AB2-B7C4-4519CC19443C\\DismHost.exe', 'SHA1': 'c34af1caf92b54737956e4940582bcce5cca2725', 'MD5': '2a1ee8df1dd0335605dcc5015c60ebc0', 'InitiatingProcessAccountDomain': 'hcltech', 'InitiatingProcessAccountName': 'annishprashan.stevi', 'InitiatingProcessAccountSid': 'S-1-5-21-333653013-2304839960-3876203932-1269283', 'InitiatingProcessMD5': '062ec57fe7f4463161d9e6ef400b2a3e', 'InitiatingProcessSHA1': '2eb39003998f0e518ad937db120b87e81d5a5893', 'InitiatingProcessFolderPath': 'c:\\windows\\system32\\cleanmgr.exe', 'InitiatingProcessFileName': 'cleanmgr.exe', 'InitiatingProcessId': 19224, 'InitiatingProcessCommandLine': 'cleanmgr.exe /autoclean /d C:', 'InitiatingProcessCreationTime': '2019-10-30T08:44:33.2836067Z', 'InitiatingProcessIntegrityLevel': 'High', 'InitiatingProcessTokenElevation': 'TokenElevationTypeFull', 'InitiatingProcessParentId': 2392, 'InitiatingProcessParentFileName': 'svchost.exe', 'InitiatingProcessParentCreationTime': '2019-10-28T18:44:18.1496667Z', 'RequestProtocol': 'Unknown', 'ReportId': 17363, 'rn': 1, 'event_count': '1'}}, {'FileCreationEvents': {'Timestamp': '2019-10-30T08:40:53.6099192Z', 'DeviceId': 'fc0842373e54e76f5c55830e47526f6f1c187be6', 'DeviceName': 'car-dev-win', 'ActionType': 'FileCreated', 'FileName': 'DismHost.exe', 'FolderPath': 'C:\\Windows\\Temp\\9C957DFF-D551-4542-9D22-556A347F3B5B\\DismHost.exe', 'SHA1': 'b01d428264a51ae803814644ea5ea43e7d7781d5', 'MD5': 'e8007eb8977e83d29f30a122771c09aa', 'InitiatingProcessAccountDomain': 'nt authority', 'InitiatingProcessAccountName': 'system', 'InitiatingProcessAccountSid': 'S-1-5-18', 'InitiatingProcessMD5': '3cb2aa46bb2f56352ee7d39886db1160', 'InitiatingProcessSHA1': '68931a7ec5bdfb7f6b7a85d1273b100456ae9ae7', 'InitiatingProcessFolderPath': 'c:\\program files\\microsoft monitoring agent\\agent\\health service state\\monitoring host temporary files 48\\761\\mssenses.exe', 'InitiatingProcessFileName': 'MsSenseS.exe', 'InitiatingProcessId': 4956, 'InitiatingProcessCommandLine': '"MsSenseS.exe"', 'InitiatingProcessCreationTime': '2019-10-23T06:55:53.1651027Z', 'InitiatingProcessIntegrityLevel': 'System', 'InitiatingProcessTokenElevation': 'TokenElevationTypeDefault', 'InitiatingProcessParentId': 3540, 'InitiatingProcessParentFileName': 'MonitoringHost.exe', 'InitiatingProcessParentCreationTime': '2019-10-22T10:55:48.4451712Z', 'RequestProtocol': 'Unknown', 'ReportId': 462, 'rn': 2, 'event_count': '1'}}] +[{ + 'DeviceFileEvents': + { + 'TableName': 'DeviceFileEvents' + 'Timestamp': '2019-10-30T08:44:56.8059397Z', + 'DeviceId': 'babec47c12ad732b595e803c3320cc32aa26af33', + 'DeviceName': 'lp-5cd84714zn.hclt.corp.hcl.in', + 'ActionType': 'FileCreated', + 'FileName': 'DismHost.exe', + 'FolderPath': 'C:\\Users\\annishprashan.stevi\\AppData\\Local\\Temp\\44556507-AAE8-4AB2-B7C4-4519CC19443C\\DismHost.exe', + 'SHA1': 'c34af1caf92b54737956e4940582bcce5cca2725', + 'MD5': '2a1ee8df1dd0335605dcc5015c60ebc0', + 'InitiatingProcessAccountDomain': 'hcltech', + 'InitiatingProcessAccountName': 'annishprashan.stevi', + 'InitiatingProcessAccountUpn': 'annishprashan.stevi@example.com', + 'InitiatingProcessAccountSid': 'S-1-5-21-333653013-2304839960-3876203932-1269283', + 'InitiatingProcessMD5': '062ec57fe7f4463161d9e6ef400b2a3e', + 'InitiatingProcessSHA1': '2eb39003998f0e518ad937db120b87e81d5a5893', + 'InitiatingProcessFolderPath': 'c:\\windows\\system32\\cleanmgr.exe', + 'InitiatingProcessFileName': 'cleanmgr.exe', + 'InitiatingProcessId': 19224, + 'InitiatingProcessCommandLine': 'cleanmgr.exe /autoclean /d C:', + 'InitiatingProcessCreationTime': '2019-10-30T08:44:33.2836067Z', + 'InitiatingProcessIntegrityLevel': 'High', + 'InitiatingProcessTokenElevation': 'TokenElevationTypeFull', + 'InitiatingProcessParentId': 2392, + 'InitiatingProcessParentFileName': 'svchost.exe', + 'InitiatingProcessParentCreationTime': '2019-10-28T18:44:18.1496667Z', + 'RequestProtocol': 'Unknown', + 'ReportId': 17363, + 'AdditionalFields': '' + 'rn': 1, + 'event_count': '1' + } +}] ``` #### STIX observable output: @@ -129,74 +715,6 @@ results "name": "msatp", "identity_class ": "events" }, - { - "id": "observed-data--c7a04ed8-755f-468e-b9ec-41897182ea18", - "type": "observed-data", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "created": "2019-11-04T07:40:17.431Z", - "modified": "2019-11-04T07:40:17.431Z", - "objects": { - "0": { - "type": "file", - "name": "DismHost.exe", - "parent_directory_ref": "1", - "hashes": { - "SHA-1": "c34af1caf92b54737956e4940582bcce5cca2725", - "MD5": "2a1ee8df1dd0335605dcc5015c60ebc0" - } - }, - "1": { - "type": "directory", - "path": "C:\\Users\\annishprashan.stevi\\AppData\\Local\\Temp\\44556507-AAE8-4AB2-B7C4-4519CC19443C" - }, - "2": { - "type": "user-account", - "account_login": "annishprashan.stevi", - "user_id": "S-1-5-21-333653013-2304839960-3876203932-1269283" - }, - "3": { - "type": "process", - "creator_user_ref": "2", - "binary_ref": "4", - "name": "cleanmgr.exe", - "pid": 19224, - "command_line": "cleanmgr.exe /autoclean /d C:", - "created": "2019-10-30T08:44:33.283Z", - "parent_ref": "6" - }, - "4": { - "type": "file", - "hashes": { - "MD5": "062ec57fe7f4463161d9e6ef400b2a3e", - "SHA-1": "2eb39003998f0e518ad937db120b87e81d5a5893" - }, - "parent_directory_ref": "5", - "name": "cleanmgr.exe" - }, - "5": { - "type": "directory", - "path": "c:\\windows\\system32" - }, - "6": { - "type": "process", - "pid": 2392, - "name": "svchost.exe", - "binary_ref": "7", - "created": "2019-10-28T18:44:18.149Z" - }, - "7": { - "type": "file", - "name": "svchost.exe" - } - }, - "first_observed": "2019-10-30T08:44:56.805Z", - "last_observed": "2019-10-30T08:44:56.805Z", - "x-msatp": { - "machine_id": "babec47c12ad732b595e803c3320cc32aa26af33", - "computer_name": "lp-5cd84714zn.hclt.corp.hcl.in" - }, - "number_observed": 1 - }, { "id": "observed-data--f337a503-71b1-4592-bf5c-e1b43a91cf66", "type": "observed-data", @@ -219,8 +737,8 @@ results }, "2": { "type": "user-account", - "account_login": "system", - "user_id": "S-1-5-18" + "account_login": "annishprashan.stevi@example.com", + "user_id": "annishprashan.stevi" }, "3": { "type": "process", @@ -255,201 +773,37 @@ results "7": { "type": "file", "name": "MonitoringHost.exe" - } - }, - "first_observed": "2019-10-30T08:40:53.609Z", - "last_observed": "2019-10-30T08:40:53.609Z", - "x-msatp": { - "machine_id": "fc0842373e54e76f5c55830e47526f6f1c187be6", - "computer_name": "car-dev-win" - }, - "number_observed": 1 - } - ] -} - -``` - -### Combined observation expression: - -#### STIX patterns: - -``` -([file:name = 'AM_Delta_Patch_1.301.613.0.exe' AND file:hashes.'SHA-1' = 'c98dbe4cb8caad5a521915f6e3f82197d53030ee'] AND [file:name MATCHES 'mpas.*' AND file:hashes.'MD5' = 'b3b863d8c5c2f3605a5b25adec80f0de']) START t'2019-09-01T08:43:10.003Z' STOP t'2019-10-25T10:43:10.003Z' -``` - -#### Translated query: - -``` -union (find withsource = TableName in (DeviceFileEvents) where Timestamp >= datetime(2019-09-01T08:43:10.003Z) and Timestamp < datetime(2019-10-25T10:43:10.003Z) | order by Timestamp desc | where (SHA1 =~ "c98dbe4cb8caad5a521915f6e3f82197d53030ee" or InitiatingProcessSHA1 =~ "c98dbe4cb8caad5a521915f6e3f82197d53030ee") and (FileName =~ "AM_Delta_Patch_1.301.613.0.exe" or InitiatingProcessFileName =~ "AM_Delta_Patch_1.301.613.0.exe" or InitiatingProcessParentFileName =~ "AM_Delta_Patch_1.301.613.0.exe")),(find withsource = TableName in (FileCreationEvents) where Timestamp >= datetime(2019-09-01T08:43:10.003Z) and Timestamp < datetime(2019-10-25T10:43:10.003Z) | order by Timestamp desc | where (MD5 =~ "b3b863d8c5c2f3605a5b25adec80f0de" or InitiatingProcessMD5 =~ "b3b863d8c5c2f3605a5b25adec80f0de") and (FileName matches regex"(mpas.*)" or InitiatingProcessFileName matches regex"(mpas.*)" or InitiatingProcessParentFileName matches regex"(mpas.*)")) -``` - -#### Above translated query is passed as parameter to STIX transmission module - -``` -transmit msatp "{\"host\":\"xx.xx.xx.xx\",\"port\": \"xxxx\"}" -"{\"auth\":{\"tenant\": \"xxxxx\", \"clientId\": \"xxxxx\", \"clientSecret\":\"xxxxxxxx\"}}" -results -"union (find withsource = TableName in (DeviceFileEvents) where Timestamp >= datetime(2019-09-01T08:43:10.003Z) and Timestamp < datetime(2019-10-25T10:43:10.003Z) | order by Timestamp desc | where (SHA1 =~ \"c98dbe4cb8caad5a521915f6e3f82197d53030ee\" or InitiatingProcessSHA1 =~ \"c98dbe4cb8caad5a521915f6e3f82197d53030ee\") and (FileName =~ \"AM_Delta_Patch_1.301.613.0.exe\" or InitiatingProcessFileName =~ \"AM_Delta_Patch_1.301.613.0.exe\" or InitiatingProcessParentFileName =~ \"AM_Delta_Patch_1.301.613.0.exe\")),(find withsource = TableName in (FileCreationEvents) where Timestamp >= datetime(2019-09-01T08:43:10.003Z) and Timestamp < datetime(2019-10-25T10:43:10.003Z) | order by Timestamp desc | where (MD5 =~ \"b3b863d8c5c2f3605a5b25adec80f0de\" or InitiatingProcessMD5 =~ \"b3b863d8c5c2f3605a5b25adec80f0de\") and (FileName matches regex\"(mpas.*)\" or InitiatingProcessFileName matches regex\"(mpas.*)\" or InitiatingProcessParentFileName matches regex\"(mpas.*)\"))" 0 2 -``` - -#### MSATP query result (Result is formatted by STIX transmission result module): - -``` -[{'DeviceFileEvents': {'Timestamp': '2019-09-05T23:01:31.7318577Z', 'DeviceId': '6a55e432bd1d390e9080fa0930f4e45292b95bf8', 'DeviceName': 'testmachine2', 'ActionType': 'FileCreated', 'FileName': 'AM_Delta_Patch_1.301.613.0.exe', 'FolderPath': 'C:\\Windows\\SoftwareDistribution\\Download\\Install\\AM_Delta_Patch_1.301.613.0.exe', 'SHA1': 'c98dbe4cb8caad5a521915f6e3f82197d53030ee', 'MD5': 'c59babc2f0acfb68a2f2d51657387a3b', 'InitiatingProcessAccountDomain': 'nt authority', 'InitiatingProcessAccountName': 'system', 'InitiatingProcessAccountSid': 'S-1-5-18', 'InitiatingProcessMD5': '9520a99e77d6196d0d09833146424113', 'InitiatingProcessSHA1': '75c5a97f521f760e32a4a9639a653eed862e9c61', 'InitiatingProcessFolderPath': 'c:\\windows\\system32\\svchost.exe', 'InitiatingProcessFileName': 'svchost.exe', 'InitiatingProcessId': 8876, 'InitiatingProcessCommandLine': 'svchost.exe -k netsvcs -p -s wuauserv', 'InitiatingProcessCreationTime': '2019-09-05T00:01:00.3681956Z', 'InitiatingProcessIntegrityLevel': 'System', 'InitiatingProcessTokenElevation': 'TokenElevationTypeDefault', 'InitiatingProcessParentId': 772, 'InitiatingProcessParentFileName': 'services.exe', 'InitiatingProcessParentCreationTime': '2019-09-04T09:22:24.2724516Z', 'RequestProtocol': 'Unknown', 'ReportId': 10583, 'rn': 1, 'event_count': '1'}}, {'FileCreationEvents': {'Timestamp': '2019-09-05T23:00:19.8520589Z', 'DeviceId': '77725e40c068f4649a772db7512dc6da80bd4214', 'DeviceName': 'testmachine1', 'ActionType': 'FileCreated', 'FileName': 'AM_Delta_Patch_1.301.613.0.exe', 'FolderPath': 'C:\\Windows\\SoftwareDistribution\\Download\\Install\\AM_Delta_Patch_1.301.613.0.exe', 'SHA1': 'c98dbe4cb8caad5a521915f6e3f82197d53030ee', 'MD5': 'c59babc2f0acfb68a2f2d51657387a3b', 'InitiatingProcessAccountDomain': 'nt authority', 'InitiatingProcessAccountName': 'system', 'InitiatingProcessAccountSid': 'S-1-5-18', 'InitiatingProcessMD5': '9520a99e77d6196d0d09833146424113', 'InitiatingProcessSHA1': '75c5a97f521f760e32a4a9639a653eed862e9c61', 'InitiatingProcessFolderPath': 'c:\\windows\\system32\\svchost.exe', 'InitiatingProcessFileName': 'svchost.exe', 'InitiatingProcessId': 9952, 'InitiatingProcessCommandLine': 'svchost.exe -k netsvcs -p -s wuauserv', 'InitiatingProcessCreationTime': '2019-09-05T00:21:00.1715094Z', 'InitiatingProcessIntegrityLevel': 'System', 'InitiatingProcessTokenElevation': 'TokenElevationTypeDefault', 'InitiatingProcessParentId': 776, 'InitiatingProcessParentFileName': 'services.exe', 'InitiatingProcessParentCreationTime': '2019-09-04T09:06:09.0434386Z', 'RequestProtocol': 'Unknown', 'ReportId': 11638, 'rn': 2, 'event_count': '1'}}] -``` - -#### STIX observable output: - -``` -{ - "type": "bundle", - "id": "bundle--a6674df5-feed-422e-8b5b-4725409d8f7e", - "objects": [ - { - "type": "identity", - "id": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "name": "msatp", - "identity_class": "events" - }, - { - "id": "observed-data--c9e9a101-9e96-4fe4-aba7-42ee3b958cd8", - "type": "observed-data", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "created": "2019-10-01T06:08:39.245Z", - "modified": "2019-10-01T06:08:39.245Z", - "objects": { - "0": { - "type": "file", - "name": "AM_Delta_Patch_1.301.613.0.exe", - "parent_directory_ref": "1", - "hashes": { - "SHA-1": "c98dbe4cb8caad5a521915f6e3f82197d53030ee", - "MD5": "c59babc2f0acfb68a2f2d51657387a3b" - } - }, - "1": { - "type": "directory", - "path": "C:\\Windows\\SoftwareDistribution\\Download\\Install" - }, - "2": { - "type": "user-account", - "account_login": "system", - "user_id": "S-1-5-18" - }, - "3": { - "type": "process", - "creator_user_ref": "2", - "binary_ref": "4", - "name": "svchost.exe", - "pid": 8876, - "command_line": "svchost.exe -k netsvcs -p -s wuauserv", - "created": "2019-09-05T00:01:00.368Z", - "parent_ref": "6" - }, - "4": { - "type": "file", - "hashes": { - "MD5": "9520a99e77d6196d0d09833146424113", - "SHA-1": "75c5a97f521f760e32a4a9639a653eed862e9c61" - }, - "parent_directory_ref": "5", - "name": "svchost.exe" - }, - "5": { - "type": "directory", - "path": "c:\\windows\\system32" - }, - "6": { - "type": "process", - "pid": 772, - "name": "services.exe", - "binary_ref": "7", - "created": "2019-09-04T09:22:24.272Z" - }, - "7": { - "type": "file", - "name": "services.exe" - } - }, - "first_observed": "2019-09-05T23:01:31.731Z", - "last_observed": "2019-09-05T23:01:31.731Z", - "x-msatp": { - "machine_id": "6a55e432bd1d390e9080fa0930f4e45292b95bf8", - "computer_name": "testmachine2" - }, - "number_observed": 1 - }, - { - "id": "observed-data--324d9ed3-2be5-4e59-8e5b-ec252651aa1b", - "type": "observed-data", - "created_by_ref": "identity--f431f809-377b-45e0-aa1c-6a4751cae5ff", - "created": "2019-10-01T06:08:39.245Z", - "modified": "2019-10-01T06:08:39.245Z", - "objects": { - "0": { - "type": "file", - "name": "AM_Delta_Patch_1.301.613.0.exe", - "parent_directory_ref": "1", - "hashes": { - "SHA-1": "c98dbe4cb8caad5a521915f6e3f82197d53030ee", - "MD5": "c59babc2f0acfb68a2f2d51657387a3b" - } - }, - "1": { - "type": "directory", - "path": "C:\\Windows\\SoftwareDistribution\\Download\\Install" }, - "2": { - "type": "user-account", - "account_login": "system", - "user_id": "S-1-5-18" - }, - "3": { - "type": "process", - "creator_user_ref": "2", - "binary_ref": "4", - "name": "svchost.exe", - "pid": 9952, - "command_line": "svchost.exe -k netsvcs -p -s wuauserv", - "created": "2019-09-05T00:21:00.171Z", - "parent_ref": "6" - }, - "4": { - "type": "file", - "hashes": { - "MD5": "9520a99e77d6196d0d09833146424113", - "SHA-1": "75c5a97f521f760e32a4a9639a653eed862e9c61" - }, - "parent_directory_ref": "5", - "name": "svchost.exe" + "8": { + "type": "x-oca-event", + "action": "FileCreated", + "created": "2019-10-30T08:44:56.8059397Z" + "provider": "Microsoft Defender for Endpoint", + "external_ref": "11", + "host_ref": "9", + "file_ref": "0", + "process_ref": "3", + "user_ref": "2" }, - "5": { - "type": "directory", - "path": "c:\\windows\\system32" + "9": { + "type": "x-oca-asset", + "hostname": "lp-5cd84714zn.hclt.corp.hcl.in", + "device_id": "babec47c12ad732b595e803c3320cc32aa26af33" }, - "6": { - "type": "process", - "pid": 776, - "name": "services.exe", - "binary_ref": "7", - "created": "2019-09-04T09:06:09.043Z" + "10": { + "type": "x-msatp", + "Table": "DeviceFileEvents", + "ReportId": 1234 }, - "7": { - "type": "file", - "name": "services.exe" + "11": { + "type": "external-reference", + "url": "https://security.microsoft.com/machines/babec47c12ad732b595e803c3320cc32aa26af33/timeline?from=2019-10-30T08:40:52.000Z&to=2019-10-30T08:40:54.000Z' } }, - "first_observed": "2019-09-05T23:00:19.852Z", - "last_observed": "2019-09-05T23:00:19.852Z", - "x-msatp": { - "machine_id": "77725e40c068f4649a772db7512dc6da80bd4214", - "computer_name": "testmachine1" - }, + "first_observed": "2019-10-30T08:40:53.609Z", + "last_observed": "2019-10-30T08:40:53.609Z", "number_observed": 1 } ] } - - ``` diff --git a/stix_shifter_modules/msatp/stix_translation/json/from_stix_map.json b/stix_shifter_modules/msatp/stix_translation/json/from_stix_map.json index 276cad7af..9abdc5713 100644 --- a/stix_shifter_modules/msatp/stix_translation/json/from_stix_map.json +++ b/stix_shifter_modules/msatp/stix_translation/json/from_stix_map.json @@ -368,11 +368,8 @@ "DeviceNetworkEvents.DeviceId", "DeviceRegistryEvents.DeviceId", "DeviceEvents.DeviceId", - "DeviceInfo.DeviceId", - "DeviceAlertEvents.DeviceId", "DeviceImageLoadEvents.DeviceId", - "DeviceLogonEvents.DeviceId", - "DeviceNetworkInfo.DeviceId" + "DeviceLogonEvents.DeviceId" ], "hostname": [ "DeviceFileEvents.DeviceName", @@ -380,11 +377,8 @@ "DeviceNetworkEvents.DeviceName", "DeviceRegistryEvents.DeviceName", "DeviceEvents.DeviceName", - "DeviceInfo.DeviceName", - "DeviceAlertEvents.DeviceName", "DeviceImageLoadEvents.DeviceName", - "DeviceLogonEvents.DeviceName", - "DeviceNetworkInfo.DeviceName" + "DeviceLogonEvents.DeviceName" ], "ip_refs[*].value": [ "DeviceNetworkEvents.LocalIP" @@ -539,11 +533,8 @@ "DeviceNetworkEvents.DeviceName", "DeviceRegistryEvents.DeviceName", "DeviceEvents.DeviceName", - "DeviceInfo.DeviceName", - "DeviceAlertEvents.DeviceName", "DeviceImageLoadEvents.DeviceName", - "DeviceLogonEvents.DeviceName", - "DeviceNetworkInfo.DeviceName" + "DeviceLogonEvents.DeviceName" ], "host_ref.device_id": [ "DeviceFileEvents.DeviceId", @@ -551,11 +542,8 @@ "DeviceNetworkEvents.DeviceId", "DeviceRegistryEvents.DeviceId", "DeviceEvents.DeviceId", - "DeviceInfo.DeviceId", - "DeviceAlertEvents.DeviceId", "DeviceImageLoadEvents.DeviceId", - "DeviceLogonEvents.DeviceId", - "DeviceNetworkInfo.DeviceId" + "DeviceLogonEvents.DeviceId" ] } }, diff --git a/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/from_stix_map.json b/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/from_stix_map.json index b8919e0ae..9abdc5713 100644 --- a/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/from_stix_map.json +++ b/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/from_stix_map.json @@ -3,7 +3,9 @@ "fields": { "value": [ "DeviceNetworkEvents.LocalIP", - "DeviceNetworkEvents.RemoteIP" + "DeviceNetworkEvents.RemoteIP", + "DeviceEvents.RemoteIP", + "DeviceEvents.LocalIP" ] } }, @@ -11,50 +13,51 @@ "fields": { "value": [ "DeviceNetworkEvents.LocalIP", - "DeviceNetworkEvents.RemoteIP" + "DeviceNetworkEvents.RemoteIP", + "DeviceEvents.RemoteIP", + "DeviceEvents.LocalIP" ] } }, "network-traffic": { "fields": { "src_port": [ - "DeviceNetworkEvents.LocalPort" + "DeviceNetworkEvents.LocalPort", + "DeviceEvents.LocalPort" ], "dst_port": [ - "DeviceNetworkEvents.RemotePort" + "DeviceNetworkEvents.RemotePort", + "DeviceEvents.RemotePort" ], "protocols[*]": [ "DeviceNetworkEvents.Protocol" ], "src_ref.value": [ "DeviceNetworkEvents.LocalIP", - "DeviceNetworkInfo.MacAddress" + "DeviceNetworkInfo.MacAddress", + "DeviceEvents.LocalIP" ], "dst_ref.value": [ - "DeviceNetworkEvents.RemoteIP" + "DeviceNetworkEvents.RemoteIP", + "DeviceEvents.RemoteIP" ] } }, "url": { "fields": { "value": [ - "DeviceNetworkEvents.RemoteUrl" + "DeviceNetworkEvents.RemoteUrl", + "DeviceEvents.RemoteUrl", + "DeviceFileEvents.FileOriginUrl", + "DeviceFileEvents.FileOriginReferrerUrl" ] } }, "domain-name": { "fields": { "value": [ - "DeviceFileEvents.DeviceName", - "DeviceProcessEvents.DeviceName", - "DeviceNetworkEvents.DeviceName", - "DeviceRegistryEvents.DeviceName", - "DeviceEvents.DeviceName", - "DeviceInfo.DeviceName", - "DeviceAlertEvents.DeviceName", - "DeviceImageLoadEvents.DeviceName", - "DeviceLogonEvents.DeviceName", - "DeviceNetworkInfo.DeviceName" + "DeviceNetworkEvents.RemoteUrl", + "DeviceEvents.RemoteUrl" ] } }, @@ -70,15 +73,25 @@ "DeviceNetworkEvents.InitiatingProcessFileName", "DeviceNetworkEvents.InitiatingProcessParentFileName", "DeviceRegistryEvents.InitiatingProcessFileName", - "DeviceRegistryEvents.InitiatingProcessParentFileName" + "DeviceRegistryEvents.InitiatingProcessParentFileName", + "DeviceEvents.FileName", + "DeviceEvents.InitiatingProcessFileName", + "DeviceEvents.InitiatingProcessParentFileName", + "DeviceImageLoadEvents.FileName", + "DeviceImageLoadEvents.InitiatingProcessFileName", + "DeviceImageLoadEvents.InitiatingProcessParentFileName" ], "hashes.'SHA-1'": [ - "DeviceFileEvents.'SHA1'", + "DeviceFileEvents.SHA1", "DeviceFileEvents.InitiatingProcessSHA1", "DeviceProcessEvents.SHA1", "DeviceProcessEvents.InitiatingProcessSHA1", "DeviceNetworkEvents.InitiatingProcessSHA1", - "DeviceRegistryEvents.InitiatingProcessSHA1" + "DeviceRegistryEvents.InitiatingProcessSHA1", + "DeviceEvents.SHA1", + "DeviceEvents.InitiatingProcessSHA1", + "DeviceImageLoadEvents.SHA1", + "DeviceImageLoadEvents.InitiatingProcessSHA1" ], "hashes.'SHA-256'": [ "DeviceFileEvents.SHA256", @@ -86,7 +99,11 @@ "DeviceProcessEvents.SHA256", "DeviceProcessEvents.InitiatingProcessSHA256", "DeviceNetworkEvents.InitiatingProcessSHA256", - "DeviceRegistryEvents.InitiatingProcessSHA256" + "DeviceRegistryEvents.InitiatingProcessSHA256", + "DeviceEvents.SHA256", + "DeviceEvents.InitiatingProcessSHA256", + "DeviceImageLoadEvents.SHA256", + "DeviceImageLoadEvents.InitiatingProcessSHA256" ], "hashes.MD5": [ "DeviceFileEvents.MD5", @@ -94,7 +111,11 @@ "DeviceProcessEvents.MD5", "DeviceProcessEvents.InitiatingProcessMD5", "DeviceNetworkEvents.InitiatingProcessMD5", - "DeviceRegistryEvents.InitiatingProcessMD5" + "DeviceRegistryEvents.InitiatingProcessMD5", + "DeviceEvents.MD5", + "DeviceEvents.InitiatingProcessMD5", + "DeviceImageLoadEvents.MD5", + "DeviceImageLoadEvents.InitiatingProcessMD5" ], "parent_directory_ref.path": [ "DeviceFileEvents.FolderPath", @@ -102,143 +123,217 @@ "DeviceProcessEvents.FolderPath", "DeviceProcessEvents.InitiatingProcessFolderPath", "DeviceNetworkEvents.InitiatingProcessFolderPath", - "DeviceRegistryEvents.InitiatingProcessFolderPath" + "DeviceRegistryEvents.InitiatingProcessFolderPath", + "DeviceEvents.FolderPath", + "DeviceEvents.InitiatingProcessFolderPath", + "DeviceImageLoadEvents.FolderPath", + "DeviceImageLoadEvents.InitiatingProcessFolderPath" ] } }, "process": { "fields": { - "parent_ref.name": [ - "DeviceProcessEvents.InitiatingProcessFileName", + "name": [ + "DeviceProcessEvents.FileName", + "DeviceEvents.FileName", "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceEvents.InitiatingProcessFileName", + "DeviceFileEvents.InitiatingProcessFileName", "DeviceNetworkEvents.InitiatingProcessFileName", "DeviceRegistryEvents.InitiatingProcessFileName", - "DeviceProcessEvents.InitiatingProcessParentFileName", - "DeviceProcessEvents.InitiatingProcessParentFileName", - "DeviceNetworkEvents.InitiatingProcessParentFileName", - "DeviceRegistryEvents.InitiatingProcessParentFileName" + "DeviceImageLoadEvents.InitiatingProcessFileName" ], "command_line": [ "DeviceProcessEvents.ProcessCommandLine", "DeviceProcessEvents.InitiatingProcessCommandLine", + "DeviceEvents.ProcessCommandLine", + "DeviceEvents.InitiatingProcessCommandLine", + "DeviceFileEvents.InitiatingProcessCommandLine", "DeviceNetworkEvents.InitiatingProcessCommandLine", - "DeviceRegistryEvents.InitiatingProcessCommandLine" + "DeviceRegistryEvents.InitiatingProcessCommandLine", + "DeviceImageLoadEvents.InitiatingProcessCommandLine" ], "pid": [ "DeviceProcessEvents.ProcessId", + "DeviceEvents.ProcessId", + "DeviceProcessEvents.InitiatingProcessId", + "DeviceEvents.InitiatingProcessId", "DeviceProcessEvents.InitiatingProcessId", - "DeviceProcessEvents.InitiatingProcessParentId", "DeviceNetworkEvents.InitiatingProcessId", - "DeviceNetworkEvents.InitiatingProcessParentId", "DeviceRegistryEvents.InitiatingProcessId", - "DeviceRegistryEvents.InitiatingProcessParentId", "DeviceFileEvents.InitiatingProcessId", - "DeviceFileEvents.InitiatingProcessParentId" + "DeviceImageLoadEvents.InitiatingProcessId" + ], + "created": [ + "DeviceProcessEvents.ProcessCreationTime", + "DeviceEvents.ProcessCreationTime", + "DeviceNetworkEvents.InitiatingProcessCreationTime", + "DeviceRegistryEvents.InitiatingProcessCreationTime", + "DeviceFileEvents.InitiatingProcessCreationTime", + "DeviceImageLoadEvents.InitiatingProcessCreationTime" + ], + "parent_ref.name": [ + "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceEvents.InitiatingProcessFileName", + "DeviceFileEvents.InitiatingProcessParentFileName", + "DeviceNetworkEvents.InitiatingProcessParentFileName", + "DeviceRegistryEvents.InitiatingProcessParentFileName", + "DeviceImageLoadEvents.InitiatingProcessParentFileName" ], "parent_ref.pid": [ "DeviceProcessEvents.InitiatingProcessId", "DeviceNetworkEvents.InitiatingProcessId", - "DeviceRegistryEvents.InitiatingProcessId", - "DeviceFileEvents.InitiatingProcessId", - "DeviceProcessEvents.InitiatingProcessParentId", - "DeviceNetworkEvents.InitiatingProcessParentId", "DeviceRegistryEvents.InitiatingProcessParentId", - "DeviceFileEvents.InitiatingProcessParentId" + "DeviceFileEvents.InitiatingProcessParentId", + "DeviceEvents.InitiatingProcessParentId", + "DeviceImageLoadEvents.InitiatingProcessParentId" ], - "child_refs.pid": [ - "DeviceProcessEvents.ProcessId", - "DeviceProcessEvents.InitiatingProcessId", - "DeviceNetworkEvents.InitiatingProcessId", - "DeviceRegistryEvents.InitiatingProcessId", - "DeviceFileEvents.InitiatingProcessId" + "parent_ref.created": [ + "DeviceProcessEvents.InitiatingProcessCreationTime", + "DeviceEvents.InitiatingProcessCreationTime", + "DeviceNetworkEvents.InitiatingProcessParentCreationTime", + "DeviceRegistryEvents.InitiatingProcessParentCreationTime", + "DeviceFileEvents.InitiatingProcessParentCreationTime", + "DeviceImageLoadEvents.InitiatingProcessParentCreationTime" ], - "creator_user_ref.user_id": [ - "DeviceProcessEvents.AccountSid", - "DeviceProcessEvents.InitiatingProcessAccountSid" + "parent_ref.parent_ref.name": [ + "DeviceProcessEvents.InitiatingProcessParentFileName", + "DeviceEvents.InitiatingProcessParentFileName" ], - "child_refs.creator_user_ref.account_login": ["DeviceProcessEvents.AccountName"], - "creator_user_ref.account_login": ["DeviceProcessEvents.InitiatingProcessAccountName"], - "image_ref.hashes.'SHA-1'": [ - "DeviceFileEvents.SHA1", - "DeviceFileEvents.InitiatingProcessSHA1", + "parent_ref.parent_ref.pid": [ + "DeviceProcessEvents.InitiatingProcessParentId", + "DeviceNetworkEvents.InitiatingProcessParentId" + ], + "parent_ref.parent_ref.created": [ + "DeviceProcessEvents.InitiatingProcessParentCreationTime", + "DeviceEvents.InitiatingProcessParentCreationTime" + ], + "creator_user_ref.user_id": [ + "DeviceProcessEvents.AccountName", + "DeviceEvents.AccountName", + "DeviceNetworkEvents.InitiatingProcessAccountName", + "DeviceRegistryEvents.InitiatingProcessAccountName", + "DeviceFileEvents.InitiatingProcessAccountName", + "DeviceImageLoadEvents.InitiatingProcessAccountName" + ], + "creator_user_ref.account_login": [ + "DeviceProcessEvents.AccountUpn", + "DeviceEvents.AccountUpn", + "DeviceNetworkEvents.InitiatingProcessAccountUpn", + "DeviceRegistryEvents.InitiatingProcessAccountUpn", + "DeviceFileEvents.InitiatingProcessAccountUpn", + "DeviceImageLoadEvents.InitiatingProcessAccountUpn" + ], + "parent_ref.creator_user_ref.user_id": [ + "DeviceProcessEvents.InitiatingProcessAccountName", + "DeviceEvents.InitiatingProcessAccountName" + ], + "parent_ref.creator_user_ref.account_login": [ + "DeviceProcessEvents.InitiatingProcessAccountUpn", + "DeviceEvents.InitiatingProcessAccountUpn" + ], + "binary_ref.hashes.'SHA-1'": [ "DeviceProcessEvents.SHA1", - "DeviceProcessEvents.InitiatingProcessSHA1", + "DeviceEvents.SHA1", + "DeviceFileEvents.InitiatingProcessSHA1", "DeviceNetworkEvents.InitiatingProcessSHA1", - "DeviceRegistryEvents.InitiatingProcessSHA1" + "DeviceRegistryEvents.InitiatingProcessSHA1", + "DeviceImageLoadEvents.InitiatingProcessSHA1" ], - "image_ref.hashes.'SHA-256'": [ - "DeviceFileEvents.SHA256", - "DeviceFileEvents.InitiatingProcessSHA256", + "binary_ref.hashes.'SHA-256'": [ "DeviceProcessEvents.SHA256", - "DeviceProcessEvents.InitiatingProcessSHA256", + "DeviceEvents.SHA256", + "DeviceFileEvents.InitiatingProcessSHA256", "DeviceNetworkEvents.InitiatingProcessSHA256", - "DeviceRegistryEvents.InitiatingProcessSHA256" + "DeviceRegistryEvents.InitiatingProcessSHA256", + "DeviceImageLoadEvents.InitiatingProcessSHA256" ], - "child_refs.image_ref.hashes.MD5": [ - "DeviceProcessEvents.MD5" + "binary_ref.hashes.MD5": [ + "DeviceProcessEvents.MD5", + "DeviceEvents.MD5", + "DeviceFileEvents.InitiatingProcessMD5", + "DeviceNetworkEvents.InitiatingProcessMD5", + "DeviceRegistryEvents.InitiatingProcessMD5", + "DeviceImageLoadEvents.InitiatingProcessMD5" ], - "child_refs.image_ref.hashes.'SHA-256'": [ - "DeviceProcessEvents.SHA256" + "binary_ref.parent_directory_ref.path": [ + "DeviceProcessEvents.FolderPath", + "DeviceEvents.FolderPath", + "DeviceNetworkEvents.InitiatingProcessFolderPath", + "DeviceRegistryEvents.InitiatingProcessFolderPath", + "DeviceFileEvents.InitiatingProcessFolderPath", + "DeviceImageLoadEvents.InitiatingProcessFolderPath" ], - "child_refs.image_ref.hashes.'SHA-1'": [ - "DeviceProcessEvents.SHA1" + "parent_ref.binary_ref.hashes.'SHA-1'": [ + "DeviceProcessEvents.InitiatingProcessSHA1", + "DeviceEvents.InitiatingProcessSHA1" ], - "image_ref.hashes.MD5": [ - "DeviceFileEvents.MD5", - "DeviceFileEvents.InitiatingProcessMD5", - "DeviceProcessEvents.MD5", + "parent_ref.binary_ref.hashes.'SHA-256'": [ + "DeviceProcessEvents.InitiatingProcessSHA256", + "DeviceEvents.InitiatingProcessSHA256" + ], + "parent_ref.binary_ref.hashes.MD5": [ "DeviceProcessEvents.InitiatingProcessMD5", - "DeviceNetworkEvents.InitiatingProcessMD5", - "DeviceRegistryEvents.InitiatingProcessMD5" + "DeviceEvents.InitiatingProcessMD5" ], - "created_time": [ - "DeviceProcessEvents.ProcessCreationTime", - "DeviceProcessEvents.InitiatingProcessParentCreationTime", - "DeviceNetworkEvents.InitiatingProcessCreationTime", - "DeviceNetworkEvents.InitiatingProcessParentCreationTime", - "DeviceRegistryEvents.InitiatingProcessCreationTime", - "DeviceRegistryEvents.InitiatingProcessParentCreationTime", - "DeviceFileEvents.InitiatingProcessCreationTime", - "DeviceFileEvents.InitiatingProcessParentCreationTime" + "parent_ref.binary_ref.parent_directory_ref.path": [ + "DeviceProcessEvents.InitiatingProcessFolderPath", + "DeviceEvents.InitiatingProcessFolderPath" ], - "parent_ref.created_time": [ - "DeviceProcessEvents.InitiatingProcessParentCreationTime", - "DeviceNetworkEvents.InitiatingProcessParentCreationTime", - "DeviceRegistryEvents.InitiatingProcessParentCreationTime", - "DeviceFileEvents.InitiatingProcessParentCreationTime" + "child_refs.binary_ref.hashes.MD5": [ + "DeviceProcessEvents.MD5" ], - "image_ref.parent_directory_ref.path": [ - "DeviceProcessEvents.InitiatingProcessFolderPath", - "DeviceNetworkEvents.InitiatingProcessFolderPath", - "DeviceRegistryEvents.InitiatingProcessFolderPath", - "DeviceFileEvents.InitiatingProcessFolderPath" + "child_refs.binary_ref.hashes.'SHA-256'": [ + "DeviceProcessEvents.SHA256" + ], + "child_refs.binary_ref.hashes.'SHA-1'": [ + "DeviceProcessEvents.SHA1" ], - "child_refs.image_ref.parent_directory_ref.path": [ + "child_refs.binary_ref.parent_directory_ref.path": [ "DeviceProcessEvents.FolderPath" + ], + "child_refs.creator_user_ref.account_login": [ + "DeviceProcessEvents.AccountName" + ], + "child_refs.pid": [ + "DeviceProcessEvents.ProcessId" ] } }, "user-account": { "fields": { "user_id": [ - "DeviceProcessEvents.AccountSid", - "DeviceNetworkEvents.InitiatingProcessAccountSid", - "DeviceRegistryEvents.InitiatingProcessAccountSid" - ], - "account_login": [ "DeviceProcessEvents.AccountName", + "DeviceFileEvents.RequestAccountName", + "DeviceEvents.AccountName", + "DeviceProcessEvents.InitiatingProcessAccountName", "DeviceNetworkEvents.InitiatingProcessAccountName", - "DeviceRegistryEvents.InitiatingProcessAccountName" + "DeviceRegistryEvents.InitiatingProcessAccountName", + "DeviceFileEvents.InitiatingProcessAccountName", + "DeviceEvents.InitiatingProcessAccountName", + "DeviceImageLoadEvents.InitiatingProcessAccountName" + ], + "account_login": [ + "DeviceProcessEvents.AccountUpn", + "DeviceEvents.AccountUpn", + "DeviceProcessEvents.InitiatingProcessAccountUpn", + "DeviceNetworkEvents.InitiatingProcessAccountUpn", + "DeviceRegistryEvents.InitiatingProcessAccountUpn", + "DeviceFileEvents.InitiatingProcessAccountUpn", + "DeviceEvents.InitiatingProcessAccountUpn", + "DeviceImageLoadEvents.InitiatingProcessAccountUpn" ] } }, "windows-registry-key": { "fields": { "key": [ - "DeviceRegistryEvents.RegistryKey" + "DeviceRegistryEvents.RegistryKey", + "DeviceEvents.RegistryKey" ], "values[*]": [ - "DeviceRegistryEvents.RegistryValueName" + "DeviceRegistryEvents.RegistryValueName", + "DeviceEvents.RegistryValueName" ] } }, @@ -249,22 +344,6 @@ ] } }, - "x-msatp": { - "fields": { - "computer_name": [ - "DeviceProcessEvents.DeviceName", - "DeviceNetworkEvents.DeviceName", - "DeviceRegistryEvents.DeviceName", - "DeviceFileEvents.DeviceName" - ], - "machine_id": [ - "DeviceProcessEvents.DeviceId", - "DeviceNetworkEvents.DeviceId", - "DeviceRegistryEvents.DeviceId", - "DeviceFileEvents.DeviceId" - ] - } - }, "directory": { "fields": { "path": [ @@ -272,24 +351,25 @@ "DeviceFileEvents.InitiatingProcessFolderPath", "DeviceProcessEvents.FolderPath", "DeviceProcessEvents.InitiatingProcessFolderPath", + "DeviceEvents.FolderPath", + "DeviceEvents.InitiatingProcessFolderPath", "DeviceNetworkEvents.InitiatingProcessFolderPath", - "DeviceRegistryEvents.InitiatingProcessFolderPath" + "DeviceRegistryEvents.InitiatingProcessFolderPath", + "DeviceImageLoadEvents.FolderPath", + "DeviceImageLoadEvents.InitiatingProcessFolderPath" ] } }, "x-oca-asset": { "fields": { - "domain": [ - "DeviceFileEvents.DeviceName", - "DeviceProcessEvents.DeviceName", - "DeviceNetworkEvents.DeviceName", - "DeviceRegistryEvents.DeviceName", - "DeviceEvents.DeviceName", - "DeviceInfo.DeviceName", - "DeviceAlertEvents.DeviceName", - "DeviceImageLoadEvents.DeviceName", - "DeviceLogonEvents.DeviceName", - "DeviceNetworkInfo.DeviceName" + "device_id": [ + "DeviceFileEvents.DeviceId", + "DeviceProcessEvents.DeviceId", + "DeviceNetworkEvents.DeviceId", + "DeviceRegistryEvents.DeviceId", + "DeviceEvents.DeviceId", + "DeviceImageLoadEvents.DeviceId", + "DeviceLogonEvents.DeviceId" ], "hostname": [ "DeviceFileEvents.DeviceName", @@ -297,25 +377,12 @@ "DeviceNetworkEvents.DeviceName", "DeviceRegistryEvents.DeviceName", "DeviceEvents.DeviceName", - "DeviceInfo.DeviceName", - "DeviceAlertEvents.DeviceName", "DeviceImageLoadEvents.DeviceName", - "DeviceLogonEvents.DeviceName", - "DeviceNetworkInfo.DeviceName" + "DeviceLogonEvents.DeviceName" ], - "name": [ - "DeviceFileEvents.DeviceName", - "DeviceProcessEvents.DeviceName", - "DeviceNetworkEvents.DeviceName", - "DeviceRegistryEvents.DeviceName", - "DeviceEvents.DeviceName", - "DeviceInfo.DeviceName", - "DeviceAlertEvents.DeviceName", - "DeviceImageLoadEvents.DeviceName", - "DeviceLogonEvents.DeviceName", - "DeviceNetworkInfo.DeviceName" + "ip_refs[*].value": [ + "DeviceNetworkEvents.LocalIP" ], - "ip": ["DeviceNetworkEvents.LocalIP"], "os.name": [ "DeviceInfo.OSPlatform" ], @@ -326,100 +393,136 @@ }, "x-oca-event": { "fields": { + "action": [ + "DeviceProcessEvents.ActionType", + "DeviceEvents.ActionType", + "DeviceNetworkEvents.ActionType", + "DeviceRegistryEvents.ActionType", + "DeviceFileEvents.ActionType", + "DeviceImageLoadEvents.ActionType" + ], "process_ref.pid": [ - "DeviceProcessEvents.InitiatingProcessId", + "DeviceProcessEvents.ProcessId", + "DeviceEvents.ProcessId", "DeviceNetworkEvents.InitiatingProcessId", "DeviceRegistryEvents.InitiatingProcessId", - "DeviceFileEvents.InitiatingProcessId" - ], - "process_ref.child_refs.pid": [ - "DeviceProcessEvents.ProcessId" + "DeviceFileEvents.InitiatingProcessId", + "DeviceImageLoadEvents.InitiatingProcessId" ], "process_ref.name": [ - "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceProcessEvents.FileName", + "DeviceEvents.FileName", "DeviceNetworkEvents.InitiatingProcessFileName", "DeviceRegistryEvents.InitiatingProcessFileName", - "DeviceFileEvents.InitiatingProcessFileName" - ], - "process_ref.child_refs.name": [ - "DeviceProcessEvents.FileName" + "DeviceFileEvents.InitiatingProcessFileName", + "DeviceImageLoadEvents.InitiatingProcessFileName" ], - "process_ref.image_ref.name": [ - "DeviceProcessEvents.InitiatingProcessFileName", + "process_ref.binary_ref.name": [ + "DeviceProcessEvents.FileName", + "DeviceEvents.FileName", "DeviceNetworkEvents.InitiatingProcessFileName", "DeviceRegistryEvents.InitiatingProcessFileName", - "DeviceFileEvents.InitiatingProcessFileName" + "DeviceFileEvents.InitiatingProcessFileName", + "DeviceImageLoadEvents.InitiatingProcessFileName" ], "process_ref.creator_user_ref.account_login": [ - "DeviceProcessEvents.AccountName" + "DeviceProcessEvents.AccountUpn", + "DeviceEvents.AccountUpn", + "DeviceNetworkEvents.InitiatingProcessAccountUpn", + "DeviceRegistryEvents.InitiatingProcessAccountUpn", + "DeviceFileEvents.InitiatingProcessAccountUpn", + "DeviceImageLoadEvents.InitiatingProcessAccountUpn" + ], + "process_ref.creator_user_ref.user_id": [ + "DeviceProcessEvents.AccountName", + "DeviceEvents.AccountName", + "DeviceNetworkEvents.InitiatingProcessAccountName", + "DeviceRegistryEvents.InitiatingProcessAccountName", + "DeviceFileEvents.InitiatingProcessAccountName", + "DeviceImageLoadEvents.InitiatingProcessAccountName" + ], + "process_ref.command_line": [ + "DeviceProcessEvents.ProcessCommandLine", + "DeviceEvents.ProcessCommandLine", + "DeviceNetworkEvents.InitiatingProcessCommandLine", + "DeviceRegistryEvents.InitiatingProcessCommandLine", + "DeviceFileEvents.InitiatingProcessCommandLine", + "DeviceImageLoadEvents.InitiatingProcessCommandLine" ], "process_ref.parent_ref.name": [ - "DeviceProcessEvents.InitiatingProcessParentFileName", + "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceEvents.InitiatingProcessFileName", "DeviceNetworkEvents.InitiatingProcessParentFileName", "DeviceRegistryEvents.InitiatingProcessParentFileName", - "DeviceFileEvents.InitiatingProcessParentFileName" + "DeviceFileEvents.InitiatingProcessParentFileName", + "DeviceImageLoadEvents.InitiatingProcessParentFileName" ], "process_ref.parent_ref.pid": [ - "DeviceProcessEvents.InitiatingProcessParentId", + "DeviceProcessEvents.InitiatingProcessId", + "DeviceEvents.InitiatingProcessId", "DeviceNetworkEvents.InitiatingProcessParentId", "DeviceRegistryEvents.InitiatingProcessParentId", - "DeviceFileEvents.InitiatingProcessParentId" + "DeviceFileEvents.InitiatingProcessParentId", + "DeviceImageLoadEvents.InitiatingProcessParentId" ], - "process_ref.command_line": [ + "process_ref.parent_ref.command_line": [ "DeviceProcessEvents.InitiatingProcessCommandLine", - "DeviceNetworkEvents.InitiatingProcessCommandLine", - "DeviceRegistryEvents.InitiatingProcessCommandLine", - "DeviceFileEvents.InitiatingProcessCommandLine" + "DeviceEvents.InitiatingProcessCommandLine" ], - "process_ref.image_ref.hashes.'SHA-256'": [ - "DeviceProcessEvents.InitiatingProcessSHA256", + "process_ref.binary_ref.hashes.'SHA-256'": [ + "DeviceProcessEvents.SHA256", + "DeviceEvents.SHA256", "DeviceNetworkEvents.InitiatingProcessSHA256", "DeviceRegistryEvents.InitiatingProcessSHA256", - "DeviceFileEvents.InitiatingProcessSHA256" + "DeviceFileEvents.InitiatingProcessSHA256", + "DeviceImageLoadEvents.InitiatingProcessSHA256" ], - "process_ref.process_ref.creator_user_ref.account_login": [ - "DeviceProcessEvents.AccountName" + "process_ref.binary_ref.hashes.MD5": [ + "DeviceProcessEvents.MD5", + "DeviceEvents.MD5", + "DeviceNetworkEvents.InitiatingProcessMD5", + "DeviceRegistryEvents.InitiatingProcessMD5", + "DeviceFileEvents.InitiatingProcessMD5", + "DeviceImageLoadEvents.InitiatingProcessMD5" + ], + "process_ref.binary_ref.hashes.'SHA-1'": [ + "DeviceProcessEvents.SHA1", + "DeviceEvents.SHA1", + "DeviceNetworkEvents.InitiatingProcessSHA1", + "DeviceRegistryEvents.InitiatingProcessSHA1", + "DeviceFileEvents.InitiatingProcessSHA1", + "DeviceImageLoadEvents.InitiatingProcessSHA1" ], "parent_process_ref.name": [ - "DeviceProcessEvents.InitiatingProcessParentFileName", + "DeviceProcessEvents.InitiatingProcessFileName", + "DeviceEvents.InitiatingProcessFileName", "DeviceNetworkEvents.InitiatingProcessParentFileName", "DeviceRegistryEvents.InitiatingProcessParentFileName", - "DeviceFileEvents.InitiatingProcessParentFileName" + "DeviceFileEvents.InitiatingProcessParentFileName", + "DeviceImageLoadEvents.InitiatingProcessParentFileName" ], "parent_process_ref.pid": [ - "DeviceProcessEvents.InitiatingProcessParentId", + "DeviceProcessEvents.InitiatingProcessId", + "DeviceEvents.InitiatingProcessId", "DeviceNetworkEvents.InitiatingProcessParentId", "DeviceRegistryEvents.InitiatingProcessParentId", - "DeviceFileEvents.InitiatingProcessParentId" - ], - "process_ref.image_ref.hashes.MD5": [ - "DeviceProcessEvents.InitiatingProcessMD5", - "DeviceNetworkEvents.InitiatingProcessMD5", - "DeviceRegistryEvents.InitiatingProcessMD5", - "DeviceFileEvents.InitiatingProcessMD5" - ], - "process_ref.image_ref.hashes.'SHA-256'": [ - "DeviceProcessEvents.InitiatingProcessSHA256", - "DeviceNetworkEvents.InitiatingProcessSHA256", - "DeviceRegistryEvents.InitiatingProcessSHA256", - "DeviceFileEvents.InitiatingProcessSHA256" - ], - "process_ref.creator_user_ref.user_id": [ - "DeviceProcessEvents.InitiatingProcessAccountSid", - "DeviceNetworkEvents.InitiatingProcessAccountSid", - "DeviceRegistryEvents.InitiatingProcessAccountSid", - "DeviceFileEvents.InitiatingProcessAccountSid" + "DeviceFileEvents.InitiatingProcessParentId", + "DeviceImageLoadEvents.InitiatingProcessParentId" ], "domain_ref.value": [ - "DeviceProcessEvents.DeviceName", - "DeviceNetworkEvents.DeviceName", - "DeviceRegistryEvents.DeviceName", - "DeviceFileEvents.DeviceName" + "DeviceNetworkEvents.RemoteUrl", + "DeviceEvents.RemoteUrl" + ], + "url_ref.value": [ + "DeviceNetworkEvents.RemoteUrl", + "DeviceEvents.RemoteUrl", + "DeviceEvents.FileOriginUrl", + "DeviceFileEvents.FileOriginUrl", + "DeviceFileEvents.FileOriginReferrerUrl" ], "file_ref.name": [ - "DeviceProcessEvents.InitiatingProcessFileName", - "DeviceNetworkEvents.InitiatingProcessFileName", - "DeviceRegistryEvents.InitiatingProcessFileName" + "DeviceFileEvents.FileName", + "DeviceImageLoadEvents.FileName" ], "registry_ref.key": [ "DeviceRegistryEvents.RegistryKey" @@ -430,23 +533,30 @@ "DeviceNetworkEvents.DeviceName", "DeviceRegistryEvents.DeviceName", "DeviceEvents.DeviceName", - "DeviceInfo.DeviceName", - "DeviceAlertEvents.DeviceName", "DeviceImageLoadEvents.DeviceName", - "DeviceLogonEvents.DeviceName", - "DeviceNetworkInfo.DeviceName" + "DeviceLogonEvents.DeviceName" ], - "host_ref.name": [ - "DeviceFileEvents.DeviceName", - "DeviceProcessEvents.DeviceName", - "DeviceNetworkEvents.DeviceName", - "DeviceRegistryEvents.DeviceName", - "DeviceEvents.DeviceName", - "DeviceInfo.DeviceName", - "DeviceAlertEvents.DeviceName", - "DeviceImageLoadEvents.DeviceName", - "DeviceLogonEvents.DeviceName", - "DeviceNetworkInfo.DeviceName" + "host_ref.device_id": [ + "DeviceFileEvents.DeviceId", + "DeviceProcessEvents.DeviceId", + "DeviceNetworkEvents.DeviceId", + "DeviceRegistryEvents.DeviceId", + "DeviceEvents.DeviceId", + "DeviceImageLoadEvents.DeviceId", + "DeviceLogonEvents.DeviceId" + ] + } + }, + "x-ibm-finding": { + "fields": { + "alert_id": [ + "DeviceAlertEvents.AlertId" + ], + "name": [ + "DeviceAlertEvents.Title" + ], + "time_observed": [ + "DeviceAlertEvents.Timestamp" ] } } diff --git a/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/to_stix_map.json b/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/to_stix_map.json index 7a64ab440..80ba234e8 100644 --- a/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/to_stix_map.json +++ b/stix_shifter_modules/msatp/stix_translation/json/stix_2_1/to_stix_map.json @@ -1,6 +1,269 @@ { + "DeviceAlertEvents": { + "AlertId": [ + { + "key": "x-ibm-finding.alert_id", + "object": "finding" + }, + { + "key": "x-oca-event.finding_refs", + "object": "event", + "references": [ + "finding" + ], + "group": true + } + ], + "Severity": { + "key": "x-ibm-finding.severity", + "object": "finding", + "transformer": "SeverityToNumericVal" + }, + "AttackTechniques": [ + { + "key": "x-ibm-ttp-tagging.extensions.mitre-attack-ext.technique_name", + "object": "ttp_tagging", + "unwrap": true + }, + { + "key": "x-ibm-finding.ttp_tagging_refs", + "object": "finding", + "references": [ + "ttp_tagging" + ], + "unwrap": true, + "group": true + } + ], + "Category": [ + { + "key": "x-ibm-ttp-tagging.kill_chain_phases.phase_name", + "object": "ttp_tagging_phase" + }, + { + "key": "x-ibm-finding.ttp_tagging_refs", + "object": "finding", + "references": [ + "ttp_tagging_phase" + ], + "unwrap": true, + "group": true + } + ], + "Title": [ + { + "key": "x-ibm-finding.name", + "object": "finding" + }, + { + "key": "x-oca-event.action", + "object": "event" + }, + { + "key": "x-oca-event.category", + "object": "event", + "transformer": "Alert" + }, + { + "key": "x-ibm-finding.finding_type", + "object": "finding", + "transformer": "Alert" + } + ], + "RemoteUrl": [ + { + "key": "url.value", + "object": "url", + "transformer": "IfValidUrl" + }, + { + "key": "domain-name.value", + "object": "domain-name", + "transformer": "GetDomainName" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain-name" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" + }, + { + "key": "x-ibm-finding.ioc_refs", + "object": "finding", + "references": [ + "url", + "domain-name" + ], + "group": true + } + ], + "RemoteIP": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + }, + { + "key": "x-oca-event.ip_refs", + "object": "event", + "references": [ + "dst_ip" + ], + "group": true + }, + { + "key": "x-ibm-finding.ioc_refs", + "object": "finding", + "references": [ + "dst_ip" + ], + "group": true + } + ], + "Timestamp": [ + { + "key": "x-oca-event.created", + "object": "event" + }, + { + "key": "x-ibm-finding.time_observed", + "object": "finding" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + }, + { + "key": "last_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + } + ], + "FileName": [ + { + "key": "file.name", + "object": "file", + "transformer": "ToFileName" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "file" + }, + { + "key": "directory.path", + "object": "file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "file_dir" + } + ], + "SHA1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "SHA256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "MD5": { + "key": "file.hashes.MD5", + "object": "file" + }, + "DeviceName": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "DeviceId": [ + { + "key": "x-oca-asset.device_id", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "original_ref": [ + { + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + } + ], + "event_link": [ + { + "key": "external-reference.url", + "object": "device_external_ref" + }, + { + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + } + }, "DeviceNetworkEvents": { + "Alerts": { + "key": "x-json-alert.data" + }, "Timestamp": [ + { + "key": "x-oca-event.created", + "object": "event" + }, { "key": "first_observed", "cybox": false, @@ -25,6 +288,19 @@ "key": "network-traffic.src_ref", "object": "nt", "references": "src_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "src_ip" + ], + "group": true + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" } ], "RemoteIP": [ @@ -40,6 +316,33 @@ "key": "network-traffic.dst_ref", "object": "nt", "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "RemoteUrl": [ + { + "key": "url.value", + "object": "url", + "transformer": "IfValidUrl" + }, + { + "key": "domain-name.value", + "object": "domain-name", + "transformer": "GetDomainName" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain-name" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" } ], "LocalPort": [ @@ -47,6 +350,11 @@ "key": "network-traffic.src_port", "object": "nt", "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" } ], "RemotePort": [ @@ -54,6 +362,11 @@ "key": "network-traffic.dst_port", "object": "nt", "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" } ], "Protocol": { @@ -61,18 +374,20 @@ "object": "nt", "transformer": "FormatTCPProtocol" }, - "RemoteUrl": { - "key": "url.value" - }, "InitiatingProcessSHA1": [ { "key": "file.hashes.SHA-1", "object": "nc_process_file" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "nc_process", "references": "nc_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "nc_process" } ], "InitiatingProcessSHA256": [ @@ -81,9 +396,14 @@ "object": "nc_process_file" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "nc_process", "references": "nc_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "nc_process" } ], "InitiatingProcessMD5": [ @@ -92,26 +412,58 @@ "object": "nc_process_file" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "nc_process", "references": "nc_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "nc_process" } ], "InitiatingProcessFileName": [ { "key": "file.name", - "object": "nc_process_file" + "object": "nc_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "nc_process", + "transformer": "ToFileName" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "nc_process", "references": "nc_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "nc_process" + }, + { + "key": "directory.path", + "object": "nc_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "nc_process_file", + "references": "nc_process_file_dir" } ], "InitiatingProcessParentFileName": [ { "key": "file.name", - "object": "nc_parent_process_file" + "object": "nc_parent_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "nc_parent_process", + "transformer": "ToFileName" }, { "key": "process.parent_ref", @@ -119,16 +471,33 @@ "references": "nc_parent_process" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "nc_parent_process", "references": "nc_parent_process_file" + }, + { + "key": "directory.path", + "object": "nc_parent_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "nc_parent_process_file", + "references": "nc_parent_process_file_dir" + } + ], + "InitiatingProcessId": [ + { + "key": "process.pid", + "object": "nc_process", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "nc_process" } ], - "InitiatingProcessId": { - "key": "process.pid", - "object": "nc_process", - "transformer": "ToInteger" - }, "InitiatingProcessParentId": [ { "key": "process.pid", @@ -141,18 +510,25 @@ "references": "nc_parent_process" } ], - "InitiatingProcessCommandLine": { - "key": "process.command_line", - "object": "nc_process" - }, + "InitiatingProcessCommandLine": [ + { + "key": "process.command_line", + "object": "nc_process" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "nc_process" + } + ], "InitiatingProcessCreationTime": { - "key": "process.created_time", + "key": "process.created", "object": "nc_process", "transformer": "MsatpToTimestamp" }, "InitiatingProcessParentCreationTime": [ { - "key": "process.created_time", + "key": "process.created", "object": "nc_parent_process", "transformer": "MsatpToTimestamp" }, @@ -162,7 +538,7 @@ "references": "nc_parent_process" } ], - "InitiatingProcessAccountSid": [ + "InitiatingProcessAccountName": [ { "key": "user-account.user_id", "object": "nc_process_creator" @@ -173,7 +549,7 @@ "references": "nc_process_creator" } ], - "InitiatingProcessAccountName": [ + "InitiatingProcessAccountUpn": [ { "key": "user-account.account_login", "object": "nc_process_creator" @@ -188,7 +564,7 @@ { "key": "directory.path", "object": "nc_process_file_dir", - "transformer": "ToDirectoryPath" + "transformer": "ToMSATPDirectoryPath" }, { "key": "file.parent_directory_ref", @@ -196,7 +572,7 @@ "references": "nc_process_file_dir" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "nc_process", "references": "nc_process_file" } @@ -210,7 +586,9 @@ { "key": "ipv4-addr.resolves_to_refs", "object": "src_ip", - "references": ["mac"] + "references": [ + "mac" + ] }, { "key": "network-traffic.src_ref", @@ -223,19 +601,150 @@ "references": "nt" } ], - "DeviceName": { - "key": "x-msatp.computer_name" + "DeviceName": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "DeviceId": [ + { + "key": "x-oca-asset.device_id", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "PublicIP": [ + { + "key": "ipv4-addr.value", + "object": "public-ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "public-ip" + ], + "group": true + } + ], + "OSPlatform": { + "key": "x-oca-asset.os_name", + "object": "host" + }, + "OSArchitecture": { + "key": "x-oca-asset.architecture", + "object": "host" }, - "DeviceId": { - "key": "x-msatp.machine_id" + "OSVersion": { + "key": "x-oca-asset.os_version", + "object": "host" }, "event_count": { "key": "number_observed", "cybox": false, "transformer": "ToInteger" + }, + "original_ref": [ + { + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + } + ], + "ActionType": { + "key": "x-oca-event.action", + "object": "event" + }, + "event_link": [ + { + "key": "external-reference.url", + "object": "device_external_ref" + }, + { + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" + } + ], + "MacAddressSet": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacList", + "unwrap": true + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac" + ], + "unwrap": true, + "group": true + } + ], + "IPAddresses": [ + { + "key": "ipv4-addr.value", + "object": "host_ip4", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "host_ip6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip4", + "host_ip6" + ], + "unwrap": true, + "group": true + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + }, + "AdditionalFields": { + "key": "x-msatp.AdditionalFields", + "object": "x-msatp", + "transformer": "JsonToString" } }, "DeviceProcessEvents": { + "Alerts": { + "key": "x-json-alert.data" + }, "Timestamp": [ { "key": "x-oca-event.created", @@ -261,12 +770,18 @@ "FileName": [ { "key": "file.name", - "object": "child_file" + "object": "child_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "child_process", + "transformer": "ToFileName" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "child_process", - "references": "child_file" + "references": "child_process_file" }, { "key": "process.child_refs", @@ -278,32 +793,42 @@ { "key": "x-oca-event.process_ref", "object": "event", - "references": "process" + "references": "child_process" + }, + { + "key": "directory.path", + "object": "child_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "child_process_file", + "references": "child_process_file_dir" } ], "FolderPath": [ { "key": "directory.path", - "object": "child_file_dir", - "transformer": "ToDirectoryPath" + "object": "child_process_file_dir", + "transformer": "ToMSATPDirectoryPath" }, { "key": "file.parent_directory_ref", - "object": "child_file", - "references": "child_file_dir" + "object": "child_process_file", + "references": "child_process_file_dir" } ], "SHA1": { "key": "file.hashes.SHA-1", - "object": "child_file" + "object": "child_process_file" }, "SHA256": { "key": "file.hashes.SHA-256", - "object": "child_file" + "object": "child_process_file" }, "MD5": { "key": "file.hashes.MD5", - "object": "child_file" + "object": "child_process_file" }, "ProcessId": [ { @@ -321,7 +846,7 @@ { "key": "x-oca-event.process_ref", "object": "event", - "references": "process" + "references": "child_process" } ], "ProcessCommandLine": [ @@ -337,27 +862,18 @@ ] } ], - "ProcessCreationTime": [ - { - "key": "process.created_time", - "object": "child_process", - "transformer": "MsatpToTimestamp" - }, - { - "key": "process.child_refs", - "object": "process", - "references": [ - "child_process" - ] - } - ], + "ProcessCreationTime": { + "key": "process.created", + "object": "child_process", + "transformer": "MsatpToTimestamp" + }, "InitiatingProcessSHA1": [ { "key": "file.hashes.SHA-1", "object": "process_file" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "process", "references": "process_file" } @@ -368,7 +884,7 @@ "object": "process_file" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "process", "references": "process_file" } @@ -379,7 +895,7 @@ "object": "process_file" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "process", "references": "process_file" } @@ -387,45 +903,45 @@ "InitiatingProcessFileName": [ { "key": "file.name", - "object": "process_file" + "object": "process_file", + "transformer": "ToFileName" }, { - "key": "process.image_ref", + "key": "process.name", "object": "process", - "references": "process_file" - } - ], - "InitiatingProcessParentFileName": [ + "transformer": "ToFileName" + }, { - "key": "file.name", - "object": "parent_process_file" + "key": "process.binary_ref", + "object": "process", + "references": "process_file" }, { "key": "process.parent_ref", - "object": "process", - "references": "parent_process" + "object": "child_process", + "references": "process" }, { - "key": "process.image_ref", - "object": "parent_process", - "references": "parent_process_file" + "key": "directory.path", + "object": "process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "process_file_dir" } ], - "InitiatingProcessId": { - "key": "process.pid", - "object": "process", - "transformer": "ToInteger" - }, - "InitiatingProcessParentId": [ + "InitiatingProcessId": [ { "key": "process.pid", - "object": "parent_process", + "object": "process", "transformer": "ToInteger" }, { "key": "process.parent_ref", - "object": "process", - "references": "parent_process" + "object": "child_process", + "references": "process" } ], "InitiatingProcessCommandLine": { @@ -433,13 +949,67 @@ "object": "process" }, "InitiatingProcessCreationTime": { - "key": "process.created_time", + "key": "process.created", "object": "process", "transformer": "MsatpToTimestamp" }, + "InitiatingProcessFolderPath": [ + { + "key": "directory.path", + "object": "process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "process_file_dir" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + } + ], + "InitiatingProcessParentId": { + "key": "process.pid", + "object": "parent_process", + "transformer": "ToInteger" + }, + "InitiatingProcessParentFileName": [ + { + "key": "file.name", + "object": "parent_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "parent_process", + "transformer": "ToFileName" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "process.binary_ref", + "object": "parent_process", + "references": "parent_process_file" + }, + { + "key": "directory.path", + "object": "parent_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "parent_process_file", + "references": "parent_process_file_dir" + } + ], "InitiatingProcessParentCreationTime": [ { - "key": "process.created_time", + "key": "process.created", "object": "parent_process", "transformer": "MsatpToTimestamp" }, @@ -449,7 +1019,7 @@ "references": "parent_process" } ], - "AccountSid": [ + "AccountName": [ { "key": "user-account.user_id", "object": "child_process_creator" @@ -458,9 +1028,14 @@ "key": "process.creator_user_ref", "object": "child_process", "references": "child_process_creator" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "child_process_creator" } ], - "InitiatingProcessAccountSid": [ + "InitiatingProcessAccountName": [ { "key": "user-account.user_id", "object": "process_creator" @@ -471,7 +1046,7 @@ "references": "process_creator" } ], - "AccountName": [ + "AccountUpn": [ { "key": "user-account.account_login", "object": "child_process_creator" @@ -480,9 +1055,14 @@ "key": "process.creator_user_ref", "object": "child_process", "references": "child_process_creator" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "child_process_creator" } ], - "InitiatingProcessAccountName": [ + "InitiatingProcessAccountUpn": [ { "key": "user-account.account_login", "object": "process_creator" @@ -493,36 +1073,146 @@ "references": "process_creator" } ], - "InitiatingProcessFolderPath": [ + "DeviceName": [ { - "key": "directory.path", - "object": "process_file_dir", - "transformer": "ToDirectoryPath" + "key": "x-oca-asset.hostname", + "object": "host" }, { - "key": "file.parent_directory_ref", - "object": "process_file", - "references": "process_file_dir" + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "DeviceId": [ + { + "key": "x-oca-asset.device_id", + "object": "host" }, { - "key": "process.image_ref", - "object": "process", - "references": "process_file" + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" } ], - "DeviceName": { - "key": "x-msatp.computer_name" - }, - "DeviceId": { - "key": "x-msatp.machine_id" - }, - "event_count": { - "key": "number_observed", - "cybox": false, - "transformer": "ToInteger" - } + "PublicIP": [ + { + "key": "ipv4-addr.value", + "object": "public-ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "public-ip" + ], + "group": true + } + ], + "OSPlatform": { + "key": "x-oca-asset.os_name", + "object": "host" + }, + "OSArchitecture": { + "key": "x-oca-asset.architecture", + "object": "host" + }, + "OSVersion": { + "key": "x-oca-asset.os_version", + "object": "host" + }, + "original_ref": [ + { + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + } + ], + "event_link": [ + { + "key": "external-reference.url", + "object": "device_external_ref" + }, + { + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" + } + ], + "MacAddressSet": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacList", + "unwrap": true + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac" + ], + "unwrap": true, + "group": true + } + ], + "IPAddresses": [ + { + "key": "ipv4-addr.value", + "object": "host_ip4", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "host_ip6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip4", + "host_ip6" + ], + "unwrap": true, + "group": true + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + }, + "AdditionalFields": { + "key": "x-msatp.AdditionalFields", + "object": "x-msatp", + "transformer": "JsonToString" + } }, "DeviceFileEvents": { + "Alerts": { + "key": "x-json-alert.data" + }, "Timestamp": [ { "key": "x-oca-event.created", @@ -548,12 +1238,23 @@ "FileName": [ { "key": "file.name", - "object": "file" + "object": "file", + "transformer": "ToFileName" }, { "key": "x-oca-event.file_ref", "object": "event", "references": "file" + }, + { + "key": "directory.path", + "object": "file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "file_dir" } ], "SHA1": { @@ -568,13 +1269,63 @@ "key": "file.hashes.MD5", "object": "file" }, + "FileOriginReferrerUrl": [ + { + "key": "url.value", + "object": "url_ref", + "transformer": "IfValidUrl" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url_ref" + }, + { + "key": "domain-name.value", + "object": "file-origin-ref-domain", + "transformer": "GetDomainName" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "file-origin-ref-domain" + } + ], + "FileOriginIP": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + }, + { + "key": "x-oca-event.ips_ref", + "object": "event", + "references": [ + "dst_ip" + ], + "group": true + } + ], "InitiatingProcessSHA1": [ { "key": "file.hashes.SHA-1", "object": "fc_process_file" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "fc_process", "references": "fc_process_file" } @@ -585,7 +1336,7 @@ "object": "fc_process_file" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "fc_process", "references": "fc_process_file" } @@ -596,7 +1347,7 @@ "object": "fc_process_file" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "fc_process", "references": "fc_process_file" } @@ -604,18 +1355,45 @@ "InitiatingProcessFileName": [ { "key": "file.name", - "object": "fc_process_file" + "object": "fc_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "fc_process", + "transformer": "ToFileName" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "fc_process", "references": "fc_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "fc_process" + }, + { + "key": "directory.path", + "object": "fc_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "fc_process_file", + "references": "fc_process_file_dir" } ], "InitiatingProcessParentFileName": [ { "key": "file.name", - "object": "fc_parent_process_file" + "object": "fc_parent_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "fc_parent_process", + "transformer": "ToFileName" }, { "key": "process.parent_ref", @@ -623,9 +1401,19 @@ "references": "fc_parent_process" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "fc_parent_process", "references": "fc_parent_process_file" + }, + { + "key": "directory.path", + "object": "fc_parent_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "fc_parent_process_file", + "references": "fc_parent_process_file_dir" } ], "InitiatingProcessId": [ @@ -662,13 +1450,13 @@ "object": "fc_process" }, "InitiatingProcessCreationTime": { - "key": "process.created_time", + "key": "process.created", "object": "fc_process", "transformer": "MsatpToTimestamp" }, "InitiatingProcessParentCreationTime": [ { - "key": "process.created_time", + "key": "process.created", "object": "fc_parent_process", "transformer": "MsatpToTimestamp" }, @@ -678,7 +1466,7 @@ "references": "fc_parent_process" } ], - "InitiatingProcessAccountSid": [ + "InitiatingProcessAccountName": [ { "key": "user-account.user_id", "object": "fc_process_creator" @@ -694,7 +1482,7 @@ "references": "fc_process_creator" } ], - "InitiatingProcessAccountName": [ + "InitiatingProcessAccountUpn": [ { "key": "user-account.account_login", "object": "fc_process_creator" @@ -709,7 +1497,7 @@ { "key": "directory.path", "object": "file_dir", - "transformer": "ToDirectoryPath" + "transformer": "ToMSATPDirectoryPath" }, { "key": "file.parent_directory_ref", @@ -721,7 +1509,7 @@ { "key": "directory.path", "object": "fc_process_file_dir", - "transformer": "ToDirectoryPath" + "transformer": "ToMSATPDirectoryPath" }, { "key": "file.parent_directory_ref", @@ -729,15 +1517,11 @@ "references": "fc_process_file_dir" }, { - "key": "process.image_ref", + "key": "process.binary_ref", "object": "fc_process", "references": "fc_process_file" } ], - "DeviceId": { - "key": "x-oca-asset.device_id", - "object": "host" - }, "DeviceName": [ { "key": "x-oca-asset.hostname", @@ -749,197 +1533,394 @@ "references": "host" } ], - "event_count": { - "key": "number_observed", - "cybox": false, - "transformer": "ToInteger" - } - }, - "DeviceRegistryEvents": { - "Timestamp": [ - { - "key": "x-oca-event.created", - "object": "event" - }, + "DeviceId": [ { - "key": "first_observed", - "cybox": false, - "transformer": "MsatpToTimestamp" + "key": "x-oca-asset.device_id", + "object": "host" }, { - "key": "last_observed", - "cybox": false, - "transformer": "MsatpToTimestamp" + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" } ], - "ActionType": [ + "PublicIP": [ { - "key": "x-oca-event.action", - "object": "event" + "key": "ipv4-addr.value", + "object": "public-ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "public-ip" + ], + "group": true } ], - "RegistryKey": { - "key": "windows-registry-key.key", - "object": "re_key" + "OSPlatform": { + "key": "x-oca-asset.os_name", + "object": "host" }, - "RegistryValues": { - "key": "windows-registry-key.values", - "object": "re_key", - "transformer": "MsatpToRegistryValue" + "OSArchitecture": { + "key": "x-oca-asset.architecture", + "object": "host" }, - "InitiatingProcessSHA1": [ + "OSVersion": { + "key": "x-oca-asset.os_version", + "object": "host" + }, + "original_ref": [ { - "key": "file.hashes.SHA-1", - "object": "re_process_file" + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" }, { - "key": "process.image_ref", - "object": "re_process", - "references": "re_process_file" + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" } ], - "InitiatingProcessSHA256": [ + "event_link": [ { - "key": "file.hashes.SHA-256", - "object": "re_process_file" + "key": "external-reference.url", + "object": "device_external_ref" }, { - "key": "process.image_ref", - "object": "re_process", - "references": "re_process_file" + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" } ], - "InitiatingProcessMD5": [ + "MacAddressSet": [ { - "key": "file.hashes.MD5", - "object": "re_process_file" + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacList", + "unwrap": true }, { - "key": "process.image_ref", - "object": "re_process", - "references": "re_process_file" + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac" + ], + "unwrap": true, + "group": true } ], - "InitiatingProcessFileName": [ + "IPAddresses": [ { - "key": "file.name", - "object": "re_process_file" + "key": "ipv4-addr.value", + "object": "host_ip4", + "unwrap": true, + "transformer": "FilterIPv4List" }, { - "key": "process.image_ref", - "object": "re_process", - "references": "re_process_file" + "key": "ipv6-addr.value", + "object": "host_ip6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip4", + "host_ip6" + ], + "unwrap": true, + "group": true } ], - "InitiatingProcessParentFileName": [ + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + }, + "AdditionalFields": { + "key": "x-msatp.AdditionalFields", + "object": "x-msatp", + "transformer": "JsonToString" + } + }, + "DeviceImageLoadEvents": { + "Alerts": { + "key": "x-json-alert.data" + }, + "Timestamp": [ { - "key": "file.name", - "object": "re_parent_process_file" + "key": "x-oca-event.created", + "object": "event" }, { - "key": "process.parent_ref", - "object": "re_process", - "references": "re_parent_process" + "key": "first_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" }, { - "key": "process.image_ref", - "object": "re_parent_process", - "references": "re_parent_process_file" + "key": "last_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" } ], - "InitiatingProcessId": [ + "ActionType": [ + { + "key": "x-oca-event.action", + "object": "event" + } + ], + "FileName": [ + { + "key": "file.name", + "object": "file", + "transformer": "ToFileName" + }, + { + "key": "x-oca-event.file_ref", + "object": "event", + "references": "file" + }, + { + "key": "directory.path", + "object": "file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "file_dir" + } + ], + "SHA1": { + "key": "file.hashes.SHA-1", + "object": "file" + }, + "SHA256": { + "key": "file.hashes.SHA-256", + "object": "file" + }, + "MD5": { + "key": "file.hashes.MD5", + "object": "file" + }, + "InitiatingProcessSHA1": [ + { + "key": "file.hashes.SHA-1", + "object": "fc_process_file" + }, + { + "key": "process.binary_ref", + "object": "fc_process", + "references": "fc_process_file" + } + ], + "InitiatingProcessSHA256": [ + { + "key": "file.hashes.SHA-256", + "object": "fc_process_file" + }, + { + "key": "process.binary_ref", + "object": "fc_process", + "references": "fc_process_file" + } + ], + "InitiatingProcessMD5": [ + { + "key": "file.hashes.MD5", + "object": "fc_process_file" + }, + { + "key": "process.binary_ref", + "object": "fc_process", + "references": "fc_process_file" + } + ], + "InitiatingProcessFileName": [ + { + "key": "file.name", + "object": "fc_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "fc_process", + "transformer": "ToFileName" + }, + { + "key": "process.binary_ref", + "object": "fc_process", + "references": "fc_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "fc_process" + }, + { + "key": "directory.path", + "object": "fc_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "fc_process_file", + "references": "fc_process_file_dir" + } + ], + "InitiatingProcessParentFileName": [ + { + "key": "file.name", + "object": "fc_parent_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "fc_parent_process", + "transformer": "ToFileName" + }, + { + "key": "process.parent_ref", + "object": "fc_process", + "references": "fc_parent_process" + }, + { + "key": "process.binary_ref", + "object": "fc_parent_process", + "references": "fc_parent_process_file" + }, + { + "key": "directory.path", + "object": "fc_parent_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "fc_parent_process_file", + "references": "fc_parent_process_file_dir" + } + ], + "InitiatingProcessId": [ { "key": "process.pid", - "object": "re_process", + "object": "fc_process", "transformer": "ToInteger" }, { "key": "x-oca-event.process_ref", "object": "event", - "references": "re_process" + "references": "fc_process" } ], "InitiatingProcessParentId": [ { "key": "process.pid", - "object": "re_parent_process", + "object": "fc_parent_process", "transformer": "ToInteger" }, { "key": "process.parent_ref", - "object": "re_process", - "references": "re_parent_process" + "object": "fc_process", + "references": "fc_parent_process" }, { "key": "x-oca-event.parent_process_ref", "object": "event", - "references": "re_parent_process" + "references": "fc_parent_process" } ], "InitiatingProcessCommandLine": { "key": "process.command_line", - "object": "re_process" + "object": "fc_process" }, "InitiatingProcessCreationTime": { - "key": "process.created_time", - "object": "re_process", + "key": "process.created", + "object": "fc_process", "transformer": "MsatpToTimestamp" }, "InitiatingProcessParentCreationTime": [ { - "key": "process.created_time", - "object": "re_parent_process", + "key": "process.created", + "object": "fc_parent_process", "transformer": "MsatpToTimestamp" }, { "key": "process.parent_ref", - "object": "re_process", - "references": "re_parent_process" + "object": "fc_process", + "references": "fc_parent_process" } ], - "InitiatingProcessAccountSid": [ + "InitiatingProcessAccountName": [ { "key": "user-account.user_id", - "object": "re_process_creator" + "object": "fc_process_creator" }, { "key": "process.creator_user_ref", - "object": "re_process", - "references": "re_process_creator" + "object": "fc_process", + "references": "fc_process_creator" + }, + { + "key": "x-oca-event.user_ref", + "object": "event", + "references": "fc_process_creator" } ], - "InitiatingProcessAccountName": [ + "InitiatingProcessAccountUpn": [ { "key": "user-account.account_login", - "object": "re_process_creator" + "object": "fc_process_creator" }, { "key": "process.creator_user_ref", - "object": "re_process", - "references": "re_process_creator" + "object": "fc_process", + "references": "fc_process_creator" + } + ], + "FolderPath": [ + { + "key": "directory.path", + "object": "file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "file", + "references": "file_dir" } ], "InitiatingProcessFolderPath": [ { "key": "directory.path", - "object": "re_process_file_dir", - "transformer": "ToDirectoryPath" + "object": "fc_process_file_dir", + "transformer": "ToMSATPDirectoryPath" }, { "key": "file.parent_directory_ref", - "object": "re_process_file", - "references": "re_process_file_dir" + "object": "fc_process_file", + "references": "fc_process_file_dir" }, { - "key": "process.image_ref", - "object": "re_process", - "references": "re_process_file" + "key": "process.binary_ref", + "object": "fc_process", + "references": "fc_process_file" } ], - "DeviceId": { - "key": "x-oca-asset.device_id", - "object": "host" - }, "DeviceName": [ { "key": "x-oca-asset.hostname", @@ -951,10 +1932,1086 @@ "references": "host" } ], + "DeviceId": [ + { + "key": "x-oca-asset.device_id", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "PublicIP": [ + { + "key": "ipv4-addr.value", + "object": "public-ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "public-ip" + ], + "group": true + } + ], + "OSPlatform": { + "key": "x-oca-asset.os_name", + "object": "host" + }, + "OSArchitecture": { + "key": "x-oca-asset.architecture", + "object": "host" + }, + "OSVersion": { + "key": "x-oca-asset.os_version", + "object": "host" + }, + "original_ref": [ + { + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + } + ], + "event_link": [ + { + "key": "external-reference.url", + "object": "device_external_ref" + }, + { + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" + } + ], + "MacAddressSet": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacList", + "unwrap": true + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac" + ], + "unwrap": true, + "group": true + } + ], + "IPAddresses": [ + { + "key": "ipv4-addr.value", + "object": "host_ip4", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "host_ip6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip4", + "host_ip6" + ], + "unwrap": true, + "group": true + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, "event_count": { "key": "number_observed", "cybox": false, "transformer": "ToInteger" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + }, + "AdditionalFields": { + "key": "x-msatp.AdditionalFields", + "object": "x-msatp", + "transformer": "JsonToString" + } + }, + "DeviceRegistryEvents": { + "Alerts": { + "key": "x-json-alert.data" + }, + "Timestamp": [ + { + "key": "x-oca-event.created", + "object": "event" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + }, + { + "key": "last_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + } + ], + "ActionType": [ + { + "key": "x-oca-event.action", + "object": "event" + } + ], + "RegistryKey": [ + { + "key": "windows-registry-key.key", + "object": "re_key" + }, + { + "key": "x-oca-event.registry_ref", + "object": "event", + "references": "re_key" + } + ], + "RegistryValues": { + "key": "windows-registry-key.values", + "object": "re_key", + "transformer": "MsatpToRegistryValue" + }, + "InitiatingProcessSHA1": [ + { + "key": "file.hashes.SHA-1", + "object": "re_process_file" + }, + { + "key": "process.binary_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "InitiatingProcessSHA256": [ + { + "key": "file.hashes.SHA-256", + "object": "re_process_file" + }, + { + "key": "process.binary_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "InitiatingProcessMD5": [ + { + "key": "file.hashes.MD5", + "object": "re_process_file" + }, + { + "key": "process.binary_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "InitiatingProcessFileName": [ + { + "key": "file.name", + "object": "re_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "re_process", + "transformer": "ToFileName" + }, + { + "key": "process.binary_ref", + "object": "re_process", + "references": "re_process_file" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "re_process" + }, + { + "key": "directory.path", + "object": "re_process_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "re_process_file", + "references": "re_process_file_dir" + } + ], + "InitiatingProcessParentFileName": [ + { + "key": "file.name", + "object": "re_parent_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "re_parent_process", + "transformer": "ToFileName" + }, + { + "key": "process.parent_ref", + "object": "re_process", + "references": "re_parent_process" + }, + { + "key": "process.binary_ref", + "object": "re_parent_process", + "references": "re_parent_process_file" + }, + { + "key": "directory.path", + "object": "re_parent_process_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "re_parent_process_file", + "references": "re_parent_process_file_dir" + } + ], + "InitiatingProcessId": [ + { + "key": "process.pid", + "object": "re_process", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "re_process" + } + ], + "InitiatingProcessParentId": [ + { + "key": "process.pid", + "object": "re_parent_process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "re_process", + "references": "re_parent_process" + } + ], + "InitiatingProcessCommandLine": { + "key": "process.command_line", + "object": "re_process" + }, + "InitiatingProcessCreationTime": { + "key": "process.created", + "object": "re_process", + "transformer": "MsatpToTimestamp" + }, + "InitiatingProcessParentCreationTime": [ + { + "key": "process.created", + "object": "re_parent_process", + "transformer": "MsatpToTimestamp" + }, + { + "key": "process.parent_ref", + "object": "re_process", + "references": "re_parent_process" + } + ], + "InitiatingProcessAccountName": [ + { + "key": "user-account.user_id", + "object": "re_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "re_process", + "references": "re_process_creator" + } + ], + "InitiatingProcessAccountUpn": [ + { + "key": "user-account.account_login", + "object": "re_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "re_process", + "references": "re_process_creator" + } + ], + "InitiatingProcessFolderPath": [ + { + "key": "directory.path", + "object": "re_process_file_dir", + "transformer": "ToDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "re_process_file", + "references": "re_process_file_dir" + }, + { + "key": "process.binary_ref", + "object": "re_process", + "references": "re_process_file" + } + ], + "DeviceName": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "DeviceId": [ + { + "key": "x-oca-asset.device_id", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "PublicIP": [ + { + "key": "ipv4-addr.value", + "object": "public-ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "public-ip" + ], + "group": true + } + ], + "OSPlatform": { + "key": "x-oca-asset.os_name", + "object": "host" + }, + "OSArchitecture": { + "key": "x-oca-asset.architecture", + "object": "host" + }, + "OSVersion": { + "key": "x-oca-asset.os_version", + "object": "host" + }, + "original_ref": [ + { + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + } + ], + "event_link": [ + { + "key": "external-reference.url", + "object": "device_external_ref" + }, + { + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" + } + ], + "MacAddressSet": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacList", + "unwrap": true + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac" + ], + "unwrap": true, + "group": true + } + ], + "IPAddresses": [ + { + "key": "ipv4-addr.value", + "object": "host_ip4", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "host_ip6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip4", + "host_ip6" + ], + "unwrap": true, + "group": true + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + }, + "AdditionalFields": { + "key": "x-msatp.AdditionalFields", + "object": "x-msatp", + "transformer": "JsonToString" + } + }, + "DeviceEvents": { + "Alerts": { + "key": "x-json-alert.data" + }, + "Timestamp": [ + { + "key": "x-oca-event.created", + "object": "event" + }, + { + "key": "first_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + }, + { + "key": "last_observed", + "cybox": false, + "transformer": "MsatpToTimestamp" + } + ], + "ActionType": [ + { + "key": "x-oca-event.action", + "object": "event" + } + ], + "LocalIP": [ + { + "key": "ipv4-addr.value", + "object": "src_ip" + }, + { + "key": "ipv6-addr.value", + "object": "src_ip" + }, + { + "key": "network-traffic.src_ref", + "object": "nt", + "references": "src_ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "src_ip" + ], + "group": true + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "RemoteIP": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "RemoteUrl": [ + { + "key": "url.value", + "object": "url", + "transformer": "IfValidUrl" + }, + { + "key": "domain-name.value", + "object": "domain-name", + "transformer": "GetDomainName" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "domain-name" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url" + } + ], + "LocalPort": [ + { + "key": "network-traffic.src_port", + "object": "nt", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "RemotePort": [ + { + "key": "network-traffic.dst_port", + "object": "nt", + "transformer": "ToInteger" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "FileOriginUrl": [ + { + "key": "url.value", + "object": "url_ref", + "transformer": "IfValidUrl" + }, + { + "key": "x-oca-event.url_ref", + "object": "event", + "references": "url_ref" + }, + { + "key": "domain-name.value", + "object": "file-origin-domain", + "transformer": "GetDomainName" + }, + { + "key": "x-oca-event.domain_ref", + "object": "event", + "references": "file-origin-domain" + } + ], + "FileOriginIP": [ + { + "key": "ipv4-addr.value", + "object": "dst_ip" + }, + { + "key": "ipv6-addr.value", + "object": "dst_ip" + }, + { + "key": "network-traffic.dst_ref", + "object": "nt", + "references": "dst_ip" + }, + { + "key": "x-oca-event.network_ref", + "object": "event", + "references": "nt" + } + ], + "FileName": [ + { + "key": "file.name", + "object": "child_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "child_process", + "transformer": "ToFileName" + }, + { + "key": "process.binary_ref", + "object": "child_process", + "references": "child_process_file" + }, + { + "key": "process.child_refs", + "object": "process", + "references": [ + "child_process" + ] + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "child_process" + }, + { + "key": "directory.path", + "object": "child_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "child_process_file", + "references": "child_process_file_dir" + } + ], + "FolderPath": [ + { + "key": "directory.path", + "object": "child_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "child_process_file", + "references": "child_process_file_dir" + } + ], + "SHA1": { + "key": "file.hashes.SHA-1", + "object": "child_process_file" + }, + "SHA256": { + "key": "file.hashes.SHA-256", + "object": "child_process_file" + }, + "MD5": { + "key": "file.hashes.MD5", + "object": "child_process_file" + }, + "ProcessId": [ + { + "key": "process.pid", + "object": "child_process", + "transformer": "ToInteger" + }, + { + "key": "process.child_refs", + "object": "process", + "references": [ + "child_process" + ] + }, + { + "key": "x-oca-event.process_ref", + "object": "event", + "references": "child_process" + } + ], + "ProcessCommandLine": [ + { + "key": "process.command_line", + "object": "child_process" + }, + { + "key": "process.child_refs", + "object": "process", + "references": [ + "child_process" + ] + } + ], + "ProcessCreationTime": { + "key": "process.created", + "object": "child_process", + "transformer": "MsatpToTimestamp" + }, + "InitiatingProcessSHA1": [ + { + "key": "file.hashes.SHA-1", + "object": "process_file" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + } + ], + "InitiatingProcessSHA256": [ + { + "key": "file.hashes.SHA-256", + "object": "process_file" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + } + ], + "InitiatingProcessMD5": [ + { + "key": "file.hashes.MD5", + "object": "process_file" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + } + ], + "InitiatingProcessFileName": [ + { + "key": "file.name", + "object": "process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "process", + "transformer": "ToFileName" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + }, + { + "key": "process.parent_ref", + "object": "child_process", + "references": "process" + }, + { + "key": "directory.path", + "object": "process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "process_file_dir" + } + ], + "InitiatingProcessId": [ + { + "key": "process.pid", + "object": "process", + "transformer": "ToInteger" + }, + { + "key": "process.parent_ref", + "object": "child_process", + "references": "process" + } + ], + "InitiatingProcessCommandLine": { + "key": "process.command_line", + "object": "process" + }, + "InitiatingProcessCreationTime": { + "key": "process.created", + "object": "process", + "transformer": "MsatpToTimestamp" + }, + "InitiatingProcessFolderPath": [ + { + "key": "directory.path", + "object": "process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "process_file", + "references": "process_file_dir" + }, + { + "key": "process.binary_ref", + "object": "process", + "references": "process_file" + } + ], + "missingChildShouldMapInitiatingPid": { + "key": "x-oca-event.missingChildShouldMapInitiatingPid", + "object": "event" + }, + "InitiatingProcessParentId": { + "key": "process.pid", + "object": "parent_process", + "transformer": "ToInteger" + }, + "InitiatingProcessParentFileName": [ + { + "key": "file.name", + "object": "parent_process_file", + "transformer": "ToFileName" + }, + { + "key": "process.name", + "object": "parent_process", + "transformer": "ToFileName" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + }, + { + "key": "process.binary_ref", + "object": "parent_process", + "references": "parent_process_file" + }, + { + "key": "directory.path", + "object": "parent_process_file_dir", + "transformer": "ToMSATPDirectoryPath" + }, + { + "key": "file.parent_directory_ref", + "object": "parent_process_file", + "references": "parent_process_file_dir" + } + ], + "InitiatingProcessParentCreationTime": [ + { + "key": "process.created", + "object": "parent_process", + "transformer": "MsatpToTimestamp" + }, + { + "key": "process.parent_ref", + "object": "process", + "references": "parent_process" + } + ], + "AccountName": [ + { + "key": "user-account.user_id", + "object": "child_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "child_process", + "references": "child_process_creator" + } + ], + "InitiatingProcessAccountName": [ + { + "key": "user-account.user_id", + "object": "process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "process_creator" + } + ], + "AccountUpn": [ + { + "key": "user-account.account_login", + "object": "child_process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "child_process", + "references": "child_process_creator" + } + ], + "InitiatingProcessAccountUpn": [ + { + "key": "user-account.account_login", + "object": "process_creator" + }, + { + "key": "process.creator_user_ref", + "object": "process", + "references": "process_creator" + } + ], + "DeviceName": [ + { + "key": "x-oca-asset.hostname", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "DeviceId": [ + { + "key": "x-oca-asset.device_id", + "object": "host" + }, + { + "key": "x-oca-event.host_ref", + "object": "event", + "references": "host" + } + ], + "PublicIP": [ + { + "key": "ipv4-addr.value", + "object": "public-ip" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "public-ip" + ], + "group": true + } + ], + "OSPlatform": { + "key": "x-oca-asset.os_name", + "object": "host" + }, + "OSArchitecture": { + "key": "x-oca-asset.architecture", + "object": "host" + }, + "OSVersion": { + "key": "x-oca-asset.os_version", + "object": "host" + }, + "original_ref": [ + { + "key": "artifact.payload_bin", + "object": "original_ref", + "transformer": "ToBase64" + }, + { + "key": "x-oca-event.original_ref", + "object": "event", + "references": "original_ref" + } + ], + "event_link": [ + { + "key": "external-reference.url", + "object": "device_external_ref" + }, + { + "key": "x-oca-event.external_ref", + "object": "event", + "references": "device_external_ref" + } + ], + "MacAddressSet": [ + { + "key": "mac-addr.value", + "object": "mac", + "transformer": "FormatMacList", + "unwrap": true + }, + { + "key": "x-oca-asset.mac_refs", + "object": "host", + "references": [ + "mac" + ], + "unwrap": true, + "group": true + } + ], + "IPAddresses": [ + { + "key": "ipv4-addr.value", + "object": "host_ip4", + "unwrap": true, + "transformer": "FilterIPv4List" + }, + { + "key": "ipv6-addr.value", + "object": "host_ip6", + "unwrap": true, + "transformer": "FilterIPv6List" + }, + { + "key": "x-oca-asset.ip_refs", + "object": "host", + "references": [ + "host_ip4", + "host_ip6" + ], + "unwrap": true, + "group": true + } + ], + "provider": { + "key": "x-oca-event.provider", + "object": "event", + "value": "Microsoft Defender for Endpoint" + }, + "event_count": { + "key": "number_observed", + "cybox": false, + "transformer": "ToInteger" + }, + "ReportId": { + "key": "x-msatp.ReportId", + "object": "x-msatp" + }, + "TableName": { + "key": "x-msatp.Table", + "object": "x-msatp" + }, + "AdditionalFields": { + "key": "x-msatp.AdditionalFields", + "object": "x-msatp", + "transformer": "JsonToString" } }, "DeviceNetworkInfo": { diff --git a/stix_shifter_modules/msatp/stix_translation/query_constructor.py b/stix_shifter_modules/msatp/stix_translation/query_constructor.py index bef1a3266..ee911dbc3 100644 --- a/stix_shifter_modules/msatp/stix_translation/query_constructor.py +++ b/stix_shifter_modules/msatp/stix_translation/query_constructor.py @@ -180,6 +180,8 @@ def _parse_mapped_fields(expression, value, comparator, mapped_fields_array): if is_int_field or is_date_field: mapped_field = 'tostring({mapped_field})'.format(mapped_field=mapped_field) elif expression.object_path == "domain-name:value": + # since msatp has one field RemoteUrl that sometimes contains urls and sometime only domain names + # in order to find a domain name we need to replace the normal equality comparator to contains comparator = 'contains' elif expression.comparator in [ComparisonComparators.GreaterThan, ComparisonComparators.GreaterThanOrEqual, ComparisonComparators.LessThan, diff --git a/stix_shifter_modules/msatp/stix_translation/results_translator.py b/stix_shifter_modules/msatp/stix_translation/results_translator.py index fde49d412..46d0e4871 100644 --- a/stix_shifter_modules/msatp/stix_translation/results_translator.py +++ b/stix_shifter_modules/msatp/stix_translation/results_translator.py @@ -39,6 +39,32 @@ def get_first_object_by_type(objects, type_name): return None, None +def delete_object(objects, remove_ref): + """removes an object with all its references from objects""" + index_to_remove = int(remove_ref) + objects.pop(remove_ref) + renames = [] + for ref, sco in objects.items(): + if int(ref) > index_to_remove: + renames.append(ref) + remove = [] + for key, value in sco.items(): + if key.endswith("_ref"): + if int(value) == index_to_remove: + remove.append(key) + elif int(value) > index_to_remove: + sco[key] = str(int(value) - 1) + elif key.endswith("_refs"): + sco[key] = [str(int(item) - 1) if int(item) > index_to_remove else item for item in value if + int(item) != index_to_remove] + if len(sco[key]) == 0: + remove.append(key) + for i in remove: + sco.pop(i) + for ref in renames: + objects[str(int(ref) - 1)] = objects.pop(ref) + + def get_next_index(objects): """returns the next available index in the objects dictionary""" i = 0 @@ -62,6 +88,15 @@ def get_next_index(objects): return next_ref +def add_to_objects(observed, obj_to_add): + objects = observed['objects'] + index = get_next_index(objects) + objects[index] = obj_to_add + if int(index) < len(objects) - 1: + observed['objects'] = sort_objects(objects) + return index + + def parse_technique(technique): match = re.match(r'^(.+) \((T.+)\)$', technique) return { @@ -112,8 +147,7 @@ def fix_alerts(observed): 'severity': SeverityToNumericVal.transform(alert.get("Severity")), 'ttp_tagging_refs': [] } - finding_ref = get_next_index(objects) - objects[finding_ref] = finding + finding_ref = add_to_objects(observed, finding) if 'finding_refs' not in event: event['finding_refs'] = [] event['finding_refs'].append(finding_ref) @@ -121,16 +155,14 @@ def fix_alerts(observed): cat = alert['Category'] if cat not in ttps: cat_ttp = create_ttp_from_category(cat) - cat_ref = get_next_index(objects) - objects[cat_ref] = cat_ttp + cat_ref = add_to_objects(observed, cat_ttp) ttps[cat] = cat_ref finding['ttp_tagging_refs'].append(ttps[cat]) if 'AttackTechniques' in alert: for technique in alert['AttackTechniques']: if technique not in ttps: ttp = create_ttps_from_technique(technique) - ttp_ref = get_next_index(objects) - objects[ttp_ref] = ttp + ttp_ref = add_to_objects(observed, ttp) ttps[technique] = ttp_ref finding['ttp_tagging_refs'].append(ttps[technique]) if len(finding['ttp_tagging_refs']) == 0: @@ -179,11 +211,10 @@ def validate_process_ref_in_event(event, objects): if 'process_ref' in event: proc_ref = event['process_ref'] proc = get_reference(objects, event, 'process_ref', 'process') - event['process_ref'] = proc['parent_ref'] ref = proc['binary_ref'] event['file_ref'] = ref - del objects[proc_ref] - if pid is not None and pid != "": + delete_object(objects, proc_ref) + if pid is not None and pid != "-1": init_proc = [key for key, value in objects.items() if value.get("type") == "process" and value.get("pid") == event['missingChildShouldMapInitiatingPid']] @@ -192,6 +223,10 @@ def validate_process_ref_in_event(event, objects): del event['missingChildShouldMapInitiatingPid'] +def sort_objects(objects): + return {k: objects[k] for k in sorted(objects, key=lambda x: int(x))} + + class ResultsTranslator(JSONToStix): def translate_results(self, data_source, data): @@ -200,5 +235,5 @@ def translate_results(self, data_source, data): if observed["type"] == "observed-data" and "objects" in observed: fix_alerts(observed) fix_device_event_refs(observed) - observed['objects'] = {k: observed['objects'][k] for k in sorted(observed['objects'], key=lambda x: int(x))} + observed['objects'] = sort_objects(observed['objects']) return result diff --git a/stix_shifter_modules/msatp/stix_transmission/connector.py b/stix_shifter_modules/msatp/stix_transmission/connector.py index 753e158bf..55c09d0be 100644 --- a/stix_shifter_modules/msatp/stix_transmission/connector.py +++ b/stix_shifter_modules/msatp/stix_transmission/connector.py @@ -18,7 +18,7 @@ def __init__(self, connection, configuration): :param connection: dict, connection dict :param configuration: dict,config dict""" self.connector = __name__.split('.')[1] - self.configuration = configuration + self.options = connection['options'] self.adal_response = Connector.generate_token(self, connection, configuration) if self.adal_response['success']: configuration['auth']['access_token'] = self.adal_response['access_token'] @@ -72,7 +72,7 @@ def create_results_connection(self, query, offset, length): :param offset: int,offset value :param length: int,length value""" - util = ConnectorPostProcessing(self.configuration, False) + util = ConnectorPostProcessing(self.options, False) response_txt = None return_obj = { 'success': True, diff --git a/stix_shifter_modules/msatp/stix_transmission/connector_post_processing.py b/stix_shifter_modules/msatp/stix_transmission/connector_post_processing.py index 8f6ff3f27..ed4e31225 100644 --- a/stix_shifter_modules/msatp/stix_transmission/connector_post_processing.py +++ b/stix_shifter_modules/msatp/stix_transmission/connector_post_processing.py @@ -105,7 +105,10 @@ def unify_alert_fields(event_data): if 'AttackTechniques' in event_data: for techniques_lst in event_data['AttackTechniques']: try: - attack_techniques = json.loads(techniques_lst) + if techniques_lst == '': + attack_techniques = '' + else: + attack_techniques = json.loads(techniques_lst) except json.decoder.JSONDecodeError: attack_techniques = '' finally: @@ -151,13 +154,14 @@ class ConnectorPostProcessing: EVENTS_TABLES = ['DeviceNetworkEvents', 'DeviceProcessEvents', 'DeviceFileEvents', 'DeviceRegistryEvents', 'DeviceEvents', 'DeviceImageLoadEvents'] - def __init__(self, configuration, alert_mode): + def __init__(self, options, alert_mode): """Initialization. - :param configuration: dict,config dict""" + :param options: dict,config dict""" self.alert_mode = alert_mode - self.should_include_alerts = configuration.get("includeAlerts") - self.should_include_network_info = configuration.get("includeNetworkInfo") - self.should_include_host_os = configuration.get("includeHostOs") + self.should_include_alerts = options.get("includeAlerts") + self.should_include_network_info = options.get("includeNetworkInfo") + self.should_include_host_os = options.get("includeHostOs") + self.should_retain_original = options.get("retainOriginal") def join_alert_with_events(self, timestamp, device_name, report_id): events_query = "union {}".format(','.join( @@ -232,9 +236,10 @@ def post_process(self, response_data, return_obj, api_client_run_search): if table == "DeviceEvents": if 'ProcessId' not in event_data or event_data['ProcessId'] is None or \ event_data['ProcessId'] == "": - event_data["missingChildShouldMapInitiatingPid"] = event_data.get("InitiatingProcessId") + event_data["missingChildShouldMapInitiatingPid"] = -1 if event_data.get("InitiatingProcessId") is None else event_data.get("InitiatingProcessId") event_data['event_count'] = '1' remove_duplicate_ips(event_data) remove_duplicate_and_empty_fields(event_data) - event_data['original_ref'] = json.dumps(event_data) + if self.should_retain_original: + event_data['original_ref'] = json.dumps(event_data) return return_obj diff --git a/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_json_to_stix.py b/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_json_to_stix.py index 48d583a8a..cea1a9b61 100644 --- a/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_json_to_stix.py +++ b/stix_shifter_modules/msatp/tests/stix_translation/test_msatp_json_to_stix.py @@ -662,3 +662,161 @@ def test_extract_pipe_name(self): results_translator.extract_pipe_name(xmsatp, event) assert 'pipe_name' in event assert event['pipe_name'] == r'\\Device\\NamedPipe\\PrinterProviderProbePipe01' + + def test_delete_object(self): + objects = { + '0': { + 'type': 'x-oca-event', + 'ip_ref': '1', + 'other_ref': '2', + 'ip_refs': ['1', '2'] + }, + '1': { + 'type': 'ipv4-addr', + 'value': '9.9.9.9' + }, + '2': { + 'type': 'ipv4-addr', + 'value': '9.9.9.1' + } + } + from stix_shifter_modules.msatp.stix_translation import results_translator + results_translator.delete_object(objects, '1') + assert len(objects) == 2 + event = TestMsatpResultsToStix.get_first_of_type(objects.values(), 'x-oca-event') + assert event is not None, 'event object type not found' + assert 'ip_ref' not in event, 'ip_ref should have been removed' + assert 'other_ref' in event, 'other_ref should not have been removed' + assert event['other_ref'] == '1', 'other_ref reference was not decreased by one' + assert 'ip_refs' in event + assert len(event['ip_refs']) == 1 + assert event['ip_refs'][0] == '1' + assert objects['1']['value'] == '9.9.9.1' + + def test_sort_objects(self): + objects = { + '0': { + 'type': 'x-oca-event' + }, + '2': { + 'type': 'ipv4-addr', + 'value': '9.9.9.9' + }, + '1': { + 'type': 'ipv4-addr', + 'value': '9.9.9.1' + } + } + wanted_result = { + '0': { + 'type': 'x-oca-event' + }, + '1': { + 'type': 'ipv4-addr', + 'value': '9.9.9.1' + }, + '2': { + 'type': 'ipv4-addr', + 'value': '9.9.9.9' + } + } + from stix_shifter_modules.msatp.stix_translation import results_translator + sorted_objects = results_translator.sort_objects(objects) + list1 = [(key, value) for key, value in sorted_objects.items()] + list2 = [(key, value) for key, value in wanted_result.items()] + + assert list1 == list2 + + def test_add_object(self): + observed = { + 'objects': { + '0': { + 'type': 'x-oca-event', + 'ip_ref': '1', + 'other_ref': '2', + 'ip_refs': ['1', '2'] + }, + '1': { + 'type': 'ipv4-addr', + 'value': '9.9.9.9' + }, + '2': { + 'type': 'ipv4-addr', + 'value': '9.9.9.1' + } + } + } + new_obj = { + 'type': 'url', + 'value': 'https://quad9.net' + } + from stix_shifter_modules.msatp.stix_translation import results_translator + index = results_translator.add_to_objects(observed, new_obj) + assert index == '3' + assert observed['objects'] == { + '0': { + 'type': 'x-oca-event', + 'ip_ref': '1', + 'other_ref': '2', + 'ip_refs': ['1', '2'] + }, + '1': { + 'type': 'ipv4-addr', + 'value': '9.9.9.9' + }, + '2': { + 'type': 'ipv4-addr', + 'value': '9.9.9.1' + }, + '3': { + 'type': 'url', + 'value': 'https://quad9.net' + } + } + + def test_add_object_unordered(self): + observed = { + 'objects': { + '0': { + 'type': 'x-oca-event', + 'ip_ref': '2', + 'other_ref': '3', + 'ip_refs': ['2', '3'] + }, + '2': { + 'type': 'ipv4-addr', + 'value': '9.9.9.9' + }, + '3': { + 'type': 'ipv4-addr', + 'value': '9.9.9.1' + } + } + } + new_obj = { + 'type': 'url', + 'value': 'https://quad9.net' + } + from stix_shifter_modules.msatp.stix_translation import results_translator + index = results_translator.add_to_objects(observed, new_obj) + assert index == '1' + assert observed['objects'] == { + '0': { + 'type': 'x-oca-event', + 'ip_ref': '2', + 'other_ref': '3', + 'ip_refs': ['2', '3'] + }, + '1': { + 'type': 'url', + 'value': 'https://quad9.net' + }, + '2': { + 'type': 'ipv4-addr', + 'value': '9.9.9.9' + }, + '3': { + 'type': 'ipv4-addr', + 'value': '9.9.9.1' + } + } diff --git a/stix_shifter_modules/msatp/tests/stix_transmission/test_connector_post_processing.py b/stix_shifter_modules/msatp/tests/stix_transmission/test_connector_post_processing.py index 9a63331aa..7fc9340e7 100644 --- a/stix_shifter_modules/msatp/tests/stix_transmission/test_connector_post_processing.py +++ b/stix_shifter_modules/msatp/tests/stix_transmission/test_connector_post_processing.py @@ -17,10 +17,7 @@ def config(self): "tenant": "bla", "clientId": "bla", "clientSecret": "bla" - }, - "includeAlerts": True, - "includeHostOs": True, - "includeNetworkInfo": True + } } def connection(self): @@ -29,7 +26,10 @@ def connection(self): "port": 8080, "selfSignedCert": "cert", "options": { - + "includeAlerts": True, + "includeHostOs": True, + "includeNetworkInfo": True, + "retainOriginal": True } } @@ -158,7 +158,7 @@ def test_get_table_name(self, mock_adal_auth): def test_join_alerts_with_events(self, mock_adal_auth): mock_adal_auth.return_value = get_adal_mock_response() - util = ConnectorPostProcessing(self.config(), False) + util = ConnectorPostProcessing(self.connection()['options'], False) joined_query = util.join_alert_with_events('<>', 'devicename', 1234) assert joined_query == ('(union (find withsource = TableName in (DeviceNetworkEvents) where ' '(Timestamp == datetime(<>)) and (DeviceName == "devicename") and ' @@ -186,7 +186,7 @@ def test_join_alerts_with_events(self, mock_adal_auth): def test_join_query_with_alerts(self, mock_adal_auth): mock_adal_auth.return_value = get_adal_mock_response() query = 'union (find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (LocalIP =~ "9.9.9.9") or (RemoteIP =~ "9.9.9.9")),(find withsource = TableName in (DeviceEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (RemoteIP =~ "9.9.9.9") or (LocalIP =~ "9.9.9.9"))' - entry_point = ConnectorPostProcessing(self.config(), False) + entry_point = ConnectorPostProcessing(self.connection()['options'], False) joined_query = entry_point.join_query_with_other_tables(query) assert joined_query == ( "(union (find withsource = TableName in (DeviceNetworkEvents) " @@ -210,7 +210,7 @@ def test_join_query_with_alerts(self, mock_adal_auth): ) query = '(find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z) and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc | where AlertId =~ "123123")' - entry_point = ConnectorPostProcessing(self.config(), False) + entry_point = ConnectorPostProcessing(self.connection()['options'], False) joined_query = entry_point.join_query_with_other_tables(query) assert joined_query == ( '((find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z)' @@ -228,10 +228,13 @@ def test_join_query_with_alerts(self, mock_adal_auth): def test_join_query_no_info(self, mock_adal_auth): mock_adal_auth.return_value = get_adal_mock_response() query = 'union (find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (LocalIP =~ "9.9.9.9") or (RemoteIP =~ "9.9.9.9")),(find withsource = TableName in (DeviceEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (RemoteIP =~ "9.9.9.9") or (LocalIP =~ "9.9.9.9"))' - config = json.loads(json.dumps(self.config())) - config['includeHostOs'] = False - config['includeNetworkInfo'] = False - entry_point = ConnectorPostProcessing(config, False) + opts = { + "includeAlerts": True, + "includeHostOs": False, + "includeNetworkInfo": False, + "retainOriginal": True + } + entry_point = ConnectorPostProcessing(opts, False) joined_query = entry_point.join_query_with_other_tables(query) assert joined_query == ( "(union (find withsource = TableName in (DeviceNetworkEvents) " @@ -248,7 +251,7 @@ def test_join_query_no_info(self, mock_adal_auth): ) query = '(find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z) and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc | where AlertId =~ "123123")' - entry_point = ConnectorPostProcessing(config, False) + entry_point = ConnectorPostProcessing(opts, False) joined_query = entry_point.join_query_with_other_tables(query) assert joined_query == ( '((find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z)' @@ -259,9 +262,13 @@ def test_join_query_no_info(self, mock_adal_auth): def test_join_query_no_alerts(self, mock_adal_auth): mock_adal_auth.return_value = get_adal_mock_response() query = 'union (find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (LocalIP =~ "9.9.9.9") or (RemoteIP =~ "9.9.9.9")),(find withsource = TableName in (DeviceEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (RemoteIP =~ "9.9.9.9") or (LocalIP =~ "9.9.9.9"))' - config = json.loads(json.dumps(self.config())) - config['includeAlerts'] = False - entry_point = ConnectorPostProcessing(config, False) + opts = { + "includeAlerts": False, + "includeHostOs": True, + "includeNetworkInfo": True, + "retainOriginal": True + } + entry_point = ConnectorPostProcessing(opts, False) joined_query = entry_point.join_query_with_other_tables(query) assert joined_query == ( "(union (find withsource = TableName in (DeviceNetworkEvents) " @@ -281,7 +288,7 @@ def test_join_query_no_alerts(self, mock_adal_auth): ) query = '(find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z) and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc | where AlertId =~ "123123")' - entry_point = ConnectorPostProcessing(config, False) + entry_point = ConnectorPostProcessing(opts, False) joined_query = entry_point.join_query_with_other_tables(query) assert joined_query == ( '((find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z)' @@ -299,11 +306,13 @@ def test_join_query_no_alerts(self, mock_adal_auth): def test_join_query_only_events(self, mock_adal_auth): mock_adal_auth.return_value = get_adal_mock_response() query = 'union (find withsource = TableName in (DeviceNetworkEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (LocalIP =~ "9.9.9.9") or (RemoteIP =~ "9.9.9.9")),(find withsource = TableName in (DeviceEvents) where Timestamp >= datetime(2023-02-13T14:25:46.000Z) and Timestamp < datetime(2023-02-13T14:26:55.500Z) | order by Timestamp desc | where (RemoteIP =~ "9.9.9.9") or (LocalIP =~ "9.9.9.9"))' - config = json.loads(json.dumps(self.config())) - config['includeAlerts'] = False - config['includeHostOs'] = False - config['includeNetworkInfo'] = False - entry_point = ConnectorPostProcessing(config, False) + opts = { + "includeAlerts": False, + "includeHostOs": False, + "includeNetworkInfo": False, + "retainOriginal": True + } + entry_point = ConnectorPostProcessing(opts, False) joined_query = entry_point.join_query_with_other_tables(query) assert joined_query == ( "(union (find withsource = TableName in (DeviceNetworkEvents) " @@ -316,7 +325,7 @@ def test_join_query_only_events(self, mock_adal_auth): ) query = '(find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z) and Timestamp < datetime(2023-03-18T17:30:36.000Z) | order by Timestamp desc | where AlertId =~ "123123")' - entry_point = ConnectorPostProcessing(config, False) + entry_point = ConnectorPostProcessing(opts, False) joined_query = entry_point.join_query_with_other_tables(query) assert joined_query == ( '((find withsource = TableName in (DeviceAlertEvents) where Timestamp >= datetime(2023-03-16T17:21:30.000Z)'