diff --git a/splunk_addon/backup/assetnote_assets_download.py b/splunk_addon/backup/assetnote_assets_download.py index 4a8491b..176fb91 100644 --- a/splunk_addon/backup/assetnote_assets_download.py +++ b/splunk_addon/backup/assetnote_assets_download.py @@ -12,186 +12,71 @@ """Special prefix added to each log message""" LOG_PREFIX = "AssetNote" -"""Number of assets to load per page""" -ASSETS_PER_PAGE_COUNT = 20 +"""Number of exposures to load per page""" +EXPOSURES_PER_PAGE_COUNT = 20 -"""Limit the number of pages returned with assets (for testing. -If set to 0, then all the pages are returned.""" +"""Limit the number of pages returned with exposures (for testing)""" LIMIT_PAGES_RETURNED = 0 -# Graphql Query template to pull down assets. -# {page_count} is set and {page_num} is incremented to pull down the assets -# listing -ASSETS_GRAPHQL_QUERY_TEMPLATE = """ +# Graph Query templates to pull vulnerability and indicators +EXPOSURES_GRAPHQL_QUERY_TEMPLATE = """ query {{ - assets(s:[{{rel:"assetGroup", field:"name", dir:ASC}}],count:{page_count},page:{page_num}) {{ + exposures(count:{page_count},page:{page_num}) {{ edges {{ node {{ - ... on CloudAsset {{ - activeARecords {{ - edges {{ - node {{ - ... on ADnsRecord {{ - id, - ipAddress - }} - }} - }} - }}, - activeCnameRecords {{ - edges {{ - node {{ - ... on CnameDnsRecord {{ - id, - subdomain, - rawRecord - }} - }} - }} - }} - }}, - ... on IpAsset {{ - activeARecords {{ - edges {{ - node {{ - ... on ADnsRecord {{ - id, - ipAddress - }} - }} - }} - }}, - activeCnameRecords {{ - edges {{ - node {{ - ... on CnameDnsRecord {{ - id, - subdomain, - rawRecord - }} - }} - }} - }} - }}, - ... on SubdomainAsset {{ - activeARecords {{ - edges {{ - node {{ - ... on ADnsRecord {{ - id, - ipAddress - }} - }} - }} - }}, - activeCnameRecords {{ - edges {{ - node {{ - ... on CnameDnsRecord {{ - id, - subdomain, - rawRecord - }} - }} - }} - }} - }}, __typename, - ... on BaseAsset {{ - humanName, - activeARecordCount, - activeCnameRecordCount, - exposureRating, - hasUnmanagedExposures, - activeARecordCount, - onlinePortEntryCount, - isOnline, - onlineDnsEntryCount, - onlineTechnologyCount, - canBeMonitored, - assetGroupId, - assetGroupName, - assetType, - created, - geoData {{ - id, - city, - country - }}, - host, - ... on IpAsset {{ - ipAddress, - technologies {{ - edges {{ - node {{ - name - }} - }} - }}, - services {{ - edges {{ - node {{ - name, - port, - isActive, - lastActive - }} - }} - }} - }}, - ... on CloudAsset {{ - ipAddress, - technologies {{ - edges {{ - node {{ - name - }} - }} - }}, - services {{ - edges {{ - node {{ - name, - port, - isActive, - lastActive - }} + ... on BaseExposure {{ + id, + name, + exposureUrl, + isIgnored, + currentIncidentUuid, + lastDetected, + latestExposureEventType, + assetGroup, + category, + created, + definition, + lastUpdated, + probeId, + severity, + severityNormalized, + signature {{ + ... on HTTPSignature {{ + categoryId, + categoryName, + created, + cve, + definitionName, + description, + enabled, + followRedirects, + hasTemplate, + lastUpdated, + name, + recommendations, + references, + severity, + signatureType, + definition {{ + id, + name, + categoryId, + categoryName, + created, + description, + lastUpdated, + signatureCount }} }} - }}, - ... on SubdomainAsset {{ - ipAddress, - technologies {{ - edges {{ - node {{ - name - }} - }} - }}, - services {{ - edges {{ - node {{ - name, - port, - isActive, - lastActive - }} - }} + }} + asset {{ + ... on SubdomainAsset {{ + id, + host }} - }}, - id, - importance, - isMonitored, - lastUpdated, - notificationsEnabled, - parentName, - risk, - verifiedStatus, - assetGroup {{ - id, - name }} - }} + }} }} }}, pageInfo {{ @@ -246,24 +131,13 @@ def collect_events(helper, ew): opt_assetnote_api_key = helper.get_arg('assetnote_api_key') opt_assetnote_instance = helper.get_arg('assetnote_instance') - # Add all the parameters in this config to be used for printing - # and maintaining configuration - #all_params = {'assetnote_index': opt_assetnote_index, - # 'assetnote_sourcetype': opt_assetnote_sourcetype, - # 'assetnote_source': helper.get_input_type(), - # 'assetnote_api_key': opt_assetnote_api_key, - # 'assetnote_instance': opt_assetnote_instance, - # 'sleep_time': SLEEP_TIMEOUT_PER_PAGE, - # 'page_count': ASSETS_PER_PAGE_COUNT, - # 'limit_pages_returned': LIMIT_PAGES_RETURNED} - all_params = {'assetnote_index': helper.get_output_index(), 'assetnote_sourcetype': helper.get_sourcetype(), 'assetnote_source': helper.get_input_type(), 'assetnote_api_key': opt_assetnote_api_key, 'assetnote_instance': opt_assetnote_instance, 'sleep_time': SLEEP_TIMEOUT_PER_PAGE, - 'page_count': ASSETS_PER_PAGE_COUNT, + 'page_count': EXPOSURES_PER_PAGE_COUNT, 'limit_pages_returned': LIMIT_PAGES_RETURNED} # Printing all the current parameters to internal log @@ -274,16 +148,16 @@ def collect_events(helper, ew): info(helper, all_params, msg) info(helper, all_params, - "Requesting assets for instance: {assetnote_instance} page-wise...") - assets = [] + "Requesting exposures for instance: {assetnote_instance} page-wise...") + exposures = [] get_next_page = True all_params['page_num'] = 1 while get_next_page: info(helper, all_params, - "Requesting page: {page_num} for assets from AssetNote...") - graphql_query = ASSETS_GRAPHQL_QUERY_TEMPLATE.format(**all_params) + "Requesting page: {page_num} for exposures from AssetNote...") + graphql_query = EXPOSURES_GRAPHQL_QUERY_TEMPLATE.format(**all_params) url_to_call = "https://{assetnote_instance}.assetnotecloud.com/api/v2/graphql".format(**all_params) method = "POST" headers={ @@ -314,31 +188,31 @@ def collect_events(helper, ew): else: info(helper, all_params, - "Parsing page: {page_num} response for assets...") + "Parsing page: {page_num} response for exposures...") #info(helper, all_params, str(resp_json), # format_params=False) info(helper, all_params, - "Listing the number of assets on the page obtained...") - assets_on_page = resp_json['data']['assets']['edges'] - for asset_on_page in assets_on_page: - assets.append(asset_on_page) - all_params['assets_count'] = len(assets) + "Listing the number of exposures on the page obtained...") + exposures_on_page = resp_json['data']['exposures']['edges'] + for exposure_on_page in exposures_on_page: + exposures.append(exposure_on_page) + all_params['exposures_count'] = len(exposures) info(helper, all_params, - "Number of assets after page: {page_num} is: {assets_count}") + "Number of exposures after page: {page_num} is: {exposures_count}") info(helper, all_params, "Checking if another page exists from page: {page_num} response...") info(helper, all_params, - "Calculating the number of assets in the page...") - all_params['asset_count_per_page'] = len(assets_on_page) + "Calculating the number of exposures in the page...") + all_params['exposure_count_per_page'] = len(exposures_on_page) info(helper, all_params, - "Creating all {asset_count_per_page} assets as an event...") - new_event = helper.new_event(json.dumps(assets_on_page, indent=4), + "Creating all {exposure_count_per_page} exposures as an event...") + new_event = helper.new_event(json.dumps(exposures_on_page, indent=4), index=all_params['assetnote_index'], sourcetype=all_params['assetnote_sourcetype'], source=all_params['assetnote_source']) @@ -350,7 +224,7 @@ def collect_events(helper, ew): info(helper, all_params, "Checking if next page should be obtained...") - get_next_page_in_resp = resp_json['data']['assets']['pageInfo']['hasNextPage'] + get_next_page_in_resp = resp_json['data']['exposures']['pageInfo']['hasNextPage'] if get_next_page_in_resp: info(helper, all_params, @@ -377,3 +251,4 @@ def collect_events(helper, ew): "Sleeping for {sleep_time}s before requesting next page...") time.sleep(all_params['sleep_time']) + diff --git a/splunk_addon/backup/assetnote_exposures_download.py b/splunk_addon/backup/assetnote_exposures_download.py index ddb4a3b..fb3d747 100644 --- a/splunk_addon/backup/assetnote_exposures_download.py +++ b/splunk_addon/backup/assetnote_exposures_download.py @@ -12,70 +12,186 @@ """Special prefix added to each log message""" LOG_PREFIX = "AssetNote" -"""Number of exposures to load per page""" -EXPOSURES_PER_PAGE_COUNT = 20 +"""Number of assets to load per page""" +ASSETS_PER_PAGE_COUNT = 20 -"""Limit the number of pages returned with exposures (for testing)""" -LIMIT_PAGES_RETURNED = 1 +"""Limit the number of pages returned with assets (for testing. +If set to 0, then all the pages are returned.""" +LIMIT_PAGES_RETURNED = 0 -# Graph Query templates to pull vulnerability and indicators -EXPOSURES_GRAPHQL_QUERY_TEMPLATE = """ +# Graphql Query template to pull down assets. +# {page_count} is set and {page_num} is incremented to pull down the assets +# listing +ASSETS_GRAPHQL_QUERY_TEMPLATE = """ query {{ - exposures(count:{page_count},page:{page_num}) {{ + assets(s:[{{rel:"assetGroup", field:"name", dir:ASC}}],count:{page_count},page:{page_num}) {{ edges {{ node {{ - ... on BaseExposure {{ - id, - name, - exposureUrl, - isIgnored, - currentIncidentUuid, - lastDetected, - latestExposureEventType, - assetGroup, - category, - created, - definition, - lastUpdated, - probeId, - severity, - severityNormalized, - signature {{ - ... on HTTPSignature {{ - categoryId, - categoryName, - created, - cve, - definitionName, - description, - enabled, - followRedirects, - hasTemplate, - lastUpdated, - name, - recommendations, - references, - severity, - signatureType, - definition {{ - id, - name, - categoryId, - categoryName, - created, - description, - lastUpdated, - signatureCount + ... on CloudAsset {{ + activeARecords {{ + edges {{ + node {{ + ... on ADnsRecord {{ + id, + ipAddress + }} + }} + }} + }}, + activeCnameRecords {{ + edges {{ + node {{ + ... on CnameDnsRecord {{ + id, + subdomain, + rawRecord + }} + }} + }} + }} + }}, + ... on IpAsset {{ + activeARecords {{ + edges {{ + node {{ + ... on ADnsRecord {{ + id, + ipAddress + }} + }} + }} + }}, + activeCnameRecords {{ + edges {{ + node {{ + ... on CnameDnsRecord {{ + id, + subdomain, + rawRecord + }} }} }} }} - asset {{ - ... on SubdomainAsset {{ - id, - host + }}, + ... on SubdomainAsset {{ + activeARecords {{ + edges {{ + node {{ + ... on ADnsRecord {{ + id, + ipAddress + }} + }} + }} + }}, + activeCnameRecords {{ + edges {{ + node {{ + ... on CnameDnsRecord {{ + id, + subdomain, + rawRecord + }} + }} }} }} - }} + }}, + __typename, + ... on BaseAsset {{ + humanName, + activeARecordCount, + activeCnameRecordCount, + exposureRating, + hasUnmanagedExposures, + activeARecordCount, + onlinePortEntryCount, + isOnline, + onlineDnsEntryCount, + onlineTechnologyCount, + canBeMonitored, + assetGroupId, + assetGroupName, + assetType, + created, + geoData {{ + id, + city, + country + }}, + host, + ... on IpAsset {{ + ipAddress, + technologies {{ + edges {{ + node {{ + name + }} + }} + }}, + services {{ + edges {{ + node {{ + name, + port, + isActive, + lastActive + }} + }} + }} + }}, + ... on CloudAsset {{ + ipAddress, + technologies {{ + edges {{ + node {{ + name + }} + }} + }}, + services {{ + edges {{ + node {{ + name, + port, + isActive, + lastActive + }} + }} + }} + }}, + ... on SubdomainAsset {{ + ipAddress, + technologies {{ + edges {{ + node {{ + name + }} + }} + }}, + services {{ + edges {{ + node {{ + name, + port, + isActive, + lastActive + }} + }} + }} + }}, + id, + importance, + isMonitored, + lastUpdated, + notificationsEnabled, + parentName, + risk, + verifiedStatus, + assetGroup {{ + id, + name + }} + }} }} }}, pageInfo {{ @@ -130,13 +246,24 @@ def collect_events(helper, ew): opt_assetnote_api_key = helper.get_arg('assetnote_api_key') opt_assetnote_instance = helper.get_arg('assetnote_instance') + # Add all the parameters in this config to be used for printing + # and maintaining configuration + #all_params = {'assetnote_index': opt_assetnote_index, + # 'assetnote_sourcetype': opt_assetnote_sourcetype, + # 'assetnote_source': helper.get_input_type(), + # 'assetnote_api_key': opt_assetnote_api_key, + # 'assetnote_instance': opt_assetnote_instance, + # 'sleep_time': SLEEP_TIMEOUT_PER_PAGE, + # 'page_count': ASSETS_PER_PAGE_COUNT, + # 'limit_pages_returned': LIMIT_PAGES_RETURNED} + all_params = {'assetnote_index': helper.get_output_index(), 'assetnote_sourcetype': helper.get_sourcetype(), 'assetnote_source': helper.get_input_type(), 'assetnote_api_key': opt_assetnote_api_key, 'assetnote_instance': opt_assetnote_instance, 'sleep_time': SLEEP_TIMEOUT_PER_PAGE, - 'page_count': EXPOSURES_PER_PAGE_COUNT, + 'page_count': ASSETS_PER_PAGE_COUNT, 'limit_pages_returned': LIMIT_PAGES_RETURNED} # Printing all the current parameters to internal log @@ -147,16 +274,16 @@ def collect_events(helper, ew): info(helper, all_params, msg) info(helper, all_params, - "Requesting exposures for instance: {assetnote_instance} page-wise...") - exposures = [] + "Requesting assets for instance: {assetnote_instance} page-wise...") + assets = [] get_next_page = True all_params['page_num'] = 1 while get_next_page: info(helper, all_params, - "Requesting page: {page_num} for exposures from AssetNote...") - graphql_query = EXPOSURES_GRAPHQL_QUERY_TEMPLATE.format(**all_params) + "Requesting page: {page_num} for assets from AssetNote...") + graphql_query = ASSETS_GRAPHQL_QUERY_TEMPLATE.format(**all_params) url_to_call = "https://{assetnote_instance}.assetnotecloud.com/api/v2/graphql".format(**all_params) method = "POST" headers={ @@ -187,31 +314,31 @@ def collect_events(helper, ew): else: info(helper, all_params, - "Parsing page: {page_num} response for exposures...") + "Parsing page: {page_num} response for assets...") #info(helper, all_params, str(resp_json), # format_params=False) info(helper, all_params, - "Listing the number of exposures on the page obtained...") - exposures_on_page = resp_json['data']['exposures']['edges'] - for exposure_on_page in exposures_on_page: - exposures.append(exposure_on_page) - all_params['exposures_count'] = len(exposures) + "Listing the number of assets on the page obtained...") + assets_on_page = resp_json['data']['assets']['edges'] + for asset_on_page in assets_on_page: + assets.append(asset_on_page) + all_params['assets_count'] = len(assets) info(helper, all_params, - "Number of exposures after page: {page_num} is: {exposures_count}") + "Number of assets after page: {page_num} is: {assets_count}") info(helper, all_params, "Checking if another page exists from page: {page_num} response...") info(helper, all_params, - "Calculating the number of exposures in the page...") - all_params['exposure_count_per_page'] = len(exposures_on_page) + "Calculating the number of assets in the page...") + all_params['asset_count_per_page'] = len(assets_on_page) info(helper, all_params, - "Creating all {exposure_count_per_page} exposures as an event...") - new_event = helper.new_event(json.dumps(exposures_on_page, indent=4), + "Creating all {asset_count_per_page} assets as an event...") + new_event = helper.new_event(json.dumps(assets_on_page, indent=4), index=all_params['assetnote_index'], sourcetype=all_params['assetnote_sourcetype'], source=all_params['assetnote_source']) @@ -223,7 +350,7 @@ def collect_events(helper, ew): info(helper, all_params, "Checking if next page should be obtained...") - get_next_page_in_resp = resp_json['data']['exposures']['pageInfo']['hasNextPage'] + get_next_page_in_resp = resp_json['data']['assets']['pageInfo']['hasNextPage'] if get_next_page_in_resp: info(helper, all_params, @@ -246,6 +373,9 @@ def collect_events(helper, ew): "Incrementing page counter...") all_params['page_num'] += 1 + info(helper, all_params, + "Next page to get: {page_num}...") + info(helper, all_params, "Sleeping for {sleep_time}s before requesting next page...") time.sleep(all_params['sleep_time'])