Skip to content

Commit

Permalink
assetnote_assets_download.py,assetnote_exposures_download.py(update):…
Browse files Browse the repository at this point in the history 
… added up-to-date scripts as backup for the Splunk Add-On
  • Loading branch information
Manas Bellani committed Jan 29, 2020
1 parent bb14509 commit e13fa1b
Show file tree
Hide file tree
Showing 2 changed files with 273 additions and 268 deletions.
269 changes: 72 additions & 197 deletions splunk_addon/backup/assetnote_assets_download.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,186 +12,71 @@
"""Special prefix added to each log message"""
LOG_PREFIX = "AssetNote"

"""Number of assets to load per page"""
ASSETS_PER_PAGE_COUNT = 20
"""Number of exposures to load per page"""
EXPOSURES_PER_PAGE_COUNT = 20

"""Limit the number of pages returned with assets (for testing.
If set to 0, then all the pages are returned."""
"""Limit the number of pages returned with exposures (for testing)"""
LIMIT_PAGES_RETURNED = 0

# Graphql Query template to pull down assets.
# {page_count} is set and {page_num} is incremented to pull down the assets
# listing
ASSETS_GRAPHQL_QUERY_TEMPLATE = """
# Graph Query templates to pull vulnerability and indicators
EXPOSURES_GRAPHQL_QUERY_TEMPLATE = """
query {{
assets(s:[{{rel:"assetGroup", field:"name", dir:ASC}}],count:{page_count},page:{page_num}) {{
exposures(count:{page_count},page:{page_num}) {{
edges {{
node {{
... on CloudAsset {{
activeARecords {{
edges {{
node {{
... on ADnsRecord {{
id,
ipAddress
}}
}}
}}
}},
activeCnameRecords {{
edges {{
node {{
... on CnameDnsRecord {{
id,
subdomain,
rawRecord
}}
}}
}}
}}
}},
... on IpAsset {{
activeARecords {{
edges {{
node {{
... on ADnsRecord {{
id,
ipAddress
}}
}}
}}
}},
activeCnameRecords {{
edges {{
node {{
... on CnameDnsRecord {{
id,
subdomain,
rawRecord
}}
}}
}}
}}
}},
... on SubdomainAsset {{
activeARecords {{
edges {{
node {{
... on ADnsRecord {{
id,
ipAddress
}}
}}
}}
}},
activeCnameRecords {{
edges {{
node {{
... on CnameDnsRecord {{
id,
subdomain,
rawRecord
}}
}}
}}
}}
}},
__typename,
... on BaseAsset {{
humanName,
activeARecordCount,
activeCnameRecordCount,
exposureRating,
hasUnmanagedExposures,
activeARecordCount,
onlinePortEntryCount,
isOnline,
onlineDnsEntryCount,
onlineTechnologyCount,
canBeMonitored,
assetGroupId,
assetGroupName,
assetType,
created,
geoData {{
id,
city,
country
}},
host,
... on IpAsset {{
ipAddress,
technologies {{
edges {{
node {{
name
}}
}}
}},
services {{
edges {{
node {{
name,
port,
isActive,
lastActive
}}
}}
}}
}},
... on CloudAsset {{
ipAddress,
technologies {{
edges {{
node {{
name
}}
}}
}},
services {{
edges {{
node {{
name,
port,
isActive,
lastActive
}}
... on BaseExposure {{
id,
name,
exposureUrl,
isIgnored,
currentIncidentUuid,
lastDetected,
latestExposureEventType,
assetGroup,
category,
created,
definition,
lastUpdated,
probeId,
severity,
severityNormalized,
signature {{
... on HTTPSignature {{
categoryId,
categoryName,
created,
cve,
definitionName,
description,
enabled,
followRedirects,
hasTemplate,
lastUpdated,
name,
recommendations,
references,
severity,
signatureType,
definition {{
id,
name,
categoryId,
categoryName,
created,
description,
lastUpdated,
signatureCount
}}
}}
}},
... on SubdomainAsset {{
ipAddress,
technologies {{
edges {{
node {{
name
}}
}}
}},
services {{
edges {{
node {{
name,
port,
isActive,
lastActive
}}
}}
}}
asset {{
... on SubdomainAsset {{
id,
host
}}
}},
id,
importance,
isMonitored,
lastUpdated,
notificationsEnabled,
parentName,
risk,
verifiedStatus,
assetGroup {{
id,
name
}}
}}
}}
}}
}},
pageInfo {{
Expand Down Expand Up @@ -246,24 +131,13 @@ def collect_events(helper, ew):
opt_assetnote_api_key = helper.get_arg('assetnote_api_key')
opt_assetnote_instance = helper.get_arg('assetnote_instance')

# Add all the parameters in this config to be used for printing
# and maintaining configuration
#all_params = {'assetnote_index': opt_assetnote_index,
# 'assetnote_sourcetype': opt_assetnote_sourcetype,
# 'assetnote_source': helper.get_input_type(),
# 'assetnote_api_key': opt_assetnote_api_key,
# 'assetnote_instance': opt_assetnote_instance,
# 'sleep_time': SLEEP_TIMEOUT_PER_PAGE,
# 'page_count': ASSETS_PER_PAGE_COUNT,
# 'limit_pages_returned': LIMIT_PAGES_RETURNED}

all_params = {'assetnote_index': helper.get_output_index(),
'assetnote_sourcetype': helper.get_sourcetype(),
'assetnote_source': helper.get_input_type(),
'assetnote_api_key': opt_assetnote_api_key,
'assetnote_instance': opt_assetnote_instance,
'sleep_time': SLEEP_TIMEOUT_PER_PAGE,
'page_count': ASSETS_PER_PAGE_COUNT,
'page_count': EXPOSURES_PER_PAGE_COUNT,
'limit_pages_returned': LIMIT_PAGES_RETURNED}

# Printing all the current parameters to internal log
Expand All @@ -274,16 +148,16 @@ def collect_events(helper, ew):
info(helper, all_params, msg)

info(helper, all_params,
"Requesting assets for instance: {assetnote_instance} page-wise...")
assets = []
"Requesting exposures for instance: {assetnote_instance} page-wise...")
exposures = []
get_next_page = True
all_params['page_num'] = 1

while get_next_page:

info(helper, all_params,
"Requesting page: {page_num} for assets from AssetNote...")
graphql_query = ASSETS_GRAPHQL_QUERY_TEMPLATE.format(**all_params)
"Requesting page: {page_num} for exposures from AssetNote...")
graphql_query = EXPOSURES_GRAPHQL_QUERY_TEMPLATE.format(**all_params)
url_to_call = "https://{assetnote_instance}.assetnotecloud.com/api/v2/graphql".format(**all_params)
method = "POST"
headers={
Expand Down Expand Up @@ -314,31 +188,31 @@ def collect_events(helper, ew):
else:

info(helper, all_params,
"Parsing page: {page_num} response for assets...")
"Parsing page: {page_num} response for exposures...")

#info(helper, all_params, str(resp_json),
# format_params=False)

info(helper, all_params,
"Listing the number of assets on the page obtained...")
assets_on_page = resp_json['data']['assets']['edges']
for asset_on_page in assets_on_page:
assets.append(asset_on_page)
all_params['assets_count'] = len(assets)
"Listing the number of exposures on the page obtained...")
exposures_on_page = resp_json['data']['exposures']['edges']
for exposure_on_page in exposures_on_page:
exposures.append(exposure_on_page)
all_params['exposures_count'] = len(exposures)

info(helper, all_params,
"Number of assets after page: {page_num} is: {assets_count}")
"Number of exposures after page: {page_num} is: {exposures_count}")

info(helper, all_params,
"Checking if another page exists from page: {page_num} response...")

info(helper, all_params,
"Calculating the number of assets in the page...")
all_params['asset_count_per_page'] = len(assets_on_page)
"Calculating the number of exposures in the page...")
all_params['exposure_count_per_page'] = len(exposures_on_page)

info(helper, all_params,
"Creating all {asset_count_per_page} assets as an event...")
new_event = helper.new_event(json.dumps(assets_on_page, indent=4),
"Creating all {exposure_count_per_page} exposures as an event...")
new_event = helper.new_event(json.dumps(exposures_on_page, indent=4),
index=all_params['assetnote_index'],
sourcetype=all_params['assetnote_sourcetype'],
source=all_params['assetnote_source'])
Expand All @@ -350,7 +224,7 @@ def collect_events(helper, ew):

info(helper, all_params,
"Checking if next page should be obtained...")
get_next_page_in_resp = resp_json['data']['assets']['pageInfo']['hasNextPage']
get_next_page_in_resp = resp_json['data']['exposures']['pageInfo']['hasNextPage']
if get_next_page_in_resp:

info(helper, all_params,
Expand All @@ -377,3 +251,4 @@ def collect_events(helper, ew):
"Sleeping for {sleep_time}s before requesting next page...")
time.sleep(all_params['sleep_time'])


Loading

0 comments on commit e13fa1b

Please sign in to comment.