From d0317e0f9d0cd0c78cb1dee6a7e4d9b074c0fb7c Mon Sep 17 00:00:00 2001 From: Alex Chantavy Date: Tue, 5 Nov 2019 12:36:14 -0800 Subject: [PATCH] Revert "Crxcavator add permission nodes (#185)" (#197) This reverts commit a5463521a6d453ba13c98c1cbdf8aa1849c1c860. --- cartography/data/indexes.cypher | 1 - .../cleanup/crxcavator_import_cleanup.json | 10 -- cartography/intel/crxcavator/crxcavator.py | 108 +++--------------- docs/schema/crxcavator.md | 38 ++---- tests/data/crxcavator/crxcavator.py | 15 --- .../intel/crxcavator/test_crxcavator.py | 41 +------ 6 files changed, 26 insertions(+), 187 deletions(-) diff --git a/cartography/data/indexes.cypher b/cartography/data/indexes.cypher index 158813568..e6c74239c 100644 --- a/cartography/data/indexes.cypher +++ b/cartography/data/indexes.cypher @@ -14,7 +14,6 @@ CREATE INDEX ON :AWSVpc(id); CREATE INDEX ON :AccountAccessKey(accesskeyid); CREATE INDEX ON :AutoScalingGroup(arn); CREATE INDEX ON :ChromeExtension(id); -CREATE INDEX ON :ChromeExtensionPermission(id); CREATE INDEX ON :DBGroup(name); CREATE INDEX ON :DNSRecord(id); CREATE INDEX ON :DNSZone(name); diff --git a/cartography/data/jobs/cleanup/crxcavator_import_cleanup.json b/cartography/data/jobs/cleanup/crxcavator_import_cleanup.json index 3b917ac21..e9ce0e2c9 100644 --- a/cartography/data/jobs/cleanup/crxcavator_import_cleanup.json +++ b/cartography/data/jobs/cleanup/crxcavator_import_cleanup.json @@ -9,20 +9,10 @@ "iterative": true, "iterationsize": 100 }, - { - "query": "MATCH (n:ChromeExtensionPermission) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted", - "iterative": true, - "iterationsize": 100 - }, { "query": "MATCH (GSuiteUser)-[r:INSTALLS]->(:ChromeExtension) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted", "iterative": true, "iterationsize": 100 - }, - { - "query": "MATCH (ChromeExtensionPermission)<-[r:DECLARES]-(:ChromeExtension) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted", - "iterative": true, - "iterationsize": 100 }], "name": "cleanup CRXcavator extensions" } diff --git a/cartography/intel/crxcavator/crxcavator.py b/cartography/intel/crxcavator/crxcavator.py index f140ad3dd..ae6019c0b 100644 --- a/cartography/intel/crxcavator/crxcavator.py +++ b/cartography/intel/crxcavator/crxcavator.py @@ -69,10 +69,7 @@ def get_extensions(crxcavator_api_key, crxcavator_base_url, extensions_list): details = get_extension_details(crxcavator_api_key, crxcavator_base_url, extension_id, version) if not details: # we only have the name and version from group API, create minimal version - logger.debug( - f"CRXcavator ingest - No results returned from report API for " - f"extension {extension_id} {version}", - ) + logger.debug(f"No results returned from report API for extension {extension_id} {version}") details = { 'data': dict( webstore={ @@ -82,7 +79,7 @@ def get_extensions(crxcavator_api_key, crxcavator_base_url, extensions_list): } extensions_details.append(details) except exceptions.RequestException as e: - logger.info(f"CRXcavator ingest - API error retrieving details for extension {extension_id}", e) + logger.info(f"API error retrieving details for extension {extension_id}", e) return extensions_details @@ -90,43 +87,22 @@ def transform_extensions(extension_details): """ Transforms the raw extensions JSON from the API into a list of extensions data :param extension_details: List containing the extension details - :return extension: List containing extension info for ingestion - :return permissions: Unique list of permissions returned from crxcavator - :return extension_permissions: Dictionary linking extensions to permissions + :return: List containing extension info for ingestion """ # the JSON returned from the CRXcavator API does not return well formatted objects # instead, each object is named after it's key, making enumeration more difficult # will build a cleaner object for import into graph extensions = [] - permissions_set = set() - extensions_permissions = [] for extension in extension_details: extension_id = extension['extension_id'] version = extension['version'] data = extension.get('data') if not data: - logger.warning(f'CRXcavator ingest - Could not retrieve details for extension {extension}') + logger.warning(f'Could not retrieve details for extension {extension}') continue risk = data.get('risk', {}) webstore = data.get('webstore', {}) - manifest = data.get('manifest', {}) - permissions = manifest.get('permissions', {}) - for permission in permissions: - if type(permission) is dict: - parsed_permissions = parse_permissions_dict(permission) - permissions_set.update(parsed_permissions) - for sub_permission in parsed_permissions: - extensions_permissions.append({ - 'id': f"{extension_id}|{version}", - 'permission': sub_permission, - }) - continue - permissions_set.add(permission) - extensions_permissions.append({ - 'id': f"{extension_id}|{version}", - 'permission': permission, - }) extensions.append({ 'id': f"{extension_id}|{version}", 'extension_id': extension_id, @@ -158,34 +134,7 @@ def transform_extensions(extension_details): 'price': webstore.get('price'), 'report_link': f"https://crxcavator.io/report/{extension_id}/{version}", }) - return extensions, list(permissions_set), extensions_permissions - - -def parse_permissions_dict(permissions_dict): - """ - Parses the possible complex permissions objects into a list of objects - Parsing cases based on results from crxcavator API as of October 2019 - :param permissions_dict: a dict object from the permissions key - :return: a list of individual permissions - """ - permissions = [] - usb_devices = permissions_dict.get('usbDevices', []) - for device in usb_devices: - permissions.append(f"usbdevice-{device.get('productId', 'none')}-{device.get('vendorId', 'none')}") - filesystem = permissions_dict.get('fileSystem', []) - for filesystem_permission in filesystem: - permissions.append(f"filesystem-{filesystem_permission}") - socket = permissions_dict.get('socket', []) - for socket_permission in socket: - permissions.append(f"socket-{socket_permission}") - media_galleries = permissions_dict.get('mediaGalleries', []) - for media in media_galleries: - permissions.append(f"mediagalleries-{media}") - if len(permissions) == 0: - # this is a case not currently handled, so just log it and do not ingest it - permission = json.dumps(permissions_dict) - logger.warning(f"CRXcavator ingest - Unknown permissions dict type {permission}") - return permissions + return extensions def get_risk_data(data_dict, key): @@ -200,17 +149,15 @@ def get_risk_data(data_dict, key): return data_score -def load_extensions(extensions, permissions, extension_permissions, session, update_tag): +def load_extensions(extensions, session, update_tag): """ Ingests the extension details into Neo4J :param extensions: List of extension data to load to Neo4J - :param permissions: unique list of extension permissions - :param extension_permissions: Dictionary of extension-permission pairings :param session: Neo4J session object for server communication :param update_tag: Timestamp used to determine data freshness :return: None """ - extensions_ingestion_cypher = """ + ingestion_cypher = """ UNWIND {ExtensionsData} as extension MERGE (e:ChromeExtension{id: extension.id}) ON CREATE SET @@ -248,34 +195,8 @@ def load_extensions(extensions, permissions, extension_permissions, session, upd e.lastupdated = {UpdateTag} """ - permissions_ingestion_cypher = """ - UNWIND {Permissions} as permission - MERGE (e:ChromeExtensionPermission{id: permission}) - ON CREATE SET - e.firstseen = timestamp() - SET - e.name = permission, - e.lastupdated = {UpdateTag} - """ - - extensions_permissions_cypher = """ - UNWIND {ExtensionPermissions} as extension_permission - MATCH (perm:ChromeExtensionPermission{id: extension_permission.permission}), - (ext:ChromeExtension{id:extension_permission.id}) - MERGE (ext)-[r:DECLARES]->(perm) - ON CREATE SET - r.firstseen = timestamp() - SET r.lastupdated = {UpdateTag} - """ - - logger.info(f'CRXcavator ingest - Ingesting {len(extensions)} extensions') - session.run(extensions_ingestion_cypher, ExtensionsData=extensions, UpdateTag=update_tag) - - logger.info(f'CRXcavator ingest - Ingesting {len(permissions)} Chrome extension permissions') - session.run(permissions_ingestion_cypher, Permissions=permissions, UpdateTag=update_tag) - - logger.info(f'CRXcavator ingest - Ingesting {len(extension_permissions)} extension to permission links') - session.run(extensions_permissions_cypher, ExtensionPermissions=extension_permissions, UpdateTag=update_tag) + logger.info(f'Ingesting {len(extensions)} extensions') + session.run(ingestion_cypher, ExtensionsData=extensions, UpdateTag=update_tag) def transform_user_extensions(user_extension_json): @@ -340,9 +261,9 @@ def load_user_extensions(users, extensions_by_user, session, update_tag): SET r.lastupdated = {UpdateTag} """ - logger.info(f'CRXcavator ingest - Ingesting {len(users)} users') + logger.info(f'Ingesting {len(users)} users') session.run(user_ingestion_cypher, Users=users, UpdateTag=update_tag) - logger.info(f'CRXcavator ingest - Ingesting {len(extensions_by_user)} user->extension relationships') + logger.info(f'Ingesting {len(extensions_by_user)} user->extension relationships') session.run(extension_ingestion_cypher, ExtensionsUsers=extensions_by_user, UpdateTag=update_tag) @@ -359,9 +280,6 @@ def sync_extensions(neo4j_session, common_job_parameters, crxcavator_api_key, cr user_extensions_json = get_users_extensions(crxcavator_api_key, crxcavator_base_url) users, extensions_list, user_extensions = transform_user_extensions(user_extensions_json) extension_details = get_extensions(crxcavator_api_key, crxcavator_base_url, extensions_list) - extensions, permissions_list, extension_permissions = transform_extensions(extension_details) - load_extensions( - extensions, permissions_list, extension_permissions, - neo4j_session, common_job_parameters['UPDATE_TAG'], - ) + extensions = transform_extensions(extension_details) + load_extensions(extensions, neo4j_session, common_job_parameters['UPDATE_TAG']) load_user_extensions(users, user_extensions, neo4j_session, common_job_parameters['UPDATE_TAG']) diff --git a/docs/schema/crxcavator.md b/docs/schema/crxcavator.md index eaf7e1e38..69b3ff73e 100644 --- a/docs/schema/crxcavator.md +++ b/docs/schema/crxcavator.md @@ -4,15 +4,18 @@ **Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* +- [Table of contents](#table-of-contents) - [GSuiteUser](#gsuiteuser) - [Relationships](#relationships) -- [ChromeExtension](#chromeextension) - [Relationships](#relationships-1) -- [ChromeExtensionPermission](#chromeextensionpermission) - - [Relationships](#relationships-2) +## Table of contents + +- [GSuiteUser](#gsuiteuser) +- [ChromeExtension](#chromeextension) + ## GSuiteUser Placeholder representation of a single G Suite [user object](https://developers.google.com/admin-sdk/directory/v1/reference/users). This node is the minimal data necessary to map who has extensions installed until full G Suite data is imported. @@ -33,7 +36,7 @@ Placeholder representation of a single G Suite [user object](https://developers. (GSuiteUser)-[INSTALLS]->(ChromeExtension) ``` -## ChromeExtension + ## ChromeExtension Representation of a CRXcavator Chrome Extension [Report](https://crxcavator.io/apidocs#tag/report). @@ -41,7 +44,7 @@ Placeholder representation of a single G Suite [user object](https://developers. |-------|--------------| | firstseen| Timestamp of when a sync job first discovered this node | | lastupdated | Timestamp of the last time the node was updated | -| id | The combined extension name and version e.g. "Docs|1.0" | +| id | The combined extension name and version e.g. "Docs|1.0" | | extension_id | CRXcavator id for extension. | | version | The versions of the extension in this report | | risk_total | CRXcavator risk score for the extension | @@ -71,31 +74,10 @@ Placeholder representation of a single G Suite [user object](https://developers. | price | Extension price in webstore if applicable | | report_link | URL of full extension report on crxcavator.io -### Relationships + ### Relationships -- GSuiteUsers install ChromeExtensions. +- GSuiteUsers install ChromeExternsions. ``` (GSuiteUser)-[INSTALLS]->(ChromeExtension) ``` - - -## ChromeExtensionPermission - -Individual permission entries from the extension manifest file. This information is returned from the manifest object in the [Report](https://crxcavator.io/apidocs#tag/report) API. These permissions are defined in the [Chrome documentation](https://developer.chrome.com/apps/declare_permissions#manifest). - - -| Field | Description | -|-------|--------------| -| firstseen| Timestamp of when a sync job first discovered this node | -| lastupdated | Timestamp of the last time the node was updated | -| id | The permission name" | -| name | The specific permission declared in the manifest - -### Relationships - -- ChromeExtension declares ChromeExtensionPermission. - - ``` - (ChromeExtension)-[DECLARES]->(ChromeExtensionPermission) - ``` diff --git a/tests/data/crxcavator/crxcavator.py b/tests/data/crxcavator/crxcavator.py index c78155c5f..ebfc9b1f0 100644 --- a/tests/data/crxcavator/crxcavator.py +++ b/tests/data/crxcavator/crxcavator.py @@ -42,12 +42,6 @@ 'type': 'Extension', 'price': '', }, - 'manifest': - { - 'permissions': [ - '*://*/*', - ], - }, }, 'extension_id': 'f06981cbc72a3c6e2e9e736cbdaef4865a4571bc', 'version': '1.0', @@ -87,15 +81,6 @@ }, ] -TRANSFORMED_PERMISSIONS_DATA = ['*://*/*'] - -TRANSFORMED_EXTENSION_PERMISSIONS_DATA = [ - { - 'id': 'f06981cbc72a3c6e2e9e736cbdaef4865a4571bc|1.0', - 'permission': '*://*/*', - }, -] - USER_RESPONSE = { 'f06981cbc72a3c6e2e9e736cbdaef4865a4571bc': { diff --git a/tests/integration/cartography/intel/crxcavator/test_crxcavator.py b/tests/integration/cartography/intel/crxcavator/test_crxcavator.py index 4886c0379..e3e544d61 100644 --- a/tests/integration/cartography/intel/crxcavator/test_crxcavator.py +++ b/tests/integration/cartography/intel/crxcavator/test_crxcavator.py @@ -8,8 +8,6 @@ def _ensure_local_neo4j_has_test_extensions_data(neo4j_session): cartography.intel.crxcavator.crxcavator.load_extensions( tests.data.crxcavator.crxcavator.TRANSFORMED_EXTENSIONS_DATA, - tests.data.crxcavator.crxcavator.TRANSFORMED_PERMISSIONS_DATA, - tests.data.crxcavator.crxcavator.TRANSFORMED_EXTENSION_PERMISSIONS_DATA, neo4j_session, TEST_UPDATE_TAG, ) @@ -29,12 +27,9 @@ def test_transform_and_load_extensions(neo4j_session): Test that we can correctly transform and load ChromeExtension nodes to Neo4j. """ extension_res = tests.data.crxcavator.crxcavator.REPORT_RESPONSE - extension_list, permissions, extension_permissions = \ - cartography.intel.crxcavator.crxcavator.transform_extensions(extension_res) + extension_list = cartography.intel.crxcavator.crxcavator.transform_extensions(extension_res) cartography.intel.crxcavator.crxcavator.load_extensions( extension_list, - permissions, - extension_permissions, neo4j_session, TEST_UPDATE_TAG, ) @@ -110,6 +105,7 @@ def test_transform_and_load_extensions(neo4j_session): n['ext.report_link'], ) for n in nodes ]) + print(actual_nodes) expected_nodes = list([ ( expected_extension_id, @@ -146,43 +142,12 @@ def test_transform_and_load_extensions(neo4j_session): assert actual_nodes == expected_nodes -def test_permission_to_extension(neo4j_session): - """ - Ensure that permissions are connected to extensions. - """ - _ensure_local_neo4j_has_test_extensions_data(neo4j_session) - query = """ - MATCH(permission:ChromeExtensionPermission)<-[:DECLARES]-(ext:ChromeExtension{id:{ExtensionId}}) - RETURN permission.name, ext.id, ext.name - """ - expected_extension_id = 'f06981cbc72a3c6e2e9e736cbdaef4865a4571bc|1.0' - nodes = neo4j_session.run( - query, - ExtensionId=expected_extension_id, - ) - actual_nodes = { - ( - n['permission.name'], - n['ext.id'], - n['ext.name'], - ) for n in nodes - } - - expected_nodes = { - ( - '*://*/*', - 'f06981cbc72a3c6e2e9e736cbdaef4865a4571bc|1.0', - 'CartographyIntegrationTest', - ), - } - assert actual_nodes == expected_nodes - - def test_transform_and_load_user_extensions(neo4j_session): """ Ensure we can transform and load users and extension mapping. """ users_res = tests.data.crxcavator.crxcavator.USER_RESPONSE + type(users_res) users_list, extensions_list, user_extensions_list = \ cartography.intel.crxcavator.crxcavator.transform_user_extensions(users_res) cartography.intel.crxcavator.crxcavator.load_user_extensions(