Skip to content

Commit

Permalink
Revert "Crxcavator add permission nodes (#185)" (#197)
Browse files Browse the repository at this point in the history 
This reverts commit a546352.
  • Loading branch information
@achantavy
achantavy authored Nov 5, 2019
1 parent a546352 commit d0317e0
Show file tree
Hide file tree
Showing 6 changed files with 26 additions and 187 deletions.
1 change: 0 additions & 1 deletion cartography/data/indexes.cypher
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ CREATE INDEX ON :AWSVpc(id);
CREATE INDEX ON :AccountAccessKey(accesskeyid);
CREATE INDEX ON :AutoScalingGroup(arn);
CREATE INDEX ON :ChromeExtension(id);
CREATE INDEX ON :ChromeExtensionPermission(id);
CREATE INDEX ON :DBGroup(name);
CREATE INDEX ON :DNSRecord(id);
CREATE INDEX ON :DNSZone(name);
Expand Down
10 changes: 0 additions & 10 deletions cartography/data/jobs/cleanup/crxcavator_import_cleanup.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,20 +9,10 @@
"iterative": true,
"iterationsize": 100
},
{
"query": "MATCH (n:ChromeExtensionPermission) WHERE n.lastupdated <> {UPDATE_TAG} WITH n LIMIT {LIMIT_SIZE} DETACH DELETE (n) return COUNT(*) as TotalCompleted",
"iterative": true,
"iterationsize": 100
},
{
"query": "MATCH (GSuiteUser)-[r:INSTALLS]->(:ChromeExtension) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted",
"iterative": true,
"iterationsize": 100
},
{
"query": "MATCH (ChromeExtensionPermission)<-[r:DECLARES]-(:ChromeExtension) WHERE r.lastupdated <> {UPDATE_TAG} WITH r LIMIT {LIMIT_SIZE} DELETE (r) return COUNT(*) as TotalCompleted",
"iterative": true,
"iterationsize": 100
}],
"name": "cleanup CRXcavator extensions"
}
108 changes: 13 additions & 95 deletions cartography/intel/crxcavator/crxcavator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,7 @@ def get_extensions(crxcavator_api_key, crxcavator_base_url, extensions_list):
details = get_extension_details(crxcavator_api_key, crxcavator_base_url, extension_id, version)
if not details:
# we only have the name and version from group API, create minimal version
logger.debug(
f"CRXcavator ingest - No results returned from report API for "
f"extension {extension_id} {version}",
)
logger.debug(f"No results returned from report API for extension {extension_id} {version}")
details = {
'data': dict(
webstore={
Expand All @@ -82,51 +79,30 @@ def get_extensions(crxcavator_api_key, crxcavator_base_url, extensions_list):
}
extensions_details.append(details)
except exceptions.RequestException as e:
logger.info(f"CRXcavator ingest - API error retrieving details for extension {extension_id}", e)
logger.info(f"API error retrieving details for extension {extension_id}", e)
return extensions_details


def transform_extensions(extension_details):
"""
Transforms the raw extensions JSON from the API into a list of extensions data
:param extension_details: List containing the extension details
:return extension: List containing extension info for ingestion
:return permissions: Unique list of permissions returned from crxcavator
:return extension_permissions: Dictionary linking extensions to permissions
:return: List containing extension info for ingestion
"""
# the JSON returned from the CRXcavator API does not return well formatted objects
# instead, each object is named after it's key, making enumeration more difficult
# will build a cleaner object for import into graph

extensions = []
permissions_set = set()
extensions_permissions = []
for extension in extension_details:
extension_id = extension['extension_id']
version = extension['version']
data = extension.get('data')
if not data:
logger.warning(f'CRXcavator ingest - Could not retrieve details for extension {extension}')
logger.warning(f'Could not retrieve details for extension {extension}')
continue
risk = data.get('risk', {})
webstore = data.get('webstore', {})
manifest = data.get('manifest', {})
permissions = manifest.get('permissions', {})
for permission in permissions:
if type(permission) is dict:
parsed_permissions = parse_permissions_dict(permission)
permissions_set.update(parsed_permissions)
for sub_permission in parsed_permissions:
extensions_permissions.append({
'id': f"{extension_id}|{version}",
'permission': sub_permission,
})
continue
permissions_set.add(permission)
extensions_permissions.append({
'id': f"{extension_id}|{version}",
'permission': permission,
})
extensions.append({
'id': f"{extension_id}|{version}",
'extension_id': extension_id,
Expand Down Expand Up @@ -158,34 +134,7 @@ def transform_extensions(extension_details):
'price': webstore.get('price'),
'report_link': f"https://crxcavator.io/report/{extension_id}/{version}",
})
return extensions, list(permissions_set), extensions_permissions


def parse_permissions_dict(permissions_dict):
"""
Parses the possible complex permissions objects into a list of objects
Parsing cases based on results from crxcavator API as of October 2019
:param permissions_dict: a dict object from the permissions key
:return: a list of individual permissions
"""
permissions = []
usb_devices = permissions_dict.get('usbDevices', [])
for device in usb_devices:
permissions.append(f"usbdevice-{device.get('productId', 'none')}-{device.get('vendorId', 'none')}")
filesystem = permissions_dict.get('fileSystem', [])
for filesystem_permission in filesystem:
permissions.append(f"filesystem-{filesystem_permission}")
socket = permissions_dict.get('socket', [])
for socket_permission in socket:
permissions.append(f"socket-{socket_permission}")
media_galleries = permissions_dict.get('mediaGalleries', [])
for media in media_galleries:
permissions.append(f"mediagalleries-{media}")
if len(permissions) == 0:
# this is a case not currently handled, so just log it and do not ingest it
permission = json.dumps(permissions_dict)
logger.warning(f"CRXcavator ingest - Unknown permissions dict type {permission}")
return permissions
return extensions


def get_risk_data(data_dict, key):
Expand All @@ -200,17 +149,15 @@ def get_risk_data(data_dict, key):
return data_score


def load_extensions(extensions, permissions, extension_permissions, session, update_tag):
def load_extensions(extensions, session, update_tag):
"""
Ingests the extension details into Neo4J
:param extensions: List of extension data to load to Neo4J
:param permissions: unique list of extension permissions
:param extension_permissions: Dictionary of extension-permission pairings
:param session: Neo4J session object for server communication
:param update_tag: Timestamp used to determine data freshness
:return: None
"""
extensions_ingestion_cypher = """
ingestion_cypher = """
UNWIND {ExtensionsData} as extension
MERGE (e:ChromeExtension{id: extension.id})
ON CREATE SET
Expand Down Expand Up @@ -248,34 +195,8 @@ def load_extensions(extensions, permissions, extension_permissions, session, upd
e.lastupdated = {UpdateTag}
"""

permissions_ingestion_cypher = """
UNWIND {Permissions} as permission
MERGE (e:ChromeExtensionPermission{id: permission})
ON CREATE SET
e.firstseen = timestamp()
SET
e.name = permission,
e.lastupdated = {UpdateTag}
"""

extensions_permissions_cypher = """
UNWIND {ExtensionPermissions} as extension_permission
MATCH (perm:ChromeExtensionPermission{id: extension_permission.permission}),
(ext:ChromeExtension{id:extension_permission.id})
MERGE (ext)-[r:DECLARES]->(perm)
ON CREATE SET
r.firstseen = timestamp()
SET r.lastupdated = {UpdateTag}
"""

logger.info(f'CRXcavator ingest - Ingesting {len(extensions)} extensions')
session.run(extensions_ingestion_cypher, ExtensionsData=extensions, UpdateTag=update_tag)

logger.info(f'CRXcavator ingest - Ingesting {len(permissions)} Chrome extension permissions')
session.run(permissions_ingestion_cypher, Permissions=permissions, UpdateTag=update_tag)

logger.info(f'CRXcavator ingest - Ingesting {len(extension_permissions)} extension to permission links')
session.run(extensions_permissions_cypher, ExtensionPermissions=extension_permissions, UpdateTag=update_tag)
logger.info(f'Ingesting {len(extensions)} extensions')
session.run(ingestion_cypher, ExtensionsData=extensions, UpdateTag=update_tag)


def transform_user_extensions(user_extension_json):
Expand Down Expand Up @@ -340,9 +261,9 @@ def load_user_extensions(users, extensions_by_user, session, update_tag):
SET r.lastupdated = {UpdateTag}
"""

logger.info(f'CRXcavator ingest - Ingesting {len(users)} users')
logger.info(f'Ingesting {len(users)} users')
session.run(user_ingestion_cypher, Users=users, UpdateTag=update_tag)
logger.info(f'CRXcavator ingest - Ingesting {len(extensions_by_user)} user->extension relationships')
logger.info(f'Ingesting {len(extensions_by_user)} user->extension relationships')
session.run(extension_ingestion_cypher, ExtensionsUsers=extensions_by_user, UpdateTag=update_tag)


Expand All @@ -359,9 +280,6 @@ def sync_extensions(neo4j_session, common_job_parameters, crxcavator_api_key, cr
user_extensions_json = get_users_extensions(crxcavator_api_key, crxcavator_base_url)
users, extensions_list, user_extensions = transform_user_extensions(user_extensions_json)
extension_details = get_extensions(crxcavator_api_key, crxcavator_base_url, extensions_list)
extensions, permissions_list, extension_permissions = transform_extensions(extension_details)
load_extensions(
extensions, permissions_list, extension_permissions,
neo4j_session, common_job_parameters['UPDATE_TAG'],
)
extensions = transform_extensions(extension_details)
load_extensions(extensions, neo4j_session, common_job_parameters['UPDATE_TAG'])
load_user_extensions(users, user_extensions, neo4j_session, common_job_parameters['UPDATE_TAG'])
38 changes: 10 additions & 28 deletions docs/schema/crxcavator.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,18 @@
<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)*

- [Table of contents](#table-of-contents)
- [GSuiteUser](#gsuiteuser)
- [Relationships](#relationships)
- [ChromeExtension](#chromeextension)
- [Relationships](#relationships-1)
- [ChromeExtensionPermission](#chromeextensionpermission)
- [Relationships](#relationships-2)

<!-- END doctoc generated TOC please keep comment here to allow auto update -->

## Table of contents

- [GSuiteUser](#gsuiteuser)
- [ChromeExtension](#chromeextension)

## GSuiteUser

Placeholder representation of a single G Suite [user object](https://developers.google.com/admin-sdk/directory/v1/reference/users). This node is the minimal data necessary to map who has extensions installed until full G Suite data is imported.
Expand All @@ -33,15 +36,15 @@ Placeholder representation of a single G Suite [user object](https://developers.
(GSuiteUser)-[INSTALLS]->(ChromeExtension)
```

## ChromeExtension
## ChromeExtension

Representation of a CRXcavator Chrome Extension [Report](https://crxcavator.io/apidocs#tag/report).

| Field | Description |
|-------|--------------|
| firstseen| Timestamp of when a sync job first discovered this node |
| lastupdated | Timestamp of the last time the node was updated |
| id | The combined extension name and version e.g. "Docs&#124;1.0" |
| id | The combined extension name and version e.g. "Docs|1.0" |
| extension_id | CRXcavator id for extension. |
| version | The versions of the extension in this report |
| risk_total | CRXcavator risk score for the extension |
Expand Down Expand Up @@ -71,31 +74,10 @@ Placeholder representation of a single G Suite [user object](https://developers.
| price | Extension price in webstore if applicable |
| report_link | URL of full extension report on crxcavator.io

### Relationships
### Relationships

- GSuiteUsers install ChromeExtensions.
- GSuiteUsers install ChromeExternsions.

```
(GSuiteUser)-[INSTALLS]->(ChromeExtension)
```


## ChromeExtensionPermission

Individual permission entries from the extension manifest file. This information is returned from the manifest object in the [Report](https://crxcavator.io/apidocs#tag/report) API. These permissions are defined in the [Chrome documentation](https://developer.chrome.com/apps/declare_permissions#manifest).


| Field | Description |
|-------|--------------|
| firstseen| Timestamp of when a sync job first discovered this node |
| lastupdated | Timestamp of the last time the node was updated |
| id | The permission name" |
| name | The specific permission declared in the manifest

### Relationships

- ChromeExtension declares ChromeExtensionPermission.

```
(ChromeExtension)-[DECLARES]->(ChromeExtensionPermission)
```
15 changes: 0 additions & 15 deletions tests/data/crxcavator/crxcavator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,6 @@
'type': 'Extension',
'price': '',
},
'manifest':
{
'permissions': [
'*://*/*',
],
},
},
'extension_id': 'f06981cbc72a3c6e2e9e736cbdaef4865a4571bc',
'version': '1.0',
Expand Down Expand Up @@ -87,15 +81,6 @@
},
]

TRANSFORMED_PERMISSIONS_DATA = ['*://*/*']

TRANSFORMED_EXTENSION_PERMISSIONS_DATA = [
{
'id': 'f06981cbc72a3c6e2e9e736cbdaef4865a4571bc|1.0',
'permission': '*://*/*',
},
]

USER_RESPONSE = {
'f06981cbc72a3c6e2e9e736cbdaef4865a4571bc':
{
Expand Down
41 changes: 3 additions & 38 deletions tests/integration/cartography/intel/crxcavator/test_crxcavator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@
def _ensure_local_neo4j_has_test_extensions_data(neo4j_session):
cartography.intel.crxcavator.crxcavator.load_extensions(
tests.data.crxcavator.crxcavator.TRANSFORMED_EXTENSIONS_DATA,
tests.data.crxcavator.crxcavator.TRANSFORMED_PERMISSIONS_DATA,
tests.data.crxcavator.crxcavator.TRANSFORMED_EXTENSION_PERMISSIONS_DATA,
neo4j_session,
TEST_UPDATE_TAG,
)
Expand All @@ -29,12 +27,9 @@ def test_transform_and_load_extensions(neo4j_session):
Test that we can correctly transform and load ChromeExtension nodes to Neo4j.
"""
extension_res = tests.data.crxcavator.crxcavator.REPORT_RESPONSE
extension_list, permissions, extension_permissions = \
cartography.intel.crxcavator.crxcavator.transform_extensions(extension_res)
extension_list = cartography.intel.crxcavator.crxcavator.transform_extensions(extension_res)
cartography.intel.crxcavator.crxcavator.load_extensions(
extension_list,
permissions,
extension_permissions,
neo4j_session,
TEST_UPDATE_TAG,
)
Expand Down Expand Up @@ -110,6 +105,7 @@ def test_transform_and_load_extensions(neo4j_session):
n['ext.report_link'],
) for n in nodes
])
print(actual_nodes)
expected_nodes = list([
(
expected_extension_id,
Expand Down Expand Up @@ -146,43 +142,12 @@ def test_transform_and_load_extensions(neo4j_session):
assert actual_nodes == expected_nodes


def test_permission_to_extension(neo4j_session):
"""
Ensure that permissions are connected to extensions.
"""
_ensure_local_neo4j_has_test_extensions_data(neo4j_session)
query = """
MATCH(permission:ChromeExtensionPermission)<-[:DECLARES]-(ext:ChromeExtension{id:{ExtensionId}})
RETURN permission.name, ext.id, ext.name
"""
expected_extension_id = 'f06981cbc72a3c6e2e9e736cbdaef4865a4571bc|1.0'
nodes = neo4j_session.run(
query,
ExtensionId=expected_extension_id,
)
actual_nodes = {
(
n['permission.name'],
n['ext.id'],
n['ext.name'],
) for n in nodes
}

expected_nodes = {
(
'*://*/*',
'f06981cbc72a3c6e2e9e736cbdaef4865a4571bc|1.0',
'CartographyIntegrationTest',
),
}
assert actual_nodes == expected_nodes


def test_transform_and_load_user_extensions(neo4j_session):
"""
Ensure we can transform and load users and extension mapping.
"""
users_res = tests.data.crxcavator.crxcavator.USER_RESPONSE
type(users_res)
users_list, extensions_list, user_extensions_list = \
cartography.intel.crxcavator.crxcavator.transform_user_extensions(users_res)
cartography.intel.crxcavator.crxcavator.load_user_extensions(
Expand Down

0 comments on commit d0317e0

Please sign in to comment.