set up guardduty intel module and integration tests

cartography/data/jobs/cleanup/aws_ingest_guardduty_findings_cleanup.json

@@ -0,0 +1,9 @@
+{
+    "statements": [{
+        "query": "MATCH (:AWSAccount{id: $AWS_ID})-[:RESOURCE]->(finding:GuardDutyFinding) WHERE finding.lastupdated <> $UPDATE_TAG WITH finding LIMIT $LIMIT_SIZE DETACH DELETE (finding)",
+        "iterative": true,
+        "iterationsize": 100
+      }],
+    "name": "cleanup GuardDuty Findings"
+  }
+

cartography/intel/aws/guardduty.py

@@ -0,0 +1,104 @@
+import logging
+import time
+from datetime import datetime
+from typing import Any
+from typing import Dict
+from typing import List
+
+import boto3
+import neo4j
+
+from cartography.util import aws_handle_regions
+from cartography.util import run_cleanup_job
+from cartography.util import timeit
+
+logger = logging.getLogger(__name__)
+
+
+@timeit
+@aws_handle_regions
+def get_guardduty_findings(boto3_session: boto3.session.Session) -> List[Dict]:
+    client = boto3_session.client('guardduty')
+    # paginator = client.get_paginator('get_findings')
+    detectorResponse = client.list_detectors()
+    detectorIds = detectorResponse['DetectorIds']
+
+    findingsList = []
+    for detectorId in detectorIds:
+        list_finding_params = {
+            'DetectorId': detectorId
+        }
+        findingIdsResponse = client.list_findings(**list_finding_params)
+        findingIds = findingIdsResponse["FindingIds"]
+
+        get_findings_params = {
+            'DetectorId': detectorId,
+            'FindingIds':findingIds
+        }
+        findings: List[Dict] 
+        findingsResponse = client.get_findings(**get_findings_params)
+        findings=findingsResponse['Findings']
+
+        findingsList.append(findings)
+
+    return findingsList
+
+
+@timeit
+def load_guardduty_findings(
+    neo4j_session: neo4j.Session, findingsList: List[List[Dict]], current_aws_account_id: str, aws_update_tag: int,
+) -> None:
+
+    ingest_findings = """
+    UNWIND $Findings as finding
+    WITH finding
+    WHERE EXISTS(finding.Resource.InstanceDetails)
+    MERGE (f:GuardDutyFinding {arn: finding.Arn})
+    ON CREATE SET f.firstseen = timestamp()
+    SET f.lastupdated = $update_tag,
+    f.id = finding.Id,
+    f.severity = finding.Severity,
+    f.title = finding.Title,
+    f.type = finding.Type,
+    f.created_at = finding.CreatedAt,
+    f.description = finding.Description
+    WITH f,finding
+    MATCH (a:AWSAccount {id: $AccountId})
+    MERGE (a)-[r_af:RESOURCE]->(f)
+    ON CREATE SET r_af.firstseen = timestamp()
+    SET r_af.lastupdated = $update_tag
+    WITH f,finding
+    MERGE (ec2:EC2Instance {instanceid:finding.Resource.InstanceDetails.InstanceId})
+    ON CREATE SET ec2.firstseen = timestamp()
+    SET ec2.lastupdated = $update_tag
+    WITH f,ec2
+    MERGE (f)-[r_fr:ASSOCIATED_RESOURCE]->(ec2)
+    ON CREATE SET r_fr.firstseen = timestamp()
+    SET r_fr.lastupdated = $update_tag
+
+    """
+
+    for findings in findingsList:
+
+        neo4j_session.run(
+            ingest_findings,
+            Findings = findings,
+            AccountId=current_aws_account_id,
+            update_tag=aws_update_tag
+        )
+
+@timeit
+def cleanup_guardduty_findings(neo4j_session: neo4j.Session, common_job_parameters: Dict) -> None:
+    run_cleanup_job('aws_ingest_guardduty_findings_cleanup.json', neo4j_session, common_job_parameters)
+
+
+
+@timeit
+def sync(
+        neo4j_session: neo4j.Session, boto3_session: boto3.session.Session, regions: List[str],
+        current_aws_account_id: str, update_tag: int, common_job_parameters: Dict,
+) -> None:
+    logger.info("Syncing guard duty findings for account '%s'.",current_aws_account_id)
+    data = get_guardduty_findings(boto3_session)
+    load_guardduty_findings(neo4j_session, data, current_aws_account_id, update_tag)
+    cleanup_guardduty_findings(neo4j_session, common_job_parameters)

cartography/intel/aws/resources.py

@@ -24,6 +24,7 @@
 from . import securityhub
 from . import sqs
 from . import ssm
+from . import guardduty
 from .ec2.auto_scaling_groups import sync_ec2_auto_scaling_groups
 from .ec2.elastic_ip_addresses import sync_elastic_ip_addresses
 from .ec2.images import sync_ec2_images
@@ -91,4 +92,5 @@
     'config': config.sync,
     'cloudtrail': cloudtrail.sync,
     'cloudwatch': cloudwatch.sync,
+    'guardduty': guardduty.sync,
 }

tests/data/aws/guardduty.py

@@ -0,0 +1,115 @@
+FINDINGS = {
+    "Findings": [
+        {
+            "Resource": {
+                "AccessKeyDetails": {
+                    "UserName": "testuser",
+                    "UserType": "IAMUser",
+                    "PrincipalId": "AIDACKCEVSQ6C2EXAMPLE",
+                    "AccessKeyId": "ASIASZ4SI7REEEXAMPLE"
+                },
+                'InstanceDetails': {
+                    'AvailabilityZone': 'string',
+                    'IamInstanceProfile': {
+                        'Arn': 'string',
+                        'Id': 'string'
+                    },
+                    'ImageDescription': 'string',
+                    'ImageId': 'string',
+                    'InstanceId': 'string',
+                    'InstanceState': 'string',
+                    'InstanceType': 'string',
+                    'OutpostArn': 'string',
+                    'LaunchTime': 'string',
+                    'NetworkInterfaces': [
+                        {
+                            'Ipv6Addresses': [
+                                'string',
+                            ],
+                            'NetworkInterfaceId': 'string',
+                            'PrivateDnsName': 'string',
+                            'PrivateIpAddress': 'string',
+                            'PrivateIpAddresses': [
+                                {
+                                    'PrivateDnsName': 'string',
+                                    'PrivateIpAddress': 'string'
+                                },
+                            ],
+                            'PublicDnsName': 'string',
+                            'PublicIp': 'string',
+                            'SecurityGroups': [
+                                {
+                                    'GroupId': 'string',
+                                    'GroupName': 'string'
+                                },
+                            ],
+                            'SubnetId': 'string',
+                            'VpcId': 'string'
+                        },
+                    ],
+                    'Platform': 'string',
+                    'ProductCodes': [
+                        {
+                            'Code': 'string',
+                            'ProductType': 'string'
+                        },
+                    ],
+                    'Tags': [
+                        {
+                            'Key': 'string',
+                            'Value': 'string'
+                        },
+                    ]
+                },
+            },
+            "Description": "APIs commonly used to discover the users, groups, policies and permissions in an account, was invoked by IAM principal testuser under unusual circumstances. Such activity is not typically seen from this principal.",
+            "Service": {
+                "Count": 5,
+                "Archived": False,
+                "ServiceName": "guardduty",
+                "EventFirstSeen": "2020-05-26T22:02:24Z",
+                "ResourceRole": "TARGET",
+                "EventLastSeen": "2020-05-26T22:33:55Z",
+                "DetectorId": "d4b040365221be2b54a6264dcexample",
+                "Action": {
+                    "ActionType": "AWS_API_CALL",
+                    "AwsApiCallAction": {
+                        "RemoteIpDetails": {
+                            "GeoLocation": {
+                                "Lat": 51.5164,
+                                "Lon": -0.093
+                            },
+                            "City": {
+                                "CityName": "London"
+                            },
+                            "IpAddressV4": "52.94.36.7",
+                            "Organization": {
+                                "Org": "Amazon.com",
+                                "Isp": "Amazon.com",
+                                "Asn": "16509",
+                                "AsnOrg": "AMAZON-02"
+                            },
+                            "Country": {
+                                "CountryName": "United Kingdom"
+                            }
+                        },
+                        "Api": "ListPolicyVersions",
+                        "ServiceName": "iam.amazonaws.com",
+                        "CallerType": "Remote IP"
+                    }
+                }
+            },
+            "Title": "Unusual user permission reconnaissance activity by testuser.",
+            "Type": "Recon:IAMUser/UserPermissions",
+            "Region": "us-east-1",
+            "Partition": "aws",
+            "Arn": "arn:aws:guardduty:us-east-1:111122223333:detector/d4b040365221be2b54a6264dcexample/finding/1ab92989eaf0e742df4a014d5example",
+            "UpdatedAt": "2020-05-26T22:55:21.703Z",
+            "SchemaVersion": "2.0",
+            "Severity": 5,
+            "Id": "f-01",
+            "CreatedAt": "2020-05-26T22:21:48.385Z",
+            "AccountId": "111122223333"
+        }
+    ]
+}

tests/integration/cartography/intel/aws/test_guardduty.py

@@ -0,0 +1,30 @@
+import tests.data.aws.guardduty
+import cartography.intel.aws.guardduty
+
+TEST_ACCOUNT_ID = '000000000000'
+TEST_UPDATE_TAG = 123456789
+
+def test_load_guardduty_findings(neo4j_session, *args):
+    dataList = []
+    data = tests.data.aws.guardduty.FINDINGS['Findings']
+    dataList.append(data)
+    cartography.intel.aws.guardduty.load_guardduty_findings(
+        neo4j_session,dataList,TEST_ACCOUNT_ID,TEST_UPDATE_TAG
+    )
+
+    expected_nodes = {
+        "f-01"
+    }
+
+    nodes = neo4j_session.run(
+        """
+        MATCH (f:GuardDutyFinding) return f.id
+        """
+    )
+
+    actual_nodes = {
+        n['f.id']
+        for n in nodes
+    }
+
+    assert actual_nodes == expected_nodes