Skip to content

Commit

Permalink
Remove FleetDM Software connector, fix ordering. Formatting. (#53)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hcpadkins
hcpadkins authored Sep 25, 2024
1 parent 146358e commit 138f3e2
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 244 deletions.
94 changes: 21 additions & 73 deletions grove/connectors/fleetdm/api.py
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0

"""FleetDM Vulnerability API client.
"""
"""FleetDM Vulnerability API client."""

import logging
import time
from datetime import datetime
from typing import Any, Dict, Optional

import jmespath
Expand All @@ -30,7 +28,8 @@ def __init__(
:param token: FleetDM API Bearer token.
:param retry: Automatically retry if recoverable errors are encountered, such as
rate-limiting.
:param params: FleetDM parameters for the REST API query - configurable in the config file
:param params: FleetDM parameters for the REST API query - configurable in the
config file
:param jmespath_queries: A jmespath query string used to filter the responses
:param api_uri: The base URI to communicate with Fleet
"""
Expand All @@ -44,7 +43,6 @@ def __init__(
self.jmespath_queries = jmespath_queries
self.api_uri = api_uri


def _get(
self,
url: str,
Expand Down Expand Up @@ -86,14 +84,19 @@ def get_hosts(
cursor: Optional[str],
) -> AuditLogEntries:
"""Fetches a list of hosts which match the provided filters.
:param params: get parameters json dict from https://fleetdm.com/docs/rest-api/rest-api#list-hosts
:param jmespath_queries: jmespath query to filter JSON response as specified at https://jmespath.org/
:param cursor: The cursor passed from host_logs.py that includes the last update date of systems in Grove
:param params: get parameters json dict from
https://fleetdm.com/docs/rest-api/rest-api#list-hosts
:param jmespath_queries: jmespath query to filter JSON response as specified at
https://jmespath.org/
:param cursor: The cursor passed from host_logs.py that includes the last update
date of systems in Grove
:return: AuditLogEntries object containing a pagination cursor, and log entries.
"""

# Ensuring that the parameters include the correct After, Order Key, and Order Direction
# for the pagination and cursor to work correctly
# Ensuring that the parameters include the correct After, Order Key, and Order
# Direction for the pagination and cursor to work correctly
params["after"] = cursor
params["order_key"] = "software_updated_at"
params["order_direction"] = "asc"
Expand All @@ -106,76 +109,21 @@ def get_hosts(
)

filteredResults = []

# FleetDM returns an empty hosts array if there's no more pages of results,
# so swap this for a None object
# Otherwise, grab the last seen update date/time to use in the next page of dates
if len(result.body.get("hosts", [])) == 0:
cursor = None
elif cursor is not None:
# The default response for a Fleet Hosts call including software returns a large
# volume - much more than required. We define a jmespath query string to filter
# the response from the API down to just the fields we need. The default if none
# is set is "*" which returns the whole API response
# The default response for a Fleet Hosts call including software returns a
# large volume - much more than required. We define a jmespath query string
# to filter the response from the API down to just the fields we need. The
# default if none is set is "*" which returns the whole API response
for host in result.body.get("hosts"):
filteredResults.append(jmespath.search(jmespath_queries,host))
filteredResults.append(jmespath.search(jmespath_queries, host))
cursor = host.get("software_updated_at")

# Return the cursor of the last processed date and the results to allow the caller to page as required.
return AuditLogEntries(cursor=cursor, entries=filteredResults)



def get_software(
self,
params: Dict[str, Any], # get parameters json dict from https://fleetdm.com/docs/rest-api/rest-api#list-hosts
jmespath_queries: str,
api_uri: str,
cursor: Optional[int],
pointer: datetime
) -> AuditLogEntries:
jmespath_queries = jmespath_queries

"""Fetches a list of audit logs which match the provided filters.
:return: AuditLogEntries object containing a pagination cursor, and log entries.
"""

# Set the page value to the current cursor
params["page"] = str(cursor)

# See psf/requests issue #2651 for why we can happily pass in None values and
# not have the request key added to the URI.
result = self._get(
f"{api_uri}/api/v1/fleet/software/versions",
params,
)

filteredResults = []
# FleetDM returns an empty software array if there's no more pages of results,
# so swap this for a None object
# Otherwise, increment the page and continue
if len(result.body.get("software",[])) == 0:
cursor = None
elif cursor is not None:
cursor = int(cursor) + 1
# The default response for a Fleet Software call including software returns a large
# volume - much more than required. We define a jmespath query string to filter
# the response from the API down to just the fields we need. The default if none
# is set is "*" which returns the whole API response
updated_at = result.body.get("counts_updated_at")
# There's no way in the REST api to ask for only software that has been updated since
# the last time the software table has last been refreshed. So we manually check
# the counts_updated_at value against the pointer stored as the last processed time
# to see if we should log the software again
if(datetime.fromisoformat(str(updated_at)) > datetime.fromisoformat(str(pointer))):
for software in result.body.get("software"):
s = jmespath.search(jmespath_queries,software)
# The software object itself doesn't include an updated at datetime,
# so we inherit the counts_updated_at datetime from the base response
# and include it in the software object to allow grove to correctly
# know the last time the software counts were updated
s["updated_at"]=updated_at
filteredResults.append(s)


# Return the cursor and the results to allow the caller to page as required.
# Return the cursor of the last processed date and the results to allow the
# caller to page as required.
return AuditLogEntries(cursor=cursor, entries=filteredResults)
72 changes: 31 additions & 41 deletions grove/connectors/fleetdm/host_logs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,52 +7,35 @@

from grove.connectors import BaseConnector
from grove.connectors.fleetdm.api import Client
from grove.constants import CHRONOLOGICAL
from grove.exceptions import NotFoundException


class Connector(BaseConnector):
NAME = "fleetdm_host_logs"
POINTER_PATH = "software_updated_at"

LOG_ORDER = CHRONOLOGICAL

@property
def jmespath_queries(self):
"""Fetches the parameters for the jmespath filters of the data response.
Jmespath query language is defined and can be tested at https://jmespath.org/
This allows you to configure what data to include or filter from the FleetDM response.
This allows you to configure what data to include or filter from the FleetDM
response.
An example is:
"jmespath_queries": "{hostname:hostname,updated_at:updated_at,software_updated_at:software_updated_at,
uuid:uuid,hardware_serial:hardware_serial,computer_name:computer_name,
os_version:osversion,software:software[?vulnerabilities].{name:name,
version:version,source:source,vulnerabilities:vulnerabilities[?cvss_score].{cve:cve,cvss_score:cvss_score,
epss_probability:epss_probability,cisa_known_exploit:cisa_known_exploit,resolved_in_version:resolved_in_version}}}"
"jmespath_queries": "{hostname:hostname,updated_at:updated_at}"
This returns the following structure of data about each host:
{
updated_at:
software_updated_at:
hostname:
uuid:
hardware_serial:
computer_name:
os_version:
software:{ (filtered to only software with vulnerabilities which have a CVSS score)
name:
version:
source:
vulnerabilities:{
cve:
cvss_score:
epss_probability:
cisa_known_exploit:
resolved_in_version:
{
updated_at:
hostname:
}
}
:return: A string of the Jmespath response that should define the JSON object to return.
Default is *, the full set of JSON response
:return: A string of the Jmespath response that should define the JSON object
to return. Default is *, the full set of JSON response
"""
try:
p = self.configuration.jmespath_queries
Expand All @@ -75,11 +58,14 @@ def params(self):
return None

return p

@property
def api_uri(self):
"""The URI for the API call
For example: https://panel.fleetdm.example.com
:return: The configured FleetDM API URI.
"""
try:
p = self.configuration.api_uri
Expand All @@ -89,30 +75,34 @@ def api_uri(self):
return p

def collect(self):
"""Collects all hosts from the FleetDM API.
"""
"""Collects all hosts from the FleetDM API."""
client = Client(
token=self.key,
params=self.params,
api_uri=self.api_uri,
jmespath_queries=self.jmespath_queries,
api_uri=self.api_uri
)
)

# We load hosts as they've had their software inventory updated. Grove requires
# a default pointer to be set, so set the pointer to a week ago.
# We start at the datetime set by the pointer and loop forward in time until the present
# a default pointer to be set, so set the pointer to a week ago. We start at the
# datetime set by the pointer and loop forward in time until the present.
try:
cursor = str(datetime.fromisoformat(self.pointer))
_ = str(datetime.fromisoformat(self.pointer))
except NotFoundException:
cursor = str(datetime.now(timezone.utc) - timedelta(days=7))
self.pointer = cursor
self.pointer = str(datetime.now(timezone.utc) - timedelta(days=7))

# Page over data using the cursor, saving returned data page by page.
while True:
log = client.get_hosts(cursor=cursor,params=self.params,jmespath_queries=self.jmespath_queries,api_uri=self.api_uri)
log = client.get_hosts(
cursor=self.pointer,
params=self.params,
jmespath_queries=self.jmespath_queries,
api_uri=self.api_uri,
)

# Save this batch of log entries.
self.save(log.entries)

# Check if we need to continue paging.
if log.cursor is None:
break
else:
cursor = str(log.cursor)
129 changes: 0 additions & 129 deletions grove/connectors/fleetdm/software_logs.py
View file

This file was deleted.

Loading

0 comments on commit 138f3e2

Please sign in to comment.