From 9366f7435a079e0a77259743a4710bd1eb74adce Mon Sep 17 00:00:00 2001 From: James Kitson Date: Mon, 8 May 2023 16:13:07 -0400 Subject: [PATCH] add v2 log exporting --- dynatrace/environment_v2/logs.py | 54 ++++++++++++++++++- setup.py | 2 +- ...ET_api_v2_logs_export_8b3579556e13525.json | 1 + test/test_logs.py | 22 ++++++++ 4 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 test/mock_data/GET_api_v2_logs_export_8b3579556e13525.json create mode 100644 test/test_logs.py diff --git a/dynatrace/environment_v2/logs.py b/dynatrace/environment_v2/logs.py index 0023550..44c771a 100644 --- a/dynatrace/environment_v2/logs.py +++ b/dynatrace/environment_v2/logs.py @@ -13,20 +13,52 @@ See the License for the specific language governing permissions and limitations under the License. """ +from enum import Enum from typing import Dict, Any, Union, List from requests import Response +from datetime import datetime +from typing import Optional, Union, Dict, Any, List from dynatrace.http_client import HttpClient +from dynatrace.dynatrace_object import DynatraceObject +from dynatrace.pagination import PaginatedList +from dynatrace.utils import timestamp_to_string class LogService: - # TODO - Add search and get + # TODO - Add search and aggregate ENDPOINT = "/api/v2/logs" def __init__(self, http_client: HttpClient): self.__http_client = http_client + def export( + self, + query: Optional[str] = None, + time_from: Optional[Union[datetime, str]] = None, + time_to: Optional[Union[datetime, str]] = None, + sort: Optional[str] = None, + page_size: Optional[int] = None, + ) -> PaginatedList["LogRecord"]: + """ + Gets the log records matching the provided criteria. Retrieves all records using pagination. + :param query: The log search query + :param page_size: Number of results per page + :param time_from: Start of the requested timeframe + :param time_to: End of the requested timefram + :param sort: Defines the ordering of log records + :return A list of log records + """ + params = { + "query": query, + "pageSize": page_size, + "from": timestamp_to_string(time_from), + "to": timestamp_to_string(time_to), + "sort": sort, + } + return PaginatedList(LogRecord, self.__http_client, "/api/v2/logs/export", params, list_item="results") + def ingest(self, payload: Union[Dict[str, Any], List[Dict[str, Any]]]) -> Response: """ Ingests logs into the Dynatrace log store. @@ -35,3 +67,23 @@ def ingest(self, payload: Union[Dict[str, Any], List[Dict[str, Any]]]) -> Respon """ headers = {"Content-Type": "application/json; charset=utf-8"} return self.__http_client.make_request(f"{self.ENDPOINT}/ingest", params=payload, method="POST", headers=headers) + +class LogRecord(DynatraceObject): + def _create_from_raw_data(self, raw_element: Dict[str, Any]): + self.additional_columns: dict = raw_element.get("additionalColumns") + self.event_type: EventType = EventType(raw_element.get("eventType")) + self.timestamp: datetime = datetime.utcfromtimestamp(raw_element.get("timestamp") / 1000) + self.content: str = raw_element.get("content") + self.status: LogRecordStatus = LogRecordStatus(raw_element.get("status")) + +class EventType(Enum): + K8S = "K8S" + LOG = "LOG" + SFM = "SFM" + +class LogRecordStatus(Enum): + ERROR = "ERROR" + INFO = "INFO" + NONE = "NONE" + NOT_APPLICABLE = "NOT_APPLICABLE" + WARN = "WARN" \ No newline at end of file diff --git a/setup.py b/setup.py index 0bc0352..d81fe05 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="dt", - version="1.1.59", + version="1.1.60", packages=find_packages(), install_requires=["requests>=2.22"], tests_require=["pytest", "mock", "tox"], diff --git a/test/mock_data/GET_api_v2_logs_export_8b3579556e13525.json b/test/mock_data/GET_api_v2_logs_export_8b3579556e13525.json new file mode 100644 index 0000000..b3c3fa0 --- /dev/null +++ b/test/mock_data/GET_api_v2_logs_export_8b3579556e13525.json @@ -0,0 +1 @@ +{"results": [{"timestamp": 1683574915193, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574855122, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574795002, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574764953, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Cannot extract extension from C:\\ProgramData\\dynatrace\\remotepluginmodule\\/agent/runtime\\extensions\\download\\custom_sybase-ase-health-ci: checking signature failed", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:sybase-ase-health-ci"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["64132242-27f9-39b7-a456-2b7c3bc359db"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574734903, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574704874, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Extension custom:sybase-ase-health-ci(0.0.6) not available in cache yet (queued for download)", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:sybase-ase-health-ci"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["64132242-27f9-39b7-a456-2b7c3bc359db"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574674822, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574614732, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574554719, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574494522, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574459170, "content": "OK", "status": "INFO", "eventType": "SFM", "additionalColumns": {"loglevel": ["INFO"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.active_gate.id": ["0x16f6e00b"], "dt.extension.name": ["custom:vcenter-extension"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "host.name": ["TAG009444368559"], "dt.extension.config.id": ["834f045c-02d3-31b7-91f8-8bfb3f6c050e"], "dt.extension.status": ["OK"]}}, {"timestamp": 1683574434460, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574409411, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Fast check failed for 1 endpoint(s). Cannot connect to database localhost:0. Exception occured: Socket fail to connect to host:address=(host=localhost)(port=3306)(type=primary). Connection refused: connect. ", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["sqlMySql"], "dt.extension.name": ["com.dynatrace.extension.mysql"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["7fb25156-5681-3f94-8f01-41a46f3d8935"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574405008, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Cannot extract extension from C:\\ProgramData\\dynatrace\\remotepluginmodule\\/agent/runtime\\extensions\\download\\custom_sybase-ase-health-ci: checking signature failed", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:sybase-ase-health-ci"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["64132242-27f9-39b7-a456-2b7c3bc359db"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574399171, "content": "Method=query_vsphere: Callback Method=query_vsphere took 69.6077 seconds to execute, which is longer than the interval of 60.0s", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.active_gate.id": ["0x16f6e00b"], "dt.extension.name": ["custom:vcenter-extension"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "host.name": ["TAG009444368559"], "dt.extension.config.id": ["834f045c-02d3-31b7-91f8-8bfb3f6c050e"], "dt.extension.status": ["GENERIC_ERROR"]}}, {"timestamp": 1683574397440, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: The monitoring configuration requires ActiveGate version 1.256.0 or later that supports data source python (runtime version min: 3.9.0) and belongs to group \"windows\"", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["windows"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:vlocity-query"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["7f3d5e87-0a6c-33db-8e4a-6b137f7d1114"], "dt.extension.status": ["NO_SUITABLE_ACTIVE_MONITOR"]}}, {"timestamp": 1683574397440, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: The monitoring configuration requires ActiveGate version 1.259.0 or later that supports data source python (runtime version min: 3.9.0) and belongs to group \"windows\"", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["windows"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:lloyds-status-page-extension"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["b1f1e72e-1b0e-3d98-aac6-5bae991eb425"], "dt.extension.status": ["NO_SUITABLE_ACTIVE_MONITOR"]}}, {"timestamp": 1683574397439, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: The monitoring configuration requires ActiveGate version 1.256.0 or later that supports data source python (runtime version min: 3.9.0) and belongs to group \"windows\"", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["windows"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:remote-unix"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["f1d59951-5fa2-35bf-9ead-1117e4e31981"], "dt.extension.status": ["NO_SUITABLE_ACTIVE_MONITOR"]}}], "pageSize": 18, "totalCount": 18} \ No newline at end of file diff --git a/test/test_logs.py b/test/test_logs.py new file mode 100644 index 0000000..bce82fc --- /dev/null +++ b/test/test_logs.py @@ -0,0 +1,22 @@ +from dynatrace import Dynatrace +from datetime import datetime + +from dynatrace.environment_v2.logs import LogRecord, EventType, LogRecordStatus +from dynatrace.pagination import PaginatedList + + +def test_export(dt: Dynatrace): + logs = dt.logs.export(time_from="now-10m") + assert isinstance(logs, PaginatedList) + + logs = list(logs) + assert len(logs) == 18 + + first = logs[0] + assert isinstance(first, LogRecord) + assert first.additional_columns['dt.extension.ds'][0] == "python" + assert first.content.startswith("Failed to assign") + assert first.event_type == EventType.SFM + assert first.status == LogRecordStatus.ERROR + assert first.timestamp == datetime.utcfromtimestamp(1683574915193 / 1000) +