Stripe Events Connector (#43)

* stripe connector work
* clean fixture json files
* update settings and connector and test suite
* update connector and test suite to work correctly
* cleanup settings.json

.vscode/settings.json

@@ -4,7 +4,7 @@
     "[python]": {
         "editor.formatOnSave": true,
         "editor.codeActionsOnSave": {
-            "source.organizeImports": true,
+            "source.organizeImports": "explicit"
         },
         "editor.defaultFormatter": "ms-python.black-formatter",
     },

grove/connectors/stripe/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+"""Stripe connectors for Grove."""

grove/connectors/stripe/events.py

@@ -0,0 +1,58 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+"""Stripe Events connector for Grove."""
+
+from stripe import StripeClient
+
+from grove.connectors import BaseConnector
+from grove.constants import REVERSE_CHRONOLOGICAL
+from grove.exceptions import NotFoundException
+
+
+class Connector(BaseConnector):
+    NAME = "stripe_events"
+    POINTER_PATH = "id"
+    LOG_ORDER = REVERSE_CHRONOLOGICAL
+
+    def collect(self):
+        """Collects all events from the Stripe Events API.
+
+        Stripe's list API methods use cursor-based pagination with a unique ID value
+        for each event. The Stripe API allows up to 100 read operations per second,
+        which is set as the limit in the params object below.
+        """
+        client = StripeClient(self.key)
+        params = {
+            "type": self.operation,
+            "limit": 100,
+        }
+        entries = None
+
+        try:
+            _ = self.pointer
+        except NotFoundException:
+            # Stripe does not use a timestamp for filtering, so this will collect as
+            # many results as Stripe is willing to give us during the first collection.
+            self.pointer = ""
+
+        # Page over data using the cursor, saving returned data page by page.
+        while True:
+            # If this has not run before, just collect all information Stripe will give
+            # us which is 30-days by default.
+            if self.pointer:
+                params["starting_after"] = self.pointer
+
+            # Pagination is handled a little differently for Stripe due to their SDK,
+            # where we call a next_page rather than tracking a cursor.
+            if not entries:
+                entries = client.events.list(params=params)  # type:ignore
+            else:
+                entries = entries.next_page()  # type:ignore
+
+            # Save this batch of log entries.
+            self.save(entries.data)
+
+            # If Stripe doesn't tell us we have more data, we're complete.
+            if not entries.has_more:
+                break

pyproject.toml

@@ -24,6 +24,7 @@ dependencies = [
     "twilio>=7.15,<8.0",
     "pydantic>=1.10,<2.0",
     "jmespath>=1.0.0,<2.0",
+    "stripe>=8.4.0,<9.0",
 ]
 
 [project.optional-dependencies]
@@ -74,6 +75,7 @@ sf_event_log = "grove.connectors.sf.event_log:Connector"
 sfmc_audit_events = "grove.connectors.sfmc.audit_events:Connector"
 sfmc_security_events = "grove.connectors.sfmc.security_events:Connector"
 slack_audit_logs = "grove.connectors.slack.audit_logs:Connector"
+stripe_events = "grove.connectors.stripe.events:Connector"
 tines_audit_logs = "grove.connectors.tines.audit_logs:Connector"
 tfc_audit_trails = "grove.connectors.tfc.audit_trails:Connector"
 torq_activity_logs = "grove.connectors.torq.activity_logs:Connector"

templates/configuration/stripe/events.json

@@ -0,0 +1,7 @@
+{
+    "key": "BEARER_TOKEN_HERE",
+    "identity": "ORGANIZATION_ID_HERE",
+    "name": "stripe-events-example",
+    "connector": "stripe_events",
+    "operation": "*"
+}

tests/fixtures/stripe/events/001.json

@@ -0,0 +1,71 @@
+{
+    "object": "list",
+    "data": [
+      {
+        "id": "evt_1OonDcGsC9LSOJDcarfWbL08",
+        "object": "event",
+        "api_version": "2020-08-27",
+        "created": 1709127043,
+        "data": {
+          "object": {
+            "id": "cus_XXXXXXXXXXXXXX",
+            "object": "customer",
+            "address": {
+              "city": "Any Town",
+              "country": "US",
+              "line1": "123 Fake Street",
+              "line2": "",
+              "postal_code": "12345",
+              "state": "TX"
+            },
+            "balance": 0,
+            "created": 1709127043,
+            "currency": null,
+            "default_currency": null,
+            "default_source": null,
+            "delinquent": false,
+            "description": "Billing Account Description",
+            "discount": null,
+            "email": "[email protected]",
+            "invoice_prefix": "XXXXXXX",
+            "invoice_settings": {
+              "custom_fields": null,
+              "default_payment_method": "pm_XXXXXXXXXXX",
+              "footer": null,
+              "rendering_options": null
+            },
+            "livemode": true,
+            "metadata": {
+              "Address_City": "Any Town",
+              "Address_Country": "US",
+              "Address_Line1": "123 Fake Street",
+              "Address_PostalCode": "12345",
+              "Address_State": "TX",
+              "billing_account_id": "123779ca-d622-2276-2207-40fc12773c61",
+              "organization_id": "123601b9-1a8c-22bb-2210-22bc55fb6ef9"
+            },
+            "name": "Purchasing 1",
+            "next_invoice_sequence": 1,
+            "phone": null,
+            "preferred_locales": [],
+            "shipping": {
+              "address": {
+                "city": "Any Town",
+                "country": "US",
+                "line1": "123 Fake Street",
+                "line2": "",
+                "postal_code": "12345",
+                "state": "TX"
+              },
+              "name": "Purchasing 1",
+              "phone": null
+            },
+            "tax_exempt": "exempt",
+            "test_clock": null
+          }
+        }
+      }
+    ],
+    "has_more": true,
+    "url": "/v1/events"
+  }

tests/fixtures/stripe/events/002.json

@@ -0,0 +1,71 @@
+{
+    "object": "list",
+    "data": [
+      {
+        "id": "evt_1OonDcGsC9LSOJDcarfWbL10",
+        "object": "event",
+        "api_version": "2020-08-27",
+        "created": 1709127043,
+        "data": {
+          "object": {
+            "id": "cus_XXXXXXXXXXXXXX",
+            "object": "customer",
+            "address": {
+              "city": "Any Town",
+              "country": "US",
+              "line1": "123 Fake Street",
+              "line2": "",
+              "postal_code": "12345",
+              "state": "TX"
+            },
+            "balance": 0,
+            "created": 1709127043,
+            "currency": null,
+            "default_currency": null,
+            "default_source": null,
+            "delinquent": false,
+            "description": "Billing Account Description",
+            "discount": null,
+            "email": "[email protected]",
+            "invoice_prefix": "XXXXXXX",
+            "invoice_settings": {
+              "custom_fields": null,
+              "default_payment_method": "pm_XXXXXXXXXXX",
+              "footer": null,
+              "rendering_options": null
+            },
+            "livemode": true,
+            "metadata": {
+              "Address_City": "Any Town",
+              "Address_Country": "US",
+              "Address_Line1": "123 Fake Street",
+              "Address_PostalCode": "12345",
+              "Address_State": "TX",
+              "billing_account_id": "123779ca-d622-2276-2207-40fc12773c61",
+              "organization_id": "123601b9-1a8c-22bb-2210-22bc55fb6ef9"
+            },
+            "name": "Purchasing 1",
+            "next_invoice_sequence": 1,
+            "phone": null,
+            "preferred_locales": [],
+            "shipping": {
+              "address": {
+                "city": "Any Town",
+                "country": "US",
+                "line1": "123 Fake Street",
+                "line2": "",
+                "postal_code": "12345",
+                "state": "TX"
+              },
+              "name": "Purchasing 1",
+              "phone": null
+            },
+            "tax_exempt": "exempt",
+            "test_clock": null
+          }
+        }
+      }
+    ],
+    "has_more": false,
+    "url": "/v1/events"
+  }

tests/test_connectors_stripe_events.py

@@ -0,0 +1,91 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+"""Implements integration tests for the Stripe Events collector."""
+
+import os
+import re
+import unittest
+from unittest.mock import patch
+
+import responses
+
+from grove.connectors.stripe.events import Connector
+from grove.models import ConnectorConfig
+from tests import mocks
+
+
+class StripeEventsTestCase(unittest.TestCase):
+    """Implements integration tests for the Stripe Events collector."""
+
+    @patch("grove.helpers.plugin.load_handler", mocks.load_handler)
+    def setUp(self):
+        """Ensure the application is setup for testing."""
+        self.dir = os.path.dirname(os.path.abspath(__file__))
+        self.connector = Connector(
+            config=ConnectorConfig(
+                identity="1FEEDFEED1",
+                key="token",
+                name="test",
+                connector="test",
+            ),
+            context={
+                "runtime": "test_harness",
+                "runtime_id": "NA",
+            },
+        )
+
+    @responses.activate
+    def test_collect_pagination(self):
+        """Ensure pagination is working as expected."""
+        # Succeed with a cursor returned (to indicate paging is required).
+        responses.add(
+            responses.GET,
+            re.compile(r"https://.*"),
+            status=200,
+            content_type="application/json",
+            body=bytes(
+                open(
+                    os.path.join(self.dir, "fixtures/stripe/events/001.json"), "r"
+                ).read(),
+                "utf-8",
+            ),
+        )
+
+        # The last "page" returns an empty cursor.
+        responses.add(
+            responses.GET,
+            re.compile(r"https://.*"),
+            status=200,
+            content_type="application/json",
+            body=bytes(
+                open(
+                    os.path.join(self.dir, "fixtures/stripe/events/002.json"), "r"
+                ).read(),
+                "utf-8",
+            ),
+        )
+
+        self.connector.run()
+        self.assertEqual(self.connector._saved["logs"], 2)
+        self.assertEqual(self.connector.pointer, "evt_1OonDcGsC9LSOJDcarfWbL08")
+
+    @responses.activate
+    def test_collect_no_pagination(self):
+        """Ensure collection without pagination is working as expected."""
+        responses.add(
+            responses.GET,
+            re.compile(r"https://.*"),
+            status=200,
+            content_type="application/json",
+            body=bytes(
+                open(
+                    os.path.join(self.dir, "fixtures/stripe/events/002.json"), "r"
+                ).read(),
+                "utf-8",
+            ),
+        )
+
+        self.connector.run()
+        self.assertEqual(self.connector._saved["logs"], 1)
+        self.assertEqual(self.connector.pointer, "evt_1OonDcGsC9LSOJDcarfWbL10")