[TDL-24743] Experiment various sync strategies for clients stream

README.md

@@ -58,13 +58,11 @@ This tap:
   - Bookmark: last_modified_date (date-time)
 - Transformations: Fields camelCase to snake_case, Abstract/generalize custom_field_sets
 
-[**clients (GET v2)**](https://api.mambu.com/?http#clients-search)
+[**clients (GET v2)**](https://api.mambu.com/?http#clients-getall)
 - Endpoint: https://instance.sandbox.mambu.com/api/clients
 - Primary keys: id
 - Foreign keys: assigned_user_key (users), assigned_centre_key (centres), custom_field_set_id, custom_field_id (custom_field_sets)
-- Replication strategy: Incremental (query all, filter results)
-  - Sort by: lastModifiedDate:ASC
-  - Bookmark: last_modified_date (date-time)
+- Replication strategy: Full table
 - Transformations: Fields camelCase to snake_case, Abstract/generalize custom_field_sets
 
 [**credit_arrangements (GET v2)**](https://api.mambu.com/?http#creditarrangements-getAll)
@@ -270,7 +268,6 @@ This tap:
             "branches": "2019-06-11T13:37:51Z",
             "communications": "2019-06-19T19:48:42Z",
             "centres": "2019-06-18T18:23:53Z",
-            "clients": "2019-06-20T00:52:44Z",
             "credit_arrangements": "2019-06-19T19:48:45Z",
             "custom_field_sets": "2019-06-11T13:37:56Z",
             "deposit_accounts": "2019-06-19T19:48:47Z",

tap_mambu/helpers/schema.py

@@ -26,8 +26,7 @@
     },
     'clients': {
         'key_properties': ['id'],
-        'replication_method': 'INCREMENTAL',
-        'replication_keys': ['last_modified_date']
+        'replication_method': 'FULL_TABLE'
     },
     'credit_arrangements': {
         'key_properties': ['id'],

tap_mambu/tap_generators/clients_generator.py

@@ -1,30 +1,8 @@
-from .multithreaded_bookmark_generator import MultithreadedBookmarkGenerator
-from ..helpers import get_bookmark
-from ..helpers.datetime_utils import str_to_localized_datetime, datetime_to_local_str
+from .generator import TapGenerator
 
 
-class ClientsGenerator(MultithreadedBookmarkGenerator):
-    def _init_config(self):
-        super()._init_config()
-        self.max_threads = 10
-
+class ClientsGenerator(TapGenerator):
     def _init_endpoint_config(self):
         super(ClientsGenerator, self)._init_endpoint_config()
-        self.endpoint_path = "clients:search"
-        self.endpoint_bookmark_field = "lastModifiedDate"
-        self.endpoint_sorting_criteria = {
-            "field": "lastModifiedDate",
-            "order": "ASC"
-        }
-        self.endpoint_filter_criteria = [
-            {
-                "field": "lastModifiedDate",
-                "operator": "AFTER",
-                "value": datetime_to_local_str(str_to_localized_datetime(
-                    get_bookmark(self.state, self.stream_name, self.sub_type, self.start_date)))
-            }
-        ]
-
-    def prepare_batch_params(self):
-        super(ClientsGenerator, self).prepare_batch_params()
-        self.endpoint_filter_criteria[0]["value"] = datetime_to_local_str(self.endpoint_intermediary_bookmark_value)
+        self.endpoint_path = "clients"
+        self.endpoint_api_method = "GET"

tests/base.py

@@ -82,10 +82,7 @@ def expected_metadata(self):
                 self.PRIMARY_KEYS: {
                     "id"
                 },
-                self.REPLICATION_METHOD: "INCREMENTAL",
-                self.REPLICATION_KEYS: {
-                    "last_modified_date"
-                }
+                self.REPLICATION_METHOD: "FULL_TABLE"
             },
             "credit_arrangements": {
                 self.PRIMARY_KEYS: {