From bc527a164c0e646015c8c2cd599bdb7dc62a55c0 Mon Sep 17 00:00:00 2001 From: John Dwyer Date: Thu, 21 Jan 2021 11:29:41 -0800 Subject: [PATCH 1/4] add redirects stream --- CHANGELOG.md | 3 +++ setup.py | 2 +- tap_shopify/schemas/redirects.json | 23 +++++++++++++++++ tap_shopify/streams/__init__.py | 1 + tap_shopify/streams/redirects.py | 41 ++++++++++++++++++++++++++++++ 5 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 tap_shopify/schemas/redirects.json create mode 100644 tap_shopify/streams/redirects.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f4dceab..d5c1762f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 1.3.0 + * Adds a `redirects` stream [#80] (https://github.com/singer-io/tap-shopify/pull/80) + ## 1.2.6 * Accepts any string for `accepts_marketing_updated_at` field on the `customers` stream [#69] (https://github.com/singer-io/tap-shopify/pull/69) diff --git a/setup.py b/setup.py index 955cf4bb..cc21d12d 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.2.6", + version="1.3.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/schemas/redirects.json b/tap_shopify/schemas/redirects.json new file mode 100644 index 00000000..18b0c3e2 --- /dev/null +++ b/tap_shopify/schemas/redirects.json @@ -0,0 +1,23 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "path": { + "type": [ + "null", + "string" + ] + }, + "target": { + "type": [ + "null", + "string" + ] + } + } +} diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index dd73ba7a..694c2f36 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -7,3 +7,4 @@ import tap_shopify.streams.products import tap_shopify.streams.collects import tap_shopify.streams.custom_collections +import tap_shopify.streams.redirects diff --git a/tap_shopify/streams/redirects.py b/tap_shopify/streams/redirects.py new file mode 100644 index 00000000..08cc2fb9 --- /dev/null +++ b/tap_shopify/streams/redirects.py @@ -0,0 +1,41 @@ +import shopify +from tap_shopify.streams.base import (Stream, + RESULTS_PER_PAGE, + OutOfOrderIdsError) +from tap_shopify.context import Context + + +class Redirects(Stream): + name = 'redirects' + replication_object = shopify.Redirect + # Redirects have no timestamps, but can be updated after creation + # So the only option is to use full table replication + replication_method = 'FULL_TABLE' + replication_key = 'id' + + def get_objects(self): + # Override base function as this is a full sync every time + since_id = 1 + while True: + query_params = { + "since_id": since_id, + "limit": RESULTS_PER_PAGE, + } + + objects = self.call_api(query_params) + + for obj in objects: + if obj.id < since_id: + raise OutOfOrderIdsError("obj.id < since_id: {} < {}".format( + obj.id, since_id)) + yield obj + + if len(objects) < RESULTS_PER_PAGE: + break + if objects[-1].id != max([o.id for o in objects]): + raise OutOfOrderIdsError("{} is not the max id in objects ({})".format( + objects[-1].id, max([o.id for o in objects]))) + since_id = objects[-1].id + + +Context.stream_objects['redirects'] = Redirects From 5310ff85707f57d0f9f02da2dbf47cea138079f6 Mon Sep 17 00:00:00 2001 From: jbaca Date: Wed, 14 Jul 2021 19:30:03 +0000 Subject: [PATCH 2/4] got discovery to work --- tap_shopify/streams/redirects.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tap_shopify/streams/redirects.py b/tap_shopify/streams/redirects.py index 08cc2fb9..6bd440c1 100644 --- a/tap_shopify/streams/redirects.py +++ b/tap_shopify/streams/redirects.py @@ -11,7 +11,8 @@ class Redirects(Stream): # Redirects have no timestamps, but can be updated after creation # So the only option is to use full table replication replication_method = 'FULL_TABLE' - replication_key = 'id' + key_properties = ['id'] + replication_key = None def get_objects(self): # Override base function as this is a full sync every time From 29d6b09e940c0d9fb76800a4f875addc1bafdea9 Mon Sep 17 00:00:00 2001 From: jbaca Date: Wed, 14 Jul 2021 19:56:47 +0000 Subject: [PATCH 3/4] added to pylint --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1bd8b48f..000433df 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .DEFAULT_GOAL := test test: - pylint tap_shopify -d missing-docstring,too-many-branches + pylint tap_shopify -d missing-docstring,too-many-branches,duplicate-code nosetests tests/unittests From 5e488dc8c0af58a0b4f0a606073f69d1867efba5 Mon Sep 17 00:00:00 2001 From: jbaca Date: Thu, 15 Jul 2021 14:32:16 +0000 Subject: [PATCH 4/4] got tap-tester passing --- tests/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/base.py b/tests/base.py index d35ef8d3..e84182b1 100644 --- a/tests/base.py +++ b/tests/base.py @@ -104,6 +104,10 @@ def expected_metadata(self): self.PRIMARY_KEYS: {"id"}, self.FOREIGN_KEYS: {"order_id"}, self.REPLICATION_METHOD: self.INCREMENTAL, + self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, + "redirects": { + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.FULL, self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE} }