diff --git a/CHANGELOG.md b/CHANGELOG.md index 7be9aaab..e36e1226 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog + +## 1.4.0 + * Adds a `redirects` stream [#80] (https://github.com/singer-io/tap-shopify/pull/80) + ## 1.3.5 * Add `status` field to `Products` stream [#108](https://github.com/singer-io/tap-shopify/pull/108) diff --git a/Makefile b/Makefile index 1bd8b48f..000433df 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .DEFAULT_GOAL := test test: - pylint tap_shopify -d missing-docstring,too-many-branches + pylint tap_shopify -d missing-docstring,too-many-branches,duplicate-code nosetests tests/unittests diff --git a/setup.py b/setup.py index 8c9cf687..078bdc94 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.3.5", + version="1.4.0", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/schemas/redirects.json b/tap_shopify/schemas/redirects.json new file mode 100644 index 00000000..18b0c3e2 --- /dev/null +++ b/tap_shopify/schemas/redirects.json @@ -0,0 +1,23 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "path": { + "type": [ + "null", + "string" + ] + }, + "target": { + "type": [ + "null", + "string" + ] + } + } +} diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index dd73ba7a..694c2f36 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -7,3 +7,4 @@ import tap_shopify.streams.products import tap_shopify.streams.collects import tap_shopify.streams.custom_collections +import tap_shopify.streams.redirects diff --git a/tap_shopify/streams/redirects.py b/tap_shopify/streams/redirects.py new file mode 100644 index 00000000..6bd440c1 --- /dev/null +++ b/tap_shopify/streams/redirects.py @@ -0,0 +1,42 @@ +import shopify +from tap_shopify.streams.base import (Stream, + RESULTS_PER_PAGE, + OutOfOrderIdsError) +from tap_shopify.context import Context + + +class Redirects(Stream): + name = 'redirects' + replication_object = shopify.Redirect + # Redirects have no timestamps, but can be updated after creation + # So the only option is to use full table replication + replication_method = 'FULL_TABLE' + key_properties = ['id'] + replication_key = None + + def get_objects(self): + # Override base function as this is a full sync every time + since_id = 1 + while True: + query_params = { + "since_id": since_id, + "limit": RESULTS_PER_PAGE, + } + + objects = self.call_api(query_params) + + for obj in objects: + if obj.id < since_id: + raise OutOfOrderIdsError("obj.id < since_id: {} < {}".format( + obj.id, since_id)) + yield obj + + if len(objects) < RESULTS_PER_PAGE: + break + if objects[-1].id != max([o.id for o in objects]): + raise OutOfOrderIdsError("{} is not the max id in objects ({})".format( + objects[-1].id, max([o.id for o in objects]))) + since_id = objects[-1].id + + +Context.stream_objects['redirects'] = Redirects diff --git a/tests/base.py b/tests/base.py index d35ef8d3..e84182b1 100644 --- a/tests/base.py +++ b/tests/base.py @@ -104,6 +104,10 @@ def expected_metadata(self): self.PRIMARY_KEYS: {"id"}, self.FOREIGN_KEYS: {"order_id"}, self.REPLICATION_METHOD: self.INCREMENTAL, + self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}, + "redirects": { + self.PRIMARY_KEYS: {"id"}, + self.REPLICATION_METHOD: self.FULL, self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE} }