Add redirects stream clone

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+
+## 1.4.0
+  * Adds a `redirects` stream [#80] (https://github.com/singer-io/tap-shopify/pull/80)
+
 ## 1.3.5
   * Add `status` field to `Products` stream [#108](https://github.com/singer-io/tap-shopify/pull/108)
 

Makefile

@@ -1,5 +1,5 @@
 .DEFAULT_GOAL := test
 
 test:
-	pylint tap_shopify -d missing-docstring,too-many-branches
+	pylint tap_shopify -d missing-docstring,too-many-branches,duplicate-code
 	nosetests tests/unittests

setup.py

@@ -3,7 +3,7 @@
 
 setup(
     name="tap-shopify",
-    version="1.3.5",
+    version="1.4.0",
     description="Singer.io tap for extracting Shopify data",
     author="Stitch",
     url="http://github.com/singer-io/tap-shopify",

tap_shopify/schemas/redirects.json

@@ -0,0 +1,23 @@
+{
+    "type": "object",
+    "properties": {
+      "id": {
+        "type": [
+          "null",
+          "integer"
+        ]
+      },
+      "path": {
+        "type": [
+          "null",
+          "string"
+        ]
+      },
+      "target": {
+        "type": [
+          "null",
+          "string"
+        ]
+      }
+    }
+}

tap_shopify/streams/__init__.py

@@ -7,3 +7,4 @@
 import tap_shopify.streams.products
 import tap_shopify.streams.collects
 import tap_shopify.streams.custom_collections
+import tap_shopify.streams.redirects

tap_shopify/streams/redirects.py

@@ -0,0 +1,42 @@
+import shopify
+from tap_shopify.streams.base import (Stream,
+                                      RESULTS_PER_PAGE,
+                                      OutOfOrderIdsError)
+from tap_shopify.context import Context
+
+
+class Redirects(Stream):
+    name = 'redirects'
+    replication_object = shopify.Redirect
+    # Redirects have no timestamps, but can be updated after creation
+    # So the only option is to use full table replication
+    replication_method = 'FULL_TABLE'
+    key_properties = ['id']
+    replication_key = None
+
+    def get_objects(self):
+        # Override base function as this is a full sync every time
+        since_id = 1
+        while True:
+            query_params = {
+                "since_id": since_id,
+                "limit": RESULTS_PER_PAGE,
+            }
+
+            objects = self.call_api(query_params)
+
+            for obj in objects:
+                if obj.id < since_id:
+                    raise OutOfOrderIdsError("obj.id < since_id: {} < {}".format(
+                        obj.id, since_id))
+                yield obj
+
+            if len(objects) < RESULTS_PER_PAGE:
+                break
+            if objects[-1].id != max([o.id for o in objects]):
+                raise OutOfOrderIdsError("{} is not the max id in objects ({})".format(
+                    objects[-1].id, max([o.id for o in objects])))
+            since_id = objects[-1].id
+
+
+Context.stream_objects['redirects'] = Redirects

tests/base.py

@@ -104,6 +104,10 @@ def expected_metadata(self):
                 self.PRIMARY_KEYS: {"id"},
                 self.FOREIGN_KEYS: {"order_id"},
                 self.REPLICATION_METHOD: self.INCREMENTAL,
+                self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE},
+            "redirects": {
+                self.PRIMARY_KEYS: {"id"},
+                self.REPLICATION_METHOD: self.FULL,
                 self.API_LIMIT: self.DEFAULT_RESULTS_PER_PAGE}
         }