From 3f238d0d5ff035db4ddbca766ca2bda7592f8900 Mon Sep 17 00:00:00 2001 From: Hassan Syyid Date: Tue, 12 Oct 2021 15:13:02 -0400 Subject: [PATCH 01/24] Add option to use api_key or access_token in config --- setup.py | 2 +- tap_shopify/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 078bdc94..35fe00ac 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.0", + version="1.4.1", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index e864e2b0..8ff7f581 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -16,12 +16,12 @@ from tap_shopify.exceptions import ShopifyError import tap_shopify.streams # Load stream objects into Context -REQUIRED_CONFIG_KEYS = ["shop", "api_key"] +REQUIRED_CONFIG_KEYS = ["shop"] LOGGER = singer.get_logger() SDC_KEYS = {'id': 'integer', 'name': 'string', 'myshopify_domain': 'string'} def initialize_shopify_client(): - api_key = Context.config['api_key'] + api_key = Context.config.get('access_token', Context.config.get("api_key")) shop = Context.config['shop'] version = '2021-04' session = shopify.Session(shop, version, api_key) From 3d29e2553b6990598e348180b660c8f106b1d893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Tue, 21 Dec 2021 12:37:36 -0500 Subject: [PATCH 02/24] added incoming --- .gitignore | 1 + tap_shopify/schemas/inventory_levels.json | 3 +++ tap_shopify/streams/inventory_levels.py | 7 +++++++ 3 files changed, 11 insertions(+) diff --git a/.gitignore b/.gitignore index a6551241..390f49b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.secrets todo.org # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/tap_shopify/schemas/inventory_levels.json b/tap_shopify/schemas/inventory_levels.json index fb9e6f10..b0288fac 100644 --- a/tap_shopify/schemas/inventory_levels.json +++ b/tap_shopify/schemas/inventory_levels.json @@ -13,6 +13,9 @@ "location_id": { "type": ["null", "integer"] }, + "incoming": { + "type": ["null", "integer"] + }, "admin_graphql_api_id": { "type": ["null", "string"] } diff --git a/tap_shopify/streams/inventory_levels.py b/tap_shopify/streams/inventory_levels.py index 6cfdb35e..41490dbb 100644 --- a/tap_shopify/streams/inventory_levels.py +++ b/tap_shopify/streams/inventory_levels.py @@ -1,3 +1,4 @@ +import json import shopify from singer.utils import strftime, strptime_to_utc from tap_shopify.streams.base import (Stream, @@ -10,6 +11,7 @@ class InventoryLevels(Stream): replication_key = 'updated_at' key_properties = ['location_id', 'inventory_item_id'] replication_object = shopify.InventoryLevel + gql_query = "query inventoryLevel($id: ID!){inventoryLevel(id: $id){incoming}}" @shopify_error_handling def api_call_for_inventory_levels(self, parent_object_id, bookmark): @@ -41,10 +43,15 @@ def get_objects(self): yield from self.get_inventory_levels(parent_object.id, bookmark) def sync(self): + gql_client = shopify.GraphQL() bookmark = self.get_bookmark() max_bookmark = bookmark for inventory_level in self.get_objects(): inventory_level_dict = inventory_level.to_dict() + variables = dict(id = inventory_level_dict.get("admin_graphql_api_id")) + response = gql_client.execute(self.gql_query, variables) + response_dict = json.loads(response) + inventory_level_dict["incoming"] = response_dict["data"]["inventoryLevel"].get("incoming") replication_value = strptime_to_utc(inventory_level_dict[self.replication_key]) if replication_value >= bookmark: yield inventory_level_dict From 48235d787190255352fb762728bb275aae6f0a3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Tue, 21 Dec 2021 12:42:00 -0500 Subject: [PATCH 03/24] bump version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 35fe00ac..1eca48e5 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.1", + version="1.4.2", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", From 0b109354c02a38597c5078a9246864aba9f6a026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Fri, 24 Dec 2021 08:30:39 -0500 Subject: [PATCH 04/24] support for shop details --- setup.py | 2 +- tap_shopify/schemas/shop.json | 185 ++++++++++++++++++++++++++++++++ tap_shopify/streams/__init__.py | 1 + tap_shopify/streams/shop.py | 15 +++ 4 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 tap_shopify/schemas/shop.json create mode 100644 tap_shopify/streams/shop.py diff --git a/setup.py b/setup.py index 1eca48e5..4c5eef62 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.2", + version="1.4.3", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/schemas/shop.json b/tap_shopify/schemas/shop.json new file mode 100644 index 00000000..6be14d12 --- /dev/null +++ b/tap_shopify/schemas/shop.json @@ -0,0 +1,185 @@ +{ + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + }, + "domain": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "latitude": { + "type": [ + "null", + "number" + ] + }, + "longitude": { + "type": [ + "null", + "number" + ] + }, + "money_in_emails_format": { + "type": [ + "null", + "string" + ] + }, + "money_with_currency_in_emails_format": { + "type": [ + "null", + "string" + ] + }, + "eligible_for_payments": { + "type": [ + "null", + "boolean" + ] + }, + "requires_extra_payments_agreement": { + "type": [ + "null", + "boolean" + ] + }, + "password_enabled": { + "type": [ + "null", + "boolean" + ] + }, + "has_storefront": { + "type": [ + "null", + "boolean" + ] + }, + "eligible_for_card_reader_giveaway": { + "type": [ + "null", + "boolean" + ] + }, + "finances": { + "type": [ + "null", + "boolean" + ] + }, + "primary_location_id": { + "type": [ + "null", + "integer" + ] + }, + "cookie_consent_level": { + "type": [ + "null", + "string" + ] + }, + "visitor_tracking_consent_preference": { + "type": [ + "null", + "string" + ] + }, + "checkout_api_supported": { + "type": [ + "null", + "boolean" + ] + }, + "multi_location_enabled": { + "type": [ + "null", + "boolean" + ] + }, + "setup_required": { + "type": [ + "null", + "boolean" + ] + }, + "pre_launch_enabled": { + "type": [ + "null", + "boolean" + ] + }, + "enabled_presentment_currencies": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "currency": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" +} diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index 0bed4024..9906f72a 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -10,3 +10,4 @@ import tap_shopify.streams.locations import tap_shopify.streams.inventory_levels import tap_shopify.streams.inventory_items +import tap_shopify.streams.shop diff --git a/tap_shopify/streams/shop.py b/tap_shopify/streams/shop.py new file mode 100644 index 00000000..ee7ea38b --- /dev/null +++ b/tap_shopify/streams/shop.py @@ -0,0 +1,15 @@ +import shopify + +from tap_shopify.streams.base import Stream +from tap_shopify.context import Context + + +class Shop(Stream): + name = 'shop' + replication_object = shopify.Shop + + def sync(self): + response = shopify.Shop.current() + return [response.to_dict()] + +Context.stream_objects['shop'] = Shop \ No newline at end of file From 136ad344a23f8124c2673f0b4d31639c6c5a7e4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Fri, 7 Jan 2022 09:27:48 -0500 Subject: [PATCH 05/24] added fulfillment --- tap_shopify/schemas/fulfillments.json | 532 ++++++++++++++++++++++++ tap_shopify/streams/__init__.py | 1 + tap_shopify/streams/fulfillments.py | 56 +++ tap_shopify/streams/inventory_levels.py | 2 +- 4 files changed, 590 insertions(+), 1 deletion(-) create mode 100644 tap_shopify/schemas/fulfillments.json create mode 100644 tap_shopify/streams/fulfillments.py diff --git a/tap_shopify/schemas/fulfillments.json b/tap_shopify/schemas/fulfillments.json new file mode 100644 index 00000000..052dbf28 --- /dev/null +++ b/tap_shopify/schemas/fulfillments.json @@ -0,0 +1,532 @@ +{ + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "order_id": { + "type": [ + "null", + "integer" + ] + }, + "status": { + "type": [ + "null", + "string" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "service": { + "type": [ + "null", + "string" + ] + }, + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "tracking_company": { + "type": [ + "null", + "string" + ] + }, + "shipment_status": { + "type": [ + "null", + "string" + ] + }, + "location_id": { + "type": [ + "null", + "integer" + ] + }, + "tracking_number": { + "type": [ + "null", + "string" + ] + }, + "tracking_numbers": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "tracking_url": { + "type": [ + "null", + "string" + ] + }, + "tracking_urls": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "receipt": { + "type": [ + "object", + "null" + ], + "properties": { + "testcase": { + "type": [ + "boolean", + "null" + ] + }, + "authorization": { + "type": [ + "string", + "null" + ] + } + } + }, + "line_items": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "integer", + "null" + ] + }, + "variant_id": { + "type": [ + "integer", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "quantity": { + "type": [ + "integer", + "null" + ] + }, + "sku": { + "type": [ + "string", + "null" + ] + }, + "variant_title": { + "type": [ + "string", + "null" + ] + }, + "vendor": { + "type": [ + "string", + "null" + ] + }, + "fulfillment_service": { + "type": [ + "string", + "null" + ] + }, + "product_id": { + "type": [ + "integer", + "null" + ] + }, + "requires_shipping": { + "type": [ + "boolean", + "null" + ] + }, + "taxable": { + "type": [ + "boolean", + "null" + ] + }, + "gift_card": { + "type": [ + "boolean", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "variant_inventory_management": { + "type": [ + "string", + "null" + ] + }, + "properties": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "name": { + "type": [ + "string", + "null" + ] + }, + "value": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "product_exists": { + "type": [ + "boolean", + "null" + ] + }, + "fulfillable_quantity": { + "type": [ + "integer", + "null" + ] + }, + "grams": { + "type": [ + "integer", + "null" + ] + }, + "price": { + "type": [ + "string", + "null" + ] + }, + "total_discount": { + "type": [ + "string", + "null" + ] + }, + "fulfillment_status": { + "type": [ + "string", + "null" + ] + }, + "price_set": { + "type": [ + "object", + "null" + ], + "properties": { + "shop_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + }, + "presentment_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "total_discount_set": { + "type": [ + "object", + "null" + ], + "properties": { + "shop_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + }, + "presentment_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "discount_allocations": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "discount_application_index": { + "type": [ + "integer", + "null" + ] + }, + "amount_set": { + "type": [ + "object", + "null" + ], + "properties": { + "shop_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + }, + "presentment_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + } + }, + "admin_graphql_api_id": { + "type": [ + "string", + "null" + ] + }, + "tax_lines": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "price": { + "type": [ + "boolean", + "null" + ] + }, + "rate": { + "type": [ + "number", + "null" + ] + }, + "title": { + "type": [ + "boolean", + "null" + ] + }, + "price_set": { + "type": [ + "object", + "null" + ], + "properties": { + "shop_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + }, + "presentment_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index 9906f72a..56c6ee49 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -11,3 +11,4 @@ import tap_shopify.streams.inventory_levels import tap_shopify.streams.inventory_items import tap_shopify.streams.shop +import tap_shopify.streams.fulfillments diff --git a/tap_shopify/streams/fulfillments.py b/tap_shopify/streams/fulfillments.py new file mode 100644 index 00000000..f20c6c5d --- /dev/null +++ b/tap_shopify/streams/fulfillments.py @@ -0,0 +1,56 @@ +import shopify +import singer +from singer.utils import strftime, strptime_to_utc +from tap_shopify.context import Context +from tap_shopify.streams.base import (Stream, + shopify_error_handling) + +LOGGER = singer.get_logger() + +FULFILLMENT_RESULTS_PER_PAGE = 100 + + +class Fulfillments(Stream): + name = 'fulfillments' + replication_key = 'created_at' + replication_object = shopify.Fulfillment + + @shopify_error_handling + def call_api_for_fulfillments(self, parent_object): + return self.replication_object.find( + limit=FULFILLMENT_RESULTS_PER_PAGE, + order_id=parent_object.id, + ) + + def get_fulfillments(self, parent_object): + page = self.call_api_for_fulfillments(parent_object) + yield from page + + while page.has_next_page(): + page = page.next_page() + yield from page + + def get_objects(self): + selected_parent = Context.stream_objects['orders']() + selected_parent.name = "fulfillment_orders" + + for parent_object in selected_parent.get_objects(): + fulfillments = self.get_fulfillments(parent_object) + for fulfillment in fulfillments: + yield fulfillment + + def sync(self): + bookmark = self.get_bookmark() + max_bookmark = bookmark + for fulfillment in self.get_objects(): + fulfillment_dict = fulfillment.to_dict() + replication_value = strptime_to_utc(fulfillment_dict[self.replication_key]) + if replication_value >= bookmark: + yield fulfillment_dict + + if replication_value > max_bookmark: + max_bookmark = replication_value + self.update_bookmark(strftime(max_bookmark)) + + +Context.stream_objects['fulfillments'] = Fulfillments diff --git a/tap_shopify/streams/inventory_levels.py b/tap_shopify/streams/inventory_levels.py index 41490dbb..b9c521e7 100644 --- a/tap_shopify/streams/inventory_levels.py +++ b/tap_shopify/streams/inventory_levels.py @@ -11,7 +11,7 @@ class InventoryLevels(Stream): replication_key = 'updated_at' key_properties = ['location_id', 'inventory_item_id'] replication_object = shopify.InventoryLevel - gql_query = "query inventoryLevel($id: ID!){inventoryLevel(id: $id){incoming}}" + gql_query = "query inventoryLevel($id: ID!){inventoryLevel(id: $id){available, item, updatedAt, location, incoming, id}}" @shopify_error_handling def api_call_for_inventory_levels(self, parent_object_id, bookmark): From f16a0274e266720d9532ef74f6a0bee417a15f76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Mon, 10 Jan 2022 16:40:27 -0500 Subject: [PATCH 06/24] discount ans price_rule streams --- setup.py | 2 +- ...{fulfillments.json => discount_codes.json} | 0 tap_shopify/schemas/incoming_items.json | 17 ++ tap_shopify/schemas/price_rules.json | 289 ++++++++++++++++++ tap_shopify/streams/__init__.py | 4 +- tap_shopify/streams/discount_codes.py | 56 ++++ tap_shopify/streams/fulfillments.py | 56 ---- tap_shopify/streams/incoming_items.py | 44 +++ tap_shopify/streams/inventory_levels.py | 6 - tap_shopify/streams/price_rules.py | 11 + 10 files changed, 421 insertions(+), 64 deletions(-) rename tap_shopify/schemas/{fulfillments.json => discount_codes.json} (100%) create mode 100644 tap_shopify/schemas/incoming_items.json create mode 100644 tap_shopify/schemas/price_rules.json create mode 100644 tap_shopify/streams/discount_codes.py delete mode 100644 tap_shopify/streams/fulfillments.py create mode 100644 tap_shopify/streams/incoming_items.py create mode 100644 tap_shopify/streams/price_rules.py diff --git a/setup.py b/setup.py index 4c5eef62..2c78311e 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.3", + version="1.4.4", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/schemas/fulfillments.json b/tap_shopify/schemas/discount_codes.json similarity index 100% rename from tap_shopify/schemas/fulfillments.json rename to tap_shopify/schemas/discount_codes.json diff --git a/tap_shopify/schemas/incoming_items.json b/tap_shopify/schemas/incoming_items.json new file mode 100644 index 00000000..49be8887 --- /dev/null +++ b/tap_shopify/schemas/incoming_items.json @@ -0,0 +1,17 @@ +{ + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "incoming": { + "type": [ + "null", + "integer" + ] + } + }, + "type": "object" +} diff --git a/tap_shopify/schemas/price_rules.json b/tap_shopify/schemas/price_rules.json new file mode 100644 index 00000000..5e1dda24 --- /dev/null +++ b/tap_shopify/schemas/price_rules.json @@ -0,0 +1,289 @@ +{ + "properties": { + "allocation_method": { + "type": [ + "null", + "string" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "customer_selection": { + "type": [ + "null", + "string" + ] + }, + "ends_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "entitled_collection_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "entitled_country_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "entitled_product_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "entitled_variant_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "once_per_customer": { + "type": [ + "null", + "boolean" + ] + }, + "prerequisite_customer_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "prerequisite_quantity_range": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "greater_than_or_equal_to": { + "type": [ + "integer", + "null" + ] + } + } + } + }, + "prerequisite_saved_search_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "prerequisite_shipping_price_range": { + "type": [ + "object", + "null" + ], + "properties": { + "less_than_or_equal_to": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "prerequisite_subtotal_range": { + "type": [ + "object", + "null" + ], + "properties": { + "greater_than_or_equal_to": { + "type": [ + "string", + "null" + ] + } + }, + "prerequisite_to_entitlement_purchase": { + "type": [ + "object", + "null" + ], + "properties": { + "prerequisite_amount": { + "type": [ + "string", + "null" + ] + } + } + }, + "starts_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "target_type": { + "type": [ + "null", + "string" + ] + }, + "target_selection": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "usage_limit": { + "type": [ + "null", + "integer" + ] + }, + "prerequisite_product_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "prerequisite_variant_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "prerequisite_collection_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "value": { + "type": [ + "null", + "string" + ] + }, + "value_type": { + "type": [ + "null", + "string" + ] + }, + "prerequisite_to_entitlement_quantity_ratio": { + "type": [ + "object", + "null" + ], + "properties": { + "entitled_quantity": { + "type": [ + "integer", + "null" + ] + }, + "prerequisite_quantity": { + "type": [ + "integer", + "null" + ] + } + } + }, + "allocation_limit": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" +} \ No newline at end of file diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index 56c6ee49..74ab6e2b 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -11,4 +11,6 @@ import tap_shopify.streams.inventory_levels import tap_shopify.streams.inventory_items import tap_shopify.streams.shop -import tap_shopify.streams.fulfillments +import tap_shopify.streams.price_rules +import tap_shopify.streams.discount_codes +import tap_shopify.streams.incoming_items diff --git a/tap_shopify/streams/discount_codes.py b/tap_shopify/streams/discount_codes.py new file mode 100644 index 00000000..b3907106 --- /dev/null +++ b/tap_shopify/streams/discount_codes.py @@ -0,0 +1,56 @@ +import shopify +import singer +from singer.utils import strftime, strptime_to_utc +from tap_shopify.context import Context +from tap_shopify.streams.base import (Stream, + shopify_error_handling) + +LOGGER = singer.get_logger() + +DISCOUNT_CODES_RESULTS_PER_PAGE = 100 + + +class DiscountCodes(Stream): + name = 'discount_codes' + replication_key = 'created_at' + replication_object = shopify.DiscountCode + + @shopify_error_handling + def call_api_for_discount_codes(self, parent_object): + return self.replication_object.find( + limit=DISCOUNT_CODES_RESULTS_PER_PAGE, + price_rule_id=parent_object.id, + ) + + def get_discount_codes(self, parent_object): + page = self.call_api_for_discount_codes(parent_object) + yield from page + + while page.has_next_page(): + page = page.next_page() + yield from page + + def get_objects(self): + selected_parent = Context.stream_objects['price_rules']() + selected_parent.name = "discount_code_price_rules" + + for parent_object in selected_parent.get_objects(): + discount_codes = self.get_discount_codes(parent_object) + for discount_code in discount_codes: + yield discount_code + + def sync(self): + bookmark = self.get_bookmark() + self.max_bookmark = bookmark + for discount_code in self.get_objects(): + discount_code_dict = discount_code.to_dict() + replication_value = strptime_to_utc(discount_code_dict[self.replication_key]) + if replication_value >= bookmark: + yield discount_code_dict + if replication_value > self.max_bookmark: + self.max_bookmark = replication_value + + self.update_bookmark(strftime(self.max_bookmark)) + + +Context.stream_objects['discount_codes'] = DiscountCodes diff --git a/tap_shopify/streams/fulfillments.py b/tap_shopify/streams/fulfillments.py deleted file mode 100644 index f20c6c5d..00000000 --- a/tap_shopify/streams/fulfillments.py +++ /dev/null @@ -1,56 +0,0 @@ -import shopify -import singer -from singer.utils import strftime, strptime_to_utc -from tap_shopify.context import Context -from tap_shopify.streams.base import (Stream, - shopify_error_handling) - -LOGGER = singer.get_logger() - -FULFILLMENT_RESULTS_PER_PAGE = 100 - - -class Fulfillments(Stream): - name = 'fulfillments' - replication_key = 'created_at' - replication_object = shopify.Fulfillment - - @shopify_error_handling - def call_api_for_fulfillments(self, parent_object): - return self.replication_object.find( - limit=FULFILLMENT_RESULTS_PER_PAGE, - order_id=parent_object.id, - ) - - def get_fulfillments(self, parent_object): - page = self.call_api_for_fulfillments(parent_object) - yield from page - - while page.has_next_page(): - page = page.next_page() - yield from page - - def get_objects(self): - selected_parent = Context.stream_objects['orders']() - selected_parent.name = "fulfillment_orders" - - for parent_object in selected_parent.get_objects(): - fulfillments = self.get_fulfillments(parent_object) - for fulfillment in fulfillments: - yield fulfillment - - def sync(self): - bookmark = self.get_bookmark() - max_bookmark = bookmark - for fulfillment in self.get_objects(): - fulfillment_dict = fulfillment.to_dict() - replication_value = strptime_to_utc(fulfillment_dict[self.replication_key]) - if replication_value >= bookmark: - yield fulfillment_dict - - if replication_value > max_bookmark: - max_bookmark = replication_value - self.update_bookmark(strftime(max_bookmark)) - - -Context.stream_objects['fulfillments'] = Fulfillments diff --git a/tap_shopify/streams/incoming_items.py b/tap_shopify/streams/incoming_items.py new file mode 100644 index 00000000..290ab04d --- /dev/null +++ b/tap_shopify/streams/incoming_items.py @@ -0,0 +1,44 @@ +import shopify +import singer +import json +from singer.utils import strftime, strptime_to_utc +from tap_shopify.context import Context +from tap_shopify.streams.base import (Stream, + shopify_error_handling) + +LOGGER = singer.get_logger() + + +class IncomingItems(Stream): + name = 'incoming_items' + replication_key = 'createdAt' + gql_query = "query inventoryLevel($id: ID!){inventoryLevel(id: $id){id, incoming, createdAt}}" + + @shopify_error_handling + def call_api_for_incoming_items(self, parent_object): + gql_client = shopify.GraphQL() + response = gql_client.execute(self.gql_query, dict(id=parent_object.admin_graphql_api_id)) + return json.loads(response) + + def get_objects(self): + selected_parent = Context.stream_objects['inventory_levels']() + selected_parent.name = "incoming_items_inventory_levels" + + for parent_object in selected_parent.get_objects(): + incoming_item = self.call_api_for_incoming_items(parent_object) + yield incoming_item["data"].get("inventoryLevel") + + def sync(self): + bookmark = self.get_bookmark() + self.max_bookmark = bookmark + for incoming_item in self.get_objects(): + replication_value = strptime_to_utc(incoming_item[self.replication_key]) + if replication_value >= bookmark: + yield incoming_item + if replication_value > self.max_bookmark: + self.max_bookmark = replication_value + + self.update_bookmark(strftime(self.max_bookmark)) + + +Context.stream_objects['incoming_items'] = IncomingItems diff --git a/tap_shopify/streams/inventory_levels.py b/tap_shopify/streams/inventory_levels.py index b9c521e7..78f4e62f 100644 --- a/tap_shopify/streams/inventory_levels.py +++ b/tap_shopify/streams/inventory_levels.py @@ -11,7 +11,6 @@ class InventoryLevels(Stream): replication_key = 'updated_at' key_properties = ['location_id', 'inventory_item_id'] replication_object = shopify.InventoryLevel - gql_query = "query inventoryLevel($id: ID!){inventoryLevel(id: $id){available, item, updatedAt, location, incoming, id}}" @shopify_error_handling def api_call_for_inventory_levels(self, parent_object_id, bookmark): @@ -43,15 +42,10 @@ def get_objects(self): yield from self.get_inventory_levels(parent_object.id, bookmark) def sync(self): - gql_client = shopify.GraphQL() bookmark = self.get_bookmark() max_bookmark = bookmark for inventory_level in self.get_objects(): inventory_level_dict = inventory_level.to_dict() - variables = dict(id = inventory_level_dict.get("admin_graphql_api_id")) - response = gql_client.execute(self.gql_query, variables) - response_dict = json.loads(response) - inventory_level_dict["incoming"] = response_dict["data"]["inventoryLevel"].get("incoming") replication_value = strptime_to_utc(inventory_level_dict[self.replication_key]) if replication_value >= bookmark: yield inventory_level_dict diff --git a/tap_shopify/streams/price_rules.py b/tap_shopify/streams/price_rules.py new file mode 100644 index 00000000..47561bc5 --- /dev/null +++ b/tap_shopify/streams/price_rules.py @@ -0,0 +1,11 @@ +import shopify + +from tap_shopify.streams.base import Stream +from tap_shopify.context import Context + + +class PriceRules(Stream): + name = 'price_rules' + replication_object = shopify.PriceRule + +Context.stream_objects['price_rules'] = PriceRules From 022c74f9805357278790bfcb3b287d177e1967b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Mon, 10 Jan 2022 17:40:11 -0500 Subject: [PATCH 07/24] fix price_rules schema --- tap_shopify/schemas/price_rules.json | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/tap_shopify/schemas/price_rules.json b/tap_shopify/schemas/price_rules.json index 5e1dda24..deead880 100644 --- a/tap_shopify/schemas/price_rules.json +++ b/tap_shopify/schemas/price_rules.json @@ -107,21 +107,15 @@ }, "prerequisite_quantity_range": { "type": [ - "array", + "object", "null" ], - "items": { - "type": [ - "object", - "null" - ], - "properties": { - "greater_than_or_equal_to": { - "type": [ - "integer", - "null" - ] - } + "properties": { + "greater_than_or_equal_to": { + "type": [ + "integer", + "null" + ] } } }, From fb5ea575b41d1d3e6752633d6e9f7513c21f3a56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Wed, 12 Jan 2022 19:39:23 -0500 Subject: [PATCH 08/24] fix stream --- tap_shopify/schemas/price_rules.json | 240 +++++++++++++-------------- 1 file changed, 120 insertions(+), 120 deletions(-) diff --git a/tap_shopify/schemas/price_rules.json b/tap_shopify/schemas/price_rules.json index deead880..49c4272c 100644 --- a/tap_shopify/schemas/price_rules.json +++ b/tap_shopify/schemas/price_rules.json @@ -1,32 +1,93 @@ { "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "value_type": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + }, + "customer_selection": { + "type": [ + "null", + "string" + ] + }, + "target_type": { + "type": [ + "null", + "string" + ] + }, + "target_selection": { + "type": [ + "null", + "string" + ] + }, "allocation_method": { "type": [ "null", "string" ] }, - "created_at": { + "allocation_limit": { + "type": [ + "null", + "string" + ] + }, + "once_per_customer": { + "type": [ + "null", + "boolean" + ] + }, + "usage_limit": { + "type": [ + "null", + "integer" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "starts_at": { "type": [ "null", "string" ], "format": "date-time" }, - "updated_at": { + "ends_at": { "type": [ "null", "string" ], "format": "date-time" }, - "customer_selection": { + "created_at": { "type": [ "null", "string" - ] + ], + "format": "date-time" }, - "ends_at": { + "updated_at": { "type": [ "null", "string" @@ -81,18 +142,6 @@ ] } }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "once_per_customer": { - "type": [ - "null", - "boolean" - ] - }, "prerequisite_customer_ids": { "type": [ "array", @@ -158,125 +207,76 @@ "null" ] } - }, - "prerequisite_to_entitlement_purchase": { - "type": [ - "object", - "null" - ], - "properties": { - "prerequisite_amount": { - "type": [ - "string", - "null" - ] - } + } + }, + "prerequisite_to_entitlement_purchase": { + "type": [ + "object", + "null" + ], + "properties": { + "prerequisite_amount": { + "type": [ + "string", + "null" + ] } - }, - "starts_at": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "target_type": { - "type": [ - "null", - "string" - ] - }, - "target_selection": { + } + }, + "prerequisite_product_ids": { + "type": [ + "array", + "null" + ], + "items": { "type": [ "null", - "string" + "integer" ] - }, - "title": { + } + }, + "prerequisite_variant_ids": { + "type": [ + "array", + "null" + ], + "items": { "type": [ "null", - "string" + "integer" ] - }, - "usage_limit": { + } + }, + "prerequisite_collection_ids": { + "type": [ + "array", + "null" + ], + "items": { "type": [ "null", "integer" ] - }, - "prerequisite_product_ids": { - "type": [ - "array", - "null" - ], - "items": { - "type": [ - "null", - "integer" - ] - } - }, - "prerequisite_variant_ids": { - "type": [ - "array", - "null" - ], - "items": { + } + }, + "prerequisite_to_entitlement_quantity_ratio": { + "type": [ + "object", + "null" + ], + "properties": { + "entitled_quantity": { "type": [ - "null", - "integer" + "integer", + "null" ] - } - }, - "prerequisite_collection_ids": { - "type": [ - "array", - "null" - ], - "items": { + }, + "prerequisite_quantity": { "type": [ - "null", - "integer" + "integer", + "null" ] } - }, - "value": { - "type": [ - "null", - "string" - ] - }, - "value_type": { - "type": [ - "null", - "string" - ] - }, - "prerequisite_to_entitlement_quantity_ratio": { - "type": [ - "object", - "null" - ], - "properties": { - "entitled_quantity": { - "type": [ - "integer", - "null" - ] - }, - "prerequisite_quantity": { - "type": [ - "integer", - "null" - ] - } - } - }, - "allocation_limit": { - "type": [ - "null", - "string" - ] } }, "type": "object" From 6c216124f3454dfec33e27fd4983991922465e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Sat, 5 Feb 2022 08:55:41 -0500 Subject: [PATCH 09/24] changes on backoff config --- setup.py | 2 +- tap_shopify/streams/base.py | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 2c78311e..2041e152 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.4", + version="1.4.5", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 04488bf6..fc510019 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -35,7 +35,7 @@ def leaky_bucket_handler(details): details['wait']) def retry_handler(details): - LOGGER.info("Received 500 or retryable error -- Retry %s/%s", + LOGGER.info("Received 500 or retryable -- Retry %s/%s", details['tries'], MAX_RETRIES) #pylint: disable=unused-argument @@ -55,15 +55,12 @@ def shopify_error_handling(fnc): (pyactiveresource.connection.ServerError, pyactiveresource.formats.Error, simplejson.scanner.JSONDecodeError), - giveup=is_not_status_code_fn(range(500, 599)), on_backoff=retry_handler, max_tries=MAX_RETRIES) @backoff.on_exception(retry_after_wait_gen, pyactiveresource.connection.ClientError, giveup=is_not_status_code_fn([429]), - on_backoff=leaky_bucket_handler, - # No jitter as we want a constant value - jitter=None) + on_backoff=leaky_bucket_handler) @functools.wraps(fnc) def wrapper(*args, **kwargs): return fnc(*args, **kwargs) From f2ccf5846c35ded7da02099529d2ca7f8d0f2621 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Sat, 5 Feb 2022 09:29:37 -0500 Subject: [PATCH 10/24] increase retries --- tap_shopify/streams/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index fc510019..5f38d313 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -20,7 +20,7 @@ DATE_WINDOW_SIZE = 1 # We will retry a 500 error a maximum of 5 times before giving up -MAX_RETRIES = 5 +MAX_RETRIES = 10 def is_not_status_code_fn(status_code): def gen_fn(exc): @@ -54,7 +54,8 @@ def shopify_error_handling(fnc): @backoff.on_exception(backoff.expo, (pyactiveresource.connection.ServerError, pyactiveresource.formats.Error, - simplejson.scanner.JSONDecodeError), + simplejson.scanner.JSONDecodeError, + Exception), on_backoff=retry_handler, max_tries=MAX_RETRIES) @backoff.on_exception(retry_after_wait_gen, From 9dfb5b82aea183f6db9a2164ffb13431f98d5397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Mon, 7 Feb 2022 10:01:22 -0500 Subject: [PATCH 11/24] improve and mute backoff --- setup.py | 2 +- tap_shopify/__init__.py | 5 +++++ tap_shopify/streams/locations.py | 5 ++++- tap_shopify/streams/shop.py | 33 ++++++++++++++++++++++++++------ 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 2041e152..8b4f133c 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.5", + version="1.4.6", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 8ff7f581..e815a539 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -5,6 +5,7 @@ import time import math import copy +import logging import pyactiveresource import shopify @@ -14,12 +15,16 @@ from singer import Transformer from tap_shopify.context import Context from tap_shopify.exceptions import ShopifyError +from tap_shopify.streams.base import shopify_error_handling import tap_shopify.streams # Load stream objects into Context REQUIRED_CONFIG_KEYS = ["shop"] LOGGER = singer.get_logger() SDC_KEYS = {'id': 'integer', 'name': 'string', 'myshopify_domain': 'string'} +logging.getLogger('backoff').setLevel(logging.CRITICAL) + +@shopify_error_handling def initialize_shopify_client(): api_key = Context.config.get('access_token', Context.config.get("api_key")) shop = Context.config['shop'] diff --git a/tap_shopify/streams/locations.py b/tap_shopify/streams/locations.py index 3a61d0a2..1c0d2274 100644 --- a/tap_shopify/streams/locations.py +++ b/tap_shopify/streams/locations.py @@ -8,8 +8,11 @@ class Locations(Stream): replication_object = shopify.Location @shopify_error_handling + def api_call_for_locations_data(self): + return self.replication_object.find() + def get_locations_data(self): - location_page = self.replication_object.find() + location_page = self.api_call_for_locations_data() yield from location_page while location_page.has_next_page(): diff --git a/tap_shopify/streams/shop.py b/tap_shopify/streams/shop.py index ee7ea38b..0c1a4b24 100644 --- a/tap_shopify/streams/shop.py +++ b/tap_shopify/streams/shop.py @@ -1,15 +1,36 @@ import shopify - -from tap_shopify.streams.base import Stream +from singer import utils +from tap_shopify.streams.base import (Stream, shopify_error_handling) from tap_shopify.context import Context - class Shop(Stream): name = 'shop' replication_object = shopify.Shop + @shopify_error_handling + def api_call_for_shop_data(self): + return self.replication_object.current() + + def get_shop_data(self): + shop_page = [self.api_call_for_shop_data()] + yield from shop_page + def sync(self): - response = shopify.Shop.current() - return [response.to_dict()] + bookmark = self.get_bookmark() + max_bookmark = bookmark + + for shop in self.get_shop_data(): + + shop_dict = shop.to_dict() + replication_value = utils.strptime_to_utc(shop_dict[self.replication_key]) + + if replication_value >= bookmark: + yield shop_dict + + # update max bookmark if "replication_value" of current shop is greater + if replication_value > max_bookmark: + max_bookmark = replication_value + + self.update_bookmark(utils.strftime(max_bookmark)) -Context.stream_objects['shop'] = Shop \ No newline at end of file +Context.stream_objects['shop'] = Shop From bacead1512246c9e022255d56d9d37967be66a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Tue, 15 Feb 2022 15:27:51 -0500 Subject: [PATCH 12/24] create products events --- setup.py | 2 +- tap_shopify/schemas/events_products.json | 55 ++++++++++++++++++++++++ tap_shopify/streams/__init__.py | 1 + tap_shopify/streams/events_products.py | 54 +++++++++++++++++++++++ 4 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 tap_shopify/schemas/events_products.json create mode 100644 tap_shopify/streams/events_products.py diff --git a/setup.py b/setup.py index 8b4f133c..1f4deef1 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.6", + version="1.4.", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/schemas/events_products.json b/tap_shopify/schemas/events_products.json new file mode 100644 index 00000000..9c00fe30 --- /dev/null +++ b/tap_shopify/schemas/events_products.json @@ -0,0 +1,55 @@ +{ + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "subject_id": { + "type": [ + "null", + "integer" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "subject_type": { + "type": [ + "null", + "string" + ] + }, + "verb": { + "type": [ + "null", + "string" + ] + }, + "message": { + "type": [ + "null", + "string" + ] + }, + "author": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + } + \ No newline at end of file diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index 74ab6e2b..ea2167c2 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -14,3 +14,4 @@ import tap_shopify.streams.price_rules import tap_shopify.streams.discount_codes import tap_shopify.streams.incoming_items +import tap_shopify.streams.events_products diff --git a/tap_shopify/streams/events_products.py b/tap_shopify/streams/events_products.py new file mode 100644 index 00000000..51c8d1e5 --- /dev/null +++ b/tap_shopify/streams/events_products.py @@ -0,0 +1,54 @@ +import shopify +import singer +from singer.utils import strftime, strptime_to_utc +from tap_shopify.context import Context +from tap_shopify.streams.base import (Stream, + shopify_error_handling) + +LOGGER = singer.get_logger() + +EVENTS_RESULTS_PER_PAGE = 100 + + +class EventsProducts(Stream): + name = 'events_products' + replication_key = 'created_at' + replication_object = shopify.Event + + @shopify_error_handling + def call_api_for_events_products(self): + return self.replication_object.find( + limit=EVENTS_RESULTS_PER_PAGE, + filter="Product", + # verb = "destroy", + created_at_min = self.get_bookmark() + ) + + def get_events_products(self, ): + page = self.call_api_for_events_products() + yield from page + + while page.has_next_page(): + page = page.next_page() + yield from page + + def get_objects(self): + events_products = self.get_events_products() + for events_product in events_products: + yield events_product + + def sync(self): + bookmark = self.get_bookmark() + self.max_bookmark = bookmark + for events_product in self.get_objects(): + events_product_dict = events_product.to_dict() + replication_value = strptime_to_utc(events_product_dict[self.replication_key]) + if replication_value >= bookmark: + yield events_product_dict + if replication_value > self.max_bookmark: + self.max_bookmark = replication_value + + self.update_bookmark(strftime(self.max_bookmark)) + + +Context.stream_objects['events_products'] = EventsProducts From 0ea57932da663cf0f65cce6c5359cef86f5ac17e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Thu, 24 Feb 2022 07:01:06 -0500 Subject: [PATCH 13/24] fix replication key for inventory levels --- tap_shopify/streams/incoming_items.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_shopify/streams/incoming_items.py b/tap_shopify/streams/incoming_items.py index 290ab04d..3fb06264 100644 --- a/tap_shopify/streams/incoming_items.py +++ b/tap_shopify/streams/incoming_items.py @@ -22,7 +22,7 @@ def call_api_for_incoming_items(self, parent_object): def get_objects(self): selected_parent = Context.stream_objects['inventory_levels']() - selected_parent.name = "incoming_items_inventory_levels" + selected_parent.name = "inventory_levels" for parent_object in selected_parent.get_objects(): incoming_item = self.call_api_for_incoming_items(parent_object) From e731f58e3ac18899121fe49fa99e94ca5caacc92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Thu, 24 Feb 2022 11:23:56 -0500 Subject: [PATCH 14/24] fix version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1f4deef1..78ee8c04 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.", + version="1.4.7", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", From d8b5775428f603f28c116645b8721c50d4671dda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Wed, 6 Apr 2022 14:27:51 -0400 Subject: [PATCH 15/24] hide gql error print --- setup.py | 2 +- tap_shopify/streams/incoming_items.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 78ee8c04..7668668a 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.7", + version="1.4.8", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/incoming_items.py b/tap_shopify/streams/incoming_items.py index 3fb06264..c0325184 100644 --- a/tap_shopify/streams/incoming_items.py +++ b/tap_shopify/streams/incoming_items.py @@ -1,3 +1,5 @@ +import os +import sys import shopify import singer import json @@ -9,6 +11,16 @@ LOGGER = singer.get_logger() +class HiddenPrints: + def __enter__(self): + self._original_stdout = sys.stdout + sys.stdout = open(os.devnull, 'w') + + def __exit__(self, exc_type, exc_val, exc_tb): + sys.stdout.close() + sys.stdout = self._original_stdout + + class IncomingItems(Stream): name = 'incoming_items' replication_key = 'createdAt' @@ -17,7 +29,8 @@ class IncomingItems(Stream): @shopify_error_handling def call_api_for_incoming_items(self, parent_object): gql_client = shopify.GraphQL() - response = gql_client.execute(self.gql_query, dict(id=parent_object.admin_graphql_api_id)) + with HiddenPrints(): + response = gql_client.execute(self.gql_query, dict(id=parent_object.admin_graphql_api_id)) return json.loads(response) def get_objects(self): From 38dd59bbb3f656aa44bef1f092355d444212771a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Thu, 12 May 2022 08:37:37 -0400 Subject: [PATCH 16/24] added smart collections --- setup.py | 2 +- tap_shopify/schemas/smart_collections.json | 143 +++++++++++++++++++++ tap_shopify/streams/__init__.py | 1 + tap_shopify/streams/smart_collections.py | 11 ++ 4 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 tap_shopify/schemas/smart_collections.json create mode 100644 tap_shopify/streams/smart_collections.py diff --git a/setup.py b/setup.py index 7668668a..e1ffe3ae 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.8", + version="1.4.9", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/schemas/smart_collections.json b/tap_shopify/schemas/smart_collections.json new file mode 100644 index 00000000..150b3ae5 --- /dev/null +++ b/tap_shopify/schemas/smart_collections.json @@ -0,0 +1,143 @@ +{ + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "handle": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "body_html": { + "type": [ + "null", + "string" + ] + }, + "published_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "sort_order": { + "type": [ + "null", + "string" + ] + }, + "template_suffix": { + "type": [ + "null", + "string" + ] + }, + "disjunctive": { + "type": [ + "null", + "boolean" + ] + }, + "published_scope": { + "type": [ + "null", + "string" + ] + }, + "admin_graphql_api_id": { + "type": [ + "null", + "string" + ] + }, + "rules": { + "items": { + "properties": { + "column": { + "type": [ + "null", + "string" + ] + }, + "relation": { + "type": [ + "null", + "string" + ] + }, + "condition": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + }, + "image": { + "properties": { + "alt": { + "type": [ + "null", + "string" + ] + }, + "src": { + "type": [ + "null", + "string" + ] + }, + "width": { + "type": [ + "null", + "integer" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ] + }, + "height": { + "type": [ + "null", + "integer" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": "object" +} diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index ea2167c2..845cb4f3 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -15,3 +15,4 @@ import tap_shopify.streams.discount_codes import tap_shopify.streams.incoming_items import tap_shopify.streams.events_products +import tap_shopify.streams.smart_collections diff --git a/tap_shopify/streams/smart_collections.py b/tap_shopify/streams/smart_collections.py new file mode 100644 index 00000000..2ebb7236 --- /dev/null +++ b/tap_shopify/streams/smart_collections.py @@ -0,0 +1,11 @@ +import shopify + +from tap_shopify.streams.base import Stream +from tap_shopify.context import Context + + +class SmartCollections(Stream): + name = 'smart_collections' + replication_object = shopify.SmartCollection + +Context.stream_objects['smart_collections'] = SmartCollections From fb24dc875790a300ca75f6d6a1978b93577e1729 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Mon, 23 May 2022 19:30:37 -0400 Subject: [PATCH 17/24] fix backoff pagination --- tap_shopify/streams/inventory_levels.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tap_shopify/streams/inventory_levels.py b/tap_shopify/streams/inventory_levels.py index 78f4e62f..a4d68fb7 100644 --- a/tap_shopify/streams/inventory_levels.py +++ b/tap_shopify/streams/inventory_levels.py @@ -19,13 +19,17 @@ def api_call_for_inventory_levels(self, parent_object_id, bookmark): limit = RESULTS_PER_PAGE, location_ids=parent_object_id ) + + @shopify_error_handling + def get_next_page(self, inventory_page): + return inventory_page.next_page() def get_inventory_levels(self, parent_object, bookmark): inventory_page = self.api_call_for_inventory_levels(parent_object, bookmark) yield from inventory_page while inventory_page.has_next_page(): - inventory_page = inventory_page.next_page() + inventory_page = self.get_next_page(inventory_page) yield from inventory_page def get_objects(self): From 1abfbabd91b812d9b5d8d5dab316f8e5381d1a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Mon, 23 May 2022 19:31:11 -0400 Subject: [PATCH 18/24] fix backoff pagination --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e1ffe3ae..88be23ee 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.9", + version="1.4.10", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", From 40ce4da5dea7ee9c3b39fb4a2803369c47268ea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Thu, 10 Nov 2022 11:00:49 -0500 Subject: [PATCH 19/24] increase default window size --- setup.py | 2 +- tap_shopify/streams/base.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 88be23ee..afce90fe 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.10", + version="1.4.11", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 5f38d313..a71e2418 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -17,7 +17,7 @@ # We've observed 500 errors returned if this is too large (30 days was too # large for a customer) -DATE_WINDOW_SIZE = 1 +DATE_WINDOW_SIZE = 365 # We will retry a 500 error a maximum of 5 times before giving up MAX_RETRIES = 10 From 5d2eb96eba9caf8107bed0a995dc7ece79dbd369 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Wed, 30 Nov 2022 15:05:10 -0500 Subject: [PATCH 20/24] Remove inventory items filter --- setup.py | 2 +- tap_shopify/streams/inventory_items.py | 12 +----------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/setup.py b/setup.py index afce90fe..3f557660 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.11", + version="1.4.12", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", diff --git a/tap_shopify/streams/inventory_items.py b/tap_shopify/streams/inventory_items.py index 55fe047a..89088513 100644 --- a/tap_shopify/streams/inventory_items.py +++ b/tap_shopify/streams/inventory_items.py @@ -38,17 +38,7 @@ def get_objects(self): yield inventory_item def sync(self): - bookmark = self.get_bookmark() - max_bookmark = bookmark for inventory_item in self.get_objects(): - inventory_item_dict = inventory_item.to_dict() - replication_value = strptime_to_utc(inventory_item_dict[self.replication_key]) - if replication_value >= bookmark: - yield inventory_item_dict - - if replication_value > max_bookmark: - max_bookmark = replication_value - - self.update_bookmark(strftime(max_bookmark)) + yield inventory_item.to_dict() Context.stream_objects['inventory_items'] = InventoryItems From 9aad168596e7af0df5017afccb612e963cf56827 Mon Sep 17 00:00:00 2001 From: Alvaro Valarezo de la Fuente <39871126+AlvaroRaul7@users.noreply.github.com> Date: Fri, 16 Dec 2022 17:11:49 -0500 Subject: [PATCH 21/24] added new stream and update shopify library (#12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Álvaro Valarezo --- README.md | 4 +- setup.py | 4 +- tap_shopify/__init__.py | 2 +- tap_shopify/schemas/product_category.json | 20 ++++++ tap_shopify/streams/__init__.py | 1 + tap_shopify/streams/product_category.py | 80 +++++++++++++++++++++++ 6 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 tap_shopify/schemas/product_category.json create mode 100644 tap_shopify/streams/product_category.py diff --git a/README.md b/README.md index aa801952..ad160e06 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ This tap: ## Quick Start -1. Install +1. Install the tap locally - pip install tap-shopify + pip install -e . 2. Create the config file diff --git a/setup.py b/setup.py index 3f557660..56ae1ee4 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.12", + version="1.4.13", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -11,7 +11,7 @@ python_requires='>=3.5.2', py_modules=["tap_shopify"], install_requires=[ - "ShopifyAPI==8.4.1", + "ShopifyAPI==12.1.0", "singer-python==5.12.1", ], extras_require={ diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index e815a539..8083d284 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -28,7 +28,7 @@ def initialize_shopify_client(): api_key = Context.config.get('access_token', Context.config.get("api_key")) shop = Context.config['shop'] - version = '2021-04' + version = '2022-10' session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) # Shop.current() makes a call for shop details with provided shop and api_key diff --git a/tap_shopify/schemas/product_category.json b/tap_shopify/schemas/product_category.json new file mode 100644 index 00000000..684d6ebe --- /dev/null +++ b/tap_shopify/schemas/product_category.json @@ -0,0 +1,20 @@ +{ + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "category_id": { + "type": ["string", "null"] + }, + "full_name": { + "type": ["string", "null"] + }, + "is_leaf": { + "type": ["boolean", "null"] + }, + "is_root": { + "type": ["boolean", "null"] + } + } +} diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index 845cb4f3..0daa7e3e 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -16,3 +16,4 @@ import tap_shopify.streams.incoming_items import tap_shopify.streams.events_products import tap_shopify.streams.smart_collections +import tap_shopify.streams.product_category \ No newline at end of file diff --git a/tap_shopify/streams/product_category.py b/tap_shopify/streams/product_category.py new file mode 100644 index 00000000..b5517c41 --- /dev/null +++ b/tap_shopify/streams/product_category.py @@ -0,0 +1,80 @@ +import json +import os +import sys +import shopify +import singer +from singer.utils import strftime, strptime_to_utc + +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream, shopify_error_handling + +LOGGER = singer.get_logger() + + +class HiddenPrints: + def __enter__(self): + self._original_stdout = sys.stdout + sys.stdout = open(os.devnull, 'w') + + def __exit__(self, exc_type, exc_val, exc_tb): + sys.stdout.close() + sys.stdout = self._original_stdout + + +class ProductCategory(Stream): + name = 'product_category' + replication_key = 'createdAt' + gql_query = """ + query product($id: ID!){ + product(id: $id){ + id, + productType, + createdAt, + productCategory{ + productTaxonomyNode{ + id, + fullName, + isLeaf, + isRoot + + } + } + } + } + """ + + @shopify_error_handling + def call_api_for_product_categories(self, parent_object): + gql_client = shopify.GraphQL() + with HiddenPrints(): + response = gql_client.execute(self.gql_query, dict(id=parent_object.admin_graphql_api_id)) + return json.loads(response) + + def get_objects(self): + selected_parent = Context.stream_objects['products']() + selected_parent.name = "products_categories" + for parent_object in selected_parent.get_objects(): + product_category = self.call_api_for_product_categories(parent_object) + item = product_category["data"].get("product") + if item.get('productCategory'): + item['category_id'] = item['productCategory']['productTaxonomyNode']['id'] + item['full_name'] = item['productCategory']['productTaxonomyNode']['fullName'] + item['is_leaf'] = item['productCategory']['productTaxonomyNode']['isLeaf'] + item['is_root'] = item['productCategory']['productTaxonomyNode']['isRoot'] + yield item + + def sync(self): + bookmark = self.get_bookmark() + self.max_bookmark = bookmark + for incoming_item in self.get_objects(): + replication_value = strptime_to_utc(incoming_item[self.replication_key]) + if replication_value >= bookmark: + + yield incoming_item + if replication_value > self.max_bookmark: + self.max_bookmark = replication_value + + self.update_bookmark(strftime(self.max_bookmark)) + + +Context.stream_objects['product_category'] = ProductCategory From 892d72613207136ad317645d1de46cef089025f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josu=C3=A9=20M=2E=20Sehnem?= Date: Tue, 20 Dec 2022 11:01:13 -0300 Subject: [PATCH 22/24] make api version configurable (#13) --- tap_shopify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index 8083d284..b368b5bb 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -28,7 +28,7 @@ def initialize_shopify_client(): api_key = Context.config.get('access_token', Context.config.get("api_key")) shop = Context.config['shop'] - version = '2022-10' + version = Context.config.get('api_version', '2022-04') session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) # Shop.current() makes a call for shop details with provided shop and api_key From 544467e8dc13ba925cfc87858e3f88d672642f9b Mon Sep 17 00:00:00 2001 From: Keyna Rafael <95432445+keyn4@users.noreply.github.com> Date: Tue, 18 Apr 2023 12:56:07 -0500 Subject: [PATCH 23/24] add sleep time to avoid 2 req per second limit (#14) --- tap_shopify/streams/base.py | 2 ++ tap_shopify/streams/inventory_items.py | 2 ++ tap_shopify/streams/metafields.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index a71e2418..0ca54e36 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -2,6 +2,7 @@ import functools import math import sys +import time import backoff import pyactiveresource @@ -90,6 +91,7 @@ def __init__(self): self.results_per_page = Context.get_results_per_page(RESULTS_PER_PAGE) def get_bookmark(self): + time.sleep(0.5) bookmark = (singer.get_bookmark(Context.state, # name is overridden by some substreams self.name, diff --git a/tap_shopify/streams/inventory_items.py b/tap_shopify/streams/inventory_items.py index 89088513..ffd62a2b 100644 --- a/tap_shopify/streams/inventory_items.py +++ b/tap_shopify/streams/inventory_items.py @@ -3,6 +3,7 @@ from singer.utils import strftime,strptime_to_utc from tap_shopify.streams.base import (Stream, shopify_error_handling) from tap_shopify.context import Context +import time LOGGER = singer.get_logger() @@ -14,6 +15,7 @@ class InventoryItems(Stream): @shopify_error_handling def get_inventory_items(self, inventory_items_ids): + time.sleep(0.5) return self.replication_object.find( ids=inventory_items_ids, limit=RESULTS_PER_PAGE) diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index f96b3a06..620fb882 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -1,6 +1,7 @@ import json import shopify import singer +import time from tap_shopify.context import Context from tap_shopify.streams.base import (Stream, @@ -19,6 +20,7 @@ def get_selected_parents(): def get_metafields(parent_object, since_id): # This call results in an HTTP request - the parent object never has a # cache of this data so we have to issue that request. + time.sleep(0.5) return parent_object.metafields( limit=Context.get_results_per_page(RESULTS_PER_PAGE), since_id=since_id) From 9480cdfd62920f0ce773a1516ccf60643d2c55f0 Mon Sep 17 00:00:00 2001 From: Adil Ahmed Date: Wed, 19 Apr 2023 18:16:00 +0500 Subject: [PATCH 24/24] Minor firs for 429 backoff --- tap_shopify/streams/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 0ca54e36..0d4da608 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -48,8 +48,8 @@ def retry_after_wait_gen(**kwargs): # Retry-After is an undocumented header. But honoring # it was proven to work in our spikes. # It's been observed to come through as lowercase, so fallback if not present - sleep_time_str = resp.headers.get('Retry-After', resp.headers.get('retry-after')) - yield math.floor(float(sleep_time_str)) + sleep_time_str = resp.headers.get('Retry-After', resp.headers.get('retry-after',4)) + yield math.ceil(float(sleep_time_str)) def shopify_error_handling(fnc): @backoff.on_exception(backoff.expo,