diff --git a/.gitignore b/.gitignore index a6551241..390f49b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.secrets todo.org # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/README.md b/README.md index aa801952..ad160e06 100644 --- a/README.md +++ b/README.md @@ -25,9 +25,9 @@ This tap: ## Quick Start -1. Install +1. Install the tap locally - pip install tap-shopify + pip install -e . 2. Create the config file diff --git a/setup.py b/setup.py index 078bdc94..56ae1ee4 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-shopify", - version="1.4.0", + version="1.4.13", description="Singer.io tap for extracting Shopify data", author="Stitch", url="http://github.com/singer-io/tap-shopify", @@ -11,7 +11,7 @@ python_requires='>=3.5.2', py_modules=["tap_shopify"], install_requires=[ - "ShopifyAPI==8.4.1", + "ShopifyAPI==12.1.0", "singer-python==5.12.1", ], extras_require={ diff --git a/tap_shopify/__init__.py b/tap_shopify/__init__.py index e864e2b0..b368b5bb 100644 --- a/tap_shopify/__init__.py +++ b/tap_shopify/__init__.py @@ -5,6 +5,7 @@ import time import math import copy +import logging import pyactiveresource import shopify @@ -14,16 +15,20 @@ from singer import Transformer from tap_shopify.context import Context from tap_shopify.exceptions import ShopifyError +from tap_shopify.streams.base import shopify_error_handling import tap_shopify.streams # Load stream objects into Context -REQUIRED_CONFIG_KEYS = ["shop", "api_key"] +REQUIRED_CONFIG_KEYS = ["shop"] LOGGER = singer.get_logger() SDC_KEYS = {'id': 'integer', 'name': 'string', 'myshopify_domain': 'string'} +logging.getLogger('backoff').setLevel(logging.CRITICAL) + +@shopify_error_handling def initialize_shopify_client(): - api_key = Context.config['api_key'] + api_key = Context.config.get('access_token', Context.config.get("api_key")) shop = Context.config['shop'] - version = '2021-04' + version = Context.config.get('api_version', '2022-04') session = shopify.Session(shop, version, api_key) shopify.ShopifyResource.activate_session(session) # Shop.current() makes a call for shop details with provided shop and api_key diff --git a/tap_shopify/schemas/discount_codes.json b/tap_shopify/schemas/discount_codes.json new file mode 100644 index 00000000..052dbf28 --- /dev/null +++ b/tap_shopify/schemas/discount_codes.json @@ -0,0 +1,532 @@ +{ + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "order_id": { + "type": [ + "null", + "integer" + ] + }, + "status": { + "type": [ + "null", + "string" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "service": { + "type": [ + "null", + "string" + ] + }, + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "tracking_company": { + "type": [ + "null", + "string" + ] + }, + "shipment_status": { + "type": [ + "null", + "string" + ] + }, + "location_id": { + "type": [ + "null", + "integer" + ] + }, + "tracking_number": { + "type": [ + "null", + "string" + ] + }, + "tracking_numbers": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "tracking_url": { + "type": [ + "null", + "string" + ] + }, + "tracking_urls": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "receipt": { + "type": [ + "object", + "null" + ], + "properties": { + "testcase": { + "type": [ + "boolean", + "null" + ] + }, + "authorization": { + "type": [ + "string", + "null" + ] + } + } + }, + "line_items": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "id": { + "type": [ + "integer", + "null" + ] + }, + "variant_id": { + "type": [ + "integer", + "null" + ] + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "quantity": { + "type": [ + "integer", + "null" + ] + }, + "sku": { + "type": [ + "string", + "null" + ] + }, + "variant_title": { + "type": [ + "string", + "null" + ] + }, + "vendor": { + "type": [ + "string", + "null" + ] + }, + "fulfillment_service": { + "type": [ + "string", + "null" + ] + }, + "product_id": { + "type": [ + "integer", + "null" + ] + }, + "requires_shipping": { + "type": [ + "boolean", + "null" + ] + }, + "taxable": { + "type": [ + "boolean", + "null" + ] + }, + "gift_card": { + "type": [ + "boolean", + "null" + ] + }, + "name": { + "type": [ + "string", + "null" + ] + }, + "variant_inventory_management": { + "type": [ + "string", + "null" + ] + }, + "properties": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "name": { + "type": [ + "string", + "null" + ] + }, + "value": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "product_exists": { + "type": [ + "boolean", + "null" + ] + }, + "fulfillable_quantity": { + "type": [ + "integer", + "null" + ] + }, + "grams": { + "type": [ + "integer", + "null" + ] + }, + "price": { + "type": [ + "string", + "null" + ] + }, + "total_discount": { + "type": [ + "string", + "null" + ] + }, + "fulfillment_status": { + "type": [ + "string", + "null" + ] + }, + "price_set": { + "type": [ + "object", + "null" + ], + "properties": { + "shop_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + }, + "presentment_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "total_discount_set": { + "type": [ + "object", + "null" + ], + "properties": { + "shop_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + }, + "presentment_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "discount_allocations": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "discount_application_index": { + "type": [ + "integer", + "null" + ] + }, + "amount_set": { + "type": [ + "object", + "null" + ], + "properties": { + "shop_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + }, + "presentment_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + } + }, + "admin_graphql_api_id": { + "type": [ + "string", + "null" + ] + }, + "tax_lines": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "object", + "null" + ], + "properties": { + "price": { + "type": [ + "boolean", + "null" + ] + }, + "rate": { + "type": [ + "number", + "null" + ] + }, + "title": { + "type": [ + "boolean", + "null" + ] + }, + "price_set": { + "type": [ + "object", + "null" + ], + "properties": { + "shop_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + }, + "presentment_money": { + "type": [ + "object", + "null" + ], + "properties": { + "amount": { + "type": [ + "string", + "null" + ] + }, + "currency_code": { + "type": [ + "string", + "null" + ] + } + } + } + } + } + } + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/tap_shopify/schemas/events_products.json b/tap_shopify/schemas/events_products.json new file mode 100644 index 00000000..9c00fe30 --- /dev/null +++ b/tap_shopify/schemas/events_products.json @@ -0,0 +1,55 @@ +{ + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "subject_id": { + "type": [ + "null", + "integer" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "subject_type": { + "type": [ + "null", + "string" + ] + }, + "verb": { + "type": [ + "null", + "string" + ] + }, + "message": { + "type": [ + "null", + "string" + ] + }, + "author": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" + } + \ No newline at end of file diff --git a/tap_shopify/schemas/incoming_items.json b/tap_shopify/schemas/incoming_items.json new file mode 100644 index 00000000..49be8887 --- /dev/null +++ b/tap_shopify/schemas/incoming_items.json @@ -0,0 +1,17 @@ +{ + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "incoming": { + "type": [ + "null", + "integer" + ] + } + }, + "type": "object" +} diff --git a/tap_shopify/schemas/inventory_levels.json b/tap_shopify/schemas/inventory_levels.json index fb9e6f10..b0288fac 100644 --- a/tap_shopify/schemas/inventory_levels.json +++ b/tap_shopify/schemas/inventory_levels.json @@ -13,6 +13,9 @@ "location_id": { "type": ["null", "integer"] }, + "incoming": { + "type": ["null", "integer"] + }, "admin_graphql_api_id": { "type": ["null", "string"] } diff --git a/tap_shopify/schemas/price_rules.json b/tap_shopify/schemas/price_rules.json new file mode 100644 index 00000000..49c4272c --- /dev/null +++ b/tap_shopify/schemas/price_rules.json @@ -0,0 +1,283 @@ +{ + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "value_type": { + "type": [ + "null", + "string" + ] + }, + "value": { + "type": [ + "null", + "string" + ] + }, + "customer_selection": { + "type": [ + "null", + "string" + ] + }, + "target_type": { + "type": [ + "null", + "string" + ] + }, + "target_selection": { + "type": [ + "null", + "string" + ] + }, + "allocation_method": { + "type": [ + "null", + "string" + ] + }, + "allocation_limit": { + "type": [ + "null", + "string" + ] + }, + "once_per_customer": { + "type": [ + "null", + "boolean" + ] + }, + "usage_limit": { + "type": [ + "null", + "integer" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "starts_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "ends_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "created_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "entitled_collection_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "entitled_country_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "entitled_product_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "entitled_variant_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "prerequisite_customer_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "prerequisite_quantity_range": { + "type": [ + "object", + "null" + ], + "properties": { + "greater_than_or_equal_to": { + "type": [ + "integer", + "null" + ] + } + } + }, + "prerequisite_saved_search_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "prerequisite_shipping_price_range": { + "type": [ + "object", + "null" + ], + "properties": { + "less_than_or_equal_to": { + "type": [ + "string", + "null" + ] + } + } + } + }, + "prerequisite_subtotal_range": { + "type": [ + "object", + "null" + ], + "properties": { + "greater_than_or_equal_to": { + "type": [ + "string", + "null" + ] + } + } + }, + "prerequisite_to_entitlement_purchase": { + "type": [ + "object", + "null" + ], + "properties": { + "prerequisite_amount": { + "type": [ + "string", + "null" + ] + } + } + }, + "prerequisite_product_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "prerequisite_variant_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "prerequisite_collection_ids": { + "type": [ + "array", + "null" + ], + "items": { + "type": [ + "null", + "integer" + ] + } + }, + "prerequisite_to_entitlement_quantity_ratio": { + "type": [ + "object", + "null" + ], + "properties": { + "entitled_quantity": { + "type": [ + "integer", + "null" + ] + }, + "prerequisite_quantity": { + "type": [ + "integer", + "null" + ] + } + } + }, + "type": "object" +} \ No newline at end of file diff --git a/tap_shopify/schemas/product_category.json b/tap_shopify/schemas/product_category.json new file mode 100644 index 00000000..684d6ebe --- /dev/null +++ b/tap_shopify/schemas/product_category.json @@ -0,0 +1,20 @@ +{ + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "category_id": { + "type": ["string", "null"] + }, + "full_name": { + "type": ["string", "null"] + }, + "is_leaf": { + "type": ["boolean", "null"] + }, + "is_root": { + "type": ["boolean", "null"] + } + } +} diff --git a/tap_shopify/schemas/shop.json b/tap_shopify/schemas/shop.json new file mode 100644 index 00000000..6be14d12 --- /dev/null +++ b/tap_shopify/schemas/shop.json @@ -0,0 +1,185 @@ +{ + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + }, + "domain": { + "type": [ + "null", + "string" + ] + }, + "province": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "address1": { + "type": [ + "null", + "string" + ] + }, + "zip": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "latitude": { + "type": [ + "null", + "number" + ] + }, + "longitude": { + "type": [ + "null", + "number" + ] + }, + "money_in_emails_format": { + "type": [ + "null", + "string" + ] + }, + "money_with_currency_in_emails_format": { + "type": [ + "null", + "string" + ] + }, + "eligible_for_payments": { + "type": [ + "null", + "boolean" + ] + }, + "requires_extra_payments_agreement": { + "type": [ + "null", + "boolean" + ] + }, + "password_enabled": { + "type": [ + "null", + "boolean" + ] + }, + "has_storefront": { + "type": [ + "null", + "boolean" + ] + }, + "eligible_for_card_reader_giveaway": { + "type": [ + "null", + "boolean" + ] + }, + "finances": { + "type": [ + "null", + "boolean" + ] + }, + "primary_location_id": { + "type": [ + "null", + "integer" + ] + }, + "cookie_consent_level": { + "type": [ + "null", + "string" + ] + }, + "visitor_tracking_consent_preference": { + "type": [ + "null", + "string" + ] + }, + "checkout_api_supported": { + "type": [ + "null", + "boolean" + ] + }, + "multi_location_enabled": { + "type": [ + "null", + "boolean" + ] + }, + "setup_required": { + "type": [ + "null", + "boolean" + ] + }, + "pre_launch_enabled": { + "type": [ + "null", + "boolean" + ] + }, + "enabled_presentment_currencies": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "currency": { + "type": [ + "null", + "string" + ] + } + }, + "type": "object" +} diff --git a/tap_shopify/schemas/smart_collections.json b/tap_shopify/schemas/smart_collections.json new file mode 100644 index 00000000..150b3ae5 --- /dev/null +++ b/tap_shopify/schemas/smart_collections.json @@ -0,0 +1,143 @@ +{ + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "handle": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "updated_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "body_html": { + "type": [ + "null", + "string" + ] + }, + "published_at": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "sort_order": { + "type": [ + "null", + "string" + ] + }, + "template_suffix": { + "type": [ + "null", + "string" + ] + }, + "disjunctive": { + "type": [ + "null", + "boolean" + ] + }, + "published_scope": { + "type": [ + "null", + "string" + ] + }, + "admin_graphql_api_id": { + "type": [ + "null", + "string" + ] + }, + "rules": { + "items": { + "properties": { + "column": { + "type": [ + "null", + "string" + ] + }, + "relation": { + "type": [ + "null", + "string" + ] + }, + "condition": { + "type": [ + "null", + "string" + ] + } + }, + "type": [ + "null", + "object" + ] + }, + "type": [ + "null", + "array" + ] + }, + "image": { + "properties": { + "alt": { + "type": [ + "null", + "string" + ] + }, + "src": { + "type": [ + "null", + "string" + ] + }, + "width": { + "type": [ + "null", + "integer" + ] + }, + "created_at": { + "type": [ + "null", + "string" + ] + }, + "height": { + "type": [ + "null", + "integer" + ] + } + }, + "type": [ + "null", + "object" + ] + } + }, + "type": "object" +} diff --git a/tap_shopify/streams/__init__.py b/tap_shopify/streams/__init__.py index 0bed4024..0daa7e3e 100644 --- a/tap_shopify/streams/__init__.py +++ b/tap_shopify/streams/__init__.py @@ -10,3 +10,10 @@ import tap_shopify.streams.locations import tap_shopify.streams.inventory_levels import tap_shopify.streams.inventory_items +import tap_shopify.streams.shop +import tap_shopify.streams.price_rules +import tap_shopify.streams.discount_codes +import tap_shopify.streams.incoming_items +import tap_shopify.streams.events_products +import tap_shopify.streams.smart_collections +import tap_shopify.streams.product_category \ No newline at end of file diff --git a/tap_shopify/streams/base.py b/tap_shopify/streams/base.py index 04488bf6..0d4da608 100644 --- a/tap_shopify/streams/base.py +++ b/tap_shopify/streams/base.py @@ -2,6 +2,7 @@ import functools import math import sys +import time import backoff import pyactiveresource @@ -17,10 +18,10 @@ # We've observed 500 errors returned if this is too large (30 days was too # large for a customer) -DATE_WINDOW_SIZE = 1 +DATE_WINDOW_SIZE = 365 # We will retry a 500 error a maximum of 5 times before giving up -MAX_RETRIES = 5 +MAX_RETRIES = 10 def is_not_status_code_fn(status_code): def gen_fn(exc): @@ -35,7 +36,7 @@ def leaky_bucket_handler(details): details['wait']) def retry_handler(details): - LOGGER.info("Received 500 or retryable error -- Retry %s/%s", + LOGGER.info("Received 500 or retryable -- Retry %s/%s", details['tries'], MAX_RETRIES) #pylint: disable=unused-argument @@ -47,23 +48,21 @@ def retry_after_wait_gen(**kwargs): # Retry-After is an undocumented header. But honoring # it was proven to work in our spikes. # It's been observed to come through as lowercase, so fallback if not present - sleep_time_str = resp.headers.get('Retry-After', resp.headers.get('retry-after')) - yield math.floor(float(sleep_time_str)) + sleep_time_str = resp.headers.get('Retry-After', resp.headers.get('retry-after',4)) + yield math.ceil(float(sleep_time_str)) def shopify_error_handling(fnc): @backoff.on_exception(backoff.expo, (pyactiveresource.connection.ServerError, pyactiveresource.formats.Error, - simplejson.scanner.JSONDecodeError), - giveup=is_not_status_code_fn(range(500, 599)), + simplejson.scanner.JSONDecodeError, + Exception), on_backoff=retry_handler, max_tries=MAX_RETRIES) @backoff.on_exception(retry_after_wait_gen, pyactiveresource.connection.ClientError, giveup=is_not_status_code_fn([429]), - on_backoff=leaky_bucket_handler, - # No jitter as we want a constant value - jitter=None) + on_backoff=leaky_bucket_handler) @functools.wraps(fnc) def wrapper(*args, **kwargs): return fnc(*args, **kwargs) @@ -92,6 +91,7 @@ def __init__(self): self.results_per_page = Context.get_results_per_page(RESULTS_PER_PAGE) def get_bookmark(self): + time.sleep(0.5) bookmark = (singer.get_bookmark(Context.state, # name is overridden by some substreams self.name, diff --git a/tap_shopify/streams/discount_codes.py b/tap_shopify/streams/discount_codes.py new file mode 100644 index 00000000..b3907106 --- /dev/null +++ b/tap_shopify/streams/discount_codes.py @@ -0,0 +1,56 @@ +import shopify +import singer +from singer.utils import strftime, strptime_to_utc +from tap_shopify.context import Context +from tap_shopify.streams.base import (Stream, + shopify_error_handling) + +LOGGER = singer.get_logger() + +DISCOUNT_CODES_RESULTS_PER_PAGE = 100 + + +class DiscountCodes(Stream): + name = 'discount_codes' + replication_key = 'created_at' + replication_object = shopify.DiscountCode + + @shopify_error_handling + def call_api_for_discount_codes(self, parent_object): + return self.replication_object.find( + limit=DISCOUNT_CODES_RESULTS_PER_PAGE, + price_rule_id=parent_object.id, + ) + + def get_discount_codes(self, parent_object): + page = self.call_api_for_discount_codes(parent_object) + yield from page + + while page.has_next_page(): + page = page.next_page() + yield from page + + def get_objects(self): + selected_parent = Context.stream_objects['price_rules']() + selected_parent.name = "discount_code_price_rules" + + for parent_object in selected_parent.get_objects(): + discount_codes = self.get_discount_codes(parent_object) + for discount_code in discount_codes: + yield discount_code + + def sync(self): + bookmark = self.get_bookmark() + self.max_bookmark = bookmark + for discount_code in self.get_objects(): + discount_code_dict = discount_code.to_dict() + replication_value = strptime_to_utc(discount_code_dict[self.replication_key]) + if replication_value >= bookmark: + yield discount_code_dict + if replication_value > self.max_bookmark: + self.max_bookmark = replication_value + + self.update_bookmark(strftime(self.max_bookmark)) + + +Context.stream_objects['discount_codes'] = DiscountCodes diff --git a/tap_shopify/streams/events_products.py b/tap_shopify/streams/events_products.py new file mode 100644 index 00000000..51c8d1e5 --- /dev/null +++ b/tap_shopify/streams/events_products.py @@ -0,0 +1,54 @@ +import shopify +import singer +from singer.utils import strftime, strptime_to_utc +from tap_shopify.context import Context +from tap_shopify.streams.base import (Stream, + shopify_error_handling) + +LOGGER = singer.get_logger() + +EVENTS_RESULTS_PER_PAGE = 100 + + +class EventsProducts(Stream): + name = 'events_products' + replication_key = 'created_at' + replication_object = shopify.Event + + @shopify_error_handling + def call_api_for_events_products(self): + return self.replication_object.find( + limit=EVENTS_RESULTS_PER_PAGE, + filter="Product", + # verb = "destroy", + created_at_min = self.get_bookmark() + ) + + def get_events_products(self, ): + page = self.call_api_for_events_products() + yield from page + + while page.has_next_page(): + page = page.next_page() + yield from page + + def get_objects(self): + events_products = self.get_events_products() + for events_product in events_products: + yield events_product + + def sync(self): + bookmark = self.get_bookmark() + self.max_bookmark = bookmark + for events_product in self.get_objects(): + events_product_dict = events_product.to_dict() + replication_value = strptime_to_utc(events_product_dict[self.replication_key]) + if replication_value >= bookmark: + yield events_product_dict + if replication_value > self.max_bookmark: + self.max_bookmark = replication_value + + self.update_bookmark(strftime(self.max_bookmark)) + + +Context.stream_objects['events_products'] = EventsProducts diff --git a/tap_shopify/streams/incoming_items.py b/tap_shopify/streams/incoming_items.py new file mode 100644 index 00000000..c0325184 --- /dev/null +++ b/tap_shopify/streams/incoming_items.py @@ -0,0 +1,57 @@ +import os +import sys +import shopify +import singer +import json +from singer.utils import strftime, strptime_to_utc +from tap_shopify.context import Context +from tap_shopify.streams.base import (Stream, + shopify_error_handling) + +LOGGER = singer.get_logger() + + +class HiddenPrints: + def __enter__(self): + self._original_stdout = sys.stdout + sys.stdout = open(os.devnull, 'w') + + def __exit__(self, exc_type, exc_val, exc_tb): + sys.stdout.close() + sys.stdout = self._original_stdout + + +class IncomingItems(Stream): + name = 'incoming_items' + replication_key = 'createdAt' + gql_query = "query inventoryLevel($id: ID!){inventoryLevel(id: $id){id, incoming, createdAt}}" + + @shopify_error_handling + def call_api_for_incoming_items(self, parent_object): + gql_client = shopify.GraphQL() + with HiddenPrints(): + response = gql_client.execute(self.gql_query, dict(id=parent_object.admin_graphql_api_id)) + return json.loads(response) + + def get_objects(self): + selected_parent = Context.stream_objects['inventory_levels']() + selected_parent.name = "inventory_levels" + + for parent_object in selected_parent.get_objects(): + incoming_item = self.call_api_for_incoming_items(parent_object) + yield incoming_item["data"].get("inventoryLevel") + + def sync(self): + bookmark = self.get_bookmark() + self.max_bookmark = bookmark + for incoming_item in self.get_objects(): + replication_value = strptime_to_utc(incoming_item[self.replication_key]) + if replication_value >= bookmark: + yield incoming_item + if replication_value > self.max_bookmark: + self.max_bookmark = replication_value + + self.update_bookmark(strftime(self.max_bookmark)) + + +Context.stream_objects['incoming_items'] = IncomingItems diff --git a/tap_shopify/streams/inventory_items.py b/tap_shopify/streams/inventory_items.py index 55fe047a..ffd62a2b 100644 --- a/tap_shopify/streams/inventory_items.py +++ b/tap_shopify/streams/inventory_items.py @@ -3,6 +3,7 @@ from singer.utils import strftime,strptime_to_utc from tap_shopify.streams.base import (Stream, shopify_error_handling) from tap_shopify.context import Context +import time LOGGER = singer.get_logger() @@ -14,6 +15,7 @@ class InventoryItems(Stream): @shopify_error_handling def get_inventory_items(self, inventory_items_ids): + time.sleep(0.5) return self.replication_object.find( ids=inventory_items_ids, limit=RESULTS_PER_PAGE) @@ -38,17 +40,7 @@ def get_objects(self): yield inventory_item def sync(self): - bookmark = self.get_bookmark() - max_bookmark = bookmark for inventory_item in self.get_objects(): - inventory_item_dict = inventory_item.to_dict() - replication_value = strptime_to_utc(inventory_item_dict[self.replication_key]) - if replication_value >= bookmark: - yield inventory_item_dict - - if replication_value > max_bookmark: - max_bookmark = replication_value - - self.update_bookmark(strftime(max_bookmark)) + yield inventory_item.to_dict() Context.stream_objects['inventory_items'] = InventoryItems diff --git a/tap_shopify/streams/inventory_levels.py b/tap_shopify/streams/inventory_levels.py index 6cfdb35e..a4d68fb7 100644 --- a/tap_shopify/streams/inventory_levels.py +++ b/tap_shopify/streams/inventory_levels.py @@ -1,3 +1,4 @@ +import json import shopify from singer.utils import strftime, strptime_to_utc from tap_shopify.streams.base import (Stream, @@ -18,13 +19,17 @@ def api_call_for_inventory_levels(self, parent_object_id, bookmark): limit = RESULTS_PER_PAGE, location_ids=parent_object_id ) + + @shopify_error_handling + def get_next_page(self, inventory_page): + return inventory_page.next_page() def get_inventory_levels(self, parent_object, bookmark): inventory_page = self.api_call_for_inventory_levels(parent_object, bookmark) yield from inventory_page while inventory_page.has_next_page(): - inventory_page = inventory_page.next_page() + inventory_page = self.get_next_page(inventory_page) yield from inventory_page def get_objects(self): diff --git a/tap_shopify/streams/locations.py b/tap_shopify/streams/locations.py index 3a61d0a2..1c0d2274 100644 --- a/tap_shopify/streams/locations.py +++ b/tap_shopify/streams/locations.py @@ -8,8 +8,11 @@ class Locations(Stream): replication_object = shopify.Location @shopify_error_handling + def api_call_for_locations_data(self): + return self.replication_object.find() + def get_locations_data(self): - location_page = self.replication_object.find() + location_page = self.api_call_for_locations_data() yield from location_page while location_page.has_next_page(): diff --git a/tap_shopify/streams/metafields.py b/tap_shopify/streams/metafields.py index f96b3a06..620fb882 100644 --- a/tap_shopify/streams/metafields.py +++ b/tap_shopify/streams/metafields.py @@ -1,6 +1,7 @@ import json import shopify import singer +import time from tap_shopify.context import Context from tap_shopify.streams.base import (Stream, @@ -19,6 +20,7 @@ def get_selected_parents(): def get_metafields(parent_object, since_id): # This call results in an HTTP request - the parent object never has a # cache of this data so we have to issue that request. + time.sleep(0.5) return parent_object.metafields( limit=Context.get_results_per_page(RESULTS_PER_PAGE), since_id=since_id) diff --git a/tap_shopify/streams/price_rules.py b/tap_shopify/streams/price_rules.py new file mode 100644 index 00000000..47561bc5 --- /dev/null +++ b/tap_shopify/streams/price_rules.py @@ -0,0 +1,11 @@ +import shopify + +from tap_shopify.streams.base import Stream +from tap_shopify.context import Context + + +class PriceRules(Stream): + name = 'price_rules' + replication_object = shopify.PriceRule + +Context.stream_objects['price_rules'] = PriceRules diff --git a/tap_shopify/streams/product_category.py b/tap_shopify/streams/product_category.py new file mode 100644 index 00000000..b5517c41 --- /dev/null +++ b/tap_shopify/streams/product_category.py @@ -0,0 +1,80 @@ +import json +import os +import sys +import shopify +import singer +from singer.utils import strftime, strptime_to_utc + +from tap_shopify.context import Context +from tap_shopify.streams.base import Stream, shopify_error_handling + +LOGGER = singer.get_logger() + + +class HiddenPrints: + def __enter__(self): + self._original_stdout = sys.stdout + sys.stdout = open(os.devnull, 'w') + + def __exit__(self, exc_type, exc_val, exc_tb): + sys.stdout.close() + sys.stdout = self._original_stdout + + +class ProductCategory(Stream): + name = 'product_category' + replication_key = 'createdAt' + gql_query = """ + query product($id: ID!){ + product(id: $id){ + id, + productType, + createdAt, + productCategory{ + productTaxonomyNode{ + id, + fullName, + isLeaf, + isRoot + + } + } + } + } + """ + + @shopify_error_handling + def call_api_for_product_categories(self, parent_object): + gql_client = shopify.GraphQL() + with HiddenPrints(): + response = gql_client.execute(self.gql_query, dict(id=parent_object.admin_graphql_api_id)) + return json.loads(response) + + def get_objects(self): + selected_parent = Context.stream_objects['products']() + selected_parent.name = "products_categories" + for parent_object in selected_parent.get_objects(): + product_category = self.call_api_for_product_categories(parent_object) + item = product_category["data"].get("product") + if item.get('productCategory'): + item['category_id'] = item['productCategory']['productTaxonomyNode']['id'] + item['full_name'] = item['productCategory']['productTaxonomyNode']['fullName'] + item['is_leaf'] = item['productCategory']['productTaxonomyNode']['isLeaf'] + item['is_root'] = item['productCategory']['productTaxonomyNode']['isRoot'] + yield item + + def sync(self): + bookmark = self.get_bookmark() + self.max_bookmark = bookmark + for incoming_item in self.get_objects(): + replication_value = strptime_to_utc(incoming_item[self.replication_key]) + if replication_value >= bookmark: + + yield incoming_item + if replication_value > self.max_bookmark: + self.max_bookmark = replication_value + + self.update_bookmark(strftime(self.max_bookmark)) + + +Context.stream_objects['product_category'] = ProductCategory diff --git a/tap_shopify/streams/shop.py b/tap_shopify/streams/shop.py new file mode 100644 index 00000000..0c1a4b24 --- /dev/null +++ b/tap_shopify/streams/shop.py @@ -0,0 +1,36 @@ +import shopify +from singer import utils +from tap_shopify.streams.base import (Stream, shopify_error_handling) +from tap_shopify.context import Context + +class Shop(Stream): + name = 'shop' + replication_object = shopify.Shop + + @shopify_error_handling + def api_call_for_shop_data(self): + return self.replication_object.current() + + def get_shop_data(self): + shop_page = [self.api_call_for_shop_data()] + yield from shop_page + + def sync(self): + bookmark = self.get_bookmark() + max_bookmark = bookmark + + for shop in self.get_shop_data(): + + shop_dict = shop.to_dict() + replication_value = utils.strptime_to_utc(shop_dict[self.replication_key]) + + if replication_value >= bookmark: + yield shop_dict + + # update max bookmark if "replication_value" of current shop is greater + if replication_value > max_bookmark: + max_bookmark = replication_value + + self.update_bookmark(utils.strftime(max_bookmark)) + +Context.stream_objects['shop'] = Shop diff --git a/tap_shopify/streams/smart_collections.py b/tap_shopify/streams/smart_collections.py new file mode 100644 index 00000000..2ebb7236 --- /dev/null +++ b/tap_shopify/streams/smart_collections.py @@ -0,0 +1,11 @@ +import shopify + +from tap_shopify.streams.base import Stream +from tap_shopify.context import Context + + +class SmartCollections(Stream): + name = 'smart_collections' + replication_object = shopify.SmartCollection + +Context.stream_objects['smart_collections'] = SmartCollections