diff --git a/tap_shopify/streams.py b/tap_shopify/streams.py index b793d3f..a5cd364 100644 --- a/tap_shopify/streams.py +++ b/tap_shopify/streams.py @@ -379,16 +379,23 @@ def __init__(self, *args, **kwargs): query_entry = kwargs.pop("query") self.name = query_entry["name"] self._configured_query = query_entry["query"] - self.primary_keys = query_entry.get("primary_keys") or [] - self.replication_key = query_entry.get("replication_key") super().__init__(*args, **kwargs) + # Must be set after super().__init__(), which resets + # self._primary_keys / self._replication_key to their defaults. + self.primary_keys = query_entry.get("primary_keys") or [] + self.replication_key = query_entry.get("replication_key") + @cached_property def schema(self) -> dict: """Discover schema by probing the API with a 1-day window.""" - probe_query = re.sub(r"\bSINCE\s+\S+", "", self._configured_query, flags=re.IGNORECASE) - probe_query = re.sub(r"\bUNTIL\s+\S+", "", probe_query, flags=re.IGNORECASE).strip() + probe_query = re.sub( + r"\bSINCE\s+\S+", "", self._configured_query, flags=re.IGNORECASE + ) + probe_query = re.sub( + r"\bUNTIL\s+\S+", "", probe_query, flags=re.IGNORECASE + ).strip() probe_query += " SINCE -1d UNTIL -0d" graphql = self._GRAPHQL_TEMPLATE.format(shopifyql=json.dumps(probe_query)) @@ -400,7 +407,13 @@ def schema(self) -> dict: response.raise_for_status() data = response.json() - result = data.get("shopifyqlQuery") or {} + gql_errors = data.get("errors") + if gql_errors: + raise RuntimeError( + f"GraphQL errors during schema discovery for '{self.name}': {gql_errors}" + ) + + result = (data.get("data") or {}).get("shopifyqlQuery") or {} parse_errors = result.get("parseErrors") or [] if parse_errors: raise RuntimeError( @@ -408,7 +421,17 @@ def schema(self) -> dict: ) columns = (result.get("tableData") or {}).get("columns") or [] - props = [th.Property(col["name"], th.StringType) for col in columns] + props = [ + th.Property( + col["name"], + ( + th.DateTimeType + if col["name"] == self.replication_key + else th.StringType + ), + ) + for col in columns + ] return th.PropertiesList(*props).to_dict() def prepare_request_payload(self, context, next_page_token): @@ -441,7 +464,7 @@ def parse_response(self, response): if gql_errors: raise RuntimeError(f"GraphQL errors: {gql_errors}") - result = data.get("shopifyqlQuery") or {} + result = (data.get("data") or {}).get("shopifyqlQuery") or {} parse_errors = result.get("parseErrors") or [] if parse_errors: @@ -462,5 +485,14 @@ def get_url_params(self, context, next_page_token): return {} def post_process(self, row, context=None): - """Return the row as-is — deduplication is not applicable here.""" + """Normalize the replication key to a full timestamp. + + ShopifyQL returns date-only strings (e.g. "2026-07-03") for TIMESERIES + columns, but the declared date-time schema type — and BigQuery's + TIMESTAMP column type on load — require a full timestamp value. + """ + if self.replication_key and self.replication_key in row: + value = row[self.replication_key] + if isinstance(value, str) and re.fullmatch(r"\d{4}-\d{2}-\d{2}", value): + row[self.replication_key] = f"{value}T00:00:00Z" return row