adjustments for the first beta

README.md

@@ -1,6 +1,9 @@
 # tap-shopify
 
-`tap-shopify` is a Singer tap for Shopify.
+`tap-shopify` is a Singer tap for Shopify that uses the GraphQL API. This tap is currently in beta. The schema 
+definition is dynamic and retrieves all nested fields that are not connections to other objects. However, it 
+may require adjustments. Since we don’t have a real use case for the tap yet, we couldn’t determine the best 
+approach. We welcome contributions and feedback
 
 Built with the [Meltano Tap SDK](https://sdk.meltano.com) for Singer Taps.
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "GraphQL-tap-shopify"
-version = "1.0.0"
+version = "0.1.0"
 description = "`tap-shopify` is a Singer tap for Shopify, built with the Meltano Singer SDK."
 readme = "README.md"
 authors = ["Josué Sehnem"]

tap_shopify/auth.py

@@ -5,4 +5,4 @@
 
 # The SingletonMeta metaclass makes the streams reuse the same authenticator instance.
 class ShopifyAuthenticator(APIKeyAuthenticator, metaclass=SingletonMeta):
- """Authenticator class for tap_shopify."""
+ """Authenticator class for tap_shopify."""

tap_shopify/client.py

@@ -7,12 +7,12 @@
 from typing import Any, Optional
 
 from singer_sdk import typing as th
-from singer_sdk.streams import GraphQLStream
-from tap_shopify.auth import ShopifyAuthenticator
-from tap_shopify.paginator import ShopifyPaginator
 from singer_sdk.pagination import SinglePagePaginator
+from singer_sdk.streams import GraphQLStream
 
+from tap_shopify.auth import ShopifyAuthenticator
 from tap_shopify.gql_queries import schema_query
+from tap_shopify.paginator import ShopifyPaginator
 
 
 def verify_recursion(func):
@@ -40,6 +40,7 @@ class ShopifyStream(GraphQLStream):
  single_object_params = None
  ignore_objs = []
  _requests_session = None
+ nested_connections = []
 
  @property
  def url_base(self) -> str:
@@ -130,7 +131,11 @@ def get_fields_schema(self, fields) -> dict:
  for field in fields:
  field_name = field["name"]
  # Ignore all the fields that need arguments
- if field.get("args") or field.get("isDeprecated"):
+ if field.get("isDeprecated") and self.config.get("ignore_deprecated"):
+ continue
+ if field.get("args"):
+ if field["args"][0]["name"] == "first":
+ self.nested_connections.append(field_name)
  continue
  if field_name in self.ignore_objs:
  continue

tap_shopify/client_bulk.py

@@ -7,11 +7,11 @@
 import requests
 import simplejson
 from singer_sdk.helpers.jsonpath import extract_jsonpath
+from singer_sdk.pagination import SinglePagePaginator
 
 from tap_shopify.client import ShopifyStream
 from tap_shopify.exceptions import InvalidOperation, OperationFailed
 from tap_shopify.gql_queries import bulk_query, bulk_query_status, simple_query
-from singer_sdk.pagination import SinglePagePaginator
 
 
 class shopifyBulkStream(ShopifyStream):

tap_shopify/client_gql.py

@@ -37,7 +37,7 @@ def get_url_params(
  ) -> Dict[str, Any]:
  """Return a dictionary of values to be used in URL parameterization."""
  params = {}
- 
+
  if next_page_token:
  params.update(next_page_token)
  else:

tap_shopify/paginator.py

@@ -1,15 +1,16 @@
-from singer_sdk.helpers.jsonpath import extract_jsonpath
-from singer_sdk.pagination import BaseAPIPaginator
 import math
-import requests
+from functools import cached_property
 from time import sleep
+
+import requests
 from requests import Response
-from functools import cached_property
+from singer_sdk.helpers.jsonpath import extract_jsonpath
+from singer_sdk.pagination import BaseAPIPaginator
 
 
 class ShopifyPaginator(BaseAPIPaginator):
  """shopify paginator class."""
- 
+
  query_cost = None
  available_points = None
  restore_rate = None
@@ -43,19 +44,19 @@ def page_size(self) -> int:
  else:
  pages = 250 if pages > 250 else pages
  return int(pages)
- 
+
  def query_name(self, response_json) -> str:
  """Set or return the GraphQL query name."""
  return list(response_json.get("data"))[0]
- 
+
  def get_next(self, response: requests.Response):
  """Get the next pagination value."""
 
  response_json = response.json()
  query_name = self.query_name(response_json)
 
  cost = response_json["extensions"].get("cost")
- 
+
  self.query_cost = cost.get("requestedQueryCost")
  self.available_points = cost["throttleStatus"].get("currentlyAvailable")
  self.restore_rate = cost["throttleStatus"].get("restoreRate")
@@ -68,9 +69,9 @@ def get_next(self, response: requests.Response):
  cursor_json_path = f"$.data.{query_name}.edges[-1].cursor"
  all_matches = extract_jsonpath(cursor_json_path, response_json)
  return next(all_matches, None)
- 
+
  return None
- 
+
  @property
  def current_value(self):
  """Get the current pagination value.

tap_shopify/streams.py

@@ -36,7 +36,6 @@ class CustomCollectionsStream(ShopifyStream):
  name = "collections"
  gql_type = "Collection"
  query_name = "collections"
- # ignore_objs = ["customer", "paymentCollectionDetails"]
  primary_keys = ["id"]
  replication_key = "updatedAt"
 

tap_shopify/tap.py

@@ -20,30 +20,36 @@ class TapShopify(Tap):
  th.StringType,
  required=True,
  secret=True,
- description="The token to authenticate against the API service",
+ description="The token to authenticate against the API service.",
  ),
  th.Property(
  "store",
  th.StringType,
  required=True,
- description="The shopify shop name",
+ description="The shopify shop name.",
  ),
  th.Property(
  "api_version",
  th.StringType,
  default="2023-04",
- description="The version of the API to use",
+ description="The version of the API to use.",
  ),
  th.Property(
  "start_date",
  th.DateTimeType,
- description="The earliest record date to sync",
+ description="The earliest record date to sync.",
  ),
  th.Property(
  "bulk",
  th.BooleanType,
  default=False,
- description="To use the bulk API or not",
+ description="To use the bulk API or not.",
+ ),
+ th.Property(
+ "ignore_deprecated",
+ th.BooleanType,
+ default=True,
+ description="To ignore deprecated fields or not.",
  ),
  ).to_dict()
 

tests/test_core.py

@@ -10,11 +10,11 @@
  "start_date": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"),
  "auth_token": "shpat_ffffffffffffffffffffffffffffffff",
  "shop": "teststore",
- "bulk": False
+ "bulk": False,
 }
 
 
 # Run standard built-in tap tests from the SDK:
 def test_standard_tap_tests():
  """Run standard tap tests from the SDK."""
- _ = get_standard_tap_tests(TapShopify, config=SAMPLE_CONFIG)
+ _ = get_standard_tap_tests(TapShopify, config=SAMPLE_CONFIG)