From 2beabfe8b78dfb61426310cabf16569e2d395341 Mon Sep 17 00:00:00 2001 From: Nathaniel Hammond Date: Wed, 5 Jun 2024 15:07:54 +0100 Subject: [PATCH 1/6] Fixed #115 nto importing all variants --- CHANGELOG.md | 8 ++++++++ src/console/controllers/SyncController.php | 23 ++++++++++++++++++++++ src/services/Api.php | 23 +++++++++++++++++++--- src/services/Products.php | 16 +++++++++++++++ 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fbbe28..89c279f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Release Notes for Shopify +## Unreleased + +- It is now possible to throttle the API calls in an attempt to avoid Shopify API rate limiting. +- Fixed a bug where syncing Shopify variants would be limited to 50. ([#115](https://github.com/craftcms/shopify/issues/115)) +- Added `craft\shopify\console\controllers\SyncController::$throttle`. +- Added `craft\shopify\services\Products::$throttle`. +- Added `craft\shopify\services\Products::$sleepSeconds`. + ## 5.1.2 - 2024-04-24 - Fixed a bug where syncing meta fields would cause Shopify API rate limiting. diff --git a/src/console/controllers/SyncController.php b/src/console/controllers/SyncController.php index cc144e5..3bb0987 100644 --- a/src/console/controllers/SyncController.php +++ b/src/console/controllers/SyncController.php @@ -24,6 +24,22 @@ class SyncController extends Controller /** @var string $defaultAction */ public $defaultAction = 'products'; + /** + * @var bool Whether to slow down API requests to avoid rate limiting. + * since + */ + public bool $throttle = false; + + /** + * @inheritdoc + */ + public function options($actionID): array + { + $options = parent::options($actionID); + $options[] = 'throttle'; + return $options; + } + /** * Sync all Shopify data. */ @@ -47,7 +63,14 @@ private function _syncProducts(): void $this->stdout('Syncing Shopify products…' . PHP_EOL . PHP_EOL, Console::FG_GREEN); // start timer $start = microtime(true); + + $originalThrottle = Plugin::getInstance()->getProducts()->throttle; + Plugin::getInstance()->getProducts()->throttle = $this->throttle; + Plugin::getInstance()->getProducts()->syncAllProducts(); + + Plugin::getInstance()->getProducts()->throttle = $originalThrottle; + // end timer $time = microtime(true) - $start; $this->stdout('Finished syncing ' . Product::find()->count() . ' product(s) in ' . round($time, 2) . 's' . PHP_EOL . PHP_EOL, Console::FG_GREEN); diff --git a/src/services/Api.php b/src/services/Api.php index 10760e9..cfbf763 100644 --- a/src/services/Api.php +++ b/src/services/Api.php @@ -18,6 +18,8 @@ use Shopify\Context; use Shopify\Rest\Admin2023_10\Metafield as ShopifyMetafield; use Shopify\Rest\Admin2023_10\Product as ShopifyProduct; +use Shopify\Rest\Admin2023_10\Variant as ShopifyVariant; +use Shopify\Rest\Base; use Shopify\Rest\Base as ShopifyBaseResource; /** @@ -146,15 +148,29 @@ public function getMetafieldsByIdAndOwnerResource(int $id, string $ownerResource } /** - * Retrieves "metafields" for the provided Shopify product ID. + * Retrieves "variants" for the provided Shopify product ID. * * @param int $id Shopify Product ID */ public function getVariantsByProductId(int $id): array { - $variants = $this->get("products/{$id}/variants"); + $resources = []; + $params = ['limit' => 250]; + + do { + $resources = array_merge($resources, ShopifyVariant::all( + $this->getSession(), + ['product_id' => $id], + ShopifyVariant::$NEXT_PAGE_QUERY ?: $params, + )); + } while (ShopifyVariant::$NEXT_PAGE_QUERY); + + $variants = []; + foreach ($resources as $resource) { + $variants[] = $resource->toArray(); + } - return $variants['variants'] ?? []; + return $variants; } /** @@ -183,6 +199,7 @@ public function getAll(string $type, array $params = []): array // Force maximum page size: $params['limit'] = 250; + /** @var Base $type */ do { $resources = array_merge($resources, $type::all( $this->getSession(), diff --git a/src/services/Products.php b/src/services/Products.php index aea33ae..90b2d83 100644 --- a/src/services/Products.php +++ b/src/services/Products.php @@ -55,6 +55,18 @@ class Products extends Component */ public const EVENT_BEFORE_SYNCHRONIZE_PRODUCT = 'beforeSynchronizeProduct'; + /** + * @var bool Whether to slow down API requests to avoid rate limiting. + * @since + */ + public bool $throttle = false; + + /** + * @var int The number of seconds to sleep between requests when `$throttle` is enabled. + * @since + */ + public int $sleepSeconds = 1; + /** * @param ShopifyProduct $product * @return void @@ -66,6 +78,10 @@ private function _updateProduct(ShopifyProduct $product): void $api = Plugin::getInstance()->getApi(); $variants = $api->getVariantsByProductId($product->id); + + if ($this->throttle) { + usleep((int) (1E6 * $this->sleepSeconds)); + } $productMetafields = $api->getMetafieldsByProductId($product->id); foreach ($variants as &$variant) { From d0b33c5d8080f102414cdabcbdc3d8cfbff7e1d8 Mon Sep 17 00:00:00 2001 From: Nathaniel Hammond Date: Wed, 5 Jun 2024 15:13:52 +0100 Subject: [PATCH 2/6] PHPstan update --- src/services/Api.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/services/Api.php b/src/services/Api.php index cfbf763..6a8c462 100644 --- a/src/services/Api.php +++ b/src/services/Api.php @@ -199,7 +199,6 @@ public function getAll(string $type, array $params = []): array // Force maximum page size: $params['limit'] = 250; - /** @var Base $type */ do { $resources = array_merge($resources, $type::all( $this->getSession(), From b16c46cf048ad20f6a1602c6c1894816ee0793af Mon Sep 17 00:00:00 2001 From: Nathaniel Hammond Date: Wed, 5 Jun 2024 15:15:28 +0100 Subject: [PATCH 3/6] fix cs --- src/services/Api.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/services/Api.php b/src/services/Api.php index 6a8c462..d3ec133 100644 --- a/src/services/Api.php +++ b/src/services/Api.php @@ -19,7 +19,6 @@ use Shopify\Rest\Admin2023_10\Metafield as ShopifyMetafield; use Shopify\Rest\Admin2023_10\Product as ShopifyProduct; use Shopify\Rest\Admin2023_10\Variant as ShopifyVariant; -use Shopify\Rest\Base; use Shopify\Rest\Base as ShopifyBaseResource; /** From 78bc53f95a843ef136da9b760fb2878577955fa8 Mon Sep 17 00:00:00 2001 From: Nathaniel Hammond Date: Fri, 7 Jun 2024 08:39:08 +0100 Subject: [PATCH 4/6] Add readme info --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index f19bf52..5f17d47 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,8 @@ php craft shopify/sync/products The [`syncProductMetafields` and `syncVariantMetafields` settings](#settings) govern what data is synchronized via this process. Going forward, your products will be automatically kept in sync via [webhooks](#set-up-webhooks). +Larger, more complex, stores may run into [rate limiting](#rate-limiting) issues during a full sync. In these cases, you can use the `--throttle` option to slow down the synchronization process. + > [!NOTE] > Smaller stores with only a few products can perform synchronization via the **Shopify Sync** utility. From 8d3cab39f5a5ee8312be21a098ad3dbb22c260a2 Mon Sep 17 00:00:00 2001 From: Nathaniel Hammond Date: Fri, 7 Jun 2024 08:41:07 +0100 Subject: [PATCH 5/6] tweak changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89c279f..aa85290 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,8 @@ ## Unreleased -- It is now possible to throttle the API calls in an attempt to avoid Shopify API rate limiting. - Fixed a bug where syncing Shopify variants would be limited to 50. ([#115](https://github.com/craftcms/shopify/issues/115)) +- `shopify/sync` commands now support a `--throttle` option. - Added `craft\shopify\console\controllers\SyncController::$throttle`. - Added `craft\shopify\services\Products::$throttle`. - Added `craft\shopify\services\Products::$sleepSeconds`. From b74960989bcfc67f3141c776d2a74dca825b3913 Mon Sep 17 00:00:00 2001 From: Nathaniel Hammond Date: Tue, 18 Jun 2024 08:02:28 +0100 Subject: [PATCH 6/6] Added since tags --- src/console/controllers/SyncController.php | 2 +- src/services/Products.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/console/controllers/SyncController.php b/src/console/controllers/SyncController.php index 3bb0987..afbb9d7 100644 --- a/src/console/controllers/SyncController.php +++ b/src/console/controllers/SyncController.php @@ -26,7 +26,7 @@ class SyncController extends Controller /** * @var bool Whether to slow down API requests to avoid rate limiting. - * since + * @since 5.2.0 */ public bool $throttle = false; diff --git a/src/services/Products.php b/src/services/Products.php index 90b2d83..f8707ce 100644 --- a/src/services/Products.php +++ b/src/services/Products.php @@ -57,13 +57,13 @@ class Products extends Component /** * @var bool Whether to slow down API requests to avoid rate limiting. - * @since + * @since 5.2.0 */ public bool $throttle = false; /** * @var int The number of seconds to sleep between requests when `$throttle` is enabled. - * @since + * @since 5.2.0 */ public int $sleepSeconds = 1;