Merge pull request #118 from craftcms/bugfix/115-fix-variant-import-l…

…imit

Fixed #115 not importing all variants

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Release Notes for Shopify
 
+## Unreleased
+
+- Fixed a bug where syncing Shopify variants would be limited to 50. ([#115](https://github.com/craftcms/shopify/issues/115))
+- `shopify/sync` commands now support a `--throttle` option.
+- Added `craft\shopify\console\controllers\SyncController::$throttle`.
+- Added `craft\shopify\services\Products::$throttle`.
+- Added `craft\shopify\services\Products::$sleepSeconds`.
+
 ## 5.1.2 - 2024-04-24
 
 - Fixed a bug where syncing meta fields would cause Shopify API rate limiting.

README.md

@@ -121,6 +121,8 @@ php craft shopify/sync/products
 
 The [`syncProductMetafields` and `syncVariantMetafields` settings](#settings) govern what data is synchronized via this process. Going forward, your products will be automatically kept in sync via [webhooks](#set-up-webhooks).
 
+Larger, more complex, stores may run into [rate limiting](#rate-limiting) issues during a full sync. In these cases, you can use the `--throttle` option to slow down the synchronization process.
+
 > [!NOTE]
 > Smaller stores with only a few products can perform synchronization via the **Shopify Sync** utility.
 

src/console/controllers/SyncController.php

@@ -24,6 +24,22 @@ class SyncController extends Controller
     /** @var string $defaultAction */
     public $defaultAction = 'products';
 
+    /**
+     * @var bool Whether to slow down API requests to avoid rate limiting.
+     * @since 5.2.0
+     */
+    public bool $throttle = false;
+
+    /**
+     * @inheritdoc
+     */
+    public function options($actionID): array
+    {
+        $options = parent::options($actionID);
+        $options[] = 'throttle';
+        return $options;
+    }
+
     /**
      * Sync all Shopify data.
      */
@@ -47,7 +63,14 @@ private function _syncProducts(): void
         $this->stdout('Syncing Shopify products…' . PHP_EOL . PHP_EOL, Console::FG_GREEN);
         // start timer
         $start = microtime(true);
+
+        $originalThrottle = Plugin::getInstance()->getProducts()->throttle;
+        Plugin::getInstance()->getProducts()->throttle = $this->throttle;
+
         Plugin::getInstance()->getProducts()->syncAllProducts();
+
+        Plugin::getInstance()->getProducts()->throttle = $originalThrottle;
+
         // end timer
         $time = microtime(true) - $start;
         $this->stdout('Finished syncing ' . Product::find()->count() . ' product(s) in ' . round($time, 2) . 's' . PHP_EOL . PHP_EOL, Console::FG_GREEN);

src/services/Api.php

@@ -18,6 +18,7 @@
 use Shopify\Context;
 use Shopify\Rest\Admin2023_10\Metafield as ShopifyMetafield;
 use Shopify\Rest\Admin2023_10\Product as ShopifyProduct;
+use Shopify\Rest\Admin2023_10\Variant as ShopifyVariant;
 use Shopify\Rest\Base as ShopifyBaseResource;
 
 /**
@@ -146,15 +147,29 @@ public function getMetafieldsByIdAndOwnerResource(int $id, string $ownerResource
     }
 
     /**
-     * Retrieves "metafields" for the provided Shopify product ID.
+     * Retrieves "variants" for the provided Shopify product ID.
      *
      * @param int $id Shopify Product ID
      */
     public function getVariantsByProductId(int $id): array
     {
-        $variants = $this->get("products/{$id}/variants");
+        $resources = [];
+        $params = ['limit' => 250];
+
+        do {
+            $resources = array_merge($resources, ShopifyVariant::all(
+                $this->getSession(),
+                ['product_id' => $id],
+                ShopifyVariant::$NEXT_PAGE_QUERY ?: $params,
+            ));
+        } while (ShopifyVariant::$NEXT_PAGE_QUERY);
+
+        $variants = [];
+        foreach ($resources as $resource) {
+            $variants[] = $resource->toArray();
+        }
 
-        return $variants['variants'] ?? [];
+        return $variants;
     }
 
     /**

src/services/Products.php

@@ -55,6 +55,18 @@ class Products extends Component
      */
     public const EVENT_BEFORE_SYNCHRONIZE_PRODUCT = 'beforeSynchronizeProduct';
 
+    /**
+     * @var bool Whether to slow down API requests to avoid rate limiting.
+     * @since 5.2.0
+     */
+    public bool $throttle = false;
+
+    /**
+     * @var int The number of seconds to sleep between requests when `$throttle` is enabled.
+     * @since 5.2.0
+     */
+    public int $sleepSeconds = 1;
+
     /**
      * @param ShopifyProduct $product
      * @return void
@@ -66,6 +78,10 @@ private function _updateProduct(ShopifyProduct $product): void
         $api = Plugin::getInstance()->getApi();
 
         $variants = $api->getVariantsByProductId($product->id);
+
+        if ($this->throttle) {
+            usleep((int) (1E6 * $this->sleepSeconds));
+        }
         $productMetafields = $api->getMetafieldsByProductId($product->id);
 
         foreach ($variants as &$variant) {