From 415c12a5cbb963a5595f760ef0dd0a7b40915cf3 Mon Sep 17 00:00:00 2001 From: Nicolas MELONI Date: Tue, 14 Nov 2023 10:20:55 +0100 Subject: [PATCH 1/4] Add retry on product model import --- ...oductAttributeAkeneoAttributeProcessor.php | 38 ++++++-- .../ProductGroup/ProductGroupProcessor.php | 1 - src/Resources/config/services.yaml | 6 ++ .../ProductModel/BatchProductModelTask.php | 87 ++++++++++++------- 4 files changed, 93 insertions(+), 39 deletions(-) diff --git a/src/Processor/ProductAttribute/ProductAttributeAkeneoAttributeProcessor.php b/src/Processor/ProductAttribute/ProductAttributeAkeneoAttributeProcessor.php index 011fb2d6..8668a0ef 100644 --- a/src/Processor/ProductAttribute/ProductAttributeAkeneoAttributeProcessor.php +++ b/src/Processor/ProductAttribute/ProductAttributeAkeneoAttributeProcessor.php @@ -12,6 +12,10 @@ use Sylius\Component\Resource\Repository\RepositoryInterface; use Synolia\SyliusAkeneoPlugin\Builder\Attribute\ProductAttributeValueValueBuilder; use Synolia\SyliusAkeneoPlugin\Component\Attribute\AttributeType\AssetAttributeType; +use Synolia\SyliusAkeneoPlugin\Exceptions\Attribute\MissingLocaleTranslationException; +use Synolia\SyliusAkeneoPlugin\Exceptions\Attribute\MissingLocaleTranslationOrScopeException; +use Synolia\SyliusAkeneoPlugin\Exceptions\Attribute\MissingScopeException; +use Synolia\SyliusAkeneoPlugin\Exceptions\Attribute\TranslationNotFoundException; use Synolia\SyliusAkeneoPlugin\Provider\AkeneoAttributeDataProviderInterface; use Synolia\SyliusAkeneoPlugin\Provider\SyliusAkeneoLocaleCodeProvider; use Synolia\SyliusAkeneoPlugin\Transformer\AkeneoAttributeToSyliusAttributeTransformerInterface; @@ -74,18 +78,34 @@ public function process(string $attributeCode, array $context = []): void /** @var AttributeInterface $attribute */ $attribute = $this->productAttributeRepository->findOneBy(['code' => $transformedAttributeCode]); - foreach ($this->syliusAkeneoLocaleCodeProvider->getUsedLocalesOnBothPlatforms() as $syliusAkeneo) { - $this->setAttributeTranslation( - $context['model'], - $attribute, - $context['data'], - $syliusAkeneo, - $attributeCode, - $context['scope'], - ); + foreach ($this->syliusAkeneoLocaleCodeProvider->getUsedLocalesOnBothPlatforms() as $syliusLocale) { + try { + $this->setAttributeTranslation( + $context['model'], + $attribute, + $context['data'], + $syliusLocale, + $attributeCode, + $context['scope'], + ); + } catch (MissingLocaleTranslationException | MissingLocaleTranslationOrScopeException|MissingScopeException|TranslationNotFoundException $error) { + $this->logger->warning('Attribute translation error', [ + 'attribute_code' => $attributeCode, + 'sylius_locale' => $syliusLocale, + 'context' => $context, + 'error' => $error->getMessage(), + 'trace' => $error->getTraceAsString(), + ]); + } } } + /** + * @throws MissingLocaleTranslationOrScopeException + * @throws MissingLocaleTranslationException + * @throws MissingScopeException + * @throws TranslationNotFoundException + */ private function setAttributeTranslation( ProductInterface $product, AttributeInterface $attribute, diff --git a/src/Processor/ProductGroup/ProductGroupProcessor.php b/src/Processor/ProductGroup/ProductGroupProcessor.php index a5f55ca0..eb7c6398 100644 --- a/src/Processor/ProductGroup/ProductGroupProcessor.php +++ b/src/Processor/ProductGroup/ProductGroupProcessor.php @@ -54,7 +54,6 @@ private function createGroupForCodeAndFamily( $productGroup->setModel($code); $productGroup->setFamily($family); $productGroup->setFamilyVariant($familyVariant); - $this->entityManager->persist($productGroup); return; } diff --git a/src/Resources/config/services.yaml b/src/Resources/config/services.yaml index 781cd3cb..20d7fb96 100644 --- a/src/Resources/config/services.yaml +++ b/src/Resources/config/services.yaml @@ -1,3 +1,7 @@ +parameters: + env(SYNOLIA_AKENEO_MAX_RETRY_COUNT): 30 + env(SYNOLIA_AKENEO_RETRY_WAIT_TIME): 5000 + services: _defaults: autowire: true @@ -5,6 +9,8 @@ services: public: false bind: $projectDir: '%kernel.project_dir%' + $maxRetryCount: '%env(int:SYNOLIA_AKENEO_MAX_RETRY_COUNT)%' + $retryWaitTime: '%env(int:SYNOLIA_AKENEO_RETRY_WAIT_TIME)%' Synolia\SyliusAkeneoPlugin\: resource: '../../*' diff --git a/src/Task/ProductModel/BatchProductModelTask.php b/src/Task/ProductModel/BatchProductModelTask.php index 4e18110d..a0a8c3cf 100644 --- a/src/Task/ProductModel/BatchProductModelTask.php +++ b/src/Task/ProductModel/BatchProductModelTask.php @@ -41,6 +41,9 @@ public function __construct( private IsProductProcessableCheckerInterface $isProductProcessableChecker, private ProductGroupProcessor $productGroupProcessor, private ManagerRegistry $managerRegistry, + private int $maxRetryCount, + private int $retryWaitTime, + private int $retryCount = 0, ) { parent::__construct($entityManager); } @@ -52,6 +55,10 @@ public function __construct( */ public function __invoke(PipelinePayloadInterface $payload): PipelinePayloadInterface { + if ($this->retryCount === $this->maxRetryCount) { + return $payload; + } + $this->logger->debug(self::class); $this->type = $payload->getType(); $this->logger->notice(Messages::createOrUpdate($this->type)); @@ -62,54 +69,76 @@ public function __invoke(PipelinePayloadInterface $payload): PipelinePayloadInte while ($results = $queryResult->fetchAllAssociative()) { foreach ($results as $result) { + $isSuccess = false; + /** @var array $resource */ $resource = json_decode($result['values'], true); - $this->handleProductGroup($resource); - - try { - $this->dispatcher->dispatch(new BeforeProcessingProductEvent($resource)); - - $this->entityManager->beginTransaction(); - - if ($this->isProductProcessableChecker->check($resource)) { - $product = $this->process($resource); - $this->dispatcher->dispatch(new AfterProcessingProductEvent($resource, $product)); + do { + try { + $this->handleProductModel($resource); + $isSuccess = true; + } catch (ORMInvalidArgumentException $ormInvalidArgumentException) { + ++$this->retryCount; + usleep($this->retryWaitTime); + + $this->logger->error('Retrying import', [ + 'product' => $result, + 'retry_count' => $this->retryCount, + 'error' => $ormInvalidArgumentException->getMessage(), + ]); + + $this->entityManager = $this->getNewEntityManager(); + } catch (\Throwable $throwable) { + ++$this->retryCount; + usleep($this->retryWaitTime); + + $this->logger->error('Error importing product', [ + 'message' => $throwable->getMessage(), + 'trace' => $throwable->getTraceAsString(), + ]); + + $this->entityManager = $this->getNewEntityManager(); } + } while (false === $isSuccess && $this->retryCount !== $this->maxRetryCount); - $this->entityManager->flush(); - $this->entityManager->commit(); - - unset($resource, $product); - $this->removeEntry($payload, (int) $result['id']); - } catch (\Throwable $throwable) { - $this->entityManager->rollback(); - $this->logger->warning($throwable->getMessage()); - $this->removeEntry($payload, (int) $result['id']); - } + unset($resource); + $this->removeEntry($payload, (int) $result['id']); + $this->retryCount = 0; } } return $payload; } + private function handleProductModel(array $resource): void + { + $this->handleProductGroup($resource); + $this->dispatcher->dispatch(new BeforeProcessingProductEvent($resource)); + + if (!$this->isProductProcessableChecker->check($resource)) { + return; + } + + $product = $this->process($resource); + $this->dispatcher->dispatch(new AfterProcessingProductEvent($resource, $product)); + $this->entityManager->flush(); + } + private function handleProductGroup(array $resource): void { try { - $this->entityManager->beginTransaction(); - $this->productGroupProcessor->process($resource); - $this->entityManager->flush(); - $this->entityManager->commit(); - } catch (ORMInvalidArgumentException) { - if ($this->entityManager->getConnection()->isTransactionActive()) { - $this->entityManager->rollback(); - } - + } catch (ORMInvalidArgumentException $ormInvalidArgumentException) { if (!$this->entityManager->isOpen()) { + $this->logger->warning('Recreating entity manager'); $this->entityManager = $this->getNewEntityManager(); } + + ++$this->retryCount; + + throw $ormInvalidArgumentException; } } From 121bd50114d276ebac3e73fbd5602f76711b4996 Mon Sep 17 00:00:00 2001 From: Nicolas MELONI Date: Thu, 16 Nov 2023 15:04:56 +0100 Subject: [PATCH 2/4] fix batching on products import --- src/Task/Product/ProcessProductsTask.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Task/Product/ProcessProductsTask.php b/src/Task/Product/ProcessProductsTask.php index e1460012..21743ccb 100644 --- a/src/Task/Product/ProcessProductsTask.php +++ b/src/Task/Product/ProcessProductsTask.php @@ -103,7 +103,7 @@ public function __invoke(PipelinePayloadInterface $payload): PipelinePayloadInte $this->process($payload); } - $this->processManager->waitForAllProcesses(); + $this->processManager->startAll(); return $payload; } @@ -114,6 +114,9 @@ private function handleProducts( int &$count = 0, array &$ids = [], ): void { + $this->processManager->setInstantProcessing($payload->getProcessAsSoonAsPossible()); + $this->processManager->setNumberOfParallelProcesses($payload->getMaxRunningProcessQueueSize()); + while ( ($page instanceof Page && $page->hasNextPage()) || ($page instanceof Page && !$page->hasPreviousPage()) || From b1ec88a824b69572fbe83b39d749a4adc7c61739 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20M=C3=A9loni?= Date: Thu, 23 Nov 2023 16:29:16 +0100 Subject: [PATCH 3/4] Lowered default max retry count --- src/Resources/config/services.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Resources/config/services.yaml b/src/Resources/config/services.yaml index 20d7fb96..39e077bb 100644 --- a/src/Resources/config/services.yaml +++ b/src/Resources/config/services.yaml @@ -1,5 +1,5 @@ parameters: - env(SYNOLIA_AKENEO_MAX_RETRY_COUNT): 30 + env(SYNOLIA_AKENEO_MAX_RETRY_COUNT): 3 env(SYNOLIA_AKENEO_RETRY_WAIT_TIME): 5000 services: From da1b329e5dc9a33140b1775ce3669c993df7bfd5 Mon Sep 17 00:00:00 2001 From: Nicolas MELONI Date: Tue, 26 Dec 2023 09:20:17 +0100 Subject: [PATCH 4/4] Added retry on product models import --- src/Task/ProductModel/BatchProductModelTask.php | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Task/ProductModel/BatchProductModelTask.php b/src/Task/ProductModel/BatchProductModelTask.php index a0a8c3cf..dfeffc26 100644 --- a/src/Task/ProductModel/BatchProductModelTask.php +++ b/src/Task/ProductModel/BatchProductModelTask.php @@ -5,7 +5,6 @@ namespace Synolia\SyliusAkeneoPlugin\Task\ProductModel; use Doctrine\DBAL\Exception; -use Doctrine\DBAL\Result; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\ORMInvalidArgumentException; use Doctrine\Persistence\ManagerRegistry; @@ -64,7 +63,6 @@ public function __invoke(PipelinePayloadInterface $payload): PipelinePayloadInte $this->logger->notice(Messages::createOrUpdate($this->type)); $query = $this->getSelectStatement($payload); - /** @var Result $queryResult */ $queryResult = $query->executeQuery(); while ($results = $queryResult->fetchAllAssociative()) { @@ -100,7 +98,7 @@ public function __invoke(PipelinePayloadInterface $payload): PipelinePayloadInte $this->entityManager = $this->getNewEntityManager(); } - } while (false === $isSuccess && $this->retryCount !== $this->maxRetryCount); + } while (false === $isSuccess && $this->retryCount < $this->maxRetryCount); unset($resource); $this->removeEntry($payload, (int) $result['id']);