From c2abb94e0ab886b15abcdd91351049a314836ae2 Mon Sep 17 00:00:00 2001
From: Pierre Gauthier <pigau@smile.fr>
Date: Fri, 9 Feb 2024 15:04:01 +0100
Subject: [PATCH] Move data cleaning code in a dedicated command
---
README.md | 6 +-
src/Command/StructureClean.php | 65 ++++++++++++++
src/Command/StructureSync.php | 2 +-
src/Resources/config/services/commands.xml | 5 ++
src/Synchronizer/AbstractSynchronizer.php | 8 ++
src/Synchronizer/CatalogSynchronizer.php | 60 +++++++++----
.../SourceFieldOptionSynchronizer.php | 52 +++++++++--
src/Synchronizer/SourceFieldSynchronizer.php | 89 +++++++------------
8 files changed, 200 insertions(+), 87 deletions(-)
create mode 100644 src/Command/StructureClean.php
diff --git a/README.md b/README.md
index 3fecc85..0c4f061 100644
--- a/README.md
+++ b/README.md
@@ -57,7 +57,7 @@
- Open Sylius Admin, head to Configuration > Gally and configure the Gally endpoint (URL, credentials)
- Run this commands from your Sylius instance. This commands must be runned only once to synchronize the structure.
```shell
- bin/console gally:structure-sync # Sync catalog et source field data with gally
+ bin/console gally:structure:sync # Sync catalog et source field data with gally
```
- Run a full index from Sylius to Gally. This command can be run only once. Afterwards, the modified products are automatically synchronized.
```shell
@@ -66,6 +66,10 @@
- At this step, you should be able to see your product and source field in the Gally backend.
- They should also appear in your Sylius frontend when searching or browsing categories.
- And you're done !
+- You can also run the command to clean data that are not present in sylius anymore:
+ ```shell
+ bin/console gally:structure:clean
+ ```
## noUiSlider
diff --git a/src/Command/StructureClean.php b/src/Command/StructureClean.php
new file mode 100644
index 0000000..5d73aa7
--- /dev/null
+++ b/src/Command/StructureClean.php
@@ -0,0 +1,65 @@
+<?php
+/**
+ * DISCLAIMER
+ *
+ * Do not edit or add to this file if you wish to upgrade Gally to newer versions in the future.
+ *
+ * @package Gally
+ * @author Stephan Hochdörfer <S.Hochdoerfer@bitexpert.de>, Gally Team <elasticsuite@smile.fr>
+ * @copyright 2022-present Smile
+ * @license Open Software License v. 3.0 (OSL-3.0)
+ */
+
+declare(strict_types=1);
+
+namespace Gally\SyliusPlugin\Command;
+
+use Gally\SyliusPlugin\Synchronizer\AbstractSynchronizer;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class StructureClean extends Command
+{
+ protected static $defaultName = 'gally:structure:clean';
+
+ /**
+ * @param AbstractSynchronizer[] $synchronizers
+ */
+ public function __construct(
+ private iterable $synchronizers
+ ) {
+ parent::__construct();
+ }
+
+ protected function configure(): void
+ {
+ $this->setDescription('Remove all entity from gally that not exist anymore on sylius side.')
+ ->addOption('force', 'f', InputOption::VALUE_NONE, 'Really remove the listed entity from the gally.')
+ ->addOption('quiet', 'q', InputOption::VALUE_NONE, 'Don\'t list deleted entities.');
+ }
+
+ protected function execute(InputInterface $input, OutputInterface $output): int
+ {
+ $output->writeln('');
+ $isDryRun = !$input->getOption('force');
+ $isQuiet = $input->getOption('quiet');
+
+ if ($isDryRun) {
+ $output->writeln("<error>Running in dry run mode, add -f to really delete entities from Gally.</error>");
+ $output->writeln('');
+ }
+
+ foreach ($this->synchronizers as $synchronizer) {
+ $time = microtime(true);
+ $message = "<comment>Clean {$synchronizer->getEntityClass()}</comment>";
+ $output->writeln("$message ...");
+ $synchronizer->cleanAll($isDryRun, $isQuiet);
+ $time = number_format(microtime(true) - $time, 2);
+ $output->writeln(" Cleaned ($time)s\n");
+ }
+
+ return 0;
+ }
+}
diff --git a/src/Command/StructureSync.php b/src/Command/StructureSync.php
index e35a345..f4f3013 100644
--- a/src/Command/StructureSync.php
+++ b/src/Command/StructureSync.php
@@ -21,7 +21,7 @@
class StructureSync extends Command
{
- protected static $defaultName = 'gally:structure-sync';
+ protected static $defaultName = 'gally:structure:sync';
/**
* @param AbstractSynchronizer[] $synchronizers
diff --git a/src/Resources/config/services/commands.xml b/src/Resources/config/services/commands.xml
index 8c728a3..a1547b7 100644
--- a/src/Resources/config/services/commands.xml
+++ b/src/Resources/config/services/commands.xml
@@ -12,5 +12,10 @@
<argument type="tagged_iterator" tag="gally.entity.indexer"/>
<tag name="console.command"/>
</service>
+
+ <service id="Gally\SyliusPlugin\Command\StructureClean">
+ <argument type="tagged_iterator" tag="gally.entity.synchronizer"/>
+ <tag name="console.command"/>
+ </service>
</services>
</container>
diff --git a/src/Synchronizer/AbstractSynchronizer.php b/src/Synchronizer/AbstractSynchronizer.php
index 15cf4d4..ec1355b 100644
--- a/src/Synchronizer/AbstractSynchronizer.php
+++ b/src/Synchronizer/AbstractSynchronizer.php
@@ -84,6 +84,14 @@ public function fetchEntity(ModelInterface $entity): ?ModelInterface
abstract protected function getIdentity(ModelInterface $entity): string;
+ /**
+ * Remove all entity from gally that not exist anymore on sylius side.
+ */
+ public function cleanAll(bool $dryRun = true, bool $quiet = false): void
+ {
+
+ }
+
protected function buildFetchAllParams(int $page): array
{
return [
diff --git a/src/Synchronizer/CatalogSynchronizer.php b/src/Synchronizer/CatalogSynchronizer.php
index 04a0c38..d4e4829 100644
--- a/src/Synchronizer/CatalogSynchronizer.php
+++ b/src/Synchronizer/CatalogSynchronizer.php
@@ -63,10 +63,7 @@ public function getIdentity(ModelInterface $entity): string
public function synchronizeAll(): void
{
$this->fetchEntities();
-
- $this->catalogCodes = array_flip($this->getAllEntityCodes());
$this->localizedCatalogSynchronizer->fetchEntities();
- $this->localizedCatalogCodes = array_flip($this->localizedCatalogSynchronizer->getAllEntityCodes());
// synchronize all channels where the Gally integration is active
$channels = $this->channelRepository->findBy(['gallyActive' => 1]);
@@ -75,18 +72,6 @@ public function synchronizeAll(): void
foreach ($channels as $channel) {
$this->synchronizeItem(['channel' => $channel]);
}
-
- foreach (array_flip($this->localizedCatalogCodes) as $localizedCatalogCode) {
- /** @var LocalizedCatalogCatalogRead $localizedCatalog */
- $localizedCatalog = $this->localizedCatalogSynchronizer->getEntityFromApi($localizedCatalogCode);
- $this->localizedCatalogSynchronizer->deleteEntity($localizedCatalog->getId());
- }
-
- foreach (array_flip($this->catalogCodes) as $catalogCode) {
- /** @var CatalogCatalogRead $catalog */
- $catalog = $this->getEntityFromApi($catalogCode);
- $this->deleteEntity($catalog->getId());
- }
}
public function synchronizeItem(array $params): ?ModelInterface
@@ -108,12 +93,51 @@ public function synchronizeItem(array $params): ?ModelInterface
'locale' => $locale,
'catalog' => $catalog,
]);
+ }
- unset($this->localizedCatalogCodes[$this->localizedCatalogSynchronizer->getIdentity($localizedCatalog)]);
+ return $catalog;
+ }
+
+ public function cleanAll(bool $dryRun = true, bool $quiet = false): void
+ {
+ $this->fetchEntities();
+
+ $this->catalogCodes = array_flip($this->getAllEntityCodes());
+ $this->localizedCatalogSynchronizer->fetchEntities();
+ $this->localizedCatalogCodes = array_flip($this->localizedCatalogSynchronizer->getAllEntityCodes());
+
+ // Synchronize all channels where the Gally integration is active
+ $channels = $this->channelRepository->findBy(['gallyActive' => 1]);
+
+ /** @var Channel[] $channels */
+ foreach ($channels as $channel) {
+ /** @var LocaleInterface $locale */
+ foreach ($channel->getLocales() as $locale) {
+ unset($this->localizedCatalogCodes[$channel->getCode() . '_' . $locale->getCode()]);
+ }
+ unset($this->catalogCodes[$channel->getCode()]);
}
- unset($this->catalogCodes[$this->getIdentity($catalog)]);
+ foreach (array_flip($this->localizedCatalogCodes) as $localizedCatalogCode) {
+ /** @var LocalizedCatalogCatalogRead $localizedCatalog */
+ $localizedCatalog = $this->localizedCatalogSynchronizer->getEntityFromApi($localizedCatalogCode);
+ if (!$quiet) {
+ print(" Delete localized catalog {$localizedCatalog->getId()}\n");
+ }
+ if (!$dryRun) {
+ $this->localizedCatalogSynchronizer->deleteEntity($localizedCatalog->getId());
+ }
+ }
- return $catalog;
+ foreach (array_flip($this->catalogCodes) as $catalogCode) {
+ /** @var CatalogCatalogRead $catalog */
+ $catalog = $this->getEntityFromApi($catalogCode);
+ if (!$quiet) {
+ print(" Delete catalog {$catalog->getId()}\n");
+ }
+ if (!$dryRun) {
+ $this->deleteEntity($catalog->getId());
+ }
+ }
}
}
diff --git a/src/Synchronizer/SourceFieldOptionSynchronizer.php b/src/Synchronizer/SourceFieldOptionSynchronizer.php
index 74afc9a..447cde6 100644
--- a/src/Synchronizer/SourceFieldOptionSynchronizer.php
+++ b/src/Synchronizer/SourceFieldOptionSynchronizer.php
@@ -72,7 +72,6 @@ public function getIdentity(ModelInterface $entity): string
public function synchronizeAll(): void
{
- $this->sourceFieldOptionCodes = array_flip($this->getAllEntityCodes());
$this->sourceFieldSynchronizer->fetchEntities();
$metadataName = strtolower((new \ReflectionClass(Product::class))->getShortName());
@@ -134,12 +133,6 @@ public function synchronizeAll(): void
}
$this->runBulk();
-
- foreach (array_flip($this->sourceFieldOptionCodes) as $sourceFieldOptionCode) {
- /** @var SourceFieldOptionSourceFieldOptionRead $sourceFieldOption */
- $sourceFieldOption = $this->getEntityFromApi($sourceFieldOptionCode);
- $this->deleteEntity($sourceFieldOption->getId());
- }
}
public function synchronizeItem(array $params): ?ModelInterface
@@ -172,11 +165,52 @@ public function synchronizeItem(array $params): ?ModelInterface
$sourceFieldOption = new SourceFieldOptionSourceFieldOptionWrite($data);
$this->addEntityToBulk($sourceFieldOption);
- unset($this->sourceFieldOptionCodes[$this->getIdentity($sourceFieldOption)]);
-
return $sourceFieldOption;
}
+ public function cleanAll(bool $dryRun = true, bool $quiet = false): void
+ {
+ $this->sourceFieldOptionCodes = array_flip($this->getAllEntityCodes());
+ $this->sourceFieldSynchronizer->fetchEntities();
+
+ $metadataName = strtolower((new \ReflectionClass(Product::class))->getShortName());
+ /** @var MetadataMetadataRead $metadata */
+ $metadata = $this->metadataSynchronizer->synchronizeItem(['entity' => $metadataName]);
+
+ /** @var ProductAttribute[] $attributes */
+ $attributes = $this->productAttributeRepository->findAll();
+ foreach ($attributes as $attribute) {
+ if ('select' === $attribute->getType()) {
+ $sourceField = $this->sourceFieldSynchronizer->getEntityByCode($metadata, $attribute->getCode());
+ $configuration = $attribute->getConfiguration();
+ foreach ($configuration['choices'] ?? [] as $code => $choice) {
+ unset($this->sourceFieldOptionCodes['/source_fields/' . $sourceField->getId() . $code]);
+ }
+ }
+ }
+
+ /** @var ProductOption[] $options */
+ $options = $this->productOptionRepository->findAll();
+ foreach ($options as $option) {
+ $sourceField = $this->sourceFieldSynchronizer->getEntityByCode($metadata, $option->getCode());
+ /** @var ProductOptionValueInterface $value */
+ foreach ($option->getValues() as $value) {
+ unset($this->sourceFieldOptionCodes['/source_fields/' . $sourceField->getId() . $value->getCode()]);
+ }
+ }
+
+ foreach (array_flip($this->sourceFieldOptionCodes) as $sourceFieldOptionCode) {
+ /** @var SourceFieldOptionSourceFieldOptionRead $sourceFieldOption */
+ $sourceFieldOption = $this->getEntityFromApi($sourceFieldOptionCode);
+ if (!$quiet) {
+ print(" Delete sourceFieldOption {$sourceFieldOption->getSourceField()} {$sourceFieldOption->getCode()}\n");
+ }
+ if (!$dryRun) {
+ $this->deleteEntity($sourceFieldOption->getId());
+ }
+ }
+ }
+
public function fetchEntity(ModelInterface $entity): ?ModelInterface
{
/** @var SourceFieldOptionSourceFieldOptionWrite $entity */
diff --git a/src/Synchronizer/SourceFieldSynchronizer.php b/src/Synchronizer/SourceFieldSynchronizer.php
index e02a332..e48b7d7 100644
--- a/src/Synchronizer/SourceFieldSynchronizer.php
+++ b/src/Synchronizer/SourceFieldSynchronizer.php
@@ -70,43 +70,18 @@ public function getIdentity(ModelInterface $entity): string
public function synchronizeAll(): void
{
- $this->sourceFieldCodes = array_flip($this->getAllEntityCodes());
-
$metadataName = strtolower((new \ReflectionClass(Product::class))->getShortName());
$metadata = $this->metadataSynchronizer->synchronizeItem(['entity' => $metadataName]);
/** @var ProductAttribute[] $attributes */
$attributes = $this->productAttributeRepository->findAll();
foreach ($attributes as $attribute) {
- $options = [];
- if ('select' === $attribute->getType()) {
- $position = 0;
- $configuration = $attribute->getConfiguration();
- $choices = $configuration['choices'] ?? [];
- foreach ($choices as $code => $choice) {
- $translations = [];
- foreach ($choice ?? [] as $locale => $translation) {
- $translations[] = [
- 'locale' => $locale,
- 'translation' => $translation,
- ];
- }
- $options[$position] = [
- 'code' => $code,
- 'translations' => $translations,
- 'position' => $position,
- ];
- ++$position;
- }
- }
-
$this->synchronizeItem([
'metadata' => $metadata,
'field' => [
'code' => $attribute->getCode(),
'type' => self::getGallyType($attribute->getType()),
'translations' => $attribute->getTranslations(),
- 'options' => $options,
],
]);
}
@@ -114,48 +89,17 @@ public function synchronizeAll(): void
/** @var ProductOption[] $options */
$options = $this->productOptionRepository->findAll();
foreach ($options as $option) {
- $optionValues = [];
- $position = 0;
- foreach ($option->getValues() as $value) {
- $translations = [];
- foreach ($value->getTranslations() as $translation) {
- /** @var ProductOptionValueTranslation $translation */
- $translations[] = [
- 'locale' => $translation->getLocale(),
- 'translation' => $translation->getValue(),
- ];
- }
-
- /** @var ProductOptionValueInterface $value */
- $optionValues[$position] = [
- 'code' => $value->getCode(),
- 'translations' => $translations,
- 'position' => $position,
- ];
-
- ++$position;
- }
-
$this->synchronizeItem([
'metadata' => $metadata,
'field' => [
'code' => $option->getCode(),
'type' => self::getGallyType('select'),
'translations' => $option->getTranslations(),
- 'options' => $optionValues,
],
]);
}
$this->runBulk();
-
- foreach (array_flip($this->sourceFieldCodes) as $sourceFieldCode) {
- /** @var SourceFieldSourceFieldRead $sourceField */
- $sourceField = $this->getEntityFromApi($sourceFieldCode);
- if (!$sourceField->getIsSystem()) {
- $this->deleteEntity($sourceField->getId());
- }
- }
}
public function synchronizeItem(array $params): ?ModelInterface
@@ -192,11 +136,40 @@ public function synchronizeItem(array $params): ?ModelInterface
$sourceField = new SourceFieldSourceFieldWrite($data);
$this->addEntityToBulk($sourceField);
- unset($this->sourceFieldCodes[$this->getIdentity($sourceField)]);
-
return $sourceField;
}
+ public function cleanAll(bool $dryRun = true, bool $quiet = false): void
+ {
+ $this->sourceFieldCodes = array_flip($this->getAllEntityCodes());
+
+ $metadataName = strtolower((new \ReflectionClass(Product::class))->getShortName());
+ $metadata = $this->metadataSynchronizer->synchronizeItem(['entity' => $metadataName]);
+
+ /** @var ProductAttribute[] $attributes */
+ $attributes = $this->productAttributeRepository->findAll();
+ foreach ($attributes as $attribute) {
+ unset($this->sourceFieldCodes['/metadata/' . $metadata->getId() . $attribute->getCode()]);
+ }
+
+ /** @var ProductOption[] $options */
+ $options = $this->productOptionRepository->findAll();
+ foreach ($options as $option) {
+ unset($this->sourceFieldCodes['/metadata/' . $metadata->getId() . $option->getCode()]);
+ }
+
+ foreach (array_flip($this->sourceFieldCodes) as $sourceFieldCode) {
+ /** @var SourceFieldSourceFieldRead $sourceField */
+ $sourceField = $this->getEntityFromApi($sourceFieldCode);
+ if (!$sourceField->getIsSystem() && !$quiet) {
+ print(" Delete sourceField {$sourceField->getMetadata()} {$sourceField->getCode()}\n");
+ }
+ if (!$sourceField->getIsSystem() && !$dryRun) {
+ $this->deleteEntity($sourceField->getId());
+ }
+ }
+ }
+
public static function getGallyType(string $type): string
{
switch ($type) {