From b19be8dbef47929ff443466c71bd1843053e7ee0 Mon Sep 17 00:00:00 2001 From: Slawomir Dolzycki-Uchto Date: Tue, 22 Oct 2024 10:46:39 +0200 Subject: [PATCH 1/4] IBX-8562: Command to remove duplicated entries after faulty IBX-5388 fix --- .../Resources/config/commands.yml | 8 + .../Twig/DebugTemplate.php | 4 +- .../VirtualFieldDuplicateFixCommand.php | 258 ++++++++++++++++++ 3 files changed, 268 insertions(+), 2 deletions(-) create mode 100644 src/bundle/Core/Command/VirtualFieldDuplicateFixCommand.php diff --git a/eZ/Bundle/EzPublishCoreBundle/Resources/config/commands.yml b/eZ/Bundle/EzPublishCoreBundle/Resources/config/commands.yml index 165639f868..29a273a993 100644 --- a/eZ/Bundle/EzPublishCoreBundle/Resources/config/commands.yml +++ b/eZ/Bundle/EzPublishCoreBundle/Resources/config/commands.yml @@ -62,3 +62,11 @@ services: $userHandler: '@ezpublish.spi.persistence.user_handler' tags: - { name: console.command } + + Ibexa\Bundle\Core\Command\VirtualFieldDuplicateFixCommand: + autowire: true + autoconfigure: true + arguments: + $connection: '@ezpublish.persistence.connection' + tags: + - { name: console.command } diff --git a/eZ/Bundle/EzPublishDebugBundle/Twig/DebugTemplate.php b/eZ/Bundle/EzPublishDebugBundle/Twig/DebugTemplate.php index 0d10379826..89f65d27d7 100644 --- a/eZ/Bundle/EzPublishDebugBundle/Twig/DebugTemplate.php +++ b/eZ/Bundle/EzPublishDebugBundle/Twig/DebugTemplate.php @@ -76,9 +76,9 @@ public function getSourceContext(): Source return new Source('', ''); } - protected function doDisplay(array $context, array $blocks = []): string + protected function doDisplay(array $context, array $blocks = []): iterable { - return ''; + return []; } /** diff --git a/src/bundle/Core/Command/VirtualFieldDuplicateFixCommand.php b/src/bundle/Core/Command/VirtualFieldDuplicateFixCommand.php new file mode 100644 index 0000000000..2f4d93f58f --- /dev/null +++ b/src/bundle/Core/Command/VirtualFieldDuplicateFixCommand.php @@ -0,0 +1,258 @@ +connection = $connection; + } + + public function configure(): void + { + $this->addOption( + 'batch-size', + 'b', + InputOption::VALUE_REQUIRED, + 'Number of attributes affected per iteration', + self::DEFAULT_BATCH_SIZE + ); + + $this->addOption( + 'max-iterations', + 'i', + InputOption::VALUE_REQUIRED, + 'Max iterations count (default or -1: unlimited)', + self::MAX_ITERATIONS_UNLIMITED + ); + + $this->addOption( + 'sleep', + 's', + InputOption::VALUE_REQUIRED, + 'Wait between iterations, in milliseconds', + self::DEFAULT_SLEEP + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $style = new SymfonyStyle($input, $output); + $stopwatch = new Stopwatch(true); + $stopwatch->start('total', 'command'); + + $batchSize = (int)$input->getOption('batch-size'); + if ($batchSize === 0) { + $style->warning('Batch size is set to 0. Nothing to do.'); + + return Command::INVALID; + } + + $maxIterations = (int)$input->getOption('max-iterations'); + if ($maxIterations === 0) { + $style->warning('Max iterations is set to 0. Nothing to do.'); + + return Command::INVALID; + } + + $sleep = (int)$input->getOption('sleep'); + + $totalCount = $this->getDuplicatedAttributeTotalCount($style, $stopwatch); + + if ($totalCount === 0) { + $style->success('Database is clean of attribute duplicates. Nothing to do.'); + + return Command::SUCCESS; + } + + if ($input->isInteractive()) { + $confirmation = $this->askForConfirmation($style); + if (!$confirmation) { + $style->info('Confirmation rejected. Terminating.'); + + return Command::FAILURE; + } + } + + $iteration = 1; + $totalDeleted = 0; + do { + $deleted = 0; + $stopwatch->start('iteration', 'sql'); + + $attributes = $this->getDuplicatedAttributesBatch($batchSize); + foreach ($attributes as $attribute) { + $attributeIds = $this->getDuplicatedAttributeIds($attribute); + + if (!empty($attributeIds)) { + $iterationDeleted = $this->deleteAttributes($attributeIds); + + $deleted += $iterationDeleted; + $totalDeleted += $iterationDeleted; + } + } + + $style->info( + sprintf( + 'Iteration %d: Removed %d duplicate database rows (total removed this execution: %d). [Debug %s]', + $iteration, + $deleted, + $totalDeleted, + $stopwatch->stop('iteration') + ) + ); + + if ($maxIterations !== self::MAX_ITERATIONS_UNLIMITED && ++$iteration > $maxIterations) { + $style->warning('Max iterations count reached. Terminating.'); + + return self::SUCCESS; + } + + // Wait, if needed, before moving to next iteration + usleep($sleep * 1000); + } while ($batchSize === count($attributes)); + + $style->success(sprintf( + 'Operation successful. Removed total of %d duplicate database rows. [Debug %s]', + $totalDeleted, + $stopwatch->stop('total') + )); + + return Command::SUCCESS; + } + + private function getDuplicatedAttributeTotalCount( + SymfonyStyle $style, + Stopwatch $stopwatch + ): int { + $stopwatch->start('total_count', 'sql'); + $query = $this->connection->createQueryBuilder() + ->select('COUNT(a.id) as instances') + ->groupBy('version', 'contentclassattribute_id', 'contentobject_id', 'language_id') + ->from('ezcontentobject_attribute', 'a') + ->having('instances > 1'); + + $count = $query->execute()->rowCount(); + + if ($count > 0) { + $style->warning( + sprintf( + 'Found %d of affected attributes. [Debug: %s]', + $count, + $stopwatch->stop('total_count') + ) + ); + } + + return $count; + } + + /** + * @phpstan-return array + */ + private function getDuplicatedAttributesBatch(int $batchSize): array + { + $query = $this->connection->createQueryBuilder(); + + $query + ->select('version', 'contentclassattribute_id', 'contentobject_id', 'language_id') + ->groupBy('version', 'contentclassattribute_id', 'contentobject_id', 'language_id') + ->from('ezcontentobject_attribute') + ->having('COUNT(id) > 1') + ->setFirstResult(0) + ->setMaxResults($batchSize); + + return $query->execute()->fetchAllAssociative(); + } + + /** + * @phpstan-param array{ + * version: int, + * contentclassattribute_id: int, + * contentobject_id: int, + * language_id: int + * } $attribute + * + * @return int[] + */ + private function getDuplicatedAttributeIds(array $attribute): array + { + $query = $this->connection->createQueryBuilder(); + + $query + ->select('id') + ->from('ezcontentobject_attribute') + ->andWhere('version = :version') + ->andWhere('contentclassattribute_id = :contentclassattribute_id') + ->andWhere('contentobject_id = :contentobject_id') + ->andWhere('language_id = :language_id') + ->orderBy('id', 'ASC') + // Keep the original attribute row, the very first one + ->setFirstResult(1); + + $query->setParameters($attribute); + $result = $query->execute()->fetchFirstColumn(); + + return array_map('intval', $result); + } + + private function askForConfirmation(SymfonyStyle $style): bool + { + $style->warning('Operation is irreversible.'); + + return $style->askQuestion( + new ConfirmationQuestion( + 'Proceed with deletion?', + false + ) + ); + } + + private function deleteAttributes($ids): int + { + $query = $this->connection->createQueryBuilder(); + + $query + ->delete('ezcontentobject_attribute') + ->andWhere($query->expr()->in('id', $ids)); + + return (int)$query->execute(); + } +} From 2fbf8ca9ad32165d30662ec52623469dc47b6c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dawid=20Parafi=C5=84ski?= Date: Tue, 22 Oct 2024 11:42:08 +0200 Subject: [PATCH 2/4] IBX-9103: Fixed cache tag name not including relation type (#437) For more details see https://issues.ibexa.co/browse/IBX-9103 and https://github.com/ibexa/core/pull/437 Key changes: * IBX-9103: Fixed cache tag name not including relation type --- src/lib/Persistence/Cache/ContentHandler.php | 6 ++--- .../settings/storage_engines/cache.yml | 2 +- .../Persistence/Cache/ContentHandlerTest.php | 23 +++++++++++++++---- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/lib/Persistence/Cache/ContentHandler.php b/src/lib/Persistence/Cache/ContentHandler.php index 9047a4c182..a54e6bb293 100644 --- a/src/lib/Persistence/Cache/ContentHandler.php +++ b/src/lib/Persistence/Cache/ContentHandler.php @@ -620,18 +620,18 @@ public function countReverseRelations(int $destinationContentId, ?int $type = nu $cacheItem = $this->cache->getItem( $this->cacheIdentifierGenerator->generateKey( self::CONTENT_REVERSE_RELATIONS_COUNT_IDENTIFIER, - [$destinationContentId], + [$destinationContentId, $type], true ) ); if ($cacheItem->isHit()) { - $this->logger->logCacheHit(['content' => $destinationContentId]); + $this->logger->logCacheHit(['content' => $destinationContentId, 'type' => $type]); return $cacheItem->get(); } - $this->logger->logCacheMiss(['content' => $destinationContentId]); + $this->logger->logCacheMiss(['content' => $destinationContentId, 'type' => $type]); $reverseRelationsCount = $this->persistenceHandler->contentHandler()->countReverseRelations($destinationContentId, $type); $cacheItem->set($reverseRelationsCount); $tags = [ diff --git a/src/lib/Resources/settings/storage_engines/cache.yml b/src/lib/Resources/settings/storage_engines/cache.yml index 48bb148387..aa1a21fec8 100644 --- a/src/lib/Resources/settings/storage_engines/cache.yml +++ b/src/lib/Resources/settings/storage_engines/cache.yml @@ -121,7 +121,7 @@ parameters: content_locations: 'cl-%s' content_relations_count_with_by_version_type_suffix: 'crc-%%s-v-%%s-t-%%s' content_relations_list_with_by_version_type_suffix: 'crl-%%s-l-%%s-o-%%s-v-%%s-t-%%s' - content_reverse_relations_count: 'crrc-%s' + content_reverse_relations_count: 'crrc-%%s-t-%%s' content_version_info: 'cvi-%s' content_version_list: 'c-%s-vl' content_version: 'c-%%s-v-%%s' diff --git a/tests/lib/Persistence/Cache/ContentHandlerTest.php b/tests/lib/Persistence/Cache/ContentHandlerTest.php index c7c43308fa..38ccfcae05 100644 --- a/tests/lib/Persistence/Cache/ContentHandlerTest.php +++ b/tests/lib/Persistence/Cache/ContentHandlerTest.php @@ -93,7 +93,8 @@ public function providerForCachedLoadMethodsHit(): array // string $method, array $arguments, string $key, array? $tagGeneratingArguments, array? $tagGeneratingResults, array? $keyGeneratingArguments, array? $keyGeneratingResults, mixed? $data, bool $multi = false, array $additionalCalls return [ - ['countReverseRelations', [2], 'ibx-crrc-2', null, null, [['content_reverse_relations_count', [2], true]], ['ibx-crrc-2'], 10], + ['countReverseRelations', [2, null], 'ibx-crrc-2-t-', null, null, [['content_reverse_relations_count', [2, null], true]], ['ibx-crrc-2-t-'], 10], + ['countReverseRelations', [2, 8], 'ibx-crrc-2-t-8', null, null, [['content_reverse_relations_count', [2, 8], true]], ['ibx-crrc-2-t-8'], 10], ['countRelations', [2], 'ibx-crc-2-v--t-', null, null, [['content_relations_count_with_by_version_type_suffix', [2, null, null], true]], ['ibx-crc-2-v--t-'], 10], ['countRelations', [2, 2], 'ibx-crc-2-v-2-t-', null, null, [['content_relations_count_with_by_version_type_suffix', [2, 2, null], true]], ['ibx-crc-2-v-2-t-'], 10], ['countRelations', [2, null, 1], 'ibx-crc-2-v--t-1', null, null, [['content_relations_count_with_by_version_type_suffix', [2, null, 1], true]], ['ibx-crc-2-v--t-1'], 10], @@ -136,15 +137,29 @@ public function providerForCachedLoadMethodsMiss(): array [ 'countReverseRelations', [2], - 'ibx-crrc-2', + 'ibx-crrc-2-t-', [ ['content', [2], false], ], ['c-2'], [ - ['content_reverse_relations_count', [2], true], + ['content_reverse_relations_count', [2, null], true], ], - ['ibx-crrc-2'], + ['ibx-crrc-2-t-'], + 10, + ], + [ + 'countReverseRelations', + [2, 8], + 'ibx-crrc-2-t-8', + [ + ['content', [2], false], + ], + ['c-2'], + [ + ['content_reverse_relations_count', [2, 8], true], + ], + ['ibx-crrc-2-t-8'], 10, ], [ From 8c50b0e9ad755b0ba15eeff185fdecc6716c7a91 Mon Sep 17 00:00:00 2001 From: tischsoic Date: Tue, 22 Oct 2024 12:01:17 +0200 Subject: [PATCH 3/4] [PHPStan] Aligned baseline with PHPStan update --- phpstan-baseline.neon | 15 ++++++++++----- phpstan.neon.dist | 2 +- .../Command/VirtualFieldDuplicateFixCommand.php | 11 ++++++++--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon index b1e31685b4..29e79b78ae 100644 --- a/phpstan-baseline.neon +++ b/phpstan-baseline.neon @@ -420,6 +420,16 @@ parameters: count: 1 path: src/bundle/Core/Command/UpdateTimestampsToUTCCommand.php + - + message: "#^Cannot cast Doctrine\\\\DBAL\\\\ForwardCompatibility\\\\Result\\|int\\|string to int\\.$#" + count: 1 + path: src/bundle/Core/Command/VirtualFieldDuplicateFixCommand.php + + - + message: "#^Method Ibexa\\\\Bundle\\\\Core\\\\Command\\\\VirtualFieldDuplicateFixCommand\\:\\:getDuplicatedAttributesBatch\\(\\) should return array\\ but returns array\\\\>\\.$#" + count: 1 + path: src/bundle/Core/Command/VirtualFieldDuplicateFixCommand.php + - message: "#^Method Ibexa\\\\Bundle\\\\Core\\\\Converter\\\\ContentParamConverter\\:\\:getSupportedClass\\(\\) has no return type specified\\.$#" count: 1 @@ -10885,11 +10895,6 @@ parameters: count: 1 path: src/lib/IO/IOMetadataHandler/LegacyDFSCluster.php - - - message: "#^Cannot call method rowCount\\(\\) on Doctrine\\\\DBAL\\\\ForwardCompatibility\\\\Result\\|int\\|string\\.$#" - count: 3 - path: src/lib/IO/IOMetadataHandler/LegacyDFSCluster.php - - message: "#^Method Ibexa\\\\Core\\\\IO\\\\IOMetadataHandler\\\\LegacyDFSCluster\\:\\:delete\\(\\) has no return type specified\\.$#" count: 1 diff --git a/phpstan.neon.dist b/phpstan.neon.dist index 54acab26b1..a8e4451c5c 100644 --- a/phpstan.neon.dist +++ b/phpstan.neon.dist @@ -9,7 +9,7 @@ parameters: treatPhpDocTypesAsCertain: false ignoreErrors: - - message: "#^Cannot call method (fetchOne|fetchColumn|fetchAllAssociative|fetchAssociative|fetchAllKeyValue|fetchFirstColumn)\\(\\) on Doctrine\\\\DBAL\\\\ForwardCompatibility\\\\Result\\|int\\|string\\.$#" + message: "#^Cannot call method (fetchOne|fetchColumn|fetchAllAssociative|fetchAssociative|fetchAllKeyValue|fetchFirstColumn|rowCount)\\(\\) on Doctrine\\\\DBAL\\\\ForwardCompatibility\\\\Result\\|int\\|string\\.$#" paths: - src/* - tests/* diff --git a/src/bundle/Core/Command/VirtualFieldDuplicateFixCommand.php b/src/bundle/Core/Command/VirtualFieldDuplicateFixCommand.php index 2f4d93f58f..4ca0cedadc 100644 --- a/src/bundle/Core/Command/VirtualFieldDuplicateFixCommand.php +++ b/src/bundle/Core/Command/VirtualFieldDuplicateFixCommand.php @@ -164,7 +164,7 @@ private function getDuplicatedAttributeTotalCount( ->from('ezcontentobject_attribute', 'a') ->having('instances > 1'); - $count = $query->execute()->rowCount(); + $count = (int) $query->execute()->rowCount(); if ($count > 0) { $style->warning( @@ -245,13 +245,18 @@ private function askForConfirmation(SymfonyStyle $style): bool ); } - private function deleteAttributes($ids): int + /** + * @param int[] $ids + * + * @throws \Doctrine\DBAL\Exception + */ + private function deleteAttributes(array $ids): int { $query = $this->connection->createQueryBuilder(); $query ->delete('ezcontentobject_attribute') - ->andWhere($query->expr()->in('id', $ids)); + ->andWhere($query->expr()->in('id', array_map('strval', $ids))); return (int)$query->execute(); } From 08ee22ce1617b90d25da446c9ba8228f3bea5ebb Mon Sep 17 00:00:00 2001 From: tischsoic Date: Tue, 22 Oct 2024 13:16:22 +0200 Subject: [PATCH 4/4] Bumped Ibexa LTS version to v4.6.13 --- src/contracts/Ibexa.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/contracts/Ibexa.php b/src/contracts/Ibexa.php index 5ce0fe2c25..fea56a7207 100644 --- a/src/contracts/Ibexa.php +++ b/src/contracts/Ibexa.php @@ -13,5 +13,5 @@ final class Ibexa /** * Ibexa DXP Version. */ - public const VERSION = '4.6.12'; + public const VERSION = '4.6.13'; }