PHRAS-3979: fix bin/maintenance cleaner (#4443)

* fix cleaner memory leak
alchemy-fr · Dec 18, 2023 · 7d72143 · 7d72143
1 parent 02b0445
commit 7d72143
Show file tree

Hide file tree

Showing 6 changed files with 135 additions and 52 deletions.
diff --git a/lib/Alchemy/Phrasea/Command/Maintenance/CleanApiLogsCommand.php b/lib/Alchemy/Phrasea/Command/Maintenance/CleanApiLogsCommand.php
@@ -74,8 +74,16 @@ protected function doExecute(InputInterface $input, OutputInterface $output)
         $output->writeln(sprintf("%d ApiLogs will be deleted.", $n));
 
         if (!$dry) {
-            $n = $this->container->getApplicationBox()->get_connection()->exec($sql_delete);
-            $output->writeln(sprintf("%d ApiLogs have been be deleted.", $n));
+            $cnx = $this->container->getApplicationBox()->get_connection();
+            $count = 0;
+            // group delete by 1000
+            $sql_delete =  $sql_delete . ' LIMIT 1000';
+            do {
+                $nbDeletedRow = $cnx->exec($sql_delete);
+                $count += $nbDeletedRow;
+            } while ($nbDeletedRow > 0);
+
+            $output->writeln(sprintf("%d ApiLogs have been be deleted.", $count));
         } else {
             $output->writeln("Dry mode: Not executed.");
         }

diff --git a/lib/Alchemy/Phrasea/Command/Maintenance/CleanLogDocsCommand.php b/lib/Alchemy/Phrasea/Command/Maintenance/CleanLogDocsCommand.php
@@ -84,38 +84,48 @@ public function doExecute(InputInterface $input, OutputInterface $output)
             $dry = true;
         }
 
-        $sql = 'SELECT id, log_id, `date`, record_id, final, `action` FROM log_docs WHERE ' . $clauseWhere;
-        $sqlDelete = 'DELETE FROM log_docs WHERE ' . $clauseWhere;
-
         $databoxId = $input->getOption('databox_id');
 
         $foundDatabox = false;
         foreach ($this->container->getDataboxes() as $databox) {
             if (empty($databoxId) || (!empty($databoxId) && $databox->get_sbas_id() == $databoxId)) {
                 $foundDatabox = true;
-                $stmt = $databox->get_connection()->prepare($sql);
-                $stmt->execute();
-                $rows = $stmt->fetchAll(\PDO::FETCH_ASSOC);
-                $stmt->closeCursor();
 
                 if ($dry) {
                     // for delete action, delete all event for the records
-                    if (in_array('delete', $action) && !empty($rows)) {
-                        $recordsId = array_column($rows, 'record_id');
-                        $sqlActionDelete = "SELECT id, log_id, `date`, record_id, final, `action` FROM log_docs WHERE record_id IN (" . implode(', ', $recordsId). ") ORDER BY record_id, id";
+                    if (in_array('delete', $action)) {
+                        $sqlRecordIds =  'SELECT DISTINCT record_id FROM log_docs WHERE ' . $clauseWhere ;
+                        $sqlActionDelete = "SELECT id, log_id, `date`, record_id, final, `action` FROM log_docs WHERE record_id IN (" . $sqlRecordIds. ") ORDER BY record_id, id LIMIT 1000";
+
+                        $sqlCount = "SELECT COUNT(`id`) AS n FROM log_docs WHERE record_id IN (" . $sqlRecordIds. ")";
+
                         $stmt = $databox->get_connection()->prepare($sqlActionDelete);
                         $stmt->execute();
-                        $rowsActionDelete = $stmt->fetchAll(\PDO::FETCH_ASSOC);
+                        // result is limited to 1000 rows to avoid memory leak
+                        $displayedRows = $stmt->fetchAll(\PDO::FETCH_ASSOC);
                         $stmt->closeCursor();
                     } else {
-                        $rowsActionDelete = $rows;
+                        $sql = 'SELECT id, log_id, `date`, record_id, final, `action` FROM log_docs WHERE ' . $clauseWhere . ' LIMIT 1000';
+
+                        $sqlCount = 'SELECT COUNT(`id`) AS n FROM log_docs WHERE ' . $clauseWhere;
+
+                        $stmt = $databox->get_connection()->prepare($sql);
+                        $stmt->execute();
+                        // result is limited to 1000 rows to avoid memory leak
+                        $displayedRows = $stmt->fetchAll(\PDO::FETCH_ASSOC);
+                        $stmt->closeCursor();
                     }
 
-                    $output->writeln(sprintf("\n \n dry-run , %d log docs entry to delete for databox %s", count($rowsActionDelete), $databox->get_dbname()));
-                    // displayed only the 1000 first row to avoid memory leak
+                    $stmt = $databox->get_connection()->prepare($sqlCount);
+                    $stmt->execute();
+                    $count = $stmt->fetchColumn(0);
+                    $stmt->closeCursor();
+
 
-                    $displayedRows = array_slice($rowsActionDelete, 0, 1000);
-                    if (count($rows) > 1000) {
+                    $output->writeln(sprintf("\n \n dry-run , %d log docs entry to delete for databox %s", $count, $databox->get_dbname()));
+
+                    // displayed only the 1000 first row to avoid memory leak
+                    if ($count> 1000) {
                         array_push($displayedRows, array_fill_keys(['id', 'log_id', 'date', 'record_id', 'final', 'action'], ' ... '));
                         array_push($displayedRows, array_fill_keys(['id', 'log_id', 'date', 'record_id', 'final', 'action'], ' ... '));
                     }
@@ -128,16 +138,43 @@ public function doExecute(InputInterface $input, OutputInterface $output)
                         ->render($output);
 
                 } else {
-                    if (in_array('delete', $action) && !empty($rows)) {
-                        $recordsId = array_column($rows, 'record_id');
-                        $sqlDeleteAction = 'DELETE FROM log_docs WHERE record_id IN(' . implode(',', $recordsId) . ')';
+                    if (in_array('delete', $action)) {
+                        $sqlRecordIds =  'SELECT DISTINCT record_id FROM log_docs WHERE ' . $clauseWhere . '  LIMIT 1000';
+
+                        $cnx = $databox->get_connection();
+                        $count = 0;
+
+                        do {
+                            $stmt = $cnx->prepare($sqlRecordIds);
+                            $stmt->execute();
+                            $recordsId = $stmt->fetchAll(\PDO::FETCH_COLUMN);
+                            $stmt->closeCursor();
+
+                            if (empty($recordsId)) {
+                                // nothing to delete
+                                break;
+                            }
+
+                            $sqlDeleteAction = 'DELETE FROM log_docs WHERE record_id IN(' . implode(',', $recordsId) . ') LIMIT 1000';
+                            do {
+                                $nbDeletedRow = $cnx->exec($sqlDeleteAction);
+                                $count += $nbDeletedRow;
+                            } while ($nbDeletedRow > 0);
+
+                        } while (1);
+
                     } else {
-                        $sqlDeleteAction = $sqlDelete;
+                        $sqlDeleteAction = 'DELETE FROM log_docs WHERE ' . $clauseWhere . ' LIMIT 1000';
+
+                        $cnx = $databox->get_connection();
+                        $count = 0;
+                        do {
+                            $nbDeletedRow = $cnx->exec($sqlDeleteAction);
+                            $count += $nbDeletedRow;
+                        } while ($nbDeletedRow > 0);
                     }
 
-                    $stmt = $databox->get_connection()->executeQuery($sqlDeleteAction);
-
-                    $output->writeln(sprintf("%d log docs entry deleted on databox %s", $stmt->rowCount(), $databox->get_dbname()));
+                    $output->writeln(sprintf("%d log docs entry deleted on databox %s", $count, $databox->get_dbname()));
                 }
             }
         }

diff --git a/lib/Alchemy/Phrasea/Command/Maintenance/CleanLogSearchCommand.php b/lib/Alchemy/Phrasea/Command/Maintenance/CleanLogSearchCommand.php
@@ -62,25 +62,29 @@ public function doExecute(InputInterface $input, OutputInterface $output)
             $dry = true;
         }
 
-        $sql = 'SELECT id, `date`, search, results, coll_id FROM log_search WHERE ' . $clauseWhere;
-        $sqlDelete = 'DELETE FROM log_search WHERE ' . $clauseWhere;
-
         $databoxId = $input->getOption('databox_id');
         $foundDatabox = false;
         foreach ($this->container->getDataboxes() as $databox) {
             if (empty($databoxId) || (!empty($databoxId) && $databox->get_sbas_id() == $databoxId)) {
                 $foundDatabox = true;
                 if ($dry) {
+                    $sqlCount = 'SELECT COUNT(`id`) FROM log_search WHERE ' . $clauseWhere;
+                    $sql = 'SELECT id, `date`, search, results, coll_id FROM log_search WHERE ' . $clauseWhere . ' LIMIT 1000';
+
+                    $stmt = $databox->get_connection()->prepare($sqlCount);
+                    $stmt->execute();
+                    $count = $stmt->fetchColumn(0);
+                    $stmt->closeCursor();
+
                     $stmt = $databox->get_connection()->prepare($sql);
                     $stmt->execute();
-                    $rows = $stmt->fetchAll(\PDO::FETCH_ASSOC);
+                    $displayedRows = $stmt->fetchAll(\PDO::FETCH_ASSOC);
                     $stmt->closeCursor();
 
-                    $output->writeln(sprintf("\n \n dry-run , %d log search entry to delete for databox %s", count($rows), $databox->get_dbname()));
+                    $output->writeln(sprintf("\n \n dry-run , %d log search entry to delete for databox %s", $count, $databox->get_dbname()));
                     // displayed only the 1000 first row to avoid memory leak
 
-                    $displayedRows = array_slice($rows, 0, 1000);
-                    if (count($rows) > 1000) {
+                    if ($count > 1000) {
                         array_push($displayedRows, array_fill_keys(['id', 'date', 'search', 'results', 'coll_id'], ' ... '));
                         array_push($displayedRows, array_fill_keys(['id', 'date', 'search', 'results', 'coll_id'], ' ... '));
                     }
@@ -92,9 +96,16 @@ public function doExecute(InputInterface $input, OutputInterface $output)
                         ->setRows($displayedRows)
                         ->render($output);
                 } else {
-                    $stmt = $databox->get_connection()->executeQuery($sqlDelete);
-
-                    $output->writeln(sprintf("%d log search entry deleted on databox %s", $stmt->rowCount(), $databox->get_dbname()));
+                    $cnx = $databox->get_connection();
+                    $count = 0;
+                    // group delete by 1000 to avoid memory leak
+                    $sqlDelete = 'DELETE FROM log_search WHERE ' . $clauseWhere . '  LIMIT 1000';
+                    do {
+                        $nbDeletedRow = $cnx->exec($sqlDelete);
+                        $count += $nbDeletedRow;
+                    } while ($nbDeletedRow > 0);
+
+                    $output->writeln(sprintf("%d log search entry deleted on databox %s", $count, $databox->get_dbname()));
                 }
             }
         }

diff --git a/lib/Alchemy/Phrasea/Command/Maintenance/CleanLogViewCommand.php b/lib/Alchemy/Phrasea/Command/Maintenance/CleanLogViewCommand.php
@@ -62,24 +62,28 @@ public function doExecute(InputInterface $input, OutputInterface $output)
             $dry = true;
         }
 
-        $sql = 'SELECT id, `date`, record_id, coll_id FROM log_view WHERE ' . $clauseWhere;
-        $sqlDelete = 'DELETE FROM log_view WHERE ' . $clauseWhere;
-
         $databoxId = $input->getOption('databox_id');
         $foundDatabox = false;
         foreach ($this->container->getDataboxes() as $databox) {
             if (empty($databoxId) || (!empty($databoxId) && $databox->get_sbas_id() == $databoxId)) {
                 $foundDatabox = true;
                 if ($dry) {
+                    $sqlCount = 'SELECT COUNT(`id`) FROM log_view WHERE ' . $clauseWhere;
+                    $stmt = $databox->get_connection()->prepare($sqlCount);
+                    $stmt->execute();
+                    $count = $stmt->fetchColumn(0);
+
+                    $sql = 'SELECT id, `date`, record_id, coll_id FROM log_view WHERE ' . $clauseWhere . ' LIMIT 1000';
+
+                    $stmt->closeCursor();
                     $stmt = $databox->get_connection()->prepare($sql);
                     $stmt->execute();
-                    $rows = $stmt->fetchAll(\PDO::FETCH_ASSOC);
+                    $displayedRows = $stmt->fetchAll(\PDO::FETCH_ASSOC);
                     $stmt->closeCursor();
 
-                    $output->writeln(sprintf("\n \n dry-run , %d log view entry to delete for databox %s", count($rows), $databox->get_dbname()));
+                    $output->writeln(sprintf("\n \n dry-run , %d log view entry to delete for databox %s", $count, $databox->get_dbname()));
                     // displayed only the 1000 first row to avoid memory leak
-                    $displayedRows = array_slice($rows, 0, 1000);
-                    if (count($rows) > 1000) {
+                    if ($count > 1000) {
                         array_push($displayedRows, array_fill_keys(['id', 'date', 'record_id', 'coll_id'], ' ... '));
                         array_push($displayedRows, array_fill_keys(['id', 'date', 'record_id', 'coll_id'], ' ... '));
                     }
@@ -91,9 +95,16 @@ public function doExecute(InputInterface $input, OutputInterface $output)
                         ->render($output);
 
                 } else {
-                    $stmt = $databox->get_connection()->executeQuery($sqlDelete);
-
-                    $output->writeln(sprintf("%d log view entry deleted on databox %s", $stmt->rowCount(), $databox->get_dbname()));
+                    $cnx = $databox->get_connection();
+                    $count = 0;
+                    // group delete by 1000
+                    $sqlDelete = 'DELETE FROM log_view WHERE ' . $clauseWhere . ' LIMIT 1000';
+                    do {
+                        $nbDeletedRow = $cnx->exec($sqlDelete);
+                        $count += $nbDeletedRow;
+                    } while ($nbDeletedRow > 0);
+
+                    $output->writeln(sprintf("%d log view entry deleted on databox %s", $count, $databox->get_dbname()));
                 }
             }
         }

diff --git a/lib/Alchemy/Phrasea/Command/Maintenance/CleanNotificationsCommand.php b/lib/Alchemy/Phrasea/Command/Maintenance/CleanNotificationsCommand.php
@@ -118,11 +118,18 @@ protected function doExecute(InputInterface $input, OutputInterface $output)
 
         $output->writeln(sprintf("%d notifications will be deleted.", $n));
 
-        if(!$dry) {
-            $n = $this->container->getApplicationBox()->get_connection()->exec($sql_delete);
-            $output->writeln(sprintf("%d notifications have been be deleted.", $n));
-        }
-        else {
+        if (!$dry) {
+            $cnx = $this->container->getApplicationBox()->get_connection();
+            $count = 0;
+            // group delete by 1000
+            $sql_delete =  $sql_delete . ' LIMIT 1000';
+            do {
+                $nbDeletedRow = $cnx->exec($sql_delete);
+                $count += $nbDeletedRow;
+            } while ($nbDeletedRow > 0);
+
+            $output->writeln(sprintf("%d notifications have been be deleted.", $count));
+        } else {
             $output->writeln("Dry mode: Not executed.");
         }
 

diff --git a/lib/Alchemy/Phrasea/Command/Maintenance/CleanWorkerRunningJobCommand.php b/lib/Alchemy/Phrasea/Command/Maintenance/CleanWorkerRunningJobCommand.php
@@ -75,8 +75,17 @@ protected function doExecute(InputInterface $input, OutputInterface $output)
         $output->writeln(sprintf("%d WorkerRunningJob will be deleted.", $n));
 
         if (!$dry) {
-            $n = $this->container->getApplicationBox()->get_connection()->exec($sql_delete);
-            $output->writeln(sprintf("%d WorkerRunningJob have been be deleted.", $n));
+            $cnx = $this->container->getApplicationBox()->get_connection();
+
+            $count = 0;
+            // group delete by 1000
+            $sql_delete =  $sql_delete . ' LIMIT 1000';
+            do {
+                $nbDeletedRow = $cnx->exec($sql_delete);
+                $count += $nbDeletedRow;
+            } while ($nbDeletedRow > 0);
+
+            $output->writeln(sprintf("%d WorkerRunningJob have been be deleted.", $count));
         } else {
             $output->writeln("Dry mode: Not executed.");
         }