From 555646232e2e5ba88f148a68a3458c405e0e1d35 Mon Sep 17 00:00:00 2001 From: jygaulier Date: Wed, 17 Jul 2024 17:25:42 +0200 Subject: [PATCH] PHRAS-4085_data-volumes-api (#4529) * new route `/api/v3/monitor/data/?oauth_token=xxx&blocksize=16ko` * add `unit` parameter for size restults ('', 'o', ''ko', 'mo', 'go'), default '' (=octets, same as 'o') * new units "octet", "octets" * fix round error (don't sum rounded values: round the real sum) * add infos about downloads; round sizes to 2 decimals * add `...&details=1` url parameter to get values by collection and subdef name * add column size in LazaretFiles table add command bin/maintenance lazaret:set_sizes * add api monitor data by databox * fix size --- bin/maintenance | 3 + lib/Alchemy/Phrasea/Border/Manager.php | 2 + .../LazaretFilesSetSizeCommand.php | 59 ++++ .../Api/V3/V3MonitorDataController.php | 318 ++++++++++++++++++ .../Phrasea/ControllerProvider/Api/V3.php | 19 ++ .../Phrasea/Model/Entities/LazaretFile.php | 28 ++ 6 files changed, 429 insertions(+) create mode 100644 lib/Alchemy/Phrasea/Command/Maintenance/LazaretFilesSetSizeCommand.php create mode 100644 lib/Alchemy/Phrasea/Controller/Api/V3/V3MonitorDataController.php diff --git a/bin/maintenance b/bin/maintenance index ed76b69e8f..30daa3fd6f 100755 --- a/bin/maintenance +++ b/bin/maintenance @@ -15,6 +15,7 @@ use Alchemy\Phrasea\Command\Maintenance\CleanRightsCommand; use Alchemy\Phrasea\Command\Maintenance\CleanWebhookLogsCommand; use Alchemy\Phrasea\Command\Maintenance\CleanWorkerRunningJobCommand; use Alchemy\Phrasea\Command\Maintenance\SessionsCommand; +use Alchemy\Phrasea\Command\Maintenance\LazaretFilesSetSizeCommand; require_once __DIR__ . '/../lib/autoload.php'; @@ -59,4 +60,6 @@ $cli->command(new CleanLogViewCommand()); $cli->command(new CleanWebhookLogsCommand()); +$cli->command(new LazaretFilesSetSizeCommand()); + $cli->run(); diff --git a/lib/Alchemy/Phrasea/Border/Manager.php b/lib/Alchemy/Phrasea/Border/Manager.php index 001974d863..8689a229be 100644 --- a/lib/Alchemy/Phrasea/Border/Manager.php +++ b/lib/Alchemy/Phrasea/Border/Manager.php @@ -406,6 +406,8 @@ protected function createLazaret(File $file, Visa $visa, LazaretSession $session $lazaretFile->setSession($session); + $lazaretFile->setSize($file->getFile()->getSize()); + $this->app['orm.em']->persist($lazaretFile); foreach ($file->getAttributes() as $fileAttribute) { diff --git a/lib/Alchemy/Phrasea/Command/Maintenance/LazaretFilesSetSizeCommand.php b/lib/Alchemy/Phrasea/Command/Maintenance/LazaretFilesSetSizeCommand.php new file mode 100644 index 0000000000..be90198b86 --- /dev/null +++ b/lib/Alchemy/Phrasea/Command/Maintenance/LazaretFilesSetSizeCommand.php @@ -0,0 +1,59 @@ +setDescription('Set the null size in the LazaretFiles table') + ->addOption('dry', null, InputOption::VALUE_NONE, 'dry run, count') + + ->setHelp(''); + } + + public function doExecute(InputInterface $input, OutputInterface $output) + { + /** @var LazaretFileRepository $lazaretRepository */ + $lazaretRepository = $this->container['repo.lazaret-files']; + + $lazaretNullSizes = $lazaretRepository->findBy(['size' => null]); + + $path = $this->container['tmp.lazaret.path']; + /** @var EntityManager $em */ + $em = $this->container['orm.em']; + + if (!$input->getOption('dry')) { + /** @var LazaretFile $lazaretNullSize */ + foreach ($lazaretNullSizes as $lazaretNullSize) { + try { + $lazaretFileName = $path .'/'.$lazaretNullSize->getFilename(); + $media = $this->container->getMediaFromUri($lazaretFileName); + $size = $media->getFile()->getSize(); + } catch (\Exception $e) { + $size = 0; + } + + $lazaretNullSize->setSize($size); + $em->persist($lazaretNullSize); + } + + $em->flush(); + + $output->writeln(sprintf("%d LazaretFiles done!", count($lazaretNullSizes))); + } else { + $output->writeln(sprintf("%d LazaretFiles to update!", count($lazaretNullSizes))); + } + } +} diff --git a/lib/Alchemy/Phrasea/Controller/Api/V3/V3MonitorDataController.php b/lib/Alchemy/Phrasea/Controller/Api/V3/V3MonitorDataController.php new file mode 100644 index 0000000000..6c3a5b80ad --- /dev/null +++ b/lib/Alchemy/Phrasea/Controller/Api/V3/V3MonitorDataController.php @@ -0,0 +1,318 @@ +1, 'o'=>1, 'octet'=>1, 'octets'=>1, 'ko'=>1<<10, 'mo'=>1<<20, 'go'=>1<<30]; + try { + return $map[strtolower($unit)]; + } + catch (\Exception $e) { + return false; + } + } + + /** + * monitor infos for app + * + * @param Request $request + * + * @return Response + */ + public function indexAction(Request $request) + { + $stopwatch = new Stopwatch("controller"); + + list($getDetails, $blocksize, $divider, $sqlDivider, $unit, $sqlByColl, $sqlByName, $sqlByDb) = $this->getParamsFromRequest($request); + + $ret = [ + 'unit' => $divider === 1 ? $unit : ucfirst($unit), // octet => octet ; mo => Mo + 'databoxes' => [] + ]; + + foreach ($this->app->getDataboxes() as $databox) { + // get volumes by db + + $stmt = $databox->get_connection()->prepare($sqlByDb); + $stmt->execute(); + $row = $stmt->fetch(PDO::FETCH_ASSOC); + $stmt->closeCursor(); + + $ret['databoxes'][$databox->get_sbas_id()] = [ + 'sbas_id' => $databox->get_sbas_id(), + 'viewname' => $databox->get_viewname(), + 'count' => (int)$row['n'], + 'size' => round($row['size'], 2), + 'disksize' => round($row['disksize'], 2) + ]; + + if ($getDetails) { + list($collections, $subdefs) = $this->getVolumeDetails($databox, $sqlByColl, $sqlByName); + + $ret['databoxes'][$databox->get_sbas_id()]['collections'] = $collections; + $ret['databoxes'][$databox->get_sbas_id()]['subdefs'] = $subdefs; + } + } + + // get volumes of downloads + + $sql = "SELECT `data` FROM `Tokens` WHERE `type`='download'"; + $stmt = $this->getApplicationBox()->get_connection()->prepare($sql); + $stmt->execute(); + $size = 0; + $disksize = 0; + $n = 0; + while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) { + try { + $data = unserialize($row['data']); + $size += $data['size']; + $disksize += ceil($data['size'] / $blocksize) * $blocksize; + $n++; + } + catch (\Exception $e) { + // ignore + } + } + $stmt->closeCursor(); + + $sql = "SELECT DATEDIFF(NOW(), MIN(`created`)) AS `oldest`, SUM(IF(NOW()>`expiration`, 1, 0)) AS `expired` FROM `Tokens` WHERE `type`='download'"; + $stmt = $this->getApplicationBox()->get_connection()->prepare($sql); + $stmt->execute(); + $row = $stmt->fetch(PDO::FETCH_ASSOC); + $stmt->closeCursor(); + + $ret['downloads'] = [ + 'count' => $n, + 'days_oldest' => (int)$row['oldest'], + 'expired' => (int)$row['expired'], + 'size' => round($size / $divider, 2), + 'disksize' => round($disksize / $divider, 2) + ]; + + $sql = "SELECT count(*) AS n , SUM(`size`) " . $sqlDivider . " AS size, " + . " SUM(CEIL(`size` / " . $blocksize . ") * " . $blocksize . ") " . $sqlDivider . " AS disksize" + . " FROM `LazaretFiles` WHERE size IS NOT NULL"; + + $stmt = $this->getApplicationBox()->get_connection()->prepare($sql); + $stmt->execute(); + + $row = $stmt->fetch(PDO::FETCH_ASSOC); + $stmt->closeCursor(); + + $ret['lazaret'] = [ + 'count' => $row['n'], + 'size' => round($row['size'], 2), + 'disksize' => round($row['disksize'], 2), + ]; + + return Result::create($request, $ret)->createResponse([$stopwatch]); + } + + /** + * monitor info for app by databox + * @param Request $request + */ + public function perDataboxAction(Request $request) + { + $stopwatch = new Stopwatch("controller"); + $databoxId = $request->get('databox_id'); + + list($getDetails, $blocksize, $divider, $sqlDivider, $unit, $sqlByColl, $sqlByName, $sqlByDb) = $this->getParamsFromRequest($request); + + $ret = [ + 'unit' => $divider === 1 ? $unit : ucfirst($unit), // octet => octet ; mo => Mo + 'databox' => [] + ]; + + $databox = $this->findDataboxById($databoxId); + + // get volumes by db + + $stmt = $databox->get_connection()->prepare($sqlByDb); + $stmt->execute(); + $row = $stmt->fetch(PDO::FETCH_ASSOC); + $stmt->closeCursor(); + + $ret['databox'] = [ + 'sbas_id' => $databox->get_sbas_id(), + 'viewname' => $databox->get_viewname(), + 'count' => (int)$row['n'], + 'size' => round($row['size'], 2), + 'disksize' => round($row['disksize'], 2) + ]; + + if ($getDetails) { + list($collections, $subdefs) = $this->getVolumeDetails($databox, $sqlByColl, $sqlByName); + + $ret['databox']['collections'] = $collections; + $ret['databox']['subdefs'] = $subdefs; + } + + // get volumes of downloads + + $sql = "SELECT `data` FROM `Tokens` WHERE `type`='download'"; + $stmt = $this->getApplicationBox()->get_connection()->prepare($sql); + $stmt->execute(); + $size = 0; + $disksize = 0; + $n = 0; + while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) { + try { + $found = false; + $data = unserialize($row['data']); + foreach ($data['files'] as $file) { + // get only for the needed databoxId + if ($file['databox_id'] == $databoxId) { + $found = true; + foreach ($file['subdefs'] as $subdef) { + $size += $subdef['size']; + $disksize += ceil($subdef['size'] / $blocksize) * $blocksize; + } + } + } + + if ($found) { + $n++; + } + } + catch (\Exception $e) { + // ignore + } + } + + $stmt->closeCursor(); + + $ret['downloads'] = [ + 'sbas_id' => $databoxId, + 'count' => $n, + 'size' => round($size / $divider, 2), + 'disksize' => round($disksize / $divider, 2) + ]; + + // get lazaret volume for the databox + + $sql = "SELECT count(*) AS n , SUM(`L`.`size`) " . $sqlDivider . " AS size, ". + " SUM(CEIL(`L`.`size` / " . $blocksize . ") * " . $blocksize . ") " . $sqlDivider . " AS disksize" . + " FROM `LazaretFiles` AS L ". + " LEFT JOIN `bas` AS b ON L.`base_id`=b.`base_id`". + " WHERE L.`size` IS NOT NULL AND b.`sbas_id`=". $databoxId; + + + $stmt = $this->getApplicationBox()->get_connection()->prepare($sql); + $stmt->execute(); + + $row = $stmt->fetch(PDO::FETCH_ASSOC); + $stmt->closeCursor(); + + $ret['lazaret'] = [ + 'sbas_id' => $databoxId, + 'count' => $row['n'], + 'size' => round($row['size'], 2), + 'disksize' => round($row['disksize'], 2), + ]; + + return Result::create($request, $ret)->createResponse([$stopwatch]); + } + + private function getParamsFromRequest(Request $request) + { + $getDetails = $request->get('details', '0') === '1'; + + $matches = []; + if(preg_match("/^(\\d+)\\s*([a-z]*)$/i", $request->get('blocksize', '1'), $matches) !== 1) { + throw new Exception("bad 'blocksize' parameter"); + } + $matches[] = ''; // if no unit, force + if(($mutiplier = $this->unitToMultiplier($matches[2])) === false) { + throw new Exception("bad 'blocksize' unit"); + } + $blocksize = (int)($matches[1]) * $mutiplier; + + if( ($divider = $this->unitToMultiplier($unit = $request->get('unit', '')) ) === false) { + throw new Exception("bad 'unit' parameter"); + } + $sqlDivider = $divider === 1 ? '' : (' / ' . $divider); + + $sqlByColl = ""; + $sqlByName = ""; + + if ($getDetails) { + + $sqlByColl = "SELECT COALESCE(r.`coll_id`, '?') AS `coll_id`, + COALESCE(c.`asciiname`, CONCAT('_',r.`coll_id`), '?') AS `asciiname`, s.`name`, + SUM(1) AS n, SUM(s.`size`) " . $sqlDivider . " AS `size`, + SUM(CEIL(s.`size` / " . $blocksize . ") * " . $blocksize . ") " . $sqlDivider . " AS `disksize` + FROM `subdef` AS s LEFT JOIN `record` AS r ON r.`record_id`=s.`record_id` + LEFT JOIN `coll` AS c ON r.`coll_id`=c.`coll_id` + GROUP BY r.`coll_id`, s.`name`;"; + + $sqlByName = "SELECT s.`name`, + SUM(1) AS n, SUM(s.`size`) " . $sqlDivider . " AS `size`, + SUM(CEIL(s.`size` / " . $blocksize . ") * " . $blocksize . ") " . $sqlDivider . " AS `disksize` + FROM `subdef` AS s + GROUP BY s.`name`;"; + } + + $sqlByDb = "SELECT SUM(1) AS n, SUM(s.`size`) " . $sqlDivider . " AS `size`, + SUM(CEIL(s.`size` / " . $blocksize . ") * " . $blocksize . ") " . $sqlDivider . " AS `disksize` + FROM `subdef` AS s"; + + return [$getDetails, $blocksize, $divider, $sqlDivider, $unit, $sqlByColl, $sqlByName, $sqlByDb]; + } + + private function getVolumeDetails(\databox $databox, $sqlByColl, $sqlByName) + { + // get volumes grouped by collection and subdef + + $collections = []; + $stmt = $databox->get_connection()->prepare($sqlByColl); + $stmt->execute(); + while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) { + if (!array_key_exists($row['coll_id'], $collections)) { + $collections[$row['coll_id']] = [ + 'coll_id' => $row['coll_id'], + 'name' => $row['asciiname'], + 'subdefs' => [] + ]; + } + $collections[$row['coll_id']]['subdefs'][$row['name']] = [ + 'count' => (int)$row['n'], + 'size' => round($row['size'], 2), + 'disksize' => round($row['disksize'], 2) + ]; + } + $stmt->closeCursor(); + + // get volumes by subdef + + $subdefs = []; + $stmt = $databox->get_connection()->prepare($sqlByName); + $stmt->execute(); + while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) { + $subdefs[$row['name']]['count'] = (int)$row['n']; + $subdefs[$row['name']]['size'] = round($row['size'], 2); + $subdefs[$row['name']]['disksize'] = round($row['disksize'], 2); + } + $stmt->closeCursor(); + + return [$collections, $subdefs]; + } +} diff --git a/lib/Alchemy/Phrasea/ControllerProvider/Api/V3.php b/lib/Alchemy/Phrasea/ControllerProvider/Api/V3.php index dafff86a2f..211d3113a4 100644 --- a/lib/Alchemy/Phrasea/ControllerProvider/Api/V3.php +++ b/lib/Alchemy/Phrasea/ControllerProvider/Api/V3.php @@ -5,6 +5,7 @@ use Alchemy\Phrasea\Application as PhraseaApplication; use Alchemy\Phrasea\Controller\Api\V1Controller; use Alchemy\Phrasea\Controller\Api\V3\V3Controller; +use Alchemy\Phrasea\Controller\Api\V3\V3MonitorDataController; use Alchemy\Phrasea\Controller\Api\V3\V3RecordController; use Alchemy\Phrasea\Controller\Api\V3\V3ResultHelpers; use Alchemy\Phrasea\Controller\Api\V3\V3SearchController; @@ -50,6 +51,9 @@ public function register(Application $app) ->setInstanceId($app['conf']) ; }); + $app['controller.api.v3.monitorData'] = $app->share(function (PhraseaApplication $app) { + return (new V3MonitorDataController($app)); + }); $app['controller.api.v3.searchraw'] = $app->share(function (PhraseaApplication $app) { return (new V3SearchRawController($app)); }); @@ -96,6 +100,21 @@ public function connect(Application $app) ->assert('record_id', '\d+') ->value('must_be_story', true); + /** + * @uses V3MonitorDataController::indexAction() + */ + $controllers->get('/monitor/data/', 'controller.api.v3.monitorData:indexAction') + ->before('controller.api.v1:ensureAdmin') + ; + + /** + * @uses V3MonitorDataController::perDataboxAction() + */ + $controllers->get('databoxes/{databox_id}/monitor/data/', 'controller.api.v3.monitorData:perDataboxAction') + ->before('controller.api.v1:ensureAdmin') + ->assert('databox_id', '\d+') + ; + /** * @uses V3SearchController::helloAction() */ diff --git a/lib/Alchemy/Phrasea/Model/Entities/LazaretFile.php b/lib/Alchemy/Phrasea/Model/Entities/LazaretFile.php index 676c7d6625..9246eed15a 100644 --- a/lib/Alchemy/Phrasea/Model/Entities/LazaretFile.php +++ b/lib/Alchemy/Phrasea/Model/Entities/LazaretFile.php @@ -96,6 +96,11 @@ class LazaretFile */ private $session; + /** + * @ORM\Column(type="bigint", nullable=true) + */ + private $size; + /** * Constructor */ @@ -322,6 +327,29 @@ public function setUpdated(\DateTime $updated) return $this; } + /** + * Set size + * + * @param integer $size + * @return LazaretFile + */ + public function setSize($size) + { + $this->size = $size; + + return $this; + } + + /** + * Get size + * + * @return integer + */ + public function getSize() + { + return $this->size; + } + /** * Get updated *