diff --git a/core/Archive.php b/core/Archive.php index 506bb129469..43249603fd8 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -436,20 +436,21 @@ public function getDataTableExpanded($name, $idSubtable = null, $depth = null, $ } /** - * Returns the list of plugins that archive the given reports. + * Returns the given reports grouped by the plugin name that archives them. * * @param array $archiveNames - * @return array + * @return array `['MyPlugin' => ['MyPlugin_metric1', 'MyPlugin_report1'], ...]` */ private function getRequestedPlugins($archiveNames) { $result = []; foreach ($archiveNames as $name) { - $result[] = self::getPluginForReport($name); + $plugin = self::getPluginForReport($name); + $result[$plugin][] = $name; } - return array_unique($result); + return array_map('array_unique', $result); } /** @@ -601,7 +602,8 @@ protected function get($archiveNames, $archiveDataType, $idSubtable = null) */ private function getArchiveIds($archiveNames) { - $plugins = $this->getRequestedPlugins($archiveNames); + $archiveNamesByPlugin = $this->getRequestedPlugins($archiveNames); + $plugins = array_keys($archiveNamesByPlugin); // figure out which archives haven't been processed (if an archive has been processed, // then we have the archive IDs in $this->idarchives) @@ -627,15 +629,13 @@ private function getArchiveIds($archiveNames) $globalDoneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($this->params->getSegment()); $doneFlags[$globalDoneFlag] = true; - $archiveGroups = array_unique($archiveGroups); - // cache id archives for plugins we haven't processed yet if (!empty($archiveGroups)) { if ( Rules::isArchivingEnabledFor($this->params->getIdSites(), $this->params->getSegment(), $this->getPeriodLabel()) && !$this->forceFetchingWithoutLaunchingArchiving ) { - $this->cacheArchiveIdsAfterLaunching($archiveGroups, $plugins); + $this->cacheArchiveIdsAfterLaunching($archiveNamesByPlugin); } else { $this->cacheArchiveIdsWithoutLaunching($plugins); } @@ -651,10 +651,9 @@ private function getArchiveIds($archiveNames) * This function will launch the archiving process for each period/site/plugin if * metrics/reports have not been calculated/archived already. * - * @param array $archiveGroups @see getArchiveGroupOfReport - * @param array $plugins List of plugin names to archive. + * @param array $archiveNamesByPlugin @see getRequestedPlugins */ - private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins) + private function cacheArchiveIdsAfterLaunching($archiveNamesByPlugin) { foreach ($this->params->getPeriods() as $period) { $twoDaysAfterPeriod = $period->getDateEnd()->addDay(2); @@ -697,7 +696,7 @@ private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins) continue; } - $this->prepareArchive($archiveGroups, $site, $period); + $this->prepareArchive($archiveNamesByPlugin, $site, $period); } } } @@ -744,6 +743,15 @@ private function cacheArchiveIdsWithoutLaunching($plugins) */ private function getDoneStringForPlugin($plugin, $idSites) { + $requestedReport = $this->getRequestedReport(); + + $shouldOnlyProcessRequestedArchives = empty($requestedReport) + && Rules::shouldProcessOnlyReportsRequestedInArchiveQuery($this->getPeriodLabel()); + + if ($shouldOnlyProcessRequestedArchives) { + return Rules::getDoneFlagArchiveContainsOnePlugin($this->params->getSegment(), $plugin); + } + return Rules::getDoneStringFlagFor( $idSites, $this->params->getSegment(), @@ -881,18 +889,18 @@ public static function getPluginForReport($report) } /** - * @param $archiveGroups + * @param $archiveNamesByPlugin * @param $site * @param $period */ - private function prepareArchive(array $archiveGroups, Site $site, Period $period) + private function prepareArchive(array $archiveNamesByPlugin, Site $site, Period $period) { $coreAdminHomeApi = API::getInstance(); - $requestedReport = null; - if (SettingsServer::isArchivePhpTriggered()) { - $requestedReport = Common::getRequestVar('requestedReport', '', 'string'); - } + $requestedReport = $this->getRequestedReport(); + + $shouldOnlyProcessRequestedArchives = empty($requestedReport) + && Rules::shouldProcessOnlyReportsRequestedInArchiveQuery($period->getLabel()); $periodString = $period->getRangeString(); $periodDateStr = $period->getLabel() == 'range' ? $periodString : $period->getDateStart()->toString(); @@ -900,17 +908,19 @@ private function prepareArchive(array $archiveGroups, Site $site, Period $period $idSites = [$site->getId()]; // process for each plugin as well - foreach ($archiveGroups as $plugin) { + foreach ($archiveNamesByPlugin as $plugin => $archiveNames) { $doneFlag = $this->getDoneStringForPlugin($plugin, $idSites); $this->initializeArchiveIdCache($doneFlag); + $reportsToArchiveForThisPlugin = (empty($requestedReport) && $shouldOnlyProcessRequestedArchives) ? $archiveNames : $requestedReport; + $prepareResult = $coreAdminHomeApi->archiveReports( $site->getId(), $period->getLabel(), $periodDateStr, $this->params->getSegment()->getOriginalString(), $plugin, - $requestedReport + $reportsToArchiveForThisPlugin ); if ( @@ -918,7 +928,11 @@ private function prepareArchive(array $archiveGroups, Site $site, Period $period && !empty($prepareResult['idarchives']) ) { foreach ($prepareResult['idarchives'] as $idArchive) { - $this->idarchives[$doneFlag][$periodString][] = $idArchive; + if (empty($this->idarchives[$doneFlag][$periodString]) + || !in_array($idArchive, $this->idarchives[$doneFlag][$periodString]) + ) { + $this->idarchives[$doneFlag][$periodString][] = $idArchive; + } } } } @@ -956,4 +970,13 @@ public function forceFetchingWithoutLaunchingArchiving() { $this->forceFetchingWithoutLaunchingArchiving = true; } + + private function getRequestedReport(): ?string + { + $requestedReport = null; + if (SettingsServer::isArchivePhpTriggered()) { + $requestedReport = Request::fromRequest()->getStringParameter('requestedReport', ''); + } + return $requestedReport; + } } diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php index fdfef26927a..1d335840481 100644 --- a/core/ArchiveProcessor.php +++ b/core/ArchiveProcessor.php @@ -262,7 +262,8 @@ public function aggregateDataTableRecords($recordNames, * as metrics for the current period. * * @param array|string $columns Array of metric names to aggregate. - * @param bool|string $operationToApply The operation to apply to the metric. Either `'sum'`, `'max'` or `'min'`. + * @param bool|string|string[] $operationToApply The operation to apply to the metric. Either `'sum'`, `'max'` or `'min'`. + * Can also be an array mapping record names to operations. * @return array|int Returns the array of aggregate values. If only one metric was aggregated, * the aggregate value will be returned as is, not in an array. * For example, if `array('nb_visits', 'nb_hits')` is supplied for `$columns`, @@ -276,9 +277,9 @@ public function aggregateDataTableRecords($recordNames, * then `3040` would be returned. * @api */ - public function aggregateNumericMetrics($columns, $operationToApply = false) + public function aggregateNumericMetrics($columns, $operationsToApply = false) { - $metrics = $this->getAggregatedNumericMetrics($columns, $operationToApply); + $metrics = $this->getAggregatedNumericMetrics($columns, $operationsToApply); foreach ($metrics as $column => $value) { $this->insertNumericRecord($column, $value); @@ -489,7 +490,7 @@ protected function getOperationForColumns($columns, $defaultOperation) { $operationForColumn = array(); foreach ($columns as $name) { - $operation = $defaultOperation; + $operation = is_array($defaultOperation) ? ($defaultOperation[$name] ?? null) : $defaultOperation; if (empty($operation)) { $operation = $this->guessOperationForColumn($name); } @@ -688,13 +689,13 @@ public function renameColumnsAfterAggregation(DataTable $table, $columnsToRename } } - protected function getAggregatedNumericMetrics($columns, $operationToApply) + protected function getAggregatedNumericMetrics($columns, $operationsToApply) { if (!is_array($columns)) { $columns = array($columns); } - $operationForColumn = $this->getOperationForColumns($columns, $operationToApply); + $operationForColumn = $this->getOperationForColumns($columns, $operationsToApply); $dataTable = $this->getArchive()->getDataTableFromNumeric($columns); @@ -740,6 +741,12 @@ public function processDependentArchive($plugin, $segment) } $params = $this->getParams(); + // range archives are always processed on demand, so pre-processing dependent archives is not required + // here + if (Rules::shouldProcessOnlyReportsRequestedInArchiveQuery($params->getPeriod()->getLabel())) { + return; + } + $idSites = [$params->getSite()->getId()]; // important to use the original segment string when combining. As the API itself would combine the original string. diff --git a/core/ArchiveProcessor/Loader.php b/core/ArchiveProcessor/Loader.php index 0dfd5f4e250..4a3b60e8130 100644 --- a/core/ArchiveProcessor/Loader.php +++ b/core/ArchiveProcessor/Loader.php @@ -134,7 +134,7 @@ private function prepareArchiveImpl($pluginName) if (sizeof($data) == 2) { return $data; } - list($idArchives, $visits, $visitsConverted) = $data; + list($idArchives, $visits, $visitsConverted, $foundRecords) = $data; // only lock meet those conditions if (ArchiveProcessor::$isRootArchivingRequest && !SettingsServer::isArchivePhpTriggered()) { @@ -153,15 +153,15 @@ private function prepareArchiveImpl($pluginName) return $data; } - list($idArchives, $visits, $visitsConverted) = $data; + list($idArchives, $visits, $visitsConverted, $foundRecords) = $data; - return $this->insertArchiveData($visits, $visitsConverted); + return $this->insertArchiveData($visits, $visitsConverted, $idArchives, $foundRecords); } finally { $lock->unlock(); } } else { - return $this->insertArchiveData($visits, $visitsConverted); + return $this->insertArchiveData($visits, $visitsConverted, $idArchives, $foundRecords); } } @@ -171,17 +171,27 @@ private function prepareArchiveImpl($pluginName) * @param $visitsConverted * @return array|false[] */ - protected function insertArchiveData($visits, $visitsConverted) + protected function insertArchiveData($visits, $visitsConverted, $existingArchives, $foundRecords) { if (SettingsServer::isArchivePhpTriggered()) { $this->logger->info("initiating archiving via core:archive for " . $this->params); } + if (!empty($foundRecords)) { + $this->params->setFoundRequestedReports($foundRecords); + } + list($visits, $visitsConverted) = $this->prepareCoreMetricsArchive($visits, $visitsConverted); list($idArchive, $visits) = $this->prepareAllPluginsArchive($visits, $visitsConverted); - if ($this->isThereSomeVisits($visits) || PluginsArchiver::doesAnyPluginArchiveWithoutVisits()) { - return [[$idArchive], $visits]; + if ($this->isThereSomeVisits($visits) + || PluginsArchiver::doesAnyPluginArchiveWithoutVisits() + ) { + $idArchivesToQuery = [$idArchive]; + if (!empty($foundRecords)) { + $idArchivesToQuery = array_merge($idArchivesToQuery, $existingArchives ?: []); + } + return [$idArchivesToQuery, $visits]; } return [false, false]; @@ -208,11 +218,22 @@ protected function loadArchiveData() // this hack was used to check the main function goes to return or continue // NOTE: $idArchives will contain the latest DONE_OK/DONE_INVALIDATED archive as well as any partial archives // with a ts_archived >= the DONE_OK/DONE_INVALIDATED date. - list($idArchives, $visits, $visitsConverted, $isAnyArchiveExists, $tsArchived, $value) = $this->loadExistingArchiveIdFromDb(); + $archiveInfo = $this->loadExistingArchiveIdFromDb(); + $idArchives = $archiveInfo['idArchives']; + $visits = $archiveInfo['visits']; + $visitsConverted = $archiveInfo['visitsConverted']; + $tsArchived = $archiveInfo['tsArchived']; + $doneFlagValue = $archiveInfo['doneFlagValue']; + $existingArchives = $archiveInfo['existingRecords']; + + $requestedRecords = $this->params->getArchiveOnlyReportAsArray(); + $isMissingRequestedRecords = !empty($requestedRecords) && is_array($existingArchives) && count($requestedRecords) != count($existingArchives); if (!empty($idArchives) && !Rules::isActuallyForceArchivingSinglePlugin() - && !$this->shouldForceInvalidatedArchive($value, $tsArchived)) { + && !$this->shouldForceInvalidatedArchive($doneFlagValue, $tsArchived) + && !$isMissingRequestedRecords + ) { // we have a usable idarchive (it's not invalidated and it's new enough), and we are not archiving // a single report return [$idArchives, $visits]; @@ -232,7 +253,7 @@ protected function loadArchiveData() } } - return [$idArchives, $visits, $visitsConverted]; + return [$idArchives, $visits, $visitsConverted, $existingArchives]; } /** @@ -331,7 +352,14 @@ public function loadExistingArchiveIdFromDb() // return no usable archive found, and no existing archive. this will skip invalidation, which should // be fine since we just force archiving. - return [false, false, false, false, false, false]; + return [ + 'idArchives' => false, + 'visits' => false, + 'visitsConverted' => false, + 'archiveExists' => false, + 'tsArchived' => false, + 'doneFlagValue' => false, + ]; } $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed(); diff --git a/core/ArchiveProcessor/Parameters.php b/core/ArchiveProcessor/Parameters.php index 8cdb522badf..ea8c6703f23 100644 --- a/core/ArchiveProcessor/Parameters.php +++ b/core/ArchiveProcessor/Parameters.php @@ -55,6 +55,11 @@ class Parameters */ private $isArchiveOnlyReportHandled; + /** + * @var string[]|null + */ + private $foundRequestedReports; + /** * Constructor. * @@ -71,7 +76,7 @@ public function __construct(Site $site, Period $period, Segment $segment) * If we want to archive only a single report, we can request that via this method. * It is up to each plugin's archiver to respect the setting. * - * @param string $archiveOnlyReport + * @param string|string[] $archiveOnlyReport * @api */ public function setArchiveOnlyReport($archiveOnlyReport) @@ -267,7 +272,11 @@ public function logStatusDebug() public function __toString() { - return "[idSite = {$this->getSite()->getId()}, period = {$this->getPeriod()->getLabel()} {$this->getPeriod()->getRangeString()}, segment = {$this->getSegment()->getString()}, plugin = {$this->getRequestedPlugin()}, report = {$this->getArchiveOnlyReport()}]"; + $requestedReports = $this->getArchiveOnlyReport(); + if (is_array($requestedReports)) { + $requestedReports = implode(', ', $requestedReports); + } + return "[idSite = {$this->getSite()->getId()}, period = {$this->getPeriod()->getLabel()} {$this->getPeriod()->getRangeString()}, segment = {$this->getSegment()->getString()}, plugin = {$this->getRequestedPlugin()}, report = {$requestedReports}]"; } /** @@ -295,4 +304,23 @@ public function setIsPartialArchive($isArchiveOnlyReportHandled) { $this->isArchiveOnlyReportHandled = $isArchiveOnlyReportHandled; } + + public function getArchiveOnlyReportAsArray() + { + $requestedReport = $this->getArchiveOnlyReport(); + if (empty($requestedReport)) { + return []; + } + return is_array($requestedReport) ? $requestedReport : [$requestedReport]; + } + + public function setFoundRequestedReports(array $foundRecords) + { + $this->foundRequestedReports = $foundRecords; + } + + public function getFoundRequestedReports() + { + return $this->foundRequestedReports ?: []; + } } diff --git a/core/ArchiveProcessor/PluginsArchiver.php b/core/ArchiveProcessor/PluginsArchiver.php index b62c91eec2c..ed7e24c4514 100644 --- a/core/ArchiveProcessor/PluginsArchiver.php +++ b/core/ArchiveProcessor/PluginsArchiver.php @@ -137,7 +137,19 @@ public function callAggregateAllPlugins($visits, $visitsConverted, $forceArchivi $archivers = static::getPluginArchivers(); + $archiveOnlyPlugin = $this->params->getRequestedPlugin(); + $archiveOnlyReports = $this->params->getArchiveOnlyReport(); + foreach ($archivers as $pluginName => $archiverClass) { + // if we are archiving specific reports for a single plugin then we don't need or want to create + // Archiver instances, since they will set the archive to partial even if the requested reports aren't + // handled by the Archiver + if (!empty($archiveOnlyReports) + && $archiveOnlyPlugin != $pluginName + ) { + continue; + } + // We clean up below all tables created during this function call (and recursive calls) $latestUsedTableId = Manager::getInstance()->getMostRecentTableId(); diff --git a/core/ArchiveProcessor/Record.php b/core/ArchiveProcessor/Record.php new file mode 100644 index 00000000000..d66ab379f6d --- /dev/null +++ b/core/ArchiveProcessor/Record.php @@ -0,0 +1,164 @@ +setType($type); + $record->setName($name); + return $record; + } + + /** + * @param string|null $plugin + * @return Record + */ + public function setPlugin(?string $plugin): Record + { + $this->plugin = $plugin; + return $this; + } + + /** + * @param string $name + * @return Record + */ + public function setName(string $name): Record + { + if (!preg_match('/^[a-zA-Z0-9_-]+$/', $name)) { + throw new \Exception('Invalid record name: ' . $name . '. Only alphanumeric characters and underscores are allowed.'); + } + + $this->name = $name; + return $this; + } + + /** + * @param int|string $columnToSortByBeforeTruncation + * @return Record + */ + public function setColumnToSortByBeforeTruncation($columnToSortByBeforeTruncation) + { + $this->columnToSortByBeforeTruncation = $columnToSortByBeforeTruncation; + return $this; + } + + /** + * @param int|null $maxRowsInTable + * @return Record + */ + public function setMaxRowsInTable(?int $maxRowsInTable): Record + { + $this->maxRowsInTable = $maxRowsInTable; + return $this; + } + + /** + * @param int|null $maxRowsInSubtable + * @return Record + */ + public function setMaxRowsInSubtable(?int $maxRowsInSubtable): Record + { + $this->maxRowsInSubtable = $maxRowsInSubtable; + return $this; + } + + /** + * @return string|null + */ + public function getPlugin(): ?string + { + return $this->plugin; + } + + /** + * @return string + */ + public function getName(): string + { + return $this->name; + } + + /** + * @return int|string + */ + public function getColumnToSortByBeforeTruncation() + { + return $this->columnToSortByBeforeTruncation; + } + + /** + * @return int|null + */ + public function getMaxRowsInTable(): ?int + { + return $this->maxRowsInTable; + } + + /** + * @return int|null + */ + public function getMaxRowsInSubtable(): ?int + { + return $this->maxRowsInSubtable; + } + + /** + * @param string $type + * @return Record + */ + public function setType(string $type): Record + { + $this->type = $type; + return $this; + } + + /** + * @return string + */ + public function getType(): string + { + return $this->type; + } +} diff --git a/core/ArchiveProcessor/RecordBuilder.php b/core/ArchiveProcessor/RecordBuilder.php new file mode 100644 index 00000000000..87ade05c713 --- /dev/null +++ b/core/ArchiveProcessor/RecordBuilder.php @@ -0,0 +1,252 @@ +maxRowsInTable = $maxRowsInTable; + $this->maxRowsInSubtable = $maxRowsInSubtable; + $this->columnToSortByBeforeTruncation = $columnToSortByBeforeTruncation; + $this->columnAggregationOps = $columnAggregationOps; + } + + public function isEnabled(ArchiveProcessor $archiveProcessor): bool + { + return true; + } + + /** + * Uses the protected `aggregate()` function to build records by aggregating log table data directly, then + * inserts them as archive data. + * + * @param ArchiveProcessor $archiveProcessor + * @return void + */ + public function buildFromLogs(ArchiveProcessor $archiveProcessor): void + { + if (!$this->isEnabled($archiveProcessor)) { + return; + } + + $recordsBuilt = $this->getRecordMetadata($archiveProcessor); + + $recordMetadataByName = []; + foreach ($recordsBuilt as $recordMetadata) { + $recordMetadataByName[$recordMetadata->getName()] = $recordMetadata; + } + + $numericRecords = []; + + $records = $this->aggregate($archiveProcessor); + foreach ($records as $recordName => $recordValue) { + if (empty($recordMetadataByName[$recordName])) { + if ($recordValue instanceof DataTable) { + Common::destroy($recordValue); + } + continue; + } + + if ($recordValue instanceof DataTable) { + $record = $recordMetadataByName[$recordName]; + + $maxRowsInTable = $record->getMaxRowsInTable() ?? $this->maxRowsInTable; + $maxRowsInSubtable = $record->getMaxRowsInSubtable() ?? $this->maxRowsInSubtable; + $columnToSortByBeforeTruncation = $record->getColumnToSortByBeforeTruncation() ?? $this->columnToSortByBeforeTruncation; + + $this->insertBlobRecord($archiveProcessor, $recordName, $recordValue, $maxRowsInTable, $maxRowsInSubtable, $columnToSortByBeforeTruncation); + + Common::destroy($recordValue); + } else { + // collect numeric records so we can insert them all at once + $numericRecords[$recordName] = $recordValue; + } + } + unset($records); + + if (!empty($numericRecords)) { + $archiveProcessor->insertNumericRecords($numericRecords); + } + } + + /** + * Builds records for non-day periods by aggregating day records together, then inserts + * them as archive data. + * + * @param ArchiveProcessor $archiveProcessor + * @return void + */ + public function buildForNonDayPeriod(ArchiveProcessor $archiveProcessor): void + { + if (!$this->isEnabled($archiveProcessor)) { + return; + } + + $requestedReports = $archiveProcessor->getParams()->getArchiveOnlyReportAsArray(); + $foundRequestedReports = $archiveProcessor->getParams()->getFoundRequestedReports(); + + $recordsBuilt = $this->getRecordMetadata($archiveProcessor); + + $numericRecords = array_filter($recordsBuilt, function (Record $r) { return $r->getType() == Record::TYPE_NUMERIC; }); + $blobRecords = array_filter($recordsBuilt, function (Record $r) { return $r->getType() == Record::TYPE_BLOB; }); + + foreach ($blobRecords as $record) { + if (!empty($requestedReports) + && !in_array($record->getName(), $requestedReports) + && !in_array($record->getName(), $foundRequestedReports) + ) { + continue; + } + + $maxRowsInTable = $record->getMaxRowsInTable() ?? $this->maxRowsInTable; + $maxRowsInSubtable = $record->getMaxRowsInSubtable() ?? $this->maxRowsInSubtable; + $columnToSortByBeforeTruncation = $record->getColumnToSortByBeforeTruncation() ?? $this->columnToSortByBeforeTruncation; + + $archiveProcessor->aggregateDataTableRecords( + $record->getName(), + $maxRowsInTable, + $maxRowsInSubtable, + $columnToSortByBeforeTruncation, + $this->columnAggregationOps + ); + } + + if (!empty($numericRecords)) { + $numericMetrics = array_map(function (Record $r) { return $r->getName(); }, $numericRecords); + if (!empty($requestedReports)) { + $numericMetrics = array_filter($numericMetrics, function ($name) use ($requestedReports, $foundRequestedReports) { + return in_array($name, $requestedReports) && !in_array($name, $foundRequestedReports); + }); + } + $archiveProcessor->aggregateNumericMetrics($numericMetrics, $this->columnAggregationOps); + } + } + + /** + * Returns metadata for records primarily used when aggregating over non-day periods. Every numeric/blob + * record your RecordBuilder creates should have an associated piece of record metadata. + * + * @return Record[] + */ + public abstract function getRecordMetadata(ArchiveProcessor $archiveProcessor): array; + + /** + * Derived classes should define this method to aggregate log data for a single day and return the records + * to store indexed by record names. + * + * @return (DataTable|int|float|string)[] Record values indexed by their record name, eg, `['MyPlugin_MyRecord' => new DataTable()]` + */ + protected abstract function aggregate(ArchiveProcessor $archiveProcessor): array; + + protected function insertBlobRecord(ArchiveProcessor $archiveProcessor, string $recordName, DataTable $record, + ?int $maxRowsInTable, ?int $maxRowsInSubtable, ?string $columnToSortByBeforeTruncation): void + { + $serialized = $record->getSerialized( + $maxRowsInTable ?: $this->maxRowsInTable, + $maxRowsInSubtable ?: $this->maxRowsInSubtable, + $columnToSortByBeforeTruncation ?: $this->columnToSortByBeforeTruncation + ); + $archiveProcessor->insertBlobRecord($recordName, $serialized); + unset($serialized); + } + + public function getMaxRowsInTable(): ?int + { + return $this->maxRowsInTable; + } + + public function getMaxRowsInSubtable(): ?int + { + return $this->maxRowsInSubtable; + } + + public function getColumnToSortByBeforeTruncation(): ?string + { + return $this->columnToSortByBeforeTruncation; + } + + public function getPluginName(): string + { + return Piwik::getPluginNameOfMatomoClass(get_class($this)); + } + + /** + * Returns an extra hint for LogAggregator to add to log aggregation SQL. Can be overridden if you'd + * like the origin hint to have more information. + * + * @return string + */ + public function getQueryOriginHint(): string + { + $recordBuilderName = get_class($this); + $recordBuilderName = explode('\\', $recordBuilderName); + return end($recordBuilderName); + } + + /** + * Returns true if at least one of the given reports is handled by this RecordBuilder instance + * when invoked with the given ArchiveProcessor. + * + * @param ArchiveProcessor $archiveProcessor Archiving parameters, like idSite, can influence the list of + * all records a RecordBuilder produces, so it is required here. + * @param string[] $requestedReports The list of requested reports to check for. + * @return bool + */ + public function isBuilderForAtLeastOneOf(ArchiveProcessor $archiveProcessor, array $requestedReports): bool + { + $recordMetadata = $this->getRecordMetadata($archiveProcessor); + foreach ($recordMetadata as $record) { + if (in_array($record->getName(), $requestedReports)) { + return true; + } + } + return false; + } +} diff --git a/core/ArchiveProcessor/Rules.php b/core/ArchiveProcessor/Rules.php index d1ff2e0880e..82f55b5331d 100644 --- a/core/ArchiveProcessor/Rules.php +++ b/core/ArchiveProcessor/Rules.php @@ -381,4 +381,13 @@ public static function isSegmentPluginArchivingDisabled($pluginName, $siteId = n return in_array(strtolower($pluginName), $pluginArchivingSetting); } + + public static function shouldProcessOnlyReportsRequestedInArchiveQuery(string $periodLabel): bool + { + if (SettingsServer::isArchivePhpTriggered()) { + return false; + } + + return $periodLabel === 'range'; + } } diff --git a/core/CronArchive.php b/core/CronArchive.php index e815dc679b2..78068de9d9f 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -1023,7 +1023,9 @@ public function canWeSkipInvalidatingBecauseThereIsAUsablePeriod(Parameters $par Date::now()->subSeconds(Rules::getPeriodArchiveTimeToLiveDefault($params->getPeriod()->getLabel())); // empty plugins param since we only check for an 'all' archive - list($idArchive, $visits, $visitsConverted, $ignore, $tsArchived) = ArchiveSelector::getArchiveIdAndVisits($params, $minArchiveProcessedTime, $includeInvalidated = $isPeriodIncludesToday); + $archiveInfo = ArchiveSelector::getArchiveIdAndVisits($params, $minArchiveProcessedTime, $includeInvalidated = $isPeriodIncludesToday); + $idArchive = $archiveInfo['idArchives']; + $tsArchived = $archiveInfo['tsArchived']; // day has changed since the archive was created, we need to reprocess it if ($isYesterday diff --git a/core/CronArchive/QueueConsumer.php b/core/CronArchive/QueueConsumer.php index 21df1211820..00826087c75 100644 --- a/core/CronArchive/QueueConsumer.php +++ b/core/CronArchive/QueueConsumer.php @@ -616,11 +616,12 @@ public function usableArchiveExists(array $invalidatedArchive) // if valid archive already exists, do not re-archive $minDateTimeProcessedUTC = Date::now()->subSeconds(Rules::getPeriodArchiveTimeToLiveDefault($periodLabel)); $archiveIdAndVisits = ArchiveSelector::getArchiveIdAndVisits($params, $minDateTimeProcessedUTC, $includeInvalidated = false); + $idArchives = $archiveIdAndVisits['idArchives']; + $tsArchived = $archiveIdAndVisits['tsArchived']; - $tsArchived = !empty($archiveIdAndVisits[4]) ? Date::factory($archiveIdAndVisits[4])->getDatetime() : null; + $tsArchived = !empty($tsArchived) ? Date::factory($tsArchived)->getDatetime() : null; - $idArchive = $archiveIdAndVisits[0]; - if (empty($idArchive)) { + if (empty($idArchives)) { return [false, $tsArchived]; } diff --git a/core/DataAccess/ArchiveSelector.php b/core/DataAccess/ArchiveSelector.php index 28eb9f4095d..983227ed097 100644 --- a/core/DataAccess/ArchiveSelector.php +++ b/core/DataAccess/ArchiveSelector.php @@ -70,6 +70,8 @@ public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params $numericTable = ArchiveTableCreator::getNumericTable($dateStart); $requestedPlugin = $params->getRequestedPlugin(); + $requestedReport = $params->getArchiveOnlyReport(); + $segment = $params->getSegment(); $plugins = array("VisitsSummary", $requestedPlugin); $plugins = array_filter($plugins); @@ -83,7 +85,15 @@ public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, null, $doneFlags); if (empty($results)) { // no archive found - return [false, false, false, false, false, false]; + return self::archiveInfoBcResult([ + 'idArchives' => false, + 'visits' => false, + 'visitsConverted' => false, + 'archiveExists' => false, + 'tsArchived' => false, + 'doneFlagValue' => false, + 'existingRecords' => null, + ]); } $result = self::findArchiveDataWithLatestTsArchived($results, $requestedPluginDoneFlags, $allPluginsDoneFlag); @@ -93,16 +103,43 @@ public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params $visitsConverted = isset($result['nb_visits_converted']) ? $result['nb_visits_converted'] : false; $value = isset($result['value']) ? $result['value'] : false; + $existingRecords = null; + $result['idarchive'] = empty($result['idarchive']) ? [] : [$result['idarchive']]; - if (isset($result['partial'])) { - $result['idarchive'] = array_merge($result['idarchive'], $result['partial']); + if (!empty($result['partial'])) { + // when we are not looking for a specific report, or if we have found a non-partial archive + // that we expect to have the full set of reports for the requested plugin, then we can just + // return it with the additionally found partial archives. + // + // if, however, there is no full archive, and only a set of partial archives, then + // we have to check whether the requested data is actually within them. if we just report the + // partial archives, Archive.php will find no archive data and simply report this. returning no + // idarchive here, however, will initiate archiving, causing the missing data to populate. + if (empty($requestedReport) + || !empty($result['idarchive']) + ) { + $result['idarchive'] = array_merge($result['idarchive'], $result['partial']); + } else { + $existingRecords = self::getModel()->getRecordsContainedInArchives($dateStart, $result['partial'], $requestedReport); + if (!empty($existingRecords)) { + $result['idarchive'] = array_merge($result['idarchive'], $result['partial']); + } + } } if (empty($result['idarchive']) || (isset($result['value']) && !in_array($result['value'], $doneFlagValues)) ) { // the archive cannot be considered valid for this request (has wrong done flag value) - return [false, $visits, $visitsConverted, true, $tsArchived, $value]; + return self::archiveInfoBcResult([ + 'idArchives' => false, + 'visits' => $visits, + 'visitsConverted' => $visitsConverted, + 'archiveExists' => true, + 'tsArchived' => $tsArchived, + 'doneFlagValue' => $value, + 'existingRecords' => null, + ]); } if (!empty($minDatetimeArchiveProcessedUTC) && !is_object($minDatetimeArchiveProcessedUTC)) { @@ -114,12 +151,28 @@ public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params && !empty($result['idarchive']) && Date::factory($tsArchived)->isEarlier($minDatetimeArchiveProcessedUTC) ) { - return [false, $visits, $visitsConverted, true, $tsArchived, $value]; + return self::archiveInfoBcResult([ + 'idArchives' => false, + 'visits' => $visits, + 'visitsConverted' => $visitsConverted, + 'archiveExists' => true, + 'tsArchived' => $tsArchived, + 'doneFlagValue' => $value, + 'existingRecords' => null, + ]); } $idArchives = !empty($result['idarchive']) ? $result['idarchive'] : false; - return [$idArchives, $visits, $visitsConverted, true, $tsArchived, $value]; + return self::archiveInfoBcResult([ + 'idArchives' => $idArchives, + 'visits' => $visits, + 'visitsConverted' => $visitsConverted, + 'archiveExists' => true, + 'tsArchived' => $tsArchived, + 'doneFlagValue' => $value, + 'existingRecords' => $existingRecords, + ]); } /** @@ -449,6 +502,22 @@ private static function findArchiveDataWithLatestTsArchived($results, $requested return $archiveData; } + /** + * provides BC result for getArchiveIdAndVisits + * @param array $archiveInfo + * @return array + */ + private static function archiveInfoBcResult(array $archiveInfo) + { + $archiveInfo[0] = $archiveInfo['idArchives']; + $archiveInfo[1] = $archiveInfo['visits']; + $archiveInfo[2] = $archiveInfo['visitsConverted']; + $archiveInfo[3] = $archiveInfo['archiveExists']; + $archiveInfo[4] = $archiveInfo['tsArchived']; + $archiveInfo[5] = $archiveInfo['doneFlagValue']; + return $archiveInfo; + } + public static function querySingleBlob(array $archiveIds, string $recordName) { $chunk = new Chunk(); diff --git a/core/DataAccess/LogAggregator.php b/core/DataAccess/LogAggregator.php index fdf2a2d371f..6bfead967c9 100644 --- a/core/DataAccess/LogAggregator.php +++ b/core/DataAccess/LogAggregator.php @@ -201,6 +201,11 @@ public function setQueryOriginHint($nameOfOrigin) $this->queryOriginHint = $nameOfOrigin; } + public function getQueryOriginHint() + { + return $this->queryOriginHint; + } + public function getSegmentTmpTableName() { $bind = $this->getGeneralQueryBindParams(); diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index 4c42b291600..8aa952a6dd2 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -973,6 +973,40 @@ public function resetFailedArchivingJobs() return $query->rowCount(); } + public function getRecordsContainedInArchives(Date $archiveStartDate, array $idArchives, $requestedRecords): array + { + $idArchives = array_map('intval', $idArchives); + $idArchives = implode(',', $idArchives); + + $requestedRecords = is_string($requestedRecords) ? [$requestedRecords] : $requestedRecords; + $placeholders = Common::getSqlStringFieldsArray($requestedRecords); + + $countSql = "SELECT DISTINCT name FROM %s WHERE idarchive IN ($idArchives) AND name IN ($placeholders) LIMIT " . count($requestedRecords); + + $numericTable = ArchiveTableCreator::getNumericTable($archiveStartDate); + $blobTable = ArchiveTableCreator::getBlobTable($archiveStartDate); + + // if the requested metrics look numeric, prioritize the numeric table, otherwise the blob table. this way, if all the metrics are + // found in this table (which will be most of the time), we don't have to query the other table + if ($this->doRequestedRecordsLookNumeric($requestedRecords)) { + $tablesToSearch = [$numericTable, $blobTable]; + } else { + $tablesToSearch = [$blobTable, $numericTable]; + } + + $existingRecords = []; + foreach ($tablesToSearch as $tableName) { + $sql = sprintf($countSql, $tableName); + $rows = Db::fetchAll($sql, $requestedRecords); + $existingRecords = array_merge($existingRecords, array_column($rows, 'name')); + + if (count($existingRecords) == count($requestedRecords)) { + break; + } + } + return $existingRecords; + } + private function isCutOffGroupConcatResult($pair) { $position = strpos($pair, '.'); @@ -984,4 +1018,14 @@ private function getHashFromDoneFlag($doneFlag) preg_match('/^done([a-zA-Z0-9]+)/', $doneFlag, $matches); return $matches[1] ?? ''; } + + private function doRequestedRecordsLookNumeric(array $requestedRecords): bool + { + foreach ($requestedRecords as $record) { + if (preg_match('/^nb_/', $record)) { + return true; + } + } + return false; + } } diff --git a/core/Piwik.php b/core/Piwik.php index e42a00adbec..ee3749a5b65 100644 --- a/core/Piwik.php +++ b/core/Piwik.php @@ -968,4 +968,21 @@ public static function getEarliestDateToRearchive() return Date::yesterday()->subMonth($lastNMonthsToInvalidate)->setDay(1); } + + /** + * Given the fully qualified name of a class located within a Matomo plugin, + * returns the name of the plugin. + * + * Uses the fact that Matomo plugins have namespaces like Piwik\Plugins\MyPlugin. + * + * @param string $className the name of a class located within a Matomo plugin + * @return string the plugin name + */ + public static function getPluginNameOfMatomoClass(string $className): string + { + $parts = explode('\\', $className); + $parts = array_filter($parts); + $plugin = $parts[2] ?? ''; + return $plugin; + } } diff --git a/core/Plugin/Archiver.php b/core/Plugin/Archiver.php index 0566b2b1c6a..ffa6e68407e 100644 --- a/core/Plugin/Archiver.php +++ b/core/Plugin/Archiver.php @@ -10,8 +10,12 @@ namespace Piwik\Plugin; use Piwik\ArchiveProcessor; +use Piwik\Cache; +use Piwik\CacheId; use Piwik\Config as PiwikConfig; +use Piwik\Container\StaticContainer; use Piwik\ErrorHandler; +use Piwik\Piwik; /** * The base class that should be extended by plugins that compute their own @@ -83,6 +87,83 @@ public function __construct(ArchiveProcessor $processor) $this->enabled = true; } + private function getPluginName(): string + { + return Piwik::getPluginNameOfMatomoClass(get_class($this)); + } + + /** + * @return ArchiveProcessor\RecordBuilder[] + * @throws \DI\DependencyException + * @throws \DI\NotFoundException + */ + private function getRecordBuilders(string $pluginName): array + { + $transientCache = Cache::getTransientCache(); + $cacheKey = CacheId::siteAware('Archiver.RecordBuilders') . '.' . $pluginName; + + $recordBuilders = $transientCache->fetch($cacheKey); + if ($recordBuilders === false) { + $recordBuilderClasses = $this->getAllRecordBuilderClasses(); + + // only select RecordBuilders for the selected plugin + $recordBuilderClasses = array_filter($recordBuilderClasses, function ($className) use ($pluginName) { + return Piwik::getPluginNameOfMatomoClass($className) == $pluginName; + }); + + $recordBuilders = array_map(function ($className) { + return StaticContainer::getContainer()->make($className); + }, $recordBuilderClasses); + + /** + * Triggered to add new RecordBuilders that cannot be picked up automatically by the platform. + * If you define RecordBuilders that take a parameter, for example, an ID to an entity your plugin + * manages, use this event to add instances of that RecordBuilder to the global list. + * + * **Example** + * + * public function addRecordBuilders(&$recordBuilders) + * { + * $recordBuilders[] = new MyParameterizedRecordBuilder($idOfThingToArchiveFor); + * } + * + * @param ArchiveProcessor\RecordBuilder[] $recordBuilders An array of RecordBuilder instances + * @api + */ + Piwik::postEvent('Archiver.addRecordBuilders', [&$recordBuilders], false, [$pluginName]); + + $transientCache->save($cacheKey, $recordBuilders); + } + + /** + * Triggered to filter / restrict reports. + * + * **Example** + * + * public function filterRecordBuilders(&$recordBuilders) + * { + * foreach ($reports as $index => $recordBuilder) { + * if ($recordBuilders instanceof AnotherPluginRecordBuilder) { + * unset($reports[$index]); + * } + * } + * } + * + * @param ArchiveProcessor\RecordBuilder[] $recordBuilders An array of RecordBuilder instances + * @api + */ + Piwik::postEvent('Archiver.filterRecordBuilders', [&$recordBuilders]); + + $requestedReports = $this->processor->getParams()->getArchiveOnlyReportAsArray(); + if (!empty($requestedReports)) { + $recordBuilders = array_filter($recordBuilders, function (ArchiveProcessor\RecordBuilder $builder) use ($requestedReports) { + return $builder->isBuilderForAtLeastOneOf($this->processor, $requestedReports); + }); + } + + return $recordBuilders; + } + /** * @ignore */ @@ -91,6 +172,33 @@ final public function callAggregateDayReport() try { ErrorHandler::pushFatalErrorBreadcrumb(static::class); + $pluginName = $this->getPluginName(); + + if (Manager::getInstance()->isPluginLoaded($pluginName)) { + $recordBuilders = $this->getRecordBuilders($pluginName); + + foreach ($recordBuilders as $recordBuilder) { + if (!$recordBuilder->isEnabled($this->getProcessor())) { + continue; + } + + // if automatically handling "archive only report" in RecordBuilders, make sure the archive + // will be marked as partial + if ($this->processor->getParams()->getArchiveOnlyReport()) { + $this->processor->getParams()->setIsPartialArchive(true); // make sure archive will be marked as partial + } + + $originalQueryHint = $this->getProcessor()->getLogAggregator()->getQueryOriginHint(); + $newQueryHint = $originalQueryHint . ' ' . $recordBuilder->getQueryOriginHint(); + try { + $this->getProcessor()->getLogAggregator()->setQueryOriginHint($newQueryHint); + $recordBuilder->buildFromLogs($this->getProcessor()); + } finally { + $this->getProcessor()->getLogAggregator()->setQueryOriginHint($originalQueryHint); + } + } + } + $this->aggregateDayReport(); } finally { ErrorHandler::popFatalErrorBreadcrumb(); @@ -105,6 +213,32 @@ final public function callAggregateMultipleReports() try { ErrorHandler::pushFatalErrorBreadcrumb(static::class); + $pluginName = $this->getPluginName(); + + if (Manager::getInstance()->isPluginLoaded($pluginName)) { + $recordBuilders = $this->getRecordBuilders($pluginName); + foreach ($recordBuilders as $recordBuilder) { + if (!$recordBuilder->isEnabled($this->getProcessor())) { + continue; + } + + // if automatically handling "archive only report" in RecordBuilders, make sure the archive + // will be marked as partial + if ($this->processor->getParams()->getArchiveOnlyReport()) { + $this->processor->getParams()->setIsPartialArchive(true); // make sure archive will be marked as partial + } + + $originalQueryHint = $this->getProcessor()->getLogAggregator()->getQueryOriginHint(); + $newQueryHint = $originalQueryHint . ' ' . $recordBuilder->getQueryOriginHint(); + try { + $this->getProcessor()->getLogAggregator()->setQueryOriginHint($newQueryHint); + $recordBuilder->buildForNonDayPeriod($this->getProcessor()); + } finally { + $this->getProcessor()->getLogAggregator()->setQueryOriginHint($originalQueryHint); + } + } + } + $this->aggregateMultipleReports(); } finally { ErrorHandler::popFatalErrorBreadcrumb(); @@ -194,4 +328,26 @@ protected function isRequestedReport(string $reportName) return empty($requestedReport) || $requestedReport == $reportName; } + + private function getDefaultConstructibleClasses(array $classes): array + { + return array_filter($classes, function ($className) { + return (new \ReflectionClass($className))->getConstructor()->getNumberOfRequiredParameters() == 0; + }); + } + + private function getAllRecordBuilderClasses(): array + { + $transientCache = Cache::getTransientCache(); + $cacheKey = CacheId::siteAware('RecordBuilders.allRecordBuilders'); + + $recordBuilderClasses = $transientCache->fetch($cacheKey); + if ($recordBuilderClasses === false) { + $recordBuilderClasses = Manager::getInstance()->findMultipleComponents('RecordBuilders', ArchiveProcessor\RecordBuilder::class); + $recordBuilderClasses = $this->getDefaultConstructibleClasses($recordBuilderClasses); + + $transientCache->save($cacheKey, $recordBuilderClasses); + } + return $recordBuilderClasses; + } } diff --git a/plugins/Goals/Archiver.php b/plugins/Goals/Archiver.php index 2373f68d037..2a43100c3ef 100644 --- a/plugins/Goals/Archiver.php +++ b/plugins/Goals/Archiver.php @@ -9,16 +9,6 @@ namespace Piwik\Plugins\Goals; -use Piwik\ArchiveProcessor; -use Piwik\Common; -use Piwik\Config; -use Piwik\DataAccess\LogAggregator; -use Piwik\DataArray; -use Piwik\DataTable; -use Piwik\Metrics; -use Piwik\Plugin\Manager; -use Piwik\Site; -use Piwik\Tracker\GoalManager; use Piwik\Plugins\VisitFrequency\API as VisitFrequencyAPI; class Archiver extends \Piwik\Plugin\Archiver @@ -42,43 +32,6 @@ class Archiver extends \Piwik\Plugin\Archiver public static $ARCHIVE_DEPENDENT = true; - /** - * This array stores the ranges to use when displaying the 'visits to conversion' report - */ - public static $visitCountRanges = array( - array(1, 1), - array(2, 2), - array(3, 3), - array(4, 4), - array(5, 5), - array(6, 6), - array(7, 7), - array(8, 8), - array(9, 14), - array(15, 25), - array(26, 50), - array(51, 100), - array(100) - ); - /** - * This array stores the ranges to use when displaying the 'days to conversion' report - */ - public static $daysToConvRanges = array( - array(0, 0), - array(1, 1), - array(2, 2), - array(3, 3), - array(4, 4), - array(5, 5), - array(6, 6), - array(7, 7), - array(8, 14), - array(15, 30), - array(31, 60), - array(61, 120), - array(121, 364), - array(364) - ); protected $dimensionRecord = [ self::SKU_FIELD => self::ITEMS_SKU_RECORD_NAME, self::NAME_FIELD => self::ITEMS_NAME_RECORD_NAME, @@ -94,165 +47,15 @@ class Archiver extends \Piwik\Plugin\Archiver self::CATEGORY5_FIELD => 'idaction_product_cat5', ]; - /** - * Array containing one DataArray for each Ecommerce items dimension (name/sku/category abandoned carts and orders) - * @var DataArray[][] - */ - protected $itemReports = []; - - /** - * @var int - */ - private $productReportsMaximumRows; - - public function __construct(ArchiveProcessor $processor) - { - parent::__construct($processor); - - $general = Config::getInstance()->General; - $this->productReportsMaximumRows = $general['datatable_archiving_maximum_rows_products']; - } - public function aggregateDayReport() { $hasConversions = $this->getProcessor()->getNumberOfVisitsConverted() > 0; - if ($hasConversions) { - $this->aggregateGeneralGoalMetrics(); - } - - if (Manager::getInstance()->isPluginActivated('Ecommerce')) { - $this->aggregateEcommerceItems(); - } - if (self::$ARCHIVE_DEPENDENT && $hasConversions) { $this->getProcessor()->processDependentArchive('Goals', VisitFrequencyAPI::NEW_VISITOR_SEGMENT); $this->getProcessor()->processDependentArchive('Goals', VisitFrequencyAPI::RETURNING_VISITOR_SEGMENT); } } - private function hasAnyGoalOrEcommerce($idSite) - { - return $this->usesEcommerce($idSite) || !empty(GoalManager::getGoalIds($idSite)); - } - - private function usesEcommerce($idSite) - { - return Manager::getInstance()->isPluginActivated('Ecommerce') - && Site::isEcommerceEnabledFor($idSite); - } - - private function getSiteId() - { - return $this->getProcessor()->getParams()->getSite()->getId(); - } - - protected function aggregateGeneralGoalMetrics() - { - $prefixes = array( - self::VISITS_UNTIL_RECORD_NAME => 'vcv', - self::DAYS_UNTIL_CONV_RECORD_NAME => 'vdsf', - ); - - $totalConversions = $totalRevenue = 0; - $goals = new DataArray(); - $visitsToConversions = $daysToConversions = []; - - $siteHasEcommerceOrGoals = $this->hasAnyGoalOrEcommerce($this->getSiteId()); - - // Special handling for sites that contain subordinated sites, like in roll up reporting. - // A roll up site, might not have ecommerce enabled or any configured goals, - // but if a subordinated site has, we calculate the overview conversion metrics nevertheless - if ($siteHasEcommerceOrGoals === false) { - $idSitesToArchive = $this->getProcessor()->getParams()->getIdSites(); - - foreach ($idSitesToArchive as $idSite) { - if ($this->hasAnyGoalOrEcommerce($idSite)) { - $siteHasEcommerceOrGoals = true; - break; - } - } - } - - // try to query goal data only, if goals or ecommerce is actually used - // otherwise we simply insert empty records - if ($siteHasEcommerceOrGoals) { - $selects = []; - $selects = array_merge($selects, LogAggregator::getSelectsFromRangedColumn( - self::VISITS_COUNT_FIELD, self::$visitCountRanges, self::LOG_CONVERSION_TABLE, $prefixes[self::VISITS_UNTIL_RECORD_NAME] - )); - $selects = array_merge($selects, LogAggregator::getSelectsFromRangedColumn( - 'FLOOR(log_conversion.' . self::SECONDS_SINCE_FIRST_VISIT_FIELD . ' / 86400)', self::$daysToConvRanges, self::LOG_CONVERSION_TABLE, $prefixes[self::DAYS_UNTIL_CONV_RECORD_NAME] - )); - - $query = $this->getLogAggregator()->queryConversionsByDimension([], false, $selects); - if ($query === false) { - return; - } - - $conversionMetrics = $this->getLogAggregator()->getConversionsMetricFields(); - while ($row = $query->fetch()) { - $idGoal = $row['idgoal']; - unset($row['idgoal']); - unset($row['label']); - - $values = []; - foreach ($conversionMetrics as $field => $statement) { - $values[$field] = $row[$field]; - } - $goals->sumMetrics($idGoal, $values); - - if (empty($visitsToConversions[$idGoal])) { - $visitsToConversions[$idGoal] = new DataTable(); - } - $array = LogAggregator::makeArrayOneColumn($row, Metrics::INDEX_NB_CONVERSIONS, $prefixes[self::VISITS_UNTIL_RECORD_NAME]); - $visitsToConversions[$idGoal]->addDataTable(DataTable::makeFromIndexedArray($array)); - - if (empty($daysToConversions[$idGoal])) { - $daysToConversions[$idGoal] = new DataTable(); - } - $array = LogAggregator::makeArrayOneColumn($row, Metrics::INDEX_NB_CONVERSIONS, $prefixes[self::DAYS_UNTIL_CONV_RECORD_NAME]); - $daysToConversions[$idGoal]->addDataTable(DataTable::makeFromIndexedArray($array)); - - // We don't want to sum Abandoned cart metrics in the overall revenue/conversions/converted visits - // since it is a "negative conversion" - if ($idGoal != GoalManager::IDGOAL_CART) { - $totalConversions += $row[Metrics::INDEX_GOAL_NB_CONVERSIONS]; - $totalRevenue += $row[Metrics::INDEX_GOAL_REVENUE]; - } - } - } - - // Stats by goal, for all visitors - $numericRecords = $this->getConversionsNumericMetrics($goals); - $this->getProcessor()->insertNumericRecords($numericRecords); - - $this->insertReports(self::VISITS_UNTIL_RECORD_NAME, $visitsToConversions); - $this->insertReports(self::DAYS_UNTIL_CONV_RECORD_NAME, $daysToConversions); - - // Stats for all goals - $nbConvertedVisits = $this->getProcessor()->getNumberOfVisitsConverted(); - $metrics = array( - self::getRecordName('nb_conversions') => $totalConversions, - self::getRecordName('nb_visits_converted') => $nbConvertedVisits, - self::getRecordName('revenue') => $totalRevenue, - ); - $this->getProcessor()->insertNumericRecords($metrics); - } - - protected function getConversionsNumericMetrics(DataArray $goals) - { - $numericRecords = array(); - $goals = $goals->getDataArray(); - foreach ($goals as $idGoal => $array) { - foreach ($array as $metricId => $value) { - $metricName = Metrics::$mappingFromIdToNameGoal[$metricId]; - $recordName = self::getRecordName($metricName, $idGoal); - $numericRecords[$recordName] = $value; - } - } - return $numericRecords; - } - /** * @param string $recordName 'nb_conversions' * @param int|bool $idGoal idGoal to return the metrics for, or false to return overall @@ -267,219 +70,6 @@ public static function getRecordName($recordName, $idGoal = false) return 'Goal_' . $idGoalStr . $recordName; } - protected function insertReports($recordName, $visitsToConversions) - { - foreach ($visitsToConversions as $idGoal => $table) { - $record = self::getRecordName($recordName, $idGoal); - $this->getProcessor()->insertBlobRecord($record, $table->getSerialized()); - } - $overviewTable = $this->getOverviewFromGoalTables($visitsToConversions); - $this->getProcessor()->insertBlobRecord(self::getRecordName($recordName), $overviewTable->getSerialized()); - } - - protected function getOverviewFromGoalTables($tableByGoal) - { - $overview = new DataTable(); - foreach ($tableByGoal as $idGoal => $table) { - if ($this->isStandardGoal($idGoal)) { - $overview->addDataTable($table); - } - } - return $overview; - } - - protected function isStandardGoal($idGoal) - { - return !in_array($idGoal, $this->getEcommerceIdGoals()); - } - - protected function aggregateEcommerceItems() - { - $this->initItemReports(); - - // try to query ecommerce items only, if ecommerce is actually used - // otherwise we simply insert empty records - if ($this->usesEcommerce($this->getSiteId())) { - foreach ($this->getItemsDimensions() as $dimension) { - $query = $this->getLogAggregator()->queryEcommerceItems($dimension); - if ($query !== false) { - $this->aggregateFromEcommerceItems($query, $dimension); - } - - $query = $this->queryItemViewsForDimension($dimension); - if ($query !== false) { - $this->aggregateFromEcommerceViews($query, $dimension); - } - } - } - $this->insertItemReports(); - return true; - } - - protected function queryItemViewsForDimension($dimension) - { - $column = $this->actionMapping[$dimension]; - $where = "log_link_visit_action.$column is not null"; - - return $this->getLogAggregator()->queryActionsByDimension( - ['label' => 'log_action1.name'], - $where, - ['AVG(log_link_visit_action.product_price) AS `avg_price_viewed`'], - false, - null, - [$column] - ); - } - - protected function initItemReports() - { - foreach ($this->getEcommerceIdGoals() as $ecommerceType) { - foreach ($this->dimensionRecord as $dimension => $record) { - $this->itemReports[$dimension][$ecommerceType] = new DataArray(); - } - } - } - - protected function insertItemReports() - { - foreach ($this->itemReports as $dimension => $itemAggregatesByType) { - foreach ($itemAggregatesByType as $ecommerceType => $itemAggregate) { - $recordName = $this->dimensionRecord[$dimension]; - if ($ecommerceType == GoalManager::IDGOAL_CART) { - $recordName = self::getItemRecordNameAbandonedCart($recordName); - } - $table = $itemAggregate->asDataTable(); - $blobData = $table->getSerialized($this->productReportsMaximumRows, $this->productReportsMaximumRows, - Metrics::INDEX_ECOMMERCE_ITEM_REVENUE); - $this->getProcessor()->insertBlobRecord($recordName, $blobData); - - Common::destroy($table); - } - } - } - - protected function getItemsDimensions() - { - $dimensions = array_keys($this->dimensionRecord); - foreach ($this->getItemExtraCategories() as $category) { - $dimensions[] = $category; - } - return $dimensions; - } - - protected function getItemExtraCategories() - { - return array(self::CATEGORY2_FIELD, self::CATEGORY3_FIELD, self::CATEGORY4_FIELD, self::CATEGORY5_FIELD); - } - - protected function isItemExtraCategory($field) - { - return in_array($field, $this->getItemExtraCategories()); - } - - protected function aggregateFromEcommerceItems($query, $dimension) - { - while ($row = $query->fetch()) { - $ecommerceType = $row['ecommerceType']; - - $label = $this->cleanupRowGetLabel($row, $dimension); - if ($label === false) { - continue; - } - - // Aggregate extra categories in the Item categories array - if ($this->isItemExtraCategory($dimension)) { - $array = $this->itemReports[self::CATEGORY_FIELD][$ecommerceType]; - } else { - $array = $this->itemReports[$dimension][$ecommerceType]; - } - - $this->roundColumnValues($row); - $array->sumMetrics($label, $row); - } - } - - protected function aggregateFromEcommerceViews($query, $dimension) - { - while ($row = $query->fetch()) { - - $label = $this->getRowLabel($row, $dimension); - if ($label === false) { - continue; // ignore empty additional categories - } - - // Aggregate extra categories in the Item categories array - if ($this->isItemExtraCategory($dimension)) { - $array = $this->itemReports[self::CATEGORY_FIELD]; - } else { - $array = $this->itemReports[$dimension]; - } - - unset($row['label']); - - if (isset($row['avg_price_viewed'])) { - $row['avg_price_viewed'] = round($row['avg_price_viewed'], GoalManager::REVENUE_PRECISION); - } - - // add views to all types - foreach ($array as $ecommerceType => $dataArray) { - $dataArray->sumMetrics($label, $row); - } - } - } - - protected function cleanupRowGetLabel(&$row, $currentField) - { - $label = $this->getRowLabel($row, $currentField); - - if (isset($row['ecommerceType']) && $row['ecommerceType'] == GoalManager::IDGOAL_CART) { - // abandoned carts are the number of visits with an abandoned cart - $row[Metrics::INDEX_ECOMMERCE_ORDERS] = $row[Metrics::INDEX_NB_VISITS]; - } - - unset($row[Metrics::INDEX_NB_VISITS]); - unset($row['label']); - unset($row['labelIdAction']); - unset($row['ecommerceType']); - - return $label; - } - - protected function getRowLabel(&$row, $currentField) - { - $label = $row['label']; - if (empty($label)) { - // An empty additional category -> skip this iteration - if ($this->isItemExtraCategory($currentField)) { - return false; - } - $label = "Value not defined"; - } - return $label; - } - - protected function roundColumnValues(&$row) - { - $columnsToRound = array( - Metrics::INDEX_ECOMMERCE_ITEM_REVENUE, - Metrics::INDEX_ECOMMERCE_ITEM_QUANTITY, - Metrics::INDEX_ECOMMERCE_ITEM_PRICE, - Metrics::INDEX_ECOMMERCE_ITEM_PRICE_VIEWED, - ); - foreach ($columnsToRound as $column) { - if (isset($row[$column]) - && $row[$column] == round($row[$column]) - ) { - $row[$column] = round($row[$column]); - } - } - } - - protected function getEcommerceIdGoals() - { - return array(GoalManager::IDGOAL_CART, GoalManager::IDGOAL_ORDER); - } - public static function getItemRecordNameAbandonedCart($recordName) { return $recordName . '_Cart'; @@ -491,84 +81,6 @@ public static function getItemRecordNameAbandonedCart($recordName) public function aggregateMultipleReports() { $hasConversions = $this->getProcessor()->getNumberOfVisitsConverted() > 0; - - /* - * Archive Ecommerce Items - */ - if (Manager::getInstance()->isPluginActivated('Ecommerce')) { - $dataTableToSum = $this->dimensionRecord; - foreach ($this->dimensionRecord as $recordName) { - $dataTableToSum[] = self::getItemRecordNameAbandonedCart($recordName); - } - $columnsAggregationOperation = null; - - $this->getProcessor()->aggregateDataTableRecords($dataTableToSum, - $maximumRowsInDataTableLevelZero = $this->productReportsMaximumRows, - $maximumRowsInSubDataTable = $this->productReportsMaximumRows, - $columnToSortByBeforeTruncation = Metrics::INDEX_ECOMMERCE_ITEM_REVENUE, - $columnsAggregationOperation, - $columnsToRenameAfterAggregation = null, - $countRowsRecursive = []); - } - - $goalIdsToSum = []; - - /* - * Archive General Goal metrics - */ - if ($hasConversions) { - $goalIdsToSum = GoalManager::getGoalIds($this->getProcessor()->getParams()->getSite()->getId()); - } - - //Ecommerce - if (Manager::getInstance()->isPluginActivated('Ecommerce')) { - $goalIdsToSum = array_merge($goalIdsToSum, $this->getEcommerceIdGoals()); - } - - // Overall goal metrics - if ($hasConversions) { - $goalIdsToSum[] = false; - } - - // overall numeric metrics - if ($hasConversions) { - $fieldsToSum = array(); - foreach ($goalIdsToSum as $goalId) { - $metricsToSum = Goals::getGoalColumns($goalId); - foreach ($metricsToSum as $metricName) { - $fieldsToSum[] = self::getRecordName($metricName, $goalId); - } - } - $this->getProcessor()->aggregateNumericMetrics($fieldsToSum); - } - - $columnsAggregationOperation = null; - - foreach ($goalIdsToSum as $goalId) { - // sum up the visits to conversion data table & the days to conversion data table - $this->getProcessor()->aggregateDataTableRecords( - array(self::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $goalId), - self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $goalId)), - $maximumRowsInDataTableLevelZero = null, - $maximumRowsInSubDataTable = null, - $columnToSortByBeforeTruncation = null, - $columnsAggregationOperation, - $columnsToRenameAfterAggregation = null, - $countRowsRecursive = array()); - } - - $columnsAggregationOperation = null; - // sum up goal overview reports - $this->getProcessor()->aggregateDataTableRecords( - array(self::getRecordName(self::VISITS_UNTIL_RECORD_NAME), - self::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME)), - $maximumRowsInDataTableLevelZero = null, - $maximumRowsInSubDataTable = null, - $columnToSortByBeforeTruncation = null, - $columnsAggregationOperation, - $columnsToRenameAfterAggregation = null, - $countRowsRecursive = array()); - if (self::$ARCHIVE_DEPENDENT && $hasConversions) { $this->getProcessor()->processDependentArchive('Goals', VisitFrequencyAPI::NEW_VISITOR_SEGMENT); $this->getProcessor()->processDependentArchive('Goals', VisitFrequencyAPI::RETURNING_VISITOR_SEGMENT); diff --git a/plugins/Goals/Goals.php b/plugins/Goals/Goals.php index 8bae1d0715b..e1268a0e0bf 100644 --- a/plugins/Goals/Goals.php +++ b/plugins/Goals/Goals.php @@ -18,6 +18,7 @@ use Piwik\Plugin\ComputedMetric; use Piwik\Plugin\ReportsProvider; use Piwik\Plugins\CoreHome\SystemSummary; +use Piwik\Plugins\Goals\RecordBuilders\ProductRecord; use Piwik\Tracker\GoalManager; use Piwik\Category\Subcategory; @@ -105,10 +106,23 @@ public function registerEvents() 'Metric.addMetrics' => 'addMetrics', 'Metric.addComputedMetrics' => 'addComputedMetrics', 'System.addSystemSummaryItems' => 'addSystemSummaryItems', + 'Archiver.addRecordBuilders' => 'addRecordBuilders', ); return $hooks; } + public function addRecordBuilders(array &$recordBuilders): void + { + $recordBuilders[] = new ProductRecord(ProductRecord::SKU_FIELD, ProductRecord::ITEMS_SKU_RECORD_NAME); + $recordBuilders[] = new ProductRecord(ProductRecord::NAME_FIELD, ProductRecord::ITEMS_NAME_RECORD_NAME); + $recordBuilders[] = new ProductRecord(ProductRecord::CATEGORY_FIELD, ProductRecord::ITEMS_CATEGORY_RECORD_NAME, [ + ProductRecord::CATEGORY2_FIELD, + ProductRecord::CATEGORY3_FIELD, + ProductRecord::CATEGORY4_FIELD, + ProductRecord::CATEGORY5_FIELD, + ]); + } + public function addSystemSummaryItems(&$systemSummary) { $goalModel = new Model(); diff --git a/plugins/Goals/RecordBuilders/Base.php b/plugins/Goals/RecordBuilders/Base.php new file mode 100644 index 00000000000..738f4d75226 --- /dev/null +++ b/plugins/Goals/RecordBuilders/Base.php @@ -0,0 +1,40 @@ +getParams()->getSite()->getId(); + } + + protected function usesEcommerce(int $idSite): bool + { + return Manager::getInstance()->isPluginActivated('Ecommerce') + && Site::isEcommerceEnabledFor($idSite); + } + + protected function hasAnyGoalOrEcommerce(int $idSite): bool + { + return $this->usesEcommerce($idSite) || !empty(GoalManager::getGoalIds($idSite)); + } + + protected function getEcommerceIdGoals(): array + { + return array(GoalManager::IDGOAL_CART, GoalManager::IDGOAL_ORDER); + } +} diff --git a/plugins/Goals/RecordBuilders/GeneralGoalsRecords.php b/plugins/Goals/RecordBuilders/GeneralGoalsRecords.php new file mode 100644 index 00000000000..b4aa3cef973 --- /dev/null +++ b/plugins/Goals/RecordBuilders/GeneralGoalsRecords.php @@ -0,0 +1,244 @@ +getSiteId($archiveProcessor); + if (empty($idSite)) { + return []; + } + + $prefixes = [ + self::VISITS_UNTIL_RECORD_NAME => 'vcv', + self::DAYS_UNTIL_CONV_RECORD_NAME => 'vdsf', + ]; + + $totalConversions = 0; + $totalRevenue = 0; + + $goals = new DataArray(); + + $visitsToConversions = []; + $daysToConversions = []; + + $siteHasEcommerceOrGoals = $this->hasAnyGoalOrEcommerce($idSite); + + // Special handling for sites that contain subordinated sites, like in roll up reporting. + // A roll up site, might not have ecommerce enabled or any configured goals, + // but if a subordinated site has, we calculate the overview conversion metrics nevertheless + if ($siteHasEcommerceOrGoals === false) { + $idSitesToArchive = $archiveProcessor->getParams()->getIdSites(); + + foreach ($idSitesToArchive as $idSite) { + if ($this->hasAnyGoalOrEcommerce($idSite)) { + $siteHasEcommerceOrGoals = true; + break; + } + } + } + + $logAggregator = $archiveProcessor->getLogAggregator(); + + // try to query goal data only, if goals or ecommerce is actually used + // otherwise we simply insert empty records + if ($siteHasEcommerceOrGoals) { + $selects = []; + $selects = array_merge($selects, LogAggregator::getSelectsFromRangedColumn( + self::VISITS_COUNT_FIELD, self::$visitCountRanges, self::LOG_CONVERSION_TABLE, $prefixes[self::VISITS_UNTIL_RECORD_NAME] + )); + $selects = array_merge($selects, LogAggregator::getSelectsFromRangedColumn( + 'FLOOR(log_conversion.' . self::SECONDS_SINCE_FIRST_VISIT_FIELD . ' / 86400)', self::$daysToConvRanges, self::LOG_CONVERSION_TABLE, $prefixes[self::DAYS_UNTIL_CONV_RECORD_NAME] + )); + + $query = $logAggregator->queryConversionsByDimension([], false, $selects); + if ($query === false) { + return []; + } + + $conversionMetrics = $logAggregator->getConversionsMetricFields(); + while ($row = $query->fetch()) { + $idGoal = $row['idgoal']; + unset($row['idgoal']); + unset($row['label']); + + $values = []; + foreach ($conversionMetrics as $field => $statement) { + $values[$field] = $row[$field]; + } + $goals->sumMetrics($idGoal, $values); + + if (empty($visitsToConversions[$idGoal])) { + $visitsToConversions[$idGoal] = new DataTable(); + } + $array = LogAggregator::makeArrayOneColumn($row, Metrics::INDEX_NB_CONVERSIONS, $prefixes[self::VISITS_UNTIL_RECORD_NAME]); + $visitsToConversions[$idGoal]->addDataTable(DataTable::makeFromIndexedArray($array)); + + if (empty($daysToConversions[$idGoal])) { + $daysToConversions[$idGoal] = new DataTable(); + } + $array = LogAggregator::makeArrayOneColumn($row, Metrics::INDEX_NB_CONVERSIONS, $prefixes[self::DAYS_UNTIL_CONV_RECORD_NAME]); + $daysToConversions[$idGoal]->addDataTable(DataTable::makeFromIndexedArray($array)); + + // We don't want to sum Abandoned cart metrics in the overall revenue/conversions/converted visits + // since it is a "negative conversion" + if ($idGoal != GoalManager::IDGOAL_CART) { + $totalConversions += $row[Metrics::INDEX_GOAL_NB_CONVERSIONS]; + $totalRevenue += $row[Metrics::INDEX_GOAL_REVENUE]; + } + } + } + + // Stats by goal, for all visitors + $numericRecords = $this->getConversionsNumericMetrics($goals); + + $nbConvertedVisits = $archiveProcessor->getNumberOfVisitsConverted(); + + $result = array_merge([ + // Stats for all goals + Archiver::getRecordName('nb_conversions') => $totalConversions, + Archiver::getRecordName('nb_visits_converted') => $nbConvertedVisits, + Archiver::getRecordName('revenue') => $totalRevenue, + ], $numericRecords); + + foreach ($visitsToConversions as $idGoal => $table) { + $recordName = Archiver::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $idGoal); + $result[$recordName] = $table; + } + $result[Archiver::getRecordName(self::VISITS_UNTIL_RECORD_NAME)] = $this->getOverviewFromGoalTables($visitsToConversions); + + foreach ($daysToConversions as $idGoal => $table) { + $recordName = Archiver::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $idGoal); + $result[$recordName] = $table; + } + $result[Archiver::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME)] = $this->getOverviewFromGoalTables($daysToConversions); + + return $result; + } + + private function getOverviewFromGoalTables(array $tableByGoal): DataTable + { + $overview = new DataTable(); + foreach ($tableByGoal as $idGoal => $table) { + if ($this->isStandardGoal($idGoal)) { + $overview->addDataTable($table); + } + } + return $overview; + } + + private function isStandardGoal(int $idGoal): bool + { + return !in_array($idGoal, $this->getEcommerceIdGoals()); + } + + public function getRecordMetadata(ArchiveProcessor $archiveProcessor): array + { + $goals = API::getInstance()->getGoals($this->getSiteId($archiveProcessor)); + $goals = array_keys($goals); + + if (Manager::getInstance()->isPluginActivated('Ecommerce')) { + $goals = array_merge($goals, $this->getEcommerceIdGoals()); + } + + // Overall goal metrics + $goals[] = false; + + $records = []; + foreach ($goals as $idGoal) { + $metricsToSum = Goals::getGoalColumns($idGoal); + foreach ($metricsToSum as $metricName) { + $records[] = Record::make(Record::TYPE_NUMERIC, Archiver::getRecordName($metricName, $idGoal)); + } + + $records[] = Record::make(Record::TYPE_BLOB, Archiver::getRecordName(self::VISITS_UNTIL_RECORD_NAME, $idGoal)); + $records[] = Record::make(Record::TYPE_BLOB, Archiver::getRecordName(self::DAYS_UNTIL_CONV_RECORD_NAME, $idGoal)); + } + return $records; + } + + private function getConversionsNumericMetrics(DataArray $goals): array + { + $numericRecords = []; + $goals = $goals->getDataArray(); + foreach ($goals as $idGoal => $array) { + foreach ($array as $metricId => $value) { + $metricName = Metrics::$mappingFromIdToNameGoal[$metricId]; + $recordName = Archiver::getRecordName($metricName, $idGoal); + $numericRecords[$recordName] = $value; + } + } + return $numericRecords; + } + + public function isEnabled(ArchiveProcessor $archiveProcessor): bool + { + return $archiveProcessor->getNumberOfVisitsConverted() > 0; + } +} diff --git a/plugins/Goals/RecordBuilders/ProductRecord.php b/plugins/Goals/RecordBuilders/ProductRecord.php new file mode 100644 index 00000000000..dc889e8532c --- /dev/null +++ b/plugins/Goals/RecordBuilders/ProductRecord.php @@ -0,0 +1,225 @@ + 'idaction_product_sku', + self::NAME_FIELD => 'idaction_product_name', + self::CATEGORY_FIELD => 'idaction_product_cat', + self::CATEGORY2_FIELD => 'idaction_product_cat2', + self::CATEGORY3_FIELD => 'idaction_product_cat3', + self::CATEGORY4_FIELD => 'idaction_product_cat4', + self::CATEGORY5_FIELD => 'idaction_product_cat5', + ]; + + /** + * @var string + */ + private $dimension; + + /** + * @var string + */ + private $recordName; + + /** + * @var string[] + */ + private $dimensionsToAggregate; + + public function __construct($dimension, $recordName, $otherDimensionsToAggregate = []) + { + $general = Config::getInstance()->General; + $productReportsMaximumRows = $general['datatable_archiving_maximum_rows_products']; + + parent::__construct($productReportsMaximumRows, $productReportsMaximumRows, Metrics::INDEX_ECOMMERCE_ITEM_REVENUE); + + $this->dimension = $dimension; + $this->recordName = $recordName; + $this->dimensionsToAggregate = array_merge([$dimension], $otherDimensionsToAggregate); + } + + public function isEnabled(ArchiveProcessor $archiveProcessor): bool + { + return Manager::getInstance()->isPluginActivated('Ecommerce'); + } + + public function getRecordMetadata(ArchiveProcessor $archiveProcessor): array + { + $abandonedCartRecordName = Archiver::getItemRecordNameAbandonedCart($this->recordName); + + return [ + Record::make(Record::TYPE_BLOB, $this->recordName), + Record::make(Record::TYPE_BLOB, $abandonedCartRecordName), + ]; + } + + protected function aggregate(ArchiveProcessor $archiveProcessor): array + { + $itemReports = []; + foreach ($this->getEcommerceIdGoals() as $ecommerceType) { + $itemReports[$ecommerceType] = new DataArray(); + } + + $logAggregator = $archiveProcessor->getLogAggregator(); + + // try to query ecommerce items only, if ecommerce is actually used + // otherwise we simply insert empty records + if ($this->usesEcommerce($this->getSiteId($archiveProcessor))) { + foreach ($this->dimensionsToAggregate as $dimension) { + $query = $logAggregator->queryEcommerceItems($dimension); + if ($query !== false) { + $this->aggregateFromEcommerceItems($itemReports, $query, $dimension); + } + + $query = $this->queryItemViewsForDimension($logAggregator, $dimension); + if ($query !== false) { + $this->aggregateFromEcommerceViews($itemReports, $query, $dimension); + } + } + } + + $records = []; + foreach ($itemReports as $ecommerceType => $itemAggregate) { + $recordName = $this->recordName; + if ($ecommerceType == GoalManager::IDGOAL_CART) { + $recordName = Archiver::getItemRecordNameAbandonedCart($recordName); + } + + $table = $itemAggregate->asDataTable(); + $records[$recordName] = $table; + } + return $records; + } + + protected function aggregateFromEcommerceItems(array $itemReports, $query, string $dimension): void + { + while ($row = $query->fetch()) { + $ecommerceType = $row['ecommerceType']; + + $label = $this->cleanupRowGetLabel($row, $dimension); + if ($label === null) { + continue; + } + + $array = $itemReports[$ecommerceType]; + + $this->roundColumnValues($row); + $array->sumMetrics($label, $row); + } + } + + protected function aggregateFromEcommerceViews(array $itemReports, $query, string $dimension): void + { + while ($row = $query->fetch()) { + $label = $this->getRowLabel($row, $dimension); + if ($label === false) { + continue; // ignore empty additional categories + } + + unset($row['label']); + + if (isset($row['avg_price_viewed'])) { + $row['avg_price_viewed'] = round($row['avg_price_viewed'], GoalManager::REVENUE_PRECISION); + } + + // add views to all types + foreach ($itemReports as $ecommerceType => $dataArray) { + $dataArray->sumMetrics($label, $row); + } + } + } + + protected function queryItemViewsForDimension(LogAggregator $logAggregator, string $dimension) + { + $column = $this->actionMapping[$dimension]; + $where = "log_link_visit_action.$column is not null"; + + return $logAggregator->queryActionsByDimension( + ['label' => 'log_action1.name'], + $where, + ['AVG(log_link_visit_action.product_price) AS `avg_price_viewed`'], + false, + null, + [$column] + ); + } + + protected function roundColumnValues(array &$row): void + { + $columnsToRound = array( + Metrics::INDEX_ECOMMERCE_ITEM_REVENUE, + Metrics::INDEX_ECOMMERCE_ITEM_QUANTITY, + Metrics::INDEX_ECOMMERCE_ITEM_PRICE, + Metrics::INDEX_ECOMMERCE_ITEM_PRICE_VIEWED, + ); + foreach ($columnsToRound as $column) { + if (isset($row[$column]) + && $row[$column] == round($row[$column]) + ) { + $row[$column] = round($row[$column]); + } + } + } + + protected function getRowLabel(array &$row, string $dimension): ?string + { + $label = $row['label']; + if (empty($label)) { + // An empty additional category -> skip this iteration + if ($dimension != $this->dimension) { + return null; + } + $label = "Value not defined"; + } + return $label; + } + + protected function cleanupRowGetLabel(array &$row, string $dimension): ?string + { + $label = $this->getRowLabel($row, $dimension); + + if (isset($row['ecommerceType']) && $row['ecommerceType'] == GoalManager::IDGOAL_CART) { + // abandoned carts are the number of visits with an abandoned cart + $row[Metrics::INDEX_ECOMMERCE_ORDERS] = $row[Metrics::INDEX_NB_VISITS]; + } + + unset($row[Metrics::INDEX_NB_VISITS]); + unset($row['label']); + unset($row['labelIdAction']); + unset($row['ecommerceType']); + + return $label; + } +} diff --git a/plugins/Goals/Reports/GetVisitsUntilConversion.php b/plugins/Goals/Reports/GetVisitsUntilConversion.php index cd0a04d70bf..2ae3deaebc3 100644 --- a/plugins/Goals/Reports/GetVisitsUntilConversion.php +++ b/plugins/Goals/Reports/GetVisitsUntilConversion.php @@ -11,7 +11,7 @@ use Piwik\Piwik; use Piwik\Plugin\ViewDataTable; use Piwik\Plugins\Goals\Columns\VisitsUntilConversion; -use Piwik\Plugins\Goals\Archiver; +use Piwik\Plugins\Goals\RecordBuilders\GeneralGoalsRecords; class GetVisitsUntilConversion extends Base { @@ -44,7 +44,7 @@ public function configureView(ViewDataTable $view) $view->requestConfig->filter_sort_column = 'label'; $view->requestConfig->filter_sort_order = 'asc'; - $view->requestConfig->filter_limit = count(Archiver::$visitCountRanges); + $view->requestConfig->filter_limit = count(GeneralGoalsRecords::$visitCountRanges); $view->config->addTranslations(array('label' => $this->dimension->getName())); } diff --git a/plugins/SegmentEditor/SegmentEditor.php b/plugins/SegmentEditor/SegmentEditor.php index 89093deffd2..5eff8835b53 100644 --- a/plugins/SegmentEditor/SegmentEditor.php +++ b/plugins/SegmentEditor/SegmentEditor.php @@ -264,7 +264,9 @@ private function getSegmentIfIsUnprocessed() // check if segment archive does not exist $processorParams = new \Piwik\ArchiveProcessor\Parameters($site, $period, $segment); $archiveIdAndStats = ArchiveSelector::getArchiveIdAndVisits($processorParams, null); - if (!empty($archiveIdAndStats[0]) || !empty($archiveIdAndStats[1])) { + if (!empty($archiveIdAndStats['idArchives']) + || !empty($archiveIdAndStats['visits']) + ) { return null; } diff --git a/tests/PHPUnit/Integration/Archive/PartialArchiveTest.php b/tests/PHPUnit/Integration/Archive/PartialArchiveTest.php new file mode 100644 index 00000000000..7d38eb6e865 --- /dev/null +++ b/tests/PHPUnit/Integration/Archive/PartialArchiveTest.php @@ -0,0 +1,207 @@ +get(1, 'day', '2020-04-07', false, $this->idGoal); // day first + $data = GoalsApi::getInstance()->get(1, 'range', '2020-04-06,2020-04-09', false, $this->idGoal); + $this->assertEquals([ + 'nb_conversions' => 1, + 'nb_visits_converted' => 1, + 'revenue' => 0, + 'conversion_rate' => 'Intl_NumberFormatPercent', + 'nb_conversions_new_visit' => 1, + 'nb_visits_converted_new_visit' => 1, + 'revenue_new_visit' => 0, + 'conversion_rate_new_visit' => 'Intl_NumberFormatPercent', + 'nb_conversions_returning_visit' => 0, + 'nb_visits_converted_returning_visit' => 0, + 'revenue_returning_visit' => 0, + 'conversion_rate_returning_visit' => 'Intl_NumberFormatPercent', + ], $data->getFirstRow()->getColumns()); + + // check archive is all plugins archive as expected + [$idArchives, $archiveInfo] = $this->getArchiveInfo('2020_04', 5, false); + $this->assertEquals([ + ['idsite' => 1, 'date1' => '2020-04-06', 'date2' => '2020-04-09', 'period' => 5, 'name' => 'done', 'value' => 1, 'blob_count' => 57], + ], $archiveInfo); + + $maxIdArchive = $this->getMaxIdArchive('2020_04'); + + self::trackAnotherVisit(); + + // trigger browser archiving for range + GoalsApi::getInstance()->get(1, 'day', '2020-04-08', false, $this->idGoal); // day first + unset($_GET['trigger']); + StaticContainer::get(ArchiveInvalidator::class)->markArchivesAsInvalidated([1], ['2020-04-06,2020-04-09'], 'range'); + $data = GoalsApi::getInstance()->get(1, 'range', '2020-04-06,2020-04-09', false, $this->idGoal); + $this->assertEquals([ + 'nb_conversions' => 2, + 'nb_visits_converted' => 2, + 'revenue' => 0, + 'conversion_rate' => 'Intl_NumberFormatPercent', + 'nb_conversions_new_visit' => 2, + 'nb_visits_converted_new_visit' => 2, + 'revenue_new_visit' => 0, + 'conversion_rate_new_visit' => 'Intl_NumberFormatPercent', + 'nb_conversions_returning_visit' => 0, + 'nb_visits_converted_returning_visit' => 0, + 'revenue_returning_visit' => 0, + 'conversion_rate_returning_visit' => 'Intl_NumberFormatPercent', + ], $data->getFirstRow()->getColumns()); + + [$idArchives, $archiveInfo] = $this->getArchiveInfo('2020_04', 5, false, $maxIdArchive); + + $archiveNames = $this->getArchiveNames('2020_04', $idArchives[0], 'numeric'); + $this->assertEquals([ + 'Goal_1_nb_conversions', + 'Goal_1_nb_visits_converted', + ], $archiveNames); + + $blobArchiveNames = $this->getArchiveNames('2020_04', $idArchives[0]); + $this->assertEquals([], $blobArchiveNames); + + $this->assertEquals([ + // expect only one blob for new range partial archive + ['idsite' => 1, 'date1' => '2020-04-06', 'date2' => '2020-04-09', 'period' => 5, 'name' => 'done.Goals', 'value' => 5, 'blob_count' => 0], + ['idsite' => 1, 'date1' => '2020-04-06', 'date2' => '2020-04-09', 'period' => 5, 'name' => 'done.VisitsSummary', 'value' => 1, 'blob_count' => 0], + ], $archiveInfo); + } + + public function test_rangeArchiving_onlyArchivesSingleRecord_whenQueryingBlobs() + { + // first trigger all plugins archiving + $_GET['trigger'] = 'archivephp'; + GoalsApi::getInstance()->getDaysToConversion(1, 'day', '2020-04-07', false, $this->idGoal); // day first + $data = GoalsApi::getInstance()->getDaysToConversion(1, 'range', '2020-04-06,2020-04-09', false, $this->idGoal); + $this->assertEquals(['label' => '0-0', Metrics::INDEX_NB_CONVERSIONS => 1], $data->getFirstRow()->getColumns()); + + // check archive is all plugins archive as expected + [$idArchives, $archiveInfo] = $this->getArchiveInfo('2020_04', 5, false); + $this->assertEquals([ + ['idsite' => 1, 'date1' => '2020-04-06', 'date2' => '2020-04-09', 'period' => 5, 'name' => 'done', 'value' => 1, 'blob_count' => 57], + ], $archiveInfo); + + $maxIdArchive = $this->getMaxIdArchive('2020_04'); + + self::trackAnotherVisit(); + + // trigger browser archiving for range + GoalsApi::getInstance()->getDaysToConversion(1, 'day', '2020-04-08', false, $this->idGoal); // first day + unset($_GET['trigger']); + StaticContainer::get(ArchiveInvalidator::class)->markArchivesAsInvalidated([1], ['2020-04-06,2020-04-09'], 'range'); + $data = GoalsApi::getInstance()->getDaysToConversion(1, 'range', '2020-04-06,2020-04-09', false, $this->idGoal); + $this->assertEquals(['label' => '0-0', Metrics::INDEX_NB_CONVERSIONS => 2], $data->getFirstRow()->getColumns()); + + [$idArchives, $archiveInfo] = $this->getArchiveInfo('2020_04', 5, false, $maxIdArchive); + + $archiveNames = $this->getArchiveNames('2020_04', $idArchives[0]); + $this->assertEquals(['Goal_1_days_until_conv'], $archiveNames); + + $this->assertEquals([ + // expect only one blob for new range partial archive + ['idsite' => 1, 'date1' => '2020-04-06', 'date2' => '2020-04-09', 'period' => 5, 'name' => 'done.Goals', 'value' => 5, 'blob_count' => 1], + ], $archiveInfo); + } + + private static function createWebsite() + { + Fixture::createWebsite('2018-05-05 09:00:00'); + GoalsApi::getInstance()->addGoal(1, 'test goal', 'url', 'http', 'contains'); + } + + private static function trackVisit() + { + $t = Fixture::getTracker(1, self::$dateTime); + $t->setUrl('http://site.com/path'); + Fixture::checkResponse($t->doTrackPageView('page title')); + } + + private static function trackAnotherVisit() + { + $t = Fixture::getTracker(1, Date::factory(self::$dateTime)->addDay(1)->getDatetime()); + $t->setUrl('http://site.com/path2'); + Fixture::checkResponse($t->doTrackPageView('page title 2')); + } + + private function getArchiveInfo($yearMonth, $period, $segmentHash = false, $idArchiveGreaterThan = 0) + { + $sql = 'SELECT idarchive, idsite, date1, date2, period, name, value FROM ' + . Common::prefixTable('archive_numeric_' . $yearMonth) + . ' WHERE (name = \'done' . $segmentHash . '\' OR name LIKE \'done' . $segmentHash . '.%\') AND period = ? AND idarchive > ?'; + $archiveNumericInfo = Db::fetchAll($sql, [$period, $idArchiveGreaterThan]); + + $sql = 'SELECT idarchive, COUNT(DISTINCT name) AS blob_count FROM ' . Common::prefixTable('archive_blob_' . $yearMonth) + . ' GROUP BY idarchive'; + $archiveBlobInfo = Db::fetchAll($sql); + $archiveBlobInfo = array_column($archiveBlobInfo, 'blob_count', 'idarchive'); + + $idArchives = []; + foreach ($archiveNumericInfo as &$row) { + $row['blob_count'] = $archiveBlobInfo[$row['idarchive']] ?? 0; + + // archives can randomly be created out of order despite not using core:archive, so we don't check their + // value. we still need to use it, though, so we return the values. + $idArchives[] = $row['idarchive']; + unset($row['idarchive']); + } + + return [$idArchives, $archiveNumericInfo]; + } + + private function getArchiveNames($yearMonth, $idArchive, $type = 'blob') + { + $sql = 'SELECT DISTINCT name FROM ' . Common::prefixTable('archive_' . $type . '_' . $yearMonth) + . ' WHERE idarchive = ? AND name NOT LIKE \'done%\''; + $rows = Db::fetchAll($sql, [$idArchive]); + $rows = array_column($rows, 'name'); + return $rows; + } + + protected static function configureFixture($fixture) + { + parent::configureFixture($fixture); + $fixture->createSuperUser = true; + } + + private function getMaxIdArchive($yearMonth) + { + return Db::fetchOne('SELECT MAX(idarchive) FROM ' . Common::prefixTable('archive_numeric_' . $yearMonth)); + } +} diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php index f0acf13986c..0da0b323cbc 100644 --- a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php +++ b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php @@ -1057,7 +1057,23 @@ public function test_loadExistingArchiveIdFromDb_returnsFalsesIfNoArchiveFound() $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertEquals([false, false, false, false, false, false], $archiveInfo); + // unset numeric index keys kept for BC + unset($archiveInfo[0]); + unset($archiveInfo[1]); + unset($archiveInfo[2]); + unset($archiveInfo[3]); + unset($archiveInfo[4]); + unset($archiveInfo[5]); + + $this->assertEquals([ + 'idArchives' => false, + 'visits' => false, + 'visitsConverted' => false, + 'archiveExists' => false, + 'doneFlagValue' => false, + 'tsArchived' => false, + 'existingRecords' => null, + ], $archiveInfo); } /** @@ -1073,17 +1089,48 @@ public function test_loadExistingArchiveIdFromDb_returnsFalsesPeriodIsForcedToAr $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertNotEmpty($archiveInfo[4]); - $this->assertLessThanOrEqual(time(), strtotime($archiveInfo[4])); + $this->assertNotEmpty($archiveInfo['tsArchived']); + $this->assertLessThanOrEqual(time(), strtotime($archiveInfo['tsArchived'])); + + unset($archiveInfo['tsArchived']); + + // unset numeric index keys kept for BC + unset($archiveInfo[0]); + unset($archiveInfo[1]); + unset($archiveInfo[2]); + unset($archiveInfo[3]); unset($archiveInfo[4]); - $archiveInfo = array_values($archiveInfo); + unset($archiveInfo[5]); - $this->assertNotEquals([false, false, false, false, false, false], $archiveInfo); + $this->assertEquals([ + 'idArchives' => [1], + 'visits' => 10, + 'visitsConverted' => 0, + 'archiveExists' => true, + 'doneFlagValue' => 1, + 'existingRecords' => null, + ], $archiveInfo); Config::getInstance()->Debug[$configSetting] = 1; $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertEquals([false, false, false, false, false, false], $archiveInfo); + + // unset numeric index keys kept for BC + unset($archiveInfo[0]); + unset($archiveInfo[1]); + unset($archiveInfo[2]); + unset($archiveInfo[3]); + unset($archiveInfo[4]); + unset($archiveInfo[5]); + + $this->assertEquals([ + 'idArchives' => false, + 'visits' => false, + 'visitsConverted' => false, + 'archiveExists' => false, + 'doneFlagValue' => false, + 'tsArchived' => false, + ], $archiveInfo); } public function getTestDataForLoadExistingArchiveIdFromDbDebugConfig() @@ -1106,11 +1153,25 @@ public function test_loadExistingArchiveIdFromDb_returnsArchiveIfArchiveInThePas $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertNotEmpty($archiveInfo[4]); + $this->assertNotEmpty($archiveInfo['tsArchived']); + unset($archiveInfo['tsArchived']); + + // unset numeric index keys kept for BC + unset($archiveInfo[0]); + unset($archiveInfo[1]); + unset($archiveInfo[2]); + unset($archiveInfo[3]); unset($archiveInfo[4]); - $archiveInfo = array_values($archiveInfo); + unset($archiveInfo[5]); - $this->assertEquals([['1'], '10', '0', true, '1'], $archiveInfo); + $this->assertEquals([ + 'idArchives' => ['1'], + 'visits' => '10', + 'visitsConverted' => '0', + 'archiveExists' => true, + 'doneFlagValue' => '1', + 'existingRecords' => null, + ], $archiveInfo); } public function test_loadExistingArchiveIdFromDb_returnsArchiveIfForACurrentPeriod_AndNewEnough() @@ -1122,11 +1183,25 @@ public function test_loadExistingArchiveIdFromDb_returnsArchiveIfForACurrentPeri $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertNotEmpty($archiveInfo[4]); + $this->assertNotEmpty($archiveInfo['tsArchived']); + unset($archiveInfo['tsArchived']); + + // unset numeric index keys kept for BC + unset($archiveInfo[0]); + unset($archiveInfo[1]); + unset($archiveInfo[2]); + unset($archiveInfo[3]); unset($archiveInfo[4]); - $archiveInfo = array_values($archiveInfo); + unset($archiveInfo[5]); - $this->assertEquals([['1'], '10', '0', true, '1'], $archiveInfo); + $this->assertEquals([ + 'idArchives' => ['1'], + 'visits' => '10', + 'visitsConverted' => '0', + 'archiveExists' => true, + 'doneFlagValue' => '1', + 'existingRecords' => null, + ], $archiveInfo); } public function test_loadExistingArchiveIdFromDb_returnsNoArchiveIfForACurrentPeriod_AndNoneAreNewEnough() @@ -1138,11 +1213,25 @@ public function test_loadExistingArchiveIdFromDb_returnsNoArchiveIfForACurrentPe $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertNotEmpty($archiveInfo[4]); + $this->assertNotEmpty($archiveInfo['tsArchived']); + unset($archiveInfo['tsArchived']); + + // unset numeric index keys kept for BC + unset($archiveInfo[0]); + unset($archiveInfo[1]); + unset($archiveInfo[2]); + unset($archiveInfo[3]); unset($archiveInfo[4]); - $archiveInfo = array_values($archiveInfo); + unset($archiveInfo[5]); - $this->assertEquals([false, '10', '0', true, '1'], $archiveInfo); // visits are still returned as this was the original behavior + $this->assertEquals([ + 'idArchives' => false, + 'visits' => '10', + 'visitsConverted' => '0', + 'archiveExists' => true, + 'doneFlagValue' => '1', + 'existingRecords' => null, + ], $archiveInfo); // visits are still returned as this was the original behavior } /** diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php index 83d062109dc..f2c6cc212da 100644 --- a/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php @@ -187,12 +187,26 @@ public function test_getArchiveIdAndVisits_returnsCorrectResult($period, $date, $params = new \Piwik\ArchiveProcessor\Parameters(new Site(1), Factory::build($period, $date), new Segment($segment, [1])); $result = ArchiveSelector::getArchiveIdAndVisits($params, $minDateProcessed, $includeInvalidated); - if ($result[4] !== false) { - Date::factory($result[4]); + if ($result['tsArchived'] !== false) { + Date::factory($result['tsArchived']); } + unset($result['tsArchived']); + + // remove BC indexed values + unset($result[0]); + unset($result[1]); + unset($result[2]); + unset($result[3]); unset($result[4]); - $result = array_values($result); + unset($result[5]); + + if (isset($result['existingRecords'])) { + sort($result['existingRecords']); + } + if (isset($expected['existingRecords'])) { + sort($expected['existingRecords']); + } $this->assertEquals($expected, $result); } @@ -212,7 +226,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, true, - [false, false, false, false, false], + ['idArchives' => false, 'visits' => false, 'visitsConverted' => false, 'archiveExists' => false, 'doneFlagValue' => false, 'existingRecords' => null], ], [ 'day', @@ -225,7 +239,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, true, - [false, false, false, false, false], + ['idArchives' => false, 'visits' => false, 'visitsConverted' => false, 'archiveExists' => false, 'doneFlagValue' => false, 'existingRecords' => null], ], // value is not valid @@ -241,7 +255,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [false, false, false, true, '99'], + ['idArchives' => false, 'visits' => false, 'visitsConverted' => false, 'archiveExists' => true, 'doneFlagValue' => '99', 'existingRecords' => null], ], [ 'day', @@ -257,7 +271,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [false, 0, 0, true, '99'], + ['idArchives' => false, 'visits' => 0, 'visitsConverted' => 0, 'archiveExists' => true, 'doneFlagValue' => '99', 'existingRecords' => null], ], [ 'day', @@ -270,7 +284,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [false, 20, 40, true, false], + ['idArchives' => false, 'visits' => 20, 'visitsConverted' => 40, 'archiveExists' => true, 'doneFlagValue' => false, 'existingRecords' => null], ], [ 'day', @@ -283,7 +297,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [false, 30, 50, true, false], + ['idArchives' => false, 'visits' => 30, 'visitsConverted' => 50, 'archiveExists' => true, 'doneFlagValue' => false, 'existingRecords' => null], ], [ 'day', @@ -297,7 +311,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [false, false, false, true, false], + ['idArchives' => false, 'visits' => false, 'visitsConverted' => false, 'archiveExists' => true, 'doneFlagValue' => false, 'existingRecords' => null], ], // archive is too old @@ -311,7 +325,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [false, false, false, true, '1'], + ['idArchives' => false, 'visits' => false, 'visitsConverted' => false, 'archiveExists' => true, 'doneFlagValue' => '1', 'existingRecords' => null], ], [ 'day', @@ -325,7 +339,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [false, 1, false, true, '1'], + ['idArchives' => false, 'visits' => 1, 'visitsConverted' => false, 'archiveExists' => true, 'doneFlagValue' => '1', 'existingRecords' => null], ], // no archive done flags, but metric @@ -339,7 +353,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [false, false, false, false, false], + ['idArchives' => false, 'visits' => false, 'visitsConverted' => false, 'archiveExists' => false, 'doneFlagValue' => false, 'existingRecords' => null], ], [ 'day', @@ -352,7 +366,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [false, false, false, false, false], + ['idArchives' => false, 'visits' => false, 'visitsConverted' => false, 'archiveExists' => false, 'doneFlagValue' => false, 'existingRecords' => null], ], // archive exists and is usable @@ -365,7 +379,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [[1], 0, 0, true, '1'], + ['idArchives' => [1], 'visits' => 0, 'visitsConverted' => 0, 'archiveExists' => true, 'doneFlagValue' => '1', 'existingRecords' => null], ], [ 'day', @@ -378,7 +392,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [[1], 5, 10, true, '1'], + ['idArchives' => [1], 'visits' => 5, 'visitsConverted' => 10, 'archiveExists' => true, 'doneFlagValue' => '1', 'existingRecords' => null], ], // range archive, valid @@ -393,7 +407,7 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [[1], 5, 10, true, '1'], + ['idArchives' => [1], 'visits' => 5, 'visitsConverted' => 10, 'archiveExists' => true, 'doneFlagValue' => '1', 'existingRecords' => null], ], // range archive, invalid @@ -408,7 +422,244 @@ public function getTestDataForGetArchiveIdAndVisits() '', $minDateProcessed, false, - [false, 5, 10, true, '4'], // forcing archiving since invalid + browser archiving of ranges allowed + // forcing archiving since invalid + browser archiving of ranges allowed + ['idArchives' => false, 'visits' => 5, 'visitsConverted' => 10, 'archiveExists' => true, 'doneFlagValue' => '4', 'existingRecords' => null], + ], + ]; + } + + /** + * @dataProvider getTestDataForGetArchiveIdAndVisitsWithOnlyPartialArchives + */ + public function test_getArchiveIdAndVisits_whenThereAreOnlyPartialArchives($archiveRows, $requestedReports, $expected, $minDatetimeArchiveProcessedUTC = false) + { + Fixture::createWebsite('2010-02-02 00:00:00'); + + Rules::setBrowserTriggerArchiving(false); + API::getInstance()->add('test segment', self::TEST_SEGMENT, 0, 0); // processed in real time + + $this->insertArchiveData($archiveRows); + + $params = new \Piwik\ArchiveProcessor\Parameters(new Site(1), Factory::build('range', '2020-03-04,2020-03-08'), new Segment('', [1])); + $params->setRequestedPlugin('TestPlugin'); + $params->setArchiveOnlyReport($requestedReports); + + $result = ArchiveSelector::getArchiveIdAndVisits($params, $minDatetimeArchiveProcessedUTC); + + if ($result['tsArchived'] !== false) { + Date::factory($result['tsArchived']); + } + + unset($result['tsArchived']); + + // remove BC indexed values + unset($result[0]); + unset($result[1]); + unset($result[2]); + unset($result[3]); + unset($result[4]); + unset($result[5]); + + if (isset($result['existingRecords'])) { + sort($result['existingRecords']); + } + if (isset($expected['existingRecords'])) { + sort($expected['existingRecords']); + } + + $this->assertEquals($expected, $result); + } + + public function getTestDataForGetArchiveIdAndVisitsWithOnlyPartialArchives() + { + // $archiveRows, $plugin, $requestedReports, $expected + return [ + // only partial archives, no requested reports + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf', 'is_blob_data' => true], + + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf2', 'is_blob_data' => true], + + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf3', 'is_blob_data' => true], + ], + null, + [ + 'idArchives' => [1], + 'visits' => false, + 'visitsConverted' => false, + 'archiveExists' => true, + 'doneFlagValue' => false, + 'existingRecords' => null, + ], + ], + + // only partial archives, requested reports, no existing reports + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf', 'is_blob_data' => true], + + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf2', 'is_blob_data' => true], + + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf3', 'is_blob_data' => true], + ], + 'TestPlugin_otherMetric', + [ + 'idArchives' => false, + 'visits' => false, + 'visitsConverted' => false, + 'archiveExists' => true, + 'doneFlagValue' => false, + 'existingRecords' => null, + ], + ], + + // only partial archives, requested reports, some existing reports (both numeric and blob) + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'done.TestPlugin', 'value' => 5, 'ts_archived' => '2020-03-08 03:00:00'], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_metric', 'value' => 5, 'ts_archived' => '2020-03-08 03:00:00'], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf 1', 'is_blob_data' => true, 'ts_archived' => '2020-03-08 03:00:00'], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_metric2', 'value' => 5, 'ts_archived' => '2020-03-08 03:00:00'], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_blob2', 'value' => 'slkdjf 2', 'is_blob_data' => true, 'ts_archived' => '2020-03-08 03:00:00'], + + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf 3', 'is_blob_data' => true], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_metric2', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_blob2', 'value' => 'slkdjf 4', 'is_blob_data' => true], + + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf 5', 'is_blob_data' => true], + ['idarchive' => 3, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_metric2', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_blob2', 'value' => 'slkdjf 6', 'is_blob_data' => true], + ], + ['TestPlugin_metric', 'TestPlugin_blob'], + [ + 'idArchives' => [1], + 'visits' => false, + 'visitsConverted' => false, + 'archiveExists' => true, + 'doneFlagValue' => false, + 'existingRecords' => ['TestPlugin_metric', 'TestPlugin_blob'], + ], + '2020-03-08 00:00:00', + ], + + // only partial archives, requested reports, some existing reports (both numeric and blob), but archive is too old + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'done.TestPlugin', 'value' => 5, 'ts_archived' => '2020-03-08 03:00:00'], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_metric', 'value' => 5, 'ts_archived' => '2020-03-08 03:00:00'], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf 1', 'is_blob_data' => true, 'ts_archived' => '2020-03-08 03:00:00'], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_metric2', 'value' => 5, 'ts_archived' => '2020-03-08 03:00:00'], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_blob2', 'value' => 'slkdjf 2', 'is_blob_data' => true, 'ts_archived' => '2020-03-08 03:00:00'], + + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf 3', 'is_blob_data' => true], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_metric2', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_blob2', 'value' => 'slkdjf 4', 'is_blob_data' => true], + + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf 5', 'is_blob_data' => true], + ['idarchive' => 3, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_metric2', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_blob2', 'value' => 'slkdjf 6', 'is_blob_data' => true], + ], + ['TestPlugin_metric', 'TestPlugin_blob'], + [ + 'idArchives' => false, + 'visits' => false, + 'visitsConverted' => false, + 'archiveExists' => true, + 'doneFlagValue' => false, + 'existingRecords' => null, + ], + '2020-03-08 09:00:00', + ], + + // only partial archives, requested reports, all existing reports (both numeric and blob) + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf', 'is_blob_data' => true], + + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf2', 'is_blob_data' => true], + + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf3', 'is_blob_data' => true], + ], + ['TestPlugin_metric', 'TestPlugin_blob'], + [ + 'idArchives' => [1], + 'visits' => false, + 'visitsConverted' => false, + 'archiveExists' => true, + 'doneFlagValue' => false, + 'existingRecords' => ['TestPlugin_metric', 'TestPlugin_blob'], + ], + ], + + // only partial archives, requested reports, some existing reports (both numeric and blob) across multiple partial archives + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_metric', 'value' => 5], + + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf 1', 'is_blob_data' => true], + ['idarchive' => 2, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_metric2', 'value' => 5], + + ['idarchive' => 3, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 3, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-08', 'name' => 'TestPlugin_blob2', 'value' => 'slkdjf 2', 'is_blob_data' => true], + + ['idarchive' => 4, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 4, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_metric', 'value' => 5], + + ['idarchive' => 5, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 5, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf 3', 'is_blob_data' => true], + + ['idarchive' => 6, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 6, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_metric2', 'value' => 5], + ['idarchive' => 6, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_blob2', 'value' => 'slkdjf 4', 'is_blob_data' => true], + + ['idarchive' => 7, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 7, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'TestPlugin_metric', 'value' => 5], + ['idarchive' => 7, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'TestPlugin_blob', 'value' => 'slkdjf 5', 'is_blob_data' => true], + + ['idarchive' => 8, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 8, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_metric2', 'value' => 5], + + ['idarchive' => 9, 'idsite' => 1, 'period' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'name' => 'done.TestPlugin', 'value' => 5], + ['idarchive' => 9, 'idsite' => 1, 'period' => 5, 'date1' => '2020-03-03', 'date2' => '2020-03-09', 'name' => 'TestPlugin_blob2', 'value' => 'slkdjf 6', 'is_blob_data' => true], + ], + ['TestPlugin_metric', 'TestPlugin_blob', 'TestPlugin_blob5'], + [ + 'idArchives' => [3, 2, 1], + 'visits' => false, + 'visitsConverted' => false, + 'archiveExists' => true, + 'doneFlagValue' => false, + 'existingRecords' => ['TestPlugin_metric', 'TestPlugin_blob'], + ], ], ]; } diff --git a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php index baf0fd2fb56..1b0fba5e356 100644 --- a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php +++ b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php @@ -125,24 +125,18 @@ public function test_checkArchiveRecords_whenPeriodIsRange() + 2 /* VisitTime */) * 3, /** - * In Each "Period=range" Archive, we expect following non zero numeric entries: - * 5 metrics + 1 flag // VisitsSummary - * + 2 metrics + 1 flag // Actions - * + 1 flag // Resolution - * + 1 flag // VisitTime - * = 11 + * segments: 9 (including all visits) + * plugins: 4 different plugins + * VisitsSummary: 9 archives (8 segments + all visits) (4 metrics in each + 3 bounce_counts across 3 archives) + * Actions: 3 archives (2 segments + all visits) (2 metrics in each) + * Resolution: 3 archives (2 segments + all visits) (0 metrics in each) + * VisitTime: 3 archives (2 segments + all visits) (0 metrics in each) * - * because we call VisitFrequency.get, this creates archives for visitorType==returning - * and visitorType==new segment. - * -> There are two archives for each segment (one for "countryCode!=aa" - * and VisitFrequency creates two more. - * - * So each period=range will have = 11 records + (5 metrics + 2 flags // VisitsSummary + 3 metrics // VisitorInterest) - * = 18 - * - * Total expected records = count unique archives * records per archive - * = 3 * 21 - * = 21 + * Total: 9 VisitsSummary done flags + ((4 * 9) + 3) VisitsSummary metrics + * + 3 Actions done flags + 3 * 2 Actions metrics + * + 3 Resolution done flags + * + 3 VisitTime done flags + * = 63 */ 'archive_numeric_2010_12' => 63,