introduce RecordBuilder concept to split up Archiver code

core/Archive.php

@@ -436,20 +436,23 @@ public function getDataTableExpanded($name, $idSubtable = null, $depth = null, $
     }
 
     /**
-     * Returns the list of plugins that archive the given reports.
+     * Returns the given reports grouped by the plugin name that archives them.
      *
      * @param array $archiveNames
-     * @return array
+     * @return array `['MyPlugin' => ['MyPlugin_metric1', 'MyPlugin_report1'], ...]`
      */
     private function getRequestedPlugins($archiveNames)
     {
         $result = [];
 
         foreach ($archiveNames as $name) {
-            $result[] = self::getPluginForReport($name);
+            $plugin = self::getPluginForReport($name);
+            $result[$plugin][] = $name;
         }
 
-        return array_unique($result);
+        $result = array_map('array_unique', $result);
+
+        return $result;
     }
 
     /**
@@ -601,7 +604,8 @@ protected function get($archiveNames, $archiveDataType, $idSubtable = null)
      */
     private function getArchiveIds($archiveNames)
     {
-        $plugins = $this->getRequestedPlugins($archiveNames);
+        $archiveNamesByPlugin = $this->getRequestedPlugins($archiveNames);
+        $plugins = array_keys($archiveNamesByPlugin);
 
         // figure out which archives haven't been processed (if an archive has been processed,
         // then we have the archive IDs in $this->idarchives)
@@ -627,15 +631,13 @@ private function getArchiveIds($archiveNames)
         $globalDoneFlag = Rules::getDoneFlagArchiveContainsAllPlugins($this->params->getSegment());
         $doneFlags[$globalDoneFlag] = true;
 
-        $archiveGroups = array_unique($archiveGroups);
-
         // cache id archives for plugins we haven't processed yet
         if (!empty($archiveGroups)) {
             if (
                 Rules::isArchivingEnabledFor($this->params->getIdSites(), $this->params->getSegment(), $this->getPeriodLabel())
                 && !$this->forceFetchingWithoutLaunchingArchiving
             ) {
-                $this->cacheArchiveIdsAfterLaunching($archiveGroups, $plugins);
+                $this->cacheArchiveIdsAfterLaunching($archiveNamesByPlugin);
             } else {
                 $this->cacheArchiveIdsWithoutLaunching($plugins);
             }
@@ -651,10 +653,9 @@ private function getArchiveIds($archiveNames)
      * This function will launch the archiving process for each period/site/plugin if
      * metrics/reports have not been calculated/archived already.
      *
-     * @param array $archiveGroups @see getArchiveGroupOfReport
-     * @param array $plugins List of plugin names to archive.
+     * @param array $archiveNamesByPlugin @see getRequestedPlugins
      */
-    private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins)
+    private function cacheArchiveIdsAfterLaunching($archiveNamesByPlugin)
     {
         foreach ($this->params->getPeriods() as $period) {
             $twoDaysAfterPeriod = $period->getDateEnd()->addDay(2);
@@ -697,7 +698,7 @@ private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins)
                     continue;
                 }
 
-                $this->prepareArchive($archiveGroups, $site, $period);
+                $this->prepareArchive($archiveNamesByPlugin, $site, $period);
             }
         }
     }
@@ -744,6 +745,15 @@ private function cacheArchiveIdsWithoutLaunching($plugins)
      */
     private function getDoneStringForPlugin($plugin, $idSites)
     {
+        $requestedReport = $this->getRequestedReport();
+
+        $shouldOnlyProcessRequestedRecords = empty($requestedReport)
+            && Rules::shouldProcessOnlyReportsRequestedInArchiveQuery($this->getPeriodLabel());
+
+        if ($shouldOnlyProcessRequestedRecords) {
+            return Rules::getDoneFlagArchiveContainsOnePlugin($this->params->getSegment(), $plugin);
+        }
+
         return Rules::getDoneStringFlagFor(
             $idSites,
             $this->params->getSegment(),
@@ -881,26 +891,26 @@ public static function getPluginForReport($report)
     }
 
     /**
-     * @param $archiveGroups
+     * @param $archiveNamesByPlugin
      * @param $site
      * @param $period
      */
-    private function prepareArchive(array $archiveGroups, Site $site, Period $period)
+    private function prepareArchive(array $archiveNamesByPlugin, Site $site, Period $period)
     {
         $coreAdminHomeApi = API::getInstance();
 
-        $requestedReport = null;
-        if (SettingsServer::isArchivePhpTriggered()) {
-            $requestedReport = Common::getRequestVar('requestedReport', '', 'string');
-        }
+        $requestedReport = $this->getRequestedReport();
+
+        $shouldOnlyProcessRequestedArchives = empty($requestedReport)
+            && Rules::shouldProcessOnlyReportsRequestedInArchiveQuery($period->getLabel());
 
         $periodString = $period->getRangeString();
         $periodDateStr = $period->getLabel() == 'range' ? $periodString : $period->getDateStart()->toString();
 
         $idSites = [$site->getId()];
 
         // process for each plugin as well
-        foreach ($archiveGroups as $plugin) {
+        foreach ($archiveNamesByPlugin as $plugin => $archiveNames) {
             $doneFlag = $this->getDoneStringForPlugin($plugin, $idSites);
             $this->initializeArchiveIdCache($doneFlag);
 
@@ -910,15 +920,19 @@ private function prepareArchive(array $archiveGroups, Site $site, Period $period
                 $periodDateStr,
                 $this->params->getSegment()->getOriginalString(),
                 $plugin,
-                $requestedReport
+                empty($requestedReport) && $shouldOnlyProcessRequestedArchives ? $archiveNames : $requestedReport
             );
 
             if (
                 !empty($prepareResult)
                 && !empty($prepareResult['idarchives'])
             ) {
                 foreach ($prepareResult['idarchives'] as $idArchive) {
-                    $this->idarchives[$doneFlag][$periodString][] = $idArchive;
+                    if (empty($this->idarchives[$doneFlag][$periodString])
+                        || !in_array($idArchive, $this->idarchives[$doneFlag][$periodString])
+                    ) {
+                        $this->idarchives[$doneFlag][$periodString][] = $idArchive;
+                    }
                 }
             }
         }
@@ -956,4 +970,13 @@ public function forceFetchingWithoutLaunchingArchiving()
     {
         $this->forceFetchingWithoutLaunchingArchiving = true;
     }
+
+    private function getRequestedReport(): ?string
+    {
+        $requestedReport = null;
+        if (SettingsServer::isArchivePhpTriggered()) {
+            $requestedReport = Request::fromRequest()->getStringParameter('requestedReport', '');
+        }
+        return $requestedReport;
+    }
 }

core/ArchiveProcessor.php

@@ -740,6 +740,12 @@ public function processDependentArchive($plugin, $segment)
         }
 
         $params = $this->getParams();
+        // range archives are always processed on demand, so pre-processing dependent archives is not required
+        // here
+        if (Rules::shouldProcessOnlyReportsRequestedInArchiveQuery($params->getPeriod()->getLabel())) {
+            return;
+        }
+
         $idSites = [$params->getSite()->getId()];
 
         // important to use the original segment string when combining. As the API itself would combine the original string.

core/ArchiveProcessor/Loader.php

@@ -134,7 +134,7 @@ private function prepareArchiveImpl($pluginName)
         if (sizeof($data) == 2) {
             return $data;
         }
-        list($idArchives, $visits, $visitsConverted) = $data;
+        list($idArchives, $visits, $visitsConverted, $foundRecords) = $data;
 
         // only lock meet those conditions
         if (ArchiveProcessor::$isRootArchivingRequest && !SettingsServer::isArchivePhpTriggered()) {
@@ -153,15 +153,15 @@ private function prepareArchiveImpl($pluginName)
                     return $data;
                 }
 
-                list($idArchives, $visits, $visitsConverted) = $data;
+                list($idArchives, $visits, $visitsConverted, $foundRecords) = $data;
 
-                return $this->insertArchiveData($visits, $visitsConverted);
+                return $this->insertArchiveData($visits, $visitsConverted, $idArchives, $foundRecords);
             } finally {
                 $lock->unlock();
             }
         } else {
 
-            return $this->insertArchiveData($visits, $visitsConverted);
+            return $this->insertArchiveData($visits, $visitsConverted, $idArchives, $foundRecords);
         }
     }
 
@@ -171,17 +171,27 @@ private function prepareArchiveImpl($pluginName)
      * @param $visitsConverted
      * @return array|false[]
      */
-    protected function insertArchiveData($visits, $visitsConverted)
+    protected function insertArchiveData($visits, $visitsConverted, $existingArchives, $foundRecords)
     {
         if (SettingsServer::isArchivePhpTriggered()) {
             $this->logger->info("initiating archiving via core:archive for " . $this->params);
         }
 
+        if (!empty($foundRecords)) {
+            $this->params->setFoundRequestedReports($foundRecords);
+        }
+
         list($visits, $visitsConverted) = $this->prepareCoreMetricsArchive($visits, $visitsConverted);
         list($idArchive, $visits) = $this->prepareAllPluginsArchive($visits, $visitsConverted);
 
-        if ($this->isThereSomeVisits($visits) || PluginsArchiver::doesAnyPluginArchiveWithoutVisits()) {
-            return [[$idArchive], $visits];
+        if ($this->isThereSomeVisits($visits)
+            || PluginsArchiver::doesAnyPluginArchiveWithoutVisits()
+        ) {
+            $idArchivesToQuery = [$idArchive];
+            if (!empty($foundRecords)) {
+                $idArchivesToQuery = array_merge($idArchivesToQuery, $existingArchives ?: []);
+            }
+            return [$idArchivesToQuery, $visits];
         }
 
         return [false, false];
@@ -208,11 +218,22 @@ protected function loadArchiveData()
         // this hack was used to check the main function goes to return or continue
         // NOTE: $idArchives will contain the latest DONE_OK/DONE_INVALIDATED archive as well as any partial archives
         // with a ts_archived >= the DONE_OK/DONE_INVALIDATED date.
-        list($idArchives, $visits, $visitsConverted, $isAnyArchiveExists, $tsArchived, $value) = $this->loadExistingArchiveIdFromDb();
+        $archiveInfo = $this->loadExistingArchiveIdFromDb();
+        $idArchives = $archiveInfo['idArchives'];
+        $visits = $archiveInfo['visits'];
+        $visitsConverted = $archiveInfo['visitsConverted'];
+        $tsArchived = $archiveInfo['tsArchived'];
+        $value = $archiveInfo['doneFlagValue'];
+        $existingArchives = $archiveInfo['existingRecords'];
+
+        $requestedRecords = $this->params->getArchiveOnlyReportAsArray();
+        $isMissingRequestedRecords = !empty($requestedRecords) && is_array($existingArchives) && count($requestedRecords) != count($existingArchives);
 
         if (!empty($idArchives)
             && !Rules::isActuallyForceArchivingSinglePlugin()
-            && !$this->shouldForceInvalidatedArchive($value, $tsArchived)) {
+            && !$this->shouldForceInvalidatedArchive($value, $tsArchived)
+            && !$isMissingRequestedRecords
+        ) {
             // we have a usable idarchive (it's not invalidated and it's new enough), and we are not archiving
             // a single report
             return [$idArchives, $visits];
@@ -232,7 +253,7 @@ protected function loadArchiveData()
             }
         }
 
-        return [$idArchives, $visits, $visitsConverted];
+        return [$idArchives, $visits, $visitsConverted, $existingArchives];
     }
 
     /**
@@ -331,7 +352,14 @@ public function loadExistingArchiveIdFromDb()
 
             // return no usable archive found, and no existing archive. this will skip invalidation, which should
             // be fine since we just force archiving.
-            return [false, false, false, false, false, false];
+            return [
+                'idArchives' => false,
+                'visits' => false,
+                'visitsConverted' => false,
+                'archiveExists' => false,
+                'tsArchived' => false,
+                'doneFlagValue' => false,
+            ];
         }
 
         $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed();

core/ArchiveProcessor/Parameters.php

@@ -55,6 +55,11 @@ class Parameters
      */
     private $isArchiveOnlyReportHandled;
 
+    /**
+     * @var string[]|null
+     */
+    private $foundRequestedReports;
+
     /**
      * Constructor.
      *
@@ -71,7 +76,7 @@ public function __construct(Site $site, Period $period, Segment $segment)
      * If we want to archive only a single report, we can request that via this method.
      * It is up to each plugin's archiver to respect the setting.
      *
-     * @param string $archiveOnlyReport
+     * @param string|string[] $archiveOnlyReport
      * @api
      */
     public function setArchiveOnlyReport($archiveOnlyReport)
@@ -267,7 +272,11 @@ public function logStatusDebug()
 
     public function __toString()
     {
-        return "[idSite = {$this->getSite()->getId()}, period = {$this->getPeriod()->getLabel()} {$this->getPeriod()->getRangeString()}, segment = {$this->getSegment()->getString()}, plugin = {$this->getRequestedPlugin()}, report = {$this->getArchiveOnlyReport()}]";
+        $requestedReports = $this->getArchiveOnlyReport();
+        if (is_array($requestedReports)) {
+            $requestedReports = json_encode($requestedReports);
+        }
+        return "[idSite = {$this->getSite()->getId()}, period = {$this->getPeriod()->getLabel()} {$this->getPeriod()->getRangeString()}, segment = {$this->getSegment()->getString()}, plugin = {$this->getRequestedPlugin()}, report = {$requestedReports}]";
     }
 
     /**
@@ -295,4 +304,23 @@ public function setIsPartialArchive($isArchiveOnlyReportHandled)
     {
         $this->isArchiveOnlyReportHandled = $isArchiveOnlyReportHandled;
     }
+
+    public function getArchiveOnlyReportAsArray()
+    {
+        $requestedReport = $this->getArchiveOnlyReport();
+        if (empty($requestedReport)) {
+            return [];
+        }
+        return is_array($requestedReport) ? $requestedReport : [$requestedReport];
+    }
+
+    public function setFoundRequestedReports(array $foundRecords)
+    {
+        $this->foundRequestedReports = $foundRecords;
+    }
+
+    public function getFoundRequestedReports()
+    {
+        return $this->foundRequestedReports ?: [];
+    }
 }

core/ArchiveProcessor/PluginsArchiver.php

@@ -137,7 +137,19 @@ public function callAggregateAllPlugins($visits, $visitsConverted, $forceArchivi
 
         $archivers = static::getPluginArchivers();
 
+        $archiveOnlyPlugin = $this->params->getRequestedPlugin();
+        $archiveOnlyReports = $this->params->getArchiveOnlyReport();
+
         foreach ($archivers as $pluginName => $archiverClass) {
+            // if we are archiving specific reports for a single plugin then we don't need or want to create
+            // Archiver instances, since they will set the archive to partial even if the requested reports aren't
+            // handled by the Archiver
+            if (!empty($archiveOnlyReports)
+                && $archiveOnlyPlugin != $pluginName
+            ) {
+                continue;
+            }
+
             // We clean up below all tables created during this function call (and recursive calls)
             $latestUsedTableId = Manager::getInstance()->getMostRecentTableId();