JE-400: Optimized sync memory usage

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Added choices.js to dropdowns with many options.
 * Added epic filter to worklog selection page.
 * Removed time from period selections on worklog selection page.
+* Optimized sync memory usage.
 
 ## [1.1.0] - 2023-12-14
 

src/Model/Invoices/PagedResult.php

@@ -0,0 +1,14 @@
+<?php
+
+namespace App\Model\Invoices;
+
+class PagedResult
+{
+    public function __construct(
+        public readonly array $items,
+        public readonly int $startAt,
+        public readonly int $maxResults,
+        public readonly int $total
+    ) {
+    }
+}

src/Service/ApiServiceInterface.php

@@ -5,6 +5,7 @@
 use App\Model\Invoices\AccountData;
 use App\Model\Invoices\ClientData;
 use App\Model\Invoices\IssueData;
+use App\Model\Invoices\PagedResult;
 use App\Model\Invoices\ProjectData;
 use App\Model\Invoices\WorklogData;
 use App\Model\Planning\PlanningData;
@@ -75,6 +76,8 @@ public function getWorklogDataForProject(string $projectId): array;
     /** @return array<IssueData> */
     public function getIssuesDataForProject(string $projectId): array;
 
+    public function getIssuesDataForProjectPaged(string $projectId, int $startAt = 0, $maxResults = 50): PagedResult;
+
     /**
      * @return array<AccountData>
      */

src/Service/BillingService.php

@@ -28,6 +28,9 @@
 
 class BillingService
 {
+    private const BATCH_SIZE = 200;
+    private const MAX_RESULTS = 50;
+
     public function __construct(
         private readonly ApiServiceInterface $apiService,
         private readonly ProjectRepository $projectRepository,
@@ -84,48 +87,68 @@ public function syncIssuesForProject(int $projectId, callable $progressCallback
             throw new \Exception('ProjectTrackerId not set');
         }
 
-        $issueData = $this->apiService->getIssuesDataForProject($projectTrackerId);
         $issuesProcessed = 0;
 
-        foreach ($issueData as $issueDatum) {
-            $issue = $this->issueRepository->findOneBy(['projectTrackerId' => $issueDatum->projectTrackerId]);
+        $startAt = 0;
 
-            if (!$issue) {
-                $issue = new Issue();
+        do {
+            $project = $this->projectRepository->find($projectId);
 
-                $this->entityManager->persist($issue);
+            if (!$project) {
+                throw new \Exception('Project not found');
             }
 
-            $issue->setName($issueDatum->name);
-            $issue->setAccountId($issueDatum->accountId);
-            $issue->setAccountKey($issueDatum->accountKey);
-            $issue->setEpicKey($issueDatum->epicKey);
-            $issue->setEpicName($issueDatum->epicName);
-            $issue->setProject($project);
-            $issue->setProjectTrackerId($issueDatum->projectTrackerId);
-            $issue->setProjectTrackerKey($issueDatum->projectTrackerKey);
-            $issue->setResolutionDate($issueDatum->resolutionDate);
-            $issue->setStatus($issueDatum->status);
-
-            if (null == $issue->getSource()) {
-                $issue->setSource($this->apiService->getProjectTrackerIdentifier());
-            }
+            $pagedIssueData = $this->apiService->getIssuesDataForProjectPaged($projectTrackerId, $startAt, self::MAX_RESULTS);
+            $total = $pagedIssueData->total;
 
-            foreach ($issueDatum->versions as $versionData) {
-                $version = $this->versionRepository->findOneBy(['projectTrackerId' => $versionData->projectTrackerId]);
+            $issueData = $pagedIssueData->items;
 
-                if (null !== $version) {
-                    $issue->addVersion($version);
+            foreach ($issueData as $issueDatum) {
+                $issue = $this->issueRepository->findOneBy(['projectTrackerId' => $issueDatum->projectTrackerId]);
+
+                if (!$issue) {
+                    $issue = new Issue();
+
+                    $this->entityManager->persist($issue);
                 }
-            }
 
-            if (null !== $progressCallback) {
-                $progressCallback($issuesProcessed, count($issueData));
-                ++$issuesProcessed;
+                $issue->setName($issueDatum->name);
+                $issue->setAccountId($issueDatum->accountId);
+                $issue->setAccountKey($issueDatum->accountKey);
+                $issue->setEpicKey($issueDatum->epicKey);
+                $issue->setEpicName($issueDatum->epicName);
+                $issue->setProject($project);
+                $issue->setProjectTrackerId($issueDatum->projectTrackerId);
+                $issue->setProjectTrackerKey($issueDatum->projectTrackerKey);
+                $issue->setResolutionDate($issueDatum->resolutionDate);
+                $issue->setStatus($issueDatum->status);
+
+                if (null == $issue->getSource()) {
+                    $issue->setSource($this->apiService->getProjectTrackerIdentifier());
+                }
+
+                foreach ($issueDatum->versions as $versionData) {
+                    $version = $this->versionRepository->findOneBy(['projectTrackerId' => $versionData->projectTrackerId]);
+
+                    if (null !== $version) {
+                        $issue->addVersion($version);
+                    }
+                }
+
+                if (null !== $progressCallback) {
+                    $progressCallback($issuesProcessed, $total);
+                    ++$issuesProcessed;
+                }
             }
-        }
+
+            $startAt += self::MAX_RESULTS;
+
+            $this->entityManager->flush();
+            $this->entityManager->clear();
+        } while ($startAt < $total);
 
         $this->entityManager->flush();
+        $this->entityManager->clear();
     }
 
     /**
@@ -149,6 +172,12 @@ public function syncWorklogsForProject(int $projectId, callable $progressCallbac
         $worklogsAdded = 0;
 
         foreach ($worklogData as $worklogDatum) {
+            $project = $this->projectRepository->find($projectId);
+
+            if (!$project) {
+                throw new \Exception('Project not found');
+            }
+
             $worklog = $this->worklogRepository->findOneBy(['worklogId' => $worklogDatum->projectTrackerId]);
 
             if (!$worklog) {
@@ -185,9 +214,16 @@ public function syncWorklogsForProject(int $projectId, callable $progressCallbac
 
                 ++$worklogsAdded;
             }
+
+            // Flush and clear for each batch.
+            if (0 === $worklogsAdded % self::BATCH_SIZE) {
+                $this->entityManager->flush();
+                $this->entityManager->clear();
+            }
         }
 
         $this->entityManager->flush();
+        $this->entityManager->clear();
     }
 
     public function updateInvoiceEntryTotalPrice(InvoiceEntry $invoiceEntry): void
@@ -247,13 +283,17 @@ public function syncAccounts(callable $progressCallback): void
             $account->setStatus($accountDatum->status);
             $account->setCategory($accountDatum->category);
 
-            $this->entityManager->flush();
-            $this->entityManager->clear();
+            // Flush and clear for each batch.
+            if (0 === intval($index) % self::BATCH_SIZE) {
+                $this->entityManager->flush();
+                $this->entityManager->clear();
+            }
 
             $progressCallback($index, count($allAccountData));
         }
 
         $this->entityManager->flush();
+        $this->entityManager->clear();
     }
 
     public function syncProjects(callable $progressCallback): void
@@ -313,13 +353,17 @@ public function syncProjects(callable $progressCallback): void
                 }
             }
 
-            $this->entityManager->flush();
-            $this->entityManager->clear();
+            // Flush and clear for each batch.
+            if (0 === intval($index) % self::BATCH_SIZE) {
+                $this->entityManager->flush();
+                $this->entityManager->clear();
+            }
 
             $progressCallback($index, count($allProjectData));
         }
 
         $this->entityManager->flush();
+        $this->entityManager->clear();
     }
 
     /**

src/Service/JiraApiService.php

@@ -7,6 +7,7 @@
 use App\Model\Invoices\AccountData;
 use App\Model\Invoices\ClientData;
 use App\Model\Invoices\IssueData;
+use App\Model\Invoices\PagedResult;
 use App\Model\Invoices\ProjectData;
 use App\Model\Invoices\VersionData;
 use App\Model\Invoices\WorklogData;
@@ -1238,6 +1239,102 @@ private function getProjectIssues($projectId): array
         return $issues;
     }
 
+    /**
+     * @throws ApiServiceException
+     */
+    private function getProjectIssuesPaged($projectId, $startAt, $maxResults = 50): array
+    {
+        // Get customFields from Jira.
+        $customFieldEpicLink = $this->getCustomFieldId('Epic Link');
+        $customFieldAccount = $this->getCustomFieldId('Account');
+
+        // Get all issues for version.
+        $fields = implode(
+            ',',
+            [
+                'timetracking',
+                'worklog',
+                'timespent',
+                'timeoriginalestimate',
+                'summary',
+                'assignee',
+                'status',
+                'resolutiondate',
+                'fixVersions',
+                $customFieldEpicLink,
+                $customFieldAccount,
+            ]
+        );
+
+        $results = $this->get(
+            self::API_PATH_SEARCH,
+            [
+                'jql' => "project = $projectId",
+                'maxResults' => $maxResults,
+                // 'fields' => $fields,
+                'startAt' => $startAt,
+            ]
+        );
+
+        return [
+            'issues' => $results->issues,
+            'total' => $results->total,
+            'startAt' => $startAt,
+            'maxResults' => $maxResults,
+        ];
+    }
+
+    public function getIssuesDataForProjectPaged(string $projectId, $startAt = 0, $maxResults = 50): PagedResult
+    {
+        // Get customFields from Jira.
+        $customFieldEpicLinkId = $this->getCustomFieldId('Epic Link');
+        $customFieldAccount = $this->getCustomFieldId('Account');
+
+        $result = [];
+
+        $pagedResult = $this->getProjectIssuesPaged($projectId, $startAt, $maxResults);
+
+        $issues = $pagedResult['issues'];
+
+        $epicsRetrieved = [];
+
+        foreach ($issues as $issue) {
+            $fields = $issue->fields;
+
+            $issueData = new IssueData();
+            $issueData->name = $fields->summary;
+            $issueData->status = $fields->status->name;
+            $issueData->projectTrackerId = $issue->id;
+            $issueData->projectTrackerKey = $issue->key;
+            $issueData->resolutionDate = isset($fields->resolutiondate) ? new \DateTime($fields->resolutiondate) : null;
+
+            $issueData->accountId = $fields->{$customFieldAccount}->id ?? null;
+            $issueData->accountKey = $fields->{$customFieldAccount}->key ?? null;
+
+            if (isset($fields->{$customFieldEpicLinkId})) {
+                $epicKey = $fields->{$customFieldEpicLinkId};
+
+                if (isset($epicsRetrieved[$epicKey])) {
+                    $epicData = $epicsRetrieved[$epicKey];
+                } else {
+                    $epicData = $this->getIssue($epicKey);
+                    $epicsRetrieved[$epicKey] = $epicData;
+                }
+
+                $issueData->epicKey = $epicKey;
+                $issueData->epicName = $epicData->fields->summary ?? null;
+            }
+
+            foreach ($fields->fixVersions ?? [] as $fixVersion) {
+                $issueData->versions->add(new VersionData($fixVersion->id, $fixVersion->name));
+            }
+
+            $result[] = $issueData;
+        }
+
+        return new PagedResult($result, $startAt, $maxResults, $pagedResult['total']);
+    }
+
     /**
      * @throws ApiServiceException
      * @throws \Exception