From 7027117c7c7449b5d33a739a9db65e7a63f2cdbf Mon Sep 17 00:00:00 2001 From: Troels Ugilt Jensen <6103205+tuj@users.noreply.github.com> Date: Fri, 15 Dec 2023 13:12:28 +0100 Subject: [PATCH 1/8] JE-400: Optimized sync memory usage --- CHANGELOG.md | 1 + src/Service/ApiServiceInterface.php | 2 + src/Service/BillingService.php | 110 +++++++++++++++++++--------- src/Service/JiraApiService.php | 101 +++++++++++++++++++++++++ 4 files changed, 181 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16d6ad77..2248fd31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Added choices.js to dropdowns with many options. * Added epic filter to worklog selection page. * Removed time from period selections on worklog selection page. +* Optimized sync memory usage. ## [1.1.0] - 2023-12-14 diff --git a/src/Service/ApiServiceInterface.php b/src/Service/ApiServiceInterface.php index 3b4fd6a0..d5ddee64 100644 --- a/src/Service/ApiServiceInterface.php +++ b/src/Service/ApiServiceInterface.php @@ -75,6 +75,8 @@ public function getWorklogDataForProject(string $projectId): array; /** @return array */ public function getIssuesDataForProject(string $projectId): array; + public function getIssuesDataForProjectPaged(string $projectId, int $startAt = 0, $maxResults = 50): array; + /** * @return array */ diff --git a/src/Service/BillingService.php b/src/Service/BillingService.php index eaea6dfd..850da8e6 100644 --- a/src/Service/BillingService.php +++ b/src/Service/BillingService.php @@ -28,6 +28,9 @@ class BillingService { + private const BATCH_SIZE = 200; + private const MAX_RESULTS = 50; + public function __construct( private readonly ApiServiceInterface $apiService, private readonly ProjectRepository $projectRepository, @@ -84,48 +87,68 @@ public function syncIssuesForProject(int $projectId, callable $progressCallback throw new \Exception('ProjectTrackerId not set'); } - $issueData = $this->apiService->getIssuesDataForProject($projectTrackerId); $issuesProcessed = 0; - foreach ($issueData as $issueDatum) { - $issue = $this->issueRepository->findOneBy(['projectTrackerId' => $issueDatum->projectTrackerId]); + $startAt = 0; - if (!$issue) { - $issue = new Issue(); + do { + $project = $this->projectRepository->find($projectId); - $this->entityManager->persist($issue); + if (!$project) { + throw new \Exception('Project not found'); } - $issue->setName($issueDatum->name); - $issue->setAccountId($issueDatum->accountId); - $issue->setAccountKey($issueDatum->accountKey); - $issue->setEpicKey($issueDatum->epicKey); - $issue->setEpicName($issueDatum->epicName); - $issue->setProject($project); - $issue->setProjectTrackerId($issueDatum->projectTrackerId); - $issue->setProjectTrackerKey($issueDatum->projectTrackerKey); - $issue->setResolutionDate($issueDatum->resolutionDate); - $issue->setStatus($issueDatum->status); - - if (null == $issue->getSource()) { - $issue->setSource($this->apiService->getProjectTrackerIdentifier()); - } + $pagedIssueData = $this->apiService->getIssuesDataForProjectPaged($projectTrackerId, $startAt, self::MAX_RESULTS); + $total = $pagedIssueData['total']; - foreach ($issueDatum->versions as $versionData) { - $version = $this->versionRepository->findOneBy(['projectTrackerId' => $versionData->projectTrackerId]); + $issueData = $pagedIssueData['issues']; - if (null !== $version) { - $issue->addVersion($version); + foreach ($issueData as $issueDatum) { + $issue = $this->issueRepository->findOneBy(['projectTrackerId' => $issueDatum->projectTrackerId]); + + if (!$issue) { + $issue = new Issue(); + + $this->entityManager->persist($issue); } - } - if (null !== $progressCallback) { - $progressCallback($issuesProcessed, count($issueData)); - ++$issuesProcessed; + $issue->setName($issueDatum->name); + $issue->setAccountId($issueDatum->accountId); + $issue->setAccountKey($issueDatum->accountKey); + $issue->setEpicKey($issueDatum->epicKey); + $issue->setEpicName($issueDatum->epicName); + $issue->setProject($project); + $issue->setProjectTrackerId($issueDatum->projectTrackerId); + $issue->setProjectTrackerKey($issueDatum->projectTrackerKey); + $issue->setResolutionDate($issueDatum->resolutionDate); + $issue->setStatus($issueDatum->status); + + if (null == $issue->getSource()) { + $issue->setSource($this->apiService->getProjectTrackerIdentifier()); + } + + foreach ($issueDatum->versions as $versionData) { + $version = $this->versionRepository->findOneBy(['projectTrackerId' => $versionData->projectTrackerId]); + + if (null !== $version) { + $issue->addVersion($version); + } + } + + if (null !== $progressCallback) { + $progressCallback($issuesProcessed, $total); + ++$issuesProcessed; + } } - } + + $startAt += self::MAX_RESULTS; + + $this->entityManager->flush(); + $this->entityManager->clear(); + } while ($startAt < $total); $this->entityManager->flush(); + $this->entityManager->clear(); } /** @@ -149,6 +172,12 @@ public function syncWorklogsForProject(int $projectId, callable $progressCallbac $worklogsAdded = 0; foreach ($worklogData as $worklogDatum) { + $project = $this->projectRepository->find($projectId); + + if (!$project) { + throw new \Exception('Project not found'); + } + $worklog = $this->worklogRepository->findOneBy(['worklogId' => $worklogDatum->projectTrackerId]); if (!$worklog) { @@ -185,9 +214,16 @@ public function syncWorklogsForProject(int $projectId, callable $progressCallbac ++$worklogsAdded; } + + // Flush and clear for each batch. + if (0 == $worklogsAdded % self::BATCH_SIZE) { + $this->entityManager->flush(); + $this->entityManager->clear(); + } } $this->entityManager->flush(); + $this->entityManager->clear(); } public function updateInvoiceEntryTotalPrice(InvoiceEntry $invoiceEntry): void @@ -247,13 +283,17 @@ public function syncAccounts(callable $progressCallback): void $account->setStatus($accountDatum->status); $account->setCategory($accountDatum->category); - $this->entityManager->flush(); - $this->entityManager->clear(); + // Flush and clear for each batch. + if (0 == intval($index) % self::BATCH_SIZE) { + $this->entityManager->flush(); + $this->entityManager->clear(); + } $progressCallback($index, count($allAccountData)); } $this->entityManager->flush(); + $this->entityManager->clear(); } public function syncProjects(callable $progressCallback): void @@ -313,13 +353,17 @@ public function syncProjects(callable $progressCallback): void } } - $this->entityManager->flush(); - $this->entityManager->clear(); + // Flush and clear for each batch. + if (0 == intval($index) % self::BATCH_SIZE) { + $this->entityManager->flush(); + $this->entityManager->clear(); + } $progressCallback($index, count($allProjectData)); } $this->entityManager->flush(); + $this->entityManager->clear(); } /** diff --git a/src/Service/JiraApiService.php b/src/Service/JiraApiService.php index a562f549..30ba0ddd 100644 --- a/src/Service/JiraApiService.php +++ b/src/Service/JiraApiService.php @@ -1238,6 +1238,107 @@ private function getProjectIssues($projectId): array return $issues; } + /** + * @throws ApiServiceException + */ + private function getProjectIssuesPaged($projectId, $startAt, $maxResults = 50): array + { + // Get customFields from Jira. + $customFieldEpicLink = $this->getCustomFieldId('Epic Link'); + $customFieldAccount = $this->getCustomFieldId('Account'); + + // Get all issues for version. + $fields = implode( + ',', + [ + 'timetracking', + 'worklog', + 'timespent', + 'timeoriginalestimate', + 'summary', + 'assignee', + 'status', + 'resolutiondate', + 'fixVersions', + $customFieldEpicLink, + $customFieldAccount, + ] + ); + + $results = $this->get( + self::API_PATH_SEARCH, + [ + 'jql' => "project = $projectId", + 'maxResults' => $maxResults, + // 'fields' => $fields, + 'startAt' => $startAt, + ] + ); + + return [ + 'issues' => $results->issues, + 'total' => $results->total, + 'startAt' => $startAt, + 'maxResults' => $maxResults, + ]; + } + + public function getIssuesDataForProjectPaged(string $projectId, $startAt = 0, $maxResults = 50): array + { + // Get customFields from Jira. + $customFieldEpicLinkId = $this->getCustomFieldId('Epic Link'); + $customFieldAccount = $this->getCustomFieldId('Account'); + + $result = []; + + $pagedResult = $this->getProjectIssuesPaged($projectId, $startAt, $maxResults); + + $issues = $pagedResult['issues']; + + $epicsRetrieved = []; + + foreach ($issues as $issue) { + $fields = $issue->fields; + + $issueData = new IssueData(); + $issueData->name = $fields->summary; + $issueData->status = $fields->status->name; + $issueData->projectTrackerId = $issue->id; + $issueData->projectTrackerKey = $issue->key; + $issueData->resolutionDate = isset($fields->resolutiondate) ? new \DateTime($fields->resolutiondate) : null; + + $issueData->accountId = $fields->{$customFieldAccount}->id ?? null; + $issueData->accountKey = $fields->{$customFieldAccount}->key ?? null; + + if (isset($fields->{$customFieldEpicLinkId})) { + $epicKey = $fields->{$customFieldEpicLinkId}; + + if (isset($epicsRetrieved[$epicKey])) { + $epicData = $epicsRetrieved[$epicKey]; + } else { + $epicData = $this->getIssue($epicKey); + $epicsRetrieved[$epicKey] = $epicData; + } + + $issueData->epicKey = $epicKey; + $issueData->epicName = $epicData->fields->summary ?? null; + } + + foreach ($fields->fixVersions ?? [] as $fixVersion) { + $issueData->versions->add(new VersionData($fixVersion->id, $fixVersion->name)); + } + + $result[] = $issueData; + } + + return [ + 'issues' => $result, + 'startAt' => $startAt, + 'maxResults' => $maxResults, + 'total' => $pagedResult['total'], + ]; + } + /** * @throws ApiServiceException * @throws \Exception From df11b00247133b46c65ab611eedf86207aa9f799 Mon Sep 17 00:00:00 2001 From: Troels Ugilt Jensen <6103205+tuj@users.noreply.github.com> Date: Fri, 15 Dec 2023 14:36:03 +0100 Subject: [PATCH 2/8] JE-400: Changed output to PagedResult instead of array --- src/Model/Invoices/PagedResult.php | 19 +++++++++++++++++++ src/Service/ApiServiceInterface.php | 3 ++- src/Service/BillingService.php | 4 ++-- src/Service/JiraApiService.php | 10 +++------- 4 files changed, 26 insertions(+), 10 deletions(-) create mode 100644 src/Model/Invoices/PagedResult.php diff --git a/src/Model/Invoices/PagedResult.php b/src/Model/Invoices/PagedResult.php new file mode 100644 index 00000000..aa7ffc29 --- /dev/null +++ b/src/Model/Invoices/PagedResult.php @@ -0,0 +1,19 @@ +items = $items; + $this->startAt = $startAt; + $this->maxResults = $maxResults; + $this->total = $total; + } +} diff --git a/src/Service/ApiServiceInterface.php b/src/Service/ApiServiceInterface.php index d5ddee64..5e5dcbe7 100644 --- a/src/Service/ApiServiceInterface.php +++ b/src/Service/ApiServiceInterface.php @@ -5,6 +5,7 @@ use App\Model\Invoices\AccountData; use App\Model\Invoices\ClientData; use App\Model\Invoices\IssueData; +use App\Model\Invoices\PagedResult; use App\Model\Invoices\ProjectData; use App\Model\Invoices\WorklogData; use App\Model\Planning\PlanningData; @@ -75,7 +76,7 @@ public function getWorklogDataForProject(string $projectId): array; /** @return array */ public function getIssuesDataForProject(string $projectId): array; - public function getIssuesDataForProjectPaged(string $projectId, int $startAt = 0, $maxResults = 50): array; + public function getIssuesDataForProjectPaged(string $projectId, int $startAt = 0, $maxResults = 50): PagedResult; /** * @return array diff --git a/src/Service/BillingService.php b/src/Service/BillingService.php index 850da8e6..9a105f58 100644 --- a/src/Service/BillingService.php +++ b/src/Service/BillingService.php @@ -99,9 +99,9 @@ public function syncIssuesForProject(int $projectId, callable $progressCallback } $pagedIssueData = $this->apiService->getIssuesDataForProjectPaged($projectTrackerId, $startAt, self::MAX_RESULTS); - $total = $pagedIssueData['total']; + $total = $pagedIssueData->total; - $issueData = $pagedIssueData['issues']; + $issueData = $pagedIssueData->items; foreach ($issueData as $issueDatum) { $issue = $this->issueRepository->findOneBy(['projectTrackerId' => $issueDatum->projectTrackerId]); diff --git a/src/Service/JiraApiService.php b/src/Service/JiraApiService.php index 30ba0ddd..9a491bcc 100644 --- a/src/Service/JiraApiService.php +++ b/src/Service/JiraApiService.php @@ -7,6 +7,7 @@ use App\Model\Invoices\AccountData; use App\Model\Invoices\ClientData; use App\Model\Invoices\IssueData; +use App\Model\Invoices\PagedResult; use App\Model\Invoices\ProjectData; use App\Model\Invoices\VersionData; use App\Model\Invoices\WorklogData; @@ -1283,7 +1284,7 @@ private function getProjectIssuesPaged($projectId, $startAt, $maxResults = 50): ]; } - public function getIssuesDataForProjectPaged(string $projectId, $startAt = 0, $maxResults = 50): array + public function getIssuesDataForProjectPaged(string $projectId, $startAt = 0, $maxResults = 50): PagedResult { // Get customFields from Jira. $customFieldEpicLinkId = $this->getCustomFieldId('Epic Link'); @@ -1331,12 +1332,7 @@ public function getIssuesDataForProjectPaged(string $projectId, $startAt = 0, $m $result[] = $issueData; } - return [ - 'issues' => $result, - 'startAt' => $startAt, - 'maxResults' => $maxResults, - 'total' => $pagedResult['total'], - ]; + return new PagedResult($result, $startAt, $maxResults, $pagedResult['total']); } /** From 6ac92889c4af52e612a2958c1e846534984cc79d Mon Sep 17 00:00:00 2001 From: Troels Ugilt Jensen <6103205+tuj@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:20:39 +0100 Subject: [PATCH 3/8] Update src/Service/BillingService.php Co-authored-by: Mikkel Ricky --- src/Service/BillingService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Service/BillingService.php b/src/Service/BillingService.php index 9a105f58..bc317b09 100644 --- a/src/Service/BillingService.php +++ b/src/Service/BillingService.php @@ -354,7 +354,7 @@ public function syncProjects(callable $progressCallback): void } // Flush and clear for each batch. - if (0 == intval($index) % self::BATCH_SIZE) { + if (0 === intval($index) % self::BATCH_SIZE) { $this->entityManager->flush(); $this->entityManager->clear(); } From 2d67f44ce8a7c078cb37180b86afeb2cf3942d3d Mon Sep 17 00:00:00 2001 From: Troels Ugilt Jensen <6103205+tuj@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:20:47 +0100 Subject: [PATCH 4/8] Update src/Service/BillingService.php Co-authored-by: Mikkel Ricky --- src/Service/BillingService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Service/BillingService.php b/src/Service/BillingService.php index bc317b09..dfeb556b 100644 --- a/src/Service/BillingService.php +++ b/src/Service/BillingService.php @@ -284,7 +284,7 @@ public function syncAccounts(callable $progressCallback): void $account->setCategory($accountDatum->category); // Flush and clear for each batch. - if (0 == intval($index) % self::BATCH_SIZE) { + if (0 === intval($index) % self::BATCH_SIZE) { $this->entityManager->flush(); $this->entityManager->clear(); } From dbc9c495972898c9bf847447df539bd236301aa2 Mon Sep 17 00:00:00 2001 From: Troels Ugilt Jensen <6103205+tuj@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:20:55 +0100 Subject: [PATCH 5/8] Update src/Service/BillingService.php Co-authored-by: Mikkel Ricky --- src/Service/BillingService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Service/BillingService.php b/src/Service/BillingService.php index dfeb556b..fbbf51db 100644 --- a/src/Service/BillingService.php +++ b/src/Service/BillingService.php @@ -216,7 +216,7 @@ public function syncWorklogsForProject(int $projectId, callable $progressCallbac } // Flush and clear for each batch. - if (0 == $worklogsAdded % self::BATCH_SIZE) { + if (0 === $worklogsAdded % self::BATCH_SIZE) { $this->entityManager->flush(); $this->entityManager->clear(); } From 640c1855616179653cdf1a4dc87532be63048201 Mon Sep 17 00:00:00 2001 From: Troels Ugilt Jensen <6103205+tuj@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:21:14 +0100 Subject: [PATCH 6/8] Update src/Model/Invoices/PagedResult.php Co-authored-by: Mikkel Ricky --- src/Model/Invoices/PagedResult.php | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/Model/Invoices/PagedResult.php b/src/Model/Invoices/PagedResult.php index aa7ffc29..5a09cfb2 100644 --- a/src/Model/Invoices/PagedResult.php +++ b/src/Model/Invoices/PagedResult.php @@ -4,16 +4,12 @@ class PagedResult { - public array $items = []; - public int $startAt = 0; - public int $maxResults = 0; - public int $total = 0; - - public function __construct(array $items, int $startAt, int $maxResults, int $total) + public function __construct( + public array $items, + public int $startAt, + public int $maxResults, + public int $total + ) { - $this->items = $items; - $this->startAt = $startAt; - $this->maxResults = $maxResults; - $this->total = $total; } } From 400e7849a79bba52c0295ae74d3661ed36a9f500 Mon Sep 17 00:00:00 2001 From: Troels Ugilt Jensen <6103205+tuj@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:28:44 +0100 Subject: [PATCH 7/8] JE-400: Added readonly to properties --- src/Model/Invoices/PagedResult.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Model/Invoices/PagedResult.php b/src/Model/Invoices/PagedResult.php index 5a09cfb2..c97d237c 100644 --- a/src/Model/Invoices/PagedResult.php +++ b/src/Model/Invoices/PagedResult.php @@ -5,10 +5,10 @@ class PagedResult { public function __construct( - public array $items, - public int $startAt, - public int $maxResults, - public int $total + public readonly array $items, + public readonly int $startAt, + public readonly int $maxResults, + public readonly int $total ) { } From ac07ea9739aed24acf41d17017084d15120420fe Mon Sep 17 00:00:00 2001 From: Troels Ugilt Jensen <6103205+tuj@users.noreply.github.com> Date: Fri, 15 Dec 2023 15:30:09 +0100 Subject: [PATCH 8/8] JE-400: Applied coding standards --- src/Model/Invoices/PagedResult.php | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Model/Invoices/PagedResult.php b/src/Model/Invoices/PagedResult.php index c97d237c..c914823a 100644 --- a/src/Model/Invoices/PagedResult.php +++ b/src/Model/Invoices/PagedResult.php @@ -5,11 +5,10 @@ class PagedResult { public function __construct( - public readonly array $items, - public readonly int $startAt, - public readonly int $maxResults, - public readonly int $total - ) - { + public readonly array $items, + public readonly int $startAt, + public readonly int $maxResults, + public readonly int $total + ) { } }