diff --git a/CHANGELOG.md b/CHANGELOG.md index 16d6ad77..2248fd31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 * Added choices.js to dropdowns with many options. * Added epic filter to worklog selection page. * Removed time from period selections on worklog selection page. +* Optimized sync memory usage. ## [1.1.0] - 2023-12-14 diff --git a/src/Model/Invoices/PagedResult.php b/src/Model/Invoices/PagedResult.php new file mode 100644 index 00000000..c914823a --- /dev/null +++ b/src/Model/Invoices/PagedResult.php @@ -0,0 +1,14 @@ + */ public function getIssuesDataForProject(string $projectId): array; + public function getIssuesDataForProjectPaged(string $projectId, int $startAt = 0, $maxResults = 50): PagedResult; + /** * @return array */ diff --git a/src/Service/BillingService.php b/src/Service/BillingService.php index eaea6dfd..fbbf51db 100644 --- a/src/Service/BillingService.php +++ b/src/Service/BillingService.php @@ -28,6 +28,9 @@ class BillingService { + private const BATCH_SIZE = 200; + private const MAX_RESULTS = 50; + public function __construct( private readonly ApiServiceInterface $apiService, private readonly ProjectRepository $projectRepository, @@ -84,48 +87,68 @@ public function syncIssuesForProject(int $projectId, callable $progressCallback throw new \Exception('ProjectTrackerId not set'); } - $issueData = $this->apiService->getIssuesDataForProject($projectTrackerId); $issuesProcessed = 0; - foreach ($issueData as $issueDatum) { - $issue = $this->issueRepository->findOneBy(['projectTrackerId' => $issueDatum->projectTrackerId]); + $startAt = 0; - if (!$issue) { - $issue = new Issue(); + do { + $project = $this->projectRepository->find($projectId); - $this->entityManager->persist($issue); + if (!$project) { + throw new \Exception('Project not found'); } - $issue->setName($issueDatum->name); - $issue->setAccountId($issueDatum->accountId); - $issue->setAccountKey($issueDatum->accountKey); - $issue->setEpicKey($issueDatum->epicKey); - $issue->setEpicName($issueDatum->epicName); - $issue->setProject($project); - $issue->setProjectTrackerId($issueDatum->projectTrackerId); - $issue->setProjectTrackerKey($issueDatum->projectTrackerKey); - $issue->setResolutionDate($issueDatum->resolutionDate); - $issue->setStatus($issueDatum->status); - - if (null == $issue->getSource()) { - $issue->setSource($this->apiService->getProjectTrackerIdentifier()); - } + $pagedIssueData = $this->apiService->getIssuesDataForProjectPaged($projectTrackerId, $startAt, self::MAX_RESULTS); + $total = $pagedIssueData->total; - foreach ($issueDatum->versions as $versionData) { - $version = $this->versionRepository->findOneBy(['projectTrackerId' => $versionData->projectTrackerId]); + $issueData = $pagedIssueData->items; - if (null !== $version) { - $issue->addVersion($version); + foreach ($issueData as $issueDatum) { + $issue = $this->issueRepository->findOneBy(['projectTrackerId' => $issueDatum->projectTrackerId]); + + if (!$issue) { + $issue = new Issue(); + + $this->entityManager->persist($issue); } - } - if (null !== $progressCallback) { - $progressCallback($issuesProcessed, count($issueData)); - ++$issuesProcessed; + $issue->setName($issueDatum->name); + $issue->setAccountId($issueDatum->accountId); + $issue->setAccountKey($issueDatum->accountKey); + $issue->setEpicKey($issueDatum->epicKey); + $issue->setEpicName($issueDatum->epicName); + $issue->setProject($project); + $issue->setProjectTrackerId($issueDatum->projectTrackerId); + $issue->setProjectTrackerKey($issueDatum->projectTrackerKey); + $issue->setResolutionDate($issueDatum->resolutionDate); + $issue->setStatus($issueDatum->status); + + if (null == $issue->getSource()) { + $issue->setSource($this->apiService->getProjectTrackerIdentifier()); + } + + foreach ($issueDatum->versions as $versionData) { + $version = $this->versionRepository->findOneBy(['projectTrackerId' => $versionData->projectTrackerId]); + + if (null !== $version) { + $issue->addVersion($version); + } + } + + if (null !== $progressCallback) { + $progressCallback($issuesProcessed, $total); + ++$issuesProcessed; + } } - } + + $startAt += self::MAX_RESULTS; + + $this->entityManager->flush(); + $this->entityManager->clear(); + } while ($startAt < $total); $this->entityManager->flush(); + $this->entityManager->clear(); } /** @@ -149,6 +172,12 @@ public function syncWorklogsForProject(int $projectId, callable $progressCallbac $worklogsAdded = 0; foreach ($worklogData as $worklogDatum) { + $project = $this->projectRepository->find($projectId); + + if (!$project) { + throw new \Exception('Project not found'); + } + $worklog = $this->worklogRepository->findOneBy(['worklogId' => $worklogDatum->projectTrackerId]); if (!$worklog) { @@ -185,9 +214,16 @@ public function syncWorklogsForProject(int $projectId, callable $progressCallbac ++$worklogsAdded; } + + // Flush and clear for each batch. + if (0 === $worklogsAdded % self::BATCH_SIZE) { + $this->entityManager->flush(); + $this->entityManager->clear(); + } } $this->entityManager->flush(); + $this->entityManager->clear(); } public function updateInvoiceEntryTotalPrice(InvoiceEntry $invoiceEntry): void @@ -247,13 +283,17 @@ public function syncAccounts(callable $progressCallback): void $account->setStatus($accountDatum->status); $account->setCategory($accountDatum->category); - $this->entityManager->flush(); - $this->entityManager->clear(); + // Flush and clear for each batch. + if (0 === intval($index) % self::BATCH_SIZE) { + $this->entityManager->flush(); + $this->entityManager->clear(); + } $progressCallback($index, count($allAccountData)); } $this->entityManager->flush(); + $this->entityManager->clear(); } public function syncProjects(callable $progressCallback): void @@ -313,13 +353,17 @@ public function syncProjects(callable $progressCallback): void } } - $this->entityManager->flush(); - $this->entityManager->clear(); + // Flush and clear for each batch. + if (0 === intval($index) % self::BATCH_SIZE) { + $this->entityManager->flush(); + $this->entityManager->clear(); + } $progressCallback($index, count($allProjectData)); } $this->entityManager->flush(); + $this->entityManager->clear(); } /** diff --git a/src/Service/JiraApiService.php b/src/Service/JiraApiService.php index a562f549..9a491bcc 100644 --- a/src/Service/JiraApiService.php +++ b/src/Service/JiraApiService.php @@ -7,6 +7,7 @@ use App\Model\Invoices\AccountData; use App\Model\Invoices\ClientData; use App\Model\Invoices\IssueData; +use App\Model\Invoices\PagedResult; use App\Model\Invoices\ProjectData; use App\Model\Invoices\VersionData; use App\Model\Invoices\WorklogData; @@ -1238,6 +1239,102 @@ private function getProjectIssues($projectId): array return $issues; } + /** + * @throws ApiServiceException + */ + private function getProjectIssuesPaged($projectId, $startAt, $maxResults = 50): array + { + // Get customFields from Jira. + $customFieldEpicLink = $this->getCustomFieldId('Epic Link'); + $customFieldAccount = $this->getCustomFieldId('Account'); + + // Get all issues for version. + $fields = implode( + ',', + [ + 'timetracking', + 'worklog', + 'timespent', + 'timeoriginalestimate', + 'summary', + 'assignee', + 'status', + 'resolutiondate', + 'fixVersions', + $customFieldEpicLink, + $customFieldAccount, + ] + ); + + $results = $this->get( + self::API_PATH_SEARCH, + [ + 'jql' => "project = $projectId", + 'maxResults' => $maxResults, + // 'fields' => $fields, + 'startAt' => $startAt, + ] + ); + + return [ + 'issues' => $results->issues, + 'total' => $results->total, + 'startAt' => $startAt, + 'maxResults' => $maxResults, + ]; + } + + public function getIssuesDataForProjectPaged(string $projectId, $startAt = 0, $maxResults = 50): PagedResult + { + // Get customFields from Jira. + $customFieldEpicLinkId = $this->getCustomFieldId('Epic Link'); + $customFieldAccount = $this->getCustomFieldId('Account'); + + $result = []; + + $pagedResult = $this->getProjectIssuesPaged($projectId, $startAt, $maxResults); + + $issues = $pagedResult['issues']; + + $epicsRetrieved = []; + + foreach ($issues as $issue) { + $fields = $issue->fields; + + $issueData = new IssueData(); + $issueData->name = $fields->summary; + $issueData->status = $fields->status->name; + $issueData->projectTrackerId = $issue->id; + $issueData->projectTrackerKey = $issue->key; + $issueData->resolutionDate = isset($fields->resolutiondate) ? new \DateTime($fields->resolutiondate) : null; + + $issueData->accountId = $fields->{$customFieldAccount}->id ?? null; + $issueData->accountKey = $fields->{$customFieldAccount}->key ?? null; + + if (isset($fields->{$customFieldEpicLinkId})) { + $epicKey = $fields->{$customFieldEpicLinkId}; + + if (isset($epicsRetrieved[$epicKey])) { + $epicData = $epicsRetrieved[$epicKey]; + } else { + $epicData = $this->getIssue($epicKey); + $epicsRetrieved[$epicKey] = $epicData; + } + + $issueData->epicKey = $epicKey; + $issueData->epicName = $epicData->fields->summary ?? null; + } + + foreach ($fields->fixVersions ?? [] as $fixVersion) { + $issueData->versions->add(new VersionData($fixVersion->id, $fixVersion->name)); + } + + $result[] = $issueData; + } + + return new PagedResult($result, $startAt, $maxResults, $pagedResult['total']); + } + /** * @throws ApiServiceException * @throws \Exception