Skip to content

Commit

Permalink
Merge pull request #46 from itk-dev/feature/je-400-optimize-sync
Browse files Browse the repository at this point in the history 
JE-400: Optimized sync memory usage
  • Loading branch information
@tuj
tuj authored Dec 16, 2023
2 parents 96972eb + ac07ea9 commit 42403a9
Show file tree
Hide file tree
Showing 5 changed files with 192 additions and 33 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* Added choices.js to dropdowns with many options.
* Added epic filter to worklog selection page.
* Removed time from period selections on worklog selection page.
* Optimized sync memory usage.

## [1.1.0] - 2023-12-14

Expand Down
14 changes: 14 additions & 0 deletions src/Model/Invoices/PagedResult.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<?php

namespace App\Model\Invoices;

class PagedResult
{
public function __construct(
public readonly array $items,
public readonly int $startAt,
public readonly int $maxResults,
public readonly int $total
) {
}
}
3 changes: 3 additions & 0 deletions src/Service/ApiServiceInterface.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
use App\Model\Invoices\AccountData;
use App\Model\Invoices\ClientData;
use App\Model\Invoices\IssueData;
use App\Model\Invoices\PagedResult;
use App\Model\Invoices\ProjectData;
use App\Model\Invoices\WorklogData;
use App\Model\Planning\PlanningData;
Expand Down Expand Up @@ -75,6 +76,8 @@ public function getWorklogDataForProject(string $projectId): array;
/** @return array<IssueData> */
public function getIssuesDataForProject(string $projectId): array;

public function getIssuesDataForProjectPaged(string $projectId, int $startAt = 0, $maxResults = 50): PagedResult;

/**
* @return array<AccountData>
*/
Expand Down
110 changes: 77 additions & 33 deletions src/Service/BillingService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@

class BillingService
{
private const BATCH_SIZE = 200;
private const MAX_RESULTS = 50;

public function __construct(
private readonly ApiServiceInterface $apiService,
private readonly ProjectRepository $projectRepository,
Expand Down Expand Up @@ -84,48 +87,68 @@ public function syncIssuesForProject(int $projectId, callable $progressCallback
throw new \Exception('ProjectTrackerId not set');
}

$issueData = $this->apiService->getIssuesDataForProject($projectTrackerId);
$issuesProcessed = 0;

foreach ($issueData as $issueDatum) {
$issue = $this->issueRepository->findOneBy(['projectTrackerId' => $issueDatum->projectTrackerId]);
$startAt = 0;

if (!$issue) {
$issue = new Issue();
do {
$project = $this->projectRepository->find($projectId);

$this->entityManager->persist($issue);
if (!$project) {
throw new \Exception('Project not found');
}

$issue->setName($issueDatum->name);
$issue->setAccountId($issueDatum->accountId);
$issue->setAccountKey($issueDatum->accountKey);
$issue->setEpicKey($issueDatum->epicKey);
$issue->setEpicName($issueDatum->epicName);
$issue->setProject($project);
$issue->setProjectTrackerId($issueDatum->projectTrackerId);
$issue->setProjectTrackerKey($issueDatum->projectTrackerKey);
$issue->setResolutionDate($issueDatum->resolutionDate);
$issue->setStatus($issueDatum->status);

if (null == $issue->getSource()) {
$issue->setSource($this->apiService->getProjectTrackerIdentifier());
}
$pagedIssueData = $this->apiService->getIssuesDataForProjectPaged($projectTrackerId, $startAt, self::MAX_RESULTS);
$total = $pagedIssueData->total;

foreach ($issueDatum->versions as $versionData) {
$version = $this->versionRepository->findOneBy(['projectTrackerId' => $versionData->projectTrackerId]);
$issueData = $pagedIssueData->items;

if (null !== $version) {
$issue->addVersion($version);
foreach ($issueData as $issueDatum) {
$issue = $this->issueRepository->findOneBy(['projectTrackerId' => $issueDatum->projectTrackerId]);

if (!$issue) {
$issue = new Issue();

$this->entityManager->persist($issue);
}
}

if (null !== $progressCallback) {
$progressCallback($issuesProcessed, count($issueData));
++$issuesProcessed;
$issue->setName($issueDatum->name);
$issue->setAccountId($issueDatum->accountId);
$issue->setAccountKey($issueDatum->accountKey);
$issue->setEpicKey($issueDatum->epicKey);
$issue->setEpicName($issueDatum->epicName);
$issue->setProject($project);
$issue->setProjectTrackerId($issueDatum->projectTrackerId);
$issue->setProjectTrackerKey($issueDatum->projectTrackerKey);
$issue->setResolutionDate($issueDatum->resolutionDate);
$issue->setStatus($issueDatum->status);

if (null == $issue->getSource()) {
$issue->setSource($this->apiService->getProjectTrackerIdentifier());
}

foreach ($issueDatum->versions as $versionData) {
$version = $this->versionRepository->findOneBy(['projectTrackerId' => $versionData->projectTrackerId]);

if (null !== $version) {
$issue->addVersion($version);
}
}

if (null !== $progressCallback) {
$progressCallback($issuesProcessed, $total);
++$issuesProcessed;
}
}
}

$startAt += self::MAX_RESULTS;

$this->entityManager->flush();
$this->entityManager->clear();
} while ($startAt < $total);

$this->entityManager->flush();
$this->entityManager->clear();
}

/**
Expand All @@ -149,6 +172,12 @@ public function syncWorklogsForProject(int $projectId, callable $progressCallbac
$worklogsAdded = 0;

foreach ($worklogData as $worklogDatum) {
$project = $this->projectRepository->find($projectId);

if (!$project) {
throw new \Exception('Project not found');
}

$worklog = $this->worklogRepository->findOneBy(['worklogId' => $worklogDatum->projectTrackerId]);

if (!$worklog) {
Expand Down Expand Up @@ -185,9 +214,16 @@ public function syncWorklogsForProject(int $projectId, callable $progressCallbac

++$worklogsAdded;
}

// Flush and clear for each batch.
if (0 === $worklogsAdded % self::BATCH_SIZE) {
$this->entityManager->flush();
$this->entityManager->clear();
}
}

$this->entityManager->flush();
$this->entityManager->clear();
}

public function updateInvoiceEntryTotalPrice(InvoiceEntry $invoiceEntry): void
Expand Down Expand Up @@ -247,13 +283,17 @@ public function syncAccounts(callable $progressCallback): void
$account->setStatus($accountDatum->status);
$account->setCategory($accountDatum->category);

$this->entityManager->flush();
$this->entityManager->clear();
// Flush and clear for each batch.
if (0 === intval($index) % self::BATCH_SIZE) {
$this->entityManager->flush();
$this->entityManager->clear();
}

$progressCallback($index, count($allAccountData));
}

$this->entityManager->flush();
$this->entityManager->clear();
}

public function syncProjects(callable $progressCallback): void
Expand Down Expand Up @@ -313,13 +353,17 @@ public function syncProjects(callable $progressCallback): void
}
}

$this->entityManager->flush();
$this->entityManager->clear();
// Flush and clear for each batch.
if (0 === intval($index) % self::BATCH_SIZE) {
$this->entityManager->flush();
$this->entityManager->clear();
}

$progressCallback($index, count($allProjectData));
}

$this->entityManager->flush();
$this->entityManager->clear();
}

/**
Expand Down
97 changes: 97 additions & 0 deletions src/Service/JiraApiService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
use App\Model\Invoices\AccountData;
use App\Model\Invoices\ClientData;
use App\Model\Invoices\IssueData;
use App\Model\Invoices\PagedResult;
use App\Model\Invoices\ProjectData;
use App\Model\Invoices\VersionData;
use App\Model\Invoices\WorklogData;
Expand Down Expand Up @@ -1238,6 +1239,102 @@ private function getProjectIssues($projectId): array
return $issues;
}

/**
* @throws ApiServiceException
*/
private function getProjectIssuesPaged($projectId, $startAt, $maxResults = 50): array
{
// Get customFields from Jira.
$customFieldEpicLink = $this->getCustomFieldId('Epic Link');
$customFieldAccount = $this->getCustomFieldId('Account');

// Get all issues for version.
$fields = implode(
',',
[
'timetracking',
'worklog',
'timespent',
'timeoriginalestimate',
'summary',
'assignee',
'status',
'resolutiondate',
'fixVersions',
$customFieldEpicLink,
$customFieldAccount,
]
);

$results = $this->get(
self::API_PATH_SEARCH,
[
'jql' => "project = $projectId",
'maxResults' => $maxResults,
// 'fields' => $fields,
'startAt' => $startAt,
]
);

return [
'issues' => $results->issues,
'total' => $results->total,
'startAt' => $startAt,
'maxResults' => $maxResults,
];
}

public function getIssuesDataForProjectPaged(string $projectId, $startAt = 0, $maxResults = 50): PagedResult
{
// Get customFields from Jira.
$customFieldEpicLinkId = $this->getCustomFieldId('Epic Link');
$customFieldAccount = $this->getCustomFieldId('Account');

$result = [];

$pagedResult = $this->getProjectIssuesPaged($projectId, $startAt, $maxResults);

$issues = $pagedResult['issues'];

$epicsRetrieved = [];

foreach ($issues as $issue) {
$fields = $issue->fields;

$issueData = new IssueData();
$issueData->name = $fields->summary;
$issueData->status = $fields->status->name;
$issueData->projectTrackerId = $issue->id;
$issueData->projectTrackerKey = $issue->key;
$issueData->resolutionDate = isset($fields->resolutiondate) ? new \DateTime($fields->resolutiondate) : null;

$issueData->accountId = $fields->{$customFieldAccount}->id ?? null;
$issueData->accountKey = $fields->{$customFieldAccount}->key ?? null;

if (isset($fields->{$customFieldEpicLinkId})) {
$epicKey = $fields->{$customFieldEpicLinkId};

if (isset($epicsRetrieved[$epicKey])) {
$epicData = $epicsRetrieved[$epicKey];
} else {
$epicData = $this->getIssue($epicKey);
$epicsRetrieved[$epicKey] = $epicData;
}

$issueData->epicKey = $epicKey;
$issueData->epicName = $epicData->fields->summary ?? null;
}

foreach ($fields->fixVersions ?? [] as $fixVersion) {
$issueData->versions->add(new VersionData($fixVersion->id, $fixVersion->name));
}

$result[] = $issueData;
}

return new PagedResult($result, $startAt, $maxResults, $pagedResult['total']);
}

/**
* @throws ApiServiceException
* @throws \Exception
Expand Down

0 comments on commit 42403a9

Please sign in to comment.