Skip to content

Commit

Permalink
some fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@aynsix
aynsix committed Nov 21, 2024
1 parent 3bf097f commit fc981fd
Show file tree
Hide file tree
Showing 5 changed files with 49 additions and 131 deletions.
105 changes: 47 additions & 58 deletions databox/api/src/Integration/Phrasea/Expose/ExposeClient.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,27 +130,6 @@ public function postAsset(IntegrationConfig $config, IntegrationToken $integrati

$fetchedFilePath = $this->fileFetcher->getFile($source);
try {
$data = array_merge([
'publication_id' => $publicationId,
'asset_id' => $asset->getId(),
'title' => $resolvedTitle,
'description' => $description,
'translations' => $translations,
'upload' => [
'type' => $source->getType(),
'size' => $source->getSize(),
'name' => $source->getOriginalName(),
],
], $extraData);

$pubAsset = $this->create($config, $integrationToken)
->request('POST', '/assets', [
'json' => $data,
])
->toArray()
;
$exposeAssetId = $pubAsset['id'];

$uploadsData = [
'filename' => $source->getOriginalName(),
'type' => $source->getType(),
Expand All @@ -165,38 +144,28 @@ public function postAsset(IntegrationConfig $config, IntegrationToken $integrati
;

$mUploadId = $resUploads['id'];

$parts['Parts'] = [];
$isPartsComplete = false;


// Upload the file in parts.
try {
$file = fopen($fetchedFilePath, 'r');
$partNumber = 1;

// part size is up to 5Mo https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
$partSize = 10 * 1024 * 1024; // 10Mo

$retryCount = 3;

while (!feof($file)) {
$resUploadPart = $this->create($config, $integrationToken)
->request('POST', '/uploads/'. $mUploadId .'/part', [
'json' => ['part' => $partNumber],
])
->toArray()
;

try {
$headerPutPart = $this->uploadClient->request('PUT', $resUploadPart['url'], [
'body' => fread($file, 10 * 1024 * 1024), // 10Mo
])->getHeaders()
;

} catch (\Exception $e) {
sleep (1); // retry after 1 second
try {
$headerPutPart = $this->uploadClient->request('PUT', $resUploadPart['url'], [
'body' => fread($file, 10 * 1024 * 1024), // 10Mo
])->getHeaders()
;
} catch (\Exception $e) {
throw $e;
}
}

$headerPutPart = $this->putPart($resUploadPart['url'], $file, $partSize, $retryCount);

$parts['Parts'][$partNumber] = [
'PartNumber' => $partNumber,
Expand All @@ -206,31 +175,34 @@ public function postAsset(IntegrationConfig $config, IntegrationToken $integrati
$partNumber++;
}

$isPartsComplete = true;
fclose($file);
} catch (\Exception $e) {
} catch (\Throwable $e) {
$this->create($config, $integrationToken)
->request('DELETE', '/uploads/'. $mUploadId);

throw $e;
}

if ($isPartsComplete) {
$result = $this->create($config, $integrationToken)
->request('POST', '/uploads/'. $mUploadId.'/complete', [
'json' => [
'parts' => $parts['Parts']
],
$data = array_merge([
'publication_id' => $publicationId,
'asset_id' => $asset->getId(),
'title' => $resolvedTitle,
'description' => $description,
'translations' => $translations,
'multipart' => [
'uploadId' => $mUploadId,
'parts' => $parts['Parts'],
],
], $extraData);

$pubAsset = $this->create($config, $integrationToken)
->request('POST', '/assets', [
'json' => $data,
])
->toArray()
;
;

$this->create($config, $integrationToken)
->request('PUT', '/assets/'. $exposeAssetId, [
'json' => [
'path' => $result['path']
],
])
;
}
$exposeAssetId = $pubAsset['id'];

foreach ([
'preview',
Expand Down Expand Up @@ -282,4 +254,21 @@ public function deleteAsset(IntegrationConfig $config, IntegrationToken $integra
->request('DELETE', '/assets/'.$assetId)
;
}

private function putPart(string $url, mixed $handleFile, int $partSize, int $retryCount)
{
if ($retryCount > 0) {
$retryCount--;
try {
return $this->uploadClient->request('PUT', $url, [
'body' => fread($handleFile, 10 * 1024 * 1024), // 10Mo
])->getHeaders();
} catch (\Throwable $e) {
if ($retryCount == 0) {
throw $e; // retry unsuccess
}
$this->putPart($url, $handleFile, $partSize, $retryCount);
}
}
}
}
4 changes: 2 additions & 2 deletions lib/php/storage-bundle/Controller/MultipartUploadCancelAction.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ public function __invoke(MultipartUpload $data, Request $request)
{
try {
$this->uploadManager->cancelMultipartUpload($data->getPath(), $data->getUploadId());
} catch (\Exception $e) {

} catch (\Throwable $e) {
// S3 storage will clean up its uncomplete uploads automatically
}

$this->em->remove($data);
Expand Down
39 changes: 0 additions & 39 deletions lib/php/storage-bundle/Controller/MultipartUploadCompleteAction.php
View file

This file was deleted.

28 changes: 0 additions & 28 deletions lib/php/storage-bundle/Entity/MultipartUpload.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,34 +69,6 @@
]],
],
]),
new Post(
uriTemplate: '/uploads/{id}/complete',
controller: MultipartUploadCompleteAction::class,
openapiContext: [
'summary' => 'Complete a multi part upload.',
'requestBody' => [
'content' => [
'application/json' => [
'schema' => [
'type' => 'object',
'properties' => [
'parts' => [
'type' => 'array',
'items' => [
'type' => 'object',
'properties' => [
'ETag' => ['type' => 'string'],
'PartNumber' => ['type' => 'integer'],
],
],
]
]
]
]
]
]
]
),
new Delete(
controller: MultipartUploadCancelAction::class,
openapiContext: [
Expand Down
4 changes: 0 additions & 4 deletions lib/php/storage-bundle/Resources/config/services.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,6 @@ services:
tags:
- { name: controller.service_arguments }

Alchemy\StorageBundle\Controller\MultipartUploadCompleteAction:
tags:
- { name: controller.service_arguments }

Alchemy\StorageBundle\Doctrine\MultipartUploadListener: ~

Alchemy\StorageBundle\Storage\PathGenerator: ~
Expand Down

0 comments on commit fc981fd

Please sign in to comment.