diff --git a/databox/api/src/Integration/Phrasea/Expose/ExposeClient.php b/databox/api/src/Integration/Phrasea/Expose/ExposeClient.php index 3e5787b44..9196ce42f 100644 --- a/databox/api/src/Integration/Phrasea/Expose/ExposeClient.php +++ b/databox/api/src/Integration/Phrasea/Expose/ExposeClient.php @@ -2,16 +2,17 @@ namespace App\Integration\Phrasea\Expose; -use App\Asset\Attribute\AssetTitleResolver; -use App\Asset\Attribute\AttributesResolver; use App\Asset\FileFetcher; -use App\Attribute\AttributeInterface; use App\Entity\Core\Asset; use App\Entity\Core\Attribute; -use App\Entity\Integration\IntegrationToken; +use App\Storage\RenditionManager; +use App\Attribute\AttributeInterface; use App\Integration\IntegrationConfig; +use App\Asset\Attribute\AssetTitleResolver; +use App\Asset\Attribute\AttributesResolver; +use App\Entity\Integration\IntegrationToken; +use Alchemy\StorageBundle\Upload\UploadManager; use App\Integration\Phrasea\PhraseaClientFactory; -use App\Storage\RenditionManager; use Symfony\Contracts\HttpClient\HttpClientInterface; final readonly class ExposeClient @@ -23,6 +24,7 @@ public function __construct( private AssetTitleResolver $assetTitleResolver, private AttributesResolver $attributesResolver, private RenditionManager $renditionManager, + private UploadManager $uploadManager ) { } @@ -149,13 +151,86 @@ public function postAsset(IntegrationConfig $config, IntegrationToken $integrati ; $exposeAssetId = $pubAsset['id']; - $this->uploadClient->request('PUT', $pubAsset['uploadURL'], [ - 'headers' => [ - 'Content-Type' => $source->getType(), - 'Content-Length' => filesize($fetchedFilePath), - ], - 'body' => fopen($fetchedFilePath, 'r'), - ]); + $uploadsData = [ + 'filename' => $source->getOriginalName(), + 'type' => $source->getType(), + 'size' => (int)$source->getSize(), + ]; + + $resUploads = $this->create($config, $integrationToken) + ->request('POST', '/uploads', [ + 'json' => $uploadsData, + ]) + ->toArray() + ; + + $mUploadId = $resUploads['id']; + $parts['Parts'] = []; + $isPartsComplete = false; + + // Upload the file in parts. + try { + $file = fopen($fetchedFilePath, 'r'); + $partNumber = 1; + while (!feof($file)) { + $resUploadPart = $this->create($config, $integrationToken) + ->request('POST', '/uploads/'. $mUploadId .'/part', [ + 'json' => ['part' => $partNumber], + ]) + ->toArray() + ; + + try { + $headerPutPart = $this->uploadClient->request('PUT', $resUploadPart['url'], [ + 'body' => fread($file, 10 * 1024 * 1024), // 10Mo + ])->getHeaders() + ; + + } catch (\Exception $e) { + sleep (1); // retry after 1 second + try { + $headerPutPart = $this->uploadClient->request('PUT', $resUploadPart['url'], [ + 'body' => fread($file, 10 * 1024 * 1024), // 10Mo + ])->getHeaders() + ; + } catch (\Exception $e) { + throw $e; + } + } + + $parts['Parts'][$partNumber] = [ + 'PartNumber' => $partNumber, + 'ETag' => current($headerPutPart['etag']), + ]; + + $partNumber++; + } + + $isPartsComplete = true; + fclose($file); + } catch (\Exception $e) { + $this->create($config, $integrationToken) + ->request('DELETE', '/uploads/'. $mUploadId); + } + + if ($isPartsComplete) { + $result = $this->create($config, $integrationToken) + ->request('POST', '/uploads/'. $mUploadId.'/complete', [ + 'json' => [ + 'parts' => $parts['Parts'] + ], + ]) + ->toArray() + ; + + $this->create($config, $integrationToken) + ->request('PUT', '/assets/'. $exposeAssetId, [ + 'json' => [ + 'path' => $result['path'] + ], + ]) + ; + } foreach ([ 'preview', diff --git a/lib/php/storage-bundle/Controller/MultipartUploadCancelAction.php b/lib/php/storage-bundle/Controller/MultipartUploadCancelAction.php new file mode 100644 index 000000000..f57f26862 --- /dev/null +++ b/lib/php/storage-bundle/Controller/MultipartUploadCancelAction.php @@ -0,0 +1,29 @@ +uploadManager->cancelMultipartUpload($data->getPath(), $data->getUploadId()); + } catch (\Exception $e) { + + } + + $this->em->remove($data); + } +} diff --git a/lib/php/storage-bundle/Controller/MultipartUploadCompleteAction.php b/lib/php/storage-bundle/Controller/MultipartUploadCompleteAction.php new file mode 100644 index 000000000..aff6a8c38 --- /dev/null +++ b/lib/php/storage-bundle/Controller/MultipartUploadCompleteAction.php @@ -0,0 +1,39 @@ +request->all('parts'); + + if (empty($parts)) { + throw new BadRequestHttpException('Missing parts'); + } + + $res = $this->uploadManager->markComplete($data->getUploadId(), $data->getPath(), (array) $parts); + + $data->setComplete(true); + $this->em->persist($data); + $this->em->flush(); + + return new JsonResponse([ + 'path' => $res['Key'], + ]); + } +} diff --git a/lib/php/storage-bundle/Entity/MultipartUpload.php b/lib/php/storage-bundle/Entity/MultipartUpload.php index 8933a32ff..c3dd0403b 100644 --- a/lib/php/storage-bundle/Entity/MultipartUpload.php +++ b/lib/php/storage-bundle/Entity/MultipartUpload.php @@ -4,18 +4,20 @@ namespace Alchemy\StorageBundle\Entity; -use Alchemy\StorageBundle\Controller\MultipartUploadPartAction; -use ApiPlatform\Metadata\ApiProperty; -use ApiPlatform\Metadata\ApiResource; -use ApiPlatform\Metadata\Delete; +use Ramsey\Uuid\Uuid; use ApiPlatform\Metadata\Get; -use ApiPlatform\Metadata\GetCollection; use ApiPlatform\Metadata\Post; use Doctrine\DBAL\Types\Types; +use ApiPlatform\Metadata\Delete; use Doctrine\ORM\Mapping as ORM; use Ramsey\Uuid\Doctrine\UuidType; -use Ramsey\Uuid\Uuid; +use ApiPlatform\Metadata\ApiProperty; +use ApiPlatform\Metadata\ApiResource; +use ApiPlatform\Metadata\GetCollection; use Symfony\Component\Serializer\Annotation\Groups; +use Alchemy\StorageBundle\Controller\MultipartUploadPartAction; +use Alchemy\StorageBundle\Controller\MultipartUploadCancelAction; +use Alchemy\StorageBundle\Controller\MultipartUploadCompleteAction; #[ApiResource( shortName: 'Upload', @@ -67,7 +69,42 @@ ]], ], ]), - new Delete(openapiContext: ['summary' => 'Cancel an upload', 'description' => 'Cancel an upload.']), + new Post( + uriTemplate: '/uploads/{id}/complete', + controller: MultipartUploadCompleteAction::class, + openapiContext: [ + 'summary' => 'Complete a multi part upload.', + 'requestBody' => [ + 'content' => [ + 'application/json' => [ + 'schema' => [ + 'type' => 'object', + 'properties' => [ + 'parts' => [ + 'type' => 'array', + 'items' => [ + 'type' => 'object', + 'properties' => [ + 'ETag' => ['type' => 'string'], + 'PartNumber' => ['type' => 'integer'], + ], + ], + ] + ] + ] + ] + ] + ] + ] + ), + new Delete( + controller: MultipartUploadCancelAction::class, + openapiContext: [ + 'summary' => 'Cancel an upload', + 'description' => 'Cancel an upload.' + ] + ), + new GetCollection(security: 'is_granted(\'ROLE_ADMIN\')'), ], normalizationContext: ['groups' => ['upload:read']], @@ -149,6 +186,7 @@ public function getUploadId(): string public function setUploadId(string $uploadId): void { + file_put_contents("log.txt", "nandalo setuploadid"); $this->uploadId = $uploadId; } diff --git a/lib/php/storage-bundle/Resources/config/services.yaml b/lib/php/storage-bundle/Resources/config/services.yaml index 2e1adf8c3..fe3869df2 100644 --- a/lib/php/storage-bundle/Resources/config/services.yaml +++ b/lib/php/storage-bundle/Resources/config/services.yaml @@ -42,6 +42,14 @@ services: tags: - { name: controller.service_arguments } + Alchemy\StorageBundle\Controller\MultipartUploadCancelAction: + tags: + - { name: controller.service_arguments } + + Alchemy\StorageBundle\Controller\MultipartUploadCompleteAction: + tags: + - { name: controller.service_arguments } + Alchemy\StorageBundle\Doctrine\MultipartUploadListener: ~ Alchemy\StorageBundle\Storage\PathGenerator: ~ diff --git a/lib/php/storage-bundle/Upload/UploadManager.php b/lib/php/storage-bundle/Upload/UploadManager.php index 7d66e74f2..23ad2361e 100644 --- a/lib/php/storage-bundle/Upload/UploadManager.php +++ b/lib/php/storage-bundle/Upload/UploadManager.php @@ -4,14 +4,14 @@ namespace Alchemy\StorageBundle\Upload; -use Alchemy\StorageBundle\Entity\MultipartUpload; -use Aws\Api\DateTimeResult; use Aws\S3\S3Client; -use Doctrine\ORM\EntityManagerInterface; +use Aws\Api\DateTimeResult; use Psr\Log\LoggerInterface; +use Doctrine\ORM\EntityManagerInterface; use Symfony\Component\HttpFoundation\Request; -use Symfony\Component\HttpKernel\Exception\BadRequestHttpException; +use Alchemy\StorageBundle\Entity\MultipartUpload; use Symfony\Component\HttpKernel\Exception\NotFoundHttpException; +use Symfony\Component\HttpKernel\Exception\BadRequestHttpException; final readonly class UploadManager { @@ -62,7 +62,7 @@ public function getSignedUrl(string $uploadId, string $path, int $partNumber): s return (string) $request->getUri(); } - public function markComplete(string $uploadId, string $filename, array $parts): void + public function markComplete(string $uploadId, string $filename, array $parts) { $params = [ 'Bucket' => $this->uploadBucket, @@ -73,7 +73,7 @@ public function markComplete(string $uploadId, string $filename, array $parts): 'UploadId' => $uploadId, ]; - $this->client->completeMultipartUpload($params); + return $this->client->completeMultipartUpload($params); } public function createPutObjectSignedURL(string $path, string $contentType): string