From 19ba8f0a3a98614562be644703b0c23f025d8f26 Mon Sep 17 00:00:00 2001 From: Robert Lemke Date: Mon, 13 Mar 2017 14:09:07 +0100 Subject: [PATCH 1/5] WIP: Raise dependency on AWS SDK to ~3.0 --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 6564b3b..88b4a8f 100644 --- a/composer.json +++ b/composer.json @@ -15,7 +15,7 @@ ], "require": { "typo3/flow": "~3.1", - "aws/aws-sdk-php": "2.8.*" + "aws/aws-sdk-php": "~3.0" }, "autoload": { "psr-4": { From 5d14875a2e1da59d1deaed5672f9a434b5fcb2dc Mon Sep 17 00:00:00 2001 From: Robert Lemke Date: Mon, 13 Mar 2017 15:14:29 +0100 Subject: [PATCH 2/5] TASK: Adjust to AWS SDK 3.x --- Classes/Command/S3CommandController.php | 19 +++++----- Classes/S3Storage.php | 48 ++++++++++++++++--------- Classes/S3Target.php | 2 +- Configuration/Settings.yaml | 8 ++++- README.md | 9 ++--- 5 files changed, 56 insertions(+), 30 deletions(-) diff --git a/Classes/Command/S3CommandController.php b/Classes/Command/S3CommandController.php index cae7ba4..0d47b15 100644 --- a/Classes/Command/S3CommandController.php +++ b/Classes/Command/S3CommandController.php @@ -5,6 +5,7 @@ * This script belongs to the package "Flownative.Aws.S3". * * */ +use Aws\S3\BatchDelete; use Aws\S3\Model\ClearBucket; use Aws\S3\S3Client; use TYPO3\Flow\Annotations as Flow; @@ -39,13 +40,12 @@ class S3CommandController extends CommandController public function connectCommand($bucket = null, $prefix = '') { try { - $s3Client = S3Client::factory($this->s3DefaultProfile); + $s3Client = new S3Client($this->s3DefaultProfile); if ($bucket !== null) { $s3Client->registerStreamWrapper(); $this->outputLine('Access list of objects in bucket "%s" with key prefix "%s" ...', [$bucket, $prefix]); - $iterator = $s3Client->getListObjectsIterator(['Bucket' => $bucket, 'Prefix' => $prefix]); - $iterator->toArray(); + $s3Client->getPaginator('ListObjects', ['Bucket' => $bucket, 'Prefix' => $prefix]); $options = array( 'Bucket' => $bucket, @@ -87,11 +87,12 @@ public function connectCommand($bucket = null, $prefix = '') public function listBucketsCommand() { try { - $s3Client = S3Client::factory($this->s3DefaultProfile); + $s3Client = new S3Client($this->s3DefaultProfile); $result = $s3Client->listBuckets(); } catch (\Exception $e) { $this->outputLine($e->getMessage()); $this->quit(1); + exit; } if (count($result['Buckets']) === 0) { @@ -119,13 +120,15 @@ public function listBucketsCommand() public function flushBucketCommand($bucket) { try { - $s3Client = S3Client::factory($this->s3DefaultProfile); - $clearBucket = new ClearBucket($s3Client, $bucket); - $clearBucket->clear(); + $s3Client = new S3Client($this->s3DefaultProfile); + $batchDelete = BatchDelete::fromListObjects($s3Client, ['Bucket' => $bucket]); + $promise = $batchDelete->promise(); } catch (\Exception $e) { $this->outputLine($e->getMessage()); $this->quit(1); + exit; } + $promise->wait(); $this->outputLine('Successfully flushed bucket %s.', array($bucket)); } @@ -153,7 +156,7 @@ public function uploadCommand($bucket, $file, $key = '') } try { - $s3Client = S3Client::factory($this->s3DefaultProfile); + $s3Client = new S3Client($this->s3DefaultProfile); $s3Client->putObject(array( 'Key' => $key, 'Bucket' => $bucket, diff --git a/Classes/S3Storage.php b/Classes/S3Storage.php index b4b5825..6814abe 100644 --- a/Classes/S3Storage.php +++ b/Classes/S3Storage.php @@ -6,6 +6,7 @@ * * * */ +use Aws\S3\Exception\S3Exception; use Aws\S3\S3Client; use TYPO3\Flow\Annotations as Flow; use TYPO3\Flow\Resource\CollectionInterface; @@ -114,7 +115,7 @@ public function initializeObject() { $clientOptions = $this->s3DefaultProfile; - $this->s3Client = S3Client::factory($clientOptions); + $this->s3Client = new S3Client($clientOptions); $this->s3Client->registerStreamWrapper(); } @@ -292,7 +293,8 @@ public function deleteResource(Resource $resource) * stored in this storage. * * @param \TYPO3\Flow\Resource\Resource $resource The resource stored in this storage - * @return resource | boolean A URI (for example the full path and filename) leading to the resource file or FALSE if it does not exist + * @return bool|resource A URI (for example the full path and filename) leading to the resource file or FALSE if it does not exist + * @throws Exception * @api */ public function getStreamByResource(Resource $resource) @@ -305,7 +307,7 @@ public function getStreamByResource(Resource $resource) } $message = sprintf('Could not retrieve stream for resource %s (s3://%s/%s%s). %s', $resource->getFilename(), $this->bucketName, $this->keyPrefix, $resource->getSha1(), $e->getMessage()); $this->systemLogger->log($message, \LOG_ERR); - throw new Exception($message, 1445682605); + return false; } } @@ -314,7 +316,8 @@ public function getStreamByResource(Resource $resource) * stored in this storage. * * @param string $relativePath A path relative to the storage root, for example "MyFirstDirectory/SecondDirectory/Foo.css" - * @return resource | boolean A URI (for example the full path and filename) leading to the resource file or FALSE if it does not exist + * @return bool|resource A URI (for example the full path and filename) leading to the resource file or FALSE if it does not exist + * @throws Exception * @api */ public function getStreamByResourcePath($relativePath) @@ -327,7 +330,7 @@ public function getStreamByResourcePath($relativePath) } $message = sprintf('Could not retrieve stream for resource (s3://%s/%s%s). %s', $this->bucketName, $this->keyPrefix, ltrim('/', $relativePath), $e->getMessage()); $this->systemLogger->log($message, \LOG_ERR); - throw new Exception($message, 1445682606); + return false; } } @@ -365,7 +368,9 @@ public function getObjectsByCollection(CollectionInterface $collection) $object = new Object(); $object->setFilename($resource->getFilename()); $object->setSha1($resource->getSha1()); - $object->setStream(function () use ($that, $bucketName, $resource) { return fopen('s3://' . $bucketName . '/' . $this->keyPrefix . $resource->getSha1(), 'r'); }); + $object->setStream(function () use ($that, $bucketName, $resource) { + return fopen('s3://' . $bucketName . '/' . $this->keyPrefix . $resource->getSha1(), 'r'); + }); $objects[] = $object; } @@ -383,6 +388,7 @@ protected function importTemporaryFile($temporaryPathAndFilename, $collectionNam { $sha1Hash = sha1_file($temporaryPathAndFilename); $md5Hash = md5_file($temporaryPathAndFilename); + $objectName = $this->keyPrefix . $sha1Hash; $resource = new Resource(); $resource->setFileSize(filesize($temporaryPathAndFilename)); @@ -390,17 +396,27 @@ protected function importTemporaryFile($temporaryPathAndFilename, $collectionNam $resource->setSha1($sha1Hash); $resource->setMd5($md5Hash); - $objectName = $this->keyPrefix . $sha1Hash; - $options = array( - 'Bucket' => $this->bucketName, - 'Body' => fopen($temporaryPathAndFilename, 'rb'), - 'ContentLength' => $resource->getFileSize(), - 'ContentType' => $resource->getMediaType(), - 'Key' => $objectName - ); + try { + $this->s3Client->headObject([ + 'Bucket' => $this->bucketName, + 'Key' => $objectName + ]); + $objectAlreadyExists = true; + } catch (S3Exception $e) { + if ($e->getAwsErrorCode() !== 'NotFound') { + throw $e; + } + $objectAlreadyExists = false; + } - if (!$this->s3Client->doesObjectExist($this->bucketName, $this->keyPrefix . $sha1Hash)) { - $this->s3Client->putObject($options); + if (!$objectAlreadyExists) { + $this->s3Client->putObject([ + 'Bucket' => $this->bucketName, + 'Body' => fopen($temporaryPathAndFilename, 'rb'), + 'ContentLength' => $resource->getFileSize(), + 'ContentType' => $resource->getMediaType(), + 'Key' => $objectName + ]); $this->systemLogger->log(sprintf('Successfully imported resource as object "%s" into bucket "%s" with MD5 hash "%s"', $objectName, $this->bucketName, $resource->getMd5() ?: 'unknown'), LOG_INFO); } else { $this->systemLogger->log(sprintf('Did not import resource as object "%s" into bucket "%s" because that object already existed.', $objectName, $this->bucketName), LOG_INFO); diff --git a/Classes/S3Target.php b/Classes/S3Target.php index cfed80b..c3eafff 100644 --- a/Classes/S3Target.php +++ b/Classes/S3Target.php @@ -130,7 +130,7 @@ public function initializeObject() { $clientOptions = $this->s3DefaultProfile; - $this->s3Client = S3Client::factory($clientOptions); + $this->s3Client = new S3Client($clientOptions); $this->s3Client->registerStreamWrapper(); } diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 1e48267..9698462 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -5,8 +5,14 @@ Flownative: # Default credentials and client options - override these in your settings with real values # For more documentation regarding options, see http://docs.aws.amazon.com/aws-sdk-php/v2/guide/configuration.html#client-configuration-options default: + + # Select the API version to use + version: '2006-03-01' + # Don't use the old signature v2 - certain regions don't support them - signature: 'v4' + signature_version: 'v4' + + # Define credentials for authentication credentials: key: 'ABCD123EFG456HIJ7890' secret: 'aBc123DEf456GHi789JKlMNopqRsTuVWXyz12345' diff --git a/README.md b/README.md index 1ac2669..a5e8bc2 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,12 @@ [packagist]: https://img.shields.io/packagist/v/flownative/aws-s3.svg -# AWS S3 Adaptor for Neos 2.x and Flow 3.x +# AWS S3 Adaptor for Neos and Flow -This [Flow](https://flow.typo3.org) package allows you to store assets (resources) in [Amazon's S3](https://aws.amazon.com/s3/) -and publish resources to S3 or [Cloudfront](https://aws.amazon.com/cloudfront/). Because [Neos CMS](https://www.neos.io) is -using Flow's resource management under the hood, this adaptor also works nicely for all kinds of assets in Neos. +This [Flow](https://flow.neos.io) package allows you to store assets (resources) in [Amazon's S3](https://aws.amazon.com/s3/) +or S3-compatible storages and publish resources to S3 or [Cloudfront](https://aws.amazon.com/cloudfront/). Because +[Neos CMS](https://www.neos.io) is using Flow's resource management under the hood, this adaptor also works nicely for +all kinds of assets in Neos. ## Key Features From 5ef1e291c9777869a0325077d7580df1b1a08a24 Mon Sep 17 00:00:00 2001 From: Karsten Dambekalns Date: Sun, 2 Apr 2017 15:41:57 +0200 Subject: [PATCH 3/5] TASK: Make S3Target more forgiving when publishing Instead of throwing exceptions that block publishing of remaining resources, an error will be logged and the code continues to run. In addition this adds support for a callback to `publishCollection` (it is passed down to `getObjects` like in the `FileSystemTarget`.) --- Classes/S3Target.php | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/Classes/S3Target.php b/Classes/S3Target.php index c3eafff..c758a01 100644 --- a/Classes/S3Target.php +++ b/Classes/S3Target.php @@ -11,6 +11,7 @@ use TYPO3\Flow\Annotations as Flow; use TYPO3\Flow\Resource\CollectionInterface; use TYPO3\Flow\Resource\Exception; +use TYPO3\Flow\Resource\Publishing\MessageCollector; use TYPO3\Flow\Resource\Resource; use TYPO3\Flow\Resource\ResourceManager; use TYPO3\Flow\Resource\ResourceMetaDataInterface; @@ -78,6 +79,12 @@ class S3Target implements TargetInterface */ protected $systemLogger; + /** + * @Flow\Inject + * @var MessageCollector + */ + protected $messageCollector; + /** * @Flow\Inject * @var ResourceManager @@ -158,10 +165,11 @@ public function getKeyPrefix() * Publishes the whole collection to this target * * @param \TYPO3\Flow\Resource\CollectionInterface $collection The collection to publish + * @param callable $callback Function called after each resource publishing * @return void * @throws Exception */ - public function publishCollection(CollectionInterface $collection) + public function publishCollection(CollectionInterface $collection, callable $callback = null) { if (!isset($this->existingObjectsInfo)) { $this->existingObjectsInfo = array(); @@ -188,7 +196,7 @@ public function publishCollection(CollectionInterface $collection) if ($storageBucketName === $this->bucketName && $storage->getKeyPrefix() === $this->keyPrefix) { throw new Exception(sprintf('Could not publish collection %s because the source and target S3 bucket is the same, with identical key prefixes. Either choose a different bucket or at least key prefix for the target.', $collection->getName()), 1428929137); } - foreach ($collection->getObjects() as $object) { + foreach ($collection->getObjects($callback) as $object) { /** @var \TYPO3\Flow\Resource\Storage\Object $object */ $objectName = $this->keyPrefix . $this->getRelativePublicationPathAndFilename($object); $options = array( @@ -202,7 +210,10 @@ public function publishCollection(CollectionInterface $collection) try { $this->s3Client->copyObject($options); } catch (S3Exception $e) { - throw new Exception(sprintf('Could not copy resource with SHA1 hash %s of collection %s from bucket %s to %s: %s', $object->getSha1(), $collection->getName(), $storageBucketName, $this->bucketName, $e->getMessage()), 1431009234); + $message = sprintf('Could not copy resource with SHA1 hash %s of collection %s from bucket %s to %s: %s', $object->getSha1(), $collection->getName(), $storageBucketName, $this->bucketName, $e->getMessage()); + $this->systemLogger->logException($e); + $this->messageCollector->append($message); + continue; } $this->systemLogger->log(sprintf('Successfully copied resource as object "%s" (MD5: %s) from bucket "%s" to bucket "%s"', $objectName, $object->getMd5() ?: 'unknown', $storageBucketName, $this->bucketName), LOG_DEBUG); unset($obsoleteObjects[$this->getRelativePublicationPathAndFilename($object)]); @@ -263,12 +274,16 @@ public function publishResource(Resource $resource, CollectionInterface $collect $this->s3Client->copyObject($options); $this->systemLogger->log(sprintf('Successfully published resource as object "%s" (MD5: %s) by copying from bucket "%s" to bucket "%s"', $objectName, $resource->getMd5() ?: 'unknown', $storage->getBucketName(), $this->bucketName), LOG_DEBUG); } catch (S3Exception $e) { - throw new Exception(sprintf('Could not publish resource with SHA1 hash %s of collection %s (source object: %s) through "CopyObject" because the S3 client reported an error: %s', $resource->getSha1(), $collection->getName(), $sourceObjectArn, $e->getMessage()), 1428999574); + $message = sprintf('Could not publish resource with SHA1 hash %s of collection %s (source object: %s) through "CopyObject" because the S3 client reported an error: %s', $resource->getSha1(), $collection->getName(), $sourceObjectArn, $e->getMessage())); + $this->systemLogger->logException($e); + $this->messageCollector->append($message); } } else { $sourceStream = $resource->getStream(); if ($sourceStream === false) { - throw new Exception(sprintf('Could not publish resource with SHA1 hash %s of collection %s because there seems to be no corresponding data in the storage.', $resource->getSha1(), $collection->getName()), 1428929649); + $message = sprintf('Could not publish resource with SHA1 hash %s of collection %s because there seems to be no corresponding data in the storage.', $resource->getSha1(), $collection->getName(); + $this->messageCollector->append($message); + return; } $this->publishFile($sourceStream, $this->getRelativePublicationPathAndFilename($resource), $resource); } From 7d9d42735aa08a36443f9430d5305d76a667b204 Mon Sep 17 00:00:00 2001 From: Lienhart Woitok Date: Fri, 9 Jun 2017 14:44:24 +0200 Subject: [PATCH 4/5] [BUGFIX] Fix fetching more than 1000 existing resources on publish When publishing a collection the existing resources in the target bucket are fetched for later clean up. AWS SDK returns up to 1000 objects in one request and offers pagination to fetch more. Using pagination parameter with lowercase spelling causes S3 to always return the first page, providing a marker for the next page, causing an endless loop. This fixes the parameter. --- Classes/S3Target.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Classes/S3Target.php b/Classes/S3Target.php index c3eafff..c25e573 100644 --- a/Classes/S3Target.php +++ b/Classes/S3Target.php @@ -175,7 +175,7 @@ public function publishCollection(CollectionInterface $collection) $result = $this->s3Client->listObjects($requestArguments); $this->existingObjectsInfo[] = $result->get('Contents'); if ($result->get('IsTruncated')) { - $requestArguments['marker'] = $result->get('NextMarker'); + $requestArguments['Marker'] = $result->get('NextMarker'); } } while ($result->get('IsTruncated')); } From 06efcf5cfe07ab15429b7497d0238191bbab6729 Mon Sep 17 00:00:00 2001 From: Lienhart Woitok Date: Fri, 9 Jun 2017 16:49:43 +0200 Subject: [PATCH 5/5] [BUGFIX] Fix setting base uri for static resources This change allows setting a base uri for static resources like the existing behaviour with base uri for persistent resources. --- Classes/S3Target.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Classes/S3Target.php b/Classes/S3Target.php index c3eafff..9a0a51a 100644 --- a/Classes/S3Target.php +++ b/Classes/S3Target.php @@ -231,7 +231,11 @@ public function publishCollection(CollectionInterface $collection) */ public function getPublicStaticResourceUri($relativePathAndFilename) { - return $this->s3Client->getObjectUrl($this->bucketName, $this->keyPrefix . $relativePathAndFilename); + if ($this->baseUri != '') { + return $this->baseUri . $relativePathAndFilename; + } else { + return $this->s3Client->getObjectUrl($this->bucketName, $this->keyPrefix . $relativePathAndFilename); + } } /**