TASK: Make S3Target more forgiving when publishing

Instead of throwing exceptions that block publishing of remaining resources, an error will be logged and the code continues to run. In addition this adds support for a callback to `publishCollection` (it is passed down to `getObjects` like in the `FileSystemTarget`.)
flownative · Apr 2, 2017 · 5ef1e29 · 5ef1e29
1 parent 5d14875
commit 5ef1e29
Showing 1 changed file with 20 additions and 5 deletions.
diff --git a/Classes/S3Target.php b/Classes/S3Target.php
@@ -11,6 +11,7 @@
 use TYPO3\Flow\Annotations as Flow;
 use TYPO3\Flow\Resource\CollectionInterface;
 use TYPO3\Flow\Resource\Exception;
+use TYPO3\Flow\Resource\Publishing\MessageCollector;
 use TYPO3\Flow\Resource\Resource;
 use TYPO3\Flow\Resource\ResourceManager;
 use TYPO3\Flow\Resource\ResourceMetaDataInterface;
@@ -78,6 +79,12 @@ class S3Target implements TargetInterface
      */
     protected $systemLogger;
 
+    /**
+     * @Flow\Inject
+     * @var MessageCollector
+     */
+    protected $messageCollector;
+
     /**
      * @Flow\Inject
      * @var ResourceManager
@@ -158,10 +165,11 @@ public function getKeyPrefix()
      * Publishes the whole collection to this target
      *
      * @param \TYPO3\Flow\Resource\CollectionInterface $collection The collection to publish
+     * @param callable $callback Function called after each resource publishing
      * @return void
      * @throws Exception
      */
-    public function publishCollection(CollectionInterface $collection)
+    public function publishCollection(CollectionInterface $collection, callable $callback = null)
     {
         if (!isset($this->existingObjectsInfo)) {
             $this->existingObjectsInfo = array();
@@ -188,7 +196,7 @@ public function publishCollection(CollectionInterface $collection)
             if ($storageBucketName === $this->bucketName && $storage->getKeyPrefix() === $this->keyPrefix) {
                 throw new Exception(sprintf('Could not publish collection %s because the source and target S3 bucket is the same, with identical key prefixes. Either choose a different bucket or at least key prefix for the target.', $collection->getName()), 1428929137);
             }
-            foreach ($collection->getObjects() as $object) {
+            foreach ($collection->getObjects($callback) as $object) {
                 /** @var \TYPO3\Flow\Resource\Storage\Object $object */
                 $objectName = $this->keyPrefix . $this->getRelativePublicationPathAndFilename($object);
                 $options = array(
@@ -202,7 +210,10 @@ public function publishCollection(CollectionInterface $collection)
                 try {
                     $this->s3Client->copyObject($options);
                 } catch (S3Exception $e) {
-                    throw new Exception(sprintf('Could not copy resource with SHA1 hash %s of collection %s from bucket %s to %s: %s', $object->getSha1(), $collection->getName(), $storageBucketName, $this->bucketName, $e->getMessage()), 1431009234);
+                    $message = sprintf('Could not copy resource with SHA1 hash %s of collection %s from bucket %s to %s: %s', $object->getSha1(), $collection->getName(), $storageBucketName, $this->bucketName, $e->getMessage());
+                    $this->systemLogger->logException($e);
+                    $this->messageCollector->append($message);
+                    continue;
                 }
                 $this->systemLogger->log(sprintf('Successfully copied resource as object "%s" (MD5: %s) from bucket "%s" to bucket "%s"', $objectName, $object->getMd5() ?: 'unknown', $storageBucketName, $this->bucketName), LOG_DEBUG);
                 unset($obsoleteObjects[$this->getRelativePublicationPathAndFilename($object)]);
@@ -263,12 +274,16 @@ public function publishResource(Resource $resource, CollectionInterface $collect
                 $this->s3Client->copyObject($options);
                 $this->systemLogger->log(sprintf('Successfully published resource as object "%s" (MD5: %s) by copying from bucket "%s" to bucket "%s"', $objectName, $resource->getMd5() ?: 'unknown', $storage->getBucketName(), $this->bucketName), LOG_DEBUG);
             } catch (S3Exception $e) {
-                throw new Exception(sprintf('Could not publish resource with SHA1 hash %s of collection %s (source object: %s) through "CopyObject" because the S3 client reported an error: %s', $resource->getSha1(), $collection->getName(), $sourceObjectArn, $e->getMessage()), 1428999574);
+                $message = sprintf('Could not publish resource with SHA1 hash %s of collection %s (source object: %s) through "CopyObject" because the S3 client reported an error: %s', $resource->getSha1(), $collection->getName(), $sourceObjectArn, $e->getMessage()));
+                $this->systemLogger->logException($e);
+                $this->messageCollector->append($message);
             }
         } else {
             $sourceStream = $resource->getStream();
             if ($sourceStream === false) {
-                throw new Exception(sprintf('Could not publish resource with SHA1 hash %s of collection %s because there seems to be no corresponding data in the storage.', $resource->getSha1(), $collection->getName()), 1428929649);
+                $message = sprintf('Could not publish resource with SHA1 hash %s of collection %s because there seems to be no corresponding data in the storage.', $resource->getSha1(), $collection->getName();
+                $this->messageCollector->append($message);
+                return;
             }
             $this->publishFile($sourceStream, $this->getRelativePublicationPathAndFilename($resource), $resource);
         }