From 5ef1e291c9777869a0325077d7580df1b1a08a24 Mon Sep 17 00:00:00 2001
From: Karsten Dambekalns <karsten@dambekalns.de>
Date: Sun, 2 Apr 2017 15:41:57 +0200
Subject: [PATCH] TASK: Make S3Target more forgiving when publishing

Instead of throwing exceptions that block publishing of remaining resources,
an error will be logged and the code continues to run.

In addition this adds support for a callback to `publishCollection` (it is passed
down to `getObjects` like in the `FileSystemTarget`.)
---
 Classes/S3Target.php | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/Classes/S3Target.php b/Classes/S3Target.php
index c3eafff..c758a01 100644
--- a/Classes/S3Target.php
+++ b/Classes/S3Target.php
@@ -11,6 +11,7 @@
 use TYPO3\Flow\Annotations as Flow;
 use TYPO3\Flow\Resource\CollectionInterface;
 use TYPO3\Flow\Resource\Exception;
+use TYPO3\Flow\Resource\Publishing\MessageCollector;
 use TYPO3\Flow\Resource\Resource;
 use TYPO3\Flow\Resource\ResourceManager;
 use TYPO3\Flow\Resource\ResourceMetaDataInterface;
@@ -78,6 +79,12 @@ class S3Target implements TargetInterface
      */
     protected $systemLogger;
 
+    /**
+     * @Flow\Inject
+     * @var MessageCollector
+     */
+    protected $messageCollector;
+
     /**
      * @Flow\Inject
      * @var ResourceManager
@@ -158,10 +165,11 @@ public function getKeyPrefix()
      * Publishes the whole collection to this target
      *
      * @param \TYPO3\Flow\Resource\CollectionInterface $collection The collection to publish
+     * @param callable $callback Function called after each resource publishing
      * @return void
      * @throws Exception
      */
-    public function publishCollection(CollectionInterface $collection)
+    public function publishCollection(CollectionInterface $collection, callable $callback = null)
     {
         if (!isset($this->existingObjectsInfo)) {
             $this->existingObjectsInfo = array();
@@ -188,7 +196,7 @@ public function publishCollection(CollectionInterface $collection)
             if ($storageBucketName === $this->bucketName && $storage->getKeyPrefix() === $this->keyPrefix) {
                 throw new Exception(sprintf('Could not publish collection %s because the source and target S3 bucket is the same, with identical key prefixes. Either choose a different bucket or at least key prefix for the target.', $collection->getName()), 1428929137);
             }
-            foreach ($collection->getObjects() as $object) {
+            foreach ($collection->getObjects($callback) as $object) {
                 /** @var \TYPO3\Flow\Resource\Storage\Object $object */
                 $objectName = $this->keyPrefix . $this->getRelativePublicationPathAndFilename($object);
                 $options = array(
@@ -202,7 +210,10 @@ public function publishCollection(CollectionInterface $collection)
                 try {
                     $this->s3Client->copyObject($options);
                 } catch (S3Exception $e) {
-                    throw new Exception(sprintf('Could not copy resource with SHA1 hash %s of collection %s from bucket %s to %s: %s', $object->getSha1(), $collection->getName(), $storageBucketName, $this->bucketName, $e->getMessage()), 1431009234);
+                    $message = sprintf('Could not copy resource with SHA1 hash %s of collection %s from bucket %s to %s: %s', $object->getSha1(), $collection->getName(), $storageBucketName, $this->bucketName, $e->getMessage());
+                    $this->systemLogger->logException($e);
+                    $this->messageCollector->append($message);
+                    continue;
                 }
                 $this->systemLogger->log(sprintf('Successfully copied resource as object "%s" (MD5: %s) from bucket "%s" to bucket "%s"', $objectName, $object->getMd5() ?: 'unknown', $storageBucketName, $this->bucketName), LOG_DEBUG);
                 unset($obsoleteObjects[$this->getRelativePublicationPathAndFilename($object)]);
@@ -263,12 +274,16 @@ public function publishResource(Resource $resource, CollectionInterface $collect
                 $this->s3Client->copyObject($options);
                 $this->systemLogger->log(sprintf('Successfully published resource as object "%s" (MD5: %s) by copying from bucket "%s" to bucket "%s"', $objectName, $resource->getMd5() ?: 'unknown', $storage->getBucketName(), $this->bucketName), LOG_DEBUG);
             } catch (S3Exception $e) {
-                throw new Exception(sprintf('Could not publish resource with SHA1 hash %s of collection %s (source object: %s) through "CopyObject" because the S3 client reported an error: %s', $resource->getSha1(), $collection->getName(), $sourceObjectArn, $e->getMessage()), 1428999574);
+                $message = sprintf('Could not publish resource with SHA1 hash %s of collection %s (source object: %s) through "CopyObject" because the S3 client reported an error: %s', $resource->getSha1(), $collection->getName(), $sourceObjectArn, $e->getMessage()));
+                $this->systemLogger->logException($e);
+                $this->messageCollector->append($message);
             }
         } else {
             $sourceStream = $resource->getStream();
             if ($sourceStream === false) {
-                throw new Exception(sprintf('Could not publish resource with SHA1 hash %s of collection %s because there seems to be no corresponding data in the storage.', $resource->getSha1(), $collection->getName()), 1428929649);
+                $message = sprintf('Could not publish resource with SHA1 hash %s of collection %s because there seems to be no corresponding data in the storage.', $resource->getSha1(), $collection->getName();
+                $this->messageCollector->append($message);
+                return;
             }
             $this->publishFile($sourceStream, $this->getRelativePublicationPathAndFilename($resource), $resource);
         }