Skip to content

Commit

Permalink
Merge branch 'skurfuerst-skip-unpublishing'
Browse files Browse the repository at this point in the history
  • Loading branch information
@kitsunet
kitsunet committed Jul 28, 2021
2 parents 1731bfe + be9bf60 commit 2d50403
Show file tree
Hide file tree
Showing 2 changed files with 77 additions and 7 deletions.
41 changes: 38 additions & 3 deletions Classes/S3Target.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,21 @@ class S3Target implements TargetInterface
*/
protected $baseUri;

/**
* if TRUE (default), resources which are not anymore part of the storage will be removed
* from the target as well. If set to FALSE, your target will only ever grow, never shrink.
*
* @var boolean
*/
protected $unpublishResources = true;

/**
* If `true` (default) the S3 ACL is set to `public-read`. If `false` no ACL option will be set.
*
* @var boolean
*/
protected $accessPolicyEnabled = true;

/**
* Internal cache for known storages, indexed by storage name
*
Expand Down Expand Up @@ -122,6 +137,12 @@ public function __construct($name, array $options = array())
case 'baseUri':
$this->baseUri = $value;
break;
case 'unpublishResources':
$this->unpublishResources = (bool)$value;
break;
case 'accessPolicyEnabled':
$this->accessPolicyEnabled = (bool)$value;
break;
default:
if ($value !== null) {
throw new Exception(sprintf('An unknown option "%s" was specified in the configuration of the "%s" resource S3Target. Please check your settings.', $key, $name), 1428928226);
Expand Down Expand Up @@ -209,13 +230,15 @@ public function publishCollection(CollectionInterface $collection, callable $cal
$potentiallyObsoleteObjects[$objectName] = false;
} else {
$options = [
'ACL' => 'public-read',
'Bucket' => $this->bucketName,
'CopySource' => urlencode($storageBucketName . '/' . $storage->getKeyPrefix() . $object->getSha1()),
'ContentType' => $object->getMediaType(),
'MetadataDirective' => 'REPLACE',
'Key' => $objectName
];
if ($this->accessPolicyEnabled !== false) {
$options['ACL'] = 'public-read';
}
try {
$this->s3Client->copyObject($options);
$this->systemLogger->debug(sprintf('Successfully copied resource as object "%s" (SHA1: %s) from bucket "%s" to bucket "%s"', $objectName, $object->getSha1() ?: 'unknown', $storageBucketName, $this->bucketName));
Expand All @@ -235,6 +258,11 @@ public function publishCollection(CollectionInterface $collection, callable $cal
}
}

if ($this->unpublishResources === false) {
$this->systemLogger->debug(sprintf('Skipping resource unpublishing from bucket "%s", because configuration option "unpublishResources" is FALSE.', $this->bucketName));
return;
}

foreach (array_keys($potentiallyObsoleteObjects) as $relativePathAndFilename) {
if (!$potentiallyObsoleteObjects[$relativePathAndFilename]) {
continue;
Expand Down Expand Up @@ -281,13 +309,15 @@ public function publishResource(PersistentResource $resource, CollectionInterfac
$sourceObjectArn = $storage->getBucketName() . '/' . $storage->getKeyPrefix() . $resource->getSha1();
$objectName = $this->keyPrefix . $this->getRelativePublicationPathAndFilename($resource);
$options = [
'ACL' => 'public-read',
'Bucket' => $this->bucketName,
'CopySource' => urlencode($sourceObjectArn),
'ContentType'=> $resource->getMediaType(),
'MetadataDirective' => 'REPLACE',
'Key' => $objectName
];
if ($this->accessPolicyEnabled !== false) {
$options['ACL'] = 'public-read';
}
$this->s3Client->copyObject($options);
$this->systemLogger->debug(sprintf('Successfully published resource as object "%s" (SHA1: %s) by copying from bucket "%s" to bucket "%s"', $objectName, $resource->getSha1() ?: 'unknown', $storage->getBucketName(), $this->bucketName));
} catch (S3Exception $e) {
Expand All @@ -314,6 +344,11 @@ public function publishResource(PersistentResource $resource, CollectionInterfac
*/
public function unpublishResource(PersistentResource $resource)
{
if ($this->unpublishResources === false) {
$this->systemLogger->debug(sprintf('Skipping resource unpublishing %s from bucket "%s", because configuration option "unpublishResources" is FALSE.', $resource->getMd5() ?: 'unknown', $this->bucketName));
return;
}

try {
$objectName = $this->keyPrefix . $this->getRelativePublicationPathAndFilename($resource);
$this->s3Client->deleteObject(array(
Expand Down Expand Up @@ -371,7 +406,7 @@ protected function publishFile($sourceStream, $relativeTargetPathAndFilename, Re
);

try {
$this->s3Client->upload($this->bucketName, $objectName, $sourceStream, 'public-read', $options);
$this->s3Client->upload($this->bucketName, $objectName, $sourceStream, $this->accessPolicyEnabled !== false ? 'public-read' : null, $options);
$this->systemLogger->debug(sprintf('Successfully published resource as object "%s" in bucket "%s" with SHA1 hash "%s"', $objectName, $this->bucketName, $metaData->getSha1() ?: 'unknown'));
} catch (\Exception $e) {
$this->systemLogger->debug(sprintf('Failed publishing resource as object "%s" in bucket "%s" with SHA1 hash "%s": %s', $objectName, $this->bucketName, $metaData->getSha1() ?: 'unknown', $e->getMessage()));
Expand Down
43 changes: 39 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ In order to communicate with the AWS web service, you need to provide the creden
to S3 (see next section for instructions for setting up the user in AWS IAM). Add the following configuration to the
`Settings.yaml` for your desired Flow context (for example in `Configuration/Production/Settings.yaml`) and make sure
to replace key, secret and region with your own data:

```yaml
Flownative:
Aws:
Expand All @@ -46,14 +46,14 @@ Flownative:
```
You can test your settings by executing the `connect` command. If you restricted access to a particular sub path of
a bucket, you must specify the bucket and key prefix:
a bucket, you must specify the bucket and key prefix:

```bash
$ ./flow s3:connect --bucket test.storage.net --prefix sites/s3-test/
Access list of objects in bucket "test.storage.neos" with key prefix "sites/s3-test/" ...
Writing test object into bucket (arn:aws:s3:::test.storage.neos/sites/s3-test/Flownative.Aws.S3.ConnectionTest.txt) ...
Deleting test object from bucket ...
OK
OK
```

Note that it does make a difference if you specify the prefix with a leading slash "/" or without, because the corresponding
Expand Down Expand Up @@ -300,7 +300,7 @@ Flownative:
Google Cloud Storage (GCS) is an offering by Google which is very similar to AWS S3. In fact, GCS supports an S3-compatible
endpoint which allows you to use Google's storage as a replacement for Amazon's S3. However, note that if you
access GCS through the S3 compatible service endpoint, you won't be able to use the full feature set of Google Cloud
Storage and you cannot easily restrict access for different users to specific buckets or sub paths.
Storage and you cannot easily restrict access for different users to specific buckets or sub paths.

GCS does not have a limit for the number of buckets you can have for one account, therefore you don't necessarily need
to share buckets across sites or projects. The following instructions assume that you use one dedicated bucket for
Expand Down Expand Up @@ -343,3 +343,38 @@ Flownative:
# Prevents the AWS client to prepend the bucket name to the hostname
use_path_style_endpoint: true
```

## Preventing unpublishing of resources in the target

There are certain situations (e.g. when having a two-stack CMS setup), where one needs to prevent unpublishing of images
or other resources, for some time.

Thus, the S3 Target option `unpublishResources` can be set to `false`, to prevent removing data from the S3 Target:

```yaml
Neos:
Flow:
resource:
targets:
s3PersistentResourcesTarget:
target: 'Flownative\Aws\S3\S3Target'
targetOptions:
unpublishResources: false
# ... other options here ...
```

## Disable public-read ACL

The canned ACL "public-read" is not useful in some cases, e.g. when using CloudFront with conflicting restrictive policies.
With this option the ACL setting for the target can be disabled/removed.

```yaml
Neos:
Flow:
resource:
targets:
s3PersistentResourcesTarget:
target: 'Flownative\Aws\S3\S3Target'
targetOptions:
accessPolicyEnabled: false
```

0 comments on commit 2d50403

Please sign in to comment.