From caf5754c667b06ab8095b026f17d24663d5c5d86 Mon Sep 17 00:00:00 2001 From: Will Rossiter Date: Wed, 12 Jun 2024 09:17:05 +1200 Subject: [PATCH 1/2] feat: implement CacheItemsTrait for better performance handling --- README.md | 158 +++++---- _config/assets.yml | 73 ++-- composer.json | 5 +- doc/en/setting-local-dev-environment.md | 109 +++--- src/Adapter/CacheAdapter.php | 11 - src/Adapter/CachedAwsS3V3Adapter.php | 429 ++++++++++++++++++++++++ src/Adapter/ProtectedAdapter.php | 19 +- src/Adapter/ProtectedCachedAdapter.php | 13 - src/Adapter/PublicAdapter.php | 6 +- src/Adapter/PublicCDNAdapter.php | 12 +- src/Adapter/PublicCachedAdapter.php | 13 - src/Adapter/TinyMceAdapter.php | 2 +- src/Cache/CacheItemsTrait.php | 157 +++++++++ src/S3FlysystemAssetStore.php | 180 ++++++++++ 14 files changed, 970 insertions(+), 217 deletions(-) delete mode 100644 src/Adapter/CacheAdapter.php create mode 100644 src/Adapter/CachedAwsS3V3Adapter.php delete mode 100755 src/Adapter/ProtectedCachedAdapter.php delete mode 100755 src/Adapter/PublicCachedAdapter.php create mode 100644 src/Cache/CacheItemsTrait.php create mode 100644 src/S3FlysystemAssetStore.php diff --git a/README.md b/README.md index cdb7971..600d3fe 100644 --- a/README.md +++ b/README.md @@ -2,48 +2,59 @@ SilverStripe module to store assets in S3 rather than on the local filesystem. -Note: This module does not currently implement any kind of bucket policy for -protected assets. It is up to you to implement this yourself using AWS bucket -policy. +```sh +composer require silverstripe/s3 +``` + +> [!WARNING] +> This module does not currently implement any kind of bucket policy for +> protected assets. It is up to you to implement this yourself using AWS bucket +> policy. + +> [!CAUTION] +> This replaces the built-in local asset store that comes with SilverStripe +> with one based on S3. Any files that had previously been uploaded to an +> existing asset store will be unavailable (though they won't be lost - just +> run `composer remove silverstripe/s3` to remove the module and restore +> access). ## Environment setup -The module requires a few environment variables to be set. Full details can be -seen in the `SilverStripeS3AdapterTrait` trait. These are mandatory. +The module requires a few environment variables to be set. -- `AWS_REGION`: The AWS region your S3 bucket is hosted in (e.g. `eu-central-1`) -- `AWS_BUCKET_NAME`: The name of the S3 bucket to store assets in. +- `AWS_REGION`: The AWS region your S3 bucket is hosted in (e.g. `eu-central-1`) +- `AWS_BUCKET_NAME`: The name of the S3 bucket to store assets in. If running outside of an EC2 instance it will be necessary to specify an API key and secret. -- `AWS_ACCESS_KEY_ID`: Your AWS access key that has access to the bucket you - want to access -- `AWS_SECRET_ACCESS_KEY`: Your AWS secret corresponding to the access key +- `AWS_ACCESS_KEY_ID`: Your AWS access key that has access to the bucket you + want to access +- `AWS_SECRET_ACCESS_KEY`: Your AWS secret corresponding to the access key **Example YML Config when running outside of EC2:** ```yml --- Only: - envvarset: AWS_BUCKET_NAME + envvarset: AWS_BUCKET_NAME After: - - "#assetsflysystem" + - "#assetsflysystem" --- SilverStripe\Core\Injector\Injector: - Aws\S3\S3Client: - constructor: - configuration: - region: "`AWS_REGION`" - version: latest - credentials: - key: "`AWS_ACCESS_KEY_ID`" - secret: "`AWS_SECRET_ACCESS_KEY`" + Aws\S3\S3Client: + constructor: + configuration: + region: "`AWS_REGION`" + version: latest + credentials: + key: "`AWS_ACCESS_KEY_ID`" + secret: "`AWS_SECRET_ACCESS_KEY`" ``` ## (Optional) CDN Implementation -If you're serving assets from S3, it's recommended that you utilise CloudFront. +If you're serving assets from S3, it's recommended that you utilize CloudFront. This improves performance and security over exposing from S3 directly. Once you've set up your CloudFront distribution, ensure that assets are @@ -51,8 +62,8 @@ reachable within the `assets` directory of the cdn (for example; https://cdn.example.com/assets/Uploads/file.jpg) and set the following environment variable: -- `AWS_PUBLIC_CDN_PREFIX`: Your CloudFront distribution domain that has access - to the bucket you want to access +- `AWS_PUBLIC_CDN_PREFIX`: Your CloudFront distribution domain that has access + to the bucket you want to access For example, adding this to your `.env`: @@ -66,47 +77,33 @@ to something like: `https://cdn.example.com/assets/Uploads/file.jpg` -You can override the default `/assets/` path by redeclaring the PublicCDNAdapter constructor, with the paramater for the `cdnAssetsDir` set to a string of your folder name: +You can override the default `/assets/` path by declaring the PublicCDNAdapter +constructor, with the parameter for the `cdnAssetsDir` set to a string of your +folder name: ```yml --- Name: silverstripes3-cdn Only: - envvarset: AWS_PUBLIC_CDN_PREFIX + envvarset: AWS_PUBLIC_CDN_PREFIX After: - - "#assetsflysystem" - - "#silverstripes3-flysystem" + - "#assetsflysystem" + - "#silverstripes3-flysystem" --- SilverStripe\Core\Injector\Injector: - SilverStripe\S3\Adapter\PublicAdapter: - class: SilverStripe\S3\Adapter\PublicCDNAdapter - constructor: - s3Client: '%$Aws\S3\S3Client' - bucket: '`AWS_BUCKET_NAME`' - prefix: '`AWS_PUBLIC_BUCKET_PREFIX`' - visibility: null - mimeTypeDetector: null - cdnPrefix: '`AWS_PUBLIC_CDN_PREFIX`' - options: [] - cdnAssetsDir: "cms-assets" # example of a custom assets folder name, which will produce https://cdn.example.com/cms-assets/Uploads/file.jpg + SilverStripe\S3\Adapter\PublicAdapter: + class: SilverStripe\S3\Adapter\PublicCDNAdapter + constructor: + s3Client: '%$Aws\S3\S3Client' + bucket: "`AWS_BUCKET_NAME`" + prefix: "`AWS_PUBLIC_BUCKET_PREFIX`" + visibility: null + mimeTypeDetector: null + cdnPrefix: "`AWS_PUBLIC_CDN_PREFIX`" + options: [] + cdnAssetsDir: "cms-assets" # example of a custom assets folder name ``` -## Installation - -- Define the environment variables listed above. -- [Install Composer from - https://getcomposer.org](https://getcomposer.org/download/) -- Run `composer require silverstripe/s3` - -This will install the most recent applicable version of the module given your -other Composer requirements. - -**Note:** This currently immediately replaces the built-in local asset store -that comes with SilverStripe with one based on S3. Any files that had previously -been uploaded to an existing asset store will be unavailable (though they won't -be lost - just run `composer remove silverstripe/s3` to remove the module and -restore access). - ## Configuration Assets are classed as either 'public' or 'protected' by SilverStripe. Public @@ -132,22 +129,22 @@ Make sure you replace `` below with the appropriate values. ```json { - "Policy": { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AddPerm", - "Effect": "Allow", - "Principal": "*", - "Action": "s3:GetObject", - "Resource": "arn:aws:s3:::/public/*" - } - ] - } + "Policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AddPerm", + "Effect": "Allow", + "Principal": "*", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::/public/*" + } + ] + } } ``` -If you are utilising a CloudFront distribution for your public assets, you will +If you are utilizing a CloudFront distribution for your public assets, you will have the option of securing your S3 bucket against all public access while still allowing access to your `public` files via your CloudFront distribution and access to your `protected` files via signed URLs. @@ -158,6 +155,31 @@ Read [Setting up a local sandbox for developing the Silverstripe S3 module](doc/en/setting-local-dev-environment.md) if you wish to do some local development. +### Performance + +This module comes with a basic in-memory cache for calls to S3. It is highly +recommended to add an additional layer of caching to achieve the best results. + +See https://docs.silverstripe.org/en/5/developer_guides/performance/caching/ for +more information. + +```yaml +Name: silverstripes3-flysystem-memcached +After: + - "#silverstripes3-flysystem" +--- +SilverStripe\Core\Injector\Injector: + MemcachedClient: + class: "Memcached" + calls: + - [addServer, ["localhost", 11211]] + MemcachedCacheFactory: + class: 'SilverStripe\Core\Cache\MemcachedCacheFactory' + constructor: + client: "%$MemcachedClient" + SilverStripe\Core\Cache\CacheFactory: "%$MemcachedCacheFactory" +``` + ## Uninstalling -- Run `composer remove silverstripe/s3` to remove the module. +Run `composer remove silverstripe/s3` to remove the module. diff --git a/_config/assets.yml b/_config/assets.yml index ac3aed4..311a1bc 100644 --- a/_config/assets.yml +++ b/_config/assets.yml @@ -3,74 +3,46 @@ Name: silverstripes3-flysystem Only: envvarset: AWS_BUCKET_NAME After: - - '#assetsflysystem' + - "#assetsflysystem" --- SilverStripe\Core\Injector\Injector: Aws\S3\S3Client: constructor: configuration: - region: '`AWS_REGION`' + region: "`AWS_REGION`" version: latest + Psr\SimpleCache\CacheInterface.s3Cache: + factory: SilverStripe\Core\Cache\CacheFactory + constructor: + namespace: "s3Cache" League\Flysystem\Adapter\Local: class: League\Flysystem\Adapter\Local constructor: - root: '`TEMP_PATH`' - - SilverStripe\S3\Adapter\PublicAdapter: - constructor: - s3Client: '%$Aws\S3\S3Client' - bucket: '`AWS_BUCKET_NAME`' - prefix: '`AWS_PUBLIC_BUCKET_PREFIX`' - Symfony\Component\Cache\Adapter\FilesystemAdapter.public: - class: Symfony\Component\Cache\Adapter\FilesystemAdapter - constructor: - namespace: 'flysystem-public' - expire: 2592000 + root: "`TEMP_PATH`" SilverStripe\Assets\Flysystem\PublicAdapter: - class: SilverStripe\S3\Adapter\PublicCachedAdapter - constructor: - adapter: '%$SilverStripe\S3\Adapter\PublicAdapter' - cache: '%$Symfony\Component\Cache\Adapter\FilesystemAdapter.public' - League\Flysystem\Filesystem.public: - class: SilverStripe\Assets\Flysystem\Filesystem - constructor: - FilesystemAdapter: '%$SilverStripe\Assets\Flysystem\PublicAdapter' - FilesystemConfig: - visibility: public - - SilverStripe\S3\Adapter\ProtectedAdapter: + class: SilverStripe\S3\Adapter\PublicAdapter constructor: s3Client: '%$Aws\S3\S3Client' - bucket: '`AWS_BUCKET_NAME`' - prefix: '`AWS_PROTECTED_BUCKET_PREFIX`' - Symfony\Component\Cache\Adapter\FilesystemAdapter.protected: - class: Symfony\Component\Cache\Adapter\FilesystemAdapter - constructor: - namespace: 'flysystem-protected' - expire: 2592000 + bucket: "`AWS_BUCKET_NAME`" + prefix: "`AWS_PUBLIC_BUCKET_PREFIX`" SilverStripe\Assets\Flysystem\ProtectedAdapter: - class: SilverStripe\S3\Adapter\ProtectedCachedAdapter + class: SilverStripe\S3\Adapter\ProtectedAdapter constructor: - adapter: '%$SilverStripe\S3\Adapter\ProtectedAdapter' - cache: '%$Symfony\Component\Cache\Adapter\FilesystemAdapter.protected' - League\Flysystem\Filesystem.protected: - class: SilverStripe\Assets\Flysystem\Filesystem - constructor: - FilesystemAdapter: '%$SilverStripe\Assets\Flysystem\ProtectedAdapter' - FilesystemConfig: - visibility: private + s3Client: '%$Aws\S3\S3Client' + bucket: "`AWS_BUCKET_NAME`" + prefix: "`AWS_PROTECTED_BUCKET_PREFIX`" --- Name: silverstripes3-assetscore Only: envvarset: AWS_BUCKET_NAME After: - - '#assetsflysystem' - - '#assetscore' + - "#assetsflysystem" + - "#assetscore" --- SilverStripe\Core\Injector\Injector: # Define our SS asset backend SilverStripe\Assets\Storage\AssetStore: - class: SilverStripe\Assets\Flysystem\FlysystemAssetStore + class: SilverStripe\S3\S3FlysystemAssetStore properties: PublicFilesystem: '%$League\Flysystem\Filesystem.public' ProtectedFilesystem: '%$League\Flysystem\Filesystem.protected' @@ -81,22 +53,21 @@ SilverStripe\Core\Injector\Injector: SilverStripe\View\Requirements_Backend: properties: AssetHandler: '%$SilverStripe\Assets\Storage\GeneratedAssetHandler' - --- Name: silverstripes3-cdn Only: envvarset: AWS_PUBLIC_CDN_PREFIX After: - - '#assetsflysystem' - - '#silverstripes3-flysystem' + - "#assetsflysystem" + - "#silverstripes3-flysystem" --- SilverStripe\Core\Injector\Injector: SilverStripe\S3\Adapter\PublicAdapter: class: SilverStripe\S3\Adapter\PublicCDNAdapter constructor: s3Client: '%$Aws\S3\S3Client' - bucket: '`AWS_BUCKET_NAME`' - prefix: '`AWS_PUBLIC_BUCKET_PREFIX`' + bucket: "`AWS_BUCKET_NAME`" + prefix: "`AWS_PUBLIC_BUCKET_PREFIX`" visibility: null mimeTypeDetector: null - cdnPrefix: '`AWS_PUBLIC_CDN_PREFIX`' + cdnPrefix: "`AWS_PUBLIC_CDN_PREFIX`" diff --git a/composer.json b/composer.json index 2f065b6..e6ae9fc 100644 --- a/composer.json +++ b/composer.json @@ -17,9 +17,8 @@ ], "require": { "silverstripe/framework": "^5", - "league/flysystem-aws-s3-v3": "^3.0", - "silverstripe/vendor-plugin": "^2", - "jgivoni/flysystem-cache-adapter": "^3.0" + "league/flysystem-aws-s3-v3": "^3", + "silverstripe/vendor-plugin": "^2" }, "autoload": { "psr-4": { diff --git a/doc/en/setting-local-dev-environment.md b/doc/en/setting-local-dev-environment.md index 7c82e5b..879be80 100644 --- a/doc/en/setting-local-dev-environment.md +++ b/doc/en/setting-local-dev-environment.md @@ -1,25 +1,37 @@ # Setting up a local sandbox for developing the Silverstripe S3 module -This article will guide you through configuring a local project to do development work on the `silverstripe/s3` module. +This article will guide you through configuring a local project to do +development work on the `silverstripe/s3` module. -**This set up has not been optimised or hardened for a production environment.** It may be sub-optimal or it may be too permissive for a production system. +**This set up has not been optimised or hardened for a production environment.** +It may be sub-optimal or it may be too permissive for a production system. ## Setting up a bucket -In this step, we create a publicly accessible S3 bucket. **Any file you publish on your test site will be accessible on the internet.** So be careful what test files you upload in your sandbox. -* [Access the AWS S3 console](https://s3.console.aws.amazon.com/s3/home). -* Create a new bucket. - * Note down your bucket name and region. -* Uncheck _Block all public access_ and make sure all the other sub-options are also unchecked. - * A scary warning will appear. Check the warning to acknowledge you are accepting the risk. -* Leave the other options as-is and complete the bucket creation. +In this step, we create a publicly accessible S3 bucket. **Any file you publish +on your test site will be accessible on the internet.** So be careful what test +files you upload in your sandbox. -## Creating credentials to access your bucket -In this step, we create an AWS user and grant it the permision to read and write files to our bucket. +- [Access the AWS S3 console](https://s3.console.aws.amazon.com/s3/home). +- Create a new bucket. + - Note down your bucket name and region. +- Uncheck _Block all public access_ and make sure all the other sub-options are + also unchecked. + - A scary warning will appear. Check the warning to acknowledge you are + accepting the risk. +- Leave the other options as-is and complete the bucket creation. + +## Creating credentials to access your bucket + +In this step, we create an AWS user and grant it the permision to read and write +files to our bucket. + +- [Acces the AWS _Identity and Access Management_ + console](https://console.aws.amazon.com/iam/home). +- Navigate to the _Policies_ section. +- Create a new policiy with the following JSON code, substituting + `my-silverstripe-bucket` with the actual name of your bucket. -* [Acces the AWS _Identity and Access Management_ console](https://console.aws.amazon.com/iam/home). -* Navigate to the _Policies_ section. -* Create a new policiy with the following JSON code, substituting `my-silverstripe-bucket` with the actual name of your bucket. ```json { "Version": "2012-10-17", @@ -39,26 +51,34 @@ In this step, we create an AWS user and grant it the permision to read and write ] } ``` - * Review your policy and give it a suitable name. - * Save your policy by cliking the _Create policy_ button. -* Navigate to the _Users_ section of the AWS _Identity and Access Management_ console. -* Add a user. - * Give the user a name. - * Check _Programmatic access_ under _Access type_. - * Click on _Next: Permissions_ - * Attach the policy you just created to the user. - * Keep clicking next until your user has been created. -* On the final screen, you'll be provided a _Access key ID_ and a _Secret access key_. - * Note down those values for later. - + +- Review your policy and give it a suitable name. +- Save your policy by cliking the _Create policy_ button. +- Navigate to the _Users_ section of the AWS _Identity and Access Management_ + console. +- Add a user. + - Give the user a name. + - Check _Programmatic access_ under _Access type_. + - Click on _Next: Permissions_ + - Attach the policy you just created to the user. + - Keep clicking next until your user has been created. +- On the final screen, you'll be provided a _Access key ID_ and a _Secret access + key_. + - Note down those values for later. + ## Creating a sandbox project -In this step, you will create a Silverstripe CMS sandbox project that will be configured to save files to your S3 bucket with the credentials that you just generated. You can run your project in whichever test environment suits you. +In this step, you will create a Silverstripe CMS sandbox project that will be +configured to save files to your S3 bucket with the credentials that you just +generated. You can run your project in whichever test environment suits you. + +- Create a new Silverstripe CMS project with `composer create-project +silverstripe/installer s3-sandbox 4.x-dev` +- From the project folder, run this command to install the development branch of + the module `composer require silverstripe/s3 dev-master` +- Configure a web server to run your sandbox project. +- Configure an `.env` file with these values, adapting them as need be. -* Create a new Silverstripe CMS project with `composer create-project silverstripe/installer s3-sandbox 4.x-dev` -* From the project folder, run this command to install the development branch of the module `composer require silverstripe/s3 dev-master` -* Configure a web server to run your sandbox project. -* Configure an `.env` file with these values, adapting them as need be. ```dotenv # These values should be configured to match your local environment SS_BASE_URL="http://s3-sandbox.local" @@ -73,7 +93,7 @@ SS_ENVIRONMENT_TYPE="dev" SS_DEFAULT_ADMIN_USERNAME="admin" SS_DEFAULT_ADMIN_PASSWORD="admin" -# You should have generated these values in the first step +# You should have generated these values in the first step AWS_REGION="ap-southeast-2" AWS_BUCKET_NAME="YOURBUCKETNAME" @@ -81,24 +101,27 @@ AWS_BUCKET_NAME="YOURBUCKETNAME" AWS_ACCESS_KEY_ID="ACCESID" AWS_SECRET_ACCESS_KEY="SECRET" ``` -* Add the following YML configuration to a file under `app/_config`. + +- Add the following YML configuration to a file under `app/_config`. + ```yml --- Only: envvarset: AWS_BUCKET_NAME After: - - '#silverstripes3-flysystem' + - "#silverstripes3-flysystem" --- SilverStripe\Core\Injector\Injector: - Aws\S3\S3Client: - constructor: - configuration: - region: '`AWS_REGION`' - version: latest - credentials: - key: '`AWS_ACCESS_KEY_ID`' - secret: '`AWS_SECRET_ACCESS_KEY`' + Aws\S3\S3Client: + constructor: + configuration: + region: "`AWS_REGION`" + version: latest + credentials: + key: "`AWS_ACCESS_KEY_ID`" + secret: "`AWS_SECRET_ACCESS_KEY`" ``` -* Run this command to build your sandbox project. `vendor/bin/sake dev/build` + +- Run this command to build your sandbox project. `vendor/bin/sake dev/build` Your site should be functional by this point. diff --git a/src/Adapter/CacheAdapter.php b/src/Adapter/CacheAdapter.php deleted file mode 100644 index a1ae1e4..0000000 --- a/src/Adapter/CacheAdapter.php +++ /dev/null @@ -1,11 +0,0 @@ -adapter; - } -} diff --git a/src/Adapter/CachedAwsS3V3Adapter.php b/src/Adapter/CachedAwsS3V3Adapter.php new file mode 100644 index 0000000..9662912 --- /dev/null +++ b/src/Adapter/CachedAwsS3V3Adapter.php @@ -0,0 +1,429 @@ +getCacheItem($path); + + if ($item && isset($item->extraMetadata()['fileExists'])) { + return $item->extraMetadata()['fileExists']; + } else if ($item && isset($item->extraMetadata()['directoryExists'])) { + return false; + } + + $fileExists = parent::fileExists($path); + + $state = new FileAttributes( + path: $fileExists, + extraMetadata: ['fileExists' => $fileExists] + ); + + $this->saveCacheItem($path, $state); + + return $fileExists; + } + + /** + * @inheritdoc + */ + public function directoryExists(string $path): bool + { + $item = $this->getCacheItem($path); + + if ($item && isset($item->extraMetadata()['directoryExists'])) { + return $item->extraMetadata()['directoryExists']; + } else if ($item && isset($item->extraMetadata()['fileExists'])) { + return false; + } + + $directoryExists = parent::directoryExists($path); + + $state = new FileAttributes( + path: $path, + extraMetadata: ['directoryExists' => $directoryExists] + ); + + $this->saveCacheItem($path, $state); + + return $directoryExists ?? \false; + } + + + public function publicUrl(string $path, Config $config): string + { + $item = $this->getCacheItem($path); + + if ($item && !empty($item->extraMetadata()['publicUrl'])) { + return $item->extraMetadata()['publicUrl']; + } + + $url = parent::publicUrl($path, $config); + + if ($item) { + $state = self::mergeFileAttributes( + fileAttributesBase: $item, + fileAttributesExtension: new FileAttributes( + path: $path, + extraMetadata: ['publicUrl' => $url] + ), + ); + } else { + $state = new FileAttributes( + path: $path, + extraMetadata: ['publicUrl' => $url] + ); + } + + $this->saveCacheItem($path, $state); + + return $url; + } + + + /** + * @inheritdoc + */ + public function write(string $path, string $contents, Config $config): void + { + parent::write($path, $contents, $config); + + $this->purgeCachePath($path); + } + + /** + * @inheritdoc + */ + public function writeStream(string $path, $contents, Config $config): void + { + parent::writeStream($path, $contents, $config); + + $this->purgeCachePath($path); + } + + /** + * @inheritdoc + */ + public function read(string $path): string + { + try { + $contents = parent::read($path); + $item = $this->getCacheItem($path); + } catch (UnableToReadFile $e) { + $this->purgeCachePath($path); + + throw $e; + } + + if (isset($item) && $item instanceof FileAttributes) { + $fileAttributes = self::mergeFileAttributes( + fileAttributesBase: $item, + fileAttributesExtension: new FileAttributes( + path: $path, + ), + ); + } else { + $fileSize = parent::fileSize($path); + + $fileAttributes = new FileAttributes( + path: $path, + fileSize: $fileSize ?? 0 + ); + } + + $this->saveCacheItem($path, $fileAttributes); + + return $contents; + } + + /** + * @inheritdoc + */ + public function readStream(string $path) + { + try { + $resource = parent::readStream($path); + } catch (UnableToReadFile $e) { + $this->purgeCachePath($path); + + throw $e; + } + + $item = $this->getCacheItem($path); + + if ($item && $item instanceof FileAttributes) { + $fileAttributes = self::mergeFileAttributes( + fileAttributesBase: $item, + fileAttributesExtension: new FileAttributes( + path: $path, + ), + ); + } else { + $fileAttributes = new FileAttributes( + path: $path, + ); + } + + + $this->saveCacheItem($path, $fileAttributes); + + return $resource; + } + + /** + * @inheritdoc + */ + public function delete(string $path): void + { + try { + parent::delete($path); + } finally { + $this->purgeCachePath($path); + } + } + + /** + * @inheritdoc + */ + public function deleteDirectory(string $path): void + { + try { + foreach (parent::listContents($path, true) as $storageAttributes) { + /** @var StorageAttributes $storageAttributes */ + $this->purgeCachePath($storageAttributes->path()); + } + + parent::deleteDirectory($path); + } finally { + $this->purgeCachePath($path); + } + } + + /** + * @inheritdoc + */ + public function createDirectory(string $path, Config $config): void + { + parent::createDirectory($path, $config); + + $this->purgeCachePath($path); + } + + /** + * @inheritdoc + */ + public function setVisibility(string $path, string $visibility): void + { + try { + parent::setVisibility($path, $visibility); + } catch (UnableToSetVisibility $e) { + $this->purgeCachePath($path); + + throw $e; + } + + $attributes = $this->getCacheItem($path); + + if ($attributes) { + $attributes = self::mergeFileAttributes( + fileAttributesBase: $attributes, + fileAttributesExtension: new FileAttributes( + path: $path, + visibility: $visibility, + ), + ); + } else { + $attributes = new FileAttributes( + path: $path, + visibility: $visibility, + ); + } + + $this->saveCacheItem($path, $attributes); + } + + + /** + * @inheritdoc + */ + public function visibility(string $path): FileAttributes + { + return $this->getFileAttributes( + path: $path, + loader: function () use ($path) { + try { + return parent::visibility($path); + } catch (UnableToRetrieveMetadata $e) { + return new FileAttributes($path, null, ''); + } + }, + attributeAccessor: function (FileAttributes $fileAttributes) { + return $fileAttributes->visibility(); + }, + ); + } + + /** + * @inheritdoc + */ + public function mimeType(string $path): FileAttributes + { + return $this->getFileAttributes( + path: $path, + loader: function () use ($path) { + try { + return parent::mimeType($path); + } catch (UnableToRetrieveMetadata $e) { + return new FileAttributes($path, null, null, null, ''); + } + }, + attributeAccessor: function (FileAttributes $fileAttributes) { + return $fileAttributes->mimeType(); + }, + ); + } + + /** + * @inheritdoc + */ + public function lastModified(string $path): FileAttributes + { + return $this->getFileAttributes( + path: $path, + loader: function () use ($path) { + try { + return parent::lastModified($path); + } catch (UnableToRetrieveMetadata $e) { + return new FileAttributes($path, null, null, time(), null); + } + }, + attributeAccessor: function (FileAttributes $fileAttributes) { + return $fileAttributes->lastModified(); + }, + ); + } + + /** + * @inheritdoc + */ + public function fileSize(string $path): FileAttributes + { + return $this->getFileAttributes( + path: $path, + loader: function () use ($path) { + return parent::fileSize($path); + try { + return parent::fileSize($path); + } catch (UnableToRetrieveMetadata $e) { + return new FileAttributes($path, 0); + } + }, + attributeAccessor: function (FileAttributes $fileAttributes) { + return $fileAttributes->fileSize(); + }, + ); + } + + /** + * @inheritdoc + */ + public function checksum(string $path, Config $config): string + { + $algo = $config->get('checksum_algo'); + $metadataKey = isset($algo) ? 'checksum_' . $algo : 'checksum'; + + $attributeAccessor = function (StorageAttributes $storageAttributes) use ($metadataKey) { + $eTag = $storageAttributes->extraMetadata()['ETag'] ?? \null; + if (isset($eTag)) { + $checksum = trim($eTag, '" '); + } + + return $checksum ?? $storageAttributes->extraMetadata()[$metadataKey] ?? \null; + }; + + try { + $fileAttributes = $this->getFileAttributes( + path: $path, + loader: function () use ($path, $config, $metadataKey) { + // This part is "mirrored" from FileSystem class to provide the fallback mechanism + // and be able to cache the result + try { + $checksum = $this->checksum($path, $config); + } catch (ChecksumAlgoIsNotSupported) { + $checksum = $this->calculateChecksumFromStream($path, $config); + } + + return new FileAttributes($path, extraMetadata: [$metadataKey => $checksum]); + }, + attributeAccessor: $attributeAccessor + ); + } catch (RuntimeException $e) { + return ''; + } + + return $attributeAccessor($fileAttributes); + } + + + /** + * @inheritdoc + */ + public function move(string $source, string $destination, Config $config): void + { + $this->purgeCachePath($source); + $this->purgeCachePath($destination); + + try { + parent::move($source, $destination, $config); + } catch (UnableToMoveFile $e) { + throw $e; + } + } + + /** + * @inheritdoc + */ + public function copy(string $source, string $destination, Config $config): void + { + $this->purgeCachePath($source); + $this->purgeCachePath($destination); + + try { + parent::copy($source, $destination, $config); + } catch (UnableToCopyFile $e) { + throw $e; + } + } + + + public static function flush() + { + Injector::inst()->get(CacheInterface::class . '.s3Cache')->clear(); + } +} diff --git a/src/Adapter/ProtectedAdapter.php b/src/Adapter/ProtectedAdapter.php index 67ce012..feb8e3f 100755 --- a/src/Adapter/ProtectedAdapter.php +++ b/src/Adapter/ProtectedAdapter.php @@ -4,7 +4,6 @@ use Aws\S3\S3Client; use InvalidArgumentException; -use League\Flysystem\AwsS3V3\AwsS3V3Adapter; use League\Flysystem\AwsS3V3\VisibilityConverter; use League\Flysystem\Config; use League\MimeTypeDetection\MimeTypeDetector; @@ -13,7 +12,7 @@ /** * An adapter that allows the use of AWS S3 to store and transmit assets rather than storing them locally. */ -class ProtectedAdapter extends AwsS3V3Adapter implements SilverstripeProtectedAdapter +class ProtectedAdapter extends CachedAwsS3V3Adapter implements SilverstripeProtectedAdapter { /** * Pre-signed request expiration time in seconds, or relative string @@ -27,10 +26,14 @@ public function __construct(S3Client $client, $bucket, $prefix = '', VisibilityC if (!$bucket) { throw new InvalidArgumentException("AWS_BUCKET_NAME environment variable not set"); } + if (!$prefix) { $prefix = 'protected'; } + parent::__construct($client, $bucket, $prefix, $visibility, $mimeTypeDetector, $options); + + $this->setCachePrefix('protected'); } /** @@ -41,6 +44,7 @@ public function getExpiry() return $this->expiry; } + /** * Set expiry. Supports either number of seconds (in int) or * a literal relative string. @@ -54,6 +58,7 @@ public function setExpiry($expiry) return $this; } + /** * @param string $path * @@ -62,18 +67,12 @@ public function setExpiry($expiry) public function getProtectedUrl($path) { $dt = new \DateTime(); - if(is_string($this->getExpiry())){ + if (is_string($this->getExpiry())) { $dt = $dt->setTimestamp(strtotime($this->getExpiry())); } else { - $dt = $dt->setTimestamp(strtotime('+'.$this->getExpiry().' seconds')); + $dt = $dt->setTimestamp(strtotime('+' . $this->getExpiry() . ' seconds')); } return $this->temporaryUrl($path, $dt, new Config()); } - - public function getVisibility($path) - { - // Save an API call - return ['path' => $path, 'visibility' => self::VISIBILITY_PRIVATE]; - } } diff --git a/src/Adapter/ProtectedCachedAdapter.php b/src/Adapter/ProtectedCachedAdapter.php deleted file mode 100755 index 25fef77..0000000 --- a/src/Adapter/ProtectedCachedAdapter.php +++ /dev/null @@ -1,13 +0,0 @@ -getAdapter()->getProtectedUrl($path); - } -} diff --git a/src/Adapter/PublicAdapter.php b/src/Adapter/PublicAdapter.php index 62c3d96..885654f 100755 --- a/src/Adapter/PublicAdapter.php +++ b/src/Adapter/PublicAdapter.php @@ -4,22 +4,24 @@ use Aws\S3\S3Client; use InvalidArgumentException; -use League\Flysystem\AwsS3V3\AwsS3V3Adapter; use League\Flysystem\AwsS3V3\VisibilityConverter; use League\Flysystem\Config; use League\MimeTypeDetection\MimeTypeDetector; use SilverStripe\Assets\Flysystem\PublicAdapter as SilverstripePublicAdapter; -class PublicAdapter extends AwsS3V3Adapter implements SilverstripePublicAdapter +class PublicAdapter extends CachedAwsS3V3Adapter implements SilverstripePublicAdapter { + public function __construct(S3Client $client, $bucket, $prefix = '', VisibilityConverter $visibility = null, MimeTypeDetector $mimeTypeDetector = null, array $options = []) { if (!$bucket) { throw new InvalidArgumentException("AWS_BUCKET_NAME environment variable not set"); } + if (!$prefix) { $prefix = 'public'; } + parent::__construct($client, $bucket, $prefix, $visibility, $mimeTypeDetector, $options); } diff --git a/src/Adapter/PublicCDNAdapter.php b/src/Adapter/PublicCDNAdapter.php index 0681f4b..37587a2 100644 --- a/src/Adapter/PublicCDNAdapter.php +++ b/src/Adapter/PublicCDNAdapter.php @@ -20,8 +20,16 @@ class PublicCDNAdapter extends PublicAdapter implements SilverstripePublicAdapte protected $cdnAssetsDir; - public function __construct(S3Client $client, $bucket, $prefix = '', VisibilityConverter $visibility = null, MimeTypeDetector $mimeTypeDetector = null, $cdnPrefix = '', array $options = [], $cdnAssetsDir = '') - { + public function __construct( + S3Client $client, + $bucket, + $prefix = '', + VisibilityConverter $visibility = null, + MimeTypeDetector $mimeTypeDetector = null, + $cdnPrefix = '', + array $options = [], + $cdnAssetsDir = '' + ) { $this->cdnPrefix = $cdnPrefix; $this->cdnAssetsDir = $cdnAssetsDir ? $cdnAssetsDir : ASSETS_DIR; parent::__construct($client, $bucket, $prefix, $visibility, $mimeTypeDetector, $options); diff --git a/src/Adapter/PublicCachedAdapter.php b/src/Adapter/PublicCachedAdapter.php deleted file mode 100755 index 2f0c62f..0000000 --- a/src/Adapter/PublicCachedAdapter.php +++ /dev/null @@ -1,13 +0,0 @@ -getAdapter()->getPublicUrl($path); - } -} diff --git a/src/Adapter/TinyMceAdapter.php b/src/Adapter/TinyMceAdapter.php index f4ad8d6..694912c 100644 --- a/src/Adapter/TinyMceAdapter.php +++ b/src/Adapter/TinyMceAdapter.php @@ -1,9 +1,9 @@ cachePrefix = $prefix; + + return $this; + } + + + protected function getCache(): CacheInterface + { + if (!$this->cache) { + $this->cache = Injector::inst()->get(CacheInterface::class . '.s3Cache'); + } + + return $this->cache; + } + + + protected function getCacheItem(string $path): ?FileAttributes + { + if (!$this->enabled) { + return null; + } + + $key = $this->getCacheItemKey($path); + $item = $this->getCache()->get($key); + + return $item; + } + + + protected function saveCacheItem(string $path, FileAttributes $cacheItem): self + { + $key = $this->getCacheItemKey($path); + + if ($this->enabled) { + $this->getCache()->set($key, $cacheItem); + } + + return $this; + } + + + public function purgeCachePath(string $path): self + { + $key = $this->getCacheItemKey($path); + $this->getCache()->delete($key); + + return $this; + } + + + protected function getCacheItemKey(string $path): string + { + return md5($this->cachePrefix . '-' . $path); + } + + + /** + * Returns a new FileAttributes with all properties from $fileAttributesExtension + * overriding existing properties from $fileAttributesBase (with the exception of path) + * + * For extraMetadata, each individual element in the array is also merged + */ + protected static function mergeFileAttributes( + FileAttributes $fileAttributesBase, + FileAttributes $fileAttributesExtension + ): FileAttributes { + return new FileAttributes( + path: $fileAttributesBase->path(), + fileSize: $fileAttributesExtension->fileSize() ?? + $fileAttributesBase->fileSize(), + visibility: $fileAttributesExtension->visibility() ?? + $fileAttributesBase->visibility(), + lastModified: $fileAttributesExtension->lastModified() ?? + $fileAttributesBase->lastModified(), + mimeType: $fileAttributesExtension->mimeType() ?? + $fileAttributesBase->mimeType(), + extraMetadata: array_merge( + $fileAttributesBase->extraMetadata(), + $fileAttributesExtension->extraMetadata() + ), + ); + } + + /** + * Returns FileAttributes from cache if desired attribute is found, + * or loads the desired missing attribute from the adapter and merges it with the cached attributes. + * + * @param Closure $loader Returns FileAttributes with the desired attribute loaded from adapter + * @param Closure $attributeAccessor Returns value of desired attribute from cached item + */ + protected function getFileAttributes( + string $path, + Closure $loader, + Closure $attributeAccessor, + ): FileAttributes { + $fileAttributes = $this->getCacheItem($path); + + if ($fileAttributes) { + if (!$fileAttributes instanceof FileAttributes) { + $fileAttributes = new FileAttributes( + path: $path, + ); + } + } else { + $fileAttributes = new FileAttributes( + path: $path, + ); + } + + if ($attributeAccessor($fileAttributes) === null) { + $fileAttributesExtension = $loader(); + + if ($fileAttributesExtension) { + $fileAttributes = self::mergeFileAttributes( + fileAttributesBase: $fileAttributes, + fileAttributesExtension: $fileAttributesExtension, + ); + } + + $this->saveCacheItem($path, $fileAttributes); + } + + return $fileAttributes; + } +} diff --git a/src/S3FlysystemAssetStore.php b/src/S3FlysystemAssetStore.php new file mode 100644 index 0000000..7966e83 --- /dev/null +++ b/src/S3FlysystemAssetStore.php @@ -0,0 +1,180 @@ +getPublicFilesystem(), + $this->getPublicResolutionStrategy(), + self::VISIBILITY_PUBLIC + ]; + + $protectedSet = [ + $this->getProtectedFilesystem(), + $this->getProtectedResolutionStrategy(), + self::VISIBILITY_PROTECTED + ]; + + + foreach ([$publicSet, $protectedSet] as $set) { + try { + list($fs, $strategy, $visibility) = $set; + + // Get a FileID string based on the type of FileID + $fileID = $strategy->buildFileID($parsedFileID); + + if ($fs->has($fileID)) { + $closureParsedFileID = $parsedFileID->setFileID($fileID); + + $response = $callable( + $closureParsedFileID, + $fs, + $strategy, + $visibility + ); + + if ($response !== false) { + return $response; + } + } + } catch (Exception $e) { + // not found + } + } + + return null; + } + + + protected function writeWithCallback($callback, $filename, $hash, $variant = null, $config = []) + { + $this->purgeCaches($filename); + + $result = parent::writeWithCallback($callback, $filename, $hash, $variant, $config); + + $this->purgeCaches($filename); + + return $result; + } + + + public function exists($filename, $hash, $variant = null) + { + if (empty($filename)) { + return false; + } + + $result = $this->applyToFileOnFilesystem( + function (ParsedFileID $parsedFileID, Filesystem $fs, FileResolutionStrategy $strategy) use ($filename) { + $parsedFileID = $strategy->stripVariant($parsedFileID); + + if ($parsedFileID && $originalFileID = $parsedFileID->getFileID()) { + if ($fs->has($originalFileID)) { + return true; + } + } + + return false; + }, + new ParsedFileID($filename, $hash, $variant) + ) ?: false; + + return $result; + } + + + public function getAsURL($filename, $hash, $variant = null, $grant = true) + { + $tuple = new ParsedFileID($filename, $hash, $variant); + + // Check with filesystem this asset exists in + $public = $this->getPublicFilesystem(); + $protected = $this->getProtectedFilesystem(); + + try { + if ($parsedFileID = $this->getPublicResolutionStrategy()->searchForTuple($tuple, $public)) { + /** @var PublicAdapter $publicAdapter */ + $publicAdapter = $public->getAdapter(); + + return $publicAdapter->getPublicUrl($parsedFileID->getFileID()); + } + } catch (Exception $e) { + // + } + + try { + if ($parsedFileID = $this->getProtectedResolutionStrategy()->searchForTuple($tuple, $protected)) { + if ($grant) { + $this->grant($parsedFileID->getFilename(), $parsedFileID->getHash()); + } + /** @var ProtectedAdapter $protectedAdapter */ + $protectedAdapter = $protected->getAdapter(); + return $protectedAdapter->getProtectedUrl($parsedFileID->getFileID()); + } + } catch (Exception $e) { + // + } + + $fileID = $this->getPublicResolutionStrategy()->buildFileID($tuple); + + /** @var PublicAdapter $publicAdapter */ + $publicAdapter = $public->getAdapter(); + + return $publicAdapter->getPublicUrl($fileID); + } + + + public function getVisibility($filename, $hash) + { + // Check with filesystem this asset exists in + $public = $this->getPublicFilesystem(); + $tuple = new ParsedFileID($filename, $hash); + + try { + if ($this->getPublicResolutionStrategy()->searchForTuple($tuple, $public)) { + return self::VISIBILITY_PUBLIC; + } + } catch (Exception $e) { + // + } + + return AssetStore::VISIBILITY_PROTECTED; + } + + + public function publish($filename, $hash) + { + parent::publish($filename, $hash); + + $this->purgeCaches($filename); + } + + + protected function purgeCaches($filename) + { + /** @var CachedAwsS3V3Adapter */ + $public = $this->getPublicFilesystem()->getAdapter(); + $public->purgeCachePath($filename); + + /** @var CachedAwsS3V3Adapter */ + $protected = $this->getProtectedFilesystem()->getAdapter(); + $protected->purgeCachePath($filename); + } +} From 6cd20d6d04ab8ac6f934698e4b157c7188eaea1c Mon Sep 17 00:00:00 2001 From: Joe Date: Wed, 12 Jun 2024 05:29:04 -0500 Subject: [PATCH 2/2] allow cache flushing to be configurable like intervention managers cache --- src/Adapter/CachedAwsS3V3Adapter.php | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/Adapter/CachedAwsS3V3Adapter.php b/src/Adapter/CachedAwsS3V3Adapter.php index 9662912..337566b 100644 --- a/src/Adapter/CachedAwsS3V3Adapter.php +++ b/src/Adapter/CachedAwsS3V3Adapter.php @@ -5,9 +5,8 @@ use League\Flysystem\AwsS3V3\AwsS3V3Adapter; use League\Flysystem\CalculateChecksumFromStream; use League\Flysystem\ChecksumAlgoIsNotSupported; -use League\Flysystem\FileAttributes; -use Silverstripe\S3\Cache\CacheItemsTrait; use League\Flysystem\Config; +use League\Flysystem\FileAttributes; use League\Flysystem\StorageAttributes; use League\Flysystem\UnableToCopyFile; use League\Flysystem\UnableToMoveFile; @@ -16,14 +15,25 @@ use League\Flysystem\UnableToSetVisibility; use Psr\SimpleCache\CacheInterface; use RuntimeException; +use SilverStripe\Core\Config\Config as SSConfig; +use SilverStripe\Core\Config\Configurable; use SilverStripe\Core\Flushable; use SilverStripe\Core\Injector\Injector; +use Silverstripe\S3\Cache\CacheItemsTrait; class CachedAwsS3V3Adapter extends AwsS3V3Adapter implements Flushable { use CacheItemsTrait; use CalculateChecksumFromStream; + use Configurable; + /** + * Is cache flushing enabled? + * + * @config + * @var boolean + */ + private static $flush_enabled = true; /** * @inheritdoc @@ -87,7 +97,7 @@ public function publicUrl(string $path, Config $config): string $url = parent::publicUrl($path, $config); if ($item) { - $state = self::mergeFileAttributes( + $state = CachedAwsS3V3Adapter::mergeFileAttributes( fileAttributesBase: $item, fileAttributesExtension: new FileAttributes( path: $path, @@ -142,7 +152,7 @@ public function read(string $path): string } if (isset($item) && $item instanceof FileAttributes) { - $fileAttributes = self::mergeFileAttributes( + $fileAttributes = CachedAwsS3V3Adapter::mergeFileAttributes( fileAttributesBase: $item, fileAttributesExtension: new FileAttributes( path: $path, @@ -178,7 +188,7 @@ public function readStream(string $path) $item = $this->getCacheItem($path); if ($item && $item instanceof FileAttributes) { - $fileAttributes = self::mergeFileAttributes( + $fileAttributes = CachedAwsS3V3Adapter::mergeFileAttributes( fileAttributesBase: $item, fileAttributesExtension: new FileAttributes( path: $path, @@ -251,7 +261,7 @@ public function setVisibility(string $path, string $visibility): void $attributes = $this->getCacheItem($path); if ($attributes) { - $attributes = self::mergeFileAttributes( + $attributes = CachedAwsS3V3Adapter::mergeFileAttributes( fileAttributesBase: $attributes, fileAttributesExtension: new FileAttributes( path: $path, @@ -424,6 +434,8 @@ public function copy(string $source, string $destination, Config $config): void public static function flush() { - Injector::inst()->get(CacheInterface::class . '.s3Cache')->clear(); + if (SSConfig::inst()->get(static::class, 'flush_enabled')) { + Injector::inst()->get(CacheInterface::class . '.s3Cache')->clear(); + } } }