From ed9ac08fbbac7b2164a6474c9190bc6030f1ca7a Mon Sep 17 00:00:00 2001 From: EarthlingDavey <15802017+EarthlingDavey@users.noreply.github.com> Date: Thu, 28 Mar 2024 14:39:20 +0000 Subject: [PATCH] Add minio & caddy for local S3. (#490) * Add minio & caddy for local S3. * Trim unused code and add optional comment * Add comments and consistent casing. * Fix typos and anon function for as3cf_aws_s3_console_url_prefix_param --- .env.example | 10 ++ Dockerfile | 8 +- composer.json | 3 - composer.lock | 56 +------- config/wp-offload-media.php | 12 +- deploy/config/local/Caddyfile | 2 +- docker-compose.yml | 47 +++++++ public/app/themes/clarity/functions.php | 1 + ...zon-s3-and-cloudfront-tweaks-for-minio.php | 121 ++++++++++++++++++ 9 files changed, 193 insertions(+), 67 deletions(-) create mode 100644 public/app/themes/clarity/inc/amazon-s3-and-cloudfront-tweaks-for-minio.php diff --git a/.env.example b/.env.example index a8b6c6968..0ce296f6d 100644 --- a/.env.example +++ b/.env.example @@ -40,6 +40,16 @@ LOCAL_SSH_PASSWORD=ssh-password # General template and api-key for gov notify GOV_NOTIFY_API_KEY="" +# AWS / WP Offload Media +# - Minio (starts with minio for AmazonS3AndCloudFrontTweaks) +S3_CUSTOM_DOMAIN="minio.${SERVER_NAME}" +# - S3 +S3_BUCKET_NAME=test-bucket +AWS_ACCESS_KEY_ID=myaccesskey +AWS_SECRET_ACCESS_KEY=myaccesssecret +# - CloudFront (optional) +DELIVERY_DOMAIN="cdn.${SERVER_NAME}" + # Generate your keys here: https://roots.io/salts.html AUTH_KEY='generate-key' SECURE_AUTH_KEY='generate-key' diff --git a/Dockerfile b/Dockerfile index 08630085b..410edf2f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,13 +43,15 @@ RUN chmod +x /var/www/html/composer-auth.sh && \ # non-root USER 82 -COPY ./composer.json /var/www/html/composer.json -RUN composer install --no-dev --no-scripts --no-autoloader +COPY composer.* /var/www/html/ -COPY . . RUN composer install --no-dev RUN composer dump-autoload -o +# Copy all of the files here for now. +# We can move this later. +COPY . . + ARG regex_files='\(htm\|html\|js\|css\|png\|jpg\|jpeg\|gif\|ico\|svg\|webmanifest\)' ARG regex_path='\(app\/themes\/clarity\/error\-pages\|app\/mu\-plugins\|app\/plugins\|wp\)' RUN mkdir -p ./vendor-assets && \ diff --git a/composer.json b/composer.json index e92d4b0da..8f3b47179 100644 --- a/composer.json +++ b/composer.json @@ -76,9 +76,6 @@ "wpackagist-plugin/amazon-s3-and-cloudfront": "^3.2" }, "require-dev": { - "wpackagist-plugin/query-monitor": "^3.15.0", - "wpackagist-plugin/debug-bar": "^1.1.0", - "wpackagist-plugin/debug-bar-elasticpress": "^3.1.0", "squizlabs/php_codesniffer": "^3.0.2" }, "extra": { diff --git a/composer.lock b/composer.lock index 70f439954..837832a5c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "865c0b8c813ac2f7cde9354220a48528", + "content-hash": "687ada9162cabca4eb99b794d84f9203", "packages": [ { "name": "acf/advanced-custom-fields-pro", @@ -3475,60 +3475,6 @@ } ], "time": "2024-02-16T15:06:51+00:00" - }, - { - "name": "wpackagist-plugin/debug-bar", - "version": "1.1.6", - "source": { - "type": "svn", - "url": "https://plugins.svn.wordpress.org/debug-bar/", - "reference": "tags/1.1.6" - }, - "dist": { - "type": "zip", - "url": "https://downloads.wordpress.org/plugin/debug-bar.1.1.6.zip" - }, - "require": { - "composer/installers": "^1.0 || ^2.0" - }, - "type": "wordpress-plugin", - "homepage": "https://wordpress.org/plugins/debug-bar/" - }, - { - "name": "wpackagist-plugin/debug-bar-elasticpress", - "version": "3.1.0", - "source": { - "type": "svn", - "url": "https://plugins.svn.wordpress.org/debug-bar-elasticpress/", - "reference": "tags/3.1.0" - }, - "dist": { - "type": "zip", - "url": "https://downloads.wordpress.org/plugin/debug-bar-elasticpress.3.1.0.zip" - }, - "require": { - "composer/installers": "^1.0 || ^2.0" - }, - "type": "wordpress-plugin", - "homepage": "https://wordpress.org/plugins/debug-bar-elasticpress/" - }, - { - "name": "wpackagist-plugin/query-monitor", - "version": "3.15.0", - "source": { - "type": "svn", - "url": "https://plugins.svn.wordpress.org/query-monitor/", - "reference": "tags/3.15.0" - }, - "dist": { - "type": "zip", - "url": "https://downloads.wordpress.org/plugin/query-monitor.3.15.0.zip" - }, - "require": { - "composer/installers": "^1.0 || ^2.0" - }, - "type": "wordpress-plugin", - "homepage": "https://wordpress.org/plugins/query-monitor/" } ], "aliases": [], diff --git a/config/wp-offload-media.php b/config/wp-offload-media.php index dd2c28c58..8798d7c96 100644 --- a/config/wp-offload-media.php +++ b/config/wp-offload-media.php @@ -28,13 +28,13 @@ // Append a timestamped folder to path of files offloaded to bucket to avoid filename clashes and bust CDN cache if updated 'object-versioning' => true, // Delivery Provider ('storage', 'aws', 'do', 'gcp', 'cloudflare', 'keycdn', 'stackpath', 'other') - 'delivery-provider' => env('CLOUDFRONT_URL') ? 'aws' : 'storage', + 'delivery-provider' => env('DELIVERY_DOMAIN') ? 'aws' : 'storage', // Rewrite file URLs to bucket (s3 or cloudfront) 'serve-from-s3' => true, // Use a custom domain (CNAME), not supported when using 'storage' Delivery Provider - 'enable-delivery-domain' => !!env('CLOUDFRONT_URL'), + 'enable-delivery-domain' => !!env('DELIVERY_DOMAIN'), // Custom domain (CNAME), not supported when using 'storage' Delivery Provider - 'delivery-domain' => env('CLOUDFRONT_URL'), + 'delivery-domain' => env('DELIVERY_DOMAIN'), // Enable signed URLs for Delivery Provider that uses separate key pair (currently only 'aws' supported, a.k.a. CloudFront) // 'enable-signed-urls' => false, // Access Key ID for signed URLs (aws only, replace '*') @@ -45,9 +45,9 @@ // Private Prefix for signed URLs (aws only, relative directory, no wildcards) // 'signed-urls-object-prefix' => 'private/', // Serve files over HTTPS - 'force-https' => !!env('CLOUDFRONT_URL'), + 'force-https' => !!env('DELIVERY_DOMAIN'), // Remove the local file version once offloaded to bucket - 'remove-local-file' => false, + 'remove-local-file' => true, // Access Control List for the bucket 'use-bucket-acls' => false, ); @@ -64,4 +64,6 @@ ]); } +Config::define('S3_CUSTOM_DOMAIN', env('S3_CUSTOM_DOMAIN') ?? ''); + Config::define('AS3CF_SETTINGS', serialize($as3_settings)); diff --git a/deploy/config/local/Caddyfile b/deploy/config/local/Caddyfile index e6ec3b4fd..7d996a4f9 100644 --- a/deploy/config/local/Caddyfile +++ b/deploy/config/local/Caddyfile @@ -3,6 +3,6 @@ # e.g. Request: http://cdn.justice.docker/uploads/2024/02/xyz.jpg # proxies to : http://minio:9000/bucket-name/uploads/2024/02/xyz.jpg -:2019 +:80 rewrite * /{$S3_BUCKET_NAME}{uri} reverse_proxy minio:9000 diff --git a/docker-compose.yml b/docker-compose.yml index beefcbb82..03164fb67 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ volumes: opensearch-data: database: driver: local + minio_storage: ~ services: php-fpm: @@ -19,6 +20,9 @@ services: depends_on: - mariadb - opensearch + links: + - "cdn:cdn.${SERVER_NAME}" + - "minio:minio.${SERVER_NAME}" nginx: build: @@ -119,3 +123,46 @@ services: DISABLE_SECURITY_DASHBOARDS_PLUGIN: true # disables security dashboards plugin in OpenSearch Dashboards depends_on: - opensearch + + minio: + image: minio/minio + ports: + - "9000:9000" # The AWS S3 compatible API. + - "9001:9001" # The Minio web console. + volumes: + - minio_storage:/data + environment: + MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID} + MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY} + VIRTUAL_HOST: minio.${SERVER_NAME} + VIRTUAL_PORT: 9000 + command: server --console-address ":9001" /data + healthcheck: + test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1 + start_period: 5s + interval: 10s + timeout: 5s + retries: 2 + + minio-init: + image: minio/mc + depends_on: + - minio + entrypoint: | + /bin/sh -c " + mc config host add justice-gov-uk http://minio:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY} + mc mb justice-gov-uk/${S3_BUCKET_NAME} + mc anonymous set download justice-gov-uk/${S3_BUCKET_NAME}; + exit 0 + " + + cdn: + image: caddy:2-alpine + volumes: + - ./deploy/config/local/Caddyfile:/etc/caddy/Caddyfile + environment: + S3_BUCKET_NAME: ${S3_BUCKET_NAME} + VIRTUAL_HOST: cdn.${SERVER_NAME} + VIRTUAL_PORT: 80 + depends_on: + - minio diff --git a/public/app/themes/clarity/functions.php b/public/app/themes/clarity/functions.php index b7110076d..9946280b0 100644 --- a/public/app/themes/clarity/functions.php +++ b/public/app/themes/clarity/functions.php @@ -52,6 +52,7 @@ require_once 'inc/aboutus.php'; require_once 'inc/acf.php'; +require_once 'inc/amazon-s3-and-cloudfront-tweaks-for-minio.php'; require_once 'inc/api/get-posts-rest-api.php'; require_once 'inc/api/campaign-api.php'; diff --git a/public/app/themes/clarity/inc/amazon-s3-and-cloudfront-tweaks-for-minio.php b/public/app/themes/clarity/inc/amazon-s3-and-cloudfront-tweaks-for-minio.php new file mode 100644 index 000000000..b9a2bd67d --- /dev/null +++ b/public/app/themes/clarity/inc/amazon-s3-and-cloudfront-tweaks-for-minio.php @@ -0,0 +1,121 @@ +minio_host = Config::get('S3_CUSTOM_DOMAIN'); + + /* + * Custom S3 API Example: Minio + * @see https://min.io/ + */ + add_filter('as3cf_aws_s3_client_args', array($this, 'MinioS3ClientArgs')); + add_filter('as3cf_aws_s3_url_domain', array($this, 'MinioS3UrlDomain'), 10, 5); + add_filter('as3cf_aws_s3_console_url', array($this, 'MinioS3ConsoleUrl')); + // The "prefix param" denotes what should be in the console URL before the path prefix value. + // Minio just appends the path prefix directly after the bucket name. + add_filter('as3cf_aws_s3_console_url_prefix_param', fn () => '/'); + + /* + * URL Rewrite related filters. + */ + add_filter('as3cf_use_ssl', '__return_false', 10, 1); + } + + /** + * This filter allows you to adjust the arguments passed to the provider's service specific SDK client. + * + * The service specific SDK client is created from the initial provider SDK client, and inherits most of its config. + * The service specific SDK client is re-created more often than the provider SDK client for specific scenarios, so if possible + * set overrides in the provider client rather than service client for a slight improvement in performance. + * + * @see https://docs.aws.amazon.com/aws-sdk-php/v3/api/class-Aws.S3.S3Client.html#___construct + * @see https://docs.min.io/docs/how-to-use-aws-sdk-for-php-with-minio-server.html + * + * @handles `MinioS3ClientArgs` + * + * @param array $args + * + * @return array + * + * Note: A good place for changing 'signature_version', 'use_path_style_endpoint' etc. for specific bucket/object actions. + */ + public function MinioS3ClientArgs($args) + { + // Example changes endpoint to connect to a local Minio server configured to use port 54321 (the default Minio port is 9000). + $args['endpoint'] = 'http://' . $this->minio_host . ':9000'; + + // Example forces SDK to use endpoint URLs with bucket name in path rather than domain name as required by Minio. + $args['use_path_style_endpoint'] = true; + + return $args; + } + + /** + * This filter allows you to change the URL used for serving the files. + * + * @handles `MinioS3UrlDomain` + * + * @param string $domain + * @param string $bucket + * @param string $region + * @param int $expires + * @param array $args Allows you to specify custom URL settings + * + * @return string + */ + public function MinioS3UrlDomain($domain, $bucket, $region, $expires, $args) + { + // Minio doesn't need a region prefix, and always puts the bucket in the path. + return $this->minio_host . ':9000/' . $bucket; + } + + + + /** + * This filter allows you to change the base URL used to take you to the provider's console from WP Offload Media's settings. + * + * @handles `MinioS3ConsoleUrl` + * + * @param string $url + * + * @return string + */ + public function MinioS3ConsoleUrl($url) + { + return 'http://' . $this->minio_host . ':9001/browser/'; + } + +} + +if (str_starts_with(Config::get('S3_CUSTOM_DOMAIN'), 'minio')) { + new AmazonS3AndCloudFrontTweaks(); +}