Skip to content

Commit

Permalink
Add minio & caddy for local S3. (#490)
Browse files Browse the repository at this point in the history
* Add minio & caddy for local S3.

* Trim unused code and add optional comment

* Add comments and consistent casing.

* Fix typos and anon function for as3cf_aws_s3_console_url_prefix_param
  • Loading branch information
EarthlingDavey authored Mar 28, 2024
1 parent 57448cf commit ed9ac08
Show file tree
Hide file tree
Showing 9 changed files with 193 additions and 67 deletions.
10 changes: 10 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,16 @@ LOCAL_SSH_PASSWORD=ssh-password
# General template and api-key for gov notify
GOV_NOTIFY_API_KEY=""

# AWS / WP Offload Media
# - Minio (starts with minio for AmazonS3AndCloudFrontTweaks)
S3_CUSTOM_DOMAIN="minio.${SERVER_NAME}"
# - S3
S3_BUCKET_NAME=test-bucket
AWS_ACCESS_KEY_ID=myaccesskey
AWS_SECRET_ACCESS_KEY=myaccesssecret
# - CloudFront (optional)
DELIVERY_DOMAIN="cdn.${SERVER_NAME}"

# Generate your keys here: https://roots.io/salts.html
AUTH_KEY='generate-key'
SECURE_AUTH_KEY='generate-key'
Expand Down
8 changes: 5 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,15 @@ RUN chmod +x /var/www/html/composer-auth.sh && \
# non-root
USER 82

COPY ./composer.json /var/www/html/composer.json
RUN composer install --no-dev --no-scripts --no-autoloader
COPY composer.* /var/www/html/

COPY . .
RUN composer install --no-dev
RUN composer dump-autoload -o

# Copy all of the files here for now.
# We can move this later.
COPY . .

ARG regex_files='\(htm\|html\|js\|css\|png\|jpg\|jpeg\|gif\|ico\|svg\|webmanifest\)'
ARG regex_path='\(app\/themes\/clarity\/error\-pages\|app\/mu\-plugins\|app\/plugins\|wp\)'
RUN mkdir -p ./vendor-assets && \
Expand Down
3 changes: 0 additions & 3 deletions composer.json
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,6 @@
"wpackagist-plugin/amazon-s3-and-cloudfront": "^3.2"
},
"require-dev": {
"wpackagist-plugin/query-monitor": "^3.15.0",
"wpackagist-plugin/debug-bar": "^1.1.0",
"wpackagist-plugin/debug-bar-elasticpress": "^3.1.0",
"squizlabs/php_codesniffer": "^3.0.2"
},
"extra": {
Expand Down
56 changes: 1 addition & 55 deletions composer.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 7 additions & 5 deletions config/wp-offload-media.php
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@
// Append a timestamped folder to path of files offloaded to bucket to avoid filename clashes and bust CDN cache if updated
'object-versioning' => true,
// Delivery Provider ('storage', 'aws', 'do', 'gcp', 'cloudflare', 'keycdn', 'stackpath', 'other')
'delivery-provider' => env('CLOUDFRONT_URL') ? 'aws' : 'storage',
'delivery-provider' => env('DELIVERY_DOMAIN') ? 'aws' : 'storage',
// Rewrite file URLs to bucket (s3 or cloudfront)
'serve-from-s3' => true,
// Use a custom domain (CNAME), not supported when using 'storage' Delivery Provider
'enable-delivery-domain' => !!env('CLOUDFRONT_URL'),
'enable-delivery-domain' => !!env('DELIVERY_DOMAIN'),
// Custom domain (CNAME), not supported when using 'storage' Delivery Provider
'delivery-domain' => env('CLOUDFRONT_URL'),
'delivery-domain' => env('DELIVERY_DOMAIN'),
// Enable signed URLs for Delivery Provider that uses separate key pair (currently only 'aws' supported, a.k.a. CloudFront)
// 'enable-signed-urls' => false,
// Access Key ID for signed URLs (aws only, replace '*')
Expand All @@ -45,9 +45,9 @@
// Private Prefix for signed URLs (aws only, relative directory, no wildcards)
// 'signed-urls-object-prefix' => 'private/',
// Serve files over HTTPS
'force-https' => !!env('CLOUDFRONT_URL'),
'force-https' => !!env('DELIVERY_DOMAIN'),
// Remove the local file version once offloaded to bucket
'remove-local-file' => false,
'remove-local-file' => true,
// Access Control List for the bucket
'use-bucket-acls' => false,
);
Expand All @@ -64,4 +64,6 @@
]);
}

Config::define('S3_CUSTOM_DOMAIN', env('S3_CUSTOM_DOMAIN') ?? '');

Config::define('AS3CF_SETTINGS', serialize($as3_settings));
2 changes: 1 addition & 1 deletion deploy/config/local/Caddyfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
# e.g. Request: http://cdn.justice.docker/uploads/2024/02/xyz.jpg
# proxies to : http://minio:9000/bucket-name/uploads/2024/02/xyz.jpg

:2019
:80
rewrite * /{$S3_BUCKET_NAME}{uri}
reverse_proxy minio:9000
47 changes: 47 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ volumes:
opensearch-data:
database:
driver: local
minio_storage: ~

services:
php-fpm:
Expand All @@ -19,6 +20,9 @@ services:
depends_on:
- mariadb
- opensearch
links:
- "cdn:cdn.${SERVER_NAME}"
- "minio:minio.${SERVER_NAME}"

nginx:
build:
Expand Down Expand Up @@ -119,3 +123,46 @@ services:
DISABLE_SECURITY_DASHBOARDS_PLUGIN: true # disables security dashboards plugin in OpenSearch Dashboards
depends_on:
- opensearch

minio:
image: minio/minio
ports:
- "9000:9000" # The AWS S3 compatible API.
- "9001:9001" # The Minio web console.
volumes:
- minio_storage:/data
environment:
MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID}
MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY}
VIRTUAL_HOST: minio.${SERVER_NAME}
VIRTUAL_PORT: 9000
command: server --console-address ":9001" /data
healthcheck:
test: timeout 5s bash -c ':> /dev/tcp/127.0.0.1/9000' || exit 1
start_period: 5s
interval: 10s
timeout: 5s
retries: 2

minio-init:
image: minio/mc
depends_on:
- minio
entrypoint: |
/bin/sh -c "
mc config host add justice-gov-uk http://minio:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY}
mc mb justice-gov-uk/${S3_BUCKET_NAME}
mc anonymous set download justice-gov-uk/${S3_BUCKET_NAME};
exit 0
"
cdn:
image: caddy:2-alpine
volumes:
- ./deploy/config/local/Caddyfile:/etc/caddy/Caddyfile
environment:
S3_BUCKET_NAME: ${S3_BUCKET_NAME}
VIRTUAL_HOST: cdn.${SERVER_NAME}
VIRTUAL_PORT: 80
depends_on:
- minio
1 change: 1 addition & 0 deletions public/app/themes/clarity/functions.php
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@

require_once 'inc/aboutus.php';
require_once 'inc/acf.php';
require_once 'inc/amazon-s3-and-cloudfront-tweaks-for-minio.php';

require_once 'inc/api/get-posts-rest-api.php';
require_once 'inc/api/campaign-api.php';
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
<?php

/**
* This file is cherry-picked functions from the wp-amazon-s3-and-cloudfront-tweaks plugin.
* It is the config for using Minio Locally with WP Offload Media.
* @see http://github.com/deliciousbrains/wp-amazon-s3-and-cloudfront-tweaks
*
* When accessing the WP Offload Media Lite setting page,
* the plugin will log the following errors when trying to access the Minio server:
* - AS3CF: Could not get Block All Public Access status: Error executing "GetPublicAccessBlock"
* - AS3CF: Could not get Object Ownership status: Error executing "GetBucketOwnershipControls"
* This is because Minio does not support these features.
*/

namespace DeliciousBrains\WP_Offload_Media\Tweaks;
use Roots\WPConfig\Config;

class AmazonS3AndCloudFrontTweaks
{

// Define the Minio hostnames.
private $minio_host = '';

public function __construct()
{
/*
* WP Offload Media & WP Offload Media Lite
*
* https://deliciousbrains.com/wp-offload-media/
* https://wordpress.org/plugins/amazon-s3-and-cloudfront/
*/

// If the S3_DOMAIN doesn't start with 'minio', then we are not using Minio.
$this->minio_host = Config::get('S3_CUSTOM_DOMAIN');

/*
* Custom S3 API Example: Minio
* @see https://min.io/
*/
add_filter('as3cf_aws_s3_client_args', array($this, 'MinioS3ClientArgs'));
add_filter('as3cf_aws_s3_url_domain', array($this, 'MinioS3UrlDomain'), 10, 5);
add_filter('as3cf_aws_s3_console_url', array($this, 'MinioS3ConsoleUrl'));
// The "prefix param" denotes what should be in the console URL before the path prefix value.
// Minio just appends the path prefix directly after the bucket name.
add_filter('as3cf_aws_s3_console_url_prefix_param', fn () => '/');

/*
* URL Rewrite related filters.
*/
add_filter('as3cf_use_ssl', '__return_false', 10, 1);
}

/**
* This filter allows you to adjust the arguments passed to the provider's service specific SDK client.
*
* The service specific SDK client is created from the initial provider SDK client, and inherits most of its config.
* The service specific SDK client is re-created more often than the provider SDK client for specific scenarios, so if possible
* set overrides in the provider client rather than service client for a slight improvement in performance.
*
* @see https://docs.aws.amazon.com/aws-sdk-php/v3/api/class-Aws.S3.S3Client.html#___construct
* @see https://docs.min.io/docs/how-to-use-aws-sdk-for-php-with-minio-server.html
*
* @handles `MinioS3ClientArgs`
*
* @param array $args
*
* @return array
*
* Note: A good place for changing 'signature_version', 'use_path_style_endpoint' etc. for specific bucket/object actions.
*/
public function MinioS3ClientArgs($args)
{
// Example changes endpoint to connect to a local Minio server configured to use port 54321 (the default Minio port is 9000).
$args['endpoint'] = 'http://' . $this->minio_host . ':9000';

// Example forces SDK to use endpoint URLs with bucket name in path rather than domain name as required by Minio.
$args['use_path_style_endpoint'] = true;

return $args;
}

/**
* This filter allows you to change the URL used for serving the files.
*
* @handles `MinioS3UrlDomain`
*
* @param string $domain
* @param string $bucket
* @param string $region
* @param int $expires
* @param array $args Allows you to specify custom URL settings
*
* @return string
*/
public function MinioS3UrlDomain($domain, $bucket, $region, $expires, $args)
{
// Minio doesn't need a region prefix, and always puts the bucket in the path.
return $this->minio_host . ':9000/' . $bucket;
}



/**
* This filter allows you to change the base URL used to take you to the provider's console from WP Offload Media's settings.
*
* @handles `MinioS3ConsoleUrl`
*
* @param string $url
*
* @return string
*/
public function MinioS3ConsoleUrl($url)
{
return 'http://' . $this->minio_host . ':9001/browser/';
}

}

if (str_starts_with(Config::get('S3_CUSTOM_DOMAIN'), 'minio')) {
new AmazonS3AndCloudFrontTweaks();
}

0 comments on commit ed9ac08

Please sign in to comment.