From 98724d66a3e9bfe9f163e5341458db1fdbf72319 Mon Sep 17 00:00:00 2001 From: andrewpatto Date: Tue, 28 Nov 2023 18:01:30 +1100 Subject: [PATCH] Huge number of changes for error messages And thawing --- dev/test.ts | 26 ++++++ ...an-write-lambda.js => can-write-lambda.ts} | 0 .../construct/can-write-lambda/errors.ts | 0 .../.dockerignore | 0 .../.gitignore | 0 .../Dockerfile | 8 ++ .../rclone-go-batch-copy-docker-image/go.mod | 0 .../rclone-go-batch-copy-docker-image/main.go | 0 .../test-docker-direct.sh | 0 .../test-go-direct.sh | 0 .../testfile1.txt | 1 + .../thaw-objects-lambda-step-construct.ts | 82 +++++++++++++++++++ .../construct/thaw-objects-lambda/.gitignore | 3 + .../thaw-objects-lambda/package.json | 8 ++ .../thaw-objects-lambda.js | 52 ++++++++++++ 15 files changed, 180 insertions(+) create mode 100644 dev/test.ts rename packages/aws-copy-out-sharer/construct/can-write-lambda/{can-write-lambda.js => can-write-lambda.ts} (100%) create mode 100644 packages/aws-copy-out-sharer/construct/can-write-lambda/errors.ts create mode 100644 packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/.dockerignore create mode 100644 packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/.gitignore create mode 100644 packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/Dockerfile create mode 100644 packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/go.mod create mode 100644 packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/main.go create mode 100644 packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-docker-direct.sh create mode 100644 packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-go-direct.sh create mode 100644 packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/testfile1.txt create mode 100644 packages/aws-copy-out-sharer/construct/thaw-objects-lambda-step-construct.ts create mode 100644 packages/aws-copy-out-sharer/construct/thaw-objects-lambda/.gitignore create mode 100644 packages/aws-copy-out-sharer/construct/thaw-objects-lambda/package.json create mode 100644 packages/aws-copy-out-sharer/construct/thaw-objects-lambda/thaw-objects-lambda.js diff --git a/dev/test.ts b/dev/test.ts new file mode 100644 index 0000000..97fe028 --- /dev/null +++ b/dev/test.ts @@ -0,0 +1,26 @@ +import { CopyOutStack } from "aws-copy-out-sharer"; +import { SubnetType } from "aws-cdk-lib/aws-ec2"; +import { App } from "aws-cdk-lib"; + +const app = new App(); + +const description = + "Bulk copy-out service for Elsa Data - an application for controlled genomic data sharing"; + +const devId = "ElsaDataDevCopyOutStack"; + +new CopyOutStack(app, devId, { + // the stack can only be deployed to 'dev' + env: { + account: "843407916570", + region: "ap-southeast-2", + }, + tags: { + "umccr-org:Product": "ElsaData", + "umccr-org:Stack": devId, + }, + description: description, + isDevelopment: true, + infrastructureStackName: "ElsaDataDevInfrastructureStack", + infrastructureSubnetSelection: SubnetType.PRIVATE_WITH_EGRESS, +}); diff --git a/packages/aws-copy-out-sharer/construct/can-write-lambda/can-write-lambda.js b/packages/aws-copy-out-sharer/construct/can-write-lambda/can-write-lambda.ts similarity index 100% rename from packages/aws-copy-out-sharer/construct/can-write-lambda/can-write-lambda.js rename to packages/aws-copy-out-sharer/construct/can-write-lambda/can-write-lambda.ts diff --git a/packages/aws-copy-out-sharer/construct/can-write-lambda/errors.ts b/packages/aws-copy-out-sharer/construct/can-write-lambda/errors.ts new file mode 100644 index 0000000..e69de29 diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/.dockerignore b/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/.dockerignore new file mode 100644 index 0000000..e69de29 diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/.gitignore b/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/Dockerfile b/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/Dockerfile new file mode 100644 index 0000000..8b29bee --- /dev/null +++ b/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/Dockerfile @@ -0,0 +1,8 @@ +FROM rclone/rclone:1.64.2 + +RUN apk add --no-cache aws-cli tini + +# a shell script that allows us to take more batch like input to rclone +COPY rclone-batch-copy.sh /app/ + +ENTRYPOINT ["/sbin/tini", "--", "/bin/sh", "/app/rclone-batch-copy.sh"] diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/go.mod b/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/go.mod new file mode 100644 index 0000000..e69de29 diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/main.go b/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/main.go new file mode 100644 index 0000000..e69de29 diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-docker-direct.sh b/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-docker-direct.sh new file mode 100644 index 0000000..e69de29 diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-go-direct.sh b/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/test-go-direct.sh new file mode 100644 index 0000000..e69de29 diff --git a/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/testfile1.txt b/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/testfile1.txt new file mode 100644 index 0000000..f638667 --- /dev/null +++ b/packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/testfile1.txt @@ -0,0 +1 @@ +This is test file 1 diff --git a/packages/aws-copy-out-sharer/construct/thaw-objects-lambda-step-construct.ts b/packages/aws-copy-out-sharer/construct/thaw-objects-lambda-step-construct.ts new file mode 100644 index 0000000..6430ad5 --- /dev/null +++ b/packages/aws-copy-out-sharer/construct/thaw-objects-lambda-step-construct.ts @@ -0,0 +1,82 @@ +import { Construct } from "constructs"; +import { Effect, PolicyStatement } from "aws-cdk-lib/aws-iam"; +import { Duration, Stack } from "aws-cdk-lib"; +import { LambdaInvoke } from "aws-cdk-lib/aws-stepfunctions-tasks"; +import { Runtime } from "aws-cdk-lib/aws-lambda"; +import { IVpc, SubnetType } from "aws-cdk-lib/aws-ec2"; +import { JsonPath } from "aws-cdk-lib/aws-stepfunctions"; +import { NodejsFunction } from "aws-cdk-lib/aws-lambda-nodejs"; +import { join } from "path"; + +type CanWriteLambdaStepProps = { + vpc: IVpc; + vpcSubnetSelection: SubnetType; + + requiredRegion: string; + + /** + * If true, will allow this CanWrite lambda to test a bucket that is + * in the same account. Otherwise, and by default, the CanWrite lambda + * is set up to not be able to test a bucket in the same account as it + * is installed. This is a security mechanism as writes to buckets in the + * same account is allowed implictly but is dangerous. This should only + * be set to true for development/testing. + */ + allowWriteToThisAccount?: boolean; +}; + +/** + * A construct for a Steps function that tests whether an S3 + * bucket exists, is in the correct region, and is writeable + * by us. Throws an exception if any of these conditions is not met. + */ +export class CanWriteLambdaStepConstruct extends Construct { + public readonly invocableLambda; + + constructor(scope: Construct, id: string, props: CanWriteLambdaStepProps) { + super(scope, id); + + const canWriteLambda = new NodejsFunction(this, "CanWriteFunction", { + vpc: props.vpc, + entry: join(__dirname, "can-write-lambda", "can-write-lambda.js"), + // by specifying the precise runtime - the bundler knows exactly what packages are already in + // the base image - and for us can skip bundling @aws-sdk + // if we need to move this forward beyond node 18 - then we may need to revisit this + runtime: Runtime.NODEJS_18_X, + handler: "handler", + vpcSubnets: { + subnetType: props.vpcSubnetSelection, + }, + // this seems like plenty of seconds to do a few API calls to S3 + timeout: Duration.seconds(30), + }); + + canWriteLambda.addToRolePolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ["s3:PutObject"], + resources: ["*"], + // yes - that's right - we want to give this lambda the ability to attempt the writes anywhere + // EXCEPT where we are deployed + // (under the assumption that buckets outside our account must be giving us explicit write permission, + // whilst within our account we get implicit access - in this case we don't want that ability) + conditions: props.allowWriteToThisAccount + ? undefined + : { + StringNotEquals: { + "s3:ResourceAccount": [Stack.of(this).account], + }, + }, + }), + ); + + this.invocableLambda = new LambdaInvoke( + this, + `Can Write To Destination Bucket?`, + { + lambdaFunction: canWriteLambda, + resultPath: JsonPath.DISCARD, + }, + ); + } +} diff --git a/packages/aws-copy-out-sharer/construct/thaw-objects-lambda/.gitignore b/packages/aws-copy-out-sharer/construct/thaw-objects-lambda/.gitignore new file mode 100644 index 0000000..dc19133 --- /dev/null +++ b/packages/aws-copy-out-sharer/construct/thaw-objects-lambda/.gitignore @@ -0,0 +1,3 @@ + +# this lambda is a straight JS - no typescript so we need to include the JS +!*.js diff --git a/packages/aws-copy-out-sharer/construct/thaw-objects-lambda/package.json b/packages/aws-copy-out-sharer/construct/thaw-objects-lambda/package.json new file mode 100644 index 0000000..6c62bf2 --- /dev/null +++ b/packages/aws-copy-out-sharer/construct/thaw-objects-lambda/package.json @@ -0,0 +1,8 @@ +{ + "name": "can-write-lambda", + "version": "1.0.0", + "main": "can-write-lambda.js", + "dependencies": { + "@aws-sdk/client-s3": "^3.405.0" + } +} diff --git a/packages/aws-copy-out-sharer/construct/thaw-objects-lambda/thaw-objects-lambda.js b/packages/aws-copy-out-sharer/construct/thaw-objects-lambda/thaw-objects-lambda.js new file mode 100644 index 0000000..e4646e6 --- /dev/null +++ b/packages/aws-copy-out-sharer/construct/thaw-objects-lambda/thaw-objects-lambda.js @@ -0,0 +1,52 @@ +const { PutObjectCommand, S3Client } = require("@aws-sdk/client-s3"); + +export const handler = async (event) => { + function WrongRegionError(message) { + this.name = "WrongRegionError"; + this.message = message; + } + WrongRegionError.prototype = new Error(); + + function AccessDeniedError(message) { + this.name = "AccessDeniedError"; + this.message = message; + } + AccessDeniedError.prototype = new Error(); + + console.log(event.requiredRegion); + console.log(event.destinationBucket); + + // we are being super specific here - the required region is where we are going + // to make our client - in order to ensure we get 301 Redirects for buckets outside our location + const client = new S3Client({ region: event.requiredRegion }); + + try { + const putCommand = new PutObjectCommand({ + Bucket: event.destinationBucket, + Key: "ELSA_DATA_STARTED_TRANSFER.txt", + Body: "A file created by Elsa Data copy out to ensure correct permissions", + }); + + const response = await client.send(putCommand); + } catch (e) { + if (e.Code === "PermanentRedirect") + throw new WrongRegionError( + "S3 Put failed because bucket was in the wrong region", + ); + + if (e.Code === "AccessDenied") + throw new AccessDeniedError("S3 Put failed with access denied error"); + + throw e; + } +}; + +/*handler({ + requiredRegion: "ap-southeast-2", + //destinationBucket: "elsa-data-tmp" + //destinationBucket: "cdk-hnb659fds-assets-843407916570-us-east-1" + //destinationBucket: "elsa-data-replication-target-foo" + destinationBucket: "elsa-data-replication-target" + // destinationBucket: "elsa-data-copy-target-sydney" + // destinationBucket: "elsa-data-copy-target-tokyo" +}) */