Huge number of changes for error messages

And thawing

dev/test.ts

@@ -0,0 +1,26 @@
+import { CopyOutStack } from "aws-copy-out-sharer";
+import { SubnetType } from "aws-cdk-lib/aws-ec2";
+import { App } from "aws-cdk-lib";
+
+const app = new App();
+
+const description =
+  "Bulk copy-out service for Elsa Data - an application for controlled genomic data sharing";
+
+const devId = "ElsaDataDevCopyOutStack";
+
+new CopyOutStack(app, devId, {
+  // the stack can only be deployed to 'dev'
+  env: {
+    account: "843407916570",
+    region: "ap-southeast-2",
+  },
+  tags: {
+    "umccr-org:Product": "ElsaData",
+    "umccr-org:Stack": devId,
+  },
+  description: description,
+  isDevelopment: true,
+  infrastructureStackName: "ElsaDataDevInfrastructureStack",
+  infrastructureSubnetSelection: SubnetType.PRIVATE_WITH_EGRESS,
+});

packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/Dockerfile

@@ -0,0 +1,8 @@
+FROM rclone/rclone:1.64.2
+
+RUN apk add --no-cache aws-cli tini
+
+# a shell script that allows us to take more batch like input to rclone
+COPY rclone-batch-copy.sh /app/
+
+ENTRYPOINT ["/sbin/tini", "--", "/bin/sh", "/app/rclone-batch-copy.sh"]

packages/aws-copy-out-sharer/construct/rclone-go-batch-copy-docker-image/testfile1.txt

@@ -0,0 +1 @@
+This is test file 1

packages/aws-copy-out-sharer/construct/thaw-objects-lambda-step-construct.ts

@@ -0,0 +1,82 @@
+import { Construct } from "constructs";
+import { Effect, PolicyStatement } from "aws-cdk-lib/aws-iam";
+import { Duration, Stack } from "aws-cdk-lib";
+import { LambdaInvoke } from "aws-cdk-lib/aws-stepfunctions-tasks";
+import { Runtime } from "aws-cdk-lib/aws-lambda";
+import { IVpc, SubnetType } from "aws-cdk-lib/aws-ec2";
+import { JsonPath } from "aws-cdk-lib/aws-stepfunctions";
+import { NodejsFunction } from "aws-cdk-lib/aws-lambda-nodejs";
+import { join } from "path";
+
+type CanWriteLambdaStepProps = {
+  vpc: IVpc;
+  vpcSubnetSelection: SubnetType;
+
+  requiredRegion: string;
+
+  /**
+   * If true, will allow this CanWrite lambda to test a bucket that is
+   * in the same account. Otherwise, and by default, the CanWrite lambda
+   * is set up to not be able to test a bucket in the same account as it
+   * is installed. This is a security mechanism as writes to buckets in the
+   * same account is allowed implictly but is dangerous. This should only
+   * be set to true for development/testing.
+   */
+  allowWriteToThisAccount?: boolean;
+};
+
+/**
+ * A construct for a Steps function that tests whether an S3
+ * bucket exists, is in the correct region, and is writeable
+ * by us. Throws an exception if any of these conditions is not met.
+ */
+export class CanWriteLambdaStepConstruct extends Construct {
+  public readonly invocableLambda;
+
+  constructor(scope: Construct, id: string, props: CanWriteLambdaStepProps) {
+    super(scope, id);
+
+    const canWriteLambda = new NodejsFunction(this, "CanWriteFunction", {
+      vpc: props.vpc,
+      entry: join(__dirname, "can-write-lambda", "can-write-lambda.js"),
+      // by specifying the precise runtime - the bundler knows exactly what packages are already in
+      // the base image - and for us can skip bundling @aws-sdk
+      // if we need to move this forward beyond node 18 - then we may need to revisit this
+      runtime: Runtime.NODEJS_18_X,
+      handler: "handler",
+      vpcSubnets: {
+        subnetType: props.vpcSubnetSelection,
+      },
+      // this seems like plenty of seconds to do a few API calls to S3
+      timeout: Duration.seconds(30),
+    });
+
+    canWriteLambda.addToRolePolicy(
+      new PolicyStatement({
+        effect: Effect.ALLOW,
+        actions: ["s3:PutObject"],
+        resources: ["*"],
+        // yes - that's right - we want to give this lambda the ability to attempt the writes anywhere
+        // EXCEPT where we are deployed
+        // (under the assumption that buckets outside our account must be giving us explicit write permission,
+        //  whilst within our account we get implicit access - in this case we don't want that ability)
+        conditions: props.allowWriteToThisAccount
+          ? undefined
+          : {
+              StringNotEquals: {
+                "s3:ResourceAccount": [Stack.of(this).account],
+              },
+            },
+      }),
+    );
+
+    this.invocableLambda = new LambdaInvoke(
+      this,
+      `Can Write To Destination Bucket?`,
+      {
+        lambdaFunction: canWriteLambda,
+        resultPath: JsonPath.DISCARD,
+      },
+    );
+  }
+}

packages/aws-copy-out-sharer/construct/thaw-objects-lambda/.gitignore

@@ -0,0 +1,3 @@
+
+# this lambda is a straight JS - no typescript so we need to include the JS
+!*.js

packages/aws-copy-out-sharer/construct/thaw-objects-lambda/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "can-write-lambda",
+  "version": "1.0.0",
+  "main": "can-write-lambda.js",
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.405.0"
+  }
+}

packages/aws-copy-out-sharer/construct/thaw-objects-lambda/thaw-objects-lambda.js

@@ -0,0 +1,52 @@
+const { PutObjectCommand, S3Client } = require("@aws-sdk/client-s3");
+
+export const handler = async (event) => {
+  function WrongRegionError(message) {
+    this.name = "WrongRegionError";
+    this.message = message;
+  }
+  WrongRegionError.prototype = new Error();
+
+  function AccessDeniedError(message) {
+    this.name = "AccessDeniedError";
+    this.message = message;
+  }
+  AccessDeniedError.prototype = new Error();
+
+  console.log(event.requiredRegion);
+  console.log(event.destinationBucket);
+
+  // we are being super specific here - the required region is where we are going
+  // to make our client - in order to ensure we get 301 Redirects for buckets outside our location
+  const client = new S3Client({ region: event.requiredRegion });
+
+  try {
+    const putCommand = new PutObjectCommand({
+      Bucket: event.destinationBucket,
+      Key: "ELSA_DATA_STARTED_TRANSFER.txt",
+      Body: "A file created by Elsa Data copy out to ensure correct permissions",
+    });
+
+    const response = await client.send(putCommand);
+  } catch (e) {
+    if (e.Code === "PermanentRedirect")
+      throw new WrongRegionError(
+        "S3 Put failed because bucket was in the wrong region",
+      );
+
+    if (e.Code === "AccessDenied")
+      throw new AccessDeniedError("S3 Put failed with access denied error");
+
+    throw e;
+  }
+};
+
+/*handler({
+  requiredRegion: "ap-southeast-2",
+  //destinationBucket: "elsa-data-tmp"
+  //destinationBucket: "cdk-hnb659fds-assets-843407916570-us-east-1"
+  //destinationBucket: "elsa-data-replication-target-foo"
+  destinationBucket: "elsa-data-replication-target"
+  // destinationBucket: "elsa-data-copy-target-sydney"
+  // destinationBucket: "elsa-data-copy-target-tokyo"
+}) */