Bump CDK for ECS Runtask bug

Final tidies
elsa-data · May 15, 2024 · 6ede843 · 6ede843
1 parent 452185c
commit 6ede843
Show file tree

Hide file tree

Showing 10 changed files with 2,077 additions and 1,374 deletions.
diff --git a/README.md b/README.md
@@ -55,8 +55,9 @@ test invocations.
 }
 ```
 
-The copy will fan out wide (to sensible width (~ 100)) - but there is a small cost to the startup/shutdown
-of the Fargate tasks. The maxItemsPerBatch controls how many individuals files are attempted per
+The copy will fan-out wide (to sensible width (~ 100)) - but there is a small AWS Config
+cost to the startup/shutdown
+of the Fargate tasks. Therefore the `maxItemsPerBatch` controls how many individuals files are attempted per
 Fargate task - though noting that we request SPOT tasks.
 
 So there is balance between the likelihood of SPOT interruptions v re-use of Fargate tasks. If

diff --git a/dev/EXAMPLE-COPY-README.md b/dev/EXAMPLE-COPY-README.md
@@ -8,11 +8,29 @@ Copy "example-copy-manifest.csv" to that folder.
 THE FOLDER MUST BE EXACTLY AS SPECIFIED AS THAT PERMISSION IS BAKED INTO
 THE DEV DEPLOYMENT (IN ORDER TO TEST PERMISSIONS!)
 
-Invoke the dev Steps with the input
+Invoke the dev Steps with the input (feel free to change to "0\_" if you
+want to run multiple experiments without overriding the results)
 
+```json
 {
-"sourceFilesCsvBucket": "elsa-data-tmp",
-"sourceFilesCsvKey": "example-copy-manifest.csv",
-"destinationBucket": "elsa-data-copy-target-sydney",
-"maxItemsPerBatch": 2
+  "sourceFilesCsvBucket": "elsa-data-tmp",
+  "sourceFilesCsvKey": "example-copy-manifest.csv",
+  "destinationBucket": "elsa-data-copy-target-sydney",
+  "maxItemsPerBatch": 2,
+  "destinationStartCopyRelativeKey": "0_STARTED_COPY.txt",
+  "destinationEndCopyRelativeKey": "0_ENDED_COPY.csv"
 }
+```
+
+For a test of AG (in the AG account - with public/made up data files)
+
+```json
+{
+  "sourceFilesCsvBucket": "elsa-data-copy-working",
+  "sourceFilesCsvKey": "example-copy-manifest-ag.csv",
+  "destinationBucket": "elsa-data-copy-target-sydney",
+  "maxItemsPerBatch": 1,
+  "destinationStartCopyRelativeKey": "AG_STARTED_COPY.txt",
+  "destinationEndCopyRelativeKey": "AG_ENDED_COPY.csv"
+}
+```
diff --git a/dev/dev.ts b/dev/dev.ts
@@ -87,7 +87,6 @@ class ElsaDataSimpleCopyOutStack extends Stack {
       vpc: vpc,
       vpcSubnetSelection: SubnetType.PRIVATE_WITH_EGRESS,
       workingBucket: "elsa-data-copy-working",
-      workingBucketPrefixKey: "temp/",
       aggressiveTimes: false,
       allowWriteToInstalledAccount: true,
     });

diff --git a/dev/example-copy-manifest-ag.csv b/dev/example-copy-manifest-ag.csv
@@ -0,0 +1,2 @@
+elsa-data-demo-agha-gdr-store,FLAGSHIP_A/2020-02-01/ERR251112_R1.fastq.gz
+elsa-data-demo-agha-gdr-store,FLAGSHIP_A/2020-02-01/ERR251112_R2.fastq.gz
diff --git a/dev/example-copy-manifest.csv b/dev/example-copy-manifest.csv
@@ -4,6 +4,7 @@ umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.vcf"
 umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.vcf.gz"
 umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.vcf.gz.csi"
 umccr-10f-data-dev,"ASHKENAZIM/HG002-HG003-HG004.joint.filter.vcf.gz.tbi"
+umccr-10f-data-dev,AFILETHATDOESNOTEXIST.txt
 umccr-10f-data-dev,ASHKENAZIM/HG002.bam
 umccr-10f-data-dev,ASHKENAZIM/HG002.bam.bai
 umccr-10f-data-dev,ASHKENAZIM/HG003.bam

diff --git a/dev/package.json b/dev/package.json
@@ -12,12 +12,13 @@
     "test-quick": "ts-node --prefer-ts-exts test.ts"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "3.450.0",
-    "@aws-sdk/client-servicediscovery": "3.450.0",
-    "@aws-sdk/client-sfn": "3.450.0",
+    "@aws-sdk/client-s3": "3.576.0",
+    "@aws-sdk/client-servicediscovery": "3.576.0",
+    "@aws-sdk/client-sfn": "3.576.0",
+    "@aws-sdk/client-sso-oidc": "3.574.0",
     "@elsa-data/aws-infrastructure": "1.5.1",
-    "aws-cdk": "2.113.0",
-    "aws-cdk-lib": "2.113.0",
+    "aws-cdk": "2.141.0",
+    "aws-cdk-lib": "2.141.0",
     "aws-copy-out-sharer": "link:../packages/aws-copy-out-sharer",
     "constructs": "10.3.0"
   },

diff --git a/packages/aws-copy-out-sharer/README.md b/packages/aws-copy-out-sharer/README.md
@@ -1,3 +1,3 @@
-# copy out state machine
+# Copy out state machine
 
 A construct for a AWS Steps Copy Out state machine.
diff --git a/packages/aws-copy-out-sharer/package.json b/packages/aws-copy-out-sharer/package.json
@@ -35,14 +35,14 @@
     }
   },
   "peerDependencies": {
-    "aws-cdk-lib": "^2.113.0",
+    "aws-cdk-lib": "^2.141.0",
     "constructs": "^10.3.0"
   },
   "devDependencies": {
-    "aws-cdk-lib": "2.113.0",
+    "aws-cdk-lib": "2.141.0",
     "constructs": "10.3.0",
-    "jsii": "5.2.35",
-    "jsii-pacmak": "1.92.0",
-    "publib": "0.2.756"
+    "jsii": "5.4.15",
+    "jsii-pacmak": "1.98.0",
+    "publib": "0.2.834"
   }
 }
diff --git a/packages/aws-copy-out-sharer/src/copy-out-state-machine-construct.ts b/packages/aws-copy-out-sharer/src/copy-out-state-machine-construct.ts
@@ -40,7 +40,7 @@ export class CopyOutStateMachineConstruct extends Construct {
       maxItemsPerBatch: 8,
       copyConcurrency: 80,
       requiredRegion: Stack.of(this).region,
-      // by default we just copy into the top level of the destination bucket
+      // by default, we just copy into the top level of the destination bucket
       destinationPrefixKey: "",
 
       // these are the default objects that will be created in the destination prefix area
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		elsa-data-demo-agha-gdr-store,FLAGSHIP_A/2020-02-01/ERR251112_R1.fastq.gz
		elsa-data-demo-agha-gdr-store,FLAGSHIP_A/2020-02-01/ERR251112_R2.fastq.gz