Renamed folder

Started diff permissions for app v command

dev/dev.ts

@@ -15,20 +15,14 @@ const app = new cdk.App();
 // Aspects.of(app).add(new HIPAASecurityChecks({ verbose: true }));
 // Aspects.of(app).add(new NIST80053R5Checks({ verbose: true }));
 
-// tags for our stacks
-const tags = {
-  "umccr-org:Stack": "ElsaDataApplication",
-  "umccr-org:Product": "ElsaData",
-};
-
 const descriptionWithTag = (tag?: string) =>
   `Application for Elsa Data ${
     tag ? "(" + tag + ") " : ""
   }- an application for controlled genomic data sharing`;
 
 // bring this out to the top as it is the type of thing we might want to change during dev
 // to point to other PR branches etc
-const DEV_DEPLOYED_IMAGE_TAG = "pr-468";
+const DEV_DEPLOYED_IMAGE_TAG = "0.4.1";
 
 /**
  * Stack for dev
@@ -44,7 +38,8 @@ new ElsaDataStack(
     description: descriptionWithTag(undefined),
     tags: {
       "umccr-org:ProductVersion": DEV_DEPLOYED_IMAGE_TAG,
-      ...tags,
+      "umccr-org:Stack": "ElsaDataApplication",
+      "umccr-org:Product": "ElsaData",
     },
   },
   {

packages/aws-application/app/elsa-data-application-construct.ts

@@ -12,6 +12,7 @@ import { ClusterConstruct } from "../construct/cluster-construct";
 import { ContainerConstruct } from "../construct/container-construct";
 import { TaskDefinitionConstruct } from "../construct/task-definition-construct";
 import { FargateService } from "aws-cdk-lib/aws-ecs";
+import { getPolicyStatementsFromDataBucketPaths } from "../helper/bucket-names-to-policy";
 
 interface Props extends ElsaDataApplicationSettings {
   readonly cluster: ClusterConstruct;
@@ -28,15 +29,18 @@ interface Props extends ElsaDataApplicationSettings {
   // the security group of our edgedb - that we will put ourselves in to enable access
   readonly edgeDbSecurityGroup: ISecurityGroup;
 
-  // a policy statement that we need to add to our running cloudMapService in order to give us access to the secrets
+  // a policy statement that we need to add to our app service in order to give us access to the secrets
   readonly accessSecretsPolicyStatement: PolicyStatement;
 
+  // a policy statement that we need to add to our app service in order to discover other services via cloud map
+  readonly discoverServicesPolicyStatement: PolicyStatement;
+
   // an already created temp bucket we can use
   readonly tempBucket: IBucket;
 }
 
 /**
- * A construct that deploys Elsa Data as a Fargate cloudMapService.
+ * A construct that deploys Elsa Data as a Fargate service.
  */
 export class ElsaDataApplicationConstruct extends Construct {
   private readonly privateServiceWithLoadBalancer: DockerServiceWithHttpsLoadBalancerConstruct;
@@ -63,33 +67,28 @@ export class ElsaDataApplicationConstruct extends Construct {
 
     const policy = new Policy(this, "FargateServiceTaskPolicy");
 
-    // need to be able to fetch secrets - we wildcard to every Secret that has our designated prefix of elsa*
+    // need to be able to fetch secrets but the infrastructure can give us a wildcard
+    // policy statement that does that
     policy.addStatements(props.accessSecretsPolicyStatement);
 
-    // restrict our Get operations to a very specific set of keys in the named buckets
-    // NOTE: our 'signing' is always done by a different user so this is not the only
-    // permission that has to be set correctly
-    for (const [bucketName, keyWildcards] of Object.entries(
-      props.awsPermissions.dataBucketPaths
-    )) {
-      policy.addStatements(
-        new PolicyStatement({
-          actions: ["s3:GetObject"],
-          // NOTE: we could consider restricting to region or account here in constructing the ARNS
-          // but given the bucket names are already globally specific we leave them open
-          resources: keyWildcards.map(
-            (k) => `arn:${Stack.of(this).partition}:s3:::${bucketName}/${k}`
-          ),
-        })
-      );
-    }
+    // need to be able to discover instances in the cloud map namespace - and our
+    // infrastructure can give us a policy statement to enable that
+    policy.addStatements(props.discoverServicesPolicyStatement);
 
+    // we (currently) give the application access to all the data bucket objects
+    // TODO consider subsetting even this permissions (only manifests??)
+    policy.addStatements(
+      ...getPolicyStatementsFromDataBucketPaths(
+        Stack.of(this).partition,
+        props.awsPermissions.dataBucketPaths
+      )
+    );
+
+    // allow cloudtrail queries to get data egress records
     policy.addStatements(
       new PolicyStatement({
-        actions: ["s3:ListBucket"],
-        resources: Object.keys(props.awsPermissions.dataBucketPaths).map(
-          (b) => `arn:${Stack.of(this).partition}:s3:::${b}`
-        ),
+        actions: ["cloudtrail:StartQuery", "cloudtrail:GetQueryResults"],
+        resources: ["*"],
       })
     );
 
@@ -164,14 +163,6 @@ export class ElsaDataApplicationConstruct extends Construct {
       })
     );
 
-    // allow cloudtrail queries to get data egress records
-    policy.addStatements(
-      new PolicyStatement({
-        actions: ["cloudtrail:StartQuery", "cloudtrail:GetQueryResults"],
-        resources: ["*"],
-      })
-    );
-
     // for some of our scaling out work (Beacon etc) - we are going to make Lambdas that we want to be able to invoke
     // again we wildcard to a designated prefix of elsa-data*
     // TODO parameterise this to not have a magic string

packages/aws-application/app-command/elsa-data-application-command-construct.ts → packages/aws-application/command/elsa-data-command-construct.ts

@@ -3,7 +3,7 @@ import { Construct } from "constructs";
 import { Policy, PolicyStatement } from "aws-cdk-lib/aws-iam";
 import { Function, Runtime } from "aws-cdk-lib/aws-lambda";
 import { ISecurityGroup } from "aws-cdk-lib/aws-ec2";
-import { Service as CloudMapService } from "aws-cdk-lib/aws-servicediscovery";
+import { INamespace, Service } from "aws-cdk-lib/aws-servicediscovery";
 import { ElsaDataApplicationSettings } from "../elsa-data-application-settings";
 import { IBucket } from "aws-cdk-lib/aws-s3";
 import { join } from "path";
@@ -12,6 +12,7 @@ import { ClusterConstruct } from "../construct/cluster-construct";
 import { ContainerConstruct } from "../construct/container-construct";
 import { TaskDefinitionConstruct } from "../construct/task-definition-construct";
 import { FargateService } from "aws-cdk-lib/aws-ecs";
+import { getPolicyStatementsFromDataBucketPaths } from "../helper/bucket-names-to-policy";
 
 interface Props extends ElsaDataApplicationSettings {
   readonly cluster: ClusterConstruct;
@@ -22,7 +23,7 @@ interface Props extends ElsaDataApplicationSettings {
 
   readonly appService: FargateService;
 
-  readonly cloudMapService: CloudMapService;
+  readonly cloudMapNamespace: INamespace;
 
   // the security group of our edgedb - that we will put ourselves in to enable access
   readonly edgeDbSecurityGroup: ISecurityGroup;
@@ -40,7 +41,7 @@ interface Props extends ElsaDataApplicationSettings {
  * It uses the same Elsa Data docker image as the actual web app, but
  * is invoked differently.
  */
-export class ElsaDataApplicationCommandConstruct extends Construct {
+export class ElsaDataCommandConstruct extends Construct {
   constructor(scope: Construct, id: string, props: Props) {
     super(scope, id);
 
@@ -49,31 +50,12 @@ export class ElsaDataApplicationCommandConstruct extends Construct {
     // need to be able to fetch secrets - we wildcard to every Secret that has our designated prefix of elsa*
     policy.addStatements(props.accessSecretsPolicyStatement);
 
-    // restrict our Get operations to a very specific set of keys in the named buckets
-    // NOTE: our 'signing' is always done by a different user so this is not the only
-    // permission that has to be set correctly
-    for (const [bucketName, keyWildcards] of Object.entries(
-      props.awsPermissions.dataBucketPaths
-    )) {
-      policy.addStatements(
-        new PolicyStatement({
-          actions: ["s3:GetObject"],
-          // NOTE: we could consider restricting to region or account here in constructing the ARNS
-          // but given the bucket names are already globally specific we leave them open
-          resources: keyWildcards.map(
-            (k) => `arn:${Stack.of(this).partition}:s3:::${bucketName}/${k}`
-          ),
-        })
-      );
-    }
-
+    // we give the command broad access to the data for the purposes of generating dataset
     policy.addStatements(
-      new PolicyStatement({
-        actions: ["s3:ListBucket"],
-        resources: Object.keys(props.awsPermissions.dataBucketPaths).map(
-          (b) => `arn:${Stack.of(this).partition}:s3:::${b}`
-        ),
-      })
+      ...getPolicyStatementsFromDataBucketPaths(
+        Stack.of(this).partition,
+        props.awsPermissions.dataBucketPaths
+      )
     );
 
     // allow the command Elsa's to do arbitrary things to tasks in the same cluster
@@ -104,8 +86,16 @@ export class ElsaDataApplicationCommandConstruct extends Construct {
       props.appService
     );
 
-    // we register it into the cloudmap cloudMapService so outside tools can locate it
-    props.cloudMapService.registerNonIpInstance("CommandLambda", {
+    // register a cloudMapService for the Application in our namespace
+    // chose a sensible default - but allow an alteration in case I guess someone might
+    // want to run two Elsa *in the same infrastructure*
+    const commandService = new Service(this, "CloudMapService", {
+      namespace: props.cloudMapNamespace,
+      name: "Command",
+    });
+
+    // we register it into the cloud map namespace so outside CLI tools can locate it
+    commandService.registerNonIpInstance("CloudMapLambdaInstance", {
       customAttributes: {
         lambdaArn: commandFunction.functionArn,
       },
@@ -136,6 +126,8 @@ export class ElsaDataApplicationCommandConstruct extends Construct {
 
     const f = new NodejsFunction(this, "CommandLambda", {
       entry: entry,
+      // note this is *just* the memory to launch the ECS task - so ECS task memory is
+      // set elsewhere
       memorySize: 128,
       timeout: Duration.minutes(14),
       // by specifying the precise runtime - the bundler knows exactly what packages are already in

packages/aws-application/elsa-data-stack.ts

@@ -3,7 +3,7 @@ import { Construct } from "constructs";
 import { ElsaDataApplicationConstruct } from "./app/elsa-data-application-construct";
 import { ElsaDataStackSettings } from "./elsa-data-stack-settings";
 import { InfrastructureClient } from "@elsa-data/aws-infrastructure-client";
-import { ElsaDataApplicationCommandConstruct } from "./app-command/elsa-data-application-command-construct";
+import { ElsaDataCommandConstruct } from "./command/elsa-data-command-construct";
 import { ClusterConstruct } from "./construct/cluster-construct";
 import { RetentionDays } from "aws-cdk-lib/aws-logs";
 import { ContainerConstruct } from "./construct/container-construct";
@@ -142,6 +142,8 @@ export class ElsaDataStack extends Stack {
       edgeDbSecurityGroup: edgeDbSecurityGroup,
       accessSecretsPolicyStatement:
         infraClient.getSecretPolicyStatementFromLookup(this),
+      discoverServicesPolicyStatement:
+        infraClient.getCloudMapDiscoveryPolicyStatementFromLookup(this),
       tempBucket: tempBucket,
       ...applicationProps,
     });
@@ -161,12 +163,12 @@ export class ElsaDataStack extends Stack {
       logStreamPrefix: "elsa",
     });
 
-    new ElsaDataApplicationCommandConstruct(this, "AppCommand", {
+    new ElsaDataCommandConstruct(this, "AppCommand", {
       cluster: cluster,
       container: container,
       taskDefinition: appCommandDef,
       appService: app.fargateService(),
-      cloudMapService: cloudMapService,
+      cloudMapNamespace: namespace,
       edgeDbSecurityGroup: edgeDbSecurityGroup,
       accessSecretsPolicyStatement:
         infraClient.getSecretPolicyStatementFromLookup(this),

packages/aws-application/helper/bucket-names-to-policy.ts

@@ -0,0 +1,40 @@
+import { PolicyStatement } from "aws-cdk-lib/aws-iam";
+
+/**
+ * Construct a list of PolicyStatement objects that give us the desired
+ * access to a user specified data buckets object.
+ *
+ * @param arnPartition
+ * @param dataBucketPaths
+ */
+export function getPolicyStatementsFromDataBucketPaths(
+  arnPartition: string,
+  dataBucketPaths: { [bucket: string]: string[] }
+) {
+  const stmts: PolicyStatement[] = [];
+
+  // restrict our Get operations to a very specific set of keys in the named buckets
+  for (const [bucketName, keyWildcards] of Object.entries(dataBucketPaths)) {
+    stmts.push(
+      new PolicyStatement({
+        actions: ["s3:GetObject"],
+        // NOTE: we could consider restricting to region or account here in constructing the ARNS
+        // but given the bucket names are already globally specific we leave them open
+        resources: keyWildcards.map(
+          (k) => `arn:${arnPartition}:s3:::${bucketName}/${k}`
+        ),
+      })
+    );
+  }
+
+  stmts.push(
+    new PolicyStatement({
+      actions: ["s3:ListBucket"],
+      resources: Object.keys(dataBucketPaths).map(
+        (b) => `arn:${arnPartition}:s3:::${b}`
+      ),
+    })
+  );
+
+  return stmts;
+}