Merge pull request #20 from elsa-data/feature/pre-uom-demo-changes

Lots of tidies up for uom demo

dev/README.md

@@ -1,6 +1,6 @@
 # Application manual deploy
 
-> :warning: \*_You must also build the workload CDK or this will not work_! See below
+> :warning: \*You must also build the workload CDK or this will not work! See below
 
 This CDK is for direct deployment to the UMCCR dev account
 from a developers machine.
@@ -14,13 +14,17 @@ It is designed for rapid development so does not use CI pipelines -
 it directly deploys the locally built solution - whatever is on the devs machine
 gets deployed.
 
-NOTE: THE CDK DEPLOYED IS THAT WHICH HAS BEEN BUILT IN THE WORKLOAD FOLDER
+NOTE: THE CDK DEPLOYED IS THAT WHICH HAS BEEN BUILT IN
+THE PACKAGES FOLDER - THAT IS THE COMPILED JAVASCRIPT
+FILES AND NOT THE TYPESCRIPT
 
-This project uses a local package reference in `package.json` - which means that
-the referred package is not "built" when we do an `npm install`.
+Open another window and do a `pnpm -w run watch` to make sure that
+this CDK deployment is continuously being kept up to date. Then use `cdk`
+directly.
 
-That means you need to open another window and do a `npm run build` or
-`npm run build:watch` (in the `workload-elsa-data-aws-application` folder)
-to make sure that this CDK deployment is "up to date".
+-OR-
 
-`npm run deploy`
+You can run CDK deploy/destroy steps that include a "pre-build".
+
+`pnpm run deploy`
+`pnpm run destroy`

dev/dev-docker-image/base.json5

@@ -3,25 +3,16 @@
   ontoFhirUrl: "https://onto.prod.umccr.org/fhir",
   releaseKeyPrefix: "R",
   ipLookup: {
-    maxMindDbPath: "Geoacumen-Country.mmdb",
+    maxMindDbPath: "/dev-config/Geoacumen-Country.mmdb",
   },
-  sharers: [
-    {
-      id: "object-signing",
-      type: "object-signing",
-      maxAgeInSeconds: 600,
-    },
-    {
-      id: "copy-out",
-      type: "copy-out",
-    },
-    {
-      id: "umccr-htsget",
-      type: "htsget",
-      url: "https://htsget.elsa.dev.umccr.org",
-      maxAgeInSeconds: 600,
-    },
-  ],
+  feature: {
+    enableConsentDisplay: true,
+    enableCohortConstructor: true,
+    enableDataEgressViewer: false,
+  },
+  // datasetEgress: {
+  //  updateInterval: "0 6 * * *", // 6am
+  // },
   logger: {
     level: "debug",
   },

dev/dev-docker-image/sharers.json5

@@ -15,5 +15,23 @@
       url: "https://htsget.elsa.dev.umccr.org",
       maxAgeInSeconds: 600,
     },
+    {
+      id: "parkville-ap",
+      type: "aws-access-point",
+      allowedVpcs: {
+        "UMCCR Nextflow Tower Research Group 1": {
+          accountId: "842385035780",
+          vpcId: "vpc-0d1e5b642f953920f",
+        },
+        "UMCCR Nextflow Tower Research Group 2": {
+          accountId: "842385035780",
+          vpcId: "vpc-0ae1fbadcf21859f3",
+        },
+        "UoM Demo": {
+          accountId: "534840902377",
+          vpcId: "vpc-031e3d24ea0e50664",
+        },
+      },
+    },
   ],
 }

dev/dev.ts

@@ -37,9 +37,9 @@ new ElsaDataStack(
     },
     description: descriptionWithTag(undefined),
     tags: {
-      "umccr-org:ProductVersion": DEV_DEPLOYED_IMAGE_TAG,
-      "umccr-org:Stack": "ElsaDataApplication",
-      "umccr-org:Product": "ElsaData",
+      ProductVersion: DEV_DEPLOYED_IMAGE_TAG,
+      Stack: "ElsaDataApplication",
+      Product: "ElsaData",
     },
   },
   {

dev/package.json

@@ -3,7 +3,10 @@
   "version": "0.0.0",
   "private": true,
   "description": "Manual deploy of the CDK Elsa Data workload for dev purposes",
-  "scripts": {},
+  "scripts": {
+    "deploy": "pnpm -w run build && cdk deploy",
+    "destroy": "pnpm -w run build && cdk destroy"
+  },
   "devDependencies": {
     "@tsconfig/node18": "1.0.3",
     "@types/node": "18.17.12",

packages/aws-application/README.md

@@ -0,0 +1 @@
+A README that should tell us about this application

packages/aws-application/app/elsa-data-application-app-runner-construct.ts

@@ -24,6 +24,8 @@ import * as apprunner from "@aws-cdk/aws-apprunner-alpha";
 // WIP warning
 // was used to test the concept
 // waiting on a CDK construct for settings the deployed URL and then we should revisit
+// THIS GETS SLOWLY OUT OF DATA WITH THE REAL APPLICATION CONSTRUCT - SO PLEASE DO A CHECK FOR CHANGES THERE
+// AND IMPLEMENT HERE
 //
 
 interface Props extends StackProps {

packages/aws-application/app/elsa-data-application-construct.ts

@@ -3,7 +3,7 @@ import { Construct } from "constructs";
 import { DockerServiceWithHttpsLoadBalancerConstruct } from "../construct/docker-service-with-https-load-balancer-construct";
 import { Policy, PolicyStatement } from "aws-cdk-lib/aws-iam";
 import { ISecurityGroup } from "aws-cdk-lib/aws-ec2";
-import { Service } from "aws-cdk-lib/aws-servicediscovery";
+import { INamespace, Service } from "aws-cdk-lib/aws-servicediscovery";
 import { IHostedZone } from "aws-cdk-lib/aws-route53";
 import { ICertificate } from "aws-cdk-lib/aws-certificatemanager";
 import { ElsaDataApplicationSettings } from "../elsa-data-application-settings";
@@ -21,11 +21,13 @@ interface Props extends ElsaDataApplicationSettings {
 
   readonly taskDefinition: TaskDefinitionConstruct;
 
-  readonly cloudMapService: Service;
+  readonly cloudMapNamespace: INamespace;
 
   readonly hostedZone: IHostedZone;
   readonly hostedZoneCertificate: ICertificate;
 
+  readonly deployedUrl: string;
+
   // the security group of our edgedb - that we will put ourselves in to enable access
   readonly edgeDbSecurityGroup: ISecurityGroup;
 
@@ -92,6 +94,8 @@ export class ElsaDataApplicationConstruct extends Construct {
       })
     );
 
+    // TODO consider moving all the "write" permissions here to be a CMD level (i.e. cluster admins only)
+    // and only have "read" permissions here
     if (props.awsPermissions.enableAccessPoints) {
       policy.addStatements(
         // temporarily give all S3 accesspoint perms - can we tighten?
@@ -194,6 +198,21 @@ export class ElsaDataApplicationConstruct extends Construct {
     props.tempBucket.grantReadWrite(
       this.privateServiceWithLoadBalancer.service.taskDefinition.taskRole
     );
+
+    // register a cloudMapService for the Application in our namespace
+    // chose a sensible default - but allow an alteration in case I guess someone might
+    // want to run two Elsa *in the same infrastructure*
+    const service = new Service(this, "CloudMapService", {
+      namespace: props.cloudMapNamespace,
+      name: "Application",
+      description: "Web application",
+    });
+
+    service.registerNonIpInstance("CloudMapCustomAttributes", {
+      customAttributes: {
+        deployedUrl: props.deployedUrl,
+      },
+    });
   }
 
   public fargateService(): FargateService {

packages/aws-application/command/command-lambda/index.mjs

@@ -30,6 +30,7 @@ export const handler = async (event) => {
   const clusterLogGroupName = process.env["CLUSTER_LOG_GROUP_NAME"];
   const taskDefinitionArn = process.env["TASK_DEFINITION_ARN"];
   const containerName = process.env["CONTAINER_NAME"];
+  const logStreamPrefix = process.env["LOG_STREAM_PREFIX"];
   const subnets = process.env["SUBNETS"];
   const securityGroups = process.env["SECURITY_GROUPS"];
 
@@ -38,17 +39,19 @@ export const handler = async (event) => {
     !clusterLogGroupName ||
     !taskDefinitionArn ||
     !containerName ||
+    !logStreamPrefix ||
     !subnets ||
     !securityGroups
   )
     throw new Error(
-      "Cluster settings must be passed in via environment variables"
+      "Lambda must be invoked with cluster settings passed in via environment variables"
     );
 
   console.log(clusterArn);
   console.log(clusterLogGroupName);
   console.log(taskDefinitionArn);
   console.log(containerName);
+  console.log(logStreamPrefix);
   console.log(subnets);
   console.log(securityGroups);
 
@@ -89,9 +92,7 @@ export const handler = async (event) => {
     const taskArnSplit = taskArn.split("/");
 
     if (taskArnSplit.length === 3) {
-      // TODO these come from the parent   logStreamPrefix/containername
-      //      so possibly pass these rather than fix them here
-      logStreamName = `elsa/ElsaData/${taskArnSplit[2]}`;
+      logStreamName = `${logStreamPrefix}/${containerName}/${taskArnSplit[2]}`;
     }
 
     let lastStatus = result.tasks[0].lastStatus;
@@ -111,6 +112,8 @@ export const handler = async (event) => {
       await sleep(10000);
     }
 
+    // we return the log group name and stream name so the bash invoker can fetch all
+    // the messages and display them to the user
     return {
       message: "Success",
       logGroupName: clusterLogGroupName,

packages/aws-application/command/elsa-data-command-construct.ts

@@ -72,7 +72,8 @@ export class ElsaDataCommandConstruct extends Construct {
       })
     );
 
-    // the permissions of the running container (i.e all AWS functionality used by Elsa Data code)
+    // the permissions of the running Command container
+    // (i.e all AWS functionality used by Elsa Data code FOR ADMIN TASKS)
     props.taskDefinition.taskDefinition.taskRole.attachInlinePolicy(policy);
 
     // the command function is an invocable lambda that will then go and spin up an ad-hoc Task in our
@@ -86,12 +87,11 @@ export class ElsaDataCommandConstruct extends Construct {
       props.appService
     );
 
-    // register a cloudMapService for the Application in our namespace
-    // chose a sensible default - but allow an alteration in case I guess someone might
-    // want to run two Elsa *in the same infrastructure*
+    // register a cloudMapService for the Command infrastructure in our namespace
     const commandService = new Service(this, "CloudMapService", {
       namespace: props.cloudMapNamespace,
       name: "Command",
+      description: "Command service for administration activity",
     });
 
     // we register it into the cloud map namespace so outside CLI tools can locate it
@@ -140,6 +140,7 @@ export class ElsaDataCommandConstruct extends Construct {
         TASK_DEFINITION_ARN: taskDefinition.taskDefinition.taskDefinitionArn,
         SERVICE_ARN: appService.serviceArn,
         CONTAINER_NAME: container.containerName,
+        LOG_STREAM_PREFIX: taskDefinition.logStreamPrefix,
         // we are passing to the lambda the subnets and security groups that need to be used
         // by the Fargate task it will invoke
         SUBNETS: cluster.vpc

packages/aws-application/construct/task-definition-construct.ts

@@ -28,18 +28,26 @@ type Props = {
   readonly cpu: number;
   readonly cpuArchitecture: CpuArchitecture;
 
+  // a string that is sent to our AWS logger to be the stream prefix
+  // needs to be recorded and available so that we can coordinate fetching of logs
   readonly logStreamPrefix: string;
 };
 
 /**
  * A construct for a TaskDefinition that can run our Elsa Data container.
+ * This definition gets used for both the running 24/7 website AND for
+ * one-off tasks that do "admin" work. They may get given different
+ * permissions outside of this construct.
  */
 export class TaskDefinitionConstruct extends Construct {
   public readonly taskDefinition: FargateTaskDefinition;
+  public readonly logStreamPrefix: string;
 
   constructor(scope: Construct, id: string, props: Props) {
     super(scope, id);
 
+    this.logStreamPrefix = props.logStreamPrefix;
+
     // we do the task definition by hand as we have some specialised settings
     this.taskDefinition = new FargateTaskDefinition(this, "TaskDefinition", {
       runtimePlatform: {

packages/aws-application/elsa-data-application-settings.ts

@@ -76,11 +76,6 @@ export interface ElsaDataApplicationSettings {
    * The cpu assigned to the Elsa Data application container - defaults to something sensible
    */
   readonly cpu?: number;
-
-  /**
-   * If present, an alternative CloudMapService name for the application - defaults to Application
-   */
-  readonly serviceName?: string;
 }
 
 export interface ElsaDataApplicationBuildLocal {