Merge pull request #642 from awslabs/bump/2.5.0

chore(release): 2.5.0
awslabs · Mar 30, 2022 · f438c8d · f438c8d
2 parents f65d5fe + 8ec92d9
commit f438c8d
Show file tree

Hide file tree

Showing 24 changed files with 1,152 additions and 115 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,14 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.148.0](https://github.com/awslabs/aws-solutions-constructs/compare/v1.147.0...v1.148.0) (2022-03-30)
+
+* Upgraded all patterns to CDK v1.148.0
+
+## [1.147.0](https://github.com/awslabs/aws-solutions-constructs/compare/v2.4.0...v1.147.0) (2022-03-29)
+
+* Upgraded all patterns to CDK v1.147.0
+
 ## [1.146.0](https://github.com/awslabs/aws-solutions-constructs/compare/v1.145.0...v1.146.0) (2022-03-02)
 
 * Upgraded all patterns to CDK v1.146.0

diff --git a/CHANGELOG.v2.md b/CHANGELOG.v2.md
@@ -2,11 +2,17 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [2.5.0](https://github.com/awslabs/aws-solutions-constructs/compare/v2.4.0...v2.5.0) (2022-03-30)
+
+### Features
+* Includes all functionality of V1.148.0
+* Built upon underlying CDK version V2.15.0
+
 ## [2.4.0](https://github.com/awslabs/aws-solutions-constructs/compare/v2.3.0...v2.4.0) (2022-03-29)
 
 ### Features
 * Includes all functionality of V1.146.0
-* Built upon underlying CDK version V2.10.0
+* Built upon underlying CDK version V2.15.0
 
 ## [2.3.0](https://github.com/awslabs/aws-solutions-constructs/compare/v2.2.0...v2.3.0) (2022-02-24)
 

diff --git a/deployment/v2/align-version.js b/deployment/v2/align-version.js
@@ -10,7 +10,7 @@ const findVersion = process.argv[2];
 const replaceVersion = process.argv[3];
 
 // these versions need to be sourced from a config file
-const awsCdkLibVersion = '2.10.0';
+const awsCdkLibVersion = '2.15.0';
 const constructsVersion = '10.0.0';
 const MODULE_EXEMPTIONS = new Set([
   '@aws-cdk/cloudformation-diff',

diff --git a/source/lerna.json b/source/lerna.json
@@ -6,5 +6,5 @@
     "./patterns/@aws-solutions-constructs/*"
   ],
   "rejectCycles": "true",
-  "version": "1.146.0"
+  "version": "1.148.0"
 }
diff --git a/source/lerna.v2.json b/source/lerna.v2.json
@@ -6,5 +6,5 @@
     "./patterns/@aws-solutions-constructs/*"
   ],
   "rejectCycles": "true",
-  "version": "2.4.0"
+  "version": "2.5.0"
 }
diff --git a/...solutions-constructs/aws-iot-lambda-dynamodb/test/integ.iot-lambda-dynamodb.expected.json b/...solutions-constructs/aws-iot-lambda-dynamodb/test/integ.iot-lambda-dynamodb.expected.json
@@ -80,7 +80,8 @@
                 "dynamodb:BatchWriteItem",
                 "dynamodb:PutItem",
                 "dynamodb:UpdateItem",
-                "dynamodb:DeleteItem"
+                "dynamodb:DeleteItem",
+                "dynamodb:DescribeTable"
               ],
               "Effect": "Allow",
               "Resource": [

diff --git a/...tterns/@aws-solutions-constructs/aws-iot-lambda-dynamodb/test/iot-lambda-dynamodb.test.ts b/...tterns/@aws-solutions-constructs/aws-iot-lambda-dynamodb/test/iot-lambda-dynamodb.test.ts
@@ -223,7 +223,8 @@ test('check lambda function policy ', () => {
             "dynamodb:BatchWriteItem",
             "dynamodb:PutItem",
             "dynamodb:UpdateItem",
-            "dynamodb:DeleteItem"
+            "dynamodb:DeleteItem",
+            "dynamodb:DescribeTable"
           ],
           Effect: "Allow",
           Resource: [

diff --git a/source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/README.md b/source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/README.md
@@ -30,44 +30,44 @@ This AWS Solutions Construct deploys a Kinesis Stream and configures a AWS Glue
 Here is a minimal deployable pattern definition in Typescript:
 
 ```javascript
-import * as glue from '@aws-cdk/aws-glue';
-import * as s3assets from '@aws-cdk/aws-s3-assets';
-import {KinesisstreamsToGluejob} from '@aws-solutions-constructs/aws-kinesisstreams-gluejob';
+import * as glue from "@aws-cdk/aws-glue";
+import * as s3assets from "@aws-cdk/aws-s3-assets";
+import { KinesisstreamsToGluejob } from "@aws-solutions-constructs/aws-kinesisstreams-gluejob";
 
 const fieldSchema: glue.CfnTable.ColumnProperty[] = [
-    {
-        name: 'id',
-        type: 'int',
-        comment: 'Identifier for the record',
-    },
-    {
-        name: 'name',
-        type: 'string',
-        comment: 'Name for the record',
-    },
-    {
-        name: 'address',
-        type: 'string',
-        comment: 'Address for the record',
-    },
-    {
-        name: 'value',
-        type: 'int',
-        comment: 'Value for the record',
-    },
+  {
+    name: "id",
+    type: "int",
+    comment: "Identifier for the record",
+  },
+  {
+    name: "name",
+    type: "string",
+    comment: "Name for the record",
+  },
+  {
+    name: "address",
+    type: "string",
+    comment: "Address for the record",
+  },
+  {
+    name: "value",
+    type: "int",
+    comment: "Value for the record",
+  },
 ];
 
-const customEtlJob = new KinesisstreamsToGluejob(this, 'CustomETL', {
-    glueJobProps: {
-        command: {
-            name: 'gluestreaming',
-            pythonVersion: '3',
-            scriptLocation: new s3assets.Asset(this, 'ScriptLocation', {
-                path: `${__dirname}/../etl/transform.py`,
-            }).s3ObjectUrl,
-        },
+const customEtlJob = new KinesisstreamsToGluejob(this, "CustomETL", {
+  glueJobProps: {
+    command: {
+      name: "gluestreaming",
+      pythonVersion: "3",
     },
-    fieldSchema: fieldSchema,
+  },
+  fieldSchema: fieldSchema,
+  etlCodeAsset: new s3assets.Asset(this, "ScriptLocation", {
+    path: `${__dirname}/../etl/transform.py`,
+  }),
 });
 ```
 
@@ -87,17 +87,18 @@ _Parameters_
 
 | **Name**            | **Type**                                                                                                                      | **Description**                                                                                                  |
 | :------------------ | :---------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- |
-| existingStreamObj?  | [`kinesis.Stream`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kinesis.Stream.html)                          | Existing instance of Kinesis Stream, providing both this and `kinesisStreamProps` will cause an error.                          |
+| existingStreamObj?  | [`kinesis.Stream`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kinesis.Stream.html)                          | Existing instance of Kinesis Stream, providing both this and `kinesisStreamProps` will cause an error.           |
 | kinesisStreamProps? | [`kinesis.StreamProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-kinesis.StreamProps.html)                | Optional user-provided props to override the default props for the Kinesis stream.                               |
 | glueJobProps?       | [`cfnJob.CfnJobProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnJobProps.html)                    | User provided props to override the default props for the AWS Glue Job.                                          |
-| existingGlueJob?    | [`cfnJob.CfnJob`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnJob.html)                              | Existing instance of AWS Glue Job, providing both this and `glueJobProps` will cause an error.                         |
+| existingGlueJob?    | [`cfnJob.CfnJob`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnJob.html)                              | Existing instance of AWS Glue Job, providing both this and `glueJobProps` will cause an error.                   |
 | fieldSchema?        | [`CfnTable.ColumnProperty[]`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnTable.ColumnProperty.html) | User provided schema structure to create an AWS Glue Table.                                                      |
 | existingTable?      | [`CfnTable`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnTable.html)                                 | Existing instance of AWS Glue Table. If this is set, tableProps and fieldSchema are ignored.                     |
 | tableProps?         | [`CfnTableProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.TableProps.html)                          | User provided AWS Glue Table props to override default props used to create a Glue Table.                        |
 | existingDatabase?   | [`CfnDatabase`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnDatabase.html)                           | Existing instance of AWS Glue Database. If this is set, then databaseProps is ignored.                           |
 | databaseProps?      | [`CfnDatabaseProps`](https://docs.aws.amazon.com/cdk/api/latest/docs/@aws-cdk_aws-glue.CfnDatabaseProps.html)                 | User provided Glue Database Props to override the default props used to create the Glue Database.                |
 | outputDataStore?    | [`SinkDataStoreProps`](#sinkdatastoreprops)                                                                                   | User provided properties for S3 bucket that stores Glue Job output. Current datastore types suported is only S3. |
 |createCloudWatchAlarms?|`boolean`|Whether to create recommended CloudWatch alarms for Kinesis Data Stream. Default value is set to `true`.|
+| etlCodeAsset?       | [s3assets.Asset](https://docs.aws.amazon.com/cdk/api/v1/docs/@aws-cdk_aws-s3-assets.Asset.html)                               | User provided instance of the Asset class that represents the ETL code on the local filesytem                    |
 
 ### SinkDataStoreProps
 

diff --git a/source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/lib/index.ts b/source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/lib/index.ts
@@ -11,13 +11,14 @@
  *  and limitations under the License.
  */
 
-import * as glue from '@aws-cdk/aws-glue';
-import { Effect, IRole, Policy, PolicyStatement } from '@aws-cdk/aws-iam';
-import { Stream, StreamProps } from '@aws-cdk/aws-kinesis';
-import { Bucket } from '@aws-cdk/aws-s3';
-import { Aws, Construct } from '@aws-cdk/core';
-import * as cloudwatch from '@aws-cdk/aws-cloudwatch';
-import * as defaults from '@aws-solutions-constructs/core';
+import * as glue from "@aws-cdk/aws-glue";
+import * as s3assets from "@aws-cdk/aws-s3-assets";
+import { Effect, IRole, Policy, PolicyStatement } from "@aws-cdk/aws-iam";
+import { Stream, StreamProps } from "@aws-cdk/aws-kinesis";
+import { Bucket } from "@aws-cdk/aws-s3";
+import { Aws, Construct } from "@aws-cdk/core";
+import * as cloudwatch from "@aws-cdk/aws-cloudwatch";
+import * as defaults from "@aws-solutions-constructs/core";
 
 export interface KinesisstreamsToGluejobProps {
   /**
@@ -45,7 +46,19 @@ export interface KinesisstreamsToGluejobProps {
    */
   readonly glueJobProps?: glue.CfnJobProps | any;
   /**
-   * Existing GlueJob configuration. If this property is provided, any properties provided through @glueJobProps is ignored
+   * Existing GlueJob configuration. If this property is provided, any properties provided through @glueJobProps is ignored.
+   * The ETL script can be provided either under glue.CfnJob.JobCommandProperty or set as an Asset instance under
+   * @KinesisstreamsToGluejobProps.etlCodeAsset.
+   *
+   * If an S3 location is know and exists, provide the S3 url in the `scriptLocation` attribute in glue.CfnJob.JobCommandProperty as an
+   * S3 format URL (example: `s3://bucketname/keyprefix.py`)
+   *
+   * If the ETL script exists as a local files or directories, create an instance of the Asset (aws-cdk-lib » aws_s3_assets) class
+   * set the @KinesisstreamsToGluejobProps.etlCodeAsset.
+   *
+   * In case both (JobCommandProperty.scriptLocation and @KinesisstreamsToGluejobProps.etlCodeAsset) are set,
+   * @KinesisstreamsToGluejobProps.etlCodeAsset will take higher precedence and override the JobCommandProperty.scriptLocation
+   *
    */
   readonly existingGlueJob?: glue.CfnJob;
   /**
@@ -71,7 +84,7 @@ export interface KinesisstreamsToGluejobProps {
    *
    * @default - None
    */
-  readonly fieldSchema?: glue.CfnTable.ColumnProperty [];
+  readonly fieldSchema?: glue.CfnTable.ColumnProperty[];
   /**
    * Glue Table for this construct, If not provided the construct will create a new Table in the
    * database. This table should define the schema for the records in the Kinesis Data Streams.
@@ -107,6 +120,17 @@ export interface KinesisstreamsToGluejobProps {
    * @default - Alarms are created
    */
   readonly createCloudWatchAlarms?: boolean;
+  /**
+   * Provide Asset instance corresponding to the code in the local filesystem, responsible for
+   * performing the Glue Job transformation. This property will override any S3 locations provided
+   * under glue.CfnJob.JobCommandProperty
+   *
+   * As of CDK V2, all ETL scripts sourced from local code should explicitly create an asset and provide
+   * that asset through this attribute.
+   *
+   * @default - None
+   */
+  readonly etlCodeAsset?: s3assets.Asset;
 }
 
 /**
@@ -145,6 +169,28 @@ export class KinesisstreamsToGluejob extends Construct {
     super(scope, id);
     defaults.CheckProps(props);
 
+    // custom props check
+    if (props.existingGlueJob && props.glueJobProps) {
+      throw Error("Either existingGlueJob instance or glueJobProps should be set, but found both");
+    }
+
+    if (!props.existingGlueJob) {
+      if (!props.glueJobProps.command.scriptLocation && !props.etlCodeAsset) {
+        throw Error('Either one of CfnJob.JobCommandProperty.scriptLocation or KinesisstreamsToGluejobProps.etlCodeAsset has ' +
+        'to be provided. If the ETL Job code file exists in a local filesystem, please set ' +
+        'KinesisstreamsToGluejobProps.etlCodeAsset. If the ETL Job is available in an S3 bucket, set the ' +
+        'CfnJob.JobCommandProperty.scriptLocation property');
+      }
+
+      if (!props.etlCodeAsset) {
+        const s3Url: string = props.glueJobProps.command.scriptLocation;
+        const found = s3Url.match(/^s3:\/\/\S+\/\S+/g);
+        if (!(found && found.length > 0 && found[0].length === s3Url.length)) {
+          throw Error("Invalid S3 URL provided");
+        }
+      }
+    }
+
     this.kinesisStream = defaults.buildKinesisStream(this, {
       existingStreamObj: props.existingStreamObj,
       kinesisStreamProps: props.kinesisStreamProps,
@@ -153,23 +199,24 @@ export class KinesisstreamsToGluejob extends Construct {
     this.database = props.existingDatabase !== undefined ? props.existingDatabase : defaults.createGlueDatabase(scope, props.databaseProps);
 
     if (props.fieldSchema === undefined && props.existingTable === undefined && props.tableProps === undefined) {
-      throw Error('Either fieldSchema or table property has to be set, both cannot be optional');
+      throw Error("Either fieldSchema or table property has to be set, both cannot be optional");
     }
 
     if (props.existingTable !== undefined) {
       this.table = props.existingTable;
     } else {
-      this.table = defaults.createGlueTable(scope, this.database, props.tableProps, props.fieldSchema, 'kinesis', {
-        STREAM_NAME: this.kinesisStream.streamName
+      this.table = defaults.createGlueTable(scope, this.database, props.tableProps, props.fieldSchema, "kinesis", {
+        STREAM_NAME: this.kinesisStream.streamName,
       });
     }
 
-    [ this.glueJob, this.glueJobRole, this.outputBucket ] = defaults.buildGlueJob(this, {
+    [this.glueJob, this.glueJobRole, this.outputBucket] = defaults.buildGlueJob(this, {
       existingCfnJob: props.existingGlueJob,
       glueJobProps: props.glueJobProps,
       table: this.table!,
       database: this.database!,
-      outputDataStore: props.outputDataStore!
+      outputDataStore: props.outputDataStore!,
+      etlCodeAsset: props.etlCodeAsset
     });
 
     this.glueJobRole = this.buildRolePolicy(scope, id, this.database, this.table, this.glueJob, this.glueJobRole);
@@ -194,47 +241,54 @@ export class KinesisstreamsToGluejob extends Construct {
     const _glueJobPolicy = new Policy(scope, `${id}GlueJobPolicy`, {
       statements: [ new PolicyStatement({
         effect: Effect.ALLOW,
-        actions: [ 'glue:GetJob' ],
-        resources: [ `arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:job/${glueJob.ref}` ]
-      }), new PolicyStatement({
+        actions: ["glue:GetJob"],
+        resources: [
+          `arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:job/${glueJob.ref}`,
+        ],
+      }),
+      new PolicyStatement({
         effect: Effect.ALLOW,
-        actions: [ 'glue:GetSecurityConfiguration' ],
-        resources: [ '*' ] // Security Configurations have no resource ARNs
-      }), new PolicyStatement({
+        actions: ["glue:GetSecurityConfiguration"],
+        resources: ["*"], // Security Configurations have no resource ARNs
+      }),
+      new PolicyStatement({
         effect: Effect.ALLOW,
-        actions: [ 'glue:GetTable' ],
-        resources: [ `arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:table/${glueDatabase.ref}/${glueTable.ref}`,
+        actions: ["glue:GetTable"],
+        resources: [
+          `arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:table/${glueDatabase.ref}/${glueTable.ref}`,
           `arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:database/${glueDatabase.ref}`,
-          `arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:catalog`
-        ]
-      }), new PolicyStatement({
+          `arn:${Aws.PARTITION}:glue:${Aws.REGION}:${Aws.ACCOUNT_ID}:catalog`,
+        ],
+      }),
+      new PolicyStatement({
         effect: Effect.ALLOW,
-        actions: [ 'cloudwatch:PutMetricData' ],
-        resources: [ '*' ], // Metrics do not have resource ARN and hence added conditions
+        actions: ["cloudwatch:PutMetricData"],
+        resources: ["*"], // Metrics do not have resource ARN and hence added conditions
         conditions: {
           StringEquals: {
-            "cloudwatch:namespace": "Glue"
+            "cloudwatch:namespace": "Glue",
           },
           Bool: {
-            "aws:SecureTransport": "true"
-          }
-        }
-      }), new PolicyStatement({
+            "aws:SecureTransport": "true",
+          },
+        },
+      }),
+      new PolicyStatement({
         effect: Effect.ALLOW,
-        actions: [ 'kinesis:DescribeStream', 'kinesis:DescribeStreamSummary', 'kinesis:GetRecords',
-          'kinesis:GetShardIterator', 'kinesis:ListShards', 'kinesis:SubscribeToShard' ],
-        resources: [ this.kinesisStream.streamArn ]
-      })]
+        actions: [ "kinesis:DescribeStream", "kinesis:DescribeStreamSummary", "kinesis:GetRecords",
+          "kinesis:GetShardIterator", "kinesis:ListShards", "kinesis:SubscribeToShard" ],
+        resources: [this.kinesisStream.streamArn],
+      })],
     });
 
     defaults.addCfnSuppressRules(_glueJobPolicy, [
       {
-        id: 'W12',
-        reason: "Glue Security Configuration does not have an ARN, and the policy only allows reading the configuration.            CloudWatch metrics also do not have an ARN but adding a namespace condition to the policy to allow it to            publish metrics only for AWS Glue"
+        id: "W12",
+        reason: "Glue Security Configuration does not have an ARN, and the policy only allows reading the configuration.            CloudWatch metrics also do not have an ARN but adding a namespace condition to the policy to allow it to            publish metrics only for AWS Glue",
       },
     ]);
 
     role.attachInlinePolicy(_glueJobPolicy);
     return role;
   }
-}
+}
diff --git a/source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/test/fakefile.py b/source/patterns/@aws-solutions-constructs/aws-kinesisstreams-gluejob/test/fakefile.py
@@ -0,0 +1 @@
+# empty python file for testing