forked from aws/aws-cdk
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(aws-glue): adds AWS Glue view L2 construct
- Loading branch information
Showing
7 changed files
with
1,257 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,149 @@ | ||
import { Fn, Resource } from '@aws-cdk/core'; | ||
import { Construct } from 'constructs'; | ||
import { IDatabase } from './database'; | ||
import { CfnTable } from './glue.generated'; | ||
import { Column, Schema } from './schema'; | ||
import { ITable } from './table'; | ||
|
||
/** | ||
* View props. | ||
*/ | ||
export interface ViewProps { | ||
/** | ||
* Name of the table. | ||
*/ | ||
readonly tableName: string; | ||
|
||
/** | ||
* Description of the table. | ||
* | ||
* @default generated | ||
*/ | ||
readonly description?: string; | ||
|
||
/** | ||
* Database in which to store the table. | ||
*/ | ||
readonly database: IDatabase; | ||
|
||
/** | ||
* Columns of the table. | ||
*/ | ||
readonly columns: Column[]; | ||
|
||
/** | ||
* Statement to run for this view. | ||
*/ | ||
readonly statement: string; | ||
|
||
/** | ||
* Placeholders to replace in the statement. | ||
* | ||
* @default - `database` resolves to the databaseName of {database} and can't be overridden. | ||
*/ | ||
readonly placeHolders?: { [key: string]: string }; | ||
} | ||
|
||
// the mapping of athena create table to PrestoDB types | ||
const prestoTypes: { [key: string]: string } = { | ||
[Schema.FLOAT.inputString]: 'real', | ||
[Schema.BIG_INT.inputString]: 'bigint', | ||
[Schema.INTEGER.inputString]: 'integer', | ||
[Schema.SMALL_INT.inputString]: 'smallint', | ||
[Schema.TINY_INT.inputString]: 'tinyint', | ||
[Schema.STRING.inputString]: 'varchar', | ||
[Schema.BINARY.inputString]: 'varbinary', | ||
}; | ||
|
||
/** | ||
* A View in AWS Glue, implemented by an AWS Glue table. | ||
* | ||
* @resource AWS::glue::Table | ||
*/ | ||
export class View extends Resource implements ITable { | ||
/** | ||
* Name of this table. | ||
*/ | ||
public readonly tableName: string; | ||
|
||
/** | ||
* ARN of this table. | ||
*/ | ||
public readonly tableArn: string; | ||
|
||
/** | ||
* @summary Creates a Table construct that represents a view in the AWS Glue data catalogue. | ||
* | ||
* @param scope The scope creating construct (usually `this`). | ||
* @param id The construct's id. | ||
* @param props Properties. | ||
*/ | ||
constructor(scope: Construct, id: string, props: ViewProps) { | ||
super(scope, id, { | ||
physicalName: props.tableName, | ||
}); | ||
|
||
const columns = props.columns; | ||
|
||
const placeHolders = {}; | ||
if (props.placeHolders) { | ||
Object.assign(placeHolders, props.placeHolders); | ||
} | ||
Object.assign(placeHolders, { database: props.database.databaseName }); | ||
|
||
const viewOriginalText = { | ||
originalSql: props.statement, | ||
catalog: 'awsdatacatalog', | ||
columns: this.renderColumns(columns, true), | ||
schema: '${database}', | ||
}; | ||
|
||
const tableResource = new CfnTable(this, 'Table', { | ||
catalogId: props.database.catalogId, | ||
databaseName: props.database.databaseName, | ||
tableInput: { | ||
name: this.physicalName, | ||
description: props.description || `${props.tableName} generated by CDK`, | ||
parameters: { presto_view: true }, | ||
storageDescriptor: { | ||
columns: this.renderColumns(columns), | ||
serdeInfo: {}, | ||
}, | ||
partitionKeys: [], | ||
tableType: 'VIRTUAL_VIEW', | ||
viewOriginalText: | ||
'/* Presto View: ' + Fn.base64(Fn.sub(JSON.stringify(viewOriginalText), placeHolders)) + ' */', | ||
}, | ||
}); | ||
this.node.defaultChild = tableResource; | ||
|
||
this.tableName = this.getResourceNameAttribute(tableResource.ref); | ||
this.tableArn = this.stack.formatArn({ | ||
service: 'glue', | ||
resource: 'table', | ||
resourceName: `${props.database.databaseName}/${this.tableName}`, | ||
}); | ||
} | ||
|
||
/** | ||
* Writes a json object used by glue from the list of columns. | ||
* | ||
* @param columns the columns | ||
* @param usePrestoTypes if `true`, replaces all data types with their presto types | ||
*/ | ||
private renderColumns(columns?: Column[], usePrestoTypes = false) { | ||
if (columns === undefined) { | ||
return undefined; | ||
} | ||
return columns.map((column) => { | ||
return { | ||
name: column.name, | ||
type: | ||
usePrestoTypes && column.type.inputString in prestoTypes | ||
? prestoTypes[column.type.inputString] | ||
: column.type.inputString, | ||
comment: column.comment, | ||
}; | ||
}); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
SELECT | ||
* | ||
, "$path" "file" | ||
FROM | ||
${sourceTable} | ||
WHERE ("concat"("year", "month", "day", "hour") >= "date_format"("date_trunc"('hour', ((current_timestamp - INTERVAL '15' MINUTE) - INTERVAL '1' HOUR)), '%Y%m%d%H')) | ||
UNION ALL SELECT | ||
* | ||
, "$path" "file" | ||
FROM | ||
${targetTable} | ||
WHERE ("concat"("year", "month", "day", "hour") < "date_format"("date_trunc"('hour', ((current_timestamp - INTERVAL '15' MINUTE) - INTERVAL '1' HOUR)), '%Y%m%d%H')) |
Oops, something went wrong.