Skip to content

Commit

Permalink
docs(examples): Add Kinesis Data Stream Autoscale (#146)
Browse files Browse the repository at this point in the history
  • Loading branch information
jshlbrd authored Mar 17, 2024
1 parent 7012181 commit f5dcbf5
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 0 deletions.
26 changes: 26 additions & 0 deletions examples/terraform/aws/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,32 @@ flowchart LR

# Kinesis

## Autoscale

Deploys a Kinesis Data Stream with autoscaling enabled. This can also be used without Substation to manage Kinesis Data Streams.

```mermaid
flowchart LR
kds[("Kinesis
Data Stream")]
sns("Autoscale SNS Topic")
cw_upscale("CloudWatch Upscale Alarm")
cw_downscale("CloudWatch Downscale Alarm")
autoscale("Autoscale Lambda")
autoscale -- UpdateShardCount API --> kds
autoscale -- PutMetricAlarm API ---> cw_upscale
autoscale -- PutMetricAlarm API ---> cw_downscale
cw_downscale -. notifies .- sns
cw_upscale -. notifies .- sns
sns -- notifies ---> autoscale
cw_downscale -. monitors .- kds
cw_upscale -. monitors .- kds
```

## Firehose

Deploys a [Kinesis Data Firehose](https://aws.amazon.com/kinesis/data-firehose/) delivery stream with [data transformation](https://docs.aws.amazon.com/firehose/latest/dev/data-transformation.html) enabled.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
provider "aws" {
# profile = "default"
region = "us-east-1"
}
83 changes: 83 additions & 0 deletions examples/terraform/aws/kinesis/autoscale/terraform/_resources.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# Repository for the Autoscale app.
module "ecr" {
source = "../../../../../../build/terraform/aws/ecr"

config = {
name = "autoscale"
force_delete = true
}
}

# SNS topic for Kinesis Data Stream autoscale alarms.
resource "aws_sns_topic" "autoscale" {
name = "autoscale"
}

# Kinesis Data Stream that is managed by the Autoscale app.
module "kds" {
source = "../../../../../../build/terraform/aws/kinesis_data_stream"

config = {
name = "substation"
autoscaling_topic = aws_sns_topic.autoscale.arn
}

# Min and max shards can be defined as tags to override changes made
# by the Autoscale app.
tags = {
MinimumShards = 2
MaximumShards = 4
}

# Add additional consumer and producer roles as needed.
access = [
# Autoscales the stream.
module.lambda_autoscale.role.name,
]
}

# Lambda Autoscale application that manages Kinesis Data Streams.
module "lambda_autoscale" {
source = "../../../../../../build/terraform/aws/lambda"

config = {
name = "autoscale"
description = "Autoscaler for Kinesis Data Streams."
image_uri = "${module.ecr.url}:latest" # This should use the project's release tags.
image_arm = true

# Override the default Autoscale configuration using environment variables.
# These are the default settings, included for demonstration purposes.
env = {
"AUTOSCALE_KINESIS_THRESHOLD" : 0.7,
"AUTOSCALE_KINESIS_UPSCALE_DATAPOINTS" : 5,
"AUTOSCALE_KINESIS_DOWNSCALE_DATAPOINTS" : 60,
}
}

depends_on = [
module.ecr.url,
]
}

resource "aws_sns_topic_subscription" "autoscale_subscription" {
topic_arn = aws_sns_topic.autoscale.arn
protocol = "lambda"
endpoint = module.lambda_autoscale.arn

depends_on = [
module.lambda_autoscale.name
]
}

resource "aws_lambda_permission" "autoscale_invoke" {
statement_id = "AllowExecutionFromSNS"
action = "lambda:InvokeFunction"
function_name = module.lambda_autoscale.name
principal = "sns.amazonaws.com"
source_arn = aws_sns_topic.autoscale.arn

depends_on = [
module.lambda_autoscale.name
]
}

0 comments on commit f5dcbf5

Please sign in to comment.