feat: add service for cronjob and improve scaling using same alarms o…

…k & in status

.github/workflows/ecs-migration.yml

@@ -0,0 +1,60 @@
+name: Deploy to Amazon ECS
+
+on:
+  workflow_run:
+    workflows: ["Deploy to Amazon ECS"]
+    types:
+      - completed
+
+env:
+  REGISTRY: ghcr.io
+  AWS_REGION: us-east-1
+  IMAGE_NAME: hotosm/tasking-manager-backend
+  TASK_DEFINITION: tasking-manager-hotosm-staging-fastapi
+  ECS_CLUSTER: tasking-manager-staging-cluster
+  ECS_SERVICE: tasking-manager-hotosm-staging-fastapi
+  CONTAINER_NAME: tasking-manager-hotosm-staging-fastapi
+  OIDC_ROLE_ARN: arn:aws:iam::670261699094:role/Github-AWS-OIDC
+
+jobs:
+  deploy:
+    name: Deploy to ECS
+    runs-on: ubuntu-latest
+    environment: production
+
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ env.AWS_REGION }}
+          role-to-assume: ${{ env.OIDC_ROLE_ARN }}
+          role-session-name: gh-ci-ecs-deploy
+
+      - name: Verify AWS identity
+        run: just aws-whoami
+
+      - name: Download task definition
+        run: |
+          aws ecs describe-task-definition --task-definition ${{ env.TASK_DEFINITION }} --query taskDefinition > task-definition.json
+
+      - name: Task definition rendition
+        id: task-def
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          task-definition: task-definition.json
+          container-name: ${{ env.CONTAINER_NAME }}
+          image: ${{ needs.image-build-and-push.outputs.image_tags }}
+
+      - name: Deploy task definition
+        uses: aws-actions/amazon-ecs-deploy-task-definition@v2
+        with:
+          task-definition: ${{ steps.task-def.outputs.task-definition }}
+          service: ${{ env.ECS_SERVICE }}
+          cluster: ${{ env.ECS_CLUSTER }}
+          wait-for-service-stability: true

.github/workflows/terragrunt-plan.yml

@@ -10,6 +10,7 @@ on:
         default: purgeable/ecs/
         options:
           - purgeable/ecs/
+          - purgeable/ecs-cron/
           - non-purgeable/extras/
 
 jobs:

scripts/aws/infra/_envcommon/ecs-cron.hcl

@@ -0,0 +1,64 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# COMMON TERRAGRUNT CONFIGURATION
+# This is the common component configuration for mysql. The common variables for each environment to
+# deploy mysql are defined here. This configuration will be merged into the environment configuration
+# via an include block.
+# ---------------------------------------------------------------------------------------------------------------------
+
+# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
+# working directory, into a temporary folder, and execute your Terraform commands in that folder. If any environment
+# needs to deploy a different module version, it should redefine this block with a different ref to override the
+# deployed version.
+terraform {
+  source = "${local.base_source_url}?ref=tasking-manager-infra"
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Locals are named constants that are reusable within the configuration.
+# ---------------------------------------------------------------------------------------------------------------------
+locals {
+  # Automatically load environment-level variables
+  environment_vars = read_terragrunt_config(find_in_parent_folders("deployment_env.hcl"))
+
+  # Extract out common variables for reuse
+  environment  = local.environment_vars.locals.environment
+  application  = local.environment_vars.locals.application
+  team         = local.environment_vars.locals.team
+
+  # Expose the base source URL so different versions of the module can be deployed in different environments. This will
+  # be used to construct the terraform block in the child terragrunt configurations.
+  base_source_url = "git::https://github.com/hotosm/terraform-aws-ecs/"
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# MODULE PARAMETERS
+# These are the variables we have to pass in to use the module. This defines the parameters that are common across all
+# environments.
+# ---------------------------------------------------------------------------------------------------------------------
+# Defaults,  overridden by env.hcl
+
+inputs = {
+  service_name = format("%s-%s-%s-%s", local.application, local.team, local.environment, "backend")
+
+  log_configuration = {
+    logdriver = "awslogs"
+    options = {
+      awslogs-group         = format("%s-%s-%s-%s", local.application, local.team, local.environment, "cron")
+      awslogs-region        = local.environment_vars.locals.aws_region
+      awslogs-stream-prefix = "cron"
+    }
+  }
+
+  container_settings = {
+    app_port         = 80
+    cpu_architecture = "X86_64"
+    image_url        = "ghcr.io/hotosm/tasking-manager-backend"
+    image_tag        = "fastapi"
+    service_name     = format("%s-%s-%s-%s", local.application, local.team, local.environment, "cron")
+  }
+
+  container_capacity = {
+    cpu       = 2048
+    memory_mb = 4096
+  }
+}

scripts/aws/infra/_envcommon/ecs.hcl

@@ -70,19 +70,4 @@ inputs = {
     cpu       = 2048
     memory_mb = 4096
   }
-
-  ## Scaling Policies enabled for cpu,memory in addition to ALB Count.
-  scale_by_cpu = {
-    enabled = true
-    cpu_pct = 70
-    scale_in_cooldown = 30
-    scale_out_cooldown = 60
-  }
-
-  scale_by_memory = {
-    enabled    = true
-    memory_pct = 80
-    scale_in_cooldown = 30
-    scale_out_cooldown = 30
-  }
 }

scripts/aws/infra/staging/purgeable/common-ecs-env.hcl

@@ -0,0 +1,50 @@
+locals {
+  envs = {
+    EXTRA_CORS_ORIGINS            = get_env("EXTRA_CORS_ORIGINS" ,"[\"https://tasks-stage.hotosm.org\", \"https://tm-ecs-frontend.naxa.com.np\", \"http://localhost:3000\"]")
+    TM_SMTP_HOST                  = get_env("TM_SMTP_HOST" ,"email-smtp.us-east-1.amazonaws.com")
+    TM_SMTP_PORT                  = get_env("TM_SMTP_PORT" ,"587")
+    TM_SMTP_USE_TLS               = get_env("TM_SMTP_USE_TLS" ,"1")
+    TM_SMTP_USE_SSL               = get_env("TM_SMTP_USE_SSL" ,"0")
+    TM_EMAIL_FROM_ADDRESS         = get_env("TM_EMAIL_FROM_ADDRESS", "[email protected]")
+    TM_EMAIL_CONTACT_ADDRESS      = get_env("TM_EMAIL_CONTACT_ADDRESS", "[email protected]")
+    TM_APP_BASE_URL               = get_env("TM_APP_BASE_URL" ,"https://tasks-stage.hotosm.org")
+    TM_APP_API_URL                = get_env("TM_APP_API_URL" ,"https://tasking-manager-staging-api.hotosm.org/api")
+    TM_REDIRECT_URI               = get_env("TM_REDIRECT_URI" ,"https://tasks-stage.hotosm.org/authorized")
+    TM_APP_API_VERSION            = get_env("TM_APP_API_VERSION" ,"v2")
+    TM_ORG_NAME                   = get_env("TM_ORG_NAME" ,"Humanitarian OpenStreetMap Team")
+    TM_ORG_CODE                   = get_env("TM_ORG_CODE" ,"HOT")
+    TM_ORG_LOGO                   = get_env("TM_ORG_LOGO" ,"https://cdn.hotosm.org/tasking-manager/uploads/1588741335578_hot-logo.png")
+    TM_ORG_URL                    = get_env("TM_ORG_URL" ,"https://www.hotosm.org/")
+    TM_ORG_PRIVACY_POLICY_URL     = get_env("TM_ORG_PRIVACY_POLICY_URL" ,"https://www.hotosm.org/privacy")
+    TM_ORG_TWITTER                = get_env("TM_ORG_TWITTER" ,"http://twitter.com/hotosm")
+    TM_ORG_FB                     = get_env("TM_ORG_FB" ,"https://www.facebook.com/hotosm")
+    TM_ORG_INSTAGRAM              = get_env("TM_ORG_INSTAGRAM" ,"https://www.instagram.com/open.mapping.hubs/")
+    TM_ORG_YOUTUBE                = get_env("TM_ORG_YOUTUBE" ,"https://www.youtube.com/user/hotosm")
+    TM_ORG_GITHUB                 = get_env("TM_ORG_GITHUB" ,"https://github.com/hotosm")
+    OSM_SERVER_URL                = get_env("OSM_SERVER_URL" ,"https://www.openstreetmap.org")
+    OSM_SERVER_API_URL            = get_env("OSM_SERVER_API_URL" ,"https://api.openstreetmap.org")
+    OSM_NOMINATIM_SERVER_URL      = get_env("OSM_NOMINATIM_SERVER_URL" ,"https://nominatim.openstreetmap.org")
+    OSM_REGISTER_URL              = get_env("OSM_REGISTER_URL" ,"https://www.openstreetmap.org/user/new")
+    POSTGRES_TEST_DB              = get_env("POSTGRES_TEST_DB" ,"tasking-manager-test")
+    UNDERPASS_URL                 = get_env("UNDERPASS_URL" ,"https://underpass.hotosm.org")
+    TM_SEND_PROJECT_EMAIL_UPDATES = get_env("TM_SEND_PROJECT_EMAIL_UPDATES" ,"1")
+    TM_DEFAULT_LOCALE             = get_env("TM_DEFAULT_LOCALE" ,"en")
+    TM_LOG_LEVEL                  = get_env("TM_LOG_LEVEL" , "10")
+    TM_LOG_DIR                    = get_env("TM_LOG_DIR", "/var/log/tasking-manager-logs")
+    TM_SUPPORTED_LANGUAGES_CODES  = get_env("TM_SUPPORTED_LANGUAGES_CODES", "en, es")
+    TM_SUPPORTED_LANGUAGES        = get_env("TM_SUPPORTED_LANGUAGES", "English, Español")
+    TM_DEFAULT_CHANGESET_COMMENT  = get_env("TM_DEFAULT_CHANGESET_COMMENT", "#hot-tm-stage-project")
+    TM_ENVIRONMENT                = get_env("TM_ENVIRONMENT", "tasking-manager-staging")
+    NEW_RELIC_ENVIRONMENT         = get_env("TM_ENVIRONMENT", "tasking-manager-staging")
+    NEW_RELIC_CONFIG_FILE         = get_env("NEW_RELIC_CONFIG_FILE", "./scripts/aws/cloudformation/newrelic.ini")
+    USE_SENTRY                    = get_env("USE_SENTRY", "false")
+    # Uncomment the following as needed.
+    # TM_TASK_AUTOUNLOCK_AFTER    = get_env("TM_TASK_AUTOUNLOCK_AFTER", "2h")
+    # TM_MAPPER_LEVEL_INTERMEDIATE = get_env("TM_MAPPER_LEVEL_INTERMEDIATE", "250")
+    # TM_MAPPER_LEVEL_ADVANCED   = get_env("TM_MAPPER_LEVEL_ADVANCED", "500")
+    # TM_IMPORT_MAX_FILESIZE     = get_env("TM_IMPORT_MAX_FILESIZE", "1000000")
+    # TM_MAX_AOI_AREA            = get_env("TM_MAX_AOI_AREA", "5000")
+    # EXPORT_TOOL_S3_URL         = get_env("EXPORT_TOOL_S3_URL", "https://foorawdataapi.s3.amazonaws.com")
+    # ENABLE_EXPORT_TOOL         = get_env("ENABLE_EXPORT_TOOL", "1")
+  }
+}

scripts/aws/infra/staging/purgeable/ecs-cron/terragrunt.hcl

@@ -0,0 +1,95 @@
+# ---------------------------------------------------------------------------------------------------------------------
+# TERRAGRUNT CONFIGURATION
+# This is the configuration for Terragrunt, a thin wrapper for Terraform and OpenTofu that helps keep your code DRY and
+# maintainable: https://github.com/gruntwork-io/terragrunt
+# ---------------------------------------------------------------------------------------------------------------------
+
+# Include the root `terragrunt.hcl` configuration. The root configuration contains settings that are common across all
+# components and environments, such as how to configure remote state.
+include "root" {
+  path = find_in_parent_folders("root.hcl")
+}
+
+# Include the envcommon configuration for the component. The envcommon configuration contains settings that are common
+# for the component across all environments.
+include "envcommon" {
+  path = "${dirname(find_in_parent_folders("root.hcl"))}/_envcommon/ecs-cron.hcl"
+  # We want to reference the variables from the included config in this configuration, so we expose it.
+  expose = true
+}
+
+# Configure the version of the module to use in this environment. This allows you to promote new versions one
+# environment at a time (e.g., qa -> stage -> prod).
+terraform {
+  source = "${include.envcommon.locals.base_source_url}?ref=tasking-manager-infra"
+}
+
+locals {
+  # Automatically load environment-level variables
+  environment_vars = read_terragrunt_config(find_in_parent_folders("deployment_env.hcl"))
+  common_ecs_envs = read_terragrunt_config(find_in_parent_folders("common-ecs-env.hcl"))
+}
+
+# ---------------------------------------------------------------------------------------------------------------------
+# We don't need to override any of the common parameters for this environment, so we don't specify any other parameters.
+# ---------------------------------------------------------------------------------------------------------------------
+
+dependency "vpc" {
+  config_path = "../../non-purgeable/vpc"
+}
+
+dependency "alb" {
+  config_path = "../../non-purgeable/alb"
+}
+
+dependency "rds" {
+  config_path = "../../non-purgeable/rds"
+}
+
+dependency "extras" {
+  config_path = "../../non-purgeable/extras"
+}
+
+## Add in any new inputs that you want to overide.
+inputs = {
+  # Inputs from dependencies (Rarely changed)
+  service_subnets = dependency.vpc.outputs.private_subnets
+  aws_vpc_id = dependency.vpc.outputs.vpc_id
+  service_security_groups = [ dependency.alb.outputs.load_balancer_app_security_group ]
+  deployment_environment = local.environment_vars.locals.environment
+
+  task_role_arn = dependency.extras.outputs.ecs_task_role_arn
+
+  service_security_groups = [
+    dependency.alb.outputs.load_balancer_app_security_group
+  ]
+
+  # Merge secrets with: key:ValueFrom together
+  container_secrets = concat(dependency.extras.outputs.container_secrets,
+      dependency.rds.outputs.database_config_as_ecs_secrets_inputs)
+
+  container_commands = [
+      "sh",
+      "-c",
+      "python3 backend/cron_jobs.py"
+  ]
+
+  ## Task count for ECS services.
+  tasks_count = {
+      desired_count   = 1
+      min_healthy_pct = 100
+      max_pct         = 200
+    }
+
+  ## Scaling Policy Target Values
+  scaling_target_values = {
+    container_min_count = 1
+    container_max_count = 1
+  }
+
+  # Merge non-sensetive together
+  container_envvars = merge(
+    dependency.rds.outputs.database_config_as_ecs_inputs,
+    local.common_ecs_envs.locals.envs
+    )
+}