JKG: Add GROK_PARAMETERS for Datagrok queue (#31)

* JKG: Add GROK_PARAMETERS for Datagrok queue * GitHub Actions: Refactor: Automated formatting of terraform code (#32) Co-authored-by: dnillovna <[email protected]> * Veriables: Mark db password as sensitive * JKG: Remove entrypoint * CVM: Allow S3 connection from CVM tasks * JKG: Datagrok API Url Parameter --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: dnillovna <[email protected]>
datagrok-ai · Apr 11, 2024 · 28542d5 · 28542d5
1 parent 9b661a5
commit 28542d5
Show file tree

Hide file tree

Showing 2 changed files with 104 additions and 1 deletion.
diff --git a/aws/ecs.tf b/aws/ecs.tf
@@ -229,7 +229,33 @@ resource "aws_iam_policy" "docker_hub" {
     )
   })
 }
+data "aws_s3_bucket" "datagrok" {
+  bucket = var.s3_bucket_name
+}
+resource "aws_iam_policy" "s3" {
+  name        = "${local.ecs_name}_s3"
+  description = "Datagrok CVM policy to access S3 storage from tasks"
 
+  policy = jsonencode({
+    "Version" = "2012-10-17",
+    "Statement" = [
+      {
+        "Action" = [
+          "s3:PutObject",
+          "s3:GetObject",
+          "s3:DeleteObject",
+          "s3:ListBucket"
+        ],
+        "Condition" = {},
+        "Effect"    = "Allow",
+        "Resource" = [
+          data.aws_s3_bucket.datagrok.arn,
+          "${data.aws_s3_bucket.datagrok.arn}/*"
+        ]
+      }
+    ]
+  })
+}
 resource "aws_iam_role" "exec" {
   name = "${local.ecs_name}_exec"
 
@@ -271,12 +297,14 @@ resource "aws_iam_role" "task" {
   })
   managed_policy_arns = compact(concat([
     aws_iam_policy.exec.arn,
+    aws_iam_policy.s3.arn,
     var.ecr_enabled ? aws_iam_policy.ecr[0].arn : (var.ecs_launch_type == "FARGATE" ? "" : aws_iam_policy.docker_hub[0].arn)
   ], var.task_iam_policies))
   #  managed_policy_arns = [aws_iam_policy.task.arn]
 
   tags = local.tags
 }
+
 resource "aws_ecs_task_definition" "grok_compute" {
   depends_on = [null_resource.ecr_push]
   family     = "${local.ecs_name}_grok_compute"
@@ -340,6 +368,14 @@ resource "aws_ecs_task_definition" "grok_compute" {
   task_role_arn            = aws_iam_role.task.arn
   requires_compatibilities = [var.ecs_launch_type]
 }
+
+data "aws_secretsmanager_secret" "jkg_secret" {
+  name = var.jkg_secret
+}
+data "aws_secretsmanager_secret_version" "jkg_secret" {
+  secret_id = data.aws_secretsmanager_secret.jkg_secret.id
+}
+
 resource "aws_ecs_task_definition" "jkg" {
   depends_on = [null_resource.ecr_push]
   family     = "${local.ecs_name}_jupyter_kernel_gateway"
@@ -374,7 +410,24 @@ resource "aws_ecs_task_definition" "jkg" {
         }
       ]
       essential  = true
-      entryPoint = ["./entrypoint.sh", "--main", "jupyter kernelgateway --config=.jupyter/jupyter_kernel_gateway_config.py", "--helper", "gunicorn grok_helper:app --timeout 900 --chdir /home/grok/grok_helper -b 0.0.0.0:5005  --access-logfile - --error-logfile - --workers 4"]
+      environment = [
+        {
+          name = "GROK_PARAMETERS",
+          value = jsonencode(
+            {
+              amazonStorageRegion : var.s3_bucket_region,
+              amazonStorageBucket : var.s3_bucket_name,
+              datlasApiUrl: var.datlas_api_url,
+              dbServer : var.db_instance_address,
+              dbPort : var.db_instance_port,
+              db : "datagrok",
+              dbLogin : var.db_dg_login,
+              dbPassword : var.db_dg_password,
+              jupyterToken : jsondecode(data.aws_secretsmanager_secret_version.jkg_secret.secret_string)["token"],
+              capabilities : ["jupyter"]
+          })
+        }
+      ]
       logConfiguration = {
         LogDriver = "awslogs",
         Options = {

diff --git a/aws/variables.tf b/aws/variables.tf
@@ -643,3 +643,53 @@ variable "ec2_root_volume_size" {
   nullable    = false
   description = "Specifies volume size for EC2 machine"
 }
+
+variable "db_instance_address" {
+  description = "The address of the Datagrok DB"
+  type        = string
+  nullable    = false
+}
+
+variable "db_instance_port" {
+  description = "The port of the Datagrok DB"
+  type        = number
+  nullable    = false
+}
+
+variable "db_dg_login" {
+  description = "The user to the Datagrok DB"
+  default     = "datagrok"
+  type        = string
+  nullable    = false
+}
+
+variable "db_dg_password" {
+  description = "The password to the Datagrok DB"
+  type        = string
+  nullable    = false
+  sensitive   = true
+}
+
+variable "s3_bucket_region" {
+  description = "The S3 Bucket region for a stand."
+  type        = string
+  nullable    = false
+}
+
+variable "s3_bucket_name" {
+  description = "The S3 Bucket name of a stand."
+  type        = string
+  nullable    = false
+}
+
+variable "jkg_secret" {
+  description = "The jupyter kernel gateway secret name"
+  type        = string
+  nullable    = false
+}
+
+variable "datlas_api_url" {
+  description = "API Url of Datagrok endpoint"
+  type        = string
+  nullable    = false
+}