Added files from pracs and reformatted out of app/

.gitignore

@@ -43,4 +43,7 @@ __pycache__
 *.json
 *.sqlite
 app/data
-app/spamoverflow/spamhammer
+app/spamoverflow/spamhammer
+
+data/
+spamoverflow/spamhammer

Dockerfile.deploy

@@ -0,0 +1,23 @@
+FROM --platform=linux/amd64 ubuntu:22.04
+
+ENV SQLALCHEMY_DATABASE_URI=sqlite:///:memory:
+
+# Installing dependencies and cleaning up
+RUN apt-get update && \
+        apt-get install -y python3 python3-pip postgresql-client libpq-dev libcurl4-openssl-dev libssl-dev && \
+        apt-get clean && \
+        rm -rf /var/lib/apt/lists/*
+# Install pipenv
+RUN pip3 install poetry
+# Setting the working directory
+WORKDIR /app
+# Install pipenv dependencies
+COPY pyproject.toml .
+RUN poetry install --no-root
+# Copying our application into the container
+COPY bin bin
+COPY spamoverflow spamoverflow
+
+# Running our application
+ENTRYPOINT ["/app/bin/docker-entrypoint"]
+CMD ["serve"]

app/Dockerfile → Dockerfile.dev

@@ -18,5 +18,7 @@ COPY spamoverflow spamoverflow
 # Get the version of spamhammer that matches the architecture of the container
 RUN dpkg --print-architecture | grep -q "amd64" && export SPAMHAMMER_ARCH="amd64" || export SPAMHAMMER_ARCH="arm64" && wget https://github.com/CSSE6400/SpamHammer/releases/download/v1.0.0/spamhammer-v1.0.0-linux-${SPAMHAMMER_ARCH} -O spamoverflow/spamhammer && chmod +x spamoverflow/spamhammer
 
+EXPOSE 8080
+
 # Running our application
 CMD ["poetry", "run", "flask", "--app", "spamoverflow", "run", "--host", "0.0.0.0", "--port", "8080"]

README.md

@@ -2,4 +2,18 @@
 
 
 TODO:
-1. Detach the spamhammer from the post request and add it as a subroutine that can be triggered f.ex. By doing this we remove any lag on the server and can accept incoming request much faster.
+1. Detach the spamhammer from the post request and add it as a subroutine that can be triggered f.ex. By doing this we remove any lag on the server and can accept incoming request much faster.
+
+
+TODO - deployment to cloud:
+1. Do I need autoscaling?
+2. Get the .tf files from prac 4,5,6 and combine them to this repo.
+3. Decide wether or not the main terraform file needs to build the docker image and then upload it to ECR, seems counterintuitive
+4. Find out if we need a wait_for_db script such that prac 6 had or iw we can do it inline in the docker compose file as previosuly
+
+4. Get an organized overview of all the ports etc. Maybe make a diagram 
+
+
+5. "Remove" docker-compose as it is no longer needed for cloud hosting. Only if we want to run the postgres db locally.
+
+    - because of this we will need the wait for db script

autoscaling.tf

@@ -0,0 +1,26 @@
+resource "aws_appautoscaling_target" "spamoverflow" { 
+  max_capacity        = 4 
+  min_capacity        = 1 
+  resource_id         = "service/spamoverflow/spamoverflow" 
+  scalable_dimension  = "ecs:service:DesiredCount" 
+  service_namespace   = "ecs" 
+
+  depends_on = [ aws_ecs_service.taskoverflow ] 
+} 
+
+
+resource "aws_appautoscaling_policy" "spamoverflow-cpu" { 
+  name                = "spamoverflow-cpu" 
+  policy_type         = "TargetTrackingScaling" 
+  resource_id         = aws_appautoscaling_target.todo.resource_id 
+  scalable_dimension  = aws_appautoscaling_target.todo.scalable_dimension 
+  service_namespace   = aws_appautoscaling_target.todo.service_namespace 
+
+  target_tracking_scaling_policy_configuration { 
+    predefined_metric_specification { 
+      predefined_metric_type  = "ECSServiceAverageCPUUtilization" 
+    } 
+
+    target_value              = 20 
+  } 
+}

bin/docker-entrypoint

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+set -e
+
+# If there is more than one argument then return the help
+if [ $# -gt 1 ]; then
+    echo "Usage: docker-entrypoint [COMMAND]"
+    echo "  COMMAND: The command to run. If not specified, the default command is used."
+    exit 1
+fi
+
+# If the first argument is serve then run the default command
+if [ $# -eq 0 ] || [ "$1" = "serve" ]; then
+    poetry run python3 bin/wait_for_db.py
+
+    exec poetry run gunicorn --bind 0.0.0.0:6400 'spamoverflow:create_app()'
+fi

bin/wait_for_db.py

@@ -0,0 +1,26 @@
+import time
+import os
+import sqlalchemy
+
+def wait_for_db(db_url, retries=6, timeout=15):
+    """Wait for the database to be available.
+
+    :param db_url: the database URL
+    :param timeout: the maximum number of seconds to wait
+    """
+    engine = sqlalchemy.create_engine(db_url, connect_args={'connect_timeout': 30})
+    for i in range(retries):
+        try:
+            engine.connect()
+            return
+        except sqlalchemy.exc.OperationalError:
+            print(f"Waiting for the database to be available ({i+1}/{retries})")
+            time.sleep(timeout)
+    raise RuntimeError("Timeout waiting for the database")
+
+if __name__ == "__main__":
+    # Get SQLIALCHEMY_DATABASE_URI from the environment
+    db_url = os.environ["SQLALCHEMY_DATABASE_URI"]
+    if "sqlite" in db_url:
+        exit()
+    wait_for_db(db_url)

db.tf

@@ -0,0 +1,34 @@
+resource "aws_db_instance" "database" {
+  allocated_storage      = 20
+  max_allocated_storage  = 1000
+  engine                 = "postgres"
+  engine_version         = "14"
+  instance_class         = "db.t4g.micro"
+  db_name                = "todo"
+  username               = local.database_username
+  password               = local.database_password
+  parameter_group_name   = "default.postgres14"
+  skip_final_snapshot    = true
+  vpc_security_group_ids = [aws_security_group.database.id]
+  publicly_accessible    = true
+}
+
+resource "aws_security_group" "database" {
+  name        = "spamoverflow-database"
+  description = "Allow inbound Postgres traffic"
+
+  ingress {
+    from_port        = 5432
+    to_port          = 5432
+    protocol         = "tcp"
+    cidr_blocks      = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port        = 0
+    to_port          = 0
+    protocol         = "-1"
+    cidr_blocks      = ["0.0.0.0/0"]
+    ipv6_cidr_blocks = ["::/0"]
+  }
+}

app/docker-compose.yml → docker-compose.yml

@@ -7,19 +7,21 @@
     environment: 
       POSTGRES_PASSWORD: verySecretPassword 
       POSTGRES_USER: administrator 
-      POSTGRES_DB: todo 
+      POSTGRES_DB: spamoverflow 
     volumes: 
       - ./data/db:/var/lib/postgresql/data 
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U administrator -d todo"]
+      test: ["CMD-SHELL", "pg_isready -U administrator -d spamoverflow"]
       interval: 10s
       timeout: 5s
       retries: 5
    app: 
-    build: . 
+    build:
+      context: .
+      dockerfile: Dockerfile.dev 
     restart: always 
     environment: 
-      SQLALCHEMY_DATABASE_URI: postgresql://administrator:verySecretPassword@database:5432/todo 
+      SQLALCHEMY_DATABASE_URI: postgresql://administrator:verySecretPassword@database:5432/spamoverflow 
     ports: 
       - "8080:8080" 
     depends_on: 

ecs.tf

@@ -0,0 +1,90 @@
+resource "aws_ecs_cluster" "spamoverflow" {
+  name = "taskoverflow"
+}
+
+resource "aws_ecs_task_definition" "spamoverflow" {
+  family                   = "spamoverflow"
+  network_mode             = "awsvpc"
+  requires_compatibilities = ["FARGATE"]
+  cpu                      = 1024
+  memory                   = 2048
+  execution_role_arn       = data.aws_iam_role.lab.arn
+
+  container_definitions = jsonencode([
+    {
+      image        = "${aws_ecr_repository.spamoverflow.repository_url}:latest"
+      cpu          = 1024
+      memory       = 2048
+      name         = "spamoverflow-app"
+      networkMode  = "awsvpc"
+      portMappings = [
+        {
+          containerPort = 6400
+          hostPort      = 6400
+        }
+      ]
+      environment = [
+        {
+          name  = "SQLALCHEMY_DATABASE_URI"
+          value = "postgresql://${local.database_username}:${local.database_password}@${aws_db_instance.spamoverflow_database.address}:${aws_db_instance.spamoverflow_database.port}/${aws_db_instance.spamoverflow_database.db_name}"
+        }
+      ]
+      logConfiguration = {
+        logDriver = "awslogs"
+        options = {
+          "awslogs-group"            = "/spamoverflow/todo"
+          "awslogs-region"           = "us-east-1"
+          "awslogs-stream-prefix"    = "ecs"
+          "awslogs-create-group"     = "true"
+        }
+      }
+    }
+  ])
+}
+
+resource "aws_ecs_service" "spamoverflow" {
+  name            = "spamoverflow-ecs"
+  cluster         = aws_ecs_cluster.spamoverflow.id
+  task_definition = aws_ecs_task_definition.todo.arn
+  desired_count   = 1
+  launch_type     = "FARGATE"
+
+  network_configuration {
+    subnets             = data.aws_subnets.private.ids
+    security_groups     = [aws_security_group.todo.id]
+    assign_public_ip    = true
+  }
+
+  load_balancer { 
+    target_group_arn = aws_lb_target_group.todo.arn 
+    container_name   = "spamoverflow" 
+    container_port   = 6400 
+  }
+
+}
+
+resource "aws_security_group" "spamoverflow" {
+  name = "spamoverflow"
+  description = "SpamoverFloww Security Group"
+
+  ingress {
+    from_port = 6400
+    to_port = 6400
+    protocol = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port = 22
+    to_port = 22
+    protocol = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}

lb.tf

@@ -0,0 +1,57 @@
+resource "aws_lb_target_group" "spamoverflow" { 
+  name          = "spamoverflow" 
+  port          = 6400 
+  protocol      = "HTTP" 
+  vpc_id        = aws_security_group.spamoverflow.vpc_id 
+  target_type   = "ip" 
+
+  health_check { 
+    path                = "/api/v1/health" 
+    port                = "6400" 
+    protocol            = "HTTP" 
+    healthy_threshold   = 2 
+    unhealthy_threshold = 2 
+    timeout             = 5 
+    interval            = 10 
+  } 
+}
+
+
+resource "aws_lb" "spamoverflow" { 
+  name               = "spamoverflow" 
+  internal           = false 
+  load_balancer_type = "application" 
+  subnets            = data.aws_subnets.private.ids 
+  security_groups    = [aws_security_group.spamoverflow.id] 
+} 
+
+resource "aws_security_group" "spamoverflow" { 
+  name        = "spamoverflow" 
+  description = "spamoverflow Security Group" 
+
+  ingress { 
+    from_port     = 80 
+    to_port       = 80 
+    protocol      = "tcp" 
+    cidr_blocks   = ["0.0.0.0/0"] 
+  } 
+
+  egress { 
+    from_port     = 0 
+    to_port       = 0 
+    protocol      = "-1" 
+    cidr_blocks   = ["0.0.0.0/0"] 
+  } 
+}
+
+//Listener aka entrypoint to the load balancer
+resource "aws_lb_listener" "spamoverflow" { 
+  load_balancer_arn   = aws_lb.spamoverflow.arn 
+  port                = "80" 
+  protocol            = "HTTP" 
+
+  default_action { 
+    type              = "forward" 
+    target_group_arn  = aws_lb_target_group.spamoverflow.arn 
+  } 
+}

local.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 
-#Change to the app folder
-cd app
-
 #Buildkit to make sure we know what env
 export DOCKER_BUILDKIT=1