Added cloud formation template to deploy dashboard + mongo

cf-deploy-akto

@@ -105,10 +105,10 @@ if [ ! -d "$DOCKER_CONTEXT" ]; then
     fi
   else
     log_step 'downloading...'
-    curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/segregation_2.zip
+    curl -L -XGET -o master.zip https://github.com/akto-api-security/infra/archive/refs/heads/feature/quick-setup.zip
     log_step 'unpacking...'
     unzip master.zip
-    mv infra-feature-segregation_2 infra
+    mv infra-feature-quick-setup infra
   fi
 
   # NB: this is to make onprem containers to all get named the same.

cf-deploy-akto-start

@@ -29,7 +29,7 @@ fi
 
 cd ~/akto/infra
 
-$MAYBE_SUDO docker-compose up
+$MAYBE_SUDO docker-compose up -d
 
 echo ""
 echo " -- ${GREEN}${BOLD}!! AKTO IS BOOTING !!${NORMAL} --"

data-ingestion-docker.env

@@ -0,0 +1,6 @@
+AKTO_TRAFFIC_BATCH_SIZE=100
+AKTO_TRAFFIC_BATCH_TIME_SECS=10
+AKTO_KAFKA_BROKER_URL=kafka1:19092
+AKTO_KAFKA_PRODUCER_BATCH_SIZE=10
+AKTO_KAFKA_PRODUCER_LINGER_MS=10
+AKTO_KAFKA_TOPIC_NAME="akto.api.logs"

docker-compose-context-analyser.yml

@@ -1,9 +1,9 @@
-version: '2'
+version: '2.1'
 
 services:
   zoo1:
     image: confluentinc/cp-zookeeper:6.2.1
-    restart: on-failure:10
+    restart: always
     hostname: zoo1
     user: "0"  
     volumes:
@@ -17,10 +17,12 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_SERVER_ID: 1
       ZOOKEEPER_SERVERS: zoo1:2888:3888
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"
 
   kafka1:
     image: confluentinc/cp-kafka:6.2.1
-    restart: on-failure:10
+    restart: always
     hostname: kafka1
     user: "0"
     ports:
@@ -49,17 +51,28 @@ services:
       - ./data-kafka-secrets:/etc/kafka/secrets             
     depends_on:
       - zoo1
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"
 
   akto-api-security-context-analyser:
-    image: aktosecurity/akto-api-context-analyzer:latest
+    image: public.ecr.aws/aktosecurity/akto-api-context-analyzer:latest
     env_file: ./docker-context-analyser.env
     mem_limit: 4g    
     restart: always
     depends_on:
       - kafka1
 
   akto-hello-world:
-    image: hotavneesh/hello-world-server:latest
+    image: public.ecr.aws/aktosecurity/hello-world-server:latest
     restart: always
     ports: 
       - "8000:8000"
+
+  watchtower:
+    image: containrrr/watchtower
+    restart: always
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    env_file: ./watchtower.env
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"

docker-compose-dashboard.yml

@@ -1,8 +1,8 @@
-version: '2'
+version: '2.1'
 
 services:
   akto-api-security-dashboard:
-    image: aktosecurity/akto-api-security-dashboard:latest
+    image: public.ecr.aws/aktosecurity/akto-api-security-dashboard:latest
     env_file: ./docker-dashboard.env
     restart: always
     ports: 
@@ -11,13 +11,13 @@ services:
       - kafka1
 
   akto-api-security-testing:
-    image: aktosecurity/akto-api-testing:latest
+    image: public.ecr.aws/aktosecurity/akto-api-testing:latest
     env_file: ./docker-dashboard.env
     restart: always
 
   zoo1:
     image: confluentinc/cp-zookeeper:6.2.1
-    restart: on-failure:10
+    restart: always
     hostname: zoo1
     user: "0"  
     volumes:
@@ -31,10 +31,12 @@ services:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_SERVER_ID: 1
       ZOOKEEPER_SERVERS: zoo1:2888:3888
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"
 
   kafka1:
     image: confluentinc/cp-kafka:6.2.1
-    restart: on-failure:10
+    restart: always
     hostname: kafka1
     user: "0"
     ports:
@@ -63,10 +65,27 @@ services:
       - ./data-kafka-secrets:/etc/kafka/secrets       
     depends_on:
       - zoo1
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"
 
   akto-api-security-runtime:
-    image: aktosecurity/akto-api-security-runtime:latest
+    image: public.ecr.aws/aktosecurity/akto-api-security-runtime:latest
     env_file: ./docker-dashboard.env
     restart: always
     depends_on:
-      - kafka1
+      - kafka1
+
+  akto-puppeteer-replay:
+    image: public.ecr.aws/aktosecurity/akto-puppeteer-replay:latest
+    ports:
+      - "3000:3000"
+    restart: always
+
+  watchtower:
+    image: containrrr/watchtower
+    restart: always
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    env_file: ./watchtower.env
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"

docker-compose-data-ingestion-runtime.yml

@@ -0,0 +1,81 @@
+version: '2.1'
+
+services:
+  zoo1:
+    image: confluentinc/cp-zookeeper:6.2.1
+    restart: always
+    hostname: zoo1
+    user: "0"  
+    volumes:
+      - ./data-zoo-data:/var/lib/zookeeper/data
+      - ./data-zoo-logs:/var/lib/zookeeper/log
+      - ./data-zoo-secrets:/etc/zookeeper/secrets              
+    container_name: zoo1
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_SERVER_ID: 1
+      ZOOKEEPER_SERVERS: zoo1:2888:3888
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"
+
+  kafka1:
+    image: confluentinc/cp-kafka:6.2.1
+    restart: always
+    hostname: kafka1
+    user: "0"
+    ports:
+      - "9092:9092"
+      - "19092:19092"
+      - "29092:29092"
+      - "9999:9999"
+    environment:
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_EXTERNAL_DIFFHOST://${AKTO_KAFKA_IP}:9092, LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL_LOCALHOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
+      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
+      KAFKA_BROKER_ID: 1
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_CREATE_TOPICS: "akto.api.logs:3:3"
+      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 60000
+      KAFKA_LOG_RETENTION_HOURS: 5
+      KAFKA_LOG_SEGMENT_BYTES: 104857600
+      KAFKA_LOG_CLEANER_ENABLE: "true"
+      KAFKA_CLEANUP_POLICY: "delete"
+      KAFKA_LOG_RETENTION_BYTES: 10737418240
+    volumes:
+      - ./data-kafka-data:/var/lib/kafka/data
+      - ./data-kafka-secrets:/etc/kafka/secrets             
+    depends_on:
+      - zoo1
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"
+
+  akto-api-security-runtime:
+    image: public.ecr.aws/aktosecurity/akto-api-security-mini-runtime:latest
+    env_file: ./docker-mini-runtime.env
+    mem_limit: 8g    
+    restart: always
+    depends_on:
+      - kafka1
+
+  data-ingestion-service:
+    image: aktosecurity/data-ingestion-service:latest
+    env_file: ./data-ingestion-docker.env
+    restart: always
+    ports:
+      - "9091:8080"
+    depends_on:
+      - kafka1
+
+  watchtower:
+    image: containrrr/watchtower
+    restart: always
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    env_file: ./watchtower.env
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"

docker-compose-mini-runtime.yml

@@ -0,0 +1,97 @@
+version: '2.1'
+
+services:
+  zoo1:
+    image: confluentinc/cp-zookeeper:6.2.1
+    restart: always
+    hostname: zoo1
+    user: "0"  
+    volumes:
+      - ./data-zoo-data:/var/lib/zookeeper/data
+      - ./data-zoo-logs:/var/lib/zookeeper/log
+      - ./data-zoo-secrets:/etc/zookeeper/secrets              
+    container_name: zoo1
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_SERVER_ID: 1
+      ZOOKEEPER_SERVERS: zoo1:2888:3888
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"
+
+  kafka1:
+    image: confluentinc/cp-kafka:6.2.1
+    restart: always
+    hostname: kafka1
+    user: "0"
+    ports:
+      - "9092:9092"
+      - "19092:19092"
+      - "29092:29092"
+      - "9999:9999"
+    environment:
+      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_EXTERNAL_DIFFHOST://${AKTO_KAFKA_IP}:9092, LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL_LOCALHOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_EXTERNAL_DIFFHOST:PLAINTEXT, LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL_LOCALHOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
+      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
+      KAFKA_BROKER_ID: 1
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_CREATE_TOPICS: "akto.api.logs:3:3"
+      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 60000
+      KAFKA_LOG_RETENTION_HOURS: 5
+      KAFKA_LOG_SEGMENT_BYTES: 104857600
+      KAFKA_LOG_CLEANER_ENABLE: "true"
+      KAFKA_CLEANUP_POLICY: "delete"
+      KAFKA_LOG_RETENTION_BYTES: 10737418240
+    volumes:
+      - ./data-kafka-data:/var/lib/kafka/data
+      - ./data-kafka-secrets:/etc/kafka/secrets             
+    depends_on:
+      - zoo1
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"
+
+  akto-api-security-runtime:
+    image: public.ecr.aws/aktosecurity/akto-api-security-mini-runtime:testruntime
+    env_file: ./docker-mini-runtime.env
+    mem_limit: 8g    
+    restart: always
+    depends_on:
+      - kafka1
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "2"
+
+  akto-api-security-mirror-api-logging:
+    image: public.ecr.aws/aktosecurity/mirror-api-logging:local
+    env_file: ./docker-mini-runtime.env
+    mem_limit: 4g    
+    restart: always
+    depends_on:
+      - kafka1
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "2"
+    network_mode: host
+
+  akto-hello-world:
+    image: public.ecr.aws/aktosecurity/hello-world-server:latest
+    restart: always
+    ports: 
+      - "8000:8000"
+
+  watchtower:
+    image: containrrr/watchtower
+    restart: always
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    env_file: ./watchtower.env
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"

docker-compose-mini-testing.yml

@@ -0,0 +1,28 @@
+version: '3.8'
+services:
+  akto-api-security-testing:
+    image: public.ecr.aws/aktosecurity/akto-api-security-mini-testing:latest
+    environment:
+      RUNTIME_MODE: hybrid
+      DATABASE_ABSTRACTOR_SERVICE_TOKEN: <token>
+      PUPPETEER_REPLAY_SERVICE_URL: "http://akto-puppeteer-replay:3000"
+    restart: always
+
+  akto-api-security-puppeteer-replay:
+    image: public.ecr.aws/aktosecurity/akto-puppeteer-replay:latest
+    ports:
+      - "3000:3000"
+    environment:
+      NODE_ENV: production
+    restart: always
+
+  watchtower:
+    image: containrrr/watchtower
+    restart: always
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      WATCHTOWER_CLEANUP: true
+      WATCHTOWER_POLL_INTERVAL: 1800
+    labels:
+      com.centurylinklabs.watchtower.enable: "false"

docker-compose-mongo-cluster.yml

@@ -0,0 +1,13 @@
+version: '3.3'
+
+services:
+  mongo:
+    container_name: mongo
+    image: mongo
+    restart: always
+    command: mongod --replSet "rs0" --bind_ip ${INSTANCE_IP},localhost
+    volumes:
+      - ./data:/data/db
+    ports:
+      - "27017:27017"
+    network_mode: host

docker-compose-mongo.yml

@@ -4,7 +4,7 @@ services:
   mongo:
     container_name: mongo
     image: mongo
-    restart: on-failure:10
+    restart: always
     volumes:
       - ./data:/data/db
     ports: