8 background service

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 .DS_Store
 npm-debug.log
+deploy-pipeline.sh

Makefile

@@ -1,12 +1,25 @@
-run:
-	docker-compose up -d
-
-build:
-	docker-compose build client
+up:
+	docker-compose up -d redis
+	docker-compose up -d localstack
+	docker-compose run bootstrap-resources
 	docker-compose up --no-deps -d client
+	docker-compose up --no-deps -d message-cards
+
+down:
+	docker-compose down
 
 test:
 	docker-compose build client
+	docker-compose build message-cards
 	docker-compose build test
-	docker-compose run --no-deps -d client
+	docker-compose up --no-deps -d client
+	docker-compose up --no-deps -d message-cards
 	docker-compose run --no-deps test
+
+cards:
+	docker-compose build message-cards
+	docker-compose up --no-deps -d message-cards
+	docker-compose run --no-deps test
+
+clean:
+	docker system prune -a -f

deps/dynamodb-tables/Dockerfile → deps/bootstrap-resources/Dockerfile

@@ -5,12 +5,14 @@ RUN apk -v --update add \
         groff \
         less \
         mailcap \
+        curl \
         && \
     pip install --upgrade awscli==1.14.5 s3cmd==2.0.1 python-magic && \
     apk -v --purge del py-pip && \
     rm /var/cache/apk/*
 VOLUME /root/.aws
 VOLUME /project
 WORKDIR /project
-ADD tables.sh .
-ENTRYPOINT ["sh", "tables.sh"]
+ADD wait-for-it.sh .
+ADD bootstrap.sh .
+ENTRYPOINT ["sh", "bootstrap.sh"]

deps/bootstrap-resources/bootstrap.sh

@@ -0,0 +1,119 @@
+# Make sure that the localstack resources are up first.
+./wait-for-it.sh $SNS_ENDPOINT
+./wait-for-it.sh $SQS_ENDPOINT
+./wait-for-it.sh $DYNAMODB_ENDPOINT
+
+aws dynamodb list-tables \
+  --region us-east-1 \
+  --endpoint-url $DYNAMODB_ENDPOINT \
+  --cli-connect-timeout 1 \
+  --cli-read-timeout 1
+
+ENV_NAME="test";
+
+# Local Users table
+TABLE_NAME="Users";
+RESULT=$(aws dynamodb describe-table \
+  --region us-east-1 \
+  --endpoint-url $DYNAMODB_ENDPOINT \
+  --table-name ${ENV_NAME}_${TABLE_NAME})
+if [ $? -eq 0 ]; then
+  echo "Delete existing table ${TABLE_NAME}";
+  aws dynamodb delete-table \
+    --region us-east-1 \
+    --endpoint-url $DYNAMODB_ENDPOINT \
+    --table-name ${ENV_NAME}_${TABLE_NAME}
+fi
+echo "Create table ${TABLE_NAME}";
+aws dynamodb create-table \
+  --region us-east-1 \
+  --endpoint-url $DYNAMODB_ENDPOINT \
+  --table-name ${ENV_NAME}_${TABLE_NAME} \
+  --key-schema AttributeName=username,KeyType=HASH \
+  --attribute-definitions AttributeName=username,AttributeType=S \
+  --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=10
+
+# Local Messages table
+TABLE_NAME="Messages";
+RESULT=$(aws dynamodb describe-table \
+  --region us-east-1 \
+  --endpoint-url $DYNAMODB_ENDPOINT \
+  --table-name ${ENV_NAME}_${TABLE_NAME})
+if [ $? -eq 0 ]; then
+  echo "Delete existing table ${TABLE_NAME}";
+  aws dynamodb delete-table \
+    --region us-east-1 \
+    --endpoint-url $DYNAMODB_ENDPOINT \
+    --table-name ${ENV_NAME}_${TABLE_NAME}
+fi
+echo "Create table ${TABLE_NAME}";
+aws dynamodb create-table \
+  --region us-east-1 \
+  --endpoint-url $DYNAMODB_ENDPOINT \
+  --table-name ${ENV_NAME}_${TABLE_NAME} \
+  --key-schema AttributeName=room,KeyType=HASH AttributeName=message,KeyType=RANGE \
+  --attribute-definitions AttributeName=room,AttributeType=S AttributeName=message,AttributeType=S \
+  --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=10
+
+# Local SQS queue
+QUEUE_NAME="MessagesToCardify";
+RESULT=$(aws sqs get-queue-url \
+  --region us-east-1 \
+  --endpoint-url $SQS_ENDPOINT \
+  --queue-name ${ENV_NAME}_${QUEUE_NAME})
+if [ $? -eq 0 ]; then
+  echo "Delete queue ${QUEUE_NAME}";
+  aws sqs delete-queue \
+    --region us-east-1 \
+    --endpoint-url $SQS_ENDPOINT \
+    --queue-url $SQS_ENDPOINT/queue/${ENV_NAME}_${QUEUE_NAME}
+fi
+echo "Create queue ${QUEUE_NAME}";
+aws sqs delete-queue \
+    --region us-east-1 \
+    --endpoint-url $SQS_ENDPOINT \
+    --queue-url $SQS_ENDPOINT/queue/${ENV_NAME}_${QUEUE_NAME}
+aws sqs create-queue \
+    --region us-east-1 \
+    --endpoint-url $SQS_ENDPOINT \
+    --attributes VisibilityTimeout=60 \
+    --queue-name ${ENV_NAME}_${QUEUE_NAME}
+
+# Local SNS topic
+TOPIC_NAME="MessageSent"
+TOPIC_ARN="arn:aws:sns:us-east-1:123456789012:${ENV_NAME}_${TOPIC_NAME}"
+echo "Create topic ${TOPIC_NAME}";
+SUBSCRIPTION_ARN=$(aws sns list-subscriptions-by-topic \
+  --region us-east-1 \
+  --endpoint-url $SNS_ENDPOINT \
+  --topic-arn $TOPIC_ARN \
+  --query Subscriptions[0].SubscriptionArn \
+  --output text)
+aws sns unsubscribe \
+  --region us-east-1 \
+  --endpoint-url $SNS_ENDPOINT \
+  --subscription-arn $SUBSCRIPTION_ARN
+aws sns delete-topic \
+  --region us-east-1 \
+  --endpoint-url $SNS_ENDPOINT \
+  --topic-arn $TOPIC_ARN
+aws sns create-topic \
+  --region us-east-1 \
+  --endpoint-url $SNS_ENDPOINT \
+  --name ${ENV_NAME}_${TOPIC_NAME}
+
+# Subscription between the SQS queue and the SNS topic
+echo "Create subscription between topic ${TOPIC_NAME} and queue ${QUEUE_NAME}"
+aws sns subscribe \
+  --region us-east-1 \
+  --endpoint-url $SNS_ENDPOINT \
+  --topic-arn $TOPIC_ARN \
+  --protocol sqs \
+  --notification-endpoint $SQS_ENDPOINT/queue/${ENV_NAME}_${QUEUE_NAME}
+aws sns list-subscriptions-by-topic \
+  --region us-east-1 \
+  --endpoint-url $SNS_ENDPOINT \
+  --topic-arn $TOPIC_ARN
+
+
+

deps/bootstrap-resources/wait-for-it.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+echo "Waiting for $1 (This may take a few moments)";
+
+clean=${1//http:\/\//}
+hostport=${clean//:/ }
+
+set -f; IFS=' '
+set -- $hostport
+second=$2; fourth=$4
+set +f; unset IFS
+
+HOST=$1
+PORT=$2
+
+while ! nc -w 1 -z $HOST $PORT; do sleep 1 && echo -n .; done;
+
+#while true
+#do
+#    echo "pre poll";
+#  STATUS=$(curl -s -o /dev/null -w '%{http_code}' $1)
+#  echo $STATUS
+#  if [ $STATUS -eq 404 ]; then
+#    echo "ready"
+#    break
+#  else
+#    echo -n "."
+#  fi
+#  sleep 1
+#done
+
+
+#--output /dev/null --silent --head

deps/sqs-local/Dockerfile

@@ -0,0 +1,9 @@
+FROM java:8
+
+ADD https://s3-eu-west-1.amazonaws.com/softwaremill-public/elasticmq-server-0.13.8.jar /
+COPY custom.conf /
+ENTRYPOINT ["/usr/bin/java", "-Dconfig.file=custom.conf", "-jar", "/elasticmq-server-0.13.8.jar"]
+
+EXPOSE 9324
+
+CMD ["-help"]

deps/sqs-local/custom.conf

@@ -0,0 +1,8 @@
+include classpath("application.conf")
+
+node-address {
+    protocol = http
+    host = "*"
+    port = 9324
+    context-path = ""
+}

docker-compose.yml

@@ -9,50 +9,80 @@ services:
     networks:
       - chat
     ports:
-      - 6379:6379
+      - 6379
 
-  # Launch a local version of DynamoDB
-  dynamodb-local:
+  # Launch localstack (https://github.com/localstack/localstack)
+  # This container mimics AWS services locally for offline testing
+  # and development purposes.
+  localstack:
+    image: localstack/localstack
     networks:
       - chat
-    build: ./deps/dynamodb-local
     ports:
-      - 8000:8000
+      # DynamoDB Local
+      - 4569
+      # SNS local
+      - 4575
+      # SQS local
+      - 4576
 
-  # Ephemeral container used for creating the tables in DynamoDB
-  dynamodb-tables:
+  # Ephemeral container used for creating the tables in
+  # the local DynamoDB tables, SNS topics, and SQS queues
+  bootstrap-resources:
     depends_on:
-      - dynamodb-local
+      - localstack
     networks:
       - chat
-    build: ./deps/dynamodb-tables
+    build: ./deps/bootstrap-resources
     environment:
-      DYNAMODB_ENDPOINT: http://dynamodb-local:8000
+      DYNAMODB_ENDPOINT: http://localstack:4569
+      SQS_ENDPOINT: http://localstack:4576
+      SNS_ENDPOINT: http://localstack:4575
       AWS_ACCESS_KEY_ID: test
       AWS_SECRET_ACCESS_KEY: test
 
   # The actual client application
   client:
     depends_on:
       - redis
-      - dynamodb-tables
+      - bootstrap-resources
     networks:
       - chat
     build: ./services/client
     environment:
       LOCAL: "true"
       ENV_NAME: test
       REDIS_ENDPOINT: redis
-      DYNAMODB_ENDPOINT: http://dynamodb-local:8000
+      DYNAMODB_ENDPOINT: http://localstack:4569
+      SQS_ENDPOINT: http://localstack:4576
+      SNS_ENDPOINT: http://localstack:4575
+      MESSAGE_SENT_SNS_ARN: arn:aws:sns:us-east-1:123456789012:test_MessageSent
       AWS_ACCESS_KEY_ID: test
       AWS_SECRET_ACCESS_KEY: test
     ports:
       - 3000:3000
 
+  # Background worker that cardifies messages
+  message-cards:
+    depends_on:
+      - redis
+      - bootstrap-resources
+    networks:
+      - chat
+    build: ./services/message-cards
+    environment:
+      LOCAL: "true"
+      ENV_NAME: test
+      REDIS_ENDPOINT: redis
+      QUEUE_URL: http://localstack:4576/queue/test_MessagesToCardify
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: test
+
   # The test suite
   test:
     depends_on:
       - client
+      - message-cards
     networks:
       - chat
     build: ./services/test-suite

services/client/index.js

@@ -1,6 +1,13 @@
 // Entrypoint
-var server = require('./server');
+
+// Initialize the Datadog APM tracer.
+const tracer = require('dd-trace');
+tracer.init();
+
 var config = require('./lib/config');
+config.tracer = tracer;
+
+var server = require('./server');
 
 server.listen(config.PORT, function() {
   console.log('Server listening at port %d', config.PORT);

services/client/lib/config.js

@@ -8,6 +8,9 @@ module.exports = {
 
   REDIS_ENDPOINT: process.env.REDIS_ENDPOINT,
   DYNAMODB_ENDPOINT: new AWS.Endpoint(process.env.DYNAMODB_ENDPOINT),
+  SNS_ENDPOINT: new AWS.Endpoint(process.env.SNS_ENDPOINT),
+
+  MESSAGE_SENT_SNS_ARN: process.env.MESSAGE_SENT_SNS_ARN,
 
   // Controls how often clients ping back and forth
   HEARTBEAT_TIMEOUT: 8000,