From 0934954042ea3aa93e112c5b2b39c3753432b034 Mon Sep 17 00:00:00 2001 From: Mohamed Asaker Date: Thu, 1 Feb 2024 09:47:12 -0800 Subject: [PATCH 1/2] Added a random-traffic-generator for the Python Sample App (#31) *Description of changes:* Added a random-traffic-generator for the Python Sample App. More details can be found in the README under `aws-otel-python-instrumentation/sample-applications/vehicle-dealership-sample-app/random-traffic-generator`. The traffic generator generates the following traffic: 1. Every minute, sends a single POST request to the VehicleInventoryApp and sends a single GET request. 2. Every hour, sends a burst of requests: 5 POST requests to the VehicleInventoryApp and 5 GET requests. 3. Every 5 minutes, sleeps for random amount of time between 30-60 seconds and then sends a GET request to the VehicleInventoryApp with a random throttle param between 5-20 seconds. The backend reads that throttle param and simulates throttling for that amount of time before responding to the request. 4. Every 5 minutes, sleeps for random amount of time between 30-60 seconds and then sends a GET request to the VehicleInventoryApp with an invalid car id to trigger 404 error. 5. Every 5 minutes, sleeps for random amount of time between 30-60 seconds and then sends a GET request to the ImageServiceApp with a non existent image name to trigger 500 error due to S3 Error: "An error occurred (NoSuchKey) when calling the GetObject operation: The specified key does not exist." By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. --- .../VehicleInventoryApp/MainService/url.py | 1 + .../VehicleInventoryApp/MainService/views.py | 32 ++++- .../random-traffic-generator/.gitignore | 1 + .../random-traffic-generator/Dockerfile | 26 ++++ .../random-traffic-generator/README.md | 21 +++ .../random-traffic-generator/build.sh | 22 +++ .../random-traffic-generator/deployment.yaml | 25 ++++ .../random-traffic-generator/index.js | 102 ++++++++++++++ .../package-lock.json | 126 ++++++++++++++++++ .../random-traffic-generator/package.json | 15 +++ .../scripts/push-ecr.sh | 2 +- 11 files changed, 368 insertions(+), 5 deletions(-) create mode 100644 sample-applications/vehicle-dealership-sample-app/random-traffic-generator/.gitignore create mode 100644 sample-applications/vehicle-dealership-sample-app/random-traffic-generator/Dockerfile create mode 100644 sample-applications/vehicle-dealership-sample-app/random-traffic-generator/README.md create mode 100644 sample-applications/vehicle-dealership-sample-app/random-traffic-generator/build.sh create mode 100644 sample-applications/vehicle-dealership-sample-app/random-traffic-generator/deployment.yaml create mode 100644 sample-applications/vehicle-dealership-sample-app/random-traffic-generator/index.js create mode 100644 sample-applications/vehicle-dealership-sample-app/random-traffic-generator/package-lock.json create mode 100644 sample-applications/vehicle-dealership-sample-app/random-traffic-generator/package.json diff --git a/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/MainService/url.py b/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/MainService/url.py index 12a162201..d7f3b1df1 100644 --- a/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/MainService/url.py +++ b/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/MainService/url.py @@ -8,4 +8,5 @@ path("", views.vehicle, name="vehicle"), path("", views.get_vehicle_by_id, name="get_vehicle_by_id"), path("/image", views.get_vehicle_image, name="get_vehicle_image"), + path("image/", views.get_image_by_name, name="image_by_name"), ] diff --git a/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/MainService/views.py b/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/MainService/views.py index b5d5900b8..fed9c9d2b 100644 --- a/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/MainService/views.py +++ b/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/MainService/views.py @@ -2,9 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 import json import os +import time import requests -from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotAllowed +from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotAllowed, HttpResponseNotFound from django.views.decorators.csrf import csrf_exempt from dotenv import load_dotenv from MainService.models import Vehicle @@ -27,8 +28,7 @@ def vehicle(request): make=body["make"], model=body["model"], year=body["year"], image_name=body["image_name"] ) vehicle_object.save() - print(get_image_endpoint() + "/images/name/" + body["image_name"]) - requests.post(get_image_endpoint() + "/images/name/" + body["image_name"], timeout=10) + requests.post(build_image_url(body["image_name"]), timeout=10) return HttpResponse("VehicleId = " + str(vehicle_object.id)) except KeyError as exception: return HttpResponseBadRequest("Missing key: " + str(exception)) @@ -40,7 +40,14 @@ def vehicle(request): def get_vehicle_by_id(request, vehicle_id): if request.method == "GET": + throttle_time = request.GET.get("throttle") + if throttle_time: + print("going to throttle for " + throttle_time + " seconds") + time.sleep(int(throttle_time)) + vehicle_objects = Vehicle.objects.filter(id=vehicle_id).values() + if not vehicle_objects: + return HttpResponseNotFound("Vehicle with id=" + str(vehicle_id) + " is not found") return HttpResponse(vehicle_objects) return HttpResponseNotAllowed("Only GET requests are allowed!") @@ -48,6 +55,23 @@ def get_vehicle_by_id(request, vehicle_id): def get_vehicle_image(request, vehicle_id): if request.method == "GET": vehicle_object = Vehicle.objects.filter(id=vehicle_id).first() + if not vehicle_object: + return HttpResponseNotFound("Vehicle with id=" + str(vehicle_id) + " is not found") image_name = getattr(vehicle_object, "image_name") - return HttpResponse(requests.get(get_image_endpoint() + "/images/name/" + image_name, timeout=10)) + return HttpResponse(requests.get(build_image_url(image_name), timeout=10)) + return HttpResponseNotAllowed("Only GET requests are allowed!") + + +@csrf_exempt +def get_image_by_name(request, image_name): + print(image_name) + if request.method == "GET": + response = requests.get(build_image_url(image_name), timeout=10) + if response.ok: + return HttpResponse(response) + return HttpResponseNotFound("Image with name: " + image_name + " is not found") return HttpResponseNotAllowed("Only GET requests are allowed!") + + +def build_image_url(image_name): + return get_image_endpoint() + "/images/name/" + image_name diff --git a/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/.gitignore b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/.gitignore new file mode 100644 index 000000000..b512c09d4 --- /dev/null +++ b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/.gitignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/Dockerfile b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/Dockerfile new file mode 100644 index 000000000..ca8249222 --- /dev/null +++ b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/Dockerfile @@ -0,0 +1,26 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# Use the official lightweight Node.js 16 image. +# https://hub.docker.com/_/node +FROM node:16-slim + +# Create and change to the app directory. +WORKDIR /usr/src/app + +# Copy application dependency manifests to the container image. +# A wildcard is used to ensure copying both package.json AND package-lock.json (if available). +# Copying this first prevents re-running npm install on every code change. +COPY package*.json ./ + +# Install production dependencies. +# If you have native dependencies, you'll need additional tools. +# For a full list of package.json changes, see: +# https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md +RUN npm install --only=production + +# Copy local code to the container image. +COPY . . + +# Run the web service on container startup. +CMD [ "node", "index.js" ] + diff --git a/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/README.md b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/README.md new file mode 100644 index 000000000..0c9f7cb20 --- /dev/null +++ b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/README.md @@ -0,0 +1,21 @@ +# Random Traffic Generator +The traffic generator generates the following traffic: +1. Every minute, sends a single POST request to the VehicleInventoryApp and sends a single GET request. +2. Every hour, sends a burst of requests: 5 POST requests to the VehicleInventoryApp and 5 GET requests. +3. Every 5 minutes, sleeps for random amount of time between 30-60 seconds and then sends a GET request to the VehicleInventoryApp with a random throttle param between 5-20 seconds. The backend reads that throttle param and simulates throttling for that amount of time before responding to the request. +4. Every 5 minutes, sleeps for random amount of time between 30-60 seconds and then sends a GET request to the VehicleInventoryApp with an invalid car id to trigger 404 error. +5. Every 5 minutes, sleeps for random amount of time between 30-60 seconds and then sends a GET request to the ImageServiceApp with a non existent image name to trigger 500 error due to S3 Error: "An error occurred (NoSuchKey) when calling the GetObject operation: The specified key does not exist." + +## Running locally +1. Run `npm install` +2. Run locally: + - If you are running against you application locally, just run `node index.js`. The default endpoint is `0.0.0.0:8000` for the ImageServiceApp and `0.0.0.0:8001` for VehicleInventoryApp. + - If you want to run against the application on EKS, before running the `node index.js`, run `export `. + +## Deploying to EKS +Run `bash build.sh `. This does the following: +1. This will retrieve the endpoint from EKS ingress-nginx pod +2. Build docker image of the traffic +3. Push the docker image to ECR +4. Deploy the image to EKS + diff --git a/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/build.sh b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/build.sh new file mode 100644 index 000000000..39079ad27 --- /dev/null +++ b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/build.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +account=$1 +region=$2 + +# Save the endpoint URL to a variable +endpoint=$(kubectl get svc -n ingress-nginx | grep "ingress-nginx" | awk '{print $4}') + +# Print the endpoint +echo "Endpoint: $endpoint" + +export REPOSITORY_PREFIX=${account}.dkr.ecr.${region}.amazonaws.com +aws ecr get-login-password --region ${region} | docker login --username AWS --password-stdin ${REPOSITORY_PREFIX} +aws ecr create-repository --repository-name random-traffic-generator --region ${region} || true +docker build -t random-traffic-generator:latest . +docker tag random-traffic-generator:latest ${REPOSITORY_PREFIX}/random-traffic-generator:latest +docker push ${REPOSITORY_PREFIX}/random-traffic-generator:latest + +sed -e 's#\${REPOSITORY_PREFIX}'"#${REPOSITORY_PREFIX}#g" -e 's#\${URL}'"#$endpoint#g" deployment.yaml | kubectl apply -f - + + diff --git a/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/deployment.yaml b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/deployment.yaml new file mode 100644 index 000000000..8a79b0e81 --- /dev/null +++ b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/deployment.yaml @@ -0,0 +1,25 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +apiVersion: apps/v1 +kind: Deployment +metadata: + name: random-traffic-generator + labels: + app: random-traffic-generator +spec: + replicas: 1 + selector: + matchLabels: + app: random-traffic-generator + template: + metadata: + labels: + app: random-traffic-generator + spec: + containers: + - name: random-traffic-generator + image: ${REPOSITORY_PREFIX}/random-traffic-generator:latest + env: + - name: URL + value: "${URL}" + diff --git a/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/index.js b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/index.js new file mode 100644 index 000000000..9a9d07752 --- /dev/null +++ b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/index.js @@ -0,0 +1,102 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +const axios = require('axios'); +const cron = require('node-cron'); + +const vehicleURL = process.env.URL ? `${process.env.URL}/vehicle-inventory` : 'http://0.0.0.0:8001/vehicle-inventory' + +function getRandomNumber(min, max) { + return Math.floor(Math.random() * (max - min + 1)) + min; +} + + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +console.log(vehicleURL) + +// sends two requests every minute, 1 POST request and 1 GET request +const postGetCarsTrafficTask = cron.schedule('* * * * *', async () => { + console.log('add 1 car every 1 minutes'); + const carData = {"make": "BMW", "model": "M340", "year": 2022, "image_name": "newCar.jpg"} + axios.post(`http://${vehicleURL}/`, carData, { timeout: 10000 }) + .catch(err => { + console.error(err.response && err.response.data); + }); + + // gets image from image service through the vehicle service + axios.get(`http://${vehicleURL}/1/image`, { timeout: 10000 }) + .catch(err => { + console.error(`${err.response}, ${err.response.data}`); + }); // Catch and log errors + + axios.get(`http://${vehicleURL}/1`, { timeout: 10000 }) + .catch(err => { + console.error(err.response && err.response.data); + }); // Catch and log errors +}, { scheduled: false }); +postGetCarsTrafficTask.start(); + +// sends a burst of traffic sending 10 requests every 15 mins: +// 5 POST requests and 5 GET requests. +const postGetCarsTrafficBurstTask = cron.schedule('*/15 * * * *', async () => { + console.log('add 5 cars within 1 minutes'); + const carData = {"make": "BMW", "model": "M340", "year": 2022, "image_name": "newCar.jpg"} + for (let i = 0; i < 5; i++) { + axios.post(`http://${vehicleURL}/`, carData, { timeout: 10000 }) + .catch(err => { + console.error(err.response && err.response.data); + }); // Catch and log errors + + // gets image from image service through the vehicle service + axios.get(`http://${vehicleURL}/1/image`, { timeout: 10000 }) + .catch(err => { + console.error(err.response && err.response.data); + }); // Catch and log errors + } +}, { scheduled: false }); +postGetCarsTrafficBurstTask.start(); + +// sends a GET request with custom throttle parameter in the body that mimics being throttled. The throttle time +// is going to be random between 2 - 5 secs. +const getCarThrottle = cron.schedule('*/5 * * * *', async () => { + sleepSecs = getRandomNumber(30,60); + console.log(`sleep ${sleepSecs} seconds`); + await sleep(sleepSecs*1000); + throttleSecs = getRandomNumber(2,5); + console.log(`request will be throttled for ${throttleSecs} seconds`) + axios.get(`http://${vehicleURL}/1`, {params: {"throttle": throttleSecs}}, { timeout: 10000 }) + .catch(err => { + console.error(err.response && err.response.data); + }); // Catch and log errors +}, { scheduled: false }); +getCarThrottle.start(); + +// sends an invalid GET request with a non existent car id to trigger 404 error +const getInvalidRequest = cron.schedule('*/5 * * * *', async () => { + sleepSecs = getRandomNumber(30,120); + console.log(`sleep ${sleepSecs} seconds`); + await sleep(sleepSecs*1000); + console.log("getting non existent car to trigger 404"); + axios.get(`http://${vehicleURL}/123456789`, { timeout: 10000 }) + .catch(err => { + console.error(err.response && err.response.data); + }); // Catch and log errors +}, { scheduled: false }); +getInvalidRequest.start(); + +// sends an invalid GET request with a non existent image name to trigger 500 error due to S3 Error: +// "An error occurred (NoSuchKey) when calling the GetObject operation: The specified key does not exist." +// The vehicle service will then return 404. +const getNonExistentImage = cron.schedule('*/5 * * * *', async () => { + sleepSecs = getRandomNumber(30,120); + console.log(`sleep ${sleepSecs} seconds`); + await sleep(sleepSecs*1000); + console.log('get an non existent image to trigger aws error'); + axios.get(`http://${vehicleURL}/image/doesnotexist.jpeg`) + .catch(err => { + console.error(err.response && err.response.data); + }); // Catch and log errors +}, { scheduled: false }); +getNonExistentImage.start(); \ No newline at end of file diff --git a/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/package-lock.json b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/package-lock.json new file mode 100644 index 000000000..6a7440a95 --- /dev/null +++ b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/package-lock.json @@ -0,0 +1,126 @@ +{ + "name": "random-traffic-generator", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "random-traffic-generator", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "axios": "^1.4.0", + "node-cron": "^3.0.2" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" + }, + "node_modules/axios": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.7.tgz", + "integrity": "sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==", + "dependencies": { + "follow-redirects": "^1.15.4", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", + "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/node-cron": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz", + "integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==", + "dependencies": { + "uuid": "8.3.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + } + } +} diff --git a/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/package.json b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/package.json new file mode 100644 index 000000000..74db18e0e --- /dev/null +++ b/sample-applications/vehicle-dealership-sample-app/random-traffic-generator/package.json @@ -0,0 +1,15 @@ +{ + "name": "random-traffic-generator", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "axios": "^1.4.0", + "node-cron": "^3.0.2" + } + } \ No newline at end of file diff --git a/sample-applications/vehicle-dealership-sample-app/scripts/push-ecr.sh b/sample-applications/vehicle-dealership-sample-app/scripts/push-ecr.sh index 109ae5f4b..37d050260 100755 --- a/sample-applications/vehicle-dealership-sample-app/scripts/push-ecr.sh +++ b/sample-applications/vehicle-dealership-sample-app/scripts/push-ecr.sh @@ -2,7 +2,7 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -REGION=${2:-"us-east-1"} +REGION=${1:-"us-east-1"} aws ecr get-login-password --region ${REGION} | docker login --username AWS --password-stdin ${REPOSITORY_PREFIX} From 8e3078b6afc92723e4e932cb5cabe58d5524505b Mon Sep 17 00:00:00 2001 From: Mohamed Asaker Date: Thu, 1 Feb 2024 09:47:34 -0800 Subject: [PATCH 2/2] Updated DB used by the sample app from mysql to postgres (#32) Updated the DB used by the sample app from mysql to postgres. Tested locally using docker and on EKS by running the one touch script. Was able to connect to DB and do the basic CRUD operations through vehicle-service APIs. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. --- .../ImageServiceApp/settings.py | 12 +++++- .../ImageServiceApp/requirements.txt | 3 +- .../vehicle-dealership-sample-app/README.md | 38 +++++++++---------- .../MainService/migrations/__init__.py | 0 .../VehicleInventoryApp/settings.py | 8 ++-- .../VehicleInventoryApp/requirements.txt | 2 +- .../docker-compose.yaml | 24 +++++++----- .../eks/backend-deployment.yaml | 10 ++--- .../eks/db-deployment.yaml | 31 +++++++-------- .../eks/db-service.yaml | 2 +- .../eks/image-backend-deployment.yaml | 8 +++- .../local_script.sh | 21 +++++----- .../vehicle-dealership-sample-app/script.sh | 7 ++-- .../scripts/deploy-eks.sh | 4 +- 14 files changed, 91 insertions(+), 79 deletions(-) create mode 100644 sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/MainService/migrations/__init__.py diff --git a/sample-applications/vehicle-dealership-sample-app/ImageServiceApp/ImageServiceApp/settings.py b/sample-applications/vehicle-dealership-sample-app/ImageServiceApp/ImageServiceApp/settings.py index 2d920fbb4..cb2b3833c 100644 --- a/sample-applications/vehicle-dealership-sample-app/ImageServiceApp/ImageServiceApp/settings.py +++ b/sample-applications/vehicle-dealership-sample-app/ImageServiceApp/ImageServiceApp/settings.py @@ -15,6 +15,10 @@ import os from pathlib import Path +from dotenv import load_dotenv + +load_dotenv() + # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -78,8 +82,12 @@ DATABASES = { "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": BASE_DIR / "db.sqlite3", + "ENGINE": "django.db.backends.postgresql", + "USER": os.environ.get("POSTGRES_USER"), + "NAME": os.environ.get("POSTGRES_DATABASE"), + "PASSWORD": os.environ.get("POSTGRES_PASSWORD"), + "HOST": os.environ.get("DB_SERVICE_HOST"), + "PORT": os.environ.get("DB_SERVICE_PORT"), } } diff --git a/sample-applications/vehicle-dealership-sample-app/ImageServiceApp/requirements.txt b/sample-applications/vehicle-dealership-sample-app/ImageServiceApp/requirements.txt index e5d1a2059..b329cbad0 100644 --- a/sample-applications/vehicle-dealership-sample-app/ImageServiceApp/requirements.txt +++ b/sample-applications/vehicle-dealership-sample-app/ImageServiceApp/requirements.txt @@ -1,4 +1,5 @@ Django~=5.0 requests~=2.31.0 boto3~=1.34.14 -python-dotenv~=1.0.0 \ No newline at end of file +python-dotenv~=1.0.0 +psycopg2~=2.9.9 \ No newline at end of file diff --git a/sample-applications/vehicle-dealership-sample-app/README.md b/sample-applications/vehicle-dealership-sample-app/README.md index 5b706d000..dcdbf8a83 100644 --- a/sample-applications/vehicle-dealership-sample-app/README.md +++ b/sample-applications/vehicle-dealership-sample-app/README.md @@ -23,7 +23,7 @@ This directory contains code for a microservices based sample app that is used t ### EKS To get started with AWS EKS, you can run the one touch script as below. -`bash script.sh ` +`bash script.sh ` This will create the docker images, upload them to ECR and then create pods on EKS with those images. @@ -52,7 +52,7 @@ To deploy to EC2, you will have to go through the following steps. - AmazonS3FullAccess - AmazonSQSFullAccess - Name one vehicle-service and the other image-service -5. Go to RDS and create a MySQL DB with the following configurations: +5. Go to RDS and create a Postgres DB with the following configurations: - Use the Dev/Test template - Update the Master username to `root` and create a password of your choosing. Write it down since you will need it later. - In the Connectivity settings, choose the VPC and security group created in step 3. @@ -61,18 +61,15 @@ To deploy to EC2, you will have to go through the following steps. ``` sudo dnf install python3.11 sudo dnf install python3.11-pip -sudo dnf install mariadb105 -sudo dnf install -y mariadb105-devel gcc python3.11-devel +sudo dnf install postgresql15 -mysql -h -P 3306 -u root -p +createdb vehicle_inventory -h -U root +createuser djangouser -h -U root -CREATE DATABASE vehicle_inventory; +psql -h -d vehicle_inventory -U root -CREATE USER 'djangouser'@'%' IDENTIFIED BY ''; - -GRANT ALL PRIVILEGES ON vehicle_inventory.* TO 'djangouser'@'%' WITH GRANT OPTION; - -FLUSH PRIVILEGES; +alter user djangouser with encrypted password ''; +grant all privileges on database vehicle_inventory to djangouser; aws s3 sync s3:// . @@ -81,10 +78,10 @@ cd to the vehicle microservice directory and run: python3.11 -m pip install -r requirements.txt Create a .env file with the following: -MYSQL_ROOT_PASSWORD= -MYSQL_DATABASE=vehicle_inventory -MYSQL_USER=djangouser -MYSQL_PASSWORD= +POSTGRES_ROOT_PASSWORD= +POSTGRES_DATABASE=vehicle_inventory +POSTGRES_USER=djangouser +POSTGRES_PASSWORD= DB_SERVICE_HOST= DB_SERVICE_PORT=3306 IMAGE_BACKEND_SERVICE_HOST= @@ -92,7 +89,8 @@ IMAGE_BACKEND_SERVICE_PORT=8000 python3.11 manage.py migrate --noinput && python3.11 manage.py runserver 0.0.0.0:8001 ``` -7. Connect to the `image-service` EC2 instance and run the following: +7. Go to the EC2 console and select the `image-service`, Go Actions -> Networking -> Connect RDS database. +8. Connect to the `image-service` EC2 instance and run the following: ``` sudo dnf install python3.11 sudo dnf install python3.11-pip @@ -113,7 +111,7 @@ Now you should be able to access the APIs below through the EC2 addr:port of eac ### Locally with Docker To get started, make sure you either have you AWS creds in `$HOME/.aws` or the following: `AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN` are exported. -1. Run `bash local_script.sh `. +1. Run `bash local_script.sh `. This will create docker containers, move the requirement env variables there and start them up. They should be accessible through `0.0.0.0:8000` for the image service and `0.0.0.0:8001` for the vehicle service. @@ -121,10 +119,10 @@ They should be accessible through `0.0.0.0:8000` for the image service and `0.0. ## APIs The following are the APIs and what they do: -1. GET /vehicle-inventory/: returns all the vehicles entries for mysql db -2. PUT /vehicle-inventory/: puts vehicle into db. For example: `curl -X POST http://0.0.0.0:8001/vehicle-inventory/ -d '{"make": "BMW","model": "M340","year": 2022,"image_name": "newCar.jpg"}'` +1. GET /vehicle-inventory/: returns all the vehicles entries for postgres db +2. POST /vehicle-inventory/: puts vehicle into db. For example: `curl -X POST http://0.0.0.0:8001/vehicle-inventory/ -d '{"make": "BMW","model": "M340","year": 2022,"image_name": "newCar.jpg"}'` 3. GET /vehicle-inventory/: returns vehicle entry with id = 4. GET /vehicle-inventory//image: returns image file information from S3 for the specific vehicle by calling the image microservice 5. GET /images/name/: returns image information for from S3 if present. -6. PUT /images/name/: creates an empty file in S3. This is an async endpoint since it will put image name in an SQS queue and not wait for the file to be created in S3. Instead, a long running thread will poll SQS and then create the image file later. +6. POST /images/name/: creates an empty file in S3. This is an async endpoint since it will put image name in an SQS queue and not wait for the file to be created in S3. Instead, a long running thread will poll SQS and then create the image file later. 7. GET /image/remote-image: makes a remote http call to google.com. \ No newline at end of file diff --git a/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/MainService/migrations/__init__.py b/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/MainService/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/VehicleInventoryApp/settings.py b/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/VehicleInventoryApp/settings.py index 210596c85..af06003f3 100644 --- a/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/VehicleInventoryApp/settings.py +++ b/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/VehicleInventoryApp/settings.py @@ -81,10 +81,10 @@ DATABASES = { "default": { - "ENGINE": "django.db.backends.mysql", - "USER": os.environ.get("MYSQL_USER"), - "NAME": os.environ.get("MYSQL_DATABASE"), - "PASSWORD": os.environ.get("MYSQL_PASSWORD"), + "ENGINE": "django.db.backends.postgresql", + "USER": os.environ.get("POSTGRES_USER"), + "NAME": os.environ.get("POSTGRES_DATABASE"), + "PASSWORD": os.environ.get("POSTGRES_PASSWORD"), "HOST": os.environ.get("DB_SERVICE_HOST"), "PORT": os.environ.get("DB_SERVICE_PORT"), } diff --git a/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/requirements.txt b/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/requirements.txt index 3da9e85d4..78cedc1c4 100644 --- a/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/requirements.txt +++ b/sample-applications/vehicle-dealership-sample-app/VehicleInventoryApp/requirements.txt @@ -1,4 +1,4 @@ Django~=5.0 requests~=2.31.0 -mysqlclient~=2.2.1 +psycopg2~=2.9.9 python-dotenv~=1.0.0 \ No newline at end of file diff --git a/sample-applications/vehicle-dealership-sample-app/docker-compose.yaml b/sample-applications/vehicle-dealership-sample-app/docker-compose.yaml index 20238c4da..0de46070c 100644 --- a/sample-applications/vehicle-dealership-sample-app/docker-compose.yaml +++ b/sample-applications/vehicle-dealership-sample-app/docker-compose.yaml @@ -3,24 +3,24 @@ version: '3' services: db: - image: mysql:8.0 + image: postgres:14.0 container_name: vehicle_inventory_db restart: always volumes: - - data:/var/lib/mysql + - data:/var/lib/postgres environment: - MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD} - MYSQL_DATABASE: ${MYSQL_DATABASE} - MYSQL_USER: ${MYSQL_USER} - MYSQL_PASSWORD: ${MYSQL_PASSWORD} + POSTGRES_DB: ${POSTGRES_DATABASE} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + PGUSER: ${POSTGRES_USER} ports: - - "3306:3306" + - "5432:5432" healthcheck: - test: ["CMD", "mysql", "-h", "localhost", "-u", "root", "-p${MYSQL_PASSWORD}", "-e", "SELECT 1"] + test: ["CMD", "pg_isready", "-d", "vehicle_inventory", "-U", "djangouser"] timeout: 20s retries: 10 expose: - - 3306 + - 5432 vehicle-inventory-backend: image: pythonsampleapp/vehicle-inventory-service @@ -44,13 +44,17 @@ services: context: . dockerfile: ImageServiceApp/Dockerfile container_name: image-service-backend - command: sh -c "python3 manage.py migrate --noinput && python3 manage.py collectstatic --noinput && python3 manage.py runserver 0.0.0.0:8000" + command: sh -c "python3 manage.py runserver 0.0.0.0:8000" restart: always volumes: - ./ImageServiceApp/.:/image-service-app - $HOME/.aws/credentials:/home/app/.aws/credentials:ro ports: - "8000:8000" + depends_on: + db: + condition: service_healthy + volumes: data: diff --git a/sample-applications/vehicle-dealership-sample-app/eks/backend-deployment.yaml b/sample-applications/vehicle-dealership-sample-app/eks/backend-deployment.yaml index b8fe8c99a..e67fb5ed5 100644 --- a/sample-applications/vehicle-dealership-sample-app/eks/backend-deployment.yaml +++ b/sample-applications/vehicle-dealership-sample-app/eks/backend-deployment.yaml @@ -35,13 +35,11 @@ spec: image: ${REPOSITORY_PREFIX}/pythonsampleapp/vehicle-inventory-service:latest name: vehicle-inventory-backend env: - - name: MYSQL_DATABASE + - name: POSTGRES_DATABASE value: vehicle_inventory - - name: MYSQL_PASSWORD - value: ${MYSQL_PASSWORD} - - name: MYSQL_ROOT_PASSWORD - value: ${MYSQL_PASSWORD} - - name: MYSQL_USER + - name: POSTGRES_PASSWORD + value: ${POSTGRES_PASSWORD} + - name: POSTGRES_USER value: djangouser imagePullPolicy: Always ports: diff --git a/sample-applications/vehicle-dealership-sample-app/eks/db-deployment.yaml b/sample-applications/vehicle-dealership-sample-app/eks/db-deployment.yaml index a17059ce1..5f9a7d208 100644 --- a/sample-applications/vehicle-dealership-sample-app/eks/db-deployment.yaml +++ b/sample-applications/vehicle-dealership-sample-app/eks/db-deployment.yaml @@ -29,36 +29,31 @@ spec: spec: containers: - env: - - name: MYSQL_DATABASE + - name: POSTGRES_DB value: vehicle_inventory - - name: MYSQL_PASSWORD - value: ${MYSQL_PASSWORD} - - name: MYSQL_ROOT_PASSWORD - value: ${MYSQL_PASSWORD} - - name: MYSQL_USER + - name: POSTGRES_PASSWORD + value: ${POSTGRES_PASSWORD} + - name: POSTGRES_USER value: djangouser - image: mysql:8.0 + image: postgres:14.0 livenessProbe: exec: command: - - mysql - - -h - - localhost - - -u - - root - - -p${MYSQL_PASSWORD} - - -e - - SELECT 1 + - pg_isready + - -d + - vehicle_inventory + - -U + - djangouser failureThreshold: 10 timeoutSeconds: 20 name: vehicle-inventory-db ports: - - containerPort: 3306 - hostPort: 3306 + - containerPort: 5432 + hostPort: 5432 protocol: TCP resources: {} volumeMounts: - - mountPath: /var/lib/mysql + - mountPath: /var/lib/postgres name: data restartPolicy: Always volumes: diff --git a/sample-applications/vehicle-dealership-sample-app/eks/db-service.yaml b/sample-applications/vehicle-dealership-sample-app/eks/db-service.yaml index b1188e47e..dcb601c67 100644 --- a/sample-applications/vehicle-dealership-sample-app/eks/db-service.yaml +++ b/sample-applications/vehicle-dealership-sample-app/eks/db-service.yaml @@ -12,7 +12,7 @@ metadata: name: db spec: ports: - - port: 3306 + - port: 5432 selector: io.kompose.service: db status: diff --git a/sample-applications/vehicle-dealership-sample-app/eks/image-backend-deployment.yaml b/sample-applications/vehicle-dealership-sample-app/eks/image-backend-deployment.yaml index d37a8e6ef..2e623731a 100644 --- a/sample-applications/vehicle-dealership-sample-app/eks/image-backend-deployment.yaml +++ b/sample-applications/vehicle-dealership-sample-app/eks/image-backend-deployment.yaml @@ -31,7 +31,7 @@ spec: - args: - sh - -c - - python3 manage.py migrate --noinput; python3 manage.py collectstatic --noinput; python3 manage.py runserver 0.0.0.0:8000 + - python3 manage.py runserver 0.0.0.0:8000 image: ${REPOSITORY_PREFIX}/pythonsampleapp/image-service:latest name: image-service-backend imagePullPolicy: Always @@ -43,5 +43,11 @@ spec: env: - name: S3_BUCKET value: ${S3_BUCKET} + - name: POSTGRES_DATABASE + value: vehicle_inventory + - name: POSTGRES_PASSWORD + value: ${POSTGRES_PASSWORD} + - name: POSTGRES_USER + value: djangouser restartPolicy: Always status: {} diff --git a/sample-applications/vehicle-dealership-sample-app/local_script.sh b/sample-applications/vehicle-dealership-sample-app/local_script.sh index 44c6c71ef..12e4de963 100644 --- a/sample-applications/vehicle-dealership-sample-app/local_script.sh +++ b/sample-applications/vehicle-dealership-sample-app/local_script.sh @@ -23,23 +23,26 @@ rm VehicleInventoryApp/.env rm ImageServiceApp/.env rm .env -echo "MYSQL_ROOT_PASSWORD=${password}" >> VehicleInventoryApp/.env -echo "MYSQL_DATABASE=vehicle_inventory" >> VehicleInventoryApp/.env -echo "MYSQL_USER=djangouser" >> VehicleInventoryApp/.env -echo "MYSQL_PASSWORD=${password}" >> VehicleInventoryApp/.env +echo "POSTGRES_DATABASE=vehicle_inventory" >> VehicleInventoryApp/.env +echo "POSTGRES_USER=djangouser" >> VehicleInventoryApp/.env +echo "POSTGRES_PASSWORD=${password}" >> VehicleInventoryApp/.env echo "DB_SERVICE_HOST=db" >> VehicleInventoryApp/.env -echo "DB_SERVICE_PORT=3306" >> VehicleInventoryApp/.env +echo "DB_SERVICE_PORT=5432" >> VehicleInventoryApp/.env echo "IMAGE_BACKEND_SERVICE_HOST=image-service-backend" >> VehicleInventoryApp/.env echo "IMAGE_BACKEND_SERVICE_PORT=8000" >> VehicleInventoryApp/.env +echo "POSTGRES_DATABASE=vehicle_inventory" >> ImageServiceApp/.env +echo "POSTGRES_USER=djangouser" >> ImageServiceApp/.env +echo "POSTGRES_PASSWORD=${password}" >> ImageServiceApp/.env +echo "DB_SERVICE_HOST=db" >> ImageServiceApp/.env +echo "DB_SERVICE_PORT=5432" >> ImageServiceApp/.env echo "AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}" >> ImageServiceApp/.env echo "AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}" >> ImageServiceApp/.env echo "AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN}" >> ImageServiceApp/.env echo "S3_BUCKET=${s3_bucket}" >> ImageServiceApp/.env -echo "MYSQL_ROOT_PASSWORD=${password}" >> .env -echo "MYSQL_DATABASE=vehicle_inventory" >> .env -echo "MYSQL_USER=djangouser" >> .env -echo "MYSQL_PASSWORD=${password}" >> .env +echo "POSTGRES_DATABASE=vehicle_inventory" >> .env +echo "POSTGRES_USER=djangouser" >> .env +echo "POSTGRES_PASSWORD=${password}" >> .env docker-compose up --build diff --git a/sample-applications/vehicle-dealership-sample-app/script.sh b/sample-applications/vehicle-dealership-sample-app/script.sh index 625f95469..6978b2243 100644 --- a/sample-applications/vehicle-dealership-sample-app/script.sh +++ b/sample-applications/vehicle-dealership-sample-app/script.sh @@ -14,10 +14,9 @@ rm ImageServiceApp/.env touch ImageServiceApp/.env export REPOSITORY_PREFIX=${account}.dkr.ecr.$region.amazonaws.com -export MYSQL_ROOT_PASSWORD=${password} -export MYSQL_DATABASE=vehicle_inventory -export MYSQL_USER=djangouser -export MYSQL_PASSWORD=${password} +export POSTGRES_DATABASE=vehicle_inventory +export POSTGRES_USER=djangouser +export POSTGRES_PASSWORD=${password} export S3_BUCKET=${s3_bucket} docker-compose build diff --git a/sample-applications/vehicle-dealership-sample-app/scripts/deploy-eks.sh b/sample-applications/vehicle-dealership-sample-app/scripts/deploy-eks.sh index e8ed528e3..c9f7aae8a 100755 --- a/sample-applications/vehicle-dealership-sample-app/scripts/deploy-eks.sh +++ b/sample-applications/vehicle-dealership-sample-app/scripts/deploy-eks.sh @@ -14,11 +14,11 @@ then else for config in $(ls ./eks/*.yaml) do - sed -e 's#\${REPOSITORY_PREFIX}'"#${REPOSITORY_PREFIX}#g" -e 's#\${MYSQL_PASSWORD}'"#${MYSQL_PASSWORD}#g" -e 's#\${S3_BUCKET}'"#${S3_BUCKET}#g" ${config} | kubectl ${OPERATION} -f - + sed -e 's#\${REPOSITORY_PREFIX}'"#${REPOSITORY_PREFIX}#g" -e 's#\${POSTGRES_PASSWORD}'"#${POSTGRES_PASSWORD}#g" -e 's#\${S3_BUCKET}'"#${S3_BUCKET}#g" ${config} | kubectl ${OPERATION} -f - done for config in $(ls ./eks/k8s-nginx-ingress/*.yaml) do - sed -e 's#\${REPOSITORY_PREFIX}'"#${REPOSITORY_PREFIX}#g" -e 's#\${MYSQL_PASSWORD}'"#${MYSQL_PASSWORD}#g" -e 's#\${S3_BUCKET}'"#${S3_BUCKET}#g" ${config} | kubectl ${OPERATION} -f - + sed -e 's#\${REPOSITORY_PREFIX}'"#${REPOSITORY_PREFIX}#g" -e 's#\${POSTGRES_PASSWORD}'"#${POSTGRES_PASSWORD}#g" -e 's#\${S3_BUCKET}'"#${S3_BUCKET}#g" ${config} | kubectl ${OPERATION} -f - done fi \ No newline at end of file