Merge pull request #8 from KillrVideo/kubernetisation

Kubernetisation Rework

.travis.yml

@@ -5,11 +5,14 @@ node_js:
 # Build the app and a docker image
 script:
 - npm run build
-- ./scripts/docker-build.sh
+- docker build -t ${TRAVIS_COMMIT} --build-arg KILLRVIDEO_YOUTUBE_API_KEY=$KILLRVIDEO_YOUTUBE_API_KEY .
 
 # If successful, see if we need to publish also
 after_success:
-- ./scripts/travis-publish.sh
+- test -z $TRAVIS_TAG && travis_terminate 0
+- docker tag ${TRAVIS_COMMIT} killrvideo/killrvideo-generator:${TRAVIS_TAG}
+- docker login -u $DOCKER_USER -p $DOCKER_PASS
+- docker push killrvideo/killrvideo-generator:${TRAVIS_TAG}
 
 # Sudo required for doing docker build
 sudo: required

.vscode/launch.json

@@ -1,6 +1,17 @@
 {
 	"version": "0.2.0",
 	"configurations": [
+		{
+			"name": "Docker: Attach to node",
+			"type": "node",
+			"request": "attach",
+			"port": 5858,
+			"address": "localhost",
+			"restart": true,
+			"sourceMaps": true,
+			"localRoot": "${workspaceRoot}",
+			"remoteRoot": "/opt/killrvideo-generator"
+		},
 		{
 			"name": "Launch",
 			"type": "node",
@@ -23,18 +34,6 @@
 			"console": "integratedTerminal",
 			"sourceMaps": true,
 			"outFiles": ["${workspaceRoot}/dist/*"]
-		},
-		{
-			"name": "Attach",
-			"type": "node",
-			"request": "attach",
-			"port": 5858,
-			"address": "localhost",
-			"restart": true,
-			"sourceMaps": true,
-			"outFiles": ["${workspaceRoot}/dist/*"],
-			"localRoot": "${workspaceRoot}",
-			"remoteRoot": null
 		}
 	]
 }

Dockerfile

@@ -2,7 +2,7 @@ FROM node:6.11-slim
 
 # Add killrvideo group and user
 RUN groupadd -r killrvideo --gid=999 \
-    && useradd -r -g killrvideo --uid=999 killrvideo
+    && useradd -m -r -g killrvideo --uid=999 killrvideo
 
 # Default to production environment
 ENV NODE_ENV production
@@ -19,6 +19,7 @@ COPY npm-shrinkwrap.json /opt/killrvideo-generator/
 
 # Add dependencies for node-gyp, then run npm install and remove dependencies
 RUN set -x \
+    && printf "deb http://archive.debian.org/debian/ jessie main\ndeb-src http://archive.debian.org/debian/ jessie main\ndeb http://security.debian.org jessie/updates main\ndeb-src http://security.debian.org jessie/updates main" > /etc/apt/sources.list \
     && apt-get update \
     && apt-get install -y python \
                           make \

README.md

@@ -11,23 +11,20 @@ This app is packaged and distributed as a Docker container. The typical usage is
 
 ## Setting up a Development Environment
 
-After cloning the repo, first install all dependencies:
+After cloning the repo, first install all dependencies and build the project:
 ```
-> npm install
+> docker-compose run --no-deps -e NODE_ENV=development generator npm install
+> docker-compose run --no-deps generator npm run build
 ```
-All environment dependencies can be spun up using `docker-compose` (i.e. Etcd and DataStax
-Enterprise). First you need to generate a `.env` file that contains information about your
-Docker environment.
 
-In Windows, from a Powershell command prompt run:
-```
-PS> .\lib\killrvideo-docker-common\create-environment.ps1
+If you have npm available locally, you may use it directly instead:
 ```
-Or on Mac/Linux, run:
-```
-> ./lib/killrvideo-docker-common/create-environment.sh
+> npm install
+> npm run build
 ```
-You can then start those dependencies with:
+
+All environment dependencies can be spun up using `docker-compose` (i.e. Etcd and DataStax
+Enterprise). You can start those dependencies with:
 ```
 > docker-compose up -d
 ```
@@ -55,6 +52,10 @@ in `/dist`.
 If using VS Code for development, the tasks checked into the repo under `/.vscode` should 
 allow you to start the program with debugging using `F5`.
 
+By default docker-compose runs generator with debugger enabled and opens port 5858. You can use this to attach to the launched application to debug it.
+
+If you would like to use DataStax Studio to work directly with the database, please uncomment studio definition in ./docker-compose.yaml
+
 ## Releasing
 
 The app is released as a Docker image for use with the service project implementations.
@@ -69,4 +70,27 @@ Docker image.
 
 We use Travis CI for doing continuous integration builds and it will use those scripts to 
 automatically publish any tagged Git commits to Docker Hub. You can, of course, manually
-build and publish Docker images with those scripts as well.
+build and publish Docker images with those scripts as well.
+
+## Known Issues
+
+### Error: Bad Request
+
+Generator repeatedly logs error:
+
+```
+generator_1   | 2019-01-18T13:37:35.543Z - error:  Error: Bad Request
+generator_1   |     at Request._callback (/opt/killrvideo-generator/node_modules/google-auth-library/lib/transporters.js:85:15)
+```
+
+The issue is ussually caused by unset youTubeApiKey. Check if you have set youTubeApiKey in config/local.yaml file and if it's still valid.
+
+### Could not initialize Cassandra 
+
+```
+generator_1   | 2019-01-18T13:46:21.015Z - debug:  NoHostAvailableError: All host(s) tried for query failed. First host tried, 172.26.0.2:9042: Error: connect ECONNREFUSED 172.26.0.2:9042. See innerErrors.
+...
+generator_1   | 2019-01-18T13:46:21.018Z - verbose: Could not initialize Cassandra. Retry 1 in 10000ms.
+```
+
+Start up of DSE/Cassandra takes noticeable time, so this error happens every time if you launch generator and DSE at the same time. Usually it can be ignored, but if connection takes too long time, please check if dse container running and check its logs.

config/default.yaml

@@ -1,5 +1,5 @@
 # Key required for calling YouTube APIs
-youTubeApiKey: REPLACE_WITH_YOUR_KEY
+youTubeApiKey: REPLACE_WITH_YOUR_KEY_IN config/local.yaml
 
 # Cassandra settings
 cassandra:
@@ -25,10 +25,23 @@ schedules:
   - every 20 seconds
 
   addSampleUser:
-  - every 30 minutes
+  - every 10 minutes
 
   addSampleVideo:
-  - every 8 hours
+  - every 10 minutes
 
   addSampleVideoView:
-  - every 1 seconds
+  - every 1 seconds
+
+services:
+  web:                    ['web:3000']
+  cassandra:              ['dse:9042']
+  dse-search:             ['dse:8983']
+  UploadsService:         ['backend:50101']
+  RatingsService:         ['backend:50101']
+  CommentsService:        ['backend:50101']
+  SearchService:          ['backend:50101']
+  StatisticsService:      ['backend:50101']
+  VideoCatalogService:    ['backend:50101']
+  UserManagementService:  ['backend:50101']
+  SuggestedVideoService:  ['backend:50101']

docker-compose.yaml

@@ -1,3 +1,62 @@
-# No services specific to this project, instead just uses base services defined
-# in .\lib\killrvideo-docker-common\docker-compose.yaml 
-version: '2'
+version: '3'
+services:
+  # The KillrVideo Sample Data Generator
+  generator:
+    build: .
+    #image: killrvideo/killrvideo-generator
+    volumes:
+      - .:/opt/killrvideo-generator
+    ports:
+      - "5858:5858"
+    depends_on:
+      - dse
+      - backend
+    environment:
+      KILLRVIDEO_LOGGING_LEVEL: debug
+    command: "node --debug /opt/killrvideo-generator/dist/index.js"
+
+  # DataStax Enterprise
+  # start with search and graph modes enabled ("-s -g")
+  dse:
+    image: datastax/dse-server:6.0.0
+    command: [ -s -g ]
+    ports:
+    - "9042:9042"
+    - "8983:8983"
+    - "8182:8182"
+    environment:
+      DS_LICENSE: accept
+    # Allow DSE to lock memory with mlock
+    cap_add:
+    - IPC_LOCK
+    ulimits:
+      memlock: -1
+
+  # Container to load KillrVideo schema and search config into DSE
+  # Provides options to configure secure users as well
+  # Runs only once and exits
+  dse-config:
+    image: killrvideo/killrvideo-dse-config:2.2.1
+    depends_on:
+      - dse
+    environment:
+      KILLRVIDEO_SERVICE_DISCOVERY_DISABLED: 'true'
+
+  # The KillrVideo backend application used by the generator to submit sample data
+  backend:
+    image: hadesarchitect/killrvideo-nodejs:no-etcd
+    ports:
+      - "50101:50101"
+    depends_on:
+      - dse
+    environment:
+      KILLRVIDEO_LOGGING_LEVEL: debug
+
+#  studio:
+#    image: killrvideo/killrvideo-studio:2.0.0
+#    ports:
+#      - "9091:9091"
+#    depends_on:
+#      - dse
+#    environment:
+#      DS_LICENSE: accept