From 41ad5dd6111db557e0bb2e20b59ae0f8f44471ed Mon Sep 17 00:00:00 2001 From: David Gilardi Date: Mon, 25 Jun 2018 17:08:14 -0400 Subject: [PATCH 01/10] Totally hacked version to include support for SSL connectivity to DSE nodes Dockerfile includes reference to cassandra.cert. This is the cert file to use connecting to nodes. This will NOT be file based in quite this way in the future, just a placeholder for now. Updated version to 1.3.0 along with other services to make it easy to keep them all together Hacked cassandra.js to provide very rudimentary support for using SSL via the DSE driver. This is just a starting point and not meant to actually be used. --- .dockerignore | 1 + Dockerfile | 2 ++ npm-shrinkwrap.json | 6 +++--- package.json | 4 ++-- src/utils/cassandra.js | 7 +++++++ 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/.dockerignore b/.dockerignore index dbb7c66..2720b06 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,4 @@ config/local.yaml !dist/* !package.json !npm-shrinkwrap.json +!cassandra.cert diff --git a/Dockerfile b/Dockerfile index aefccfa..b0ded8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,8 @@ WORKDIR /opt/killrvideo-generator # Copy package.json for dependencies COPY package.json /opt/killrvideo-generator/ COPY npm-shrinkwrap.json /opt/killrvideo-generator/ + +COPY cassandra.cert /opt/killrvideo-generator/ # Add dependencies for node-gyp, then run npm install and remove dependencies RUN set -x \ diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 0f43e65..1bb6ffe 100755 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "killrvideo-generator", - "version": "1.2.6", + "version": "1.3.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -2268,8 +2268,8 @@ "dev": true }, "grpc": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/grpc/-/grpc-1.10.1.tgz", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/grpc/-/grpc-1.0.0.tgz", "integrity": "sha512-xmhA11h2XhqpSVzDAmoQAYdNQ+swILXpKOiRpAEQ2kX55ioxVADc6v7SkS4zQBxm4klhQHgGqpGKvoL6LGx4VQ==", "requires": { "lodash": "4.17.5", diff --git a/package.json b/package.json index 742fab1..4bea687 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "killrvideo-generator", - "version": "1.2.6", + "version": "1.3.0", "description": "Sample Data generator for KillrVideo", "main": "dist/index.js", "scripts": { @@ -31,7 +31,7 @@ "dse-driver": "^1.6.0", "faker": "^3.1.0", "googleapis": "^12.4.0", - "grpc": "^1.10.1", + "grpc": "^1.0.0", "js-yaml": "^3.11.0", "later": "^1.2.0", "regenerator-runtime": "^0.9.6", diff --git a/src/utils/cassandra.js b/src/utils/cassandra.js index 8b12b27..2d907ec 100644 --- a/src/utils/cassandra.js +++ b/src/utils/cassandra.js @@ -81,6 +81,13 @@ export function getCassandraClientAsync(keyspace, dseUsername, dsePassword) { } else { logger.info('No detected username/password combination was passed in. DSE cluster authentication method was NOT executed.'); } + + var fs = require('fs'); + clientOpts.sslOptions = { + //key : fs.readFileSync('cassandra.cert'), + //cert : fs.readFileSync('cassandra.cert'), + ca : [fs.readFileSync('cassandra.cert')] + }; // Create a client and promisify it let client = new Client(clientOpts); From f3089833cb4b39389053a960c0ac94b4aa7eece2 Mon Sep 17 00:00:00 2001 From: David Gilardi Date: Wed, 27 Jun 2018 20:29:06 -0400 Subject: [PATCH 02/10] Removed reference to cassandra.cert from the .dockerignore file as it is no longer needed --- .dockerignore | 1 - Dockerfile | 2 -- 2 files changed, 3 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2720b06..dbb7c66 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,4 +4,3 @@ config/local.yaml !dist/* !package.json !npm-shrinkwrap.json -!cassandra.cert diff --git a/Dockerfile b/Dockerfile index b0ded8a..92a06cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,8 +17,6 @@ WORKDIR /opt/killrvideo-generator COPY package.json /opt/killrvideo-generator/ COPY npm-shrinkwrap.json /opt/killrvideo-generator/ -COPY cassandra.cert /opt/killrvideo-generator/ - # Add dependencies for node-gyp, then run npm install and remove dependencies RUN set -x \ && apt-get update \ From a79be9fdf626631d8a46f40c4172752f3f0e6caa Mon Sep 17 00:00:00 2001 From: Adron Hall Date: Mon, 23 Jul 2018 09:35:53 -0700 Subject: [PATCH 03/10] Adding logging, logic for finding/using cert file. --- src/utils/cassandra.js | 47 ++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/src/utils/cassandra.js b/src/utils/cassandra.js index 2d907ec..7f61a89 100644 --- a/src/utils/cassandra.js +++ b/src/utils/cassandra.js @@ -1,8 +1,8 @@ import Promise from 'bluebird'; import config from 'config'; -import { Client, auth, types as CassandraTypes } from 'dse-driver'; -import { logger } from './logging'; -import { lookupServiceAsync } from './lookup-service'; +import {auth, Client, types as CassandraTypes} from 'dse-driver'; +import {logger} from './logging'; +import {lookupServiceAsync} from './lookup-service'; /** * An array of CQL table strings to use for the schema. @@ -73,7 +73,7 @@ export function getCassandraClientAsync(keyspace, dseUsername, dsePassword) { */ if (dseUsername && dsePassword) { let passwordLength = dsePassword.length; - logger.info('Using supplied DSE username: "' + dseUsername + '" and password: "***' + dsePassword.substring(passwordLength - 4, passwordLength) + '" from environment variables') + logger.info('Using supplied DSE username: "' + dseUsername + '" and password: "***' + dsePassword.substring(passwordLength - 4, passwordLength) + '" from environment variables'); // Use the values passed in from the config clientOpts.authProvider = new auth.DsePlainTextAuthProvider(dseUsername, dsePassword); @@ -82,13 +82,27 @@ export function getCassandraClientAsync(keyspace, dseUsername, dsePassword) { logger.info('No detected username/password combination was passed in. DSE cluster authentication method was NOT executed.'); } - var fs = require('fs'); - clientOpts.sslOptions = { - //key : fs.readFileSync('cassandra.cert'), - //cert : fs.readFileSync('cassandra.cert'), - ca : [fs.readFileSync('cassandra.cert')] - }; - + import {fs} from 'fs' + + let sslStat = process.env.KILLRVIDEO_ENABLE_SSL; + logger.log('debug', sslStat); + + if (sslStat === "true") { + logger.log('debug', 'SSL is configured to be on.'); + if (fs.existsSync('cassandra.cert')) { + clientOpts.sslOptions = { + ca: [fs.readFileSync('cassandra.cert')] + }; + logger.log('debug', 'Found cert, read file sync.') + } else { + logger.log('debug', 'No cert found, SSL not enabled.') + } + } else if (sslStat === "false") { + logger.log('debug', 'SSL is configured to be off.') + } else { + logger.log('debug', 'SSL is not configured, should it be set?') + } + // Create a client and promisify it let client = new Client(clientOpts); client = Promise.promisifyAll(client); @@ -103,8 +117,7 @@ export function getCassandraClientAsync(keyspace, dseUsername, dsePassword) { clientPromises.set(keyspace, promise); return promise; -}; - +} /** * Creates a keyspace in Cassandra if it doesn't already exist. Pass the name of the keyspace and the * string to be used as the REPLICATION setting (i.e. after WITH REPLIACTION = ...). @@ -115,8 +128,7 @@ function createKeyspaceIfNotExistsAsync(keyspace, replication, dseUsername, dseP // Get a client, then create the keyspace return getCassandraClientAsync(null, dseUsername, dsePassword).then(client => client.executeAsync(cql)); -}; - +} /** * Create the tables if they don't already exist. */ @@ -136,8 +148,7 @@ export function getCassandraClient() { throw new Error('No client instance found. Did you forget to call initCassandraAsync?'); } return clientInstance; -}; - +} /** * Initializes the Cassandra keyspace and schema needed. */ @@ -152,4 +163,4 @@ export async function initCassandraAsync() { // Save client instance clientInstance = client; -}; \ No newline at end of file +} \ No newline at end of file From 057c482faaef0db45fd3a86b4c276d3bd967419e Mon Sep 17 00:00:00 2001 From: Adron Hall Date: Mon, 23 Jul 2018 09:38:06 -0700 Subject: [PATCH 04/10] Adding logging for finding/using cert file & notification of environment variable. Updated require to import. Added logic for finding/using cert file. --- src/utils/cassandra.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/utils/cassandra.js b/src/utils/cassandra.js index 7f61a89..853d543 100644 --- a/src/utils/cassandra.js +++ b/src/utils/cassandra.js @@ -3,6 +3,7 @@ import config from 'config'; import {auth, Client, types as CassandraTypes} from 'dse-driver'; import {logger} from './logging'; import {lookupServiceAsync} from './lookup-service'; +import {Filesystem} from 'fs' /** * An array of CQL table strings to use for the schema. @@ -82,16 +83,14 @@ export function getCassandraClientAsync(keyspace, dseUsername, dsePassword) { logger.info('No detected username/password combination was passed in. DSE cluster authentication method was NOT executed.'); } - import {fs} from 'fs' - let sslStat = process.env.KILLRVIDEO_ENABLE_SSL; logger.log('debug', sslStat); if (sslStat === "true") { logger.log('debug', 'SSL is configured to be on.'); - if (fs.existsSync('cassandra.cert')) { + if (Filesystem.existsSync('cassandra.cert')) { clientOpts.sslOptions = { - ca: [fs.readFileSync('cassandra.cert')] + ca: [Filesystem.readFileSync('cassandra.cert')] }; logger.log('debug', 'Found cert, read file sync.') } else { From ef0fe87937ffb7660638e9246f46e4a80437ccbb Mon Sep 17 00:00:00 2001 From: David Gilardi Date: Wed, 25 Jul 2018 13:36:23 -0400 Subject: [PATCH 05/10] Updated version from 1.3.0 to 2.1.0 in both package.json and npm-shrinkwrap.json to synchronize versions for SSL support --- npm-shrinkwrap.json | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 1bb6ffe..d1c6e9b 100755 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "killrvideo-generator", - "version": "1.3.0", + "version": "2.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 4bea687..eb53705 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "killrvideo-generator", - "version": "1.3.0", + "version": "2.1.0", "description": "Sample Data generator for KillrVideo", "main": "dist/index.js", "scripts": { From 585c4b8df28b4ee25daf4c412848a03f231391b4 Mon Sep 17 00:00:00 2001 From: Adron Hall Date: Wed, 25 Jul 2018 11:28:19 -0700 Subject: [PATCH 06/10] Changed logger call to logger.info for clarity. --- src/utils/cassandra.js | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/utils/cassandra.js b/src/utils/cassandra.js index 853d543..aa93f87 100644 --- a/src/utils/cassandra.js +++ b/src/utils/cassandra.js @@ -67,11 +67,11 @@ export function getCassandraClientAsync(keyspace, dseUsername, dsePassword) { clientOpts.keyspace = keyspace; } - /** - * Check for both KILLRVIDEO_DSE_USERNAME and KILLRVIDEO_DSE_PASSWORD environment - * variables. If they both exist use the values set within them. If not, - * use default values for authentication. - */ + /** + * Check for both KILLRVIDEO_DSE_USERNAME and KILLRVIDEO_DSE_PASSWORD environment + * variables. If they both exist use the values set within them. If not, + * use default values for authentication. + */ if (dseUsername && dsePassword) { let passwordLength = dsePassword.length; logger.info('Using supplied DSE username: "' + dseUsername + '" and password: "***' + dsePassword.substring(passwordLength - 4, passwordLength) + '" from environment variables'); @@ -84,22 +84,22 @@ export function getCassandraClientAsync(keyspace, dseUsername, dsePassword) { } let sslStat = process.env.KILLRVIDEO_ENABLE_SSL; - logger.log('debug', sslStat); + logger.info(sslStat); if (sslStat === "true") { - logger.log('debug', 'SSL is configured to be on.'); + logger.info('SSL is configured to be on.'); if (Filesystem.existsSync('cassandra.cert')) { clientOpts.sslOptions = { ca: [Filesystem.readFileSync('cassandra.cert')] }; - logger.log('debug', 'Found cert, read file sync.') + logger.info('Found cert, read file sync.') } else { - logger.log('debug', 'No cert found, SSL not enabled.') + logger.info('No cert found, SSL not enabled.') } } else if (sslStat === "false") { - logger.log('debug', 'SSL is configured to be off.') + logger.info('SSL is configured to be off.') } else { - logger.log('debug', 'SSL is not configured, should it be set?') + logger.info('SSL is not configured, should it be set?') } // Create a client and promisify it From a2a9cf35ec945e2bd06d3951c5abdce409ccdf5f Mon Sep 17 00:00:00 2001 From: Adron Hall Date: Wed, 25 Jul 2018 11:33:17 -0700 Subject: [PATCH 07/10] Adding .vscode to gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7d72234..17a0418 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ config/production.yaml # Webstorm/IDEA .idea/ +.vscode \ No newline at end of file From 0bb52fe62ff0a5da3cf7b7b1aeece5adb7512b25 Mon Sep 17 00:00:00 2001 From: Adron Hall Date: Wed, 25 Jul 2018 11:47:13 -0700 Subject: [PATCH 08/10] Finished logging changes. --- src/utils/cassandra.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/cassandra.js b/src/utils/cassandra.js index aa93f87..25d6c85 100644 --- a/src/utils/cassandra.js +++ b/src/utils/cassandra.js @@ -162,4 +162,4 @@ export async function initCassandraAsync() { // Save client instance clientInstance = client; -} \ No newline at end of file +} From 7a3ae3da97281da668cae82ff1cabfc3aeaf4ff5 Mon Sep 17 00:00:00 2001 From: Adron Hall Date: Wed, 25 Jul 2018 12:29:03 -0700 Subject: [PATCH 09/10] Changed to let statement for filesystem. --- src/utils/cassandra.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/cassandra.js b/src/utils/cassandra.js index 25d6c85..8b1b30f 100644 --- a/src/utils/cassandra.js +++ b/src/utils/cassandra.js @@ -3,7 +3,6 @@ import config from 'config'; import {auth, Client, types as CassandraTypes} from 'dse-driver'; import {logger} from './logging'; import {lookupServiceAsync} from './lookup-service'; -import {Filesystem} from 'fs' /** * An array of CQL table strings to use for the schema. @@ -83,6 +82,7 @@ export function getCassandraClientAsync(keyspace, dseUsername, dsePassword) { logger.info('No detected username/password combination was passed in. DSE cluster authentication method was NOT executed.'); } + let Filesystem = require("fs"); let sslStat = process.env.KILLRVIDEO_ENABLE_SSL; logger.info(sslStat); From a934ff57cb5c4e7bb6952d75b107ee769c891fb1 Mon Sep 17 00:00:00 2001 From: David Gilardi Date: Thu, 23 Aug 2018 09:49:48 -0400 Subject: [PATCH 10/10] Added requestCert and rejectUnauthorized params to sslOptions per driver documentation --- src/utils/cassandra.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/utils/cassandra.js b/src/utils/cassandra.js index 8b1b30f..e73abbe 100644 --- a/src/utils/cassandra.js +++ b/src/utils/cassandra.js @@ -90,7 +90,10 @@ export function getCassandraClientAsync(keyspace, dseUsername, dsePassword) { logger.info('SSL is configured to be on.'); if (Filesystem.existsSync('cassandra.cert')) { clientOpts.sslOptions = { - ca: [Filesystem.readFileSync('cassandra.cert')] + ca: [Filesystem.readFileSync('cassandra.cert')], + // validate server cert and reject if not trusted + requestCert: true, + rejectUnauthorized: true }; logger.info('Found cert, read file sync.') } else {