From 57eacad9c4ab4191da5ab194d3b0a8e65f5c8cbe Mon Sep 17 00:00:00 2001 From: Aleksandr Volochnev Date: Tue, 12 Feb 2019 16:01:51 +0100 Subject: [PATCH 1/2] Clean up (not needed after travis + git tags) --- build/VERSION | 1 - build/docker-build.ps1 | 16 ---------------- build/docker-build.sh | 13 ------------- build/docker-publish.sh | 31 ------------------------------- build/git-subtree-pull.ps1 | 7 ------- build/git-subtree-pull.sh | 5 ----- 6 files changed, 73 deletions(-) delete mode 100644 build/VERSION delete mode 100644 build/docker-build.ps1 delete mode 100755 build/docker-build.sh delete mode 100755 build/docker-publish.sh delete mode 100644 build/git-subtree-pull.ps1 delete mode 100755 build/git-subtree-pull.sh diff --git a/build/VERSION b/build/VERSION deleted file mode 100644 index c1d8330..0000000 --- a/build/VERSION +++ /dev/null @@ -1 +0,0 @@ -DOCKER_BUILD_TAG=2.2.1 diff --git a/build/docker-build.ps1 b/build/docker-build.ps1 deleted file mode 100644 index 81c669d..0000000 --- a/build/docker-build.ps1 +++ /dev/null @@ -1,16 +0,0 @@ -<# - .DESCRIPTION - Runs a docker build for the version specified in the VERSION file. -#> - -$scriptPath = Split-Path -parent $PSCommandPath - -# Use contents of VERSION file as variables -Get-Content "$scriptPath\VERSION" |% { - if ($_) { - $varParts = $_.Split('=') - New-Variable -Name $varParts[0] -Value $varParts[1] - } -} - -docker build -t killrvideo/killrvideo-dse-config:$DOCKER_BUILD_TAG . diff --git a/build/docker-build.sh b/build/docker-build.sh deleted file mode 100755 index 67704c7..0000000 --- a/build/docker-build.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -set -x - -# Load the version number -. "`dirname $0`/VERSION" - -# Login if user/password present -if [ ! -z "$DOCKER_USER" ] && [ ! -z "$DOCKER_PASS" ]; then - docker login -u $DOCKER_USER -p $DOCKER_PASS -fi - -# Build with docker -docker build -t killrvideo/killrvideo-dse-config:$DOCKER_BUILD_TAG . diff --git a/build/docker-publish.sh b/build/docker-publish.sh deleted file mode 100755 index cbe5896..0000000 --- a/build/docker-publish.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh - -# This script assumes the following environment variables are present -# -# DOCKER_USER=username -# DOCKER_PASS=somesecretpassword -DOCKER_IMAGE=killrvideo/killrvideo-dse-config - -# Allow git describe to fail (which it will if this isn't a tagged commit) -set +e -CURRENT_TAG=`git describe --tags --exact-match 2> /dev/null` - -set -e # Exit with nonzero exit code if anything fails - -# Only allow publish for tags -if [ -z "$CURRENT_TAG" ]; then - echo "Current commit is not for a tag, skipping publish" - exit 0 -fi - -# Make sure we have a user/pass -if [[ -z "$DOCKER_USER" ]] || [[ -z "$DOCKER_PASS" ]]; then - echo "DOCKER_USER or DOCKER_PASS not set" - exit 1 -fi - -echo "Publishing $DOCKER_IMAGE for git tag $CURRENT_TAG" - -# Login to Docker and push the image which should have been built -docker login -u $DOCKER_USER -p $DOCKER_PASS -docker push $DOCKER_IMAGE diff --git a/build/git-subtree-pull.ps1 b/build/git-subtree-pull.ps1 deleted file mode 100644 index 20cd5a2..0000000 --- a/build/git-subtree-pull.ps1 +++ /dev/null @@ -1,7 +0,0 @@ -<# - .DESCRIPTION - Pull the latest from the projects that are git subtrees. -#> - -git subtree pull --prefix lib/killrvideo-data git@github.com:KillrVideo/killrvideo-data.git master --squash -git subtree pull --prefix lib/wait-for-it git@github.com:vishnubob/wait-for-it.git master --squash \ No newline at end of file diff --git a/build/git-subtree-pull.sh b/build/git-subtree-pull.sh deleted file mode 100755 index eb7b120..0000000 --- a/build/git-subtree-pull.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -# Pull the latest from projects that are git subtrees -git subtree pull --prefix lib/killrvideo-data git@github.com:KillrVideo/killrvideo-data.git master --squash -git subtree pull --prefix lib/wait-for-it git@github.com:vishnubob/wait-for-it.git master --squash \ No newline at end of file From d5fab4065640b2c20944b9e822960b52e28ce8f3 Mon Sep 17 00:00:00 2001 From: Aleksandr Volochnev Date: Tue, 12 Feb 2019 21:06:00 +0100 Subject: [PATCH 2/2] Set-Flag Improvement - Switched bootstrap to use external flag instead of the file flag - Removed curl installation from Dockerfile as curl was needed only for ETCD --- Dockerfile | 10 +--------- README.md | 6 ++---- bootstrap.sh | 43 ++++++++++++++++++------------------------- 3 files changed, 21 insertions(+), 38 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1f3eb94..e7453e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,8 @@ FROM datastax/dse-server:6.7.0 -# Copy schema files into /opt/killrvideo-data +# Copy required files COPY [ "lib/killrvideo-data/graph/killrvideo_video_recommendations_schema.groovy", "lib/killrvideo-data/schema.cql", "lib/killrvideo-data/search/*", "keyspace.cql", "/opt/killrvideo-data/" ] - -# Copy bootstrap script(s) and make executable COPY [ "bootstrap.sh", "lib/wait-for-it/wait-for-it.sh", "/" ] -# Make sure curl command is available for registering DSE ports with etcd -USER root -RUN set -x \ - && apt-get update -qq && apt-get install -y curl -# && apt-get install -y sudo - # Set the entrypoint to the bootstrap script ENTRYPOINT [ "/bootstrap.sh" ] diff --git a/README.md b/README.md index 1606a22..74f2b0e 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,7 @@ This container supports several different options for configuration: ### DSE node location By default, this container assumes that DataStax Enterprise is being started in a Docker container -as part of a `docker-compose` configuration. The value of the `KILLRVIDEO_DOCKER_IP` -environment variable is used to register DSE services in `etcd`. +as part of a `docker-compose` configuration. If you instead wish to deploy KillrVideo with an existing external cluster, you can override this behavior by setting the value of the `KILLRVIDEO_DSE_EXTERNAL_IP` environment variable to the location @@ -65,8 +64,7 @@ please see the [KillrVideo Docker documentation page][docker-doc]. ## Builds and Releases -The `./build` folder contains a number of scripts to help with builds and releases. Continuous -integration builds are done by Travis and any commits that are tagged will also automatically +Continuous integration builds are done by Travis and any commits that are tagged will also automatically be released to the [Docker Hub][docker-hub] page. We try to follow semantic versioning, however the version numbering is not related to what version of DSE we're using. For example, version `1.0.0` uses DSE version `5.1.5`. diff --git a/bootstrap.sh b/bootstrap.sh index ed423dd..0231514 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -3,12 +3,6 @@ set -e echo '===> DSE Configuration' -# See if we've already completed bootstrapping -if [ -f killrvideo_bootstrapped ]; then - echo '=> Configuration already completed, exiting' - exit 0 -fi - # Default addresses to use for DSE cluster if starting in Docker dse_ip='dse' dse_external_ip=$KILLRVIDEO_DOCKER_IP @@ -54,7 +48,7 @@ if [ "$KILLRVIDEO_ENABLE_SSL" = 'true' ]; then echo "=> SSL encryption is ENABLED with CERT FILE: $dse_ssl_certfile" fi -# Wait for port 9042 (CQL) to be ready for up to 240 seconds +# Wait for port 9042 (CQL) to be ready for up to 300 seconds echo '=> Waiting for DSE to become available' /wait-for-it.sh -t 300 $dse_ip:9042 echo '=> DSE is available' @@ -68,6 +62,12 @@ dse_password='cassandra' # If requested, create a new superuser to replace the default superuser if [ "$KILLRVIDEO_CREATE_ADMIN_USER" = 'true' ]; then + # Check if initialisation is done already + if [ cqlsh $dse_ip 9042 -u $admin_user -p $admin_password $cql_options -e "DESCRIBE KEYSPACE kv_init_done;" 2>&1 | grep -q 'CREATE KEYSPACE kv_init_done' ]; then + echo "The database is already initialised, exiting..." + exit 0 + fi + echo "=> Creating new superuser $KILLRVIDEO_ADMIN_USERNAME" cqlsh $dse_ip 9042 -u $admin_user -p $admin_password $cql_options -e "CREATE ROLE $KILLRVIDEO_ADMIN_USERNAME with SUPERUSER = true and LOGIN = true and PASSWORD = '$KILLRVIDEO_ADMIN_PASSWORD'" # Login as new superuser to delete default superuser (cassandra) @@ -80,6 +80,15 @@ if [ ! -z "$KILLRVIDEO_ADMIN_USERNAME" ]; then admin_password=$KILLRVIDEO_ADMIN_PASSWORD fi +if cqlsh $dse_ip 9042 -u $admin_user -p $admin_password $cql_options -e "DESCRIBE KEYSPACE kv_init_done;" 2>&1 | grep -q 'CREATE KEYSPACE kv_init_done'; then + if [ ! -z "$KILLRVIDEO_FORCE_BOOTSTRAP" ]; then + echo '=> Forced bootstrap!' + else + echo "The database is already initialised, exiting..." + exit 0 + fi +fi + # If requested, create a new standard user if [ "$KILLRVIDEO_CREATE_DSE_USER" = 'true' ]; then # Create user and grant permission to create keyspaces (generator and web will need) @@ -142,23 +151,7 @@ if [ ! -z "$KILLRVIDEO_GRAPH_REPLICATION" ]; then fi dse gremlin-console -e $graph_file -# Register services in ETCD once all the schema are configured, using an IP that will be accessible -# internally and externally to the Docker environment -hostname="$(hostname)" - -if [ ! -z "$KILLRVIDEO_SERVICE_DISCOVERY_DISABLED" ]; then - echo '=> Skipping registration in ETCD as service discovery is disabled' -else - echo '=> Registering DSE DB (Cassandra) cluster in ETCD' - curl "http://etcd:2379/v2/keys/killrvideo/services/cassandra/$hostname" -XPUT -d value="$dse_external_ip:9042" - - echo '=> Registering DSE Search in ETCD' - curl "http://etcd:2379/v2/keys/killrvideo/services/dse-search/$hostname" -XPUT -d value="$dse_external_ip:8983" - - echo '=> Registering DSE Graph in ETCD' - curl "http://etcd:2379/v2/keys/killrvideo/services/gremlin/$hostname" -XPUT -d value="$dse_external_ip:8182" -fi +echo '=> Configuration of DSE users and schema complete' # Don't bootstrap next time we start -echo '=> Configuration of DSE users and schema complete' -touch killrvideo_bootstrapped +cqlsh $dse_ip 9042 -u $dse_user -p $dse_password $cql_options -e "CREATE KEYSPACE IF NOT EXISTS kv_init_done WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1};"