Enabled Authentication, authorization, and HTTPS enforcement (#19)

* Enabled Authentication, authorization, and HTTPS enforcement

* Adressed feedback

ci/release_notes.md

@@ -1,3 +1,17 @@
+# New Features
+
+- **Authentication and Authorization!**
+  SHIELD now supports options for authenticating requests to it!
+  It supports HTTP Basic authentication, OAuth2 (currently the
+  only supported provider is github), and API Keys. If no authentication
+  configuration is provided, SHIELD will default to HTTP Basic Auth,
+  using a default user/password.
+- **SSL Required**
+  SHIELD now runs behind an nginx instance doing SSL termination.
+  If you do not specify a key, one will be auto-generated for you,
+  making an easier transition. Additionally, non-encrypted requests
+  will be redirected to https for you.
+
 # Bug Fixes
 
 - Remove console.log calls from frontend Web UI Javascript,

jobs/nginx/monit

@@ -0,0 +1,5 @@
+check process nginx
+  with pidfile /var/vcap/sys/run/nginx/nginx.pid
+  start program "/var/vcap/jobs/nginx/bin/nginx_ctl start"
+  stop program "/var/vcap/jobs/nginx/bin/nginx_ctl stop"
+  group vcap

jobs/nginx/spec

@@ -0,0 +1,47 @@
+---
+name: nginx
+templates:
+  mime.types: config/mime.types
+  nginx.conf.erb: config/nginx.conf
+  nginx_ctl: bin/nginx_ctl
+  ssl_key.erb: config/ssl_key
+  ssl_crt.erb: config/ssl_crt
+
+packages:
+- nginx
+
+properties:
+  shield.daemon.port:
+    default: 443
+    description: "Port to listen on for encrypted traffic"
+  shield.daemon.http_port:
+    default: 80
+    description: "Port non-encrypted traffic should listen on. Redirects to https_port"
+  shield.daemon.domain:
+    description: "Hostname/IP SHIELD is accessed with"
+
+  nginx.worker_processes:
+    description: 'Number of nginx workers'
+    default: 2
+  nginx.worker_connections:
+    description: 'Number of nginx connections per worker'
+    default: 8192
+  nginx.keepalive_timeout:
+    description: 'Timeout for keep-alive connections'
+    default: '75 20'
+  nginx.proxy_connect_timeout:
+    description: 'Timeout for the connection to the upstream server'
+    default: 300
+  nginx.proxy_read_timeout:
+    description: 'Timeout for read queries to the upstream server'
+    default: 120
+  nginx.proxy_send_timeout:
+    description: 'Timeout for send queries to the upstream server'
+    default: 120
+  nginx.ssl_timeout:
+    description: 'Timeout for reusing the previously negotiated cryptographic parameters'
+    default: 7200
+  nginx.ssl_key:
+    description: 'SSL private key (PEM encoded)'
+  nginx.ssl_cert:
+    description: 'SSL Certificate (PEM encoded)'

jobs/nginx/templates/mime.types

@@ -0,0 +1,73 @@
+types {
+    text/html                             html htm shtml;
+    text/css                              css;
+    text/xml                              xml;
+    image/gif                             gif;
+    image/jpeg                            jpeg jpg;
+    application/x-javascript              js;
+    application/atom+xml                  atom;
+    application/rss+xml                   rss;
+
+    text/mathml                           mml;
+    text/plain                            txt;
+    text/vnd.sun.j2me.app-descriptor      jad;
+    text/vnd.wap.wml                      wml;
+    text/x-component                      htc;
+
+    image/png                             png;
+    image/tiff                            tif tiff;
+    image/vnd.wap.wbmp                    wbmp;
+    image/x-icon                          ico;
+    image/x-jng                           jng;
+    image/x-ms-bmp                        bmp;
+    image/svg+xml                         svg;
+
+    application/java-archive              jar war ear;
+    application/mac-binhex40              hqx;
+    application/msword                    doc;
+    application/pdf                       pdf;
+    application/postscript                ps eps ai;
+    application/rtf                       rtf;
+    application/vnd.ms-excel              xls;
+    application/vnd.ms-powerpoint         ppt;
+    application/vnd.wap.wmlc              wmlc;
+    application/vnd.wap.xhtml+xml         xhtml;
+    application/vnd.google-earth.kml+xml  kml;
+    application/vnd.google-earth.kmz      kmz;
+    application/x-7z-compressed           7z;
+    application/x-cocoa                   cco;
+    application/x-java-archive-diff       jardiff;
+    application/x-java-jnlp-file          jnlp;
+    application/x-makeself                run;
+    application/x-perl                    pl pm;
+    application/x-pilot                   prc pdb;
+    application/x-rar-compressed          rar;
+    application/x-redhat-package-manager  rpm;
+    application/x-sea                     sea;
+    application/x-shockwave-flash         swf;
+    application/x-stuffit                 sit;
+    application/x-tcl                     tcl tk;
+    application/x-x509-ca-cert            der pem crt;
+    application/x-xpinstall               xpi;
+    application/zip                       zip;
+
+    application/octet-stream              bin exe dll;
+    application/octet-stream              deb;
+    application/octet-stream              dmg;
+    application/octet-stream              eot;
+    application/octet-stream              iso img;
+    application/octet-stream              msi msp msm;
+
+    audio/midi                            mid midi kar;
+    audio/mpeg                            mp3;
+    audio/x-realaudio                     ra;
+
+    video/3gpp                            3gpp 3gp;
+    video/mpeg                            mpeg mpg;
+    video/quicktime                       mov;
+    video/x-flv                           flv;
+    video/x-mng                           mng;
+    video/x-ms-asf                        asx asf;
+    video/x-ms-wmv                        wmv;
+    video/x-msvideo                       avi;
+}

jobs/nginx/templates/nginx.conf.erb

@@ -0,0 +1,68 @@
+user vcap;
+worker_processes     <%= p('nginx.worker_processes') %>;
+daemon off;
+
+error_log            /var/vcap/sys/log/nginx/error.log;
+pid                  /var/vcap/sys/run/nginx/nginx.pid;
+
+events {
+  use epoll;
+  worker_connections <%= p('nginx.worker_connections') %>;
+}
+
+http {
+  include            /var/vcap/jobs/nginx/config/mime.types;
+  default_type       text/html;
+  server_tokens      off;
+
+  access_log         /var/vcap/sys/log/nginx/access.log;
+
+  sendfile           on;
+  tcp_nopush         on;
+  tcp_nodelay        on;
+
+  underscores_in_headers on;
+
+  keepalive_timeout  <%= p('nginx.keepalive_timeout') %>;
+
+  ssl_session_cache shared:SSL:10m;
+
+  gzip               on;
+  gzip_min_length    1250;
+  gzip_buffers       16 8k;
+  gzip_comp_level    2;
+  gzip_proxied       any;
+  gzip_types         application/json application/xml application/xhtml+xml application/javascript application/atom+xml application/rss+xml application/x-font-ttf application/x-javascript application/xml+rss image/svg+xml text/css text/javascript text/plain text/xml;
+  gzip_vary          on;
+  gzip_disable       "MSIE [1-6]\.(?!.*SV1)";
+
+  upstream shield {
+    server localhost:8080;
+  }
+
+  server {
+    listen <%= p('shield.daemon.http_port') %>;
+    server_name  _;
+    ssl off;
+    return 301 https://<%= p('shield.daemon.domain') %>:<%= p('shield.daemon.port') %>$request_uri;
+  }
+
+  server {
+    listen <%= p('shield.daemon.port') %>;
+    server_name  _;
+    server_name_in_redirect off;
+
+    ssl on;
+    ssl_prefer_server_ciphers on;
+    ssl_protocols             TLSv1 TLSv1.1 TLSv1.2;
+    ssl_ciphers               ECDHE-RSA-AES128-SHA256:AES128-GCM-SHA256:HIGH:!MD5:!aNULL:!EDH;
+    ssl_certificate           /var/vcap/jobs/nginx/config/ssl/ssl_crt;
+    ssl_certificate_key       /var/vcap/jobs/nginx/config/ssl/ssl_key;
+    ssl_session_timeout       <%= p('nginx.ssl_timeout') %>;
+    add_header                Strict-Transport-Security max-age=15768000;
+
+    location / {
+      proxy_pass http://shield;
+    }
+  }
+}

jobs/nginx/templates/nginx_ctl

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+RUN_DIR=/var/vcap/sys/run/nginx
+LOG_DIR=/var/vcap/sys/log/nginx
+JOB_DIR=/var/vcap/jobs/nginx
+CONF_DIR=$JOB_DIR/config
+CERTS_DIR=$CONF_DIR/ssl
+PIDFILE=$RUN_DIR/nginx.pid
+
+case $1 in
+
+  start)
+    mkdir -p $RUN_DIR $LOG_DIR $CERTS_DIR
+
+    echo $$ > $PIDFILE
+
+    # if the properties contain sslproxy.ssl.key & sslproxy.ssl.cert
+    # the two files below will contain key & cert, then copy them to
+    # the certificates dir
+    if [[ -n $(cat $CONF_DIR/ssl_key) && -n $(cat $CONF_DIR/ssl_crt) ]]; then
+      cp $CONF_DIR/ssl_key $CERTS_DIR
+      cp $CONF_DIR/ssl_crt $CERTS_DIR
+    # otherwise, if the key or cert doesn't exist create new ones
+    elif [[ ! -f $CERTS_DIR/ssl_key || ! -f $CERTS_DIR/ssl_crt ]]; then
+      openssl req -nodes -new -newkey rsa:2048 -out $CERTS_DIR/ssl.csr \
+        -keyout $CERTS_DIR/ssl_key -subj '/O=Bosh/CN=*'
+      openssl x509 -req -days 3650 -in $CERTS_DIR/ssl.csr \
+        -signkey $CERTS_DIR/ssl_key -out $CERTS_DIR/ssl_crt
+    fi
+
+    exec /var/vcap/packages/nginx/sbin/nginx -c $JOB_DIR/config/nginx.conf \
+      >>$LOG_DIR/nginx.stdout.log 2>>$LOG_DIR/nginx.stderr.log
+    ;;
+
+  stop)
+    PID=$(head -1 $PIDFILE)
+    kill $PID
+    while [ -e /proc/$PID ]; do sleep 0.1; done
+    rm -f $PIDFILE
+    ;;
+
+  *)
+    echo "Usage: nginx_ctl {start|stop}" ;;
+esac
+exit 0

jobs/nginx/templates/ssl_crt.erb

@@ -0,0 +1 @@
+<% if_p('nginx.ssl_crt') do |cert| %><%= cert %><% end %>

jobs/nginx/templates/ssl_key.erb

@@ -0,0 +1 @@
+<% if_p('nginx.ssl_key') do |key| %><%= key %><% end %>

jobs/shield-agent/spec

@@ -43,6 +43,11 @@ properties:
   shield.log_level:
     description: "Log level for shield processes"
     default: "info"
+  shield.provisioning_key:
+    description: "API Key to use when provisioning jobs, or targets. Required if shield.target or shield.job have data"
+  shield.skip_ssl_verify:
+    description: "Boolean to determine if SSL certs will be ignored when provisioning SHIELD data"
+    default: true
 
   shield.target.name:
     description: "Target name"

jobs/shield-agent/templates/bin/post-start.erb

@@ -9,21 +9,35 @@ done
 <% if_p("shield.agent.autoprovision") do |server| %>
 <% if_p("shield.target.name") do |target_name|%>
 
-export SHIELD_API=<%= server %>
+<%#
+  The SHIELD_API_TOKEN is set here so that the post-start script will fail to render
+  when provisioning_key is not specified, but the script is about to try to create targets
+  or jobs
+%>
+export SHIELD_API_TOKEN=<%= p("shield.provisioning_key") %>
 
-TARGET=$(shield --raw show target <%= target_name %> | jq -r '.uuid // empty')
+
+<% if p("shield.skip_ssl_verify") %>
+export SHIELD_SKIP_SSL_VERIFY=true
+<% end %>
+
+<%# Create a specific  config file for shield-agent stuff, to avoid race conditions
+    in case we are colocated with shield-daemon. %>
+shield -c /tmp/.shield_agent_config create backend default https://<%= p("shield.daemon.domain") %>:<%= p("shield.daemon.port") %>
+
+TARGET=$(shield -c /tmp/.shield_agent_config --raw show target <%= target_name %> | jq -r '.uuid // empty')
 if [[ -z ${TARGET} ]]; then
   echo "Creating target"
-	cat /var/vcap/jobs/shield-agent/config/target.json | shield --raw create target
+	cat /var/vcap/jobs/shield-agent/config/target.json | shield -c /tmp/.shield_agent_config --raw create target
 else
   echo "Editing target"
-	cat /var/vcap/jobs/shield-daemon/config/target.json | shield --raw edit target
+	cat /var/vcap/jobs/shield-agent/config/target.json | shield -c /tmp/.shield_agent_config --raw edit target
 fi
 <% if_p("shield.job.name", "shield.job.store", "shield.job.retention", "shield.job.schedule") do |job_name, retention, schedule, store| %>
 
-STORE=$(shield --raw show store <%= store %> | jq -r '.uuid // empty')
-RETENTION=$(shield --raw show retention policy <%= retention %> | jq -r '.uuid // empty')
-SCHEDULE=$(shield --raw show schedule <%= schedule %> | jq -r '.uuid // empty')
+STORE=$(shield -c /tmp/.shield_agent_config --raw show store <%= store %> | jq -r '.uuid // empty')
+RETENTION=$(shield -c /tmp/.shield_agent_config --raw show retention policy <%= retention %> | jq -r '.uuid // empty')
+SCHEDULE=$(shield -c /tmp/.shield_agent_config --raw show schedule <%= schedule %> | jq -r '.uuid // empty')
 
 cat <<EOF > /tmp/job.json
 {"name":      "<%= job_name %>",
@@ -35,13 +49,13 @@ cat <<EOF > /tmp/job.json
 }
 EOF
 
-JOB=$(shield --raw show job <%= job_name %> | jq -r '.uuid // empty')
+JOB=$(shield -c /tmp/.shield_agent_config --raw show job <%= job_name %> | jq -r '.uuid // empty')
 if [[ -z ${JOB} ]]; then
   echo "Creating job"
-	cat /tmp/job.json | shield --raw create job
+	cat /tmp/job.json | shield -c /tmp/.shield_agent_config --raw create job
 else
   echo "Editing job"
-  cat /tmp/job.json | shield --raw edit job
+  cat /tmp/job.json | shield -c /tmp/.shield_agent_config --raw edit job
 fi
 
 # End Job