Merge pull request #71 from jacquelinegarrahan/hard-coded-configs

Add hard coded config files.

cli/commands/add-grafana-datasource

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#
+# Author: Jacqueline Garrahan
+# Create datasource directory for a Kafka configuration
+#
+usage() {
+  cli_name=${0##*/}
+  echo "Usage: nalms add-grafana-datasource config_names"
+  echo "Accepts comma separated list of config names."
+  echo "Requires setting of following environment variables: \$NALMS_GRAFANA_DATASOURCE_FILE,"
+  echo "\$NALMS_ES_HOST, \$NALMS_ES_PORT"
+  exit 0
+}
+
+if [[ -z "${NALMS_GRAFANA_DATASOURCE_FILE}" ]]; then
+  echo "\$NALMS_GRAFANA_DATASOURCE_FILE is not set."
+  exit 0
+fi
+
+if [[ -z "${NALMS_ES_HOST}" ]]; then
+  echo "\$NALMS_ES_HOST is not set."
+  exit 0
+fi
+
+if [[ -z "${NALMS_ES_PORT}" ]]; then
+  echo "\$NALMS_ES_PORT is not set."
+  exit 0
+fi
+
+if [[ -z "$1" ]]; then
+  usage
+else
+  CONFIG_NAME="$1"
+fi
+
+export IFS=","
+
+# convert to lowercase for index
+CONFIG=$(echo "$CONFIG_NAME" | tr '[:upper:]' '[:lower:]')
+
+for word in $CONFIG_NAME; do
+    # convert to lowercase for index
+    CONFIG=$(echo "$word" | tr '[:upper:]' '[:lower:]')
+    echo "" >> $NALMS_GRAFANA_DATASOURCE_FILE
+    echo "  - name: ${word}" >> $NALMS_GRAFANA_DATASOURCE_FILE
+    echo "    type: elasticsearch" >> $NALMS_GRAFANA_DATASOURCE_FILE
+    echo "    access: proxy" >> $NALMS_GRAFANA_DATASOURCE_FILE
+    echo "    database: \"${CONFIG}_*\"" >> $NALMS_GRAFANA_DATASOURCE_FILE
+    echo "    url: http://${NALMS_ES_HOST}:${NALMS_ES_PORT}" >> $NALMS_GRAFANA_DATASOURCE_FILE
+    echo "    jsonData:" >> $NALMS_GRAFANA_DATASOURCE_FILE
+    echo "      esVersion: 60" >> $NALMS_GRAFANA_DATASOURCE_FILE
+    echo "      timeField: \"message_time\"" >> $NALMS_GRAFANA_DATASOURCE_FILE
+    echo "      logMessageField: \"id\"" >> $NALMS_GRAFANA_DATASOURCE_FILE
+done
+
+echo "Completed writing file."

cli/commands/build-grafana-dashboard

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Author: Jacqueline Garrahan
+# Create dashboard for a Kafka configuration
+#
+usage() {
+  cli_name=${0##*/}
+  echo "Usage: nalms build-grafana-dashboard config_name"
+  echo "Accepts comma separated list of config names."
+  echo "Requires setting of following environment variables: \$NALMS_GRAFANA_DASHBOARD_DIR, \$NALMS_HOME"
+  exit 0
+}
+
+if [[ -z "${NALMS_GRAFANA_DATASOURCE_DIR}" ]]; then
+  echo "\$NALMS_GRAFANA_DATASOURCE_DIR is not set."
+  exit 0
+fi
+
+if [[ -z "${NALMS_HOME}" ]]; then
+  echo "\$NALMS_HOME is not set."
+  exit 0
+fi
+
+if [[ -z "$1" ]]; then
+  usage
+else
+  CONFIG_NAME="$1"
+fi
+
+DASHBOARD_TEMPLATE="${NALMS_HOME}/grafana/dashboards/alarm_logs_dashboard.json"
+
+# create dashboards
+export IFS=","
+for CONFIG_ITEM in $CONFIG_NAME; do
+    sed 's/$DATASOURCE_NAME/'"$CONFIG_ITEM/" $DASHBOARD_TEMPLATE > "${NALMS_GRAFANA_DASHBOARD_DIR}/${CONFIG_ITEM}.json"
+done

cli/commands/start-alarm-logger

@@ -6,7 +6,8 @@
 usage() {
   cli_name=${0##*/}
   echo "Usage: nalms start-alarm-logger config_name config_file "
-  echo "Requires definition of \$NALMS_ES_HOST, \$NALMS_ES_PORT, \$NALMS_BOOTSTRAP_SERVERS"
+  echo "Requires definition of \$ALARM_LOGGER_PROPERTIES and optional definition of"
+  echo "\$NALMS_ES_HOST, \$NALMS_ES_PORT, \$NALMS_BOOTSTRAP_SERVERS"
   exit 0
 }
 
@@ -27,19 +28,8 @@ else
 fi
 
 
-
-if [[ -z "$NALMS_ES_HOST" ]]; then
-  echo "Elasticsearch host not provided."
-  usage
-fi
-
-if [[ -z "$NALMS_ES_PORT" ]]; then
-  echo "Elasticsearch port not provided."
-  usage
-fi
-
-if [[ -z "$NALMS_KAFKA_BOOTSTRAP" ]]; then
-  echo "Bootstrap servers not provided."
+if [[ -z "$NALMS_ALARM_LOGGER_PROPERTIES" ]]; then
+  echo "Property file not provided."
   usage
 fi
 
@@ -50,8 +40,10 @@ if [ "$1" == "-h" ]; then
 fi
 
 docker run -v $CONFIG_FILE:/tmp/nalms/$CONFIG_NAME.xml \
-  -e ES_HOST=$NALMS_ES_HOST \
-  -e ES_PORT=$NALMS_ES_PORT \
-  -e BOOTSTRAP_SERVERS=$NALMS_KAFKA_BOOTSTRAP \
+  -e ES_HOST="${NALMS_ES_HOST}" \
+  -e ES_PORT="${NALMS_ES_PORT}" \
+  -e BOOTSTRAP_SERVERS="${NALMS_KAFKA_BOOTSTRAP}" \
+  -e ALARM_LOGGER_PROPERTIES="/opt/nalms/config/alarm_logger.properties" \
+  -v "${NALMS_ALARM_LOGGER_PROPERTIES}:/opt/nalms/config/alarm_logger.properties" \
   --name nalms_logger_$CONFIG_NAME \
   -d jgarrahan/nalms-phoebus-alarm-logger:latest start-logger $CONFIG_NAME /tmp/nalms/$CONFIG_NAME.xml

cli/commands/start-alarm-server

@@ -30,8 +30,14 @@ if [[ -z "$NALMS_KAFKA_BOOTSTRAP" ]]; then
   usage
 fi
 
+if [[ -z "$NALMS_ALARM_SERVER_PROPERTIES" ]]; then
+  echo "No Kafka bootstrap provided."
+  usage
+fi
+
 docker run -v $CONFIG_FILE:/tmp/nalms/$CONFIG_NAME.xml \
   --name nalms_server_$CONFIG_NAME \
+  -v "${NALMS_ALARM_SERVER_PROPERTIES}:/opt/nalms/config/alarm_server.properties" \
   -e ALARM_IOC=false \
   -e KAFKA_BOOTSTRAP="${NALMS_KAFKA_BOOTSTRAP}" \
   -e EPICS_CA_ADDR_LIST="${EPICS_CA_ADDR_LIST}" \
@@ -40,4 +46,5 @@ docker run -v $CONFIG_FILE:/tmp/nalms/$CONFIG_NAME.xml \
   -e EPICS_PVA_ADDR_LIST="${EPICS_PVA_ADDR_LIST}" \
   -e EPICS_PVA_SERVER_PORT="${EPICS_PVA_SERVER_PORT}" \
   -e EPICS_PVA_REPEATER_PORT="${EPICS_PVA_REPEATER_PORT}" \
+  -e ALARM_SERVER_PROPERTIES="/opt/nalms/config/alarm_server.properties" \
   -d -t jgarrahan/nalms-phoebus-alarm-server:latest start-server $CONFIG_NAME /tmp/nalms/$CONFIG_NAME.xml

cli/commands/start-cruise-control

@@ -6,7 +6,8 @@
 usage() {
   cli_name=${0##*/}
   echo "Usage: nalms start-cruise-control"
-  echo "Requires definition of \$NALMS_BOOTSTRAP_SERVERS, \$NALMS_ZOOKEEPER_HOST, \$NALMS_ZOOKEEPER_PORT \$NALMS_CRUISE_CONTROL_PORT"
+  echo "Requires definition of \$NALMS_CRUISE_CONTROL_PORT, \$NALMS_CRUISE_CONTROL_PROPERTIES"
+  echo "Optionally requires \$NALMS_BOOTSTRAP_SERVERS, \$NALMS_ZOOKEEPER_HOST if not defined in file and using environment substitution"
   exit 0
 }
 
@@ -15,30 +16,22 @@ if [[ "$1" == "-h" ]]; then
 fi
 
 
-if [[ -z "$NALMS_KAFKA_BOOTSTRAP" ]]; then
-  echo "No bootstrap servers defined for cruise control."
-  usage
-fi
-
-if [[ -z "$NALMS_ZOOKEEPER_HOST" ]]; then
-  echo "No zookeeper host defined for cruise control."
-  usage
-fi
-
-if [[ -z "$NALMS_ZOOKEEPER_PORT" ]]; then
-  echo "No zookeeper host defined for cruise control."
+if [[ -z "$NALMS_CRUISE_CONTROL_PORT" ]]; then
+  echo "No cruise control port defined ."
   usage
 fi
 
 
-if [[ -z "$NALMS_CRUISE_CONTROL_PORT" ]]; then
-  echo "No cruise control port defined ."
+if [[ -z "$NALMS_CRUISE_CONTROL_PROPERTIES" ]]; then
+  echo "No cruise control properties file defined ."
   usage
 fi
 
 
 docker run \
-    -e BOOTSTRAP_SERVERS=$NALMS_KAFKA_BOOTSTRAP \
-    -e ZOOKEEPER_CONNECT=$NALMS_ZOOKEEPER_HOST:$NALMS_ZOOKEEPER_PORT \
+    -e BOOTSTRAP_SERVERS="${NALMS_KAFKA_BOOTSTRAP}" \
+    -e ZOOKEEPER_CONNECT="${NALMS_ZOOKEEPER_HOST}:${NALMS_ZOOKEEPER_PORT}" \
+    -e CRUISE_CONTROL_PROPERTIES="/opt/cruise-control/config/cruisecontrol.properties" \
+    -v "${NALMS_CRUISE_CONTROL_PROPERTIES}:/opt/cruise-control/config/cruisecontrol.properties" \
     --name nalms_cruise_control \
     -p "$NALMS_CRUISE_CONTROL_PORT:9090" -d jgarrahan/nalms-cruise-control:latest

cli/commands/start-elasticsearch

@@ -7,6 +7,7 @@ usage() {
   cli_name=${0##*/}
   echo "Usage: nalms start-elasticsearch [--port elasticsearch]"
   echo "Port required, but may be set using the \$NALMS_ES_PORT environment variable."
+  echo "\$NALMS_ES_CONFIG must be set."
   exit 0
 }
 
@@ -27,6 +28,11 @@ if [[ -z "$NALMS_ES_PORT" ]]; then
   usage
 fi
 
+if [[ -z "$NALMS_ES_CONFIG" ]]; then
+  echo "No configuration file provided for Elasticsearch. Define \$NALMS_ES_CONFIG"
+  usage
+fi
+
 
 docker run \
     -e node.name=node01 \
@@ -35,6 +41,7 @@ docker run \
     -e ES_JAVA_OPTS="-Xms128m -Xmx128m" \
     -e ES_HOST=localhost \
     -e ES_PORT=9200 \
+    -v "${NALMS_ES_CONFIG}:/usr/share/elasticsearch/config" \
     -p "$NALMS_ES_PORT:9200" \
     --name nalms_elasticsearch \
     -d jgarrahan/nalms-elasticsearch:latest

cli/commands/start-grafana

@@ -6,7 +6,7 @@
 usage() {
   cli_name=${0##*/}
   echo "Usage: nalms start-grafana [--port grafana_port] [--config config_name]"
-  echo "Requires definition of \$NALMS_GRAFANA_PORT, \$NALMS_CONFIGURATIONS, \$NALMS_ES_HOST, \$NALMS_ES_PORT"
+  echo "Requires definition of \$NALMS_GRAFANA_PORT, \$NALMS_GRAFANA_DASHBOARD_DIR, \$NALMS_GRAFANA_DATASOURCE_FILE, \$NALMS_GRAFANA_CONFIG"
   exit 0
 }
 
@@ -17,9 +17,6 @@ do
         --port) 
           NALMS_GRAFANA_PORT=$2
             ;;
-        --config) 
-          NALMS_CONFIGURATIONS=$2
-            ;;
         "")
     esac
     shift
@@ -29,28 +26,26 @@ if [[ -z "$NALMS_GRAFANA_PORT" ]]; then
   echo "No port provided for Grafana."
   usage
 fi
-
-if [[ -z "$NALMS_CONFIGURATIONS" ]]; then
-  echo "No configurations provided to launch Grafana."
+if [[ -z "$NALMS_GRAFANA_DASHBOARD_DIR" ]]; then
+  echo "NALMS_GRAFANA_DASHBOARD_DIR not defined."
   usage
 fi
 
-if [[ -z "$NALMS_ES_HOST" ]]; then
-  echo "Elasticsearch host not defined."
+if [[ -z "$NALMS_GRAFANA_DATASOURCE_FILE" ]]; then
+  echo "NALMS_GRAFANA_DATASOURCE_FILE not defined."
   usage
 fi
 
-
-if [[ -z "$NALMS_ES_PORT" ]]; then
-  echo "Elasticsearch port not defined."
+if [[ -z "$NALMS_GRAFANA_CONFIG" ]]; then
+  echo "NALMS_GRAFANA_CONFIG not defined."
   usage
 fi
 
 docker run \
-    -p "$NALMS_GRAFANA_PORT:3000" \
-    -e ES_HOST=$NALMS_ES_HOST \
-    -e ES_PORT=$NALMS_ES_PORT \
-    -e CONFIG_NAME=$NALMS_CONFIGURATIONS \
+    -p "${NALMS_GRAFANA_PORT}:3000" \
+    -v "${NALMS_GRAFANA_DASHBOARD_DIR}:/var/lib/grafana/dashboards" \
+    -v "${NALMS_GRAFANA_DATASOURCE_FILE}:/etc/grafana/provisioning/datasources/all.yml" \
+    -v "${NALMS_GRAFANA_CONFIG}:/etc/grafana/config.ini" \
     --name nalms_grafana \
     -d jgarrahan/nalms-grafana:latest
 

cli/commands/start-zookeeper

@@ -7,6 +7,7 @@ usage() {
   cli_name=${0##*/}
   echo "Usage: nalms start-zookeeper [--port zookeeper_port]"
   echo "Port required, but may be set using the \$NALMS_ZOOKEEPER_PORT environment variable."
+  echo "Must set \$ NALMS_ZOOKEEPER_CONFIG"
   exit 0
 }
 
@@ -28,5 +29,11 @@ if [[ -z "$NALMS_ZOOKEEPER_PORT" ]]; then
   usage
 fi
 
+if [[ -z "$NALMS_ZOOKEEPER_CONFIG" ]]; then
+  echo "No config provided for Zookeeper. Please set \$NALMS_ZOOKEEPER_CONFIG"
+  usage
+fi
 
-docker run -p "$NALMS_ZOOKEEPER_PORT:2181" --name nalms_zookeeper -d jgarrahan/nalms-zookeeper:latest
+docker run -p "${NALMS_ZOOKEEPER_PORT}:2181" -e ZOOKEEPER_CONFIG=/tmp/zoo.cfg \
+  -v "${NALMS_ZOOKEEPER_CONFIG}:/tmp/zoo.cfg" --name nalms_zookeeper \
+  -d jgarrahan/nalms-zookeeper:latest

cli/nalms

@@ -22,6 +22,8 @@ Commands:
   create-alarm-ioc configuration_file config_name output_directory
   convert-alh alh_file output_filename config_name
   delete-configurations configuration_name
+  add-grafana-datasource config_names
+  build-grafana-dashboard config_name
   list-configurations
   launch-editor
   generate-kafka-certs domain password
@@ -71,6 +73,12 @@ case "$1" in
   delete-configuration|d)
     . "$CLI_WORKDIR/commands/delete-configuration" "$2"
     ;;
+  add-grafana-datasource|d)
+    . "$CLI_WORKDIR/commands/add-grafana-datasource" "$2"
+    ;;
+  build-grafana-dashboard|d)
+    . "$CLI_WORKDIR/commands/build-grafana-dashboard" "$2"
+    ;;
   launch-editor|d)
     . "$CLI_WORKDIR/commands/launch-editor"
     ;;