diff --git a/cli/commands/add-grafana-datasource b/cli/commands/add-grafana-datasource new file mode 100644 index 0000000..06199dd --- /dev/null +++ b/cli/commands/add-grafana-datasource @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# +# Author: Jacqueline Garrahan +# Create datasource directory for a Kafka configuration +# +usage() { + cli_name=${0##*/} + echo "Usage: nalms add-grafana-datasource config_names" + echo "Accepts comma separated list of config names." + echo "Requires setting of following environment variables: \$NALMS_GRAFANA_DATASOURCE_FILE," + echo "\$NALMS_ES_HOST, \$NALMS_ES_PORT" + exit 0 +} + +if [[ -z "${NALMS_GRAFANA_DATASOURCE_FILE}" ]]; then + echo "\$NALMS_GRAFANA_DATASOURCE_FILE is not set." + exit 0 +fi + +if [[ -z "${NALMS_ES_HOST}" ]]; then + echo "\$NALMS_ES_HOST is not set." + exit 0 +fi + +if [[ -z "${NALMS_ES_PORT}" ]]; then + echo "\$NALMS_ES_PORT is not set." + exit 0 +fi + +if [[ -z "$1" ]]; then + usage +else + CONFIG_NAME="$1" +fi + +export IFS="," + +# convert to lowercase for index +CONFIG=$(echo "$CONFIG_NAME" | tr '[:upper:]' '[:lower:]') + +for word in $CONFIG_NAME; do + # convert to lowercase for index + CONFIG=$(echo "$word" | tr '[:upper:]' '[:lower:]') + echo "" >> $NALMS_GRAFANA_DATASOURCE_FILE + echo " - name: ${word}" >> $NALMS_GRAFANA_DATASOURCE_FILE + echo " type: elasticsearch" >> $NALMS_GRAFANA_DATASOURCE_FILE + echo " access: proxy" >> $NALMS_GRAFANA_DATASOURCE_FILE + echo " database: \"${CONFIG}_*\"" >> $NALMS_GRAFANA_DATASOURCE_FILE + echo " url: http://${NALMS_ES_HOST}:${NALMS_ES_PORT}" >> $NALMS_GRAFANA_DATASOURCE_FILE + echo " jsonData:" >> $NALMS_GRAFANA_DATASOURCE_FILE + echo " esVersion: 60" >> $NALMS_GRAFANA_DATASOURCE_FILE + echo " timeField: \"message_time\"" >> $NALMS_GRAFANA_DATASOURCE_FILE + echo " logMessageField: \"id\"" >> $NALMS_GRAFANA_DATASOURCE_FILE +done + +echo "Completed writing file." \ No newline at end of file diff --git a/cli/commands/build-grafana-dashboard b/cli/commands/build-grafana-dashboard new file mode 100644 index 0000000..2f72084 --- /dev/null +++ b/cli/commands/build-grafana-dashboard @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Author: Jacqueline Garrahan +# Create dashboard for a Kafka configuration +# +usage() { + cli_name=${0##*/} + echo "Usage: nalms build-grafana-dashboard config_name" + echo "Accepts comma separated list of config names." + echo "Requires setting of following environment variables: \$NALMS_GRAFANA_DASHBOARD_DIR, \$NALMS_HOME" + exit 0 +} + +if [[ -z "${NALMS_GRAFANA_DATASOURCE_DIR}" ]]; then + echo "\$NALMS_GRAFANA_DATASOURCE_DIR is not set." + exit 0 +fi + +if [[ -z "${NALMS_HOME}" ]]; then + echo "\$NALMS_HOME is not set." + exit 0 +fi + +if [[ -z "$1" ]]; then + usage +else + CONFIG_NAME="$1" +fi + +DASHBOARD_TEMPLATE="${NALMS_HOME}/grafana/dashboards/alarm_logs_dashboard.json" + +# create dashboards +export IFS="," +for CONFIG_ITEM in $CONFIG_NAME; do + sed 's/$DATASOURCE_NAME/'"$CONFIG_ITEM/" $DASHBOARD_TEMPLATE > "${NALMS_GRAFANA_DASHBOARD_DIR}/${CONFIG_ITEM}.json" +done diff --git a/cli/commands/start-alarm-logger b/cli/commands/start-alarm-logger index 121d916..9fc77a8 100644 --- a/cli/commands/start-alarm-logger +++ b/cli/commands/start-alarm-logger @@ -6,7 +6,8 @@ usage() { cli_name=${0##*/} echo "Usage: nalms start-alarm-logger config_name config_file " - echo "Requires definition of \$NALMS_ES_HOST, \$NALMS_ES_PORT, \$NALMS_BOOTSTRAP_SERVERS" + echo "Requires definition of \$ALARM_LOGGER_PROPERTIES and optional definition of" + echo "\$NALMS_ES_HOST, \$NALMS_ES_PORT, \$NALMS_BOOTSTRAP_SERVERS" exit 0 } @@ -27,19 +28,8 @@ else fi - -if [[ -z "$NALMS_ES_HOST" ]]; then - echo "Elasticsearch host not provided." - usage -fi - -if [[ -z "$NALMS_ES_PORT" ]]; then - echo "Elasticsearch port not provided." - usage -fi - -if [[ -z "$NALMS_KAFKA_BOOTSTRAP" ]]; then - echo "Bootstrap servers not provided." +if [[ -z "$NALMS_ALARM_LOGGER_PROPERTIES" ]]; then + echo "Property file not provided." usage fi @@ -50,8 +40,10 @@ if [ "$1" == "-h" ]; then fi docker run -v $CONFIG_FILE:/tmp/nalms/$CONFIG_NAME.xml \ - -e ES_HOST=$NALMS_ES_HOST \ - -e ES_PORT=$NALMS_ES_PORT \ - -e BOOTSTRAP_SERVERS=$NALMS_KAFKA_BOOTSTRAP \ + -e ES_HOST="${NALMS_ES_HOST}" \ + -e ES_PORT="${NALMS_ES_PORT}" \ + -e BOOTSTRAP_SERVERS="${NALMS_KAFKA_BOOTSTRAP}" \ + -e ALARM_LOGGER_PROPERTIES="/opt/nalms/config/alarm_logger.properties" \ + -v "${NALMS_ALARM_LOGGER_PROPERTIES}:/opt/nalms/config/alarm_logger.properties" \ --name nalms_logger_$CONFIG_NAME \ -d jgarrahan/nalms-phoebus-alarm-logger:latest start-logger $CONFIG_NAME /tmp/nalms/$CONFIG_NAME.xml \ No newline at end of file diff --git a/cli/commands/start-alarm-server b/cli/commands/start-alarm-server index 9cbc4e3..a301c00 100644 --- a/cli/commands/start-alarm-server +++ b/cli/commands/start-alarm-server @@ -30,8 +30,14 @@ if [[ -z "$NALMS_KAFKA_BOOTSTRAP" ]]; then usage fi +if [[ -z "$NALMS_ALARM_SERVER_PROPERTIES" ]]; then + echo "No Kafka bootstrap provided." + usage +fi + docker run -v $CONFIG_FILE:/tmp/nalms/$CONFIG_NAME.xml \ --name nalms_server_$CONFIG_NAME \ + -v "${NALMS_ALARM_SERVER_PROPERTIES}:/opt/nalms/config/alarm_server.properties" \ -e ALARM_IOC=false \ -e KAFKA_BOOTSTRAP="${NALMS_KAFKA_BOOTSTRAP}" \ -e EPICS_CA_ADDR_LIST="${EPICS_CA_ADDR_LIST}" \ @@ -40,4 +46,5 @@ docker run -v $CONFIG_FILE:/tmp/nalms/$CONFIG_NAME.xml \ -e EPICS_PVA_ADDR_LIST="${EPICS_PVA_ADDR_LIST}" \ -e EPICS_PVA_SERVER_PORT="${EPICS_PVA_SERVER_PORT}" \ -e EPICS_PVA_REPEATER_PORT="${EPICS_PVA_REPEATER_PORT}" \ + -e ALARM_SERVER_PROPERTIES="/opt/nalms/config/alarm_server.properties" \ -d -t jgarrahan/nalms-phoebus-alarm-server:latest start-server $CONFIG_NAME /tmp/nalms/$CONFIG_NAME.xml diff --git a/cli/commands/start-cruise-control b/cli/commands/start-cruise-control index 6962f5f..24f830d 100644 --- a/cli/commands/start-cruise-control +++ b/cli/commands/start-cruise-control @@ -6,7 +6,8 @@ usage() { cli_name=${0##*/} echo "Usage: nalms start-cruise-control" - echo "Requires definition of \$NALMS_BOOTSTRAP_SERVERS, \$NALMS_ZOOKEEPER_HOST, \$NALMS_ZOOKEEPER_PORT \$NALMS_CRUISE_CONTROL_PORT" + echo "Requires definition of \$NALMS_CRUISE_CONTROL_PORT, \$NALMS_CRUISE_CONTROL_PROPERTIES" + echo "Optionally requires \$NALMS_BOOTSTRAP_SERVERS, \$NALMS_ZOOKEEPER_HOST if not defined in file and using environment substitution" exit 0 } @@ -15,30 +16,22 @@ if [[ "$1" == "-h" ]]; then fi -if [[ -z "$NALMS_KAFKA_BOOTSTRAP" ]]; then - echo "No bootstrap servers defined for cruise control." - usage -fi - -if [[ -z "$NALMS_ZOOKEEPER_HOST" ]]; then - echo "No zookeeper host defined for cruise control." - usage -fi - -if [[ -z "$NALMS_ZOOKEEPER_PORT" ]]; then - echo "No zookeeper host defined for cruise control." +if [[ -z "$NALMS_CRUISE_CONTROL_PORT" ]]; then + echo "No cruise control port defined ." usage fi -if [[ -z "$NALMS_CRUISE_CONTROL_PORT" ]]; then - echo "No cruise control port defined ." +if [[ -z "$NALMS_CRUISE_CONTROL_PROPERTIES" ]]; then + echo "No cruise control properties file defined ." usage fi docker run \ - -e BOOTSTRAP_SERVERS=$NALMS_KAFKA_BOOTSTRAP \ - -e ZOOKEEPER_CONNECT=$NALMS_ZOOKEEPER_HOST:$NALMS_ZOOKEEPER_PORT \ + -e BOOTSTRAP_SERVERS="${NALMS_KAFKA_BOOTSTRAP}" \ + -e ZOOKEEPER_CONNECT="${NALMS_ZOOKEEPER_HOST}:${NALMS_ZOOKEEPER_PORT}" \ + -e CRUISE_CONTROL_PROPERTIES="/opt/cruise-control/config/cruisecontrol.properties" \ + -v "${NALMS_CRUISE_CONTROL_PROPERTIES}:/opt/cruise-control/config/cruisecontrol.properties" \ --name nalms_cruise_control \ -p "$NALMS_CRUISE_CONTROL_PORT:9090" -d jgarrahan/nalms-cruise-control:latest \ No newline at end of file diff --git a/cli/commands/start-elasticsearch b/cli/commands/start-elasticsearch index d137d43..bca0d52 100644 --- a/cli/commands/start-elasticsearch +++ b/cli/commands/start-elasticsearch @@ -7,6 +7,7 @@ usage() { cli_name=${0##*/} echo "Usage: nalms start-elasticsearch [--port elasticsearch]" echo "Port required, but may be set using the \$NALMS_ES_PORT environment variable." + echo "\$NALMS_ES_CONFIG must be set." exit 0 } @@ -27,6 +28,11 @@ if [[ -z "$NALMS_ES_PORT" ]]; then usage fi +if [[ -z "$NALMS_ES_CONFIG" ]]; then + echo "No configuration file provided for Elasticsearch. Define \$NALMS_ES_CONFIG" + usage +fi + docker run \ -e node.name=node01 \ @@ -35,6 +41,7 @@ docker run \ -e ES_JAVA_OPTS="-Xms128m -Xmx128m" \ -e ES_HOST=localhost \ -e ES_PORT=9200 \ + -v "${NALMS_ES_CONFIG}:/usr/share/elasticsearch/config" \ -p "$NALMS_ES_PORT:9200" \ --name nalms_elasticsearch \ -d jgarrahan/nalms-elasticsearch:latest diff --git a/cli/commands/start-grafana b/cli/commands/start-grafana index 4644d05..b16de11 100644 --- a/cli/commands/start-grafana +++ b/cli/commands/start-grafana @@ -6,7 +6,7 @@ usage() { cli_name=${0##*/} echo "Usage: nalms start-grafana [--port grafana_port] [--config config_name]" - echo "Requires definition of \$NALMS_GRAFANA_PORT, \$NALMS_CONFIGURATIONS, \$NALMS_ES_HOST, \$NALMS_ES_PORT" + echo "Requires definition of \$NALMS_GRAFANA_PORT, \$NALMS_GRAFANA_DASHBOARD_DIR, \$NALMS_GRAFANA_DATASOURCE_FILE, \$NALMS_GRAFANA_CONFIG" exit 0 } @@ -17,9 +17,6 @@ do --port) NALMS_GRAFANA_PORT=$2 ;; - --config) - NALMS_CONFIGURATIONS=$2 - ;; "") esac shift @@ -29,28 +26,26 @@ if [[ -z "$NALMS_GRAFANA_PORT" ]]; then echo "No port provided for Grafana." usage fi - -if [[ -z "$NALMS_CONFIGURATIONS" ]]; then - echo "No configurations provided to launch Grafana." +if [[ -z "$NALMS_GRAFANA_DASHBOARD_DIR" ]]; then + echo "NALMS_GRAFANA_DASHBOARD_DIR not defined." usage fi -if [[ -z "$NALMS_ES_HOST" ]]; then - echo "Elasticsearch host not defined." +if [[ -z "$NALMS_GRAFANA_DATASOURCE_FILE" ]]; then + echo "NALMS_GRAFANA_DATASOURCE_FILE not defined." usage fi - -if [[ -z "$NALMS_ES_PORT" ]]; then - echo "Elasticsearch port not defined." +if [[ -z "$NALMS_GRAFANA_CONFIG" ]]; then + echo "NALMS_GRAFANA_CONFIG not defined." usage fi docker run \ - -p "$NALMS_GRAFANA_PORT:3000" \ - -e ES_HOST=$NALMS_ES_HOST \ - -e ES_PORT=$NALMS_ES_PORT \ - -e CONFIG_NAME=$NALMS_CONFIGURATIONS \ + -p "${NALMS_GRAFANA_PORT}:3000" \ + -v "${NALMS_GRAFANA_DASHBOARD_DIR}:/var/lib/grafana/dashboards" \ + -v "${NALMS_GRAFANA_DATASOURCE_FILE}:/etc/grafana/provisioning/datasources/all.yml" \ + -v "${NALMS_GRAFANA_CONFIG}:/etc/grafana/config.ini" \ --name nalms_grafana \ -d jgarrahan/nalms-grafana:latest diff --git a/cli/commands/start-zookeeper b/cli/commands/start-zookeeper index 6e16ce9..17c9a87 100644 --- a/cli/commands/start-zookeeper +++ b/cli/commands/start-zookeeper @@ -7,6 +7,7 @@ usage() { cli_name=${0##*/} echo "Usage: nalms start-zookeeper [--port zookeeper_port]" echo "Port required, but may be set using the \$NALMS_ZOOKEEPER_PORT environment variable." + echo "Must set \$ NALMS_ZOOKEEPER_CONFIG" exit 0 } @@ -28,5 +29,11 @@ if [[ -z "$NALMS_ZOOKEEPER_PORT" ]]; then usage fi +if [[ -z "$NALMS_ZOOKEEPER_CONFIG" ]]; then + echo "No config provided for Zookeeper. Please set \$NALMS_ZOOKEEPER_CONFIG" + usage +fi -docker run -p "$NALMS_ZOOKEEPER_PORT:2181" --name nalms_zookeeper -d jgarrahan/nalms-zookeeper:latest \ No newline at end of file +docker run -p "${NALMS_ZOOKEEPER_PORT}:2181" -e ZOOKEEPER_CONFIG=/tmp/zoo.cfg \ + -v "${NALMS_ZOOKEEPER_CONFIG}:/tmp/zoo.cfg" --name nalms_zookeeper \ + -d jgarrahan/nalms-zookeeper:latest \ No newline at end of file diff --git a/cli/nalms b/cli/nalms index 5e98714..189f375 100644 --- a/cli/nalms +++ b/cli/nalms @@ -22,6 +22,8 @@ Commands: create-alarm-ioc configuration_file config_name output_directory convert-alh alh_file output_filename config_name delete-configurations configuration_name + add-grafana-datasource config_names + build-grafana-dashboard config_name list-configurations launch-editor generate-kafka-certs domain password @@ -71,6 +73,12 @@ case "$1" in delete-configuration|d) . "$CLI_WORKDIR/commands/delete-configuration" "$2" ;; + add-grafana-datasource|d) + . "$CLI_WORKDIR/commands/add-grafana-datasource" "$2" + ;; + build-grafana-dashboard|d) + . "$CLI_WORKDIR/commands/build-grafana-dashboard" "$2" + ;; launch-editor|d) . "$CLI_WORKDIR/commands/launch-editor" ;; diff --git a/config/dev_logging.properties b/config/dev_logging.properties deleted file mode 100644 index 1f112cc..0000000 --- a/config/dev_logging.properties +++ /dev/null @@ -1,31 +0,0 @@ -#logging.properties file. - - handlers= java.util.logging.FileHandler - java.util.logging.FileHandler.pattern=/var/log/nalms/java%u.log - -# Default global logging level. -# This specifies which kinds of events are logged across -# all loggers. For any given facility this global level -# can be overriden by a facility-specific level. -# Note that the ConsoleHandler also has a separate level -# setting to limit messages printed to the console. - - .level= WARNING - -# Limit the messages that are printed on the console to INFO and above. - - java.util.logging.ConsoleHandler.level = WARNING - java.util.logging.ConsoleHandler.formatter = - java.util.logging.SimpleFormatter - - java.util.logging.SimpleFormatter.format='%1$tY-%1$tm-%1$td %1$tH:%1$tM:%1$tS %4$s %2$s %5$s%6$s%n' - -# The logger with javax.jms.connection name space will write -# Level.INFO messages to its output handler(s). In this configuration -# the ouput handler is set to java.util.logging.ConsoleHandler. - - javax.jms.connection.level = WARNING - - - - org.apache.kafka.level = SEVERE \ No newline at end of file diff --git a/config/dev_zookeeper.properties b/config/dev_zookeeper.properties deleted file mode 100644 index 1273875..0000000 --- a/config/dev_zookeeper.properties +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# the directory where the snapshot is stored. -dataDir=/var/log/nalms/zookeeper -# the port at which the clients will connect -clientPort=2181 -# disable the per-ip limit on the number of connections since this is a non-production config -maxClientCnxns=0 -# Disable the adminserver by default to avoid port conflicts. -# Set the port to something non-conflicting if choosing to enable this -admin.enableServer=false -# admin.serverPort=8080 - - -#dataLogDir - -#will direct the machine to write the transaction log to thedataLogDirratherthan thedataDir \ No newline at end of file diff --git a/config/local_server.properties b/config/local_server.properties deleted file mode 100644 index b96d23c..0000000 --- a/config/local_server.properties +++ /dev/null @@ -1,154 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# see kafka.server.KafkaConfig for additional details and defaults - -############################# Server Basics ############################# - -# The id of the broker. This must be set to a unique integer for each broker. -broker.id=0 - -############################# Socket Server Settings ############################# - -# The address the socket server listens on. It will get the value returned from -# java.net.InetAddress.getCanonicalHostName() if not configured. -# FORMAT: -# listeners = listener_name://host_name:port -# EXAMPLE: -# listeners = PLAINTEXT://your.host.name:9092 -#listeners=SSL://localhost:9092 -listeners=PLAINTEXT://:9092 - -#ssl.truststore.location=/var/private/ssl/kafka.server.truststore.jks -#ssl.truststore.password=test1234 -#ssl.keystore.location=/var/private/ssl/kafka.server.keystore.jks -#ssl.keystore.password=test1234 -#ssl.key.password=test1234 - -# use ssl for inter broker communication -#security.inter.broker.protocol=SSL - -# Hostname and port the broker will advertise to producers and consumers. If not set, -# it uses the value for "listeners" if configured. Otherwise, it will use the value -# returned from java.net.InetAddress.getCanonicalHostName(). -#advertised.listeners=localhost:9092 - -# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details -#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -#listener.security.protocol.map=SSL:SSL - - -# The number of threads that the server uses for receiving requests from the network and sending responses to the network -num.network.threads=3 - -# The number of threads that the server uses for processing requests, which may include disk I/O -num.io.threads=8 - -# The send buffer (SO_SNDBUF) used by the socket server -socket.send.buffer.bytes=102400 - -# The receive buffer (SO_RCVBUF) used by the socket server -socket.receive.buffer.bytes=102400 - -# The maximum size of a request that the socket server will accept (protection against OOM) -socket.request.max.bytes=104857600 - - -############################# Log Basics ############################# - -# A comma separated list of directories under which to store log files -log.dirs=/tmp/log/kafka-logs - -# The default number of log partitions per topic. More partitions allow greater -# parallelism for consumption, but this will also result in more files across -# the brokers. -num.partitions=3 - -# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. -# This value is recommended to be increased for installations with data dirs located in RAID array. -num.recovery.threads.per.data.dir=1 - -############################# Internal Topic Settings ############################# -# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" -# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. -offsets.topic.replication.factor=1 -transaction.state.log.replication.factor=1 -transaction.state.log.min.isr=1 - -############################# Log Flush Policy ############################# - -# Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. -# There are a few important trade-offs here: -# 1. Durability: Unflushed data may be lost if you are not using replication. -# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. -# The settings below allow one to configure the flush policy to flush data after a period of time or -# every N messages (or both). This can be done globally and overridden on a per-topic basis. - -# The number of messages to accept before forcing a flush of data to disk -#log.flush.interval.messages=10000 - -# The maximum amount of time a message can sit in a log before we force a flush -#log.flush.interval.ms=1000 - -############################# Log Retention Policy ############################# - -# The following configurations control the disposal of log segments. The policy can -# be set to delete segments after a period of time, or after a given size has accumulated. -# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens -# from the end of the log. - -# The minimum age of a log file to be eligible for deletion due to age -#log.retention.hours=2 - -log.cleaner.delete.retention.ms=10000 - -# A size-based retention policy for logs. Segments are pruned from the log unless the remaining -# segments drop below log.retention.bytes. Functions independently of log.retention.hours. -#log.retention.bytes=1073741824 - -# The maximum size of a log segment file. When this size is reached a new log segment will be created. -log.segment.bytes=1073741824 - -# The interval at which log segments are checked to see if they can be deleted according -# to the retention policies -log.retention.check.interval.ms=100 - -# compaction -log.cleanup.policy=compact -log.cleaner.max.compaction.lag.ms=1000 - -############################# Zookeeper ############################# - -# Zookeeper connection string (see zookeeper docs for details). -# This is a comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". -# You can also append an optional chroot string to the urls to specify the -# root directory for all kafka znodes. -zookeeper.connect=localhost:2181 - -# Timeout in ms for connecting to zookeeper -zookeeper.connection.timeout.ms=18000 - - -############################# Group Coordinator Settings ############################# - -# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. -# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. -# The default value for this is 3 seconds. -# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. -# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. -group.initial.rebalance.delay.ms=0 \ No newline at end of file diff --git a/config/prod_server.properties b/config/prod_server.properties deleted file mode 100644 index 3a9077b..0000000 --- a/config/prod_server.properties +++ /dev/null @@ -1,156 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# see kafka.server.KafkaConfig for additional details and defaults - -############################# Server Basics ############################# - -# The id of the broker. This must be set to a unique integer for each broker. -broker.id=0 - -############################# Socket Server Settings ############################# - -# The address the socket server listens on. It will get the value returned from -# java.net.InetAddress.getCanonicalHostName() if not configured. -# FORMAT: -# listeners = listener_name://host_name:port -# EXAMPLE: -# listeners = PLAINTEXT://your.host.name:9092 -listeners=SSL://localhost:9092 - -ssl.truststore.location=/var/private/ssl/kafka.server.truststore.jks -ssl.truststore.password=test1234 -ssl.keystore.location=/var/private/ssl/kafka.server.keystore.jks -ssl.keystore.password=test1234 -ssl.key.password=test1234 - -# use ssl for inter broker communication -security.inter.broker.protocol=SSL - -# Hostname and port the broker will advertise to producers and consumers. If not set, -# it uses the value for "listeners" if configured. Otherwise, it will use the value -# returned from java.net.InetAddress.getCanonicalHostName(). -#advertised.listeners=localhost:9092 - -# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details -#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL -listener.security.protocol.map=SSL:SSL - - -# The number of threads that the server uses for receiving requests from the network and sending responses to the network -num.network.threads=3 - -# The number of threads that the server uses for processing requests, which may include disk I/O -num.io.threads=8 - -# The send buffer (SO_SNDBUF) used by the socket server -socket.send.buffer.bytes=102400 - -# The receive buffer (SO_RCVBUF) used by the socket server -socket.receive.buffer.bytes=102400 - -# The maximum size of a request that the socket server will accept (protection against OOM) -socket.request.max.bytes=104857600 - - -############################# Log Basics ############################# - -# A comma separated list of directories under which to store log files -log.dirs=/var/log/kafka-logs - -# The default number of log partitions per topic. More partitions allow greater -# parallelism for consumption, but this will also result in more files across -# the brokers. -num.partitions=3 - -# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. -# This value is recommended to be increased for installations with data dirs located in RAID array. -num.recovery.threads.per.data.dir=1 - -############################# Internal Topic Settings ############################# -# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" -# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. -offsets.topic.replication.factor=3 -transaction.state.log.replication.factor=3 -transaction.state.log.min.isr=3 - -############################# Log Flush Policy ############################# - -# Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. -# There are a few important trade-offs here: -# 1. Durability: Unflushed data may be lost if you are not using replication. -# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. -# The settings below allow one to configure the flush policy to flush data after a period of time or -# every N messages (or both). This can be done globally and overridden on a per-topic basis. - -# The number of messages to accept before forcing a flush of data to disk -#log.flush.interval.messages=10000 - -# The maximum amount of time a message can sit in a log before we force a flush -#log.flush.interval.ms=1000 - -############################# Log Retention Policy ############################# - -# The following configurations control the disposal of log segments. The policy can -# be set to delete segments after a period of time, or after a given size has accumulated. -# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens -# from the end of the log. - -# The minimum age of a log file to be eligible for deletion due to age -#log.retention.hours=2 - -log.cleaner.delete.retention.ms=10000 - -# A size-based retention policy for logs. Segments are pruned from the log unless the remaining -# segments drop below log.retention.bytes. Functions independently of log.retention.hours. -#log.retention.bytes=1073741824 - -# The maximum size of a log segment file. When this size is reached a new log segment will be created. -log.segment.bytes=1073741824 - -# The interval at which log segments are checked to see if they can be deleted according -# to the retention policies -log.retention.check.interval.ms=100 - -# compaction -log.cleanup.policy=compact -log.cleaner.max.compaction.lag.ms=1000 - -############################# Zookeeper ############################# - -# Zookeeper connection string (see zookeeper docs for details). -# This is a comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". -# You can also append an optional chroot string to the urls to specify the -# root directory for all kafka znodes. -zookeeper.connect=localhost:2181 - -# Timeout in ms for connecting to zookeeper -zookeeper.connection.timeout.ms=18000 - - -############################# Group Coordinator Settings ############################# - -# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. -# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. -# The default value for this is 3 seconds. -# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. -# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. -group.initial.rebalance.delay.ms=0 - - -#authorizer.class.name=kafka.security.authorizer.AclAuthorizer \ No newline at end of file diff --git a/config/prod_zookeeper.properties b/config/prod_zookeeper.properties deleted file mode 100644 index a8dfdbe..0000000 --- a/config/prod_zookeeper.properties +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# the directory where the snapshot is stored. -dataDir=/var/log/zookeeper -# the port at which the clients will connect -clientPort=2181 -# disable the per-ip limit on the number of connections since this is a non-production config -maxClientCnxns=0 -# Disable the adminserver by default to avoid port conflicts. -# Set the port to something non-conflicting if choosing to enable this -admin.enableServer=false -# admin.serverPort=8080 diff --git a/config/server.properties b/config/server.properties deleted file mode 100644 index 43e5220..0000000 --- a/config/server.properties +++ /dev/null @@ -1,145 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# see kafka.server.KafkaConfig for additional details and defaults - -############################# Server Basics ############################# - -# The id of the broker. This must be set to a unique integer for each broker. -broker.id=0 - -############################# Socket Server Settings ############################# - -# The address the socket server listens on. It will get the value returned from -# java.net.InetAddress.getCanonicalHostName() if not configured. -# FORMAT: -# listeners = listener_name://host_name:port -# EXAMPLE: -# listeners = PLAINTEXT://your.host.name:9092 -listeners=PLAINTEXT://localhost:9092 - -# Hostname and port the broker will advertise to producers and consumers. If not set, -# it uses the value for "listeners" if configured. Otherwise, it will use the value -# returned from java.net.InetAddress.getCanonicalHostName(). -#advertised.listeners=localhost:9092 - -# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details -#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL - -# The number of threads that the server uses for receiving requests from the network and sending responses to the network -num.network.threads=3 - -# The number of threads that the server uses for processing requests, which may include disk I/O -num.io.threads=8 - -# The send buffer (SO_SNDBUF) used by the socket server -socket.send.buffer.bytes=102400 - -# The receive buffer (SO_RCVBUF) used by the socket server -socket.receive.buffer.bytes=102400 - -# The maximum size of a request that the socket server will accept (protection against OOM) -socket.request.max.bytes=104857600 - - -############################# Log Basics ############################# - -# A comma separated list of directories under which to store log files -log.dirs=/tmp/kafka-logs - -# The default number of log partitions per topic. More partitions allow greater -# parallelism for consumption, but this will also result in more files across -# the brokers. -num.partitions=1 - -# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. -# This value is recommended to be increased for installations with data dirs located in RAID array. -num.recovery.threads.per.data.dir=1 - -############################# Internal Topic Settings ############################# -# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" -# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3. -offsets.topic.replication.factor=1 -transaction.state.log.replication.factor=1 -transaction.state.log.min.isr=1 - -############################# Log Flush Policy ############################# - -# Messages are immediately written to the filesystem but by default we only fsync() to sync -# the OS cache lazily. The following configurations control the flush of data to disk. -# There are a few important trade-offs here: -# 1. Durability: Unflushed data may be lost if you are not using replication. -# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush. -# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks. -# The settings below allow one to configure the flush policy to flush data after a period of time or -# every N messages (or both). This can be done globally and overridden on a per-topic basis. - -# The number of messages to accept before forcing a flush of data to disk -#log.flush.interval.messages=10000 - -# The maximum amount of time a message can sit in a log before we force a flush -#log.flush.interval.ms=1000 - -############################# Log Retention Policy ############################# - -# The following configurations control the disposal of log segments. The policy can -# be set to delete segments after a period of time, or after a given size has accumulated. -# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens -# from the end of the log. - -# The minimum age of a log file to be eligible for deletion due to age -#log.retention.hours=2 - -log.cleaner.delete.retention.ms=10000 - -# A size-based retention policy for logs. Segments are pruned from the log unless the remaining -# segments drop below log.retention.bytes. Functions independently of log.retention.hours. -#log.retention.bytes=1073741824 - -# The maximum size of a log segment file. When this size is reached a new log segment will be created. -log.segment.bytes=1073741824 - -# The interval at which log segments are checked to see if they can be deleted according -# to the retention policies -log.retention.check.interval.ms=100 - -# compaction -log.cleanup.policy=compact -log.cleaner.max.compaction.lag.ms=1000 - -############################# Zookeeper ############################# - -# Zookeeper connection string (see zookeeper docs for details). -# This is a comma separated host:port pairs, each corresponding to a zk -# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". -# You can also append an optional chroot string to the urls to specify the -# root directory for all kafka znodes. -zookeeper.connect=localhost:2181 - -# Timeout in ms for connecting to zookeeper -zookeeper.connection.timeout.ms=18000 - - -############################# Group Coordinator Settings ############################# - -# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance. -# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms. -# The default value for this is 3 seconds. -# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing. -# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup. -group.initial.rebalance.delay.ms=0 - - -metric.reporters=com.linkedin.kafka.cruisecontrol.metricsreporter.CruiseControlMetricsReporter \ No newline at end of file diff --git a/cruise-control/Dockerfile b/cruise-control/Dockerfile index 970877b..6b925ef 100644 --- a/cruise-control/Dockerfile +++ b/cruise-control/Dockerfile @@ -10,9 +10,6 @@ COPY gradle.properties /tmp/cruise-control/gradle.properties WORKDIR /opt -# && rm gradle.properties \ -# && cp /tmp/cruise-control/gradle.properties gradle.properties \ - RUN yum install -y git ${JAVA_BASE}-devel-${JAVA_VERSION} && \ git clone https://github.com/linkedin/cruise-control.git \ && cd cruise-control \ @@ -24,7 +21,6 @@ RUN yum install -y git ${JAVA_BASE}-devel-${JAVA_VERSION} && \ -o /tmp/cruise-control-ui.tar.gz \ && tar zxvf /tmp/cruise-control-ui.tar.gz -COPY cruisecontrol.properties /opt/cruise-control/config/cruisecontrol.properties FROM centos:7 as runtime WORKDIR /opt/cruise-control @@ -35,6 +31,7 @@ ENV JAVA_HOME /usr/lib/jvm/${JAVA_BASE}-${JAVA_VERSION} ENV PATH ${JAVA_HOME}/bin:$PATH COPY --from=build /opt/cruise-control . +COPY config.csv /opt/cruise-control/cruise-control-ui/dist/static/config.csv COPY start-cruise-control.sh /opt/cruise-control RUN yum -y install ${JAVA_BASE}-${JAVA_VERSION} && \ diff --git a/cruise-control/README.md b/cruise-control/README.md new file mode 100644 index 0000000..fd401ec --- /dev/null +++ b/cruise-control/README.md @@ -0,0 +1,3 @@ +# NALMS cruise-control image + +In order to run this image, you must mount a cruisecontrol.properties to a path specified with the $CRUISE_CONTROL_PROPERTIES env variable. The image will perform interpolation on properties files with $BOOTSTRAP_SERVERS or $ZOOKEEPER_CONNECT as placeholders and defined $BOOTSTRAP_SERVERS or $ZOOKEEPER_CONNECT environment variables. \ No newline at end of file diff --git a/cruise-control/config.csv b/cruise-control/config.csv new file mode 100644 index 0000000..c853839 --- /dev/null +++ b/cruise-control/config.csv @@ -0,0 +1 @@ +nalms,nalms,/kafkacruisecontrol diff --git a/cruise-control/start-cruise-control.sh b/cruise-control/start-cruise-control.sh index 1176461..1569308 100644 --- a/cruise-control/start-cruise-control.sh +++ b/cruise-control/start-cruise-control.sh @@ -23,8 +23,8 @@ if [[ -z "$BOOTSTRAP_SERVERS" ]]; then usage fi +# if launching using interpolation, perform substitution +sed 's/$ZOOKEEPER_CONNECT/'"$ZOOKEEPER_CONNECT/" $CRUISE_CONTROL_PROPERTIES > /tmp/cruisecontrol.properties +sed -i 's/$BOOTSTRAP_SERVERS/'"$BOOTSTRAP_SERVERS/" /tmp/cruisecontrol.properties -sed -i 's/$ZOOKEEPER_CONNECT/'"$ZOOKEEPER_CONNECT/" /opt/cruise-control/config/cruisecontrol.properties -sed -i 's/$BOOTSTRAP_SERVERS/'"$BOOTSTRAP_SERVERS/" /opt/cruise-control/config/cruisecontrol.properties - -./kafka-cruise-control-start.sh /opt/cruise-control/config/cruisecontrol.properties \ No newline at end of file +./kafka-cruise-control-start.sh /tmp/cruisecontrol.properties \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 63d1e60..4e0c0a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,15 @@ version: "3.0" services: zookeeper: - image: jgarrahan/nalms-zookeeper:v0.3 + image: jgarrahan/nalms-zookeeper:v0.4 ports: - "2181:2181" + environment: + ZOOKEEPER_CONFIG: /tmp/zoo.cfg + volumes: + - "./examples/demo/config/zoo.cfg:/tmp/zoo.cfg" kafka: - image: jgarrahan/nalms-kafka:v0.3 + image: jgarrahan/nalms-kafka:v0.4 depends_on: - zookeeper ports: @@ -27,7 +31,7 @@ services: reservations: memory: 8G kafka1: - image: jgarrahan/nalms-kafka:v0.3 + image: jgarrahan/nalms-kafka:v0.4 depends_on: - zookeeper ports: @@ -49,7 +53,7 @@ services: reservations: memory: 8G kafka2: - image: jgarrahan/nalms-kafka:v0.3 + image: jgarrahan/nalms-kafka:v0.4 depends_on: - zookeeper ports: @@ -71,7 +75,7 @@ services: reservations: memory: 8G phoebus-alarm-server: - image: jgarrahan/nalms-phoebus-alarm-server:v0.3 + image: jgarrahan/nalms-phoebus-alarm-server:v0.4 links: - kafka - example-ioc @@ -79,6 +83,7 @@ services: - kafka volumes: - "./examples/heart_of_gold/heart_of_gold.xml:/tmp/nalms/heart_of_gold.xml" + - "./examples/heart_of_gold/config/alarm_server.properties:/opt/nalms/config/alarm_server.properties" command: start-server HeartOfGold /tmp/nalms/heart_of_gold.xml environment: KAFKA_BOOTSTRAP: kafka:9092 @@ -86,10 +91,13 @@ services: ALARM_IOC: "false" EPICS_CA_SERVER_PORT: 5064 EPICS_CA_REPEATER_PORT: 5065 + ALARM_SERVER_PROPERTIES: "/opt/nalms/config/alarm_server.properties" elasticsearch: - image: jgarrahan/nalms-elasticsearch:v0.3 + image: jgarrahan/nalms-elasticsearch:v0.4 ports: - "9200:9200" + volumes: + - "./exampes/heart_of_gold/config/elasticsearch:/usr/share/elasticsearch/config" environment: node.name: node01 cluster.name: es-cluster-7 @@ -100,17 +108,21 @@ services: KAFKA_HOST: kafka KAFKA_PORT: 9092 grafana: - image: jgarrahan/nalms-grafana:v0.3 + image: jgarrahan/nalms-grafana:v0.4 ports: - '3000:3000' links: - elasticsearch + volumes: + - "./examples/heart_of_gold/config/dashboards:/var/lib/grafana/dashboards" + - "./examples/heart_of_gold/config/datasource.yml:/etc/grafana/provisioning/datasources/all.yml" + - "./examples/heart_of_gold/config/grafana.ini:/etc/grafana/config.ini" environment: CONFIG_NAME: HeartOfGold ES_HOST: elasticsearch ES_PORT: 9200 phoebus-alarm-logger: - image: jgarrahan/nalms-phoebus-alarm-logger:v0.3 + image: jgarrahan/nalms-phoebus-alarm-logger:v0.4 links: - elasticsearch depends_on: @@ -121,11 +133,13 @@ services: ES_HOST: elasticsearch ES_PORT: 9200 BOOTSTRAP_SERVERS: kafka:9092,kafka1:9092,kafka2:9092 + ALARM_LOGGER_PROPERTIES: /opt/nalms/config/alarm_logger.properties volumes: - "./examples/heart_of_gold/heart_of_gold.xml:/tmp/nalms/heart_of_gold.xml" + - "./examples/heart_of_gold/config/logger.properties:/opt/nalms/config/alarm_logger.properties" command: start-logger HeartOfGold /tmp/nalms/heart_of_gold.xml example-ioc: - image: jgarrahan/nalms-example-ioc:v0.3 + image: jgarrahan/nalms-example-ioc:v0.4 ports: - "5075:5065/tcp" - "5074:5064/tcp" @@ -135,12 +149,15 @@ services: EPICS_CA_SERVER_PORT: 5064 EPICS_CA_REPEATER_PORT: 5065 cruise-control: - image: jgarrahan/nalms-cruise-control:v0.3 + image: jgarrahan/nalms-cruise-control:v0.4 ports: - "9090:9090" environment: BOOTSTRAP_SERVERS: kafka:9092,kafka1:9092,kafka2:9092 ZOOKEEPER_CONNECT: zookeeper:2181 + CRUISE_CONTROL_PROPERTIES: /opt/cruise-control/config/cruisecontrol.properties + volumes: + - "./examples/heart_of_gold/config/cruisecontrol.properties:/opt/cruise-control/config/cruisecontrol.properties" links: - kafka - zookeeper diff --git a/docs/cli.md b/docs/cli.md index 6353b54..873749f 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -17,6 +17,14 @@ In order to abstract the deployment process and interact with the Dockerized app | NALMS_CONFIGURATIONS | Comma separated list of configurations for launching Grafana | | NALMS_HOME | Path to NALMS repository | | NALMS_CLIENT_JAR | Path to NALMS client jar file | +| NALMS_ALARM_SERVER_PROPERTIES | Path to alarm server properties file | +| NALMS_ALARM_LOGGER_PROPERTIES | Path to alarm logger properties file | +| NALMS_CRUISE_CONTROL_PROPERTIES | Path to cruise control properties file | +| NALMS_GRAFANA_DASHBOARD_DIR | Path to Grafana dashboard directory | +| NALMS_GRAFANA_CONFIG | Path to Grafana configuration file | +| NALMS_GRAFANA_DATASOURCE_FILE | Path to Grafana datasource file | +| NALMS_ZOOKEEPER_CONFIG | Path to Zookeeper configuration file | +| NALMS_ES_CONFIG | Path to elasticsearch configuration file | ## convert-alh @@ -104,4 +112,16 @@ $ bash cli/nalms start-phoebus-client Start Zookeeper, creating image named `nalms_zookeeper`. ``` $ bash cli/nalms start-zookeeper +``` + +## add-grafana-datasource +Add a Grafana datasource for a configuration(s). +``` +$ bash cli/nalms add-grafana-datasource config_names +``` + +## build-grafana-dashboard +Build a Grafana dashboard for a configuration +``` +$ bash cli/nalms build-grafana-dashboard config_name ``` \ No newline at end of file diff --git a/docs/install.md b/docs/install.md index 386ef62..d67b8a9 100644 --- a/docs/install.md +++ b/docs/install.md @@ -24,7 +24,7 @@ The current NALMS iteration consists of the following Dockerhub hosted container * Script for templating of Alarm indices ### jgarrahan/nalms-grafana -* Grafana service (7.3.0-beta1) +* Grafana service (7.5.3) * Template Grafana dashboard for any configuration * Can be launched with multiple configurations as a comma separated list * Automatic generation of elasticsearch datasources based on network configs and configuration names @@ -102,32 +102,36 @@ A reference for elasticsearch configuration files can be found [here](https://ww In order for the Elasticsearch fields to be properly formatted, a template matching the topic scheme must be posted to the server. These may be versioned and are automatically applied to newly created indices. The initial script for templating NALMS topics is hosted in `elasticsearch/scripts/create_alarm_template.sh`. This template has been taken from the Phoebus source [examples](https://github.com/ControlSystemStudio/phoebus/blob/master/app/alarm/examples/create_alarm_topics.sh). #### Docker -The elasticsearch node may be configured using an exposed port, node specific variables, and Kafka networking variables. Because this is a single node deployment, `single-node` deployment is used. Java options may be specifified using the `ES_JAVA_OPTS` variable. +The elasticsearch node may be configured using an exposed port, node specific variables, and Kafka networking variables. Because this is a single node deployment, `single-node` deployment is used. Java options may be specifified using the `ES_JAVA_OPTS` variable. The Elasticsearch docker image also creates the appropriate elasticsearch template for the configuration messages. The configuration files must be mounted to `/usr/share/elasticsearch/config`. The following Docker run command will lauch an Elasticsearch node reachable on host machine port 9200. ``` -docker run \ +$ docker run \ -e node.name=node01 \ -e cluster.name=es-cluster-7 \ -e discovery.type=single-node \ -e ES_JAVA_OPTS="-Xms128m -Xmx128m" \ -e ES_HOST=localhost \ -e ES_PORT=9200 \ - -p "9200:9200" \ + -v "${NALMS_ES_CONFIG}:/usr/share/elasticsearch/config" \ + -p "$NALMS_ES_PORT:9200" \ --name nalms_elasticsearch \ -d jgarrahan/nalms-elasticsearch:latest ``` + ### Zookeeper #### Configuration -At present, Zookeeper is launched using the default settings. For more sophisticated deployments, a configuration with mounted configuration files would be preferable. +At present, Zookeeper is launched using the default settings. For more sophisticated deployments, a configuration with mounted configuration files would be preferable. The configuration file is mounted to the Zookeeper container at runtime. A description of the zookeeper configuration may be found [here](https://zookeeper.apache.org/doc/r3.5.9/zookeeperAdmin.html). #### Docker The following command will run Zookeeper accessible on the host machine at port 2181: ``` -docker run -p "2181:2181" --name nalms_zookeeper -d jgarrahan/nalms-zookeeper +$ docker run -p "${NALMS_ZOOKEEPER_PORT}:2181" -e ZOOKEEPER_CONFIG=/tmp/zoo.cfg \ + -v "${NALMS_ZOOKEEPER_CONFIG}:/tmp/zoo.cfg" --name nalms_zookeeper \ + -d jgarrahan/nalms-zookeeper:latest ``` ### Kafka @@ -155,7 +159,7 @@ The Kafka broker images require the definition of Kafka networking variables, `K An example run command for the Kafka docker image is given below: ``` -docker run -m 8g \ +$ docker run -m 8g \ -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://${HOST_IP}:9092,CONNECTIONS_FROM_HOST://${HOST_IP}:19092 \ -e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,CONNECTIONS_FROM_HOST:PLAINTEXT \ -e KAFKA_LISTENERS=PLAINTEXT://${HOST_IP}:9092,CONNECTIONS_FROM_HOST://0.0.0.0:19092 \ @@ -191,19 +195,26 @@ org.phoebus.pv.ca/auto_addr_list=no org.phoebus.applications.alarm/server=kafka:19092 ``` -###3 Docker +### Docker + +The Phoebus alarm server requires mounting of the configuration file with the Docker volume option and the definition of environment variables indicating Kafka networking address, whether the alarm IOC is to be used, and the EPICS configuration settings to access the alarm and variable iocs. The image supports the substitution of networking variables ($KAFKA_BOOTSTRAP and EPICS variables). Alternatively, these can be defined directly in the configuration file. -The Phoebus alarm server requires mounting of the configuration file with the Docker volume option and the definition of environment variables indicating Kafka networking address, whether the alarm IOC is to be used, and the EPICS configuration settings to access the alarm and variable iocs. The Docker run command for the packaged example is given below: +The Docker run command for the packaged example is given below: ``` -$ docker run -v /full/path/to/examples/demo/demo.xml:/tmp/nalms/Demo.xml \ - --name nalms_server_Demo \ - -e ALARM_IOC=true \ - -e KAFKA_BOOTSTRAP=${HOST_IP}:19092 \ - -e EPICS_CA_ADDR_LIST=$HOST_IP \ - -e EPICS_CA_SERVER_PORT=5054 \ - -e EPICS_CA_REPEATER_PORT=5055 \ - -d -t jgarrahan/nalms-phoebus-alarm-server:latest start-server Demo /tmp/nalms/demo.xml +$ docker run -v $CONFIG_FILE:/tmp/nalms/$CONFIG_NAME.xml \ + --name nalms_server_$CONFIG_NAME \ + -v "${NALMS_ALARM_SERVER_PROPERTIES}:/opt/nalms/config/alarm_server.properties" \ + -e ALARM_IOC=false \ + -e KAFKA_BOOTSTRAP="${NALMS_KAFKA_BOOTSTRAP}" \ + -e EPICS_CA_ADDR_LIST="${EPICS_CA_ADDR_LIST}" \ + -e EPICS_CA_SERVER_PORT="${EPICS_CA_SERVER_PORT}" \ + -e EPICS_CA_REPEATER_PORT="${EPICS_CA_REPEATER_PORT}" \ + -e EPICS_PVA_ADDR_LIST="${EPICS_PVA_ADDR_LIST}" \ + -e EPICS_PVA_SERVER_PORT="${EPICS_PVA_SERVER_PORT}" \ + -e EPICS_PVA_REPEATER_PORT="${EPICS_PVA_REPEATER_PORT}" \ + -e ALARM_SERVER_PROPERTIES="/opt/nalms/config/alarm_server.properties" \ + -d -t jgarrahan/nalms-phoebus-alarm-server:latest start-server $CONFIG_NAME /tmp/nalms/$CONFIG_NAME.xml ``` The configuration file must be mounted to `/tmp/nalms/${CONFIG_NAME}, for internal identification. @@ -226,15 +237,17 @@ Additionally, logging for the logger is configurable and defined in `phoebus-ala #### Docker -The Phoebus alarm logger requires the mounting of the configuration file with the Docker volume option and the definition of Elasticsearch networking variables. The Docker run command for the packaged example is given below: +The Phoebus alarm logger requires the mounting of the configuration file with the Docker volume option. The image supports the interpolation of networking variables $NALMS_ES_HOST, $NALMS_ES_PORT, and $NALMS_KAFKA_BOOTSTRAP in this file. The Docker run command for the packaged example is given below: ``` -docker run -v /full/path/to/examples/demo/demo.xml:/tmp/nalms/Demo.xml \ - -e ES_HOST=${HOST_IP} \ - -e ES_PORT=9200 \ - -e BOOTSTRAP_SERVERS=${HOST_IP}:19092 \ - --name nalms_logger_Demo \ - -d jgarrahan/nalms-phoebus-alarm-logger:latest start-logger Demo /tmp/nalms/Demo.xml +$ docker run -v $CONFIG_FILE:/tmp/nalms/$CONFIG_NAME.xml \ + -e ES_HOST="${NALMS_ES_HOST}" \ + -e ES_PORT="${NALMS_ES_PORT}" \ + -e BOOTSTRAP_SERVERS="${NALMS_KAFKA_BOOTSTRAP}" \ + -e ALARM_LOGGER_PROPERTIES="/opt/nalms/config/alarm_logger.properties" \ + -v "${ALARM_LOGGER_PROPERTIES}:/opt/nalms/config/alarm_logger.properties" \ + --name nalms_logger_$CONFIG_NAME \ + -d jgarrahan/nalms-phoebus-alarm-logger:latest start-logger $CONFIG_NAME /tmp/nalms/$CONFIG_NAME.xml ``` The configuration file must be mounted to `/tmp/nalms/${CONFIG_NAME}, for internal identification. @@ -250,23 +263,27 @@ Like the alarm server and logger, the client also accepts a properties file that Grafana datasources and dashboards may be programatically provisioned as outlined [here](https://grafana.com/docs/grafana/latest/administration/provisioning/). Elasticsearch datasources define an index and networking variables. -For the purpose of NALMS, the Grafana image automatically generates the provisioned dashboards and datasources depending on configured Elasticsearch network settings and provided configurations using templates. The dashboard template is hosted at `grafana/dashboards/alarm_logs_dashboard.json` and the datsource template is generated from the `grafana/scripts/create-datasource-file.sh` bash script executed on startup. +General Grafana configuration is described [here](https://grafana.com/docs/grafana/v7.5/administration/configuration/). + +The dashboard template is hosted at `grafana/dashboards/alarm_logs_dashboard.json` and a configuration dashboard can be created using the `cli/nalms build-grafana-dashboard config-name` command. The datasource may be added to an existing datasource file using the `cli/nalms add-grafana-datasource config-name` command or manually created. #### Docker -The Grafana image automatically generates the provisioned dashboards and datasources depending on configured Elasticsearch network settings and provided configurations. The Docker run command for the packaged example is given below: +The Grafana image requires mounting of the dashboards, datasource file, and configuration file. The Docker run command for the packaged example is given below: ``` -docker run \ - -p "3000:3000" \ - -e ES_HOST=${HOST_IP} \ - -e ES_PORT=5064 \ - -e CONFIG_NAME=Demo \ +$ docker run \ + -p "${NALMS_GRAFANA_PORT}:3000" \ + -v "${NALMS_GRAFANA_DASHBOARD_DIR}:/var/lib/grafana/dashboards" \ + -v "${NALMS_GRAFANA_DATASOURCE_FILE}:/etc/grafana/provisioning/datasources/all.yml" \ + -v "${NALMS_GRAFANA_CONFIG}:/etc/grafana/config.ini" \ + -e ES_HOST=$NALMS_ES_HOST \ + -e ES_PORT=$NALMS_ES_PORT \ --name nalms_grafana \ -d jgarrahan/nalms-grafana:latest ``` -The Grafana dashboards are then reachable at localhost:3000 in browser. +The datasource file must be mounted to `/etc/grafana/provisioning/datasources/all.yml`, the dashboard directory must be mounted to `/var/lib/grafana/dashboards`, and the configuration must be mounted to `/etc/grafana/provisioning/datasources/all.yml`. The Grafana dashboards are then reachable at localhost:${NALMS_GRAFANA_PORT} in browser. ### Cruise Control @@ -276,19 +293,21 @@ The `cruise-control/cruisecontrol.properties` file dictates the behavior of the See wiki: https://github.com/linkedin/cruise-control/wiki - +https://github.com/linkedin/cruise-control-ui/wiki/Single-Kafka-Cluster #### Docker -The Cruise Control Image requires definition of bootstrap servers and Zookeeper addresses. The Docker run command for the packaged example is given below: +In order to run this image, you must mount a cruisecontrol.properties to a path specified with the $CRUISE_CONTROL_PROPERTIES env variable. The image will perform interpolation on properties files with $BOOTSTRAP_SERVERS or $ZOOKEEPER_CONNECT as placeholders and defined $BOOTSTRAP_SERVERS or $ZOOKEEPER_CONNECT environment variables. The Docker run command for the packaged example is given below: ``` -docker run \ - -e BOOTSTRAP_SERVERS=${HOST_IP}:5064 \ - -e ZOOKEEPER_CONNECT=${HOST_IP}:2181 \ +$ docker run \ + -e BOOTSTRAP_SERVERS="${NALMS_KAFKA_BOOTSTRAP}" \ + -e ZOOKEEPER_CONNECT="${NALMS_ZOOKEEPER_HOST}:${NALMS_ZOOKEEPER_PORT}" \ + -e CRUISE_CONTROL_PROPERTIES="/opt/cruise-control/config/cruisecontrol.properties" \ + -v "${NALMS_CRUISE_CONTROL_PROPERTIES}:/opt/cruise-control/config/cruisecontrol.properties" \ --name nalms_cruise_control \ - -p "9090:9090" -d jgarrahan/nalms-cruise-control:latest + -p "$NALMS_CRUISE_CONTROL_PORT:9090" -d jgarrahan/nalms-cruise-control:latest ``` The Cruise Control UI is then available in browser at localhost:9090. \ No newline at end of file diff --git a/elasticsearch/Dockerfile b/elasticsearch/Dockerfile index 10ea0ac..7f9beb2 100644 --- a/elasticsearch/Dockerfile +++ b/elasticsearch/Dockerfile @@ -1,8 +1,6 @@ # Use the elasticsearch image (centos7) - FROM docker.elastic.co/elasticsearch/elasticsearch:6.8.16@sha256:37994c16cea5a3c89195e77c63967f2910747bef18b272c3229da40afdc68f2c -COPY config config COPY scripts/init_es.sh /opt/nalms/ COPY scripts/create_alarm_template.sh /opt/nalms/ RUN chmod +x /opt/nalms/init_es.sh && \ diff --git a/elasticsearch/files/elasticsearch.repo b/elasticsearch/files/elasticsearch.repo deleted file mode 100644 index 82b8e47..0000000 --- a/elasticsearch/files/elasticsearch.repo +++ /dev/null @@ -1,8 +0,0 @@ -[elasticsearch-6.x] -name=Elasticsearch repository for 6.x packages -baseurl=https://artifacts.elastic.co/packages/6.x/yum -gpgcheck=1 -gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch -enabled=1 -autorefresh=1 -type=rpm-md \ No newline at end of file diff --git a/phoebus-alarm-server/server.properties b/examples/demo/config/alarm_server.properties similarity index 100% rename from phoebus-alarm-server/server.properties rename to examples/demo/config/alarm_server.properties diff --git a/examples/demo/config/client.properties b/examples/demo/config/client.properties new file mode 100644 index 0000000..a89c9c6 --- /dev/null +++ b/examples/demo/config/client.properties @@ -0,0 +1,13 @@ +# -------------------------------------- +# Package org.phoebus.applications.alarm +# -------------------------------------- +org.phoebus.applications.alarm/config_names=$CONFIG_NAME +org.phoebus.applications.alarm/server=$KAFKA_HOST:$KAFKA_PORT + +# ------------------------------------------------- +# Package org.phoebus.applications.alarm.logging.ui +# ------------------------------------------------- +org.phoebus.applications.alarm.logging.ui/es_host=$ES_HOST +org.phoebus.applications.alarm.logging.ui/es_port=$ES_PORT +org.phoebus.applications.alarm.logging.ui/es_index=$ES_INDEX + diff --git a/cruise-control/cruisecontrol.properties b/examples/demo/config/cruisecontrol.properties similarity index 100% rename from cruise-control/cruisecontrol.properties rename to examples/demo/config/cruisecontrol.properties diff --git a/elasticsearch/config/elasticsearch.keystore b/examples/demo/config/elasticsearch/elasticsearch.keystore similarity index 100% rename from elasticsearch/config/elasticsearch.keystore rename to examples/demo/config/elasticsearch/elasticsearch.keystore diff --git a/elasticsearch/config/elasticsearch.yml b/examples/demo/config/elasticsearch/elasticsearch.yml similarity index 100% rename from elasticsearch/config/elasticsearch.yml rename to examples/demo/config/elasticsearch/elasticsearch.yml diff --git a/elasticsearch/config/jvm.options b/examples/demo/config/elasticsearch/jvm.options similarity index 100% rename from elasticsearch/config/jvm.options rename to examples/demo/config/elasticsearch/jvm.options diff --git a/elasticsearch/config/log4j2.properties b/examples/demo/config/elasticsearch/log4j2.properties similarity index 100% rename from elasticsearch/config/log4j2.properties rename to examples/demo/config/elasticsearch/log4j2.properties diff --git a/elasticsearch/config/role_mapping.yml b/examples/demo/config/elasticsearch/role_mapping.yml similarity index 100% rename from elasticsearch/config/role_mapping.yml rename to examples/demo/config/elasticsearch/role_mapping.yml diff --git a/elasticsearch/config/roles.yml b/examples/demo/config/elasticsearch/roles.yml similarity index 100% rename from elasticsearch/config/roles.yml rename to examples/demo/config/elasticsearch/roles.yml diff --git a/elasticsearch/config/users b/examples/demo/config/elasticsearch/users similarity index 100% rename from elasticsearch/config/users rename to examples/demo/config/elasticsearch/users diff --git a/elasticsearch/config/users_roles b/examples/demo/config/elasticsearch/users_roles similarity index 100% rename from elasticsearch/config/users_roles rename to examples/demo/config/elasticsearch/users_roles diff --git a/phoebus-alarm-logger/logger.properties b/examples/demo/config/logger.properties similarity index 100% rename from phoebus-alarm-logger/logger.properties rename to examples/demo/config/logger.properties diff --git a/examples/demo/config/zoo.cfg b/examples/demo/config/zoo.cfg new file mode 100644 index 0000000..7e5a263 --- /dev/null +++ b/examples/demo/config/zoo.cfg @@ -0,0 +1,28 @@ +# The number of milliseconds of each tick +tickTime=2000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=5 +# the directory where the snapshot is stored. +# do not use /tmp for storage, /tmp here is just +# example sakes. +dataDir=/opt/apache-zookeeper-3.5.9-bin/data +# the port at which the clients will connect +clientPort=2181 +# the maximum number of client connections. +# increase this if you need to handle more clients +#maxClientCnxns=60 +# +# Be sure to read the maintenance section of the +# administrator guide before turning on autopurge. +# +# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance +# +# The number of snapshots to retain in dataDir +autopurge.snapRetainCount=3 +# Purge task interval in hours +# Set to "0" to disable auto purge feature +autopurge.purgeInterval=1 \ No newline at end of file diff --git a/examples/demo/demo.env b/examples/demo/demo.env index 81adc27..1a3d5d7 100644 --- a/examples/demo/demo.env +++ b/examples/demo/demo.env @@ -7,6 +7,7 @@ export EPICS_CA_REPEATER_PORT=5065 export EPICS_CA_SERVER_PORT=5064 export NALMS_KAFKA_PROPERTIES=${DEMO_DIR}/config/server.properties +export NALMS_CRUISE_CONTROL_PROPERTIES=${DEMO_DIR}/config/server.properties export NALMS_ZOOKEEPER_PORT=2181 export NALMS_ES_PORT=9200 diff --git a/examples/heart_of_gold/config/alarm_server.properties b/examples/heart_of_gold/config/alarm_server.properties new file mode 100644 index 0000000..35c0016 --- /dev/null +++ b/examples/heart_of_gold/config/alarm_server.properties @@ -0,0 +1,11 @@ +# ------------------------- +# Package org.phoebus.pv.ca +# ------------------------- +org.phoebus.pv.ca/auto_addr_list=yes +org.phoebus.pv.ca/addr_list=$EPICS_CA_ADDR_LIST +org.phoebus.pv.ca/server_port=$EPICS_CA_SERVER_PORT +org.phoebus.pv.ca/repeater_port=$EPICS_CA_REPEATER_PORT + + +# add kafka server +org.phoebus.applications.alarm/server=$KAFKA_BOOTSTRAP diff --git a/examples/heart_of_gold/config/cruisecontrol.properties b/examples/heart_of_gold/config/cruisecontrol.properties new file mode 100644 index 0000000..70f96f3 --- /dev/null +++ b/examples/heart_of_gold/config/cruisecontrol.properties @@ -0,0 +1,360 @@ +# +# Copyright 2017 LinkedIn Corp. Licensed under the BSD 2-Clause License (the "License"). See License in the project root for license information. +# + +# This is an example property file for Kafka Cruise Control. See com.linkedin.kafka.cruisecontrol.config.constants for more details. + +# Configuration for the metadata client. +# ======================================= + +# The Kafka cluster to control. +bootstrap.servers=$BOOTSTRAP_SERVERS + +# The maximum interval in milliseconds between two metadata refreshes. +#metadata.max.age.ms=300000 + +# Client id for the Cruise Control. It is used for the metadata client. +#client.id=kafka-cruise-control + +# The size of TCP send buffer bytes for the metadata client. +#send.buffer.bytes=131072 + +# The size of TCP receive buffer size for the metadata client. +#receive.buffer.bytes=131072 + +# The time to wait before disconnect an idle TCP connection. +#connections.max.idle.ms=540000 + +# The time to wait before reconnect to a given host. +#reconnect.backoff.ms=50 + +# The time to wait for a response from a host after sending a request. +#request.timeout.ms=30000 + +# The time to wait for broker logdir to respond after sending a request. +#logdir.response.timeout.ms=10000 + +# Configurations for the load monitor +# ======================================= + +# The number of metric fetcher thread to fetch metrics for the Kafka cluster +num.metric.fetchers=1 + +# The metric sampler class +metric.sampler.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.CruiseControlMetricsReporterSampler + +# True if the sampling process allows CPU capacity estimation of brokers used for CPU utilization estimation. +sampling.allow.cpu.capacity.estimation=true + +# Configurations for CruiseControlMetricsReporterSampler +metric.reporter.topic=__CruiseControlMetrics + +# The sample store class name +sample.store.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.KafkaSampleStore + +# The config for the Kafka sample store to save the partition metric samples +partition.metric.sample.store.topic=__KafkaCruiseControlPartitionMetricSamples + +# The config for the Kafka sample store to save the model training samples +broker.metric.sample.store.topic=__KafkaCruiseControlModelTrainingSamples + +# The replication factor of Kafka metric sample store topic +sample.store.topic.replication.factor=1 + +# The config for the number of Kafka sample store consumer threads +num.sample.loading.threads=8 + +# The partition assignor class for the metric samplers +metric.sampler.partition.assignor.class=com.linkedin.kafka.cruisecontrol.monitor.sampling.DefaultMetricSamplerPartitionAssignor + +# The metric sampling interval in milliseconds +metric.sampling.interval.ms=120000 + +# The partition metrics window size in milliseconds +partition.metrics.window.ms=300000 + +# The number of partition metric windows to keep in memory. Partition-load-history = num.partition.metrics.windows * partition.metrics.window.ms +num.partition.metrics.windows=5 + +# The minimum partition metric samples required for a partition in each window +min.samples.per.partition.metrics.window=1 + +# The broker metrics window size in milliseconds +broker.metrics.window.ms=300000 + +# The number of broker metric windows to keep in memory. Broker-load-history = num.broker.metrics.windows * broker.metrics.window.ms +num.broker.metrics.windows=20 + +# The minimum broker metric samples required for a partition in each window +min.samples.per.broker.metrics.window=1 + +# The configuration for the BrokerCapacityConfigFileResolver (supports JBOD, non-JBOD, and heterogeneous CPU core capacities) +#capacity.config.file=config/capacity.json +capacity.config.file=config/capacityJBOD.json + +# Configurations for the analyzer +# ======================================= + +# The list of goals to optimize the Kafka cluster for with pre-computed proposals -- consider using RackAwareDistributionGoal instead of RackAwareGoal in clusters with partitions whose replication factor > number of racks +default.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal + +# The list of supported goals +goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PotentialNwOutGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.TopicReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderReplicaDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.LeaderBytesInDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.kafkaassigner.KafkaAssignerDiskUsageDistributionGoal,com.linkedin.kafka.cruisecontrol.analyzer.kafkaassigner.KafkaAssignerEvenRackAwareGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.PreferredLeaderElectionGoal + +# The list of supported intra-broker goals +intra.broker.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.IntraBrokerDiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.IntraBrokerDiskUsageDistributionGoal + +# The list of supported hard goals -- consider using RackAwareDistributionGoal instead of RackAwareGoal in clusters with partitions whose replication factor > number of racks +hard.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + +# The minimum percentage of well monitored partitions out of all the partitions +min.valid.partition.ratio=0.95 + +# The balance threshold for CPU +cpu.balance.threshold=1.1 + +# The balance threshold for disk +disk.balance.threshold=1.1 + +# The balance threshold for network inbound utilization +network.inbound.balance.threshold=1.1 + +# The balance threshold for network outbound utilization +network.outbound.balance.threshold=1.1 + +# The balance threshold for the replica count +replica.count.balance.threshold=1.1 + +# The capacity threshold for CPU in percentage +cpu.capacity.threshold=0.7 + +# The capacity threshold for disk in percentage +disk.capacity.threshold=0.8 + +# The capacity threshold for network inbound utilization in percentage +network.inbound.capacity.threshold=0.8 + +# The capacity threshold for network outbound utilization in percentage +network.outbound.capacity.threshold=0.8 + +# The threshold to define the cluster to be in a low CPU utilization state +cpu.low.utilization.threshold=0.0 + +# The threshold to define the cluster to be in a low disk utilization state +disk.low.utilization.threshold=0.0 + +# The threshold to define the cluster to be in a low network inbound utilization state +network.inbound.low.utilization.threshold=0.0 + +# The threshold to define the cluster to be in a low network outbound utilization state +network.outbound.low.utilization.threshold=0.0 + +# The metric anomaly percentile upper threshold +metric.anomaly.percentile.upper.threshold=90.0 + +# The metric anomaly percentile lower threshold +metric.anomaly.percentile.lower.threshold=10.0 + +# How often should the cached proposal be expired and recalculated if necessary +proposal.expiration.ms=60000 + +# The maximum number of replicas that can reside on a broker at any given time. +max.replicas.per.broker=10000 + +# The number of threads to use for proposal candidate precomputing. +num.proposal.precompute.threads=1 + +# the topics that should be excluded from the partition movement. +#topics.excluded.from.partition.movement + +# The impact of having one level higher goal priority on the relative balancedness score. +#goal.balancedness.priority.weight + +# The impact of strictness on the relative balancedness score. +#goal.balancedness.strictness.weight + +# Configurations for the executor +# ======================================= + +# The zookeeper connect of the Kafka cluster +zookeeper.connect=$ZOOKEEPER_CONNECT + +# If true, appropriate zookeeper Client { .. } entry required in jaas file located at $base_dir/config/cruise_control_jaas.conf +zookeeper.security.enabled=false + +# The max number of partitions to move in/out on a given broker at a given time. +num.concurrent.partition.movements.per.broker=10 + +# The max number of partitions to move between disks within a given broker at a given time. +num.concurrent.intra.broker.partition.movements=2 + +# The max number of leadership movement within the whole cluster at a given time. +num.concurrent.leader.movements=1000 + +# Default replica movement throttle. If not specified, movements unthrottled by default. +# default.replication.throttle= + +# The interval between two execution progress checks. +execution.progress.check.interval.ms=10000 + + +# Configurations for anomaly detector +# ======================================= + +# The goal violation notifier class +anomaly.notifier.class=com.linkedin.kafka.cruisecontrol.detector.notifier.SelfHealingNotifier + +# The metric anomaly finder class +metric.anomaly.finder.class=com.linkedin.kafka.cruisecontrol.detector.KafkaMetricAnomalyFinder + +# The anomaly detection interval +#anomaly.detection.interval.ms=10000 + +# The goal violation to detect -- consider using RackAwareDistributionGoal instead of RackAwareGoal in clusters with partitions whose replication factor > number of racks +anomaly.detection.goals=com.linkedin.kafka.cruisecontrol.analyzer.goals.RackAwareGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.ReplicaCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.DiskCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkInboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.NetworkOutboundCapacityGoal,com.linkedin.kafka.cruisecontrol.analyzer.goals.CpuCapacityGoal + +# The interested metrics for metric anomaly analyzer. +metric.anomaly.analyzer.metrics=BROKER_PRODUCE_LOCAL_TIME_MS_50TH,BROKER_PRODUCE_LOCAL_TIME_MS_999TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_50TH,BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_999TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_50TH,BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_999TH,BROKER_LOG_FLUSH_TIME_MS_50TH,BROKER_LOG_FLUSH_TIME_MS_999TH + +# True if recently demoted brokers are excluded from optimizations during self healing, false otherwise +self.healing.exclude.recently.demoted.brokers=true + +# True if recently removed brokers are excluded from optimizations during self healing, false otherwise +self.healing.exclude.recently.removed.brokers=true + +# The zk path to store failed broker information. +failed.brokers.zk.path=/CruiseControlBrokerList + +# Topic config provider class +topic.config.provider.class=com.linkedin.kafka.cruisecontrol.config.KafkaTopicConfigProvider + +# The cluster configurations for the KafkaTopicConfigProvider +cluster.configs.file=config/clusterConfigs.json + +# The maximum time in milliseconds to store the response and access details of a completed kafka monitoring user task. +completed.kafka.monitor.user.task.retention.time.ms=86400000 + +# The maximum time in milliseconds to store the response and access details of a completed cruise control monitoring user task. +completed.cruise.control.monitor.user.task.retention.time.ms=86400000 + +# The maximum time in milliseconds to store the response and access details of a completed kafka admin user task. +completed.kafka.admin.user.task.retention.time.ms=604800000 + +# The maximum time in milliseconds to store the response and access details of a completed cruise control admin user task. +completed.cruise.control.admin.user.task.retention.time.ms=604800000 + +# The fallback maximum time in milliseconds to store the response and access details of a completed user task. +completed.user.task.retention.time.ms=86400000 + +# The maximum time in milliseconds to retain the demotion history of brokers. +demotion.history.retention.time.ms=1209600000 + +# The maximum time in milliseconds to retain the removal history of brokers. +removal.history.retention.time.ms=1209600000 + +# The maximum number of completed kafka monitoring user tasks for which the response and access details will be cached. +max.cached.completed.kafka.monitor.user.tasks=20 + +# The maximum number of completed cruise control monitoring user tasks for which the response and access details will be cached. +max.cached.completed.cruise.control.monitor.user.tasks=20 + +# The maximum number of completed kafka admin user tasks for which the response and access details will be cached. +max.cached.completed.kafka.admin.user.tasks=30 + +# The maximum number of completed cruise control admin user tasks for which the response and access details will be cached. +max.cached.completed.cruise.control.admin.user.tasks=30 + +# The fallback maximum number of completed user tasks of certain type for which the response and access details will be cached. +max.cached.completed.user.tasks=25 + +# The maximum number of user tasks for concurrently running in async endpoints across all users. +max.active.user.tasks=5 + +# Enable self healing for all anomaly detectors, unless the particular anomaly detector is explicitly disabled +self.healing.enabled=false + +# Enable self healing for broker failure detector +#self.healing.broker.failure.enabled=true + +# Enable self healing for goal violation detector +#self.healing.goal.violation.enabled=true + +# Enable self healing for metric anomaly detector +#self.healing.metric.anomaly.enabled=true + +# Enable self healing for disk failure detector +#self.healing.disk.failure.enabled=true + +# Enable self healing for topic anomaly detector +#self.healing.topic.anomaly.enabled=true +#topic.anomaly.finder.class=com.linkedin.kafka.cruisecontrol.detector.TopicReplicationFactorAnomalyFinder + +# Enable self healing for maintenance event detector +#self.healing.maintenance.event.enabled=true + +# The multiplier applied to the threshold of distribution goals used by goal.violation.detector. +#goal.violation.distribution.threshold.multiplier=2.50 + +# configurations for the webserver +# ================================ + +# HTTP listen port +#webserver.http.port=9090 + +# HTTP listen address +webserver.http.address=0.0.0.0 + +# Whether CORS support is enabled for API or not +webserver.http.cors.enabled=false + +# Value for Access-Control-Allow-Origin +webserver.http.cors.origin=http://localhost:*/ + +# Value for Access-Control-Request-Method +webserver.http.cors.allowmethods=OPTIONS,GET,POST + +# Headers that should be exposed to the Browser (Webapp) +# This is a special header that is used by the +# User Tasks subsystem and should be explicitly +# Enabled when CORS mode is used as part of the +# Admin Interface +webserver.http.cors.exposeheaders=User-Task-ID + +# REST API default prefix (dont forget the ending /*) +webserver.api.urlprefix=/kafkacruisecontrol/* + +# Location where the Cruise Control frontend is deployed +webserver.ui.diskpath=./cruise-control-ui/dist/ + +# URL path prefix for UI (dont forget the ending /*) +webserver.ui.urlprefix=/* + +# Time After which request is converted to Async +webserver.request.maxBlockTimeMs=10000 + +# Default Session Expiry Period +webserver.session.maxExpiryTimeMs=60000 + +# Session cookie path +webserver.session.path=/ + +# Server Access Logs +webserver.accesslog.enabled=true + +# Location of HTTP Request Logs +webserver.accesslog.path=access.log + +# HTTP Request Log retention days +webserver.accesslog.retention.days=14 + +# Configurations for servlet +# ========================== + +# Enable two-step verification for processing POST requests. +two.step.verification.enabled=false + +# The maximum time in milliseconds to retain the requests in two-step (verification) purgatory. +two.step.purgatory.retention.time.ms=1209600000 + +# The maximum number of requests in two-step (verification) purgatory. +two.step.purgatory.max.requests=25 \ No newline at end of file diff --git a/examples/heart_of_gold/config/dashboards/HeartOfGold.json b/examples/heart_of_gold/config/dashboards/HeartOfGold.json new file mode 100644 index 0000000..8e5b706 --- /dev/null +++ b/examples/heart_of_gold/config/dashboards/HeartOfGold.json @@ -0,0 +1,370 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "iteration": 1627245062536, + "links": [], + "panels": [ + { + "datasource": "HeartOfGold", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 23, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "message_time", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "_type: alarm_config", + "queryType": "randomWalk", + "refId": "A", + "timeField": "message_time" + }, + { + "alias": "", + "bucketAggs": [ + { + "field": "message_time", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "hide": false, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "_type: alarm", + "refId": "B", + "timeField": "message_time" + } + ], + "title": "Alarm Events", + "type": "timeseries" + }, + { + "alignNumbersToRightEnabled": true, + "columnAliases": [ + { + "$$hashKey": "object:192", + "alias": "path", + "name": "config" + } + ], + "columnFiltersEnabled": false, + "columnWidthHints": [], + "columns": [], + "compactRowsEnabled": false, + "datasource": "HeartOfGold", + "datatablePagingType": "simple_numbers", + "datatableTheme": "basic_theme", + "emptyData": false, + "fontSize": "100%", + "gridPos": { + "h": 14, + "w": 23, + "x": 0, + "y": 8 + }, + "hoverEnabled": true, + "id": 2, + "infoEnabled": true, + "lengthChangeEnabled": true, + "orderColumnEnabled": true, + "pagingTypes": [ + { + "$$hashKey": "object:92", + "text": "Page number buttons only", + "value": "numbers" + }, + { + "$$hashKey": "object:93", + "text": "'Previous' and 'Next' buttons only", + "value": "simple" + }, + { + "$$hashKey": "object:94", + "text": "'Previous' and 'Next' buttons, plus page numbers", + "value": "simple_numbers" + }, + { + "$$hashKey": "object:95", + "text": "'First', 'Previous', 'Next' and 'Last' buttons", + "value": "full" + }, + { + "$$hashKey": "object:96", + "text": "'First', 'Previous', 'Next' and 'Last' buttons, plus page numbers", + "value": "full_numbers" + }, + { + "$$hashKey": "object:97", + "text": "'First' and 'Last' buttons, plus page numbers", + "value": "first_last_numbers" + } + ], + "panelHeight": 388, + "rowNumbersEnabled": false, + "rowsPerPage": 10, + "scroll": false, + "scrollHeight": "default", + "searchEnabled": true, + "searchHighlightingEnabled": false, + "showCellBorders": false, + "showHeader": true, + "showRowBorders": true, + "sort": { + "col": 0, + "desc": true + }, + "sortByColumns": [ + { + "$$hashKey": "object:30", + "columnData": 0, + "sortMethod": "desc" + } + ], + "sortByColumnsData": [ + [ + 0, + "desc" + ] + ], + "stripedRowsEnabled": true, + "styles": [ + { + "$$hashKey": "object:32", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "$$hashKey": "object:33", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "splitPattern": "/ /", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "bucketAggs": [], + "metrics": [ + { + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "", + "queryType": "randomWalk", + "refId": "A", + "timeField": "message_time" + } + ], + "themeOptions": { + "dark": "./styles/dark.scss", + "light": "./styles/light.scss" + }, + "themes": [ + { + "$$hashKey": "object:67", + "disabled": false, + "text": "Basic", + "value": "basic_theme" + }, + { + "$$hashKey": "object:68", + "disabled": true, + "text": "Bootstrap", + "value": "bootstrap_theme" + }, + { + "$$hashKey": "object:69", + "disabled": true, + "text": "Foundation", + "value": "foundation_theme" + }, + { + "$$hashKey": "object:70", + "disabled": true, + "text": "ThemeRoller", + "value": "themeroller_theme" + } + ], + "title": "Alarm Event Logs", + "transform": "table", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "message_time", + "_type", + "config", + "current_message", + "current_severity", + "enabled", + "host", + "latch", + "pv", + "user", + "value" + ] + } + } + } + ], + "type": "briangann-datatable-panel" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "alarm", + "value": "alarm" + }, + "datasource": "HeartOfGold", + "definition": "{\"find\": \"terms\", \"field\": \"_type\"}", + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "event_type", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"_type\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "HeartOfGold", + "uid": "ZDFkpxW7k", + "version": 6 +} diff --git a/examples/heart_of_gold/config/datasource.yml b/examples/heart_of_gold/config/datasource.yml new file mode 100644 index 0000000..3c0d377 --- /dev/null +++ b/examples/heart_of_gold/config/datasource.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +datasources: + - name: HeartOfGold + type: elasticsearch + access: proxy + database: "heartofgold_*" + url: http://elasticsearch:9200 + jsonData: + esVersion: 60 + timeField: "message_time" + logMessageField: "id" diff --git a/examples/heart_of_gold/config/elasticsearch/elasticsearch.keystore b/examples/heart_of_gold/config/elasticsearch/elasticsearch.keystore new file mode 100644 index 0000000..a375108 Binary files /dev/null and b/examples/heart_of_gold/config/elasticsearch/elasticsearch.keystore differ diff --git a/examples/heart_of_gold/config/elasticsearch/elasticsearch.yml b/examples/heart_of_gold/config/elasticsearch/elasticsearch.yml new file mode 100644 index 0000000..43e858e --- /dev/null +++ b/examples/heart_of_gold/config/elasticsearch/elasticsearch.yml @@ -0,0 +1,2 @@ +cluster.name: "docker-cluster" +network.host: 0.0.0.0 \ No newline at end of file diff --git a/examples/heart_of_gold/config/elasticsearch/jvm.options b/examples/heart_of_gold/config/elasticsearch/jvm.options new file mode 100644 index 0000000..daaaf50 --- /dev/null +++ b/examples/heart_of_gold/config/elasticsearch/jvm.options @@ -0,0 +1,128 @@ +## JVM configuration + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## You should always set the min and max JVM heap +## size to the same value. For example, to set +## the heap to 4 GB, set: +## +## -Xms4g +## -Xmx4g +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html +## for more information +## +################################################################ + +# Xms represents the initial size of total heap space +# Xmx represents the maximum size of total heap space + +-Xms1g +-Xmx1g + +################################################################ +## Expert settings +################################################################ +## +## All settings below this section are considered +## expert settings. Don't tamper with them unless +## you understand what you are doing +## +################################################################ + +## GC configuration +8-13:-XX:+UseConcMarkSweepGC +8-13:-XX:CMSInitiatingOccupancyFraction=75 +8-13:-XX:+UseCMSInitiatingOccupancyOnly + +## G1GC Configuration +# NOTE: G1 GC is only supported on JDK version 10 or later +# to use G1GC, uncomment the next two lines and update the version on the +# following three lines to your version of the JDK +# 10-13:-XX:-UseConcMarkSweepGC +# 10-13:-XX:-UseCMSInitiatingOccupancyOnly +14-:-XX:+UseG1GC +14-:-XX:G1ReservePercent=25 +14-:-XX:InitiatingHeapOccupancyPercent=30 + +## DNS cache policy +# cache ttl in seconds for positive DNS lookups noting that this overrides the +# JDK security property networkaddress.cache.ttl; set to -1 to cache forever +-Des.networkaddress.cache.ttl=60 +# cache ttl in seconds for negative DNS lookups noting that this overrides the +# JDK security property networkaddress.cache.negative ttl; set to -1 to cache +# forever +-Des.networkaddress.cache.negative.ttl=10 + +## optimizations + +# pre-touch memory pages used by the JVM during initialization +-XX:+AlwaysPreTouch + +## basic + +# explicitly set the stack size +-Xss1m + +# set to headless, just in case +-Djava.awt.headless=true + +# ensure UTF-8 encoding by default (e.g. filenames) +-Dfile.encoding=UTF-8 + +# use our provided JNA always versus the system one +-Djna.nosys=true + +# turn off a JDK optimization that throws away stack traces for common +# exceptions because stack traces are important for debugging +-XX:-OmitStackTraceInFastThrow + +# enable helpful NullPointerExceptions (https://openjdk.java.net/jeps/358), if +# they are supported +14-:-XX:+ShowCodeDetailsInExceptionMessages + +# flags to configure Netty +-Dio.netty.noUnsafe=true +-Dio.netty.noKeySetOptimization=true +-Dio.netty.recycler.maxCapacityPerThread=0 + +# log4j 2 +-Dlog4j.shutdownHookEnabled=false +-Dlog4j2.disable.jmx=true + +-Djava.io.tmpdir=${ES_TMPDIR} + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails +# heap dumps are created in the working directory of the JVM +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps; ensure the directory exists and +# has sufficient space +-XX:HeapDumpPath=data + +# specify an alternative path for JVM fatal error logs +-XX:ErrorFile=logs/hs_err_pid%p.log + +## JDK 8 GC logging + +8:-XX:+PrintGCDetails +8:-XX:+PrintGCDateStamps +8:-XX:+PrintTenuringDistribution +8:-XX:+PrintGCApplicationStoppedTime +8:-Xloggc:logs/gc.log +8:-XX:+UseGCLogFileRotation +8:-XX:NumberOfGCLogFiles=32 +8:-XX:GCLogFileSize=64m + +# JDK 9+ GC logging +9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m +# due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise +# time/date parsing will break in an incompatible way for some date patterns and locals +9-:-Djava.locale.providers=COMPAT + +# temporary workaround for C2 bug with JDK 10 on hardware with AVX-512 +10-:-XX:UseAVX=2 diff --git a/examples/heart_of_gold/config/elasticsearch/log4j2.properties b/examples/heart_of_gold/config/elasticsearch/log4j2.properties new file mode 100644 index 0000000..1fcce23 --- /dev/null +++ b/examples/heart_of_gold/config/elasticsearch/log4j2.properties @@ -0,0 +1,9 @@ +status = error + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console \ No newline at end of file diff --git a/examples/heart_of_gold/config/elasticsearch/role_mapping.yml b/examples/heart_of_gold/config/elasticsearch/role_mapping.yml new file mode 100644 index 0000000..68c82f7 --- /dev/null +++ b/examples/heart_of_gold/config/elasticsearch/role_mapping.yml @@ -0,0 +1,14 @@ +# Role mapping configuration file which has elasticsearch roles as keys +# that map to one or more user or group distinguished names + +#roleA: this is an elasticsearch role +# - groupA-DN this is a group distinguished name +# - groupB-DN +# - user1-DN this is the full user distinguished name + +#power_user: +# - "cn=admins,dc=example,dc=com" +#user: +# - "cn=users,dc=example,dc=com" +# - "cn=admins,dc=example,dc=com" +# - "cn=John Doe,cn=other users,dc=example,dc=com" diff --git a/examples/heart_of_gold/config/elasticsearch/roles.yml b/examples/heart_of_gold/config/elasticsearch/roles.yml new file mode 100644 index 0000000..68e003b --- /dev/null +++ b/examples/heart_of_gold/config/elasticsearch/roles.yml @@ -0,0 +1,3 @@ +# The default roles file is empty as the preferred method of defining roles is +# through the API/UI. File based roles are useful in error scenarios when the +# API based roles may not be available. diff --git a/config/.tmux.conf b/examples/heart_of_gold/config/elasticsearch/users similarity index 100% rename from config/.tmux.conf rename to examples/heart_of_gold/config/elasticsearch/users diff --git a/examples/heart_of_gold/config/elasticsearch/users_roles b/examples/heart_of_gold/config/elasticsearch/users_roles new file mode 100644 index 0000000..e69de29 diff --git a/grafana/config.ini b/examples/heart_of_gold/config/grafana.ini similarity index 100% rename from grafana/config.ini rename to examples/heart_of_gold/config/grafana.ini diff --git a/examples/heart_of_gold/config/logger.properties b/examples/heart_of_gold/config/logger.properties new file mode 100644 index 0000000..bda544d --- /dev/null +++ b/examples/heart_of_gold/config/logger.properties @@ -0,0 +1,33 @@ +# Alarm topics to be logged, they can be defined as a comma separated list +# alarm_topics=Accelerator + +# location of elastic node/s +es_host=$ES_HOST +es_port=$ES_PORT +# max default size for es queries +es_max_size=1000 +# set to 'true' if sniffing to be enabled to discover other cluster nodes +es_sniff=false + +# Kafka server location +bootstrap.servers=$BOOTSTRAP_SERVERS + +# The units of the indices date span: Days (D), Weeks(W), Months(M), Years(Y). +date_span_units=M + +# The value of the indices date span. +# An integer value to be combined with the units. For example, if months were selected and the value set to 2, each index would span 2 months. +date_span_value=1 + + +# Disable the spring banner +spring.main.banner-mode=off + +# Suppress the logging from spring boot during debugging this should be set to DEBUG +logging.level.root=WARN +logging.level.org.springframework=WARN +logging.level.org.apache.catalina=WARN +logging.level.org.apache.kafka=WARN + +# Size of the thread pool for message and command loggers. Two threads per topic/configuration are required +thread_pool_size=4 diff --git a/examples/heart_of_gold/config/zoo.cfg b/examples/heart_of_gold/config/zoo.cfg new file mode 100644 index 0000000..4b34497 --- /dev/null +++ b/examples/heart_of_gold/config/zoo.cfg @@ -0,0 +1,29 @@ +# The number of milliseconds of each tick +tickTime=2000 +# The number of ticks that the initial +# synchronization phase can take +initLimit=10 +# The number of ticks that can pass between +# sending a request and getting an acknowledgement +syncLimit=5 +# the directory where the snapshot is stored. +# do not use /tmp for storage, /tmp here is just +# example sakes. +dataDir=/opt/apache-zookeeper-3.5.9-bin/data +# the port at which the clients will connect +clientPort=2181 +# the maximum number of client connections. +# increase this if you need to handle more clients +#maxClientCnxns=60 +# +# Be sure to read the maintenance section of the +# administrator guide before turning on autopurge. +# +# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance +# +# The number of snapshots to retain in dataDir +autopurge.snapRetainCount=3 +# Purge task interval in hours +# Set to "0" to disable auto purge feature +autopurge.purgeInterval=1 +~ \ No newline at end of file diff --git a/config/local_zookeeper.properties b/examples/heart_of_gold/config/zookeeper.properties similarity index 100% rename from config/local_zookeeper.properties rename to examples/heart_of_gold/config/zookeeper.properties diff --git a/examples/heart_of_gold/heart_of_gold.env b/examples/heart_of_gold/heart_of_gold.env new file mode 100644 index 0000000..f50ab90 --- /dev/null +++ b/examples/heart_of_gold/heart_of_gold.env @@ -0,0 +1,28 @@ +#!/bin/bash +export DEMO_DIR="$(cd -P "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export EPICS_CA_AUTO_ADDR_LIST=YES +export EPICS_CA_ADDR_LIST=${HOST_IP} +export EPICS_CA_REPEATER_PORT=5065 +export EPICS_CA_SERVER_PORT=5064 + +export NALMS_KAFKA_PROPERTIES=${DEMO_DIR}/config/server.properties +export NALMS_CRUISE_CONTROL_PROPERTIES=${DEMO_DIR}/config/server.properties +export NALMS_ALARM_SERVER_PROPERTIES=${DEMO_DIR}/config/alarm_server.properties +export NALMS_ALARM_LOGGER_PROPERTIES=${DEMO_DIR}/config/alarm_logger.properties +export NALMS_GRAFANA_DATASOURCE_FILE=${DEMO_DIR}/config/datasource.yml +export NALMS_GRAFANA_DASHBOARD_DIR=${DEMO_DIR}/config/dashboards +export NALMS_GRAFANA_CONFIG=${DEMO_DIR}/config/grafana.ini +export NALMS_ZOOKEEPER_CONFIG=${DEMO_DIR}/config/zoo.cfg +export NALMS_ES_CONFIG=${DEMO_DIR}/config/elasticsearch + +export NALMS_ZOOKEEPER_PORT=2181 +export NALMS_ES_PORT=9200 +export NALMS_CRUISE_CONTROL_PORT=9090 +export NALMS_KAFKA_PORT=9092 +export NALMS_GRAFANA_PORT=3000 + +export NALMS_ES_HOST=elasticsearch +export NALMS_KAFKA_BOOTSTRAP=kafka1:9092 +export NALMS_ZOOKEEPER_HOST=zookeeper +export NALMS_KAFKA_HOST=kafka1 diff --git a/grafana/Dockerfile b/grafana/Dockerfile index 711d146..4981e77 100644 --- a/grafana/Dockerfile +++ b/grafana/Dockerfile @@ -1,30 +1,16 @@ -FROM grafana/grafana +FROM grafana/grafana:7.5.10 ADD ./provisioning/dashboards /etc/grafana/provisioning/dashboards -ADD ./config.ini /etc/grafana/config.ini -COPY ./dashboards/alarm_logs_dashboard.json /opt/nalms/alarm_logs_dashboard.json - -COPY scripts/start-grafana.sh /opt/nalms/start-grafana.sh -COPY scripts/update-dashboards.sh /opt/nalms/update-dashboards.sh -COPY scripts/create-datasource-file.sh /opt/nalms/create-datasource-file.sh - USER root RUN mkdir /var/lib/grafana/dashboards && \ apk add jq curl &&\ - chmod +x /opt/nalms/start-grafana.sh && \ - chmod +x /opt/nalms/update-dashboards.sh && \ - chmod +x /opt/nalms/create-datasource-file.sh && \ grafana-cli plugins install briangann-datatable-panel -ENV DATASOURCE_DIR=/etc/grafana/provisioning/datasources -ENV DASHBOARD_TEMPLATE=/opt/nalms/alarm_logs_dashboard.json -ENV DASHBOARD_DIR=/var/lib/grafana/dashboards -ENV DASHBOARDS_DIRECTORY=/var/lib/grafana/dashboards ENV GF_AUTH_DISABLE_LOGIN_FORM "true" ENV GF_AUTH_ANONYMOUS_ENABLED "true" ENV GF_AUTH_ANONYMOUS_ORG_ROLE "Admin" - +ENV DATASOURCE_DIR=/etc/grafana/provisioning/datasources ENTRYPOINT [] -CMD /opt/nalms/start-grafana.sh \ No newline at end of file +CMD /run.sh \ No newline at end of file diff --git a/grafana/scripts/create-datasource-file.sh b/grafana/scripts/create-datasource-file.sh deleted file mode 100644 index a42fbfe..0000000 --- a/grafana/scripts/create-datasource-file.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# -# Create dynamic datasource dir for all configs in image - - -DATASOURCE_FILE="${DATASOURCE_DIR}/all.yml" -export IFS="," - -# convert to lowercase for index -CONFIG=$(echo "$CONFIG_NAME" | tr '[:upper:]' '[:lower:]') - -# Create file -echo "apiVersion: 1" >> $DATASOURCE_FILE -echo "" >> $DATASOURCE_FILE -echo "" >> $DATASOURCE_FILE -echo "datasources:" >> $DATASOURCE_FILE - - -for word in $CONFIG_NAME; do - # convert to lowercase for index - CONFIG=$(echo "$word" | tr '[:upper:]' '[:lower:]') - echo " - name: ${word}" >> $DATASOURCE_FILE - echo " type: elasticsearch" >> $DATASOURCE_FILE - echo " access: proxy" >> $DATASOURCE_FILE - echo " database: \"${CONFIG}_*\"" >> $DATASOURCE_FILE - echo " url: http://${ES_HOST}:${ES_PORT}" >> $DATASOURCE_FILE - echo " jsonData:" >> $DATASOURCE_FILE - echo " esVersion: 60" >> $DATASOURCE_FILE - echo " timeField: \"message_time\"" >> $DATASOURCE_FILE - echo " logMessageField: \"id\"" >> $DATASOURCE_FILE - -done - -echo "Completed writing file." \ No newline at end of file diff --git a/grafana/scripts/start-grafana.sh b/grafana/scripts/start-grafana.sh deleted file mode 100644 index c375906..0000000 --- a/grafana/scripts/start-grafana.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - - -# create dashboards -export IFS="," -for CONFIG_ITEM in $CONFIG_NAME; do - sed 's/$DATASOURCE_NAME/'"$CONFIG_ITEM/" $DASHBOARD_TEMPLATE > "${DASHBOARD_DIR}/${CONFIG_ITEM}.json" -done - - -# create datasource from configurations -bash /opt/nalms/create-datasource-file.sh - -# update dashboards -#(bash /opt/nalms/update-dashboards.sh) & - -#run -/run.sh \ No newline at end of file diff --git a/grafana/scripts/update-dashboards.sh b/grafana/scripts/update-dashboards.sh deleted file mode 100644 index b63407a..0000000 --- a/grafana/scripts/update-dashboards.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -# Updates local dashboard configurations by retrieving -# the new version from a Grafana instance. -# -# The script assumes that basic authentication is configured -# (change the login credentials with `LOGIN`). -# -# DASHBOARD_DIRECTORY represents the path to the directory -# where the JSON files corresponding to the dashboards exist. -# The default location is relative to the execution of the -# script. -# -# URL specifies the URL of the Grafana instance. -# - -sleep 10 - -set -o errexit - -readonly URL=${URL:-"http://localhost:3000"} -readonly LOGIN=${LOGIN:-"admin:admin"} -readonly DASHBOARDS_DIRECTORY=${DASHBOARDS_DIRECTORY:-"./grafana/dashboards"} - - -main() { - local dashboards=$(list_dashboards) - local dashboard_json - - show_config - - for dashboard in $dashboards; do - dashboard_json=$(get_dashboard "$dashboard") - - if [[ -z "$dashboard_json" ]]; then - echo "ERROR: - Couldn't retrieve dashboard $dashboard. - " - exit 1 - fi - - echo "$dashboard_json" >$DASHBOARDS_DIRECTORY/$dashboard.json - done -} - - -# Shows the global environment variables that have been configured -# for this run. -show_config() { - echo "INFO: - Starting dashboard extraction. - - URL: $URL - LOGIN: $LOGIN - DASHBOARDS_DIRECTORY: $DASHBOARDS_DIRECTORY - " -} - - -# Retrieves a dashboard from the database of dashboards. -get_dashboard() { - local dashboard=$1 - - if [[ -z "$dashboard" ]]; then - echo "ERROR: - A dashboard must be specified. - " - exit 1 - fi - - curl \ - --silent \ - --user "$LOGIN" \ - $URL/api/dashboards/db/$dashboard | - jq '.dashboard | .id = null' -} - - -# lists all the dashboards available. -# -list_dashboards() { - curl \ - --silent \ - --user "$LOGIN" \ - $URL/api/search | - jq -r '.[] | select(.type == "dash-db") | .uri' | - cut -d '/' -f2 -} - -main "$@" \ No newline at end of file diff --git a/phoebus-alarm-logger/Dockerfile b/phoebus-alarm-logger/Dockerfile index 27ccaae..4690f46 100644 --- a/phoebus-alarm-logger/Dockerfile +++ b/phoebus-alarm-logger/Dockerfile @@ -21,7 +21,6 @@ RUN yum install -y git maven ${JAVA_BASE}-devel-${JAVA_VERSION} && \ FROM centos:7 as runtime -COPY logger.properties /opt/nalms/config/alarm_logger.properties COPY logging.properties /opt/nalms/config/logging.properties COPY cli /opt/nalms/cli USER root @@ -43,7 +42,6 @@ RUN yum install -y ${JAVA_BASE}-${JAVA_VERSION} \ COPY --from=build /opt/phoebus-build /opt/phoebus # set environment -ENV ALARM_LOGGER_PROPERTIES /opt/nalms/config/alarm_logger.properties ENV ALARM_LOGGER_JAR /opt/phoebus/service-alarm-logger.jar ENV LOGGING_CONFIG_FILE /opt/nalms/config/logging.properties diff --git a/phoebus-alarm-logger/cli/commands/start-logger b/phoebus-alarm-logger/cli/commands/start-logger index 7bb7370..6b074bc 100644 --- a/phoebus-alarm-logger/cli/commands/start-logger +++ b/phoebus-alarm-logger/cli/commands/start-logger @@ -45,9 +45,9 @@ fi CONFIG_NAME=$1 CONFIG_FILE=$2 -sed -i 's/$ES_HOST/'"$ES_HOST/" $ALARM_LOGGER_PROPERTIES -sed -i 's/$ES_PORT/'"$ES_PORT/" $ALARM_LOGGER_PROPERTIES -sed -i 's/$BOOTSTRAP_SERVERS/'"$BOOTSTRAP_SERVERS/" $ALARM_LOGGER_PROPERTIES +sed 's/$ES_HOST/'"$ES_HOST/" $ALARM_LOGGER_PROPERTIES > /tmp/logger.properties +sed -i 's/$ES_PORT/'"$ES_PORT/" /tmp/logger.properties +sed -i 's/$BOOTSTRAP_SERVERS/'"$BOOTSTRAP_SERVERS/" /tmp/logger.properties -java -jar $ALARM_LOGGER_JAR -properties $ALARM_LOGGER_PROPERTIES -topics $CONFIG_NAME -noshell +java -jar $ALARM_LOGGER_JAR -properties /tmp/logger.properties -topics $CONFIG_NAME -noshell diff --git a/phoebus-alarm-server/Dockerfile b/phoebus-alarm-server/Dockerfile index 0dd4e2b..c3a7c2f 100644 --- a/phoebus-alarm-server/Dockerfile +++ b/phoebus-alarm-server/Dockerfile @@ -25,7 +25,6 @@ FROM centos:7 as runtime COPY cli /opt/nalms/cli COPY requirements.txt /opt/nalms/requirements.txt COPY scripts /opt/nalms/scripts -COPY server.properties /opt/nalms/config/alarm_server.properties COPY logging.properties /opt/nalms/config/logging.properties USER root @@ -60,7 +59,6 @@ RUN yum install -y wget ${JAVA_BASE}-${JAVA_VERSION} readline libsdc++ && \ COPY --from=build /opt/phoebus-build /opt/phoebus # set environment -ENV ALARM_SERVER_PROPERTIES /opt/nalms/config/alarm_server.properties ENV ALARM_SERVER_JAR /opt/phoebus/service-alarm-server.jar ENV LOGGING_CONFIG_FILE /opt/nalms/config/logging.properties diff --git a/phoebus-alarm-server/cli/commands/start-server b/phoebus-alarm-server/cli/commands/start-server index 68115f6..a4074fa 100644 --- a/phoebus-alarm-server/cli/commands/start-server +++ b/phoebus-alarm-server/cli/commands/start-server @@ -36,11 +36,10 @@ fi export PATH="$JAVA_HOME/bin:$PATH" -sed -i 's/$KAFKA_BOOTSTRAP/'"$KAFKA_BOOTSTRAP/" $ALARM_SERVER_PROPERTIES -sed -i 's/$EPICS_CA_ADDR_LIST/'"$EPICS_CA_ADDR_LIST/" $ALARM_SERVER_PROPERTIES -sed -i 's/$EPICS_CA_REPEATER_PORT/'"$EPICS_CA_REPEATER_PORT/" $ALARM_SERVER_PROPERTIES -sed -i 's/$EPICS_CA_SERVER_PORT/'"$EPICS_CA_SERVER_PORT/" $ALARM_SERVER_PROPERTIES - +sed 's/$KAFKA_BOOTSTRAP/'"$KAFKA_BOOTSTRAP/" $ALARM_SERVER_PROPERTIES > /tmp/alarm_server.properties +sed -i 's/$EPICS_CA_ADDR_LIST/'"$EPICS_CA_ADDR_LIST/" /tmp/alarm_server.properties +sed -i 's/$EPICS_CA_REPEATER_PORT/'"$EPICS_CA_REPEATER_PORT/" /tmp/alarm_server.properties +sed -i 's/$EPICS_CA_SERVER_PORT/'"$EPICS_CA_SERVER_PORT/" /tmp/alarm_server.properties CONFIG_NAME=$1 CONFIG_FILE=$2 @@ -48,9 +47,9 @@ if [[ "$ALARM_IOC" = true ]]; then python3 /opt/nalms/scripts/update_ioc.py $CONFIG_NAME $KAFKA_BOOTSTRAP & fi -if java -jar $ALARM_SERVER_JAR -settings $ALARM_SERVER_PROPERTIES -config $CONFIG_NAME -import $CONFIG_FILE; then +if java -jar $ALARM_SERVER_JAR -settings /tmp/alarm_server.properties -config $CONFIG_NAME -import $CONFIG_FILE; then - java -jar $ALARM_SERVER_JAR -config $CONFIG_NAME -settings $ALARM_SERVER_PROPERTIES -noshell -logging $LOGGING_CONFIG_FILE + java -jar $ALARM_SERVER_JAR -config $CONFIG_NAME -settings /tmp/alarm_server.properties -noshell -logging $LOGGING_CONFIG_FILE else echo "Unable to import configuration." diff --git a/zookeeper/Dockerfile b/zookeeper/Dockerfile index 7bd2e6b..79a4b14 100644 --- a/zookeeper/Dockerfile +++ b/zookeeper/Dockerfile @@ -1,7 +1,5 @@ FROM centos:7 - - ENV JAVA_VERSION 11.0.12.0.7-0.el7_9.x86_64 ENV JAVA_BASE java-11-openjdk ENV JAVA_HOME /usr/lib/jvm/${JAVA_BASE}-${JAVA_VERSION} @@ -16,16 +14,13 @@ RUN yum install -y wget openssh-server ${JAVA_BASE}-${JAVA_VERSION} && \ #Verify download RUN gpg --import KEYS && \ - sha512sum apache-zookeeper-${ZOOKEEPER_VERSION}.tar.gz.sha512 apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz - -#Install -RUN tar -xzf apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz -C /opt - -#Configure -RUN mv /opt/apache-zookeeper-${ZOOKEEPER_VERSION}-bin/conf/zoo_sample.cfg /opt/apache-zookeeper-${ZOOKEEPER_VERSION}-bin/conf/zoo.cfg + sha512sum apache-zookeeper-${ZOOKEEPER_VERSION}.tar.gz.sha512 apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz && \ + tar -xzf apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz -C /opt ENV JAVA_HOME /usr/lib/jvm/java-7-openjdk-amd64 ENV ZK_HOME /opt/apache-zookeeper-${ZOOKEEPER_VERSION}-bin +ENV ZOOKEEPER_CONFIG_PATH /opt/apache-zookeeper-${ZOOKEEPER_VERSION}-bin/conf/zoo.cfg + RUN sed -i "s|/tmp/zookeeper|$ZK_HOME/data|g" $ZK_HOME/conf/zoo.cfg; mkdir $ZK_HOME/data COPY start-zk.sh /usr/bin/start-zk.sh diff --git a/zookeeper/start-zk.sh b/zookeeper/start-zk.sh index eff5236..86c4c66 100644 --- a/zookeeper/start-zk.sh +++ b/zookeeper/start-zk.sh @@ -1,4 +1,10 @@ +#!/bin/sh +# Author: Jacqueline Garrahan +# +# Start Zookeeper sed -i -r 's|#(log4j.appender.ROLLINGFILE.MaxBackupIndex.*)|\1|g' $ZK_HOME/conf/log4j.properties -sed -i -r 's|#autopurge|autopurge|g' $ZK_HOME/conf/zoo.cfg + +# copy mounted config to config path +cp $ZOOKEEPER_CONFIG $ZOOKEEPER_CONFIG_PATH $ZK_HOME/bin/zkServer.sh start-foreground \ No newline at end of file