diff --git a/.gitignore b/.gitignore index c06c87b4..1ae6495b 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,4 @@ creds.txt cws-test/newCWSDir .DS_Store -install/logging/logstash-6.4.2.zip +install/logging/logstash-7.9.0.zip diff --git a/NOTICE.txt b/NOTICE.txt index fc9a1e27..cb194eff 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -17,7 +17,6 @@ This software includes third party software subject to the following copyrights: - The AWS SDK for Java is Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - The Camunda BPMN Platform is Copyright 2013-2020 Camunda Services GmbH. - The Camunda Modeler is Copyright (c) Camunda Services GmbH. - - The Elasticsearch software is Copyright 2009-2018 Elasticsearch. - The Logstash software is Copyright 2009-2018 Elasticsearch. - The Bootstrap software has the following copyright holders: - Copyright (c) 2011-2020 Twitter, Inc. diff --git a/README.md b/README.md index 393bbb97..70b5ad0c 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ See the [wiki](https://github.com/NASA-AMMOS/common-workflow-service/wiki) for m - A database for CWS to use. `cws` is a good default. - A database user with full access to the above database. - [ITerm2](https://iterm2.com/): Currently these build scripts include commands to open new terminal windows using ITerm2, so they are best run from that terminal. - - **Logstash**: You will need to place the logstash 6.4.2 zip in `install/logging/`. This is a temporary workaround while we clean up our installation process. You can find the zip download [here](https://www.elastic.co/downloads/past-releases/logstash-6-4-2). + - **Logstash 7.9+**: You will need to place the logstash 7.9.0 zip in `install/logging/`. This is a temporary workaround while we clean up our installation process. You can find the zip download [here](https://www.elastic.co/downloads/past-releases/logstash-7-9-0). + - **Elasticsearch 7.9+**: CWS requires an externally-configured elasticsearch cluster to be set up. You can use elasticsearch with or without authentication. Please note that CWS currently only supports basic HTTP authentication. - Tomcat **keystore and truststore files** (needed for CWS web console to work properly): - You will need to add your own Tomcat keystore file to the `install/` direcotory - You will need to add your own truststor file to the `install/tomcat_lib/` directory diff --git a/create_server_dist.sh b/create_server_dist.sh index f81cfe1e..f8c59613 100755 --- a/create_server_dist.sh +++ b/create_server_dist.sh @@ -59,15 +59,10 @@ fi print 'Copying Logstash zip into place...' cp ${INSTALL_DIR}/logging/logstash-${LOGSTASH_VER}.zip ${CWS}/server -# ----------------- -# ELASTICSEARCH -print 'Copying elasticsearch zip into place...' -cp ${INSTALL_DIR}/elasticsearch/elasticsearch-${ELASTICSEARCH_VER}.zip ${CWS}/server - # MOVE TEMPLATE CONFIG FILES INTO PLACE print 'Copying configuration templates...' CONFIG_TEMPLATES_DIR=${CWS}/config/templates -mkdir -p ${CONFIG_TEMPLATES_DIR}/{cws-engine,cws-ui,tomcat_bin,tomcat_lib,tomcat_conf,camunda_mods,engine-rest_mods,elasticsearch,logging} +mkdir -p ${CONFIG_TEMPLATES_DIR}/{cws-engine,cws-ui,tomcat_bin,tomcat_lib,tomcat_conf,camunda_mods,engine-rest_mods,logging} cp ${INSTALL_DIR}/tomcat_lib/css-jaas.cfg ${CONFIG_TEMPLATES_DIR}/tomcat_lib cp ${INSTALL_DIR}/tomcat_bin/setenv.sh ${CONFIG_TEMPLATES_DIR}/tomcat_bin @@ -85,8 +80,6 @@ cp ${INSTALL_DIR}/cws-ui/*.ftl ${CONFIG_TEMPLATES cp ${INSTALL_DIR}/cws-ui/sqs_dispatcher_thread_bean.xml ${CONFIG_TEMPLATES_DIR}/cws-ui cp ${INSTALL_DIR}/camunda_mods/web.xml ${CONFIG_TEMPLATES_DIR}/camunda_mods cp ${INSTALL_DIR}/engine-rest/web.xml ${CONFIG_TEMPLATES_DIR}/engine-rest_mods -cp ${INSTALL_DIR}/elasticsearch/elasticsearch.yml ${CONFIG_TEMPLATES_DIR}/elasticsearch -cp ${INSTALL_DIR}/elasticsearch/jvm.options ${CONFIG_TEMPLATES_DIR}/elasticsearch cp ${INSTALL_DIR}/logging/cws-logstash.conf ${CONFIG_TEMPLATES_DIR}/logging cp ${INSTALL_DIR}/refresh_cws_token.sh ${CONFIG_TEMPLATES_DIR} cp ${INSTALL_DIR}/stop_cws.sh ${CONFIG_TEMPLATES_DIR} @@ -211,7 +204,6 @@ cp ${INSTALL_DIR}/stop_cws.sh ${CWS} cp ${INSTALL_DIR}/refresh_cws_token.sh ${CWS} cp ${INSTALL_DIR}/deploy_proc_def.sh ${CWS} cp ${INSTALL_DIR}/launch_ls.sh ${CWS} -cp ${INSTALL_DIR}/launch_es.sh ${CWS} print 'Copying Modeller scripts and libraries...' cp -R ${INSTALL_DIR}/modeler ${CWS} diff --git a/cws-core/src/main/java/jpl/cws/core/web/WebUtils.java b/cws-core/src/main/java/jpl/cws/core/web/WebUtils.java index c66296cc..b0185ebd 100644 --- a/cws-core/src/main/java/jpl/cws/core/web/WebUtils.java +++ b/cws-core/src/main/java/jpl/cws/core/web/WebUtils.java @@ -4,6 +4,7 @@ import java.io.OutputStream; import java.net.HttpURLConnection; import java.net.URL; +import java.util.Base64; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; @@ -47,16 +48,29 @@ public static RestCallResult restCall(String urlString, String method, String po return restCall(urlString, method, postData, null, // cookie null, // acceptType - null, // contentType - false // allowInsecureRequests + null // contentType ); } + + /** + * Performs an unauthenticated REST call + */ + public static RestCallResult restCall(String urlString, String method, String data, String cookie, String acceptType, String contentType) throws Exception { + return restCall(urlString, method, data, cookie, acceptType, contentType, true, null, null); + } + + /** + * Performs an authenticated REST call + */ + public static RestCallResult restCall(String urlString, String method, String data, String cookie, String acceptType, String contentType, String username, String password) throws Exception { + return restCall(urlString, method, data, cookie, acceptType, contentType, false, username, password); + } /** * Performs a REST call * */ - public static RestCallResult restCall(String urlString, String method, String data, String cookie, String acceptType, String contentType, Boolean allowInsecureRequests) throws Exception { + public static RestCallResult restCall(String urlString, String method, String data, String cookie, String acceptType, String contentType, Boolean allowInsecureRequests, String username, String password) throws Exception { log.trace("urlString = " + urlString); HttpURLConnection connection = null; try { @@ -72,7 +86,14 @@ else if (rootAndQueryString.length > 2) { URL url = new URL(urlString); connection = (HttpURLConnection) url.openConnection(); - + + // Add authentication to request + if (username != null) { + String userpass = username + ":" + password; + String basicAuth = "Basic " + new String(Base64.getEncoder().encode(userpass.getBytes())); + connection.setRequestProperty ("Authorization", basicAuth); + } + if (allowInsecureRequests) { log.info("SSL 'insecure' mode activated."); @@ -136,10 +157,10 @@ else if (rootAndQueryString.length > 2) { out.close(); } } - + // Perform the connection connection.connect(); - + int responseCode = -1; try { responseCode = connection.getResponseCode(); @@ -149,7 +170,7 @@ else if (rootAndQueryString.length > 2) { } if (responseCode == 200) { log.trace("REST "+method+" for " + url + " was successful (HTTP 200)."); - + InputStream in = connection.getInputStream(); // FIXME: potential memory suck if response is large String response = IOUtils.toString(in); @@ -158,7 +179,7 @@ else if (rootAndQueryString.length > 2) { } else { log.warn("REST " + method + " FAILED for " + url + ". Response code was " + responseCode); - + return new RestCallResult(responseCode, null, connection.getResponseMessage()); } } @@ -179,8 +200,8 @@ else if (rootAndQueryString.length > 2) { } } } - - + + /** * */ diff --git a/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java b/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java index 67f50a82..74295bb4 100644 --- a/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java +++ b/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java @@ -38,6 +38,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; +import java.util.Arrays; import java.util.Calendar; import java.util.HashSet; import java.util.TimeZone; @@ -67,11 +68,6 @@ public class CwsInstaller { // Number of bytes per gigabyte private static final long BYTES_PER_GIG = (long) 1e+9; - // The maximum heap size in gigs we allow as a default for Elasticsearch - // This number is based on Elasticsearch's own recommendation for heap sizing - // See https://www.elastic.co/guide/en/elasticsearch/reference/6.4/heap-size.html - public static final int MAX_ES_HEAP_SIZE = 26; - // amount of time in milliseconds that a part of a CWS installation // (database, worker, or console) can be off from the system time of // this installation @@ -120,7 +116,6 @@ public class CwsInstaller { private static String installType; private static boolean installConsole; private static boolean installWorker; - private static boolean installElasticsearch; private static String cws_worker_type; private static String cws_engine_proc_start_req_listener; @@ -168,18 +163,17 @@ public class CwsInstaller { private static String aws_cloudwatch_endpoint; private static String metrics_publishing_interval; private static String cws_worker_id; - private static String elasticsearch_ver; private static String cws_server_root; private static String logstash_ver; - private static String elasticsearch_root; private static String unique_broker_group_name; private static String cws_security_filter_class; private static String startup_autoregister_process_defs; private static String cws_token_expiration_hours; - private static String user_provided_elasticsearch; private static String elasticsearch_host; private static String elasticsearch_port; - private static String elasticsearch_heapsize; + private static String elasticsearch_use_auth; + private static String elasticsearch_username; + private static String elasticsearch_password; private static String user_provided_logstash; private static String history_level; private static String history_days_to_live; @@ -254,10 +248,6 @@ public static void main(String args[]) { createFreshWorkDir(); updateFiles(); - if (!reconfigure && installElasticsearch) { - installElasticSearch(); - } - if (!reconfigure) { installLogstash(); } @@ -316,9 +306,6 @@ private static void init() { logstash_ver = getenv("LOGSTASH_VER"); logstash_root = cws_server_root + SEP + "logstash-" + logstash_ver; - elasticsearch_ver = getenv("ELASTICSEARCH_VER"); - elasticsearch_root = cws_server_root + SEP + "elasticsearch-" + elasticsearch_ver; - config_templates_dir = cws_root + SEP + "config" + SEP + "templates"; config_work_dir = cws_root + SEP + "config" + SEP + "work"; @@ -1041,84 +1028,97 @@ private static void setupLogstash() { private static void setupElasticsearch() { - // PROMPT USER FOR USER PROVIDED ELASTIC SEARCH - user_provided_elasticsearch = getPreset("user_provided_elasticsearch"); + // PROMPT USER FOR ELASTICSEARCH HOST + elasticsearch_host = getPreset("elasticsearch_host"); + + if (cws_installer_mode.equals("interactive")) { + if (elasticsearch_host == null) { + elasticsearch_host = readRequiredLine("Enter the Elasticsearch host: ", + "You must enter a hostname"); + } else { + elasticsearch_host = readLine("Enter the Elasticsearch host. " + + "Default is " + elasticsearch_host + ": ", elasticsearch_host); + } + } else { + if (elasticsearch_host == null) { + bailOutMissingOption("elasticsearch_host"); + } + } + + log.debug("elasticsearch_host: " + elasticsearch_host); + + // PROMPT USER FOR ELASTICSEARCH PORT + elasticsearch_port = getPreset("elasticsearch_port"); + + if (elasticsearch_port == null) { + elasticsearch_port = getPreset("default_elasticsearch_port"); + } + + if (cws_installer_mode.equals("interactive")) { + elasticsearch_port = readLine("Enter the Elasticsearch port. " + + "Default is " + elasticsearch_port + ": ", elasticsearch_port); + } + + log.debug("elasticsearch_port: " + elasticsearch_port); + + // PROMPT USER ELASTICSEARCH AUTH + elasticsearch_use_auth = getPreset("elasticsearch_use_auth"); - if (user_provided_elasticsearch == null) { - user_provided_elasticsearch = getPreset("default_user_provided_elasticsearch"); + if (elasticsearch_use_auth == null) { + elasticsearch_use_auth = getPreset("default_elasticsearch_use_auth"); } if (cws_installer_mode.equals("interactive")) { - String read_user_provided_elasticsearch = ""; + String read_elasticsearch_use_auth = ""; - while (!read_user_provided_elasticsearch.equalsIgnoreCase("y") && - !read_user_provided_elasticsearch.equalsIgnoreCase("n")) { - read_user_provided_elasticsearch = - readRequiredLine("Are you providing your own Elasticsearch service? (Y/N): ", + while (!read_elasticsearch_use_auth.equalsIgnoreCase("y") && + !read_elasticsearch_use_auth.equalsIgnoreCase("n")) { + read_elasticsearch_use_auth = + readRequiredLine("Does you Elasticsearch cluster require authentication? (Y/N): ", "ERROR: Must specify either 'Y' or 'N'"); } - user_provided_elasticsearch = read_user_provided_elasticsearch.toLowerCase(); + user_provided_logstash = read_elasticsearch_use_auth.toLowerCase(); } - installElasticsearch = false; + if (elasticsearch_use_auth.equalsIgnoreCase("Y")) { - // PROMPT USER FOR ELASTICSEARCH HOST AND PORT - if (user_provided_elasticsearch.equalsIgnoreCase("Y")) { - - // PROMPT USER FOR ELASTICSEARCH HOST - elasticsearch_host = getPreset("elasticsearch_host"); + elasticsearch_username = getPreset("elasticsearch_username"); + // PROMPT USER FOR ELASTICSEARCH USERNAME if (cws_installer_mode.equals("interactive")) { - if (elasticsearch_host == null) { - elasticsearch_host = readRequiredLine("Enter the Elasticsearch host: ", - "You must enter a hostname"); + if (elasticsearch_username == null) { + elasticsearch_username = readRequiredLine("Enter the elasticsearch username: ", + "Must specify an elasticsearch username!"); } else { - elasticsearch_host = readLine("Enter the Elasticsearch host. " + - "Default is " + elasticsearch_host + ": ", elasticsearch_host); + elasticsearch_username = readLine("Enter the database username. " + + "Default is " + elasticsearch_username + ": ", elasticsearch_username); } } else { - if (elasticsearch_host == null) { - bailOutMissingOption("elasticsearch_host"); + if (elasticsearch_username == null) { + bailOutMissingOption("elasticsearch_username"); } } - // PROMPT USER FOR ELASTICSEARCH PORT - elasticsearch_port = getPreset("elasticsearch_port"); + log.debug("elasticsearch_username: " + elasticsearch_username); - if (elasticsearch_port == null) { - elasticsearch_port = getPreset("default_elasticsearch_port"); - } + elasticsearch_password = getPreset("elasticsearch_password"); + // PROMPT USER FOR ELASTICSEARCH PASSWORD if (cws_installer_mode.equals("interactive")) { - elasticsearch_port = readLine("Enter the Elasticsearch port. " + - "Default is " + elasticsearch_port + ": ", elasticsearch_port); - } - } - else { - // Use CWS's own Elasticsearch--set the elasticsearch host and port to the console, and install ES if this is a console + char[] password = readPassword("Enter the elasticsearch password: "); - if (installConsole) { - installElasticsearch = true; - - elasticsearch_heapsize = getPreset("elasticsearch_heapsize"); - - if (elasticsearch_heapsize == null) { - // Determine the heap size we'll recommend as default, which is either - // one quarter of the available memory or a hard upper limit to ensure good - // performance from ES, whichever is smaller. - int recommended_heapsize = (int) Math.min(Math.ceil(physicalMemoryGigs / 4.0), MAX_ES_HEAP_SIZE); - elasticsearch_heapsize = String.valueOf(recommended_heapsize); + while (password == null || password.length < 1) { + print("Must specify an elasticsearch password!"); + password = readPassword("Enter the elasticsearch password: "); } - if (cws_installer_mode.equals("interactive")) { - elasticsearch_heapsize = readLine("Enter the desired Elasticsearch heap size in gigabytes. " + - "Default is " + elasticsearch_heapsize + ": ", elasticsearch_heapsize); + elasticsearch_password = String.valueOf(password); + } else { + if (elasticsearch_password == null) { + bailOutMissingOption("elasticsearch_password"); } } - - elasticsearch_host = cws_console_host; - elasticsearch_port = getPreset("default_elasticsearch_port"); } } @@ -1445,14 +1445,11 @@ private static void showInstallationInfo() { print("SMTP host = " + cws_smtp_hostname); print("SMTP port = " + cws_smtp_port); print("...................................................................................."); - if (user_provided_elasticsearch.equalsIgnoreCase("Y")) { - print("Elasticsearch = User Provided"); - print("Elasticsearch host = " + elasticsearch_host); - print("Elasticsearch port = " + elasticsearch_port); - } - else { - print("Elasticsearch = CWS Provided"); - print("Elasticsearch heap size = " + elasticsearch_heapsize); + print("Elasticsearch URL = " + elasticsearch_host); + print("Elasticsearch Port = " + elasticsearch_port); + if (elasticsearch_use_auth.equalsIgnoreCase("Y")) { + print("Elasticsearch User = " + elasticsearch_username); + print("Elasticsearch Password = ****** (hidden) "); } print("...................................................................................."); if (user_provided_logstash.equalsIgnoreCase("Y")) { @@ -1508,23 +1505,8 @@ private static void validateConfig() { warningCount++; } - // Validate ES setting if installing console and not user provided ES. - if (installConsole && user_provided_elasticsearch.equalsIgnoreCase("N")) { - - // ElasticSearch seems to ignore lower limits on Macs. - if (!osName.equals("Mac OS X")) { - warningCount += validateElasticsearchProcThreads(); - warningCount += validateElasticsearchFileDescriptors(); - } - - warningCount += validateElasticsearchHeapSize(); - } - // Check that user provided Elasticsearch service is up and healthy - if (user_provided_elasticsearch.equalsIgnoreCase("Y")) { - - warningCount += validateUserProvidedElasticsearch(); - } + warningCount += validateElasticsearch(); if (installWorker && !installConsole) { // Validate the AMQ host/port for worker only installations. @@ -1826,11 +1808,17 @@ else if (Integer.valueOf(cws_amq_port) < 1024) { * Validates that User Provided ElasticSearch is running * */ - private static int validateUserProvidedElasticsearch() { + private static int validateElasticsearch() { print("checking that user provided Elasticsearch (" + elasticsearch_host + ":" + elasticsearch_port + ") is running..."); try { String[] cmdArray = new String[] {"curl", "--fail", "http://" + elasticsearch_host + ":" + elasticsearch_port + "/_cluster/health"}; + + if (elasticsearch_use_auth.equalsIgnoreCase("Y")) { + // Add auth to curl + cmdArray = new String[] {"curl", "--fail", "-u", elasticsearch_username + ":" + elasticsearch_password, "https://" + elasticsearch_host + ":" + elasticsearch_port + "/_cluster/health"}; + } + Process p = Runtime.getRuntime().exec(cmdArray); // Wait for the process to complete @@ -1857,167 +1845,6 @@ private static int validateUserProvidedElasticsearch() { } } - /** - * Validates that ElasticSearch limits are met - * - */ - private static int validateElasticsearchProcThreads() { - print("checking for Elasticsearch prerequisite #1 (number of threads)..."); - - // max user processes - try { - String maxUserProcesses; - String[] cmdArray = new String[] {"bash", "-c", "ulimit -u"}; - Process p = Runtime.getRuntime().exec(cmdArray); - BufferedReader input = - new BufferedReader(new InputStreamReader(p.getInputStream())); - maxUserProcesses = input.readLine(); - input.close(); - - if (maxUserProcesses == null) { - throw new Exception("unable to determine max user processes "); - } - - if (maxUserProcesses.equals("unlimited")) { - print(" [OK] (# users processes = unlimited)"); - print(""); - return 0; // no warnings - } - - int maxUserProcsInt = Integer.parseInt(maxUserProcesses); - - if (maxUserProcsInt < 4096) { - print(" [WARNING]"); - print(" It was determined (via the \"ulimit -u\" command)"); - print(" that this user on this system can create only " + maxUserProcsInt + " threads."); - print(" Please work with a System Administrator to ensure that at user on"); - print(" this system can create is at least 4096."); - print(" See: https://www.elastic.co/guide/en/elasticsearch/reference/master/max-number-of-threads.html"); - print(""); - return 1; - } - print(" [OK] (# users processes = " + maxUserProcsInt + ")"); - print(""); - return 0; // no warnings - - } catch (Exception e) { - log.error("error during Elasticsearch thread count validation: ", e); - print(" [WARNING]"); - print(" There was a problem determining (via the \"ulimit -u\" command)"); - print(" how many processes can run on this system. "); - print(" Please work with a System Administrator to ensure that at user on"); - print(" this system can create is at least 4096."); - print(" See: https://www.elastic.co/guide/en/elasticsearch/reference/master/max-number-of-threads.html"); - print(""); - return 1; - } - } - - - /** - * Validates that ElasticSearch limits are met - * - */ - private static int validateElasticsearchFileDescriptors() { - print("checking for Elasticsearch prerequisite #2 (number of file descriptors)..."); - - // check file descriptors count - try { - String maxFileDescriptors; - String[] cmdArray = new String[] {"bash", "-c", "ulimit -n"}; - Process p = Runtime.getRuntime().exec(cmdArray); - BufferedReader input = - new BufferedReader(new InputStreamReader(p.getInputStream())); - maxFileDescriptors = input.readLine(); - input.close(); - - if (maxFileDescriptors == null) { - throw new Exception("unable to determine max file descriptors"); - } - - if (maxFileDescriptors.equals("unlimited")) { - print(" [OK] (# file descriptors = unlimited)"); - print(""); - return 0; // no warnings - } - - int maxFileDescriptorsInt = Integer.parseInt(maxFileDescriptors); - - if (maxFileDescriptorsInt < 65536) { - print(" [WARNING]"); - print(" It was determined (via the \"ulimit -n\" command)"); - print(" that the max open file descriptors on this system is " + maxFileDescriptorsInt + "."); - print(" Please work with a System Administrator to ensure that this system "); - print(" that this limit is set to 65536."); - print(" See: https://www.elastic.co/guide/en/elasticsearch/reference/current/file-descriptors.html"); - print(""); - return 1; - } - print(" [OK] (# file descriptors = " + maxFileDescriptorsInt + ")"); - print(""); - - return 0; // no warnings - - } catch (Exception e) { - log.error("error during Elasticsearch file descriptor validation: ", e);; - print(" [WARNING]"); - print(" There was a problem determining (via the \"ulimit -n\" command)"); - print(" what the open file descriptors limit is on this system."); - print(" Please work with a System Administrator to ensure that this system "); - print(" that this limit is set to 65536."); - print(" See: https://www.elastic.co/guide/en/elasticsearch/reference/current/file-descriptors.html"); - print(""); - return 1; - } - } - - /** - * Validates the user-provided heap size for Elasticsearch - * - */ - private static int validateElasticsearchHeapSize() { - print("checking for Elasticsearch prerequisite #3 (valid heap size)..."); - - try { - int elasticsearchHeapSizeInt = Integer.parseInt(elasticsearch_heapsize); - - // Make sure the user didn't request more than 50% of available RAM - if (elasticsearchHeapSizeInt > (physicalMemoryGigs / 2)) { - print(" [WARNING]"); - print(" The provided Elasticsearch heap size (" + elasticsearch_heapsize + " GB) is "); - print(" larger than 50% of the available physical memory on the current machine (" + physicalMemoryGigs + " GB)"); - print(" This could have an adverse effect on OS performance."); - print(" See: https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html"); - print(""); - return 1; - } - - // Make sure the user didn't exceed the upper limit which would prevent compression of object pointers - if (elasticsearchHeapSizeInt > MAX_ES_HEAP_SIZE) { - print(" [WARNING]"); - print(" The provided Elasticsearch heap size (" + elasticsearch_heapsize + " GB) is "); - print(" larger than the maximum recommended size of " + MAX_ES_HEAP_SIZE + "."); - print(" This could have an adverse effect on Elasticsearch JVM performance."); - print(" See: https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html"); - print(""); - return 1; - } - - print(" [OK] (heap size = " + elasticsearchHeapSizeInt + ")"); - print(""); - - return 0; // no warnings - - } catch (Exception e) { - log.error("error during Elasticsearch heap size validation: ", e); - print(" [WARNING]"); - print(" There was a problem trying to parse the provided Elasticsearch heap size: " + elasticsearch_heapsize); - print(" Please ensure the provided heap size is a positive integer."); - print(""); - return 1; - } - } - /** * Validates that some sort of time syncing service * such as NTP or chrony is running on this installation machine. @@ -2093,7 +1920,6 @@ private static void createFreshWorkDir() { mkDir(config_work_dir + SEP + "tomcat_conf"); mkDir(config_work_dir + SEP + "camunda_mods"); mkDir(config_work_dir + SEP + "engine-rest_mods"); - mkDir(config_work_dir + SEP + "elasticsearch"); mkDir(config_work_dir + SEP + "logging"); @@ -2136,10 +1962,6 @@ private static void createFreshWorkDir() { Paths.get(config_work_dir + SEP + "tomcat_conf" + SEP + "ldap_plugin_ref.xml")); copy(Paths.get( config_templates_dir + SEP + "camunda_mods" + SEP + "web.xml"), Paths.get(config_work_dir + SEP + "camunda_mods" + SEP + "web.xml")); - copy(Paths.get( config_templates_dir + SEP + "elasticsearch" + SEP + "elasticsearch.yml"), - Paths.get(config_work_dir + SEP + "elasticsearch" + SEP + "elasticsearch.yml")); - copy(Paths.get( config_templates_dir + SEP + "elasticsearch" + SEP + "jvm.options"), - Paths.get(config_work_dir + SEP + "elasticsearch" + SEP + "jvm.options")); copy(Paths.get( config_templates_dir + SEP + "logging" + SEP + "cws-logstash.conf"), Paths.get(config_work_dir + SEP + "logging" + SEP + "cws-logstash.conf")); } @@ -2412,6 +2234,7 @@ private static void updateCwsUiProperties() throws IOException { content = content.replace("__CWS_CONSOLE_SSL_PORT__", cws_console_ssl_port); content = content.replace("__CWS_ES_HOST__", elasticsearch_host); content = content.replace("__CWS_ES_PORT__", elasticsearch_port); + content = content.replace("__CWS_ES_USE_AUTH__", elasticsearch_use_auth); content = content.replace("__CWS_ENABLE_CLOUD_AUTOSCALING__", cws_enable_cloud_autoscaling); content = content.replace("__CWS_CLOUDWATCH_ENDPOINT__", aws_cloudwatch_endpoint); content = content.replace("__CWS_METRICS_PUBLISHING_INTERVAL__", metrics_publishing_interval); @@ -2431,6 +2254,12 @@ private static void updateCwsUiProperties() throws IOException { content = content.replace("__CWS_HISTORY_LEVEL__", history_level); content = content.replace("__AWS_DEFAULT_REGION__", aws_default_region); + // ES auth might not be in use + if(elasticsearch_use_auth.equalsIgnoreCase("Y")) { + content = content.replace("__CWS_ES_USERNAME__", elasticsearch_username); + content = content.replace("__CWS_ES_PASSWORD__", elasticsearch_password); + } + // S3 Initiator might not be in use if(aws_sqs_dispatcher_sqsUrl != null) { content = content.replace("__AWS_SQS_DISPATCHER_SQS_URL__", aws_sqs_dispatcher_sqsUrl); @@ -2491,6 +2320,11 @@ private static void updateCwsUiConfig() throws IOException { content = getFileContents(path); content = content.replace("__ES_HOST__", elasticsearch_host); content = content.replace("__ES_PORT__", elasticsearch_port); + content = content.replace("__ES_USE_AUTH__", elasticsearch_use_auth); + if (elasticsearch_use_auth.equalsIgnoreCase("Y")) { + content = content.replace("__ES_USERNAME__", elasticsearch_username); + content = content.replace("__ES_PASSWORD__", elasticsearch_password); + } content = content.replace("__CWS_HISTORY_DAYS_TO_LIVE__", history_days_to_live); writeToFile(path, content); copy( @@ -2586,39 +2420,6 @@ else if (cws_auth_scheme.equals("CAMUNDA")) { } - private static void installElasticSearch() throws IOException { - // UNZIP / INSTALL ELASTICSEARCH - print(" Unzipping elasticsearch-" + elasticsearch_ver + ".zip..."); - String elasticsearchZipFilePath = cws_server_root + SEP + "elasticsearch-" + elasticsearch_ver + ".zip"; - String elasticsearchDestDirectory = cws_server_root; - unzipFile(elasticsearchZipFilePath, elasticsearchDestDirectory); - - // Open up permissions of elastic search executables - openUpPermissions(elasticsearchDestDirectory + SEP + "elasticsearch-" + elasticsearch_ver + SEP + "bin" + SEP + "elasticsearch"); - openUpPermissions(elasticsearchDestDirectory + SEP + "elasticsearch-" + elasticsearch_ver + SEP + "modules" + SEP + "x-pack-ml" + SEP + "platform" + SEP + "darwin-x86_64" + SEP + "bin" + SEP + "controller"); - openUpPermissions(elasticsearchDestDirectory + SEP + "elasticsearch-" + elasticsearch_ver + SEP + "modules" + SEP + "x-pack-ml" + SEP + "platform" + SEP + "linux-x86_64" + SEP + "bin" + SEP + "controller"); - - // UPDATE elasticsearch.yml - print(" Updating elasticsearch.yml..."); - Path elasticsearchFilePath = Paths.get(config_work_dir + SEP + "elasticsearch" + SEP + "elasticsearch.yml"); - String content = getFileContents(elasticsearchFilePath); - content = content.replace("__CWS_ES_CLUSTERNAME__", unique_broker_group_name); - content = content.replace("__CWS_ES_HOST__", elasticsearch_host); - writeToFile(elasticsearchFilePath, content); - - copy(elasticsearchFilePath, Paths.get(elasticsearch_root + SEP + "config" + SEP + "elasticsearch.yml")); - - // UPDATE jvm.options - print(" Updating jvm.options..."); - elasticsearchFilePath = Paths.get(config_work_dir + SEP + "elasticsearch" + SEP + "jvm.options"); - content = getFileContents(elasticsearchFilePath); - content = content.replaceAll("__CWS_ES_HEAPSIZE__", elasticsearch_heapsize); - writeToFile(elasticsearchFilePath, content); - - copy(elasticsearchFilePath, Paths.get(elasticsearch_root + SEP + "config" + SEP + "jvm.options")); - } - - private static void deleteCwsUiWebApp() { print(" Removing cws-ui app from webapps..."); @@ -2661,6 +2462,23 @@ private static void installLogstash() throws IOException { logstashContent = logstashContent.replace("__CWS_ES_HOST__", elasticsearch_host); logstashContent = logstashContent.replace("__CWS_ES_PORT__", elasticsearch_port); + if (elasticsearch_use_auth.equalsIgnoreCase(("Y"))) { + // Construct the auth config for logstash + String user = "user => \"" + elasticsearch_username + "\""; + String pw = "password => \"" + elasticsearch_password + "\""; + logstashContent = logstashContent.replace("__LOGSTASH_ES_USERNAME__", user); + logstashContent = logstashContent.replace("__LOGSTASH_ES_PASSWORD__", pw); + + // Tell logstash to use https + logstashContent = logstashContent.replace("__LOGSTASH_ES_USE_SSL__", "true"); + } else { + // Remove these blocks if not using auth + logstashContent = logstashContent.replace("__LOGSTASH_ES_USERNAME__", ""); + logstashContent = logstashContent.replace("__LOGSTASH_ES_PASSWORD__", ""); + + // Tell logstash to use http + logstashContent = logstashContent.replace("__LOGSTASH_ES_USE_SSL__", "false"); + } writeToFile(logstashFilePath, logstashContent); print(" Put logstash conf file into place."); @@ -2710,10 +2528,11 @@ private static void writeOutConfigurationFile() { setPreset("metrics_publishing_interval", metrics_publishing_interval); setPreset("cws_notification_emails", cws_notification_emails); setPreset("cws_token_expiration_hours", cws_token_expiration_hours); - setPreset("user_provided_elasticsearch", user_provided_elasticsearch); setPreset("elasticsearch_host", elasticsearch_host); setPreset("elasticsearch_port", elasticsearch_port); - setPreset("elasticsearch_heapsize", elasticsearch_heapsize); + setPreset("elasticsearch_use_auth", elasticsearch_use_auth); + setPreset("elasticsearch_username", elasticsearch_username); + setPreset("elasticsearcH_password", elasticsearch_password); setPreset("user_provided_logstash", user_provided_logstash); setPreset("history_level", history_level); setPreset("history_days_to_live", history_days_to_live); diff --git a/cws-service/src/main/java/jpl/cws/controller/RestService.java b/cws-service/src/main/java/jpl/cws/controller/RestService.java index fcb46a5a..4805637a 100644 --- a/cws-service/src/main/java/jpl/cws/controller/RestService.java +++ b/cws-service/src/main/java/jpl/cws/controller/RestService.java @@ -112,6 +112,10 @@ public class RestService extends MvcCore { @Value("${cws.elasticsearch.hostname}") private String elasticsearchHostname; @Value("${cws.elasticsearch.port}") private String elasticsearchPort; + @Value("${cws.elasticsearch.use.auth}") private String elasticsearchUseAuth; + @Value("${cws.elasticsearch.username}") private String elasticsearchUsername; + @Value("${cws.elasticsearch.password}") private String elasticsearchPassword; + public RestService() {} @@ -392,6 +396,27 @@ private String doDeployProcessDefinition(MultipartFile file) { return e.getMessage(); } } + + /** + * Constructs Elasticsearch URL + * + * @param subPath The subPath for the elasticsearch query, e.g., /_delete_by_query + * @return fully constructed elasticsearch URL string + */ + private String constructElasticsearchUrl(String subPath) { + String urlString = elasticsearchUseAuth.equalsIgnoreCase("Y")? "https://" : "http://"; + urlString += elasticsearchHostname + ":" + elasticsearchPort + subPath; + + return urlString; + } + + /** + * + * @return boolean indicating whether elasticsearch requires authentication + */ + private Boolean elasticsearchUseAuth() { + return elasticsearchUseAuth.equalsIgnoreCase("Y"); + } /** @@ -603,14 +628,21 @@ public ModelAndView validateAndSaveSnippets( @RequestMapping(value = "/logs/get/scroll", method = POST, produces="application/json") public @ResponseBody String getLogsScroll( @RequestParam(value = "scrollId") String scrollId) { - String urlString = "http://" + elasticsearchHostname + ":" + elasticsearchPort + "/_search/scroll"; + String urlString = constructElasticsearchUrl("/_search/scroll"); String jsonData = "{ \"scroll\" : \"1m\", \"scroll_id\" : \"" + scrollId + "\" }"; log.trace("REST getLogs query = " + urlString); log.trace("REST getLogs jsonData = " + jsonData); try { - RestCallResult restCallResult = WebUtils.restCall(urlString, "POST", jsonData, null, null, "application/json; charset=utf-8", true); + RestCallResult restCallResult; + if (elasticsearchUseAuth()) { + // Authenticated call + restCallResult = WebUtils.restCall(urlString, "POST", jsonData, null, null, "application/json; charset=utf-8", elasticsearchUsername, elasticsearchPassword); + } else { + // Unauthenticated call + restCallResult = WebUtils.restCall(urlString, "POST", jsonData, null, null, "application/json; charset=utf-8"); + } if (restCallResult.getResponseCode() != 200) { return "ERROR"; } @@ -630,12 +662,19 @@ public ModelAndView validateAndSaveSnippets( @RequestMapping(value = "/logs/get", method = GET, produces="application/json") public @ResponseBody String getLogs( @RequestParam(value = "source") String source) { - String urlString = "http://" + elasticsearchHostname + ":" + elasticsearchPort + "/_search?scroll=1m&source=" + source + "&source_content_type=application/json"; + String urlString = constructElasticsearchUrl("/_search?scroll=1m&source=" + source + "&source_content_type=application/json"); log.trace("REST getLogs query = " + urlString); try { - RestCallResult restCallResult = WebUtils.restCall(urlString, "GET", null, null, null, "application/json; charset=utf-8", true); + RestCallResult restCallResult; + if (elasticsearchUseAuth()) { + // Authenticated call + restCallResult = WebUtils.restCall(urlString, "GET", null, null, null, "application/json; charset=utf-8", elasticsearchUsername, elasticsearchPassword); + } else { + // Unauthenticated call + restCallResult = WebUtils.restCall(urlString, "GET", null, null, null, "application/json; charset=utf-8"); + } if (restCallResult.getResponseCode() != 200) { return "ERROR"; } @@ -657,15 +696,21 @@ public ModelAndView validateAndSaveSnippets( HttpServletResponse response, @PathVariable String procDefKey ) { - - String urlString = "http://" + elasticsearchHostname + ":" + elasticsearchPort + "/*/_delete_by_query"; + String urlString = constructElasticsearchUrl("/*/_delete_by_query"); log.debug("REST deleteLogsByProcDefKey url = " + urlString); String data = "{ \"query\": { \"bool\": { \"must\": [ { \"match\": { \"procDefKey\": \"" + procDefKey + "\" } } ] } } }"; log.debug("REST deleteLogsByProcDefKey data = " + data); try { - RestCallResult restCallResult = WebUtils.restCall(urlString, "POST", data, null, null, "application/json", true); + RestCallResult restCallResult; + if (elasticsearchUseAuth()) { + // Authenticated call + restCallResult = WebUtils.restCall(urlString, "POST", data, null, null, "application/json", elasticsearchUsername, elasticsearchPassword); + } else { + // Unauthenticated call + restCallResult = WebUtils.restCall(urlString, "POST", data, null, null, "application/json"); + } if (restCallResult.getResponseCode() != 200) { @@ -733,12 +778,19 @@ public GsonUTCDateAdapter() { */ @RequestMapping(value = "/stats/es/indices", method = GET, produces="application/json") public @ResponseBody String getElasticsearchIndices() { - String urlString = "http://" + elasticsearchHostname + ":" + elasticsearchPort + "/_cat/indices?v&bytes=b&s=index&format=json"; + String urlString = constructElasticsearchUrl("/_cat/indices?v&bytes=b&s=index&format=json"); log.trace("REST query = " + urlString); try { - RestCallResult restCallResult = WebUtils.restCall(urlString, "GET", null, null, null, null, true); + RestCallResult restCallResult; + if (elasticsearchUseAuth()) { + // Authenticated call + restCallResult = WebUtils.restCall(urlString, "GET", null, null, null, null, elasticsearchUsername, elasticsearchPassword); + } else { + // Unauthenticated call + restCallResult = WebUtils.restCall(urlString, "GET", null, null, null, null); + } if (restCallResult.getResponseCode() != 200) { return "ERROR"; } @@ -757,12 +809,19 @@ public GsonUTCDateAdapter() { */ @RequestMapping(value = "/stats/es/cluster/health", method = GET, produces="application/json") public @ResponseBody String getElasticsearchClusterHealth() { - String urlString = "http://" + elasticsearchHostname + ":" + elasticsearchPort + "/_cluster/health"; + String urlString = constructElasticsearchUrl("/_cluster/health"); log.trace("REST query = " + urlString); try { - RestCallResult restCallResult = WebUtils.restCall(urlString, "GET", null, null, null, null, true); + RestCallResult restCallResult; + if (elasticsearchUseAuth()) { + // Authenticated call + restCallResult = WebUtils.restCall(urlString, "GET", null, null, null, null, elasticsearchUsername, elasticsearchPassword); + } else { + // Unauthenticated call + restCallResult = WebUtils.restCall(urlString, "GET", null, null, null, null); + } if (restCallResult.getResponseCode() != 200) { return "ERROR"; } @@ -781,12 +840,19 @@ public GsonUTCDateAdapter() { */ @RequestMapping(value = "/stats/es", method = GET, produces="application/json") public @ResponseBody String getElasticsearchStats() { - String urlString = "http://" + elasticsearchHostname + ":" + elasticsearchPort + "/_nodes/stats/_all"; + String urlString = constructElasticsearchUrl("/_nodes/stats/_all"); log.trace("REST query = " + urlString); try { - RestCallResult restCallResult = WebUtils.restCall(urlString, "GET", null, null, null, null, true); + RestCallResult restCallResult; + if (elasticsearchUseAuth()) { + // Authenticated call + restCallResult = WebUtils.restCall(urlString, "GET", null, null, null, null, elasticsearchUsername, elasticsearchPassword); + } else { + // Unauthenticated call + restCallResult = WebUtils.restCall(urlString, "GET", null); + } if (restCallResult.getResponseCode() != 200) { return "ERROR"; } @@ -1390,7 +1456,7 @@ public Message createMessage(Session session) throws JMSException { RestCallResult restCallResult = null; try { - restCallResult = WebUtils.restCall(url, "GET", null, null, acceptType, null, false); + restCallResult = WebUtils.restCall(url, "GET", null, null, acceptType, null); } catch (Exception e) { log.error("Exception occurred while making external GET request to: " + url, e); return "ERROR"; @@ -1420,7 +1486,7 @@ public Message createMessage(Session session) throws JMSException { RestCallResult restCallResult = null; try { - restCallResult = WebUtils.restCall(url, "POST", postPayload, null, null, contentType, false); + restCallResult = WebUtils.restCall(url, "POST", postPayload, null, null, contentType); } catch (Exception e) { log.error("Exception occurred while making external POST request to: " + url, e); return "ERROR"; @@ -1451,7 +1517,7 @@ public Message createMessage(Session session) throws JMSException { RestCallResult restCallResult = null; try { - restCallResult = WebUtils.restCall(url, "PUT", payload, null, null, contentType, false); + restCallResult = WebUtils.restCall(url, "PUT", payload, null, null, contentType); } catch (Exception e) { log.error("Exception occurred while making external PUT request to: " + url, e); return "ERROR"; diff --git a/cws-tasks/src/main/java/jpl/cws/task/RestGetTask.java b/cws-tasks/src/main/java/jpl/cws/task/RestGetTask.java index 0b6c4003..1c392c9e 100644 --- a/cws-tasks/src/main/java/jpl/cws/task/RestGetTask.java +++ b/cws-tasks/src/main/java/jpl/cws/task/RestGetTask.java @@ -3,7 +3,6 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; -import java.net.URL; import java.util.Map; import java.util.Map.Entry; import java.util.regex.Matcher; @@ -44,6 +43,10 @@ public class RestGetTask extends CwsTask { private String acceptTypeString; private Expression allowInsecureRequests; private boolean allowInsecureRequestsBoolean; + private Expression httpAuthUsername; + private String httpAuthUsernameString; + private Expression httpAuthPassword; + private String httpAuthPasswordString; // CWS token security scheme variables // @@ -53,7 +56,7 @@ public class RestGetTask extends CwsTask { private String cwsCookieNameString; private Expression cwsCookieValue; // cookie value -- this is the CWS "token" private String cwsCookieValueString; - + public RestGetTask() { log.trace("RestGetTask constructor..."); @@ -63,6 +66,8 @@ public RestGetTask() { public void initParams() throws Exception { urlString = getStringParam(url, "url"); allowInsecureRequestsBoolean = getBooleanParam(allowInsecureRequests, "allowInsecureRequests", DEFAULT_ALLOW_INSECURE_REQUESTS); + httpAuthUsernameString = getStringParam(httpAuthUsername, "httpAuthUsername", null); + httpAuthPasswordString = getStringParam(httpAuthPassword, "httpAuthPassword", null); throwOnBadResponseBoolean = getBooleanParam(throwOnBadResponse, "throwOnBadResponse", DEFAULT_THROW_ON_BAD_RESPONSE); acceptTypeString = getStringParam(acceptType, "acceptType", null); @@ -92,13 +97,26 @@ public void executeTask() throws Exception { // Make the REST GET // - RestCallResult result = WebUtils.restCall( - urlString, "GET", - null, // no POST args, since we are making a GET call - cookieToUse, - acceptTypeString, - null, - allowInsecureRequestsBoolean); + RestCallResult result; + if (httpAuthUsername == null) { + // Unauthenticated GET + result = WebUtils.restCall( + urlString, "GET", + null, // no POST args, since we are making a GET call + cookieToUse, + acceptTypeString, + null); + } else { + // Authenticated GET + result = WebUtils.restCall( + urlString, "GET", + null, // no POST args, since we are making a GET call + cookieToUse, + acceptTypeString, + null, + httpAuthUsernameString, + httpAuthPasswordString); + } int responseCode = result.getResponseCode(); log.info("httpStatusCode: " + responseCode); @@ -182,6 +200,22 @@ public Expression getAllowInsecureRequests() { public void setAllowInsecureRequests(Expression allowInsecureRequests) { this.allowInsecureRequests = allowInsecureRequests; } + + public Expression getHttpAuthUsername() { + return httpAuthUsername; + } + + public void setHttpAuthUsername(Expression httpAuthUsername) { + this.httpAuthUsername = httpAuthUsername; + } + + public Expression getHttpAuthPassword() { + return httpAuthPassword; + } + + public void setHttpAuthPassword(Expression httpAuthPassword) { + this.httpAuthPassword = httpAuthPassword; + } public Expression getThrowOnBadResponse() { return throwOnBadResponse; diff --git a/cws-tasks/src/main/java/jpl/cws/task/RestPostTask.java b/cws-tasks/src/main/java/jpl/cws/task/RestPostTask.java index fd625cd4..74472a8d 100644 --- a/cws-tasks/src/main/java/jpl/cws/task/RestPostTask.java +++ b/cws-tasks/src/main/java/jpl/cws/task/RestPostTask.java @@ -41,6 +41,10 @@ public class RestPostTask extends CwsTask { private boolean throwOnBadResponseBoolean; private Expression allowInsecureRequests; private boolean allowInsecureRequestsBoolean; + private Expression httpAuthUsername; + private String httpAuthUsernameString; + private Expression httpAuthPassword; + private String httpAuthPasswordString; // CWS token security scheme variables // @@ -60,6 +64,8 @@ public RestPostTask() { public void initParams() throws Exception { urlString = getStringParam(url, "url"); allowInsecureRequestsBoolean = getBooleanParam(allowInsecureRequests, "allowInsecureRequests", DEFAULT_ALLOW_INSECURE_REQUESTS); + httpAuthUsernameString = getStringParam(httpAuthUsername, "httpAuthUsername", null); + httpAuthPasswordString = getStringParam(httpAuthPassword, "httpAuthPassword", null); bodyString = getStringParam(body, "body", DEFAULT_BODY); contentTypeString = getStringParam(contentType, "contentType", DEFAULT_MEDIA_TYPE); throwOnBadResponseBoolean = getBooleanParam(throwOnBadResponse, "throwOnBadResponse", DEFAULT_THROW_ON_BAD_RESPONSE); @@ -91,14 +97,27 @@ public void executeTask() throws Exception { // Make the REST POST // - RestCallResult result = WebUtils.restCall( - urlString, "POST", - bodyString, - cookieToUse, - null, - contentTypeString, - allowInsecureRequestsBoolean); - + RestCallResult result; + if (httpAuthUsername == null) { + // Unauthenticated GET + result = WebUtils.restCall( + urlString, "POST", + bodyString, + cookieToUse, + null, + contentTypeString); + } else { + // Authenticated GET + result = WebUtils.restCall( + urlString, "POST", + bodyString, + cookieToUse, + null, + contentTypeString, + httpAuthUsernameString, + httpAuthPasswordString); + } + int responseCode = result.getResponseCode(); log.info("httpStatusCode: " + responseCode); this.setOutputVariable("httpStatusCode", String.valueOf(responseCode)); @@ -184,6 +203,22 @@ public Expression getAllowInsecureRequests() { public void setAllowInsecureRequests(Expression allowInsecureRequests) { this.allowInsecureRequests = allowInsecureRequests; } + + public Expression getHttpAuthUsername() { + return httpAuthUsername; + } + + public void setHttpAuthUsername(Expression httpAuthUsername) { + this.httpAuthUsername = httpAuthUsername; + } + + public Expression getHttpAuthPassword() { + return httpAuthPassword; + } + + public void setHttpAuthPassword(Expression httpAuthPassword) { + this.httpAuthPassword = httpAuthPassword; + } public Expression getBody() { return body; diff --git a/dev.sh b/dev.sh index afa53f08..96ae2d90 100755 --- a/dev.sh +++ b/dev.sh @@ -12,14 +12,19 @@ DB_PORT=${5} DB_NAME=${6} DB_USER=${7} DB_PASS=${8} -ENABLE_CLOUD_AS=${9} -SECURITY_SCHEME=${10} -THIS_HOSTNAME=${11} -NOTIFICATION_EMAILS=${12} -ADMIN_FIRSTNAME=${13} -ADMIN_LASTNAME=${14} -ADMIN_EMAIL=${15} -NUM_WORKERS=${16} +ES_HOST=${9} +ES_PORT=${10} +ES_USE_AUTH=${11} +ES_USERNAME=${12} +ES_PASSWORD=${13} +ENABLE_CLOUD_AS=${14} +SECURITY_SCHEME=${15} +THIS_HOSTNAME=${16} +NOTIFICATION_EMAILS=${17} +ADMIN_FIRSTNAME=${18} +ADMIN_LASTNAME=${19} +ADMIN_EMAIL=${20} +NUM_WORKERS=${21} source ${ROOT}/utils.sh diff --git a/install/clean_es_history.sh b/install/clean_es_history.sh index 5a6db53f..31704860 100755 --- a/install/clean_es_history.sh +++ b/install/clean_es_history.sh @@ -11,6 +11,9 @@ source ${ROOT}/utils.sh ES_HOST='__ES_HOST__' ES_PORT='__ES_PORT__' +ES_USE_AUTH='__ES_USE_AUTH__' +ES_USERNAME='__ES_USERNAME__' +ES_PASSWORD='__ES_PASSWORD__' DAYS_TO_LIVE=__CWS_HISTORY_DAYS_TO_LIVE__ # Zero padded days using %d instead of %e @@ -43,7 +46,10 @@ echo "$ALLLINES" | while read LINE if [[ ! -z "$FORMATEDLINE" ]] && [[ "$FORMATEDLINE" -lt "$DAYSAGO" ]] then TODELETE=$(echo $LINE | awk '{ print $3 }') - curl -X DELETE http://${ES_HOST}:${ES_PORT}/${TODELETE} + case $ES_USE_AUTH in + ([yY]) curl -X DELETE --user ${ES_USERNAME}:${ES_PASSWORD} http://${ES_HOST}:${ES_PORT}/${TODELETE} ;; + (*) curl -X DELETE http://${ES_HOST}:${ES_PORT}/${TODELETE} ;; + esac sleep 1 fi done diff --git a/install/cws-ui/cws-ui.properties b/install/cws-ui/cws-ui.properties index 63581a55..2394a48f 100644 --- a/install/cws-ui/cws-ui.properties +++ b/install/cws-ui/cws-ui.properties @@ -18,6 +18,9 @@ cws.db.password=__CWS_DB_PASSWORD__ cws.elasticsearch.hostname=__CWS_ES_HOST__ cws.elasticsearch.port=__CWS_ES_PORT__ +cws.elasticsearch.use.auth=__CWS_ES_USE_AUTH__ +cws.elasticsearch.username=__CWS_ES_USERNAME__ +cws.elasticsearch.password=__CWS_ES_PASSWORD__ cws.console.port.ssl=__CWS_CONSOLE_SSL_PORT__ cws.console.app.root=cws-ui diff --git a/install/docker/console-db-es-ls-kibana/config.properties b/install/docker/console-db-es-ls-kibana/config.properties index d4ad93f0..4d4ee816 100644 --- a/install/docker/console-db-es-ls-kibana/config.properties +++ b/install/docker/console-db-es-ls-kibana/config.properties @@ -103,10 +103,10 @@ cws_enable_cloud_autoscaling=n # messaging service (JMS) failures, and authentication server connection failures. cws_notification_emails=[AdminEmailAddresses(comma separated)] -# This allows CWS to use a user provided Elasticsearch service, thus CWS will not install -# nor start its own. If user_provided_elasticsearch=y, then you must provide the -# elasticsearch_host (host name) and elasticsearch_port for CWS to use -user_provided_elasticsearch=y +# This allows CWS to use a user provided Elasticsearch service. +# If you wish to use an unsecured Elasticsearch host, you may do so +# by specifying elasticsearch_use_auth=n below. +elasticsearch_use_auth=n elasticsearch_host=es elasticsearch_port=9200 diff --git a/install/docker/console-db-es-ls-kibana/docker-compose.yml b/install/docker/console-db-es-ls-kibana/docker-compose.yml index d102608f..2b1ddb9b 100644 --- a/install/docker/console-db-es-ls-kibana/docker-compose.yml +++ b/install/docker/console-db-es-ls-kibana/docker-compose.yml @@ -23,7 +23,7 @@ services: labels: com.example.service: "es" com.example.description: "For searching and indexing data" - image: elasticsearch:6.4.2 + image: elasticsearch:7.9.0 container_name: cws-es networks: - frontend @@ -53,7 +53,7 @@ services: labels: com.example.service: "kibana" com.example.description: "Data visualisation and for log aggregation" - image: kibana:6.4.2 + image: kibana:7.9.0 container_name: cws-kibana ports: - "5601:5601" @@ -68,7 +68,7 @@ services: labels: com.example.service: "logstash" com.example.description: "For logging data" - image: docker.elastic.co/logstash/logstash:6.4.2 + image: docker.elastic.co/logstash/logstash:7.9.0 container_name: cws_logstash volumes: - ./cws-logstash.conf:/home/cws_user/cws-logstash.conf:ro diff --git a/install/docker/cws-image/build.sh b/install/docker/cws-image/build.sh index 472c6016..d0ecbfff 100755 --- a/install/docker/cws-image/build.sh +++ b/install/docker/cws-image/build.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -ver='2.0' # update this each CWS release +ver='2.1-pre' # update this each CWS release echo echo "ATTENTION: If changes were made to the source code or config files, don't forget to rebuild the cws .tar.gz package file before running this script." diff --git a/install/docker/worker-ls/docker-compose.yml b/install/docker/worker-ls/docker-compose.yml index 1a4942ff..c0d5a6ed 100644 --- a/install/docker/worker-ls/docker-compose.yml +++ b/install/docker/worker-ls/docker-compose.yml @@ -5,7 +5,7 @@ services: labels: com.example.service: "logstash" com.example.description: "For logging data" - image: docker.elastic.co/logstash/logstash:6.4.2 + image: docker.elastic.co/logstash/logstash:7.9.0 container_name: cws-worker-logstash volumes: - logs-volume:/cws_logs:ro diff --git a/install/elasticsearch/elasticsearch-6.4.2.zip b/install/elasticsearch/elasticsearch-6.4.2.zip deleted file mode 100644 index 9dc6fe52..00000000 Binary files a/install/elasticsearch/elasticsearch-6.4.2.zip and /dev/null differ diff --git a/install/elasticsearch/elasticsearch.yml b/install/elasticsearch/elasticsearch.yml deleted file mode 100644 index 5c68ac4a..00000000 --- a/install/elasticsearch/elasticsearch.yml +++ /dev/null @@ -1,89 +0,0 @@ -# ======================== Elasticsearch Configuration ========================= -# -# NOTE: Elasticsearch comes with reasonable defaults for most settings. -# Before you set out to tweak and tune the configuration, make sure you -# understand what are you trying to accomplish and the consequences. -# -# The primary way of configuring a node is via this file. This template lists -# the most important settings you may want to configure for a production cluster. -# -# Please consult the documentation for further information on configuration options: -# https://www.elastic.co/guide/en/elasticsearch/reference/index.html -# -# ---------------------------------- Cluster ----------------------------------- -# -# Use a descriptive name for your cluster: -# -cluster.name: __CWS_ES_CLUSTERNAME__ -# -# ------------------------------------ Node ------------------------------------ -# -# Use a descriptive name for the node: -# -#node.name: node-1 -# -# Add custom attributes to the node: -# -#node.attr.rack: r1 -# -# ----------------------------------- Paths ------------------------------------ -# -# Path to directory where to store the data (separate multiple locations by comma): -# -#path.data: /path/to/data -# -# Path to log files: -# -#path.logs: /path/to/logs -# -# ----------------------------------- Memory ----------------------------------- -# -# Lock the memory on startup: -# -#bootstrap.memory_lock: true -# -# Make sure that the heap size is set to about half the memory available -# on the system and that the owner of the process is allowed to use this -# limit. -# -# Elasticsearch performs poorly when the system is swapping the memory. -# -# ---------------------------------- Network ----------------------------------- -# -# Set the bind address to a specific IP (IPv4 or IPv6): -# -network.host: __CWS_ES_HOST__ -# -# Set a custom port for HTTP: -# -#http.port: 9200 -# -# For more information, consult the network module documentation. -# -# --------------------------------- Discovery ---------------------------------- -# -# Pass an initial list of hosts to perform discovery when new node is started: -# The default list of hosts is ["127.0.0.1", "[::1]"] -# -#discovery.zen.ping.unicast.hosts: ["host1", "host2"] -discovery.type: single-node -# -# Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1): -# -#discovery.zen.minimum_master_nodes: -# -# For more information, consult the zen discovery module documentation. -# -# ---------------------------------- Gateway ----------------------------------- -# -# Block initial recovery after a full cluster restart until N nodes are started: -# -#gateway.recover_after_nodes: 3 -# -# For more information, consult the gateway module documentation. -# -# ---------------------------------- Various ----------------------------------- -# -# Require explicit names when deleting indices: -# -#action.destructive_requires_name: true diff --git a/install/elasticsearch/jvm.options b/install/elasticsearch/jvm.options deleted file mode 100644 index ee591e05..00000000 --- a/install/elasticsearch/jvm.options +++ /dev/null @@ -1,105 +0,0 @@ -## JVM configuration - -################################################################ -## IMPORTANT: JVM heap size -################################################################ -## -## You should always set the min and max JVM heap -## size to the same value. For example, to set -## the heap to 4 GB, set: -## -## -Xms4g -## -Xmx4g -## -## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html -## for more information -## -################################################################ - -# Xms represents the initial size of total heap space -# Xmx represents the maximum size of total heap space - --Xms__CWS_ES_HEAPSIZE__g --Xmx__CWS_ES_HEAPSIZE__g - -################################################################ -## Expert settings -################################################################ -## -## All settings below this section are considered -## expert settings. Don't tamper with them unless -## you understand what you are doing -## -################################################################ - -## GC configuration --XX:+UseConcMarkSweepGC --XX:CMSInitiatingOccupancyFraction=75 --XX:+UseCMSInitiatingOccupancyOnly - -## optimizations - -# pre-touch memory pages used by the JVM during initialization --XX:+AlwaysPreTouch - -## basic - -# explicitly set the stack size --Xss1m - -# set to headless, just in case --Djava.awt.headless=true - -# ensure UTF-8 encoding by default (e.g. filenames) --Dfile.encoding=UTF-8 - -# use our provided JNA always versus the system one --Djna.nosys=true - -# turn off a JDK optimization that throws away stack traces for common -# exceptions because stack traces are important for debugging --XX:-OmitStackTraceInFastThrow - -# flags to configure Netty --Dio.netty.noUnsafe=true --Dio.netty.noKeySetOptimization=true --Dio.netty.recycler.maxCapacityPerThread=0 - -# log4j 2 --Dlog4j.shutdownHookEnabled=false --Dlog4j2.disable.jmx=true - --Djava.io.tmpdir=${ES_TMPDIR} - -## heap dumps - -# generate a heap dump when an allocation from the Java heap fails -# heap dumps are created in the working directory of the JVM --XX:+HeapDumpOnOutOfMemoryError - -# specify an alternative path for heap dumps; ensure the directory exists and -# has sufficient space --XX:HeapDumpPath=data - -# specify an alternative path for JVM fatal error logs --XX:ErrorFile=logs/hs_err_pid%p.log - -## JDK 8 GC logging - -8:-XX:+PrintGCDetails -8:-XX:+PrintGCDateStamps -8:-XX:+PrintTenuringDistribution -8:-XX:+PrintGCApplicationStoppedTime -8:-Xloggc:logs/gc.log -8:-XX:+UseGCLogFileRotation -8:-XX:NumberOfGCLogFiles=32 -8:-XX:GCLogFileSize=64m - -# JDK 9+ GC logging -9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m -# due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise -# time/date parsing will break in an incompatible way for some date patterns and locals -9-:-Djava.locale.providers=COMPAT - -# temporary workaround for C2 bug with JDK 10 on hardware with AVX-512 -10-:-XX:UseAVX=2 diff --git a/install/example-cws-configuration.properties b/install/example-cws-configuration.properties index 5cae32e9..7f426c57 100644 --- a/install/example-cws-configuration.properties +++ b/install/example-cws-configuration.properties @@ -113,12 +113,14 @@ metrics_publishing_interval=10 # messaging service (JMS) failures, and authentication server connection failures. cws_notification_emails=[AdminEmailAddresses(comma separated)] -# This allows CWS to use a user provided Elasticsearch service, thus CWS will not install -# nor start its own. If user_provided_elasticsearch=y, then you must provide the -# elasticsearch_host (host name) and elasticsearch_port for CWS to use -user_provided_elasticsearch=n +# This allows CWS to use a user provided Elasticsearch service. +# If you wish to use an unsecured Elasticsearch host, you may do so +# by specifying elasticsearch_use_auth=n below. elasticsearch_host=[YourElasticsearchHost] elasticsearch_port=9200 +elasticsearch_use_auth=n +elasticsearch_username=[YourElasticsearchUsername] +elasticsearch_password=[YourElasticsearchPassword] # This allows CWS to use a user provided Logstash service, thus CWS will not install # nor start its own. @@ -151,4 +153,4 @@ cws_token_expiration_hours=24 # # Must be specified to use S3Initiator # aws_sqs_dispatcher_sqsUrl=[YourSQSUrl] -# aws_sqs_dispatcher_msgFetchLimit=[1-10, Defaults to 1] \ No newline at end of file +# aws_sqs_dispatcher_msgFetchLimit=[1-10, Defaults to 1] diff --git a/install/installerPresets.properties b/install/installerPresets.properties index cd4ddc5e..f6a9330d 100644 --- a/install/installerPresets.properties +++ b/install/installerPresets.properties @@ -14,7 +14,7 @@ default_startup_autoregister_process_defs=false default_cws_token_expiration_hours=24 default_smtp_hostname=smtp.localhost default_smtp_port=25 -default_user_provided_elasticsearch=n +default_elasticsearch_use_auth=n default_elasticsearch_port=9200 default_user_provided_logstash=n default_history_level=full diff --git a/install/launch_es.sh b/install/launch_es.sh deleted file mode 100755 index 1bc741ec..00000000 --- a/install/launch_es.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# ------------ -# launch_es.sh -# ------------ -# Launches an instance of Elasticsearch. - -ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -source ${ROOT}/utils.sh - -ES_VER=$1 -export JAVA_HOME="$2" - -print "ulimit = `ulimit -u`" -print "Starting up Elasticsearch (version ${ES_VER})..." - -${ROOT}/server/elasticsearch-${ES_VER}/bin/elasticsearch - -print "Now running $(ps -ef | grep [e]lasticsearch | grep -v grep |wc -l) Elasticsearch processes" diff --git a/install/logging/cws-logstash.conf b/install/logging/cws-logstash.conf index ff2130ff..4cdbe92b 100644 --- a/install/logging/cws-logstash.conf +++ b/install/logging/cws-logstash.conf @@ -46,5 +46,11 @@ filter { } output { - elasticsearch { hosts => ["__CWS_ES_HOST__:__CWS_ES_PORT__"] } + elasticsearch { + hosts => ["__CWS_ES_HOST__:__CWS_ES_PORT__"] + ilm_enabled => false + ssl => __LOGSTASH_ES_USE_SSL__ + __LOGSTASH_ES_USERNAME__ + __LOGSTASH_ES_PASSWORD__ + } } diff --git a/install/start_cws.sh b/install/start_cws.sh index aafe6be7..acc62a95 100755 --- a/install/start_cws.sh +++ b/install/start_cws.sh @@ -147,14 +147,12 @@ print "CWS install type is ${INSTALL_TYPE}" # ============================ TOMCAT_PROC_COUNT=$(remaining_procs ${CWS_TOMCAT_HOME}/bin/bootstrap.jar) LS_PROC_COUNT=$(remaining_procs ${ROOT}/server/logstash-${LOGSTASH_VER}) -ES_PROC_COUNT=$(remaining_procs ${ROOT}/server/elasticsearch-${ELASTICSEARCH_VER}) -if [[ ${TOMCAT_PROC_COUNT} -gt 0 ]] || [[ ${LS_PROC_COUNT} -gt 0 ]] || [[ ${ES_PROC_COUNT} -gt 0 ]] +if [[ ${TOMCAT_PROC_COUNT} -gt 0 ]] || [[ ${LS_PROC_COUNT} -gt 0 ]] then print "---------------------------------------------------------------------------" print "${TOMCAT_PROC_COUNT} ACTIVE TOMCAT PROCS (${CWS_TOMCAT_HOME})" print "${LS_PROC_COUNT} ACTIVE LOGSTASH PROCS (${ROOT}/server/logstash-${LOGSTASH_VER})" - print "${ES_PROC_COUNT} ACTIVE ELASTICSEARCH PROCS (${ROOT}/server/elasticsearch-${ELASTICSEARCH_VER})" print print "+--------------------------------------------------------------+" print "| It appears that some CWS process(es) are still running. |" @@ -234,25 +232,6 @@ else nohup ${CWS_TOMCAT_HOME}/bin/catalina.sh jpda start fi -# =================== -# START ELASTICSEARCH -# =================== -# Start ES only for console installations where user is not providing their own Elasticsearch -if [[ "${INSTALL_TYPE}" = "1" ]] || [[ "${INSTALL_TYPE}" = "2" ]]; then - USER_ES=`grep ^user_provided_elasticsearch ${CWS_INSTALLER_CONFIG_FILE} | grep -v "#" | cut -d"=" -f 2` - - if [[ ${USER_ES} =~ $(echo "^(n|N)$") ]]; then - nohup env -i ${ROOT}/launch_es.sh ${ELASTICSEARCH_VER} ${JAVA_HOME} > ${ROOT}/logs/cws_launch_es.log 2>&1 & - - if [[ $? -ne 0 ]]; then - print "ERROR: Problem launching Elasticsearch. Please check the log under ${ROOT}/logs/cws_launch_es.log." - exit 1 - fi - else - print "Using user provided Elasticsearch..." - fi -fi - # ============== # START LOGSTASH # ============== diff --git a/install/stop_cws.sh b/install/stop_cws.sh index b7b27959..6531bff4 100755 --- a/install/stop_cws.sh +++ b/install/stop_cws.sh @@ -101,13 +101,6 @@ LS_KILL_PATTERN=${ROOT}/server/logstash-${LOGSTASH_VER} kill_proc "Logstash" ${LS_KILL_PATTERN} -# =========================== -# KILL ELASTICSEARCH PROCESS -# =========================== -ES_KILL_PATTERN=${ROOT}/server/elasticsearch-${ELASTICSEARCH_VER} - -kill_proc "Elasticsearch" ${ES_KILL_PATTERN} - # =================== # TELL TOMCAT TO STOP # =================== @@ -148,11 +141,9 @@ fi # ======================= TOMCAT_PROC_COUNT=$(remaining_procs ${TOMCAT_KILL_PATTERN}) LS_PROC_COUNT=$(remaining_procs ${LS_KILL_PATTERN}) -ES_PROC_COUNT=$(remaining_procs ${ES_KILL_PATTERN}) print "---------------------------------------------------------------------------" print "${TOMCAT_PROC_COUNT} REMAINING TOMCAT PROCS (${CWS_TOMCAT_HOME})" print "${LS_PROC_COUNT} REMAINING LOGSTASH PROCS (${ROOT}/server/logstash-${LOGSTASH_VER})" -print "${ES_PROC_COUNT} REMAINING ELASTICSEARCH PROCS (${ROOT}/server/elasticsearch-${ELASTICSEARCH_VER})" print "If any processes still remain, they should be manually terminated." print "---------------------------------------------------------------------------" diff --git a/utils.sh b/utils.sh index ca42fb80..b87d177f 100644 --- a/utils.sh +++ b/utils.sh @@ -8,8 +8,7 @@ export CWS_VER='2.0' # update this each CWS release export CAMUNDA_VER='7.13.0' export TOMCAT_VER='9.0.33' -export LOGSTASH_VER='6.4.2' -export ELASTICSEARCH_VER='6.4.2' +export LOGSTASH_VER='7.9.0' # Prints the provided string, tagging with the script that called it function print () { @@ -190,16 +189,21 @@ function auto_conf_data () { DB_NAME=${7} DB_USER=${8} DB_PASS=${9} - ENABLE_CLOUD_AS=${10} - SECURITY_SCHEME=${11} - THIS_HOSTNAME=${12} - NOTIFICATION_EMAILS=${13} - ADMIN_FIRSTNAME=${14} - ADMIN_LASTNAME=${15} - ADMIN_EMAIL=${16} - NUM_WORKERS=${17} - - OUTPUT_FILE=${18} + ES_HOST=${10} + ES_PORT=${11} + ES_USE_AUTH=${12} + ES_USERNAME=${13} + ES_PASSWORD=${14} + ENABLE_CLOUD_AS=${15} + SECURITY_SCHEME=${16} + THIS_HOSTNAME=${17} + NOTIFICATION_EMAILS=${18} + ADMIN_FIRSTNAME=${19} + ADMIN_LASTNAME=${20} + ADMIN_EMAIL=${21} + NUM_WORKERS=${22} + + OUTPUT_FILE=${23} source ${ROOT}/utils.sh @@ -258,6 +262,8 @@ function auto_conf_data () { exit 1; fi + print ${OUTPUT_FILE} + cat > ${OUTPUT_FILE} <<- EOF hostname=${THIS_HOSTNAME} install_type=${INSTALL_TYPE_CODE} @@ -268,6 +274,11 @@ function auto_conf_data () { database_name=${DB_NAME} database_username=${DB_USER} database_password=${DB_PASS} + elasticsearch_host=${ES_HOST} + elasticsearch_port=${ES_PORT} + elasticsearch_use_auth=${ES_USE_AUTH} + elasticsearch_username=${ES_USERNAME} + elasticsearch_password=${ES_PASSWORD} admin_user=${LDAP_USERNAME} admin_firstname=${ADMIN_FIRSTNAME} admin_lastname=${ADMIN_LASTNAME} @@ -289,7 +300,6 @@ function auto_conf_data () { brand_header=Test Console project_webapp_root=proj cws_enable_cloud_autoscaling=${ENABLE_CLOUD_AS} - elasticsearch_heapsize=4 identity_plugin_type=$(echo ${SECURITY_SCHEME} | tr '[:lower:]' '[:upper:]') cws_ldap_url=${LDAP_SERVER_URL} cws_ldap_url_default=${LDAP_SERVER_URL}