New elk stack

- Updated to elk 8
- Loads of cleanup
- Swapped logstash for a python script
- New startup script sets up the kibana index pattern and dashboard
- Dashboard visualization now look for both 4.2 and 4.3 PEP names
- Must be run with --privileged

irods_audit_elk_stack/Dockerfile

@@ -3,52 +3,228 @@
 #
 # Used in iRODS Training
 #
-FROM ubuntu:latest
-MAINTAINER Justin James "[email protected]"
-
-RUN apt-get update
-RUN apt-get remove --purge openjdk-11-jre
-RUN apt-get remove --purge openjdk-11-jre-headless
-RUN apt-get install -y openjdk-8-jre-headless
-RUN update-java-alternatives --set /usr/lib/jvm/java-1.8.0-openjdk-amd64
-RUN export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::")
-RUN apt-get install -y gnupg curl
-RUN apt-get install -y wget
-
-RUN wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add -
-RUN apt-get -y install apt-transport-https
-RUN echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-6.x.list
-RUN apt-get update && apt-get -y install elasticsearch
-#curl http://localhost:9200
-#RUN curl -XPUT 'http://localhost:9200/irods_audit'
-RUN apt-get -y install logstash
-RUN /usr/share/logstash/bin/logstash-plugin install logstash-input-stomp
-
-RUN printf 'input {\n    # Read the audit_messages queue messages using the stomp protocol.\n    #stomp {\n    #    host        => "localhost"\n    #    destination => "/queue/audit_messages"\n    #    codec       => plain {\n    #                       charset => "ISO-8859-1"\n    #                   }\n    #}\n\n    rabbitmq {\n      host => "localhost"\n      queue => "audit_messages"\n    }\n}\n\nfilter {\n\n    # Remove AMQP header and footer information from message\n    #ruby {\n    #    code => "event.set('message', event.get('message').sub(/.*__BEGIN_JSON__/, ''))\n    #             event.set('message', event.get('message').sub(/__END_JSON__.*/, ''))"\n    #}\n\n    if "_jsonparsefailure" in [tags] {\n        mutate {\n                  gsub => [ "message", "[\\\\]","" ]\n                  gsub => [ "message", ".*__BEGIN_JSON__", ""]\n                  gsub => [ "message", "__END_JSON__", ""]\n\n        } \n        mutate { remove_tag => [ "tags", "_jsonparsefailure" ] }\n        json { source => "message" }\n\n    }\n\n    # Parse the JSON message\n    json {\n        source       => "message"\n        remove_field => ["message"]\n    }\n\n    # Replace @timestamp with the timestamp stored in time_stamp\n    date {\n        match => [ "time_stamp", "UNIX_MS" ]\n    }\n\n    # Convert select fields to integer\n    mutate {\n        convert => { "int" => "integer" }\n        convert => { "int__2" => "integer" }\n        convert => { "int__3" => "integer" }\n        convert => { "file_size" => "integer" }\n    }\n\n}\n\noutput {\n    # Write the output to elastic search under the irods_audit index.\n    elasticsearch {\n        hosts => ["localhost:9200"]\n        index => "irods_audit"\n    }\n    #stdout {\n    #    codec => rubydebug {}\n    #}\n}\n' > /etc/logstash/conf.d/irods_audit.conf
-
-#/usr/share/logstash/bin/logstash&
-RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh |  bash
-RUN wget https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb
-RUN apt-get update
-RUN apt-get -y install erlang
-RUN apt-get -y install rabbitmq-server
-RUN rabbitmq-plugins enable rabbitmq_amqp1_0
-RUN rabbitmq-plugins enable rabbitmq_management
-
-RUN apt-get -y install kibana
+FROM ubuntu:20.04
+
+SHELL [ "/bin/bash", "-c" ]
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Make sure we're starting with an up-to-date image
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get autoremove -y --purge && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+# To mark all installed packages as manually installed:
+#apt-mark showauto | xargs -r apt-mark manual
+
+RUN apt-get update && \
+    apt-get install -y \
+        apt-transport-https \
+        gnupg \
+        curl \
+    && \
+    apt-get install --no-install-recommends -y \
+        software-properties-common \
+        systemd \
+        systemd-sysv \
+        dbus \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+ADD https://packages.adoptium.net/artifactory/api/gpg/key/public /usr/share/keyrings/adoptium.asc
+ADD https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public /usr/share/keyrings/adoptopenjdk.asc
+RUN gpg --dearmor -o /usr/share/keyrings/adoptium.gpg /usr/share/keyrings/adoptium.asc && \
+    gpg --dearmor -o /usr/share/keyrings/adoptopenjdk.gpg /usr/share/keyrings/adoptopenjdk.asc && \
+    echo "deb [signed-by=/usr/share/keyrings/adoptium.gpg] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list && \
+    echo "deb [signed-by=/usr/share/keyrings/adoptopenjdk.gpg] https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptopenjdk.list && \
+    apt-get update && \
+    apt-get install -y \
+        adoptium-ca-certificates \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+#ARG java_ver=8
+#ARG java_ver=11
+#ARG java_vendor=adoptopenjdk
+#ARG java_dist=hotspot-jre
+ARG java_ver=17
+ARG java_vendor=temurin
+ARG java_dist=jdk
+
+RUN apt-get update && \
+    apt-get install -y \
+        ${java_vendor}-${java_ver}-${java_dist} \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+ENV JAVA_HOME=/usr/lib/jvm/${java_vendor}-${java_ver}-${java_dist}-amd64
+RUN update-java-alternatives --set ${JAVA_HOME}
+ENV ES_JAVA_HOME=${JAVA_HOME}
+
+#ARG es_ver=6
+#ARG es_ver=7
+ARG es_ver=8
+ADD https://artifacts.elastic.co/GPG-KEY-elasticsearch /usr/share/keyrings/elasticsearch-keyring.asc
+RUN gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg /usr/share/keyrings/elasticsearch-keyring.asc && \
+    echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/${es_ver}.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-${es_ver}.x.list && \
+    echo 'path-exclude=/usr/share/elasticsearch/jdk' >> /etc/dpkg/dpkg.cfg.d/excludes-elasticsearch-jvm && \
+    echo 'path-exclude=/usr/share/elasticsearch/jdk/*' >> /etc/dpkg/dpkg.cfg.d/excludes-elasticsearch-jvm
+
+RUN apt-get update && \
+    apt-get install -y \
+        elasticsearch \
+        kibana \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+RUN echo "ES_JAVA_HOME=\"${ES_JAVA_HOME}\"" >> /etc/default/elasticsearch
+
+ADD https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey /usr/share/keyrings/rabbitmq_rabbitmq-server.asc
+RUN add-apt-repository --no-update -y ppa:rabbitmq/rabbitmq-erlang && \
+    gpg --dearmor -o /usr/share/keyrings/rabbitmq_rabbitmq-server.gpg /usr/share/keyrings/rabbitmq_rabbitmq-server.asc && \
+    echo "deb [signed-by=/usr/share/keyrings/rabbitmq_rabbitmq-server.gpg] https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/rabbitmq_rabbitmq-server.list && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+ADD https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc /usr/share/keyrings/erlang_solutions.asc
+#RUN gpg --dearmor -o /usr/share/keyrings/erlang_solutions.gpg /usr/share/keyrings/erlang_solutions.asc && \
+#    echo "deb [signed-by=/usr/share/keyrings/erlang_solutions.gpg] https://packages.erlang-solutions.com/ubuntu $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) contrib" | tee /etc/apt/sources.list.d/erlang-solutions.list
+RUN gpg --dearmor -o /usr/share/keyrings/erlang_solutions.gpg /usr/share/keyrings/erlang_solutions.asc && \
+    echo "deb [signed-by=/usr/share/keyrings/erlang_solutions.gpg] http://binaries.erlang-solutions.com/debian $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) contrib" | tee /etc/apt/sources.list.d/erlang-solutions.list
+
+RUN apt-get update && \
+    apt-get install -y \
+        rabbitmq-server \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+# Hopefully these are new enough
+RUN apt-get update && \
+    apt-get install -y \
+        python3-qpid-proton \
+        python3-elasticsearch \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+RUN rabbitmq-plugins enable rabbitmq_amqp1_0 && \
+    rabbitmq-plugins enable rabbitmq_management
+
 RUN echo "server.host: \"0.0.0.0\"" >> /etc/kibana/kibana.yml
 
-RUN echo "transport.host: localhost" >> /etc/elasticsearch/elasticsearch.yml
-RUN echo "transport.tcp.port: 9300" >> /etc/elasticsearch/elasticsearch.yml
-RUN echo "http.port: 9200" >> /etc/elasticsearch/elasticsearch.yml
-RUN echo "network.host: 0.0.0.0" >> /etc/elasticsearch/elasticsearch.yml
+COPY elasticsearch.yml /etc/elasticsearch/elasticsearch.yml
+RUN /usr/share/elasticsearch/bin/elasticsearch-keystore remove \
+        xpack.security.http.ssl.keystore.secure_password \
+        xpack.security.transport.ssl.keystore.secure_password \
+        xpack.security.transport.ssl.truststore.secure_password
+
+# utils
+RUN apt-get update && \
+    apt-get install -y \
+        procps \
+        nano \
+        less \
+        iproute2 \
+        file \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+# from ubi8-init
+STOPSIGNAL SIGRTMIN+3
+
+# from ubi8-init
+RUN systemctl mask \
+        systemd-remount-fs.service \
+        dev-hugepages.mount \
+        sys-fs-fuse-connections.mount \
+        systemd-logind.service \
+        getty.target \
+        console-getty.service \
+        systemd-udev-trigger.service \
+        systemd-udevd.service \
+        systemd-random-seed.service
+
+# from ubi8-init
+#mask systemd-machine-id-commit.service - partial fix for https://bugzilla.redhat.com/show_bug.cgi?id=1472439
+RUN systemctl mask systemd-machine-id-commit.service
+
+RUN systemctl mask \
+        unattended-upgrades.service \
+        packagekit-offline-update.service \
+        systemd-timesyncd.service \
+        systemd-resolved.service \
+        apt-daily-upgrade.service \
+        apt-daily-upgrade.timer \
+        apt-daily.service \
+        apt-daily.timer \
+        e2scrub_reap.service \
+        e2scrub_all.service \
+        e2scrub_all.timer \
+        ondemand.service \
+        systemd-modules-load.service \
+        fstrim.service \
+        fstrim.timer
+
+#RUN systemctl mask \
+#        remote-fs.target \
+#        systemd-pstore.service \
+#        cryptsetup.target
+
+RUN systemctl mask \
+        getty-static.service \
+        networkd-dispatcher.service
+
+#RUN systemctl mask \
+#        kmod-static-nodes.service
+#        proc-sys-fs-binfmt_misc.mount \
+#        proc-sys-fs-binfmt_misc.automount \
+#        dev-mqueue.mount \
+#        sys-kernel-config.mount \
+#        sys-kernel-debug.mount \
+#        sys-kernel-tracing.mount \
+#        systemd-ask-password-console.path \
+#        systemd-binfmt.service \
+#        systemd-boot-system-token.service \
+#        systemd-sysctl.service \
+#        systemd-sysusers.service \
+#        systemd-update-utmp.service \
+#        systemd-initctl.socket \
+#        systemd-update-utmp-runlevel.service \
+#        systemd-ask-password-wall.path \
+#        systemd-user-sessions.service
+
+#RUN systemctl mask \
+#        systemd-tmpfiles-setup-dev.service \
+#        systemd-tmpfiles-setup.service \
+#        systemd-tmpfiles-clean.timer \
+#        systemd-tmpfiles-clean.service
+
 
+COPY startup-script.sh /var/lib/irods-elk/
+CMD ["/var/lib/irods-elk/startup-script.sh"]
 
-RUN printf 'chown rabbitmq:rabbitmq /var/lib/rabbitmq/.erlang.cookie\nservice elasticsearch start\nservice logstash start\nservice rabbitmq-server start\nservice kibana start\ncurl http://localhost:9200\ncurl -XPUT "http://localhost:9200/irods_audit"\nrabbitmqctl add_user test test\nrabbitmqctl set_user_tags test administrator\nrabbitmqctl set_permissions -p / test ".*" ".*" ".*"\n/bin/bash\nsleep 20\ncurl -XPUT http://localhost:9200/irods_audit/_settings -H \'Content-Type: application/json\' -d\'{"index.mapping.total_fields.limit": 2000}\''> /startup_script.sh
-RUN chmod +x /startup_script.sh
+RUN mkdir -p /etc/systemd/system/kibana.service.d && \
+    echo "[Unit]" >> /etc/systemd/system/kibana.service.d/elasticsearch.conf && \
+    echo "After=elasticsearch.service" >> /etc/systemd/system/kibana.service.d/elasticsearch.conf && \
+    echo "Wants=elasticsearch.service" >> /etc/systemd/system/kibana.service.d/elasticsearch.conf
 
-CMD /startup_script.sh
+COPY not-logstash.service /etc/systemd/system/
+COPY not-logstash.py /var/lib/irods-elk/
 
+COPY elk-firstrun.service /etc/systemd/system/
+COPY example_kibana_dashboard.ndjson /var/lib/irods-elk/
+COPY firstrun.sh /var/lib/irods-elk/
 
-WORKDIR /home
+RUN systemctl enable \
+        elasticsearch \
+        rabbitmq-server \
+        kibana \
+        not-logstash \
+        elk-firstrun
 
+WORKDIR /root

irods_audit_elk_stack/elasticsearch.yml

@@ -0,0 +1,104 @@
+# ======================== Elasticsearch Configuration =========================
+#
+# NOTE: Elasticsearch comes with reasonable defaults for most settings.
+#       Before you set out to tweak and tune the configuration, make sure you
+#       understand what are you trying to accomplish and the consequences.
+#
+# The primary way of configuring a node is via this file. This template lists
+# the most important settings you may want to configure for a production cluster.
+#
+# Please consult the documentation for further information on configuration options:
+# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
+#
+# ---------------------------------- Cluster -----------------------------------
+#
+# Use a descriptive name for your cluster:
+#
+#cluster.name: my-application
+#
+# ------------------------------------ Node ------------------------------------
+#
+# Use a descriptive name for the node:
+#
+node.name: irods-elk
+#
+# Add custom attributes to the node:
+#
+#node.attr.rack: r1
+#
+# ----------------------------------- Paths ------------------------------------
+#
+# Path to directory where to store the data (separate multiple locations by comma):
+#
+path.data: /var/lib/elasticsearch
+#
+# Path to log files:
+#
+path.logs: /var/log/elasticsearch
+#
+# ----------------------------------- Memory -----------------------------------
+#
+# Lock the memory on startup:
+#
+#bootstrap.memory_lock: true
+#
+# Make sure that the heap size is set to about half the memory available
+# on the system and that the owner of the process is allowed to use this
+# limit.
+#
+# Elasticsearch performs poorly when the system is swapping the memory.
+#
+# ---------------------------------- Network -----------------------------------
+#
+# By default Elasticsearch is only accessible on localhost. Set a different
+# address here to expose this node on the network:
+#
+#network.host: 192.168.0.1
+network.host: 0.0.0.0
+#
+# By default Elasticsearch listens for HTTP traffic on the first free port it
+# finds starting at 9200. Set a specific HTTP port here:
+#
+http.port: 9200
+#
+# For more information, consult the network module documentation.
+#
+# --------------------------------- Discovery ----------------------------------
+#
+# Pass an initial list of hosts to perform discovery when this node is started:
+# The default list of hosts is ["127.0.0.1", "[::1]"]
+#
+#discovery.seed_hosts: ["host1", "host2"]
+#
+# Bootstrap the cluster using an initial set of master-eligible nodes:
+#
+#cluster.initial_master_nodes: ["node-1", "node-2"]
+cluster.initial_master_nodes: ["irods-elk"]
+#
+# For more information, consult the discovery and cluster formation module documentation.
+#
+# --------------------------------- Readiness ----------------------------------
+#
+# Enable an unauthenticated TCP readiness endpoint on localhost
+#
+#readiness.port: 9399
+#
+# ---------------------------------- Various -----------------------------------
+#
+# Allow wildcard deletion of indices:
+#
+#action.destructive_requires_name: false
+#
+# ---------------------------------- Security ----------------------------------
+#
+# Enable/disable security (enabled by default since version 8.0)
+#
+xpack.security.enabled: false
+
+
+
+
+http.host: 0.0.0.0
+
+transport.host: localhost
+transport.port: 9300

irods_audit_elk_stack/elk-firstrun.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=iRODS elk stack first-run setup
+After=rabbitmq-server.service
+After=elasticsearch.service
+After=kibana.service
+Requires=rabbitmq-server.service
+Requires=elasticsearch.service
+Requires=kibana.service
+
+[Service]
+Type=oneshot
+ExecStart=/var/lib/irods-elk/firstrun.sh
+TimeoutSec=600s
+
+[Install]
+WantedBy=multi-user.target