From 9603ce951ca73cef49c1eb34ab8212d95400f1fc Mon Sep 17 00:00:00 2001 From: "Markus Kitsinger (SwooshyCueb)" Date: Wed, 15 Jun 2022 19:16:11 -0400 Subject: [PATCH] New elk stack - Updated to elk 8 - Loads of cleanup - Swapped logstash for a python script - New startup script sets up the kibana index pattern and dashboard - Dashboard visualization now look for both 4.2 and 4.3 PEP names - Must be run with --privileged --- irods_audit_elk_stack/Dockerfile | 258 +++++++++++++++--- irods_audit_elk_stack/elasticsearch.yml | 104 +++++++ irods_audit_elk_stack/elk-firstrun.service | 16 ++ .../example_kibana_dashboard.ndjson | 9 + irods_audit_elk_stack/firstrun.sh | 57 ++++ irods_audit_elk_stack/not-logstash.py | 65 +++++ irods_audit_elk_stack/not-logstash.service | 18 ++ irods_audit_elk_stack/startup-script.sh | 5 + 8 files changed, 491 insertions(+), 41 deletions(-) create mode 100644 irods_audit_elk_stack/elasticsearch.yml create mode 100644 irods_audit_elk_stack/elk-firstrun.service create mode 100644 irods_audit_elk_stack/example_kibana_dashboard.ndjson create mode 100755 irods_audit_elk_stack/firstrun.sh create mode 100755 irods_audit_elk_stack/not-logstash.py create mode 100644 irods_audit_elk_stack/not-logstash.service create mode 100755 irods_audit_elk_stack/startup-script.sh diff --git a/irods_audit_elk_stack/Dockerfile b/irods_audit_elk_stack/Dockerfile index fdf4c80..193478c 100644 --- a/irods_audit_elk_stack/Dockerfile +++ b/irods_audit_elk_stack/Dockerfile @@ -3,52 +3,228 @@ # # Used in iRODS Training # -FROM ubuntu:latest -MAINTAINER Justin James "jjames@renci.org" - -RUN apt-get update -RUN apt-get remove --purge openjdk-11-jre -RUN apt-get remove --purge openjdk-11-jre-headless -RUN apt-get install -y openjdk-8-jre-headless -RUN update-java-alternatives --set /usr/lib/jvm/java-1.8.0-openjdk-amd64 -RUN export JAVA_HOME=$(readlink -f /usr/bin/java | sed "s:bin/java::") -RUN apt-get install -y gnupg curl -RUN apt-get install -y wget - -RUN wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | apt-key add - -RUN apt-get -y install apt-transport-https -RUN echo "deb https://artifacts.elastic.co/packages/6.x/apt stable main" | tee -a /etc/apt/sources.list.d/elastic-6.x.list -RUN apt-get update && apt-get -y install elasticsearch -#curl http://localhost:9200 -#RUN curl -XPUT 'http://localhost:9200/irods_audit' -RUN apt-get -y install logstash -RUN /usr/share/logstash/bin/logstash-plugin install logstash-input-stomp - -RUN printf 'input {\n # Read the audit_messages queue messages using the stomp protocol.\n #stomp {\n # host => "localhost"\n # destination => "/queue/audit_messages"\n # codec => plain {\n # charset => "ISO-8859-1"\n # }\n #}\n\n rabbitmq {\n host => "localhost"\n queue => "audit_messages"\n }\n}\n\nfilter {\n\n # Remove AMQP header and footer information from message\n #ruby {\n # code => "event.set('message', event.get('message').sub(/.*__BEGIN_JSON__/, ''))\n # event.set('message', event.get('message').sub(/__END_JSON__.*/, ''))"\n #}\n\n if "_jsonparsefailure" in [tags] {\n mutate {\n gsub => [ "message", "[\\\\]","" ]\n gsub => [ "message", ".*__BEGIN_JSON__", ""]\n gsub => [ "message", "__END_JSON__", ""]\n\n } \n mutate { remove_tag => [ "tags", "_jsonparsefailure" ] }\n json { source => "message" }\n\n }\n\n # Parse the JSON message\n json {\n source => "message"\n remove_field => ["message"]\n }\n\n # Replace @timestamp with the timestamp stored in time_stamp\n date {\n match => [ "time_stamp", "UNIX_MS" ]\n }\n\n # Convert select fields to integer\n mutate {\n convert => { "int" => "integer" }\n convert => { "int__2" => "integer" }\n convert => { "int__3" => "integer" }\n convert => { "file_size" => "integer" }\n }\n\n}\n\noutput {\n # Write the output to elastic search under the irods_audit index.\n elasticsearch {\n hosts => ["localhost:9200"]\n index => "irods_audit"\n }\n #stdout {\n # codec => rubydebug {}\n #}\n}\n' > /etc/logstash/conf.d/irods_audit.conf - -#/usr/share/logstash/bin/logstash& -RUN curl -s https://packagecloud.io/install/repositories/rabbitmq/rabbitmq-server/script.deb.sh | bash -RUN wget https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb -RUN apt-get update -RUN apt-get -y install erlang -RUN apt-get -y install rabbitmq-server -RUN rabbitmq-plugins enable rabbitmq_amqp1_0 -RUN rabbitmq-plugins enable rabbitmq_management - -RUN apt-get -y install kibana +FROM ubuntu:20.04 + +SHELL [ "/bin/bash", "-c" ] +ENV DEBIAN_FRONTEND=noninteractive + +# Make sure we're starting with an up-to-date image +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get autoremove -y --purge && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* +# To mark all installed packages as manually installed: +#apt-mark showauto | xargs -r apt-mark manual + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + gnupg \ + curl \ + && \ + apt-get install --no-install-recommends -y \ + software-properties-common \ + systemd \ + systemd-sysv \ + dbus \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +ADD https://packages.adoptium.net/artifactory/api/gpg/key/public /usr/share/keyrings/adoptium.asc +ADD https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public /usr/share/keyrings/adoptopenjdk.asc +RUN gpg --dearmor -o /usr/share/keyrings/adoptium.gpg /usr/share/keyrings/adoptium.asc && \ + gpg --dearmor -o /usr/share/keyrings/adoptopenjdk.gpg /usr/share/keyrings/adoptopenjdk.asc && \ + echo "deb [signed-by=/usr/share/keyrings/adoptium.gpg] https://packages.adoptium.net/artifactory/deb $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptium.list && \ + echo "deb [signed-by=/usr/share/keyrings/adoptopenjdk.gpg] https://adoptopenjdk.jfrog.io/adoptopenjdk/deb/ $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/adoptopenjdk.list && \ + apt-get update && \ + apt-get install -y \ + adoptium-ca-certificates \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +#ARG java_ver=8 +#ARG java_ver=11 +#ARG java_vendor=adoptopenjdk +#ARG java_dist=hotspot-jre +ARG java_ver=17 +ARG java_vendor=temurin +ARG java_dist=jdk + +RUN apt-get update && \ + apt-get install -y \ + ${java_vendor}-${java_ver}-${java_dist} \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* +ENV JAVA_HOME=/usr/lib/jvm/${java_vendor}-${java_ver}-${java_dist}-amd64 +RUN update-java-alternatives --set ${JAVA_HOME} +ENV ES_JAVA_HOME=${JAVA_HOME} + +#ARG es_ver=6 +#ARG es_ver=7 +ARG es_ver=8 +ADD https://artifacts.elastic.co/GPG-KEY-elasticsearch /usr/share/keyrings/elasticsearch-keyring.asc +RUN gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg /usr/share/keyrings/elasticsearch-keyring.asc && \ + echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/${es_ver}.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-${es_ver}.x.list && \ + echo 'path-exclude=/usr/share/elasticsearch/jdk' >> /etc/dpkg/dpkg.cfg.d/excludes-elasticsearch-jvm && \ + echo 'path-exclude=/usr/share/elasticsearch/jdk/*' >> /etc/dpkg/dpkg.cfg.d/excludes-elasticsearch-jvm + +RUN apt-get update && \ + apt-get install -y \ + elasticsearch \ + kibana \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN echo "ES_JAVA_HOME=\"${ES_JAVA_HOME}\"" >> /etc/default/elasticsearch + +ADD https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey /usr/share/keyrings/rabbitmq_rabbitmq-server.asc +RUN add-apt-repository --no-update -y ppa:rabbitmq/rabbitmq-erlang && \ + gpg --dearmor -o /usr/share/keyrings/rabbitmq_rabbitmq-server.gpg /usr/share/keyrings/rabbitmq_rabbitmq-server.asc && \ + echo "deb [signed-by=/usr/share/keyrings/rabbitmq_rabbitmq-server.gpg] https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/rabbitmq_rabbitmq-server.list && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +ADD https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc /usr/share/keyrings/erlang_solutions.asc +#RUN gpg --dearmor -o /usr/share/keyrings/erlang_solutions.gpg /usr/share/keyrings/erlang_solutions.asc && \ +# echo "deb [signed-by=/usr/share/keyrings/erlang_solutions.gpg] https://packages.erlang-solutions.com/ubuntu $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) contrib" | tee /etc/apt/sources.list.d/erlang-solutions.list +RUN gpg --dearmor -o /usr/share/keyrings/erlang_solutions.gpg /usr/share/keyrings/erlang_solutions.asc && \ + echo "deb [signed-by=/usr/share/keyrings/erlang_solutions.gpg] http://binaries.erlang-solutions.com/debian $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) contrib" | tee /etc/apt/sources.list.d/erlang-solutions.list + +RUN apt-get update && \ + apt-get install -y \ + rabbitmq-server \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +# Hopefully these are new enough +RUN apt-get update && \ + apt-get install -y \ + python3-qpid-proton \ + python3-elasticsearch \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN rabbitmq-plugins enable rabbitmq_amqp1_0 && \ + rabbitmq-plugins enable rabbitmq_management + RUN echo "server.host: \"0.0.0.0\"" >> /etc/kibana/kibana.yml -RUN echo "transport.host: localhost" >> /etc/elasticsearch/elasticsearch.yml -RUN echo "transport.tcp.port: 9300" >> /etc/elasticsearch/elasticsearch.yml -RUN echo "http.port: 9200" >> /etc/elasticsearch/elasticsearch.yml -RUN echo "network.host: 0.0.0.0" >> /etc/elasticsearch/elasticsearch.yml +COPY elasticsearch.yml /etc/elasticsearch/elasticsearch.yml +RUN /usr/share/elasticsearch/bin/elasticsearch-keystore remove \ + xpack.security.http.ssl.keystore.secure_password \ + xpack.security.transport.ssl.keystore.secure_password \ + xpack.security.transport.ssl.truststore.secure_password + +# utils +RUN apt-get update && \ + apt-get install -y \ + procps \ + nano \ + less \ + iproute2 \ + file \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +# from ubi8-init +STOPSIGNAL SIGRTMIN+3 + +# from ubi8-init +RUN systemctl mask \ + systemd-remount-fs.service \ + dev-hugepages.mount \ + sys-fs-fuse-connections.mount \ + systemd-logind.service \ + getty.target \ + console-getty.service \ + systemd-udev-trigger.service \ + systemd-udevd.service \ + systemd-random-seed.service + +# from ubi8-init +#mask systemd-machine-id-commit.service - partial fix for https://bugzilla.redhat.com/show_bug.cgi?id=1472439 +RUN systemctl mask systemd-machine-id-commit.service + +RUN systemctl mask \ + unattended-upgrades.service \ + packagekit-offline-update.service \ + systemd-timesyncd.service \ + systemd-resolved.service \ + apt-daily-upgrade.service \ + apt-daily-upgrade.timer \ + apt-daily.service \ + apt-daily.timer \ + e2scrub_reap.service \ + e2scrub_all.service \ + e2scrub_all.timer \ + ondemand.service \ + systemd-modules-load.service \ + fstrim.service \ + fstrim.timer + +#RUN systemctl mask \ +# remote-fs.target \ +# systemd-pstore.service \ +# cryptsetup.target + +RUN systemctl mask \ + getty-static.service \ + networkd-dispatcher.service + +#RUN systemctl mask \ +# kmod-static-nodes.service +# proc-sys-fs-binfmt_misc.mount \ +# proc-sys-fs-binfmt_misc.automount \ +# dev-mqueue.mount \ +# sys-kernel-config.mount \ +# sys-kernel-debug.mount \ +# sys-kernel-tracing.mount \ +# systemd-ask-password-console.path \ +# systemd-binfmt.service \ +# systemd-boot-system-token.service \ +# systemd-sysctl.service \ +# systemd-sysusers.service \ +# systemd-update-utmp.service \ +# systemd-initctl.socket \ +# systemd-update-utmp-runlevel.service \ +# systemd-ask-password-wall.path \ +# systemd-user-sessions.service + +#RUN systemctl mask \ +# systemd-tmpfiles-setup-dev.service \ +# systemd-tmpfiles-setup.service \ +# systemd-tmpfiles-clean.timer \ +# systemd-tmpfiles-clean.service + +COPY startup-script.sh /var/lib/irods-elk/ +CMD ["/var/lib/irods-elk/startup-script.sh"] -RUN printf 'chown rabbitmq:rabbitmq /var/lib/rabbitmq/.erlang.cookie\nservice elasticsearch start\nservice logstash start\nservice rabbitmq-server start\nservice kibana start\ncurl http://localhost:9200\ncurl -XPUT "http://localhost:9200/irods_audit"\nrabbitmqctl add_user test test\nrabbitmqctl set_user_tags test administrator\nrabbitmqctl set_permissions -p / test ".*" ".*" ".*"\n/bin/bash\nsleep 20\ncurl -XPUT http://localhost:9200/irods_audit/_settings -H \'Content-Type: application/json\' -d\'{"index.mapping.total_fields.limit": 2000}\''> /startup_script.sh -RUN chmod +x /startup_script.sh +RUN mkdir -p /etc/systemd/system/kibana.service.d && \ + echo "[Unit]" >> /etc/systemd/system/kibana.service.d/elasticsearch.conf && \ + echo "After=elasticsearch.service" >> /etc/systemd/system/kibana.service.d/elasticsearch.conf && \ + echo "Wants=elasticsearch.service" >> /etc/systemd/system/kibana.service.d/elasticsearch.conf -CMD /startup_script.sh +COPY not-logstash.service /etc/systemd/system/ +COPY not-logstash.py /var/lib/irods-elk/ +COPY elk-firstrun.service /etc/systemd/system/ +COPY example_kibana_dashboard.ndjson /var/lib/irods-elk/ +COPY firstrun.sh /var/lib/irods-elk/ -WORKDIR /home +RUN systemctl enable \ + elasticsearch \ + rabbitmq-server \ + kibana \ + not-logstash \ + elk-firstrun +WORKDIR /root diff --git a/irods_audit_elk_stack/elasticsearch.yml b/irods_audit_elk_stack/elasticsearch.yml new file mode 100644 index 0000000..1dab07c --- /dev/null +++ b/irods_audit_elk_stack/elasticsearch.yml @@ -0,0 +1,104 @@ +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please consult the documentation for further information on configuration options: +# https://www.elastic.co/guide/en/elasticsearch/reference/index.html +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +#cluster.name: my-application +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +node.name: irods-elk +# +# Add custom attributes to the node: +# +#node.attr.rack: r1 +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): +# +path.data: /var/lib/elasticsearch +# +# Path to log files: +# +path.logs: /var/log/elasticsearch +# +# ----------------------------------- Memory ----------------------------------- +# +# Lock the memory on startup: +# +#bootstrap.memory_lock: true +# +# Make sure that the heap size is set to about half the memory available +# on the system and that the owner of the process is allowed to use this +# limit. +# +# Elasticsearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# By default Elasticsearch is only accessible on localhost. Set a different +# address here to expose this node on the network: +# +#network.host: 192.168.0.1 +network.host: 0.0.0.0 +# +# By default Elasticsearch listens for HTTP traffic on the first free port it +# finds starting at 9200. Set a specific HTTP port here: +# +http.port: 9200 +# +# For more information, consult the network module documentation. +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when this node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +#discovery.seed_hosts: ["host1", "host2"] +# +# Bootstrap the cluster using an initial set of master-eligible nodes: +# +#cluster.initial_master_nodes: ["node-1", "node-2"] +cluster.initial_master_nodes: ["irods-elk"] +# +# For more information, consult the discovery and cluster formation module documentation. +# +# --------------------------------- Readiness ---------------------------------- +# +# Enable an unauthenticated TCP readiness endpoint on localhost +# +#readiness.port: 9399 +# +# ---------------------------------- Various ----------------------------------- +# +# Allow wildcard deletion of indices: +# +#action.destructive_requires_name: false +# +# ---------------------------------- Security ---------------------------------- +# +# Enable/disable security (enabled by default since version 8.0) +# +xpack.security.enabled: false + + + + +http.host: 0.0.0.0 + +transport.host: localhost +transport.port: 9300 diff --git a/irods_audit_elk_stack/elk-firstrun.service b/irods_audit_elk_stack/elk-firstrun.service new file mode 100644 index 0000000..9830647 --- /dev/null +++ b/irods_audit_elk_stack/elk-firstrun.service @@ -0,0 +1,16 @@ +[Unit] +Description=iRODS elk stack first-run setup +After=rabbitmq-server.service +After=elasticsearch.service +After=kibana.service +Requires=rabbitmq-server.service +Requires=elasticsearch.service +Requires=kibana.service + +[Service] +Type=oneshot +ExecStart=/var/lib/irods-elk/firstrun.sh +TimeoutSec=600s + +[Install] +WantedBy=multi-user.target diff --git a/irods_audit_elk_stack/example_kibana_dashboard.ndjson b/irods_audit_elk_stack/example_kibana_dashboard.ndjson new file mode 100644 index 0000000..e2c2511 --- /dev/null +++ b/irods_audit_elk_stack/example_kibana_dashboard.ndjson @@ -0,0 +1,9 @@ +{"attributes":{"timeFieldName":"@timestamp","title":"irods_audit"},"coreMigrationVersion":"8.3.1","id":"irods-audit-pattern","migrationVersion":{"index-pattern":"8.0.0"},"references":[],"type":"index-pattern","updated_at":"2022-07-12T19:13:10.517Z","version":"WzksMV0="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":{\"terms\":{\"rule_name\":[\"audit_pep_auth_agent_auth_request_pre\",\"audit_pep_api_authenticate_pre\"]}},\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Unique Users Per Minute","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Unique Users Per Minute\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"params\":{\"field\":\"user_user_name.keyword\",\"emptyAsNull\":false},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"extendToTimeRange\":false,\"scaleMetricValues\":true,\"interval\":\"m\",\"used_interval\":\"1m\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"hostname.keyword\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{},\"type\":\"line\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"filter\":true},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\",\"setYExtents\":false,\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Unique count of user_user_name.keyword\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"line\",\"mode\":\"normal\",\"data\":{\"label\":\"Unique count of user_user_name.keyword\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"radiusRatio\":9,\"circlesRadius\":1}],\"legendPosition\":\"right\",\"palette\":{\"type\":\"palette\",\"name\":\"kibana_palette\"},\"isVislibVis\":true,\"detailedTooltip\":true,\"fittingFunction\":\"linear\",\"legendSize\":\"auto\",\"truncateLegend\":true,\"maxLegendLines\":1,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"coreMigrationVersion":"8.3.1","id":"Unique-Users-Per-Minute","migrationVersion":{"visualization":"8.3.0"},"references":[{"id":"irods-audit-pattern","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-07-12T19:16:37.414Z","version":"WzIwNSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":{\"terms\":{\"rule_name\":[\"audit_pep_auth_agent_auth_request_pre\",\"audit_pep_api_authenticate_pre\"]}},\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Top Users","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Top Users\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{\"emptyAsNull\":false},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"user_user_name.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"extendToTimeRange\":false,\"scaleMetricValues\":true,\"interval\":\"m\",\"used_interval\":\"1m\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"radiusRatio\":9,\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"showCircles\":true,\"smoothLines\":false,\"times\":[],\"yAxis\":{},\"type\":\"line\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"filter\":true},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\",\"setYExtents\":false,\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"line\",\"mode\":\"normal\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"radiusRatio\":9,\"circlesRadius\":1}],\"legendPosition\":\"right\",\"palette\":{\"type\":\"palette\",\"name\":\"kibana_palette\"},\"isVislibVis\":true,\"detailedTooltip\":true,\"fittingFunction\":\"linear\",\"legendSize\":\"auto\",\"truncateLegend\":true,\"maxLegendLines\":1,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"coreMigrationVersion":"8.3.1","id":"Top-Users","migrationVersion":{"visualization":"8.3.0"},"references":[{"id":"irods-audit-pattern","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-07-12T19:16:31.525Z","version":"WzE5MCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":{\"terms\":{\"rule_name\":[\"audit_pep_auth_agent_auth_request_pre\",\"audit_pep_api_authenticate_pre\"]}},\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Top Client IPs","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Top Client IPs\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{\"emptyAsNull\":false},\"schema\":\"metric\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"client_addr.keyword\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"},{\"id\":\"4\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"extendToTimeRange\":false,\"scaleMetricValues\":true,\"interval\":\"m\",\"used_interval\":\"1m\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"radiusRatio\":9,\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"showCircles\":true,\"smoothLines\":false,\"times\":[],\"yAxis\":{},\"type\":\"line\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"filter\":true},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\",\"setYExtents\":false,\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"line\",\"mode\":\"normal\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"radiusRatio\":9,\"circlesRadius\":1}],\"legendPosition\":\"right\",\"palette\":{\"type\":\"palette\",\"name\":\"kibana_palette\"},\"isVislibVis\":true,\"detailedTooltip\":true,\"fittingFunction\":\"linear\",\"legendSize\":\"auto\",\"truncateLegend\":true,\"maxLegendLines\":1,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"coreMigrationVersion":"8.3.1","id":"Top-Client-IPs","migrationVersion":{"visualization":"8.3.0"},"references":[{"id":"irods-audit-pattern","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-07-12T19:15:30.247Z","version":"WzExNywxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":{\"terms\":{\"rule_name\":[\"audit_pep_auth_agent_auth_request_pre\",\"audit_pep_api_authenticate_pre\"]}},\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Connections Per Minute","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Connections Per Minute\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"params\":{\"emptyAsNull\":false},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"@timestamp\",\"timeRange\":{\"from\":\"now-15m\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"extendToTimeRange\":false,\"scaleMetricValues\":true,\"interval\":\"m\",\"used_interval\":\"1m\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{},\"json\":\"\"},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"hostname.keyword\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":10,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"radiusRatio\":9,\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"showCircles\":true,\"smoothLines\":false,\"times\":[],\"yAxis\":{},\"type\":\"line\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"filter\":true},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\",\"setYExtents\":false,\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"line\",\"mode\":\"normal\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"radiusRatio\":9,\"circlesRadius\":1}],\"legendPosition\":\"right\",\"palette\":{\"type\":\"palette\",\"name\":\"kibana_palette\"},\"isVislibVis\":true,\"detailedTooltip\":true,\"fittingFunction\":\"linear\",\"legendSize\":\"auto\",\"truncateLegend\":true,\"maxLegendLines\":1,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"coreMigrationVersion":"8.3.1","id":"Connections-Per-Minute","migrationVersion":{"visualization":"8.3.0"},"references":[{"id":"irods-audit-pattern","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-07-12T19:15:22.518Z","version":"WzEwNCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":{\"term\":{\"rule_name\":\"audit_pep_resource_read_post\"}},\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Bytes Read Per Minute","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Bytes Read Per Minute\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"stacked\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{},\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"filter\":true},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\",\"setYExtents\":false,\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of int\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of int\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"circlesRadius\":1}],\"legendPosition\":\"right\",\"palette\":{\"type\":\"palette\",\"name\":\"kibana_palette\"},\"isVislibVis\":true,\"detailedTooltip\":true,\"legendSize\":\"auto\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"int\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"m\",\"min_doc_count\":1,\"extended_bounds\":{},\"scaleMetricValues\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"hostname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"_key\"}}]}"},"coreMigrationVersion":"8.3.1","id":"Bytes-Read-Per-Minute","migrationVersion":{"visualization":"8.3.0"},"references":[{"id":"irods-audit-pattern","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-07-12T19:13:10.517Z","version":"WzEwLDFd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":{\"term\":{\"rule_name\":\"audit_pep_resource_write_post\"}},\"language\":\"lucene\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Bytes Written Per Minute","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Bytes Written Per Minute\",\"type\":\"histogram\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"mode\":\"stacked\",\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"times\":[],\"yAxis\":{},\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100,\"filter\":true},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\",\"setYExtents\":false,\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of int\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of int\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"circlesRadius\":1}],\"legendPosition\":\"right\",\"palette\":{\"type\":\"palette\",\"name\":\"kibana_palette\"},\"isVislibVis\":true,\"detailedTooltip\":true,\"legendSize\":\"auto\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"int\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"m\",\"min_doc_count\":1,\"extended_bounds\":{},\"scaleMetricValues\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"hostname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"_key\"}}]}"},"coreMigrationVersion":"8.3.1","id":"Bytes-Written-Per-Minute","migrationVersion":{"visualization":"8.3.0"},"references":[{"id":"irods-audit-pattern","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-07-12T19:13:10.517Z","version":"WzExLDFd"} +{"attributes":{"description":"Sample iRODS Dashboard","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"highlightAll\":true,\"version\":true,\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":false,\"useMargins\":true,\"hidePanelTitles\":false}","panelsJSON":"[{\"version\":\"7.3.0\",\"type\":\"visualization\",\"gridData\":{\"w\":24,\"h\":12,\"x\":0,\"y\":0,\"i\":\"1\"},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.3.0\",\"type\":\"visualization\",\"gridData\":{\"w\":24,\"h\":12,\"x\":24,\"y\":0,\"i\":\"2\"},\"panelIndex\":\"2\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.3.0\",\"type\":\"visualization\",\"gridData\":{\"w\":24,\"h\":12,\"x\":0,\"y\":12,\"i\":\"3\"},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.3.0\",\"type\":\"visualization\",\"gridData\":{\"w\":24,\"h\":12,\"x\":24,\"y\":12,\"i\":\"4\"},\"panelIndex\":\"4\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"},{\"version\":\"7.3.0\",\"type\":\"visualization\",\"gridData\":{\"w\":24,\"h\":12,\"x\":0,\"y\":24,\"i\":\"5\"},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_5\"},{\"version\":\"7.3.0\",\"type\":\"visualization\",\"gridData\":{\"w\":24,\"h\":12,\"x\":24,\"y\":24,\"i\":\"6\"},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_6\"}]","timeRestore":false,"title":"iRODS Dashboard","version":1},"coreMigrationVersion":"8.3.1","id":"iRODS-Dashboard","migrationVersion":{"dashboard":"8.3.0"},"references":[{"id":"Bytes-Read-Per-Minute","name":"1:panel_1","type":"visualization"},{"id":"Bytes-Written-Per-Minute","name":"2:panel_2","type":"visualization"},{"id":"Connections-Per-Minute","name":"3:panel_3","type":"visualization"},{"id":"Top-Client-IPs","name":"4:panel_4","type":"visualization"},{"id":"Top-Users","name":"5:panel_5","type":"visualization"},{"id":"Unique-Users-Per-Minute","name":"6:panel_6","type":"visualization"}],"type":"dashboard","updated_at":"2022-07-12T19:13:10.517Z","version":"WzE2LDFd"} +{"excludedObjects":[],"excludedObjectsCount":0,"exportedCount":8,"missingRefCount":0,"missingReferences":[]} \ No newline at end of file diff --git a/irods_audit_elk_stack/firstrun.sh b/irods_audit_elk_stack/firstrun.sh new file mode 100755 index 0000000..0216af2 --- /dev/null +++ b/irods_audit_elk_stack/firstrun.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +if [ ! -f /var/lib/irods-elk/.firstrun_rmq_done ]; then + echo "<5>Performing rabbitmq first-run setup..." + + rabbitmqctl add_user test test + rabbitmqctl set_user_tags test administrator + rabbitmqctl set_permissions -p / test ".*" ".*" ".*" + + echo "<5>Completed rabbitmq first-run setup" + touch /var/lib/irods-elk/.firstrun_rmq_done +else + echo "<5>Skipping rabbitmq first-run setup (already done)..." +fi + +if [ ! -f /var/lib/irods-elk/.firstrun_es_done ]; then + echo "<5>Performing elasticsearch first-run setup..." + + curl -sLS http://localhost:9200 + curl -sLS -XPUT "http://localhost:9200/irods_audit" + curl -sLS -XPUT http://localhost:9200/irods_audit/_settings -H 'Content-Type: application/json' -d'{"index.mapping.total_fields.limit": 2000}' + + echo "<5>Completed elasticsearch first-run setup" + touch /var/lib/irods-elk/.firstrun_es_done +else + echo "<5>Skipping elasticsearch first-run setup (already done)..." +fi + +if [ ! -f /var/lib/irods-elk/.firstrun_kb_done ]; then + while true; do + echo "<5>Checking kibana status..." + + status_code="$(curl -sLSI -w "%{http_code}" -o /dev/null "http://localhost:5601/api/features" -H 'kbn-xsrf: true')" + curl_ret=$? + + if [[ "$curl_ret" != "0" ]]; then + echo "<4>Could not reach kibana (curl return code ${curl_ret})" + elif [[ "$status_code" != "200" ]]; then + echo "<4>Kibana is unhappy (got HTTP status ${status_code})" + else + echo "<5>Kibana seems ready" + break + fi + echo "<5>Waiting 3 seconds and trying again..." + sleep 3s + done + + echo "<5>Performing kibana first-run setup..." + + #curl -sLS -XPOST "http://localhost:5601/api/saved_objects/index-pattern/irods-audit-pattern" -H 'kbn-xsrf: true' -H 'Content-Type: application/json' -d '{ "attributes": { "title": "irods_audit*", "timeFieldName": "@timestamp" } }' + curl -sLS -X POST "http://localhost:5601/api/saved_objects/_import" -H "kbn-xsrf: true" --form file=@/var/lib/irods-elk/example_kibana_dashboard.ndjson + + echo "<5>Completed kibana first-run setup" + touch /var/lib/irods-elk/.firstrun_kb_done +else + echo "<5>Skipping kibana first-run setup (already done)..." +fi diff --git a/irods_audit_elk_stack/not-logstash.py b/irods_audit_elk_stack/not-logstash.py new file mode 100755 index 0000000..d7e31b0 --- /dev/null +++ b/irods_audit_elk_stack/not-logstash.py @@ -0,0 +1,65 @@ +#!/usr/bin/python3 + +import time +import json +from datetime import datetime +from threading import Lock + +from proton import Message +from proton.handlers import MessagingHandler +from proton.reactor import Container +from elasticsearch import Elasticsearch + +# There's way more than this that need conversion. +# I think that we have to do this at all means we're constructing our json improperly. +# Ugh. +convert_int = [ + 'int', + 'int__2', + 'int__3', + 'file_size' +] + +class ELKReader(MessagingHandler): + def __init__(self, server, address): + super(ELKReader, self).__init__() + self.server = server + self.address = address + self.id_ctr = 0 + self.id_ctr_lock = Lock() + self.es = Elasticsearch(hosts=[{'host':'localhost', 'port':9200}]) + + def on_start(self, event): + conn = event.container.connect(self.server) + event.container.create_receiver(conn, self.address) + + def on_message(self, event): + with self.id_ctr_lock: + msg_id = self.id_ctr + self.id_ctr += 1 + + msg = event.message.body + + # remove bad workaround tokens + msg = msg.replace("__BEGIN_JSON__","").replace("__END_JSON__", "") + + msg = json.loads(msg) + + timestamp = datetime.fromtimestamp(int(msg.pop('time_stamp')) / 1000.0) + msg['@timestamp'] = timestamp.isoformat() + + #msg.pop('const_char_ptr__3', None) + + for convert_int_key in convert_int: + if convert_int_key in msg: + msg[convert_int_key] = int(msg[convert_int_key]) + + self.es.index( + index = "irods_audit", + id = msg_id, + body = msg + ) + +while True: + time.sleep(1) + Container(ELKReader("localhost:5672", "audit_messages")).run() diff --git a/irods_audit_elk_stack/not-logstash.service b/irods_audit_elk_stack/not-logstash.service new file mode 100644 index 0000000..972e820 --- /dev/null +++ b/irods_audit_elk_stack/not-logstash.service @@ -0,0 +1,18 @@ +[Unit] +Description=fake logstash +After=rabbitmq-server.service +After=elasticsearch.service +Requires=rabbitmq-server.service +Requires=elasticsearch.service + +[Service] +Type=simple +User=elasticsearch +Group=elasticsearch +ExecStart=/var/lib/irods-elk/not-logstash.py +Restart=always +WorkingDirectory=/var/lib/irods-elk +LimitNOFILE=16384 + +[Install] +WantedBy=multi-user.target diff --git a/irods_audit_elk_stack/startup-script.sh b/irods_audit_elk_stack/startup-script.sh new file mode 100755 index 0000000..16a0cff --- /dev/null +++ b/irods_audit_elk_stack/startup-script.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +ip addr + +exec /sbin/init