Skip to content

Commit

Permalink
[28] Remove systemd from elk container
Browse files Browse the repository at this point in the history 
Also, optimize the image slightly
  • Loading branch information
@SwooshyCueb
SwooshyCueb committed Jul 21, 2022
1 parent 9603ce9 commit 84de099
Show file tree
Hide file tree
Showing 16 changed files with 1,013 additions and 231 deletions.
205 changes: 79 additions & 126 deletions irods_audit_elk_stack/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ RUN apt-get update && \
# To mark all installed packages as manually installed:
#apt-mark showauto | xargs -r apt-mark manual

# Install some standard stuff
RUN apt-get update && \
apt-get install -y \
apt-transport-https \
Expand All @@ -25,13 +26,22 @@ RUN apt-get update && \
&& \
apt-get install --no-install-recommends -y \
software-properties-common \
systemd \
systemd-sysv \
dbus \
gosu \
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/*

# Install yq, needed for init scripts
RUN add-apt-repository --no-update -y ppa:rmescandon/yq
RUN apt-get update && \
apt-get install --no-install-recommends -y \
yq \
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/*

# Install JDK/JRE
COPY java-excludes.dpkg.cfg /etc/dpkg/dpkg.cfg.d/java-excludes
ADD https://packages.adoptium.net/artifactory/api/gpg/key/public /usr/share/keyrings/adoptium.asc
ADD https://adoptopenjdk.jfrog.io/adoptopenjdk/api/gpg/key/public /usr/share/keyrings/adoptopenjdk.asc
RUN gpg --dearmor -o /usr/share/keyrings/adoptium.gpg /usr/share/keyrings/adoptium.asc && \
Expand All @@ -44,17 +54,15 @@ RUN gpg --dearmor -o /usr/share/keyrings/adoptium.gpg /usr/share/keyrings/adopti
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/*

#ARG java_ver=8
#ARG java_ver=11
#ARG java_vendor=adoptopenjdk
#ARG java_dist=hotspot-jre
ARG java_ver=17
ARG java_vendor=temurin
ARG java_dist=jdk

RUN apt-get update && \
apt-get install -y \
apt-get install --no-install-recommends -y \
${java_vendor}-${java_ver}-${java_dist} \
&& \
apt-get clean && \
Expand All @@ -63,66 +71,51 @@ ENV JAVA_HOME=/usr/lib/jvm/${java_vendor}-${java_ver}-${java_dist}-amd64
RUN update-java-alternatives --set ${JAVA_HOME}
ENV ES_JAVA_HOME=${JAVA_HOME}

# Install Elasticsearch and Kibana
#ARG es_ver=6
#ARG es_ver=7
ARG es_ver=8
COPY elasticsearch/exclude-jvm.dpkg.cfg /etc/dpkg/dpkg.cfg.d/elasticsearch-exclude-jvm
COPY kibana/exclude-node-stuff.dpkg.cfg /etc/dpkg/dpkg.cfg.d/kibana-exclude-node-stuff
ADD https://artifacts.elastic.co/GPG-KEY-elasticsearch /usr/share/keyrings/elasticsearch-keyring.asc
RUN gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg /usr/share/keyrings/elasticsearch-keyring.asc && \
echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/${es_ver}.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-${es_ver}.x.list && \
echo 'path-exclude=/usr/share/elasticsearch/jdk' >> /etc/dpkg/dpkg.cfg.d/excludes-elasticsearch-jvm && \
echo 'path-exclude=/usr/share/elasticsearch/jdk/*' >> /etc/dpkg/dpkg.cfg.d/excludes-elasticsearch-jvm

echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/${es_ver}.x/apt stable main" | tee /etc/apt/sources.list.d/elastic-${es_ver}.x.list
RUN apt-get update && \
apt-get install -y \
apt-get install --no-install-recommends -y \
elasticsearch \
kibana \
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/*

RUN echo "ES_JAVA_HOME=\"${ES_JAVA_HOME}\"" >> /etc/default/elasticsearch

# Install RabbitMQ
ADD https://packagecloud.io/rabbitmq/rabbitmq-server/gpgkey /usr/share/keyrings/rabbitmq_rabbitmq-server.asc
ADD https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc /usr/share/keyrings/erlang_solutions.asc
RUN add-apt-repository --no-update -y ppa:rabbitmq/rabbitmq-erlang && \
gpg --dearmor -o /usr/share/keyrings/rabbitmq_rabbitmq-server.gpg /usr/share/keyrings/rabbitmq_rabbitmq-server.asc && \
gpg --dearmor -o /usr/share/keyrings/erlang_solutions.gpg /usr/share/keyrings/erlang_solutions.asc && \
echo "deb [signed-by=/usr/share/keyrings/rabbitmq_rabbitmq-server.gpg] https://packagecloud.io/rabbitmq/rabbitmq-server/ubuntu/ $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) main" | tee /etc/apt/sources.list.d/rabbitmq_rabbitmq-server.list && \
echo "deb [signed-by=/usr/share/keyrings/erlang_solutions.gpg] https://packages.erlang-solutions.com/ubuntu $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) contrib" | tee /etc/apt/sources.list.d/erlang-solutions.list && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/*

ADD https://packages.erlang-solutions.com/ubuntu/erlang_solutions.asc /usr/share/keyrings/erlang_solutions.asc
#RUN gpg --dearmor -o /usr/share/keyrings/erlang_solutions.gpg /usr/share/keyrings/erlang_solutions.asc && \
# echo "deb [signed-by=/usr/share/keyrings/erlang_solutions.gpg] https://packages.erlang-solutions.com/ubuntu $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) contrib" | tee /etc/apt/sources.list.d/erlang-solutions.list
RUN gpg --dearmor -o /usr/share/keyrings/erlang_solutions.gpg /usr/share/keyrings/erlang_solutions.asc && \
echo "deb [signed-by=/usr/share/keyrings/erlang_solutions.gpg] http://binaries.erlang-solutions.com/debian $(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) contrib" | tee /etc/apt/sources.list.d/erlang-solutions.list

RUN apt-get update && \
apt-get install -y \
rabbitmq-server \
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/*

# Hopefully these are new enough
# Install Python modules for Logstash stand-in
RUN apt-get update && \
apt-get install -y \
apt-get install --no-install-recommends -y \
python3-qpid-proton \
python3-elasticsearch \
&& \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/*

RUN rabbitmq-plugins enable rabbitmq_amqp1_0 && \
rabbitmq-plugins enable rabbitmq_management

RUN echo "server.host: \"0.0.0.0\"" >> /etc/kibana/kibana.yml

COPY elasticsearch.yml /etc/elasticsearch/elasticsearch.yml
RUN /usr/share/elasticsearch/bin/elasticsearch-keystore remove \
xpack.security.http.ssl.keystore.secure_password \
xpack.security.transport.ssl.keystore.secure_password \
xpack.security.transport.ssl.truststore.secure_password

# utils
# Install some utils
RUN apt-get update && \
apt-get install -y \
procps \
Expand All @@ -134,97 +127,57 @@ RUN apt-get update && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* /tmp/*

# from ubi8-init
STOPSIGNAL SIGRTMIN+3

# from ubi8-init
RUN systemctl mask \
systemd-remount-fs.service \
dev-hugepages.mount \
sys-fs-fuse-connections.mount \
systemd-logind.service \
getty.target \
console-getty.service \
systemd-udev-trigger.service \
systemd-udevd.service \
systemd-random-seed.service

# from ubi8-init
#mask systemd-machine-id-commit.service - partial fix for https://bugzilla.redhat.com/show_bug.cgi?id=1472439
RUN systemctl mask systemd-machine-id-commit.service

RUN systemctl mask \
unattended-upgrades.service \
packagekit-offline-update.service \
systemd-timesyncd.service \
systemd-resolved.service \
apt-daily-upgrade.service \
apt-daily-upgrade.timer \
apt-daily.service \
apt-daily.timer \
e2scrub_reap.service \
e2scrub_all.service \
e2scrub_all.timer \
ondemand.service \
systemd-modules-load.service \
fstrim.service \
fstrim.timer

#RUN systemctl mask \
# remote-fs.target \
# systemd-pstore.service \
# cryptsetup.target

RUN systemctl mask \
getty-static.service \
networkd-dispatcher.service

#RUN systemctl mask \
# kmod-static-nodes.service
# proc-sys-fs-binfmt_misc.mount \
# proc-sys-fs-binfmt_misc.automount \
# dev-mqueue.mount \
# sys-kernel-config.mount \
# sys-kernel-debug.mount \
# sys-kernel-tracing.mount \
# systemd-ask-password-console.path \
# systemd-binfmt.service \
# systemd-boot-system-token.service \
# systemd-sysctl.service \
# systemd-sysusers.service \
# systemd-update-utmp.service \
# systemd-initctl.socket \
# systemd-update-utmp-runlevel.service \
# systemd-ask-password-wall.path \
# systemd-user-sessions.service

#RUN systemctl mask \
# systemd-tmpfiles-setup-dev.service \
# systemd-tmpfiles-setup.service \
# systemd-tmpfiles-clean.timer \
# systemd-tmpfiles-clean.service


COPY startup-script.sh /var/lib/irods-elk/
CMD ["/var/lib/irods-elk/startup-script.sh"]

RUN mkdir -p /etc/systemd/system/kibana.service.d && \
echo "[Unit]" >> /etc/systemd/system/kibana.service.d/elasticsearch.conf && \
echo "After=elasticsearch.service" >> /etc/systemd/system/kibana.service.d/elasticsearch.conf && \
echo "Wants=elasticsearch.service" >> /etc/systemd/system/kibana.service.d/elasticsearch.conf

COPY not-logstash.service /etc/systemd/system/
COPY not-logstash.py /var/lib/irods-elk/

COPY elk-firstrun.service /etc/systemd/system/
COPY example_kibana_dashboard.ndjson /var/lib/irods-elk/
COPY firstrun.sh /var/lib/irods-elk/

RUN systemctl enable \
elasticsearch \
rabbitmq-server \
kibana \
not-logstash \
elk-firstrun
# Install RabbitMQ plugins and create administrator account
RUN rabbitmq-plugins enable \
rabbitmq_amqp1_0 \
rabbitmq_management \
&& \
/etc/init.d/rabbitmq-server start && \
rabbitmqctl add_user test test && \
rabbitmqctl set_user_tags test administrator && \
rabbitmqctl set_permissions -p / test ".*" ".*" ".*" && \
/etc/init.d/rabbitmq-server stop

# Elasticsearch init script and config files
COPY --chown=root:elasticsearch elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.yml
COPY --chown=root:elasticsearch elasticsearch/jvm.options.d/oom_heap_dump.options /etc/elasticsearch/jvm.options.d/
COPY elasticsearch/elasticsearch.init /etc/init.d/elasticsearch
RUN chmod +x /etc/init.d/elasticsearch
# Since we have disabled security, we must purge our keystore of secure passwords
RUN /usr/share/elasticsearch/bin/elasticsearch-keystore remove \
xpack.security.http.ssl.keystore.secure_password \
xpack.security.transport.ssl.keystore.secure_password \
xpack.security.transport.ssl.truststore.secure_password

WORKDIR /root
# Kibana init script and config files
COPY --chown=root:kibana kibana/kibana.yml /etc/kibana/kibana.yml
COPY kibana/kibana.init /etc/init.d/kibana
RUN chmod +x /etc/init.d/kibana

# Initialize Elasticsearch and Kibana
COPY kibana/irods_dashboard.ndjson /var/lib/irods-elk/irods_dashboard.ndjson
RUN ES_JAVA_OPTS="-Xms512m -Xmx512m" /etc/init.d/elasticsearch start && \
curl -sLSf -XPUT "http://localhost:9200/irods_audit" && echo && \
curl -sLSf -XPUT "http://localhost:9200/irods_audit/_settings" \
-H 'Content-Type: application/json' \
-d'{"index.mapping.total_fields.limit": 2000}' \
&& echo && \
/etc/init.d/kibana start && \
curl -sLSf -X POST "http://localhost:5601/api/saved_objects/_import" \
-H "kbn-xsrf: true" \
--form file=@/var/lib/irods-elk/irods_dashboard.ndjson \
&& echo && \
/etc/init.d/kibana stop && \
/etc/init.d/elasticsearch stop

# not-logstash script and init script
COPY not-logstash/not-logstash.py /var/lib/irods-elk/bin/not-logstash
COPY not-logstash/not-logstash.init /etc/init.d/not-logstash
RUN chmod +x /var/lib/irods-elk/bin/not-logstash \
/etc/init.d/not-logstash

WORKDIR /var/lib/irods-elk

COPY startup-script.sh /var/lib/irods-elk/startup-script.sh
RUN chmod +x /var/lib/irods-elk/startup-script.sh
ENTRYPOINT ["/var/lib/irods-elk/startup-script.sh"]
Loading

0 comments on commit 84de099

Please sign in to comment.