From 7fc48bf1af01228fac948ccaec9164c0fc728bb0 Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Sat, 10 Feb 2024 03:21:27 +0100 Subject: [PATCH 1/5] Add test script that tests the grok pipeline by sending an event through it --- test_pipeline.sh | 69 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100755 test_pipeline.sh diff --git a/test_pipeline.sh b/test_pipeline.sh new file mode 100755 index 0000000..338c4a6 --- /dev/null +++ b/test_pipeline.sh @@ -0,0 +1,69 @@ +#!/bin/sh + +set -eux + +INPUT=$(mktemp tmp.logstash.in.XXXXX) +OUTPUT=$(mktemp tmp.logstash.out.XXXXX) +PIPELINE=$(mktemp tmp.logstash.pipeline.XXXXX) + +echo Preparing input data +echo "postfix/smtp[123]: 7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 153053D)" > $INPUT + +echo Preparing pipeline config +cat > $PIPELINE << EOF +input { + file { + path => "/tmp/logstash.in" + start_position => beginning + } +} +filter { + dissect { + mapping => { + "message" => "%{program}[%{pid}]: %{message}" + } + } +} +EOF + +cat 50-filter-postfix.conf >> $PIPELINE + +cat >> $PIPELINE << EOF +output { + file { + path => "/tmp/logstash.out" + } +} +EOF + +echo Starting logstash docker container +CONTAINER_ID=$(docker run --rm --detach \ + --volume ./${INPUT}:/tmp/logstash.in \ + --volume ./${OUTPUT}:/tmp/logstash.out \ + --volume ./postfix.grok:/etc/logstash/patterns.d/postfix.grok \ + --volume ./${PIPELINE}:/usr/share/logstash/pipeline/pipeline.conf \ + logstash:8.12.0 \ + logstash -f /usr/share/logstash/pipeline/pipeline.conf) + +echo -n "Waiting for output from logstash " +until test -s $OUTPUT; do + echo -n "." + sleep 2 +done +echo + +docker stop --time 1 $CONTAINER_ID > /dev/null + +if test "$(jq .tags[0] $OUTPUT)" = '"_grok_postfix_success"'; then + echo Grok processing successful! + jq . $OUTPUT +else + echo "Grok processing failed :<" + jq . $OUTPUT + exit 1 +fi + +echo Cleaning up +rm -f $INPUT $OUTPUT $PIPELINE + +echo Done \ No newline at end of file From c84c00e12039113e2616f7835f338558762a5914 Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Fri, 5 Jul 2024 01:25:55 +0200 Subject: [PATCH 2/5] Fix some minor posix shell scripting issues --- test_config_syntax.sh | 4 ++-- test_grok_patterns.sh | 2 +- test_pipeline.sh | 32 ++++++++++++++++---------------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/test_config_syntax.sh b/test_config_syntax.sh index a48c9c0..86a2bca 100755 --- a/test_config_syntax.sh +++ b/test_config_syntax.sh @@ -3,7 +3,7 @@ set -eux docker run --rm -it \ - --volume $(pwd)/postfix.grok:/etc/logstash/patterns.d/postfix.grok \ - --volume $(pwd)/50-filter-postfix.conf:/usr/share/logstash/pipeline/50-filter-postfix.conf \ + --volume "$(pwd)"/postfix.grok:/etc/logstash/patterns.d/postfix.grok \ + --volume "$(pwd)"/50-filter-postfix.conf:/usr/share/logstash/pipeline/50-filter-postfix.conf \ logstash:8.12.0 \ logstash --config.test_and_exit -f /usr/share/logstash/pipeline/50-filter-postfix.conf diff --git a/test_grok_patterns.sh b/test_grok_patterns.sh index 08b1280..9978c83 100755 --- a/test_grok_patterns.sh +++ b/test_grok_patterns.sh @@ -12,4 +12,4 @@ FROM ruby:slim RUN gem install jls-grok minitest EOF -docker run --volume $(pwd):"${VOLUMEPATH}" --workdir ${VOLUMEPATH} ${DOCKERIMAGE} sh -c "ruby test/test.rb" +docker run --volume "$(pwd)":"${VOLUMEPATH}" --workdir ${VOLUMEPATH} ${DOCKERIMAGE} sh -c "ruby test/test.rb" diff --git a/test_pipeline.sh b/test_pipeline.sh index 338c4a6..573dde5 100755 --- a/test_pipeline.sh +++ b/test_pipeline.sh @@ -7,10 +7,10 @@ OUTPUT=$(mktemp tmp.logstash.out.XXXXX) PIPELINE=$(mktemp tmp.logstash.pipeline.XXXXX) echo Preparing input data -echo "postfix/smtp[123]: 7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 153053D)" > $INPUT +echo "postfix/smtp[123]: 7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 153053D)" > "$INPUT" echo Preparing pipeline config -cat > $PIPELINE << EOF +cat > "$PIPELINE" << EOF input { file { path => "/tmp/logstash.in" @@ -26,9 +26,9 @@ filter { } EOF -cat 50-filter-postfix.conf >> $PIPELINE +cat 50-filter-postfix.conf >> "$PIPELINE" -cat >> $PIPELINE << EOF +cat >> "$PIPELINE" << EOF output { file { path => "/tmp/logstash.out" @@ -38,32 +38,32 @@ EOF echo Starting logstash docker container CONTAINER_ID=$(docker run --rm --detach \ - --volume ./${INPUT}:/tmp/logstash.in \ - --volume ./${OUTPUT}:/tmp/logstash.out \ + --volume ./"${INPUT}":/tmp/logstash.in \ + --volume ./"${OUTPUT}":/tmp/logstash.out \ --volume ./postfix.grok:/etc/logstash/patterns.d/postfix.grok \ - --volume ./${PIPELINE}:/usr/share/logstash/pipeline/pipeline.conf \ + --volume ./"${PIPELINE}":/usr/share/logstash/pipeline/pipeline.conf \ logstash:8.12.0 \ logstash -f /usr/share/logstash/pipeline/pipeline.conf) -echo -n "Waiting for output from logstash " -until test -s $OUTPUT; do - echo -n "." +printf "Waiting for output from logstash " +until test -s "$OUTPUT"; do + printf "." sleep 2 done echo -docker stop --time 1 $CONTAINER_ID > /dev/null +docker stop --time 1 "$CONTAINER_ID" > /dev/null -if test "$(jq .tags[0] $OUTPUT)" = '"_grok_postfix_success"'; then +if test "$(jq .tags[0] "$OUTPUT")" = '"_grok_postfix_success"'; then echo Grok processing successful! - jq . $OUTPUT + jq . "$OUTPUT" else echo "Grok processing failed :<" - jq . $OUTPUT + jq . "$OUTPUT" exit 1 fi echo Cleaning up -rm -f $INPUT $OUTPUT $PIPELINE +rm -f "$INPUT" "$OUTPUT" "$PIPELINE" -echo Done \ No newline at end of file +echo Done From 9c49ea3ef96188c1f1843bc60948976e8ef30d68 Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Fri, 5 Jul 2024 01:41:23 +0200 Subject: [PATCH 3/5] Add some documentation to the testing shell scripts --- test_config_syntax.sh | 6 ++++++ test_grok_patterns.sh | 6 ++++++ test_pipeline.sh | 7 +++++++ 3 files changed, 19 insertions(+) diff --git a/test_config_syntax.sh b/test_config_syntax.sh index 86a2bca..25a1abe 100755 --- a/test_config_syntax.sh +++ b/test_config_syntax.sh @@ -1,5 +1,11 @@ #!/bin/sh +# +# This script is used to test the config syntax of the 50-filter-postfix.conf file. +# +# The configuration file is validated using the logstash --config.test_and_exit command in a docker container. +# + set -eux docker run --rm -it \ diff --git a/test_grok_patterns.sh b/test_grok_patterns.sh index 9978c83..9bb7b01 100755 --- a/test_grok_patterns.sh +++ b/test_grok_patterns.sh @@ -1,5 +1,11 @@ #!/bin/sh +# +# This script is used to test the grok patterns in the postfix.grok file. +# +# The patterns are tested by running the test suite (in test/test.rb and test/*.yaml) +# against the patterns in the postfix.grok file in a docker container. +# set -eux DOCKERIMAGE="postfix-grok-patterns-runtests" diff --git a/test_pipeline.sh b/test_pipeline.sh index 573dde5..63f2e24 100755 --- a/test_pipeline.sh +++ b/test_pipeline.sh @@ -1,5 +1,12 @@ #!/bin/sh +# +# This script is used to test the logstash pipeline configuration. +# +# It sets up a logstash pipeline with the postfix configuration, +# sends a test logline through the pipeline and checks the results. +# + set -eux INPUT=$(mktemp tmp.logstash.in.XXXXX) From e7372816623e76cb4e5f69dbd0bf987ff52c9094 Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Fri, 5 Jul 2024 01:48:17 +0200 Subject: [PATCH 4/5] Reference test scripts in README --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a4f3e9c..4e3eed9 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,17 @@ Tests In the `test/` directory, there is a test suite that tries to make sure that no previously supported log line will break because of changing common patterns and such. It also returns results a lot faster than doing `sudo service logstash restart` :-). -The test suite needs the patterns provided by Logstash, you can easily pull these from github by running `git submodule update --init`. To run the test suite, you need a recent version of `ruby` (`2.6` or newer should work), and the `jls-grok` and `minitest` gems. Then simply execute `ruby test/test.rb`. NOTE: The whole test process can now be executed inside a docker container, simply by running the `runtests.sh` script. +The test suite needs the patterns provided by Logstash, you can easily pull these from github by running `git submodule update --init`. To run the test suite, you need a recent version of `ruby` (`2.6` or newer should work), and the `jls-grok` and `minitest` gems. Then simply execute `ruby test/test.rb`. NOTE: The whole test process can now be executed inside a docker container, simply by running the `test_grok_patterns.sh` script. Adding new test cases can easily be done by creating new yaml files in the test directory. Each file specifies a grok pattern to validate, a sample log line, and a list of expected results. Also, the example Logstash config file adds some informative tags that aid in finding grok failures and unparsed lines. If you're not interested in those, you can remove all occurrences of `add_tag` and `tag_on_failure` from the config file. +Additional test scripts are available for local tests (using docker containers): +- `test_grok_patterns.sh`: runs the test suite for the grok patterns in `postfix.grok` +- `test_logstash_config.sh`: validates the logstash config in `50-filter-postfix.conf` +- `test_pipeline.sh`: validates that the logstash config can be used in a simple logstash pipeline, and ensures that this results in parsed messages + Contributing ------------ From a0c98f7fbf64903055271fac1493a9b7e48b61e9 Mon Sep 17 00:00:00 2001 From: Tom Hendrikx Date: Fri, 5 Jul 2024 01:51:56 +0200 Subject: [PATCH 5/5] Use latest logstash --- test_config_syntax.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_config_syntax.sh b/test_config_syntax.sh index 25a1abe..6d9fe45 100755 --- a/test_config_syntax.sh +++ b/test_config_syntax.sh @@ -11,5 +11,5 @@ set -eux docker run --rm -it \ --volume "$(pwd)"/postfix.grok:/etc/logstash/patterns.d/postfix.grok \ --volume "$(pwd)"/50-filter-postfix.conf:/usr/share/logstash/pipeline/50-filter-postfix.conf \ - logstash:8.12.0 \ + logstash:8.14.1 \ logstash --config.test_and_exit -f /usr/share/logstash/pipeline/50-filter-postfix.conf