diff --git a/README.md b/README.md index a4f3e9c..4e3eed9 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,17 @@ Tests In the `test/` directory, there is a test suite that tries to make sure that no previously supported log line will break because of changing common patterns and such. It also returns results a lot faster than doing `sudo service logstash restart` :-). -The test suite needs the patterns provided by Logstash, you can easily pull these from github by running `git submodule update --init`. To run the test suite, you need a recent version of `ruby` (`2.6` or newer should work), and the `jls-grok` and `minitest` gems. Then simply execute `ruby test/test.rb`. NOTE: The whole test process can now be executed inside a docker container, simply by running the `runtests.sh` script. +The test suite needs the patterns provided by Logstash, you can easily pull these from github by running `git submodule update --init`. To run the test suite, you need a recent version of `ruby` (`2.6` or newer should work), and the `jls-grok` and `minitest` gems. Then simply execute `ruby test/test.rb`. NOTE: The whole test process can now be executed inside a docker container, simply by running the `test_grok_patterns.sh` script. Adding new test cases can easily be done by creating new yaml files in the test directory. Each file specifies a grok pattern to validate, a sample log line, and a list of expected results. Also, the example Logstash config file adds some informative tags that aid in finding grok failures and unparsed lines. If you're not interested in those, you can remove all occurrences of `add_tag` and `tag_on_failure` from the config file. +Additional test scripts are available for local tests (using docker containers): +- `test_grok_patterns.sh`: runs the test suite for the grok patterns in `postfix.grok` +- `test_logstash_config.sh`: validates the logstash config in `50-filter-postfix.conf` +- `test_pipeline.sh`: validates that the logstash config can be used in a simple logstash pipeline, and ensures that this results in parsed messages + Contributing ------------ diff --git a/test_config_syntax.sh b/test_config_syntax.sh index a48c9c0..6d9fe45 100755 --- a/test_config_syntax.sh +++ b/test_config_syntax.sh @@ -1,9 +1,15 @@ #!/bin/sh +# +# This script is used to test the config syntax of the 50-filter-postfix.conf file. +# +# The configuration file is validated using the logstash --config.test_and_exit command in a docker container. +# + set -eux docker run --rm -it \ - --volume $(pwd)/postfix.grok:/etc/logstash/patterns.d/postfix.grok \ - --volume $(pwd)/50-filter-postfix.conf:/usr/share/logstash/pipeline/50-filter-postfix.conf \ - logstash:8.12.0 \ + --volume "$(pwd)"/postfix.grok:/etc/logstash/patterns.d/postfix.grok \ + --volume "$(pwd)"/50-filter-postfix.conf:/usr/share/logstash/pipeline/50-filter-postfix.conf \ + logstash:8.14.1 \ logstash --config.test_and_exit -f /usr/share/logstash/pipeline/50-filter-postfix.conf diff --git a/test_grok_patterns.sh b/test_grok_patterns.sh index 08b1280..9bb7b01 100755 --- a/test_grok_patterns.sh +++ b/test_grok_patterns.sh @@ -1,5 +1,11 @@ #!/bin/sh +# +# This script is used to test the grok patterns in the postfix.grok file. +# +# The patterns are tested by running the test suite (in test/test.rb and test/*.yaml) +# against the patterns in the postfix.grok file in a docker container. +# set -eux DOCKERIMAGE="postfix-grok-patterns-runtests" @@ -12,4 +18,4 @@ FROM ruby:slim RUN gem install jls-grok minitest EOF -docker run --volume $(pwd):"${VOLUMEPATH}" --workdir ${VOLUMEPATH} ${DOCKERIMAGE} sh -c "ruby test/test.rb" +docker run --volume "$(pwd)":"${VOLUMEPATH}" --workdir ${VOLUMEPATH} ${DOCKERIMAGE} sh -c "ruby test/test.rb" diff --git a/test_pipeline.sh b/test_pipeline.sh new file mode 100755 index 0000000..63f2e24 --- /dev/null +++ b/test_pipeline.sh @@ -0,0 +1,76 @@ +#!/bin/sh + +# +# This script is used to test the logstash pipeline configuration. +# +# It sets up a logstash pipeline with the postfix configuration, +# sends a test logline through the pipeline and checks the results. +# + +set -eux + +INPUT=$(mktemp tmp.logstash.in.XXXXX) +OUTPUT=$(mktemp tmp.logstash.out.XXXXX) +PIPELINE=$(mktemp tmp.logstash.pipeline.XXXXX) + +echo Preparing input data +echo "postfix/smtp[123]: 7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 153053D)" > "$INPUT" + +echo Preparing pipeline config +cat > "$PIPELINE" << EOF +input { + file { + path => "/tmp/logstash.in" + start_position => beginning + } +} +filter { + dissect { + mapping => { + "message" => "%{program}[%{pid}]: %{message}" + } + } +} +EOF + +cat 50-filter-postfix.conf >> "$PIPELINE" + +cat >> "$PIPELINE" << EOF +output { + file { + path => "/tmp/logstash.out" + } +} +EOF + +echo Starting logstash docker container +CONTAINER_ID=$(docker run --rm --detach \ + --volume ./"${INPUT}":/tmp/logstash.in \ + --volume ./"${OUTPUT}":/tmp/logstash.out \ + --volume ./postfix.grok:/etc/logstash/patterns.d/postfix.grok \ + --volume ./"${PIPELINE}":/usr/share/logstash/pipeline/pipeline.conf \ + logstash:8.12.0 \ + logstash -f /usr/share/logstash/pipeline/pipeline.conf) + +printf "Waiting for output from logstash " +until test -s "$OUTPUT"; do + printf "." + sleep 2 +done +echo + +docker stop --time 1 "$CONTAINER_ID" > /dev/null + +if test "$(jq .tags[0] "$OUTPUT")" = '"_grok_postfix_success"'; then + echo Grok processing successful! + jq . "$OUTPUT" +else + echo "Grok processing failed :<" + jq . "$OUTPUT" + exit 1 +fi + +echo Cleaning up +rm -f "$INPUT" "$OUTPUT" "$PIPELINE" + +echo Done