diff --git a/Dockerfile b/Dockerfile index c18cd92..5291a2b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,35 +1,26 @@ FROM ubuntu:focal ENV DEBIAN_FRONTEND=noninteractive RUN \ + apt-get update -qq && \ + apt-get install -y -qqq software-properties-common && \ + add-apt-repository -y ppa:dns-oarc/dnsjit && \ apt-get update -qq && \ apt-get install -y -qqq \ + dnsjit \ + dnsjit-dev \ python3 \ python3-pip \ tshark \ jq \ - libck-dev \ - libluajit-5.1-dev \ - libpcap-dev \ - liblmdb-dev \ - libgnutls28-dev \ libnghttp2-dev \ luajit \ libuv1-dev \ - libgoogle-perftools-dev \ make \ automake \ libtool \ pkg-config \ git && \ rm -rf /var/lib/apt/lists/* -RUN \ - git clone https://github.com/DNS-OARC/dnsjit.git && \ - cd dnsjit && \ - ./autogen.sh && \ - ./configure && \ - make && \ - make install && \ - cd .. COPY . /shotgun WORKDIR /shotgun diff --git a/NEWS b/NEWS new file mode 100644 index 0000000..d272583 --- /dev/null +++ b/NEWS @@ -0,0 +1,66 @@ +DNS Shotgun v20210714 +===================== + +Incompatible changes +-------------------- +- dnsjit 1.2.0+ is now required +- dnssim component was moved from dnsjit to this repository and has to be + compiled separately (!52) + +Improvements +------------ +- replay: new option `stop_after_s` added to input (!40) +- tools/plot-packat-rate: new tool to plot packet rate in PCAPs (!46) +- replay/shotgun: support reading PCAP from stdin (!36) +- pcap/merge-chunks: utility to merge chunks on-the-fly (!37) +- tools/plot-response-rate: add `--rcodes-above-pct` option (!43) +- tools/plot-response-rate: add `--skip-total` option (!47) +- tools/plot-response-rate: add `--sum-rcodes` option (!47) +- tools/plot-latency: add `--group` option to plot aggregate data (!53) +- pcap/cut-pcap: new script to effetively trim a sorted PCAP (!50) +- configs: plot response rate chart with non-zero rcodes by default (!44) +- pcap/extract-clients: add --seed option for deterministic splitting (!34) +- pcap/filter-dnsq: improved malformed packet handling (!31) +- pcap/extract-clients: detect non-monotonic timestamps (!31) +- replay/shotgun: use faster mmpcap (!36) +- tools/plot-response-rate: add markers for plotting rcodes (!41) +- tools/plot-response-rate: improve color selection (!41, !42) + +Bugfixes +-------- +- pcap/extract-clients: fix imprecise timestamps and off-by-one error (!32) +- tools/plot-client-distribution: fix handling of empty query count intervals (!38) +- replay: add workaround to support /32 and /128 addresses (!39) + + +DNS Shotgun v20210203 +===================== + +- new user interface + - replay.py is now the main tool instead of shotgun.lua + - new configuration format to support complex scenarios + - mixed-protocol traffic replay is supported + - some scripts in `tools/` were renamed +- fast scripts for processing PCAP data + - can be found in `pcap/` + - these replaced the now removed pellet.py and scale.py +- added IPv4 support +- DoH is now fully supported +- comprehensive usage documentation is available at https://dns-shotgun.readthedocs.io +- minimum required dnsjit version is 1.1.0 + + +DNS Shotgun v20200914 +===================== + +- fully supported UDP, TCP and DNS-over-TLS with + [dnsjit](https://github.com/DNS-OARC/dnsjit) 1.0.0 +- fully supported DNS-over-HTTPS with development version of dnsjit +- traffic can be replayed only over IPv6 +- user interface + - may be unstable + - only very basic UI available + - more complex scenarios are no supported yet + (e.g. simultaneously using multiple protocols) +- pellet.py is functional, but it is very slow and requires python-dpkt from + master diff --git a/docs/installation.md b/docs/installation.md index fab12b1..afe2e27 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,31 +1,8 @@ # Installation -There are two options for using DNS Shotgun. You can either install the -dependencies and use the scripts from the repository directly, or use a -pre-built docker image. - -## Using script directly - -You can use the toolchain scripts directly from the git repository. You need to -ensure you have the required dependencies installed. Also make sure to check -out some tagged version, as the development happens in master branch. - -``` -$ git clone https://gitlab.nic.cz/knot/shotgun.git -$ git checkout v20210203 -``` - -### Dependencies - -When using the scripts directly, the following dependencies are needed. If you -only wish to process shotgun JSON output (e.g. plot charts), then dnsjit isn't -required. - -- [dnsjit](https://github.com/DNS-OARC/dnsjit): Can be installed from [DNS-OARC - repositories](https://dev.dns-oarc.net/packages/). -- Python 3.6 or later -- Python dependencies from [requirements.txt](https://gitlab.nic.cz/knot/shotgun/-/blob/master/requirements.txt) -- (optional) tshark/wireshark for some PCAP pre-processing +There are two options for using DNS Shotgun. You can either use a pre-built +docker image, or install the dependencies, compile the dnssim module and use +the scripts from the repository directly. ## Docker Image @@ -33,7 +10,7 @@ Pre-built image can be obtained from [CZ.NIC DNS Shotgun Registry](https://gitlab.nic.cz/knot/shotgun/container_registry/65). ``` -$ docker pull registry.nic.cz/knot/shotgun:v20210203 +$ docker pull registry.nic.cz/knot/shotgun:v20210714 ``` Alternately, you can build the image yourself from Dockerfile in the repository. @@ -48,6 +25,39 @@ Alternately, you can build the image yourself from Dockerfile in the repository. $ docker run \ --network host \ -v "$PWD:/mnt" \ - registry.nic.cz/knot/shotgun:v20210203 \ + registry.nic.cz/knot/shotgun:v20210714 \ $COMMAND ``` + +## Using scripts from sources + +You can use the toolchain scripts directly from the git repository. You need to +ensure you have the required dependencies installed and the compile and install +the dnssim module. Also make sure to check out some tagged version, as the +development happens in master branch. + +``` +$ git clone https://gitlab.nic.cz/knot/shotgun.git +$ git checkout v20210714 +$ cd shotgun/replay/dnssim +$ ./autogen.sh +$ ./configure +$ make +$ make install +``` + +### Dependencies + +When using the scripts directly, the following dependencies are needed. + +If you only wish to process shotgun JSON output (e.g. plot charts), then dnsjit +and compiling the dnssim module isn't required. + +- [dnsjit 1.2+](https://github.com/DNS-OARC/dnsjit): Can be installed from [DNS-OARC + repositories](https://dev.dns-oarc.net/packages/). +- libuv +- libnghttp2 + +- Python 3.6 or later +- Python dependencies from [requirements.txt](https://gitlab.nic.cz/knot/shotgun/-/blob/master/requirements.txt) +- (optional) tshark/wireshark for some PCAP pre-processing diff --git a/pcap/merge_chunks.py b/pcap/merge-chunks.py similarity index 100% rename from pcap/merge_chunks.py rename to pcap/merge-chunks.py diff --git a/replay/dnssim/CHANGELOG.md b/replay/dnssim/CHANGELOG.md index 9cbfa55..58b22a9 100644 --- a/replay/dnssim/CHANGELOG.md +++ b/replay/dnssim/CHANGELOG.md @@ -1,3 +1,8 @@ +dnssim v20210714 +================ +- dnssim was moved from dnsjit to shotgun, this implies new + lua name: "shotgun.output.dnssim" + dnssim v20210129 ================ diff --git a/replay/dnssim/configure.ac b/replay/dnssim/configure.ac index 6448b17..0e247cb 100644 --- a/replay/dnssim/configure.ac +++ b/replay/dnssim/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ(2.61) -AC_INIT([shotgun-output-dnssim], [20210129],,) +AC_INIT([shotgun-output-dnssim], [20210714],,) AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects]) AC_CONFIG_HEADER([src/config.h]) @@ -21,6 +21,10 @@ PKG_CHECK_MODULES([libuv], [libuv]) PKG_CHECK_MODULES([libnghttp2], [libnghttp2]) AC_CHECK_LIB([gnutls], [gnutls_init],, [AC_MSG_ERROR([libgnutls not found])]) +# Checks for support. +AC_SEARCH_LIBS([clock_gettime]) +AC_CHECK_FUNCS([clock_nanosleep]) + # Output Makefiles AC_CONFIG_FILES([ Makefile diff --git a/replay/dnssim/src/output/dnssim.c b/replay/dnssim/src/output/dnssim.c index e9e47c7..5074462 100644 --- a/replay/dnssim/src/output/dnssim.c +++ b/replay/dnssim/src/output/dnssim.c @@ -2,6 +2,8 @@ * SPDX-License-Identifier: GPL-3.0-or-later */ +#include "config.h" + #include "output/dnssim.h" #include "output/dnssim/internal.h" #include "output/dnssim/ll.h" @@ -28,8 +30,7 @@ static uint64_t _now_ms() now_ms += ts.tv_nsec / 1000000; return now_ms; #else - mlfatal("clock_gettime() not available"); - return 0; +#error "clock_gettime() is not available" #endif } diff --git a/replay/dnssim/src/output/dnssim.lua b/replay/dnssim/src/output/dnssim.lua index 7aac8bb..1dfd78d 100644 --- a/replay/dnssim/src/output/dnssim.lua +++ b/replay/dnssim/src/output/dnssim.lua @@ -149,7 +149,7 @@ local C = ffi.C local DnsSim = {} -local _DNSSIM_VERSION = 20210129 +local _DNSSIM_VERSION = 20210714 local _DNSSIM_JSON_VERSION = 20200527 -- Create a new DnsSim output for up to max_clients. diff --git a/replay/shotgun.lua b/replay/shotgun.lua index 591d9c0..7ebae7c 100755 --- a/replay/shotgun.lua +++ b/replay/shotgun.lua @@ -4,7 +4,7 @@ local object = require("dnsjit.core.objects") local log = require("dnsjit.core.log") local dnssim = require("shotgun.output.dnssim") -local DNSSIM_REQ_VERSION = 20210129 +local DNSSIM_REQ_VERSION = 20210714 local has_check_version, version = pcall(dnssim.check_version, DNSSIM_REQ_VERSION) if not has_check_version or version == nil then log.fatal(string.format( @@ -45,7 +45,7 @@ local function send_thread_main(thr) -- output must be global (per thread) to be accesible in loadstring() -- luacheck: globals output, ignore log - output = require("dnsjit.output.dnssim").new(thr:pop()) + output = require("shotgun.output.dnssim").new(thr:pop()) local log = output:log(thr:pop()) output:target(thr:pop(), thr:pop()) @@ -65,9 +65,6 @@ local function send_thread_main(thr) elseif protocol_func == "tls" then cmd = cmd .. "('" .. gnutls_priority .. "')" elseif protocol_func == "https2" then - if type(output.https2) ~= "function" then - log:fatal("https2 isn't supported with this version of dnsjit") - end cmd = cmd .. "({ method = '" .. http_method .. "' }, '" .. gnutls_priority .. "')" else log:fatal("unknown protocol_func: " .. protocol_func)