jbuf: add tooling and examples

README.md

@@ -2,3 +2,8 @@
 
 ## /ssl
 Prebuilt assets helpers (openssl/libressl)
+
+## /jbuf
+Jitter testing, tracing and analyzing tools
+
+[SIPp](https://github.com/SIPp/sipp) tooling with PCAP examples (jitter etc.)

jbuf/.gitignore

@@ -0,0 +1,2 @@
+sipp
+data

jbuf/README.md

@@ -0,0 +1,76 @@
+# SIPp - Protocol and jitter testing
+
+## Build
+
+```bash
+git clone https://github.com/SIPp/sipp
+cd sipp
+git checkout v3.7.2
+cmake -B build -DUSE_PCAP=1
+cmake --build build -j
+```
+
+## Usage
+
+```bash
+sudo sipp/build/sipp -sf scenarios/uac_pcap_opus.xml 127.0.0.1 -m 1 -key pcap pcap/opus_audio_500hz_linux_qdisc_delay_50ms.pcap
+```
+
+## PCAP's
+
+### Manipulating
+
+#### Reorder
+
+```bash
+editcap -r g711a.pcap tmp1 2-3 # dump packets 2-3
+editcap -r -t 0.07 g711a.pcap tmp2 1 # dump packet 1 and manipulate timestamp
+editcap g711a.pcap tmp3 1-3 # exclude packets 1-3
+mergecap -w out.pcap -a tmp1 tmp2 tmp3
+```
+#### Drop
+
+```bash
+editcap g711a.pcap out.pcap 2-3 # drop packets 2-3
+```
+
+#### Remove TURN (4 Bytes, Offset 42 Bytes) encapsulation
+
+editcap -L -C42:4 in.pcap tmp.pcap
+./pcap_fix_udp_len.py tmp.pcap out.pcap 4
+
+
+### Text2pcap
+
+text2pcap is useful for debugging encrypted connections like DTLS_SRTP from application context.
+
+See: https://blog.mozilla.org/webrtc/debugging-encrypted-rtp-is-more-fun-than-it-used-to-be/
+
+#### Build options (libre and baresip)
+
+Needs at least baresip v3.10.0 (current main)
+```bash
+cmake -B build -DUSE_TRACE=ON -DCMAKE_C_FLAGS="-DRE_RTP_PCAP"
+cmake --build build
+```
+
+#### Text2pcap dump
+
+Start baresip and connections normally. Traces are written to `re_trace.json` after exit you can extract the pcap traces with `jq` and `text2pcap`:
+
+```bash
+jq -r ".traceEvents[] | select (.cat == \"pcap\") | .args.pcap" re_trace.json | text2pcap -D -n -l1 -i17 -u 1000,2000 -t '%H:%M:%S.%f' - dump.pcapng
+
+```
+
+Big re_trace.json files can be streamed like this
+
+```bash
+jq -r --stream "select(.[0][3] == \"pcap\" and .[1] != null) | .[1]" re_trace.json | text2pcap -D -n -l1 -i17 -u 1000,2000 -t '%H:%M:%S.%f' - dump.pcapng
+```
+
+The dump can now be opened with `wireshark`:
+```bash
+wireshark dump.pcapng
+```
+

jbuf/aubuf.plot

@@ -0,0 +1,27 @@
+#!/usr/bin/gnuplot
+
+# Choose your preferred gnuplot terminal or use e.g. evince to view the
+# jbuf.eps!
+
+set terminal wxt title "aubuf receive"
+#set terminal postscript eps size 30,20 enhanced color
+#set output 'jbuf.eps'
+#set terminal png size 1280,480
+#set output 'jbuf.png'
+set datafile separator ","
+set key outside
+set xlabel "time/ms"
+set ylabel "delay (ms)"
+
+stats "data/aubuf_cur_sz_ms.dat" using ($2) name "N"
+stats "data/aubuf_underrun.dat" using ($1) name "U" nooutput
+stats "data/aubuf_overrun.dat" using ($1) name "O" nooutput
+stats "data/aubuf_filling.dat" using ($1) name "F" nooutput
+event_h(i) = (0.5*N_max) + 0.3*N_max*(i/6.0)
+
+plot 'data/aubuf_cur_sz_ms.dat' using ($1/1000):2 title 'size' with linespoints, \
+     'data/aubuf_underrun.dat' using ($1/1000):(event_h(1)) title sprintf("Underrun: %d", U_records)  with points, \
+     'data/aubuf_overrun.dat' using ($1/1000):(event_h(2)) title sprintf("Overrun: %d", O_records)  with points, \
+     'data/aubuf_filling.dat' using ($1/1000):(event_h(3)) title sprintf("Filling: %d", F_records)  with points
+
+pause mouse close # Comment for non-interactive terminals

jbuf/jbuf.plot

@@ -0,0 +1,28 @@
+#!/usr/bin/gnuplot
+#
+# Choose your preferred gnuplot terminal or use e.g. evince to view the
+# jbuf.eps!
+
+set terminal wxt title "jbuf delay"
+#set terminal qt persist
+#set terminal postscript eps size 30,20 enhanced color
+#set output 'jbuf.eps'
+#set terminal png size 1280,480
+#set output 'jbuf.png'
+set datafile separator ","
+set key outside
+set xlabel "time/ms"
+set ylabel "delay (ms)"
+
+stats "data/jbuf_recv_delay.dat" using ($2) name "N"
+stats "data/jbuf_late_play.dat" using ($2) name "L" nooutput
+stats "data/jbuf_lost.dat" using ($2) name "X" nooutput
+
+plot 'data/jbuf_recv_delay.dat' using ($1/1000):2 title 'receive' with linespoints, \
+  'data/jbuf_play_delay.dat' using ($1/1000):2 title 'playout' with points, \
+  'data/jbuf_jitter_adjust.dat' using ($1/1000):2 title 'adapt' with points, \
+  'data/jbuf_playout_diff.dat' using ($1/1000):2 title 'playout_diff' with linespoints, \
+  'data/jbuf_late_play.dat' using ($1/1000):2 title sprintf("Late: %d", L_records) with points, \
+  'data/jbuf_lost.dat' using ($1/1000):2 title sprintf("Lost: %d", X_sum) with points
+
+pause mouse close # Comment for non-interactive terminals

jbuf/jbuf.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+trace_file="../../re_trace.json"
+
+function gen_datfile_arg() {
+    cat=$1
+    ph=$2
+    id=$3
+    filename=data/${cat}_${ph}.dat
+
+    jqc=".traceEvents[] | select (.id == \"${id}\" and .cat == \"${cat}\" and .name == \"${ph}\") | \"\(.ts),\(.args.${ph})\""
+    jq -c "${jqc}" ${trace_file} | sed 's/"//g' > "${filename}"
+}
+
+function gen_datfile() {
+    cat=$1
+    ph=$2
+    id=$3
+    filename=data/${cat}_${ph}.dat
+
+    jqc=".traceEvents[] | select (.id == \"${id}\" and .cat == \"${cat}\" and .name == \"${ph}\") | \"\(.ts)\""
+    jq -c "${jqc}" ${trace_file} | sed 's/"//g' > "${filename}"
+}
+
+mkdir -p data
+
+gen_datfile_arg jbuf recv_delay audio
+gen_datfile_arg jbuf play_delay audio
+gen_datfile_arg jbuf jitter_adjust audio
+gen_datfile_arg jbuf playout_diff audio
+gen_datfile_arg jbuf clock_skew audio
+gen_datfile_arg jbuf get audio
+gen_datfile_arg jbuf late_play audio
+gen_datfile_arg jbuf jitter audio
+gen_datfile_arg jbuf lost audio
+gen_datfile_arg aubuf cur_sz_ms aureceiver
+gen_datfile_arg aubuf append_delay aureceiver
+gen_datfile_arg aubuf read_delay aureceiver
+gen_datfile aubuf underrun aureceiver 
+gen_datfile aubuf overrun aureceiver
+gen_datfile aubuf filling aureceiver
+
+pkill -f gnuplot
+./jbuf.plot &
+./jbuf_clocks.plot &
+./aubuf.plot &
+
+wait

jbuf/jbuf_clocks.plot

@@ -0,0 +1,19 @@
+#!/usr/bin/gnuplot
+#
+# Choose your preferred gnuplot terminal or use e.g. evince to view the
+# jbuf.eps!
+
+set terminal wxt title "jbuf clocks"
+#set terminal postscript eps size 30,20 enhanced color
+#set output 'jbuf.eps'
+#set terminal png size 1280,480
+#set output 'jbuf.png'
+set datafile separator ","
+set key outside
+set xlabel "time/ms"
+set ylabel "delay (ms)"
+
+plot 'data/jbuf_clock_skew.dat' using ($1/1000):2 title 'clock_skew' with linespoints, \
+     'data/jbuf_jitter.dat' using ($1/1000):2 title 'jitter' with linespoints
+
+pause mouse close # Comment for non-interactive terminals

jbuf/jbuf_video.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+trace_file="../../re_trace.json"
+
+function gen_datfile_arg() {
+    cat=$1
+    ph=$2
+    id=$3
+    filename=${cat}_${ph}.dat
+
+    jqc=".traceEvents[] | select (.id == \"${id}\" and .cat == \"${cat}\" and .name == \"${ph}\") | \"\(.ts),\(.args.${ph})\""
+    jq -c "${jqc}" ${trace_file} | sed 's/"//g' > "${filename}"
+}
+
+function gen_datfile() {
+    cat=$1
+    ph=$2
+    id=$3
+    filename=${cat}_${ph}.dat
+
+    jqc=".traceEvents[] | select (.id == \"${id}\" and .cat == \"${cat}\" and .name == \"${ph}\") | \"\(.ts)\""
+    jq -c "${jqc}" ${trace_file} | sed 's/"//g' > "${filename}"
+}
+
+gen_datfile_arg jbuf recv_delay video
+gen_datfile_arg jbuf play_delay video
+gen_datfile_arg jbuf jitter_adjust video
+gen_datfile_arg jbuf playout_diff video
+gen_datfile_arg jbuf clock_skew video
+gen_datfile_arg jbuf get video
+gen_datfile_arg jbuf late_play video
+gen_datfile_arg jbuf jitter video
+
+pkill -f gnuplot
+./jbuf.plot &
+./jbuf_clocks.plot &
+./aubuf.plot &
+
+wait

jbuf/pcap_fix_udp_len.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+
+import argparse
+from scapy.all import *
+
+def update_bytes_from_packet(packet, rbytes):
+    if UDP in packet:
+        # Check if the packet contains a UDP layer
+        packet[UDP].len = packet[UDP].len - int(rbytes)
+        packet[IP].len = packet[IP].len - int(rbytes)
+        packet[UDP].chksum = None  # Recalculate UDP checksum
+        packet[IP].chksum = None  # Recalculate IP checksum
+
+def main():
+    parser = argparse.ArgumentParser(description="Update IP/UDP length.")
+    parser.add_argument("input_file", help="Input PCAP file")
+    parser.add_argument("output_file", help="Output PCAP file")
+    parser.add_argument("rbytes", help="bytes to remove")
+
+    args = parser.parse_args()
+
+    # Load the PCAP file
+    packets = rdpcap(args.input_file)
+
+    for packet in packets:
+            update_bytes_from_packet(packet, args.rbytes)
+
+    # Save the modified PCAP file
+    wrpcap(args.output_file, packets)
+
+    print("PCAP file successfully modified and saved as", args.output_file)
+
+if __name__ == "__main__":
+    main()

jbuf/scenarios/uac_pcap_opus.xml

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="ISO-8859-1" ?>
+<!DOCTYPE scenario SYSTEM "sipp.dtd">
+
+<!-- This program is free software; you can redistribute it and/or      -->
+<!-- modify it under the terms of the GNU General Public License as     -->
+<!-- published by the Free Software Foundation; either version 2 of the -->
+<!-- License, or (at your option) any later version.                    -->
+<!--                                                                    -->
+<!-- This program is distributed in the hope that it will be useful,    -->
+<!-- but WITHOUT ANY WARRANTY; without even the implied warranty of     -->
+<!-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the      -->
+<!-- GNU General Public License for more details.                       -->
+<!--                                                                    -->
+<!-- You should have received a copy of the GNU General Public License  -->
+<!-- along with this program; if not, write to the                      -->
+<!-- Free Software Foundation, Inc.,                                    -->
+<!-- 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA             -->
+<!--                                                                    -->
+<!--                 Sipp 'uac' scenario with pcap (rtp) play           -->
+<!--                                                                    -->
+
+<scenario name="UAC with media">
+  <!-- In client mode (sipp placing calls), the Call-ID MUST be         -->
+  <!-- generated by sipp. To do so, use [call_id] keyword.                -->
+  <send retrans="500">
+    <![CDATA[
+
+      INVITE sip:[service]@[remote_ip]:[remote_port] SIP/2.0
+      Via: SIP/2.0/[transport] [local_ip]:[local_port];branch=[branch]
+      From: sipp <sip:sipp@[local_ip]:[local_port]>;tag=[pid]SIPpTag09[call_number]
+      To: [service] <sip:[service]@[remote_ip]:[remote_port]>
+      Call-ID: [call_id]
+      CSeq: 1 INVITE
+      Contact: sip:sipp@[local_ip]:[local_port]
+      Max-Forwards: 70
+      Subject: Performance Test
+      Content-Type: application/sdp
+      Content-Length: [len]
+
+      v=0
+      o=user1 53655765 2353687637 IN IP[local_ip_type] [local_ip]
+      s=-
+      c=IN IP[local_ip_type] [local_ip]
+      t=0 0
+      m=audio [media_port] RTP/AVP 96
+      a=rtpmap:96 opus/48000/2
+
+    ]]>
+  </send>
+
+  <recv response="100" optional="true">
+  </recv>
+
+  <recv response="180" optional="true">
+  </recv>
+
+  <recv response="200" rtd="true" crlf="true">
+  </recv>
+
+  <send>
+    <![CDATA[
+
+      ACK sip:[service]@[remote_ip]:[remote_port] SIP/2.0
+      Via: SIP/2.0/[transport] [local_ip]:[local_port];branch=[branch]
+      From: sipp <sip:sipp@[local_ip]:[local_port]>;tag=[pid]SIPpTag09[call_number]
+      To: [service] <sip:[service]@[remote_ip]:[remote_port]>[peer_tag_param]
+      Call-ID: [call_id]
+      CSeq: 1 ACK
+      Contact: sip:sipp@[local_ip]:[local_port]
+      Max-Forwards: 70
+      Subject: Performance Test
+      Content-Length: 0
+
+    ]]>
+  </send>
+
+  <!-- Play a pre-recorded PCAP file (RTP stream)                       -->
+  <nop>
+    <action>
+        <exec play_pcap_audio="[pcap]"/>
+    </action>
+  </nop>
+
+  <!-- Pause 30 seconds, which is approximately the duration of the      -->
+  <!-- PCAP file                                                        -->
+  <pause milliseconds="30000"/>
+
+  <!-- The 'crlf' option inserts a blank line in the statistics report. -->
+  <send retrans="500">
+    <![CDATA[
+
+      BYE sip:[service]@[remote_ip]:[remote_port] SIP/2.0
+      Via: SIP/2.0/[transport] [local_ip]:[local_port];branch=[branch]
+      From: sipp <sip:sipp@[local_ip]:[local_port]>;tag=[pid]SIPpTag09[call_number]
+      To: [service] <sip:[service]@[remote_ip]:[remote_port]>[peer_tag_param]
+      Call-ID: [call_id]
+      CSeq: 2 BYE
+      Contact: sip:sipp@[local_ip]:[local_port]
+      Max-Forwards: 70
+      Subject: Performance Test
+      Content-Length: 0
+
+    ]]>
+  </send>
+
+  <recv response="200" crlf="true">
+  </recv>
+</scenario>