diff --git a/resel-nsa-0.4/debian/resel-nsa.install b/resel-nsa-0.4/debian/resel-nsa.install deleted file mode 100644 index f7d2fa9..0000000 --- a/resel-nsa-0.4/debian/resel-nsa.install +++ /dev/null @@ -1,3 +0,0 @@ -etc/* etc -var/spool/* var/spool -usr/* usr diff --git a/resel-nsa-0.4/etc/nsa/config b/resel-nsa-0.4/etc/nsa/config deleted file mode 100644 index 1a1ed19..0000000 --- a/resel-nsa-0.4/etc/nsa/config +++ /dev/null @@ -1,2 +0,0 @@ -TOKEN='DEFAULT_TOKEN' -ENDPOINT='WARP_10_ENDPOINT' diff --git a/resel-nsa-0.4/usr/libexec/nsa/nsa-send b/resel-nsa-0.4/usr/libexec/nsa/nsa-send deleted file mode 100755 index 3a4b6f2..0000000 --- a/resel-nsa-0.4/usr/libexec/nsa/nsa-send +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -if [ -f /etc/nsa/.disable ] -then - exit 1 -fi - -shopt -s nullglob -source /etc/nsa/config - -{ - sleep 60 - kill $$ -} & - -for dir in /var/spool/nsa/ready/* -do - if [ -d $dir ] - then - category=$(basename $dir) - for file in /var/spool/nsa/ready/$category/* - do - source /etc/nsa/profiles.d/$category - if [ ! -s $file ] || ([ -f $file ] && curl -fs -H "X-Warp10-Token: $TOKEN" -H 'Transfer-Encoding: chunked' -T $file $ENDPOINT) - then - rm -f $file - fi - done - fi -done diff --git a/resel-nsa-0.4/debian/changelog b/resel-nsa-0.5/debian/changelog similarity index 76% rename from resel-nsa-0.4/debian/changelog rename to resel-nsa-0.5/debian/changelog index 9e1a8a2..b7f4f23 100644 --- a/resel-nsa-0.4/debian/changelog +++ b/resel-nsa-0.5/debian/changelog @@ -1,3 +1,11 @@ +resel-nsa (0.5-1) UNRELEASED; urgency=low + + * Add multiprocessing capabilities, + * Add log rotation, + * Improve code quality. + + -- Benjamin Somers Wed, 14 Oct 2020 23:22:00 +0200 + resel-nsa (0.4-1) UNRELEASED; urgency=low * Add a manager to simplify administrative tasks and create manpages. diff --git a/resel-nsa-0.4/debian/compat b/resel-nsa-0.5/debian/compat similarity index 100% rename from resel-nsa-0.4/debian/compat rename to resel-nsa-0.5/debian/compat diff --git a/resel-nsa-0.4/debian/control b/resel-nsa-0.5/debian/control similarity index 100% rename from resel-nsa-0.4/debian/control rename to resel-nsa-0.5/debian/control diff --git a/resel-nsa-0.4/debian/copyright b/resel-nsa-0.5/debian/copyright similarity index 100% rename from resel-nsa-0.4/debian/copyright rename to resel-nsa-0.5/debian/copyright diff --git a/resel-nsa-0.4/debian/docs/nsa-admin.8 b/resel-nsa-0.5/debian/docs/nsa-admin.8 similarity index 100% rename from resel-nsa-0.4/debian/docs/nsa-admin.8 rename to resel-nsa-0.5/debian/docs/nsa-admin.8 diff --git a/resel-nsa-0.4/debian/docs/resel-nsa.7 b/resel-nsa-0.5/debian/docs/resel-nsa.7 similarity index 100% rename from resel-nsa-0.4/debian/docs/resel-nsa.7 rename to resel-nsa-0.5/debian/docs/resel-nsa.7 diff --git a/resel-nsa-0.4/debian/postinst b/resel-nsa-0.5/debian/postinst similarity index 79% rename from resel-nsa-0.4/debian/postinst rename to resel-nsa-0.5/debian/postinst index e6422b8..4996f13 100755 --- a/resel-nsa-0.4/debian/postinst +++ b/resel-nsa-0.5/debian/postinst @@ -11,9 +11,14 @@ case "$1" in chown nsa:nsa /usr/libexec/nsa/nsa-collect chown nsa:nsa /usr/libexec/nsa/nsa-send + chown nsa:nsa /var/log/resel-nsa + chown nsa:nsa /var/spool/nsa + chown nsa:nsa /var/spool/nsa/error chown nsa:nsa /var/spool/nsa/pending chown nsa:nsa /var/spool/nsa/ready + chown nsa:nsa /var/spool/nsa/sending + chown nsa:nsa /var/spool/nsa/timeout ;; abort-upgrade|abort-remove|abort-deconfigure) diff --git a/resel-nsa-0.4/debian/preinst b/resel-nsa-0.5/debian/preinst similarity index 100% rename from resel-nsa-0.4/debian/preinst rename to resel-nsa-0.5/debian/preinst diff --git a/resel-nsa-0.4/debian/resel-nsa.cron.d b/resel-nsa-0.5/debian/resel-nsa.cron.d similarity index 100% rename from resel-nsa-0.4/debian/resel-nsa.cron.d rename to resel-nsa-0.5/debian/resel-nsa.cron.d diff --git a/resel-nsa-0.5/debian/resel-nsa.install b/resel-nsa-0.5/debian/resel-nsa.install new file mode 100644 index 0000000..9897613 --- /dev/null +++ b/resel-nsa-0.5/debian/resel-nsa.install @@ -0,0 +1,3 @@ +etc/* etc +var/* var +usr/* usr diff --git a/resel-nsa-0.4/debian/resel-nsa.lintian-overrides b/resel-nsa-0.5/debian/resel-nsa.lintian-overrides similarity index 100% rename from resel-nsa-0.4/debian/resel-nsa.lintian-overrides rename to resel-nsa-0.5/debian/resel-nsa.lintian-overrides diff --git a/resel-nsa-0.5/debian/resel-nsa.logrotate b/resel-nsa-0.5/debian/resel-nsa.logrotate new file mode 100644 index 0000000..0cc9be5 --- /dev/null +++ b/resel-nsa-0.5/debian/resel-nsa.logrotate @@ -0,0 +1,8 @@ +/var/log/resel-nsa/nsa-send { + compress + nocreate + daily + notifempty + rotate 7 + missingok +} diff --git a/resel-nsa-0.4/debian/resel-nsa.manpages b/resel-nsa-0.5/debian/resel-nsa.manpages similarity index 100% rename from resel-nsa-0.4/debian/resel-nsa.manpages rename to resel-nsa-0.5/debian/resel-nsa.manpages diff --git a/resel-nsa-0.4/debian/rules b/resel-nsa-0.5/debian/rules similarity index 100% rename from resel-nsa-0.4/debian/rules rename to resel-nsa-0.5/debian/rules diff --git a/resel-nsa-0.4/debian/source/format b/resel-nsa-0.5/debian/source/format similarity index 100% rename from resel-nsa-0.4/debian/source/format rename to resel-nsa-0.5/debian/source/format diff --git a/resel-nsa-0.4/etc/nsa/.disable b/resel-nsa-0.5/etc/nsa/.disable similarity index 100% rename from resel-nsa-0.4/etc/nsa/.disable rename to resel-nsa-0.5/etc/nsa/.disable diff --git a/resel-nsa-0.5/etc/nsa/config b/resel-nsa-0.5/etc/nsa/config new file mode 100644 index 0000000..c878311 --- /dev/null +++ b/resel-nsa-0.5/etc/nsa/config @@ -0,0 +1,5 @@ +#LOG_DATE_FORMAT= +#TOKEN= +#ENDPOINT= +#SEND_JOBS= +#SEND_TIMEOUT= diff --git a/resel-nsa-0.4/etc/nsa/functions/.keep b/resel-nsa-0.5/etc/nsa/functions/.keep similarity index 100% rename from resel-nsa-0.4/etc/nsa/functions/.keep rename to resel-nsa-0.5/etc/nsa/functions/.keep diff --git a/resel-nsa-0.4/etc/nsa/mods-available/.keep b/resel-nsa-0.5/etc/nsa/mods-available/.keep similarity index 100% rename from resel-nsa-0.4/etc/nsa/mods-available/.keep rename to resel-nsa-0.5/etc/nsa/mods-available/.keep diff --git a/resel-nsa-0.4/etc/nsa/mods-enabled/.keep b/resel-nsa-0.5/etc/nsa/mods-enabled/.keep similarity index 100% rename from resel-nsa-0.4/etc/nsa/mods-enabled/.keep rename to resel-nsa-0.5/etc/nsa/mods-enabled/.keep diff --git a/resel-nsa-0.4/etc/nsa/profiles.d/.keep b/resel-nsa-0.5/etc/nsa/profiles.d/.keep similarity index 100% rename from resel-nsa-0.4/etc/nsa/profiles.d/.keep rename to resel-nsa-0.5/etc/nsa/profiles.d/.keep diff --git a/resel-nsa-0.4/usr/bin/nsa-admin b/resel-nsa-0.5/usr/bin/nsa-admin similarity index 100% rename from resel-nsa-0.4/usr/bin/nsa-admin rename to resel-nsa-0.5/usr/bin/nsa-admin diff --git a/resel-nsa-0.4/usr/libexec/nsa/nsa-collect b/resel-nsa-0.5/usr/libexec/nsa/nsa-collect similarity index 62% rename from resel-nsa-0.4/usr/libexec/nsa/nsa-collect rename to resel-nsa-0.5/usr/libexec/nsa/nsa-collect index 4d3b54b..125e1b2 100755 --- a/resel-nsa-0.4/usr/libexec/nsa/nsa-collect +++ b/resel-nsa-0.5/usr/libexec/nsa/nsa-collect @@ -10,12 +10,12 @@ set -aeu source /etc/nsa/config for lib in /etc/nsa/functions/* do - source $lib + source "$lib" done declare -A DURATIONS DURATIONS=([m]=1 [h]=60 [d]=1440 [w]=10080) MINUTES=$(($(date +%s) / 60)) -TIMESTAMP=$(($MINUTES * 60))000000 +TIMESTAMP=$((MINUTES * 60))000000 set +a { @@ -30,44 +30,43 @@ rand() process() { - if [ $(($MINUTES % $2)) -ne 0 ] + if [ $((MINUTES % $2)) -ne 0 ] then return fi - base=$(basename $1) + base=$(basename "$1") category=${base%%.*} pending_dir=/var/spool/nsa/pending/$category ready_dir=/var/spool/nsa/ready/$category - mkdir -p $pending_dir - mkdir -p $ready_dir - chmod 6770 $pending_dir - chmod 6770 $ready_dir + mkdir -p "$pending_dir" "$ready_dir" + chmod 6770 "$pending_dir" + chmod 6770 "$ready_dir" ( set -a source /etc/nsa/profiles.d/$category set +a filename=$pending_dir/nsa.$TIMESTAMP.$(rand) - $1 | sed "/^{/! s/^/./; /}/! s/ /{} /; s/^/$TIMESTAMP\/\/ $base/" > $filename - mv $filename $ready_dir + $1 | sed "/^{/! s/^/./; /}/! s/ /{} /; s/^/$TIMESTAMP\/\/ $base/" > "$filename" + mv "$filename" "$ready_dir" )& } for element in /etc/nsa/mods-enabled/* do - if [ -f $element ] + if [ -f "$element" ] then - process $element 1 - elif [ -d $element ] + process "$element" 1 + elif [ -d "$element" ] then - period=$(basename $element) + period=$(basename "$element") value=${period::-1} suffix=${period: -1} - minutes=$(($value * ${DURATIONS[$suffix]})) - for file in $element/* + minutes=$((value * ${DURATIONS[$suffix]})) + for file in "$element"/* do - if [ -f $file ] + if [ -f "$file" ] then - process $file $minutes + process "$file" $minutes fi done fi diff --git a/resel-nsa-0.5/usr/libexec/nsa/nsa-send b/resel-nsa-0.5/usr/libexec/nsa/nsa-send new file mode 100755 index 0000000..165e373 --- /dev/null +++ b/resel-nsa-0.5/usr/libexec/nsa/nsa-send @@ -0,0 +1,90 @@ +#!/bin/bash + +if [ -f /etc/nsa/.disable ] +then + exit 1 +fi + +shopt -s nullglob +set -aeu +LOG_DIR=/var/log/resel-nsa +LOG_DATE_FORMAT='%Y-%m-%d %H:%M:%S' +SEND_JOBS=$(getconf _NPROCESSORS_ONLN) +SEND_TIMEOUT=1m +source /etc/nsa/config + +declare -A DURATIONS +DURATIONS=([s]=1 [m]=60 [h]=3600) +TIMEOUT_VALUE=${SEND_TIMEOUT::-1} +TIMEOUT_SUFFIX=${SEND_TIMEOUT: -1} +TIMEOUT=$((TIMEOUT_VALUE * ${DURATIONS[$TIMEOUT_SUFFIX]})) +set +a + +open_semaphore() +{ + fifo=/var/spool/nsa/pipe-$$ + mkfifo $fifo + exec 3<>/$fifo + rm $fifo + local i=$1 + for ((; i>0; i--)) + do + printf %s 000 >&3 + done +} + +run_job() +{ + read -u 3 -n 3 -r code && ((0==code)) || exit "$code" + ( + timeout_dir=/var/spool/nsa/timeout/$3 + mkdir -p "$timeout_dir" + { + sleep $TIMEOUT + echo "[$(date +"$LOG_DATE_FORMAT")] sending $2 timed out" >> "$LOG_DIR/nsa-send" + mv "/var/spool/nsa/sending/$3/$2" "$timeout_dir" + kill $BASHPID + }& + timer_pid=$! + ( "$@"; ) + kill $timer_pid + printf '%.3d' $? >&3 + )& +} + +process() +{ + error_dir=/var/spool/nsa/error/$2 + sending_dir=/var/spool/nsa/sending/$2 + mkdir -p "$error_dir" + file="$sending_dir/$1" + source "/etc/nsa/profiles.d/$2" + echo "[$(date +"$LOG_DATE_FORMAT")] processing $1" >> "$LOG_DIR/nsa-send" + if [ ! -s "$file" ] || ([ -f "$file" ] && curl -fs -H "X-Warp10-Token: $TOKEN" -H 'Transfer-Encoding: chunked' -T "$file" "$ENDPOINT") + then + rm -f "$file" + echo "[$(date +"$LOG_DATE_FORMAT")] $1 sent" >> $LOG_DIR/nsa-send + else + mv "$file" "$error_dir" + echo "[$(date +"$LOG_DATE_FORMAT")] $1 moved to error pool" >> $LOG_DIR/nsa-send + fi +} + +open_semaphore "$SEND_JOBS" +for dir in /var/spool/nsa/ready/* +do + if [ -d "$dir" ] + then + category=$(basename "$dir") + ready_dir=/var/spool/nsa/ready/$category + sending_dir=/var/spool/nsa/sending/$category + mkdir -p "$sending_dir" + files=("$ready_dir"/*) + for ((i=${#files[@]}-1; i>=0; i--)) + do + file=$(basename "${files[$i]}") + mv "$ready_dir/$file" "$sending_dir" + run_job process "$file" "$category" + done + fi +done diff --git a/resel-nsa-0.4/var/spool/nsa/pending/.keep b/resel-nsa-0.5/var/log/resel-nsa/.keep similarity index 100% rename from resel-nsa-0.4/var/spool/nsa/pending/.keep rename to resel-nsa-0.5/var/log/resel-nsa/.keep diff --git a/resel-nsa-0.4/var/spool/nsa/ready/.keep b/resel-nsa-0.5/var/spool/nsa/error/.keep similarity index 100% rename from resel-nsa-0.4/var/spool/nsa/ready/.keep rename to resel-nsa-0.5/var/spool/nsa/error/.keep diff --git a/resel-nsa-0.5/var/spool/nsa/pending/.keep b/resel-nsa-0.5/var/spool/nsa/pending/.keep new file mode 100644 index 0000000..e69de29 diff --git a/resel-nsa-0.5/var/spool/nsa/ready/.keep b/resel-nsa-0.5/var/spool/nsa/ready/.keep new file mode 100644 index 0000000..e69de29 diff --git a/resel-nsa-0.5/var/spool/nsa/sending/.keep b/resel-nsa-0.5/var/spool/nsa/sending/.keep new file mode 100644 index 0000000..e69de29 diff --git a/resel-nsa-0.5/var/spool/nsa/timeout/.keep b/resel-nsa-0.5/var/spool/nsa/timeout/.keep new file mode 100644 index 0000000..e69de29