ResEl NSA v0.5

resel-nsa-0.4/debian/changelog → resel-nsa-0.5/debian/changelog

@@ -1,3 +1,11 @@
+resel-nsa (0.5-1) UNRELEASED; urgency=low
+
+  * Add multiprocessing capabilities,
+  * Add log rotation,
+  * Improve code quality.
+
+ -- Benjamin Somers <[email protected]>  Wed, 14 Oct 2020 23:22:00 +0200
+
 resel-nsa (0.4-1) UNRELEASED; urgency=low
 
   * Add a manager to simplify administrative tasks and create manpages.

resel-nsa-0.4/debian/postinst → resel-nsa-0.5/debian/postinst

@@ -11,9 +11,14 @@ case "$1" in
     chown nsa:nsa /usr/libexec/nsa/nsa-collect
     chown nsa:nsa /usr/libexec/nsa/nsa-send
 
+    chown nsa:nsa /var/log/resel-nsa
+
     chown nsa:nsa /var/spool/nsa
+    chown nsa:nsa /var/spool/nsa/error
     chown nsa:nsa /var/spool/nsa/pending
     chown nsa:nsa /var/spool/nsa/ready
+    chown nsa:nsa /var/spool/nsa/sending
+    chown nsa:nsa /var/spool/nsa/timeout
   ;;
 
   abort-upgrade|abort-remove|abort-deconfigure)

resel-nsa-0.5/debian/resel-nsa.install

@@ -0,0 +1,3 @@
+etc/* etc
+var/* var
+usr/* usr

resel-nsa-0.5/debian/resel-nsa.logrotate

@@ -0,0 +1,8 @@
+/var/log/resel-nsa/nsa-send {
+  compress
+  nocreate
+  daily
+  notifempty
+  rotate 7
+  missingok
+}

resel-nsa-0.5/etc/nsa/config

@@ -0,0 +1,5 @@
+#LOG_DATE_FORMAT=<Date Format>
+#TOKEN=<Default WRITE Token>
+#ENDPOINT=<Warp 10 Endpoint>
+#SEND_JOBS=<Ingress Jobs>
+#SEND_TIMEOUT=<Ingress Timeout>

resel-nsa-0.4/usr/libexec/nsa/nsa-collect → resel-nsa-0.5/usr/libexec/nsa/nsa-collect

@@ -10,12 +10,12 @@ set -aeu
 source /etc/nsa/config
 for lib in /etc/nsa/functions/*
 do
-  source $lib
+  source "$lib"
 done
 declare -A DURATIONS
 DURATIONS=([m]=1 [h]=60 [d]=1440 [w]=10080)
 MINUTES=$(($(date +%s) / 60))
-TIMESTAMP=$(($MINUTES * 60))000000
+TIMESTAMP=$((MINUTES * 60))000000
 set +a
 
 {
@@ -30,44 +30,43 @@ rand()
 
 process()
 {
-  if [ $(($MINUTES % $2)) -ne 0 ]
+  if [ $((MINUTES % $2)) -ne 0 ]
   then
     return
   fi
-  base=$(basename $1)
+  base=$(basename "$1")
   category=${base%%.*}
   pending_dir=/var/spool/nsa/pending/$category
   ready_dir=/var/spool/nsa/ready/$category
-  mkdir -p $pending_dir
-  mkdir -p $ready_dir
-  chmod 6770 $pending_dir
-  chmod 6770 $ready_dir
+  mkdir -p "$pending_dir" "$ready_dir"
+  chmod 6770 "$pending_dir"
+  chmod 6770 "$ready_dir"
   (
     set -a
     source /etc/nsa/profiles.d/$category
     set +a
     filename=$pending_dir/nsa.$TIMESTAMP.$(rand)
-    $1 | sed "/^{/! s/^/./; /}/! s/ /{} /; s/^/$TIMESTAMP\/\/ $base/" > $filename
-    mv $filename $ready_dir
+    $1 | sed "/^{/! s/^/./; /}/! s/ /{} /; s/^/$TIMESTAMP\/\/ $base/" > "$filename"
+    mv "$filename" "$ready_dir"
   )&
 }
 
 for element in /etc/nsa/mods-enabled/*
 do
-  if [ -f $element ]
+  if [ -f "$element" ]
   then
-    process $element 1
-  elif [ -d $element ]
+    process "$element" 1
+  elif [ -d "$element" ]
   then
-    period=$(basename $element)
+    period=$(basename "$element")
     value=${period::-1}
     suffix=${period: -1}
-    minutes=$(($value * ${DURATIONS[$suffix]}))
-    for file in $element/*
+    minutes=$((value * ${DURATIONS[$suffix]}))
+    for file in "$element"/*
     do
-      if [ -f $file ]
+      if [ -f "$file" ]
       then
-        process $file $minutes
+        process "$file" $minutes
       fi
     done
   fi

resel-nsa-0.5/usr/libexec/nsa/nsa-send

@@ -0,0 +1,90 @@
+#!/bin/bash
+
+if [ -f /etc/nsa/.disable ]
+then
+  exit 1
+fi
+
+shopt -s nullglob
+set -aeu
+LOG_DIR=/var/log/resel-nsa
+LOG_DATE_FORMAT='%Y-%m-%d %H:%M:%S'
+SEND_JOBS=$(getconf _NPROCESSORS_ONLN)
+SEND_TIMEOUT=1m
+source /etc/nsa/config
+
+declare -A DURATIONS
+DURATIONS=([s]=1 [m]=60 [h]=3600)
+TIMEOUT_VALUE=${SEND_TIMEOUT::-1}
+TIMEOUT_SUFFIX=${SEND_TIMEOUT: -1}
+TIMEOUT=$((TIMEOUT_VALUE * ${DURATIONS[$TIMEOUT_SUFFIX]}))
+set +a
+
+open_semaphore()
+{
+  fifo=/var/spool/nsa/pipe-$$
+  mkfifo $fifo
+  exec 3<>/$fifo
+  rm $fifo
+  local i=$1
+  for ((; i>0; i--))
+  do
+    printf %s 000 >&3
+  done
+}
+
+run_job()
+{
+  read -u 3 -n 3 -r code && ((0==code)) || exit "$code"
+  (
+    timeout_dir=/var/spool/nsa/timeout/$3
+    mkdir -p "$timeout_dir"
+    {
+      sleep $TIMEOUT
+      echo "[$(date +"$LOG_DATE_FORMAT")] sending $2 timed out" >> "$LOG_DIR/nsa-send"
+      mv "/var/spool/nsa/sending/$3/$2" "$timeout_dir"
+      kill $BASHPID
+    }&
+    timer_pid=$!
+    ( "$@"; )
+    kill $timer_pid
+    printf '%.3d' $? >&3
+  )&
+}
+
+process()
+{
+  error_dir=/var/spool/nsa/error/$2
+  sending_dir=/var/spool/nsa/sending/$2
+  mkdir -p "$error_dir"
+  file="$sending_dir/$1"
+  source "/etc/nsa/profiles.d/$2"
+  echo "[$(date +"$LOG_DATE_FORMAT")] processing $1" >> "$LOG_DIR/nsa-send"
+  if [ ! -s "$file" ] || ([ -f "$file" ] && curl -fs -H "X-Warp10-Token: $TOKEN" -H 'Transfer-Encoding: chunked' -T "$file" "$ENDPOINT")
+  then
+    rm -f "$file"
+    echo "[$(date +"$LOG_DATE_FORMAT")] $1 sent" >> $LOG_DIR/nsa-send
+  else
+    mv "$file" "$error_dir"
+    echo "[$(date +"$LOG_DATE_FORMAT")] $1 moved to error pool" >> $LOG_DIR/nsa-send
+  fi
+}
+
+open_semaphore "$SEND_JOBS"
+for dir in /var/spool/nsa/ready/*
+do
+  if [ -d "$dir" ]
+  then
+    category=$(basename "$dir")
+    ready_dir=/var/spool/nsa/ready/$category
+    sending_dir=/var/spool/nsa/sending/$category
+    mkdir -p "$sending_dir"
+    files=("$ready_dir"/*)
+    for ((i=${#files[@]}-1; i>=0; i--))
+    do
+      file=$(basename "${files[$i]}")
+      mv "$ready_dir/$file" "$sending_dir"
+      run_job process "$file" "$category"
+    done
+  fi
+done