watch_status

#! /bin/bash -

# This script estimates the current state of the hashing
# process by using `du` on all search directories
# and prints it with the current time every 10 minutes.
# Usage: ./watch_state <hash file> <dir1> <dir2> ...

trap - EXIT
trap 'exit 0' SIGUSR1

startTime=$(date +%s)
minWait=10
maxWait=600
export LC_ALL=C # faster grep

hashFile="$1"
if [[ ! -z "$SLURM_JOB_ID" ]]; then
    printf 'Starting progress monitoring at %s for %s.\n' "$(date)" "$hashFile"
    printf 'Direcories: %s\n' "$(printf "'%s' " "${@:2}")"
    printf 'Calculating total size in background...\n'
    cd "$SLURM_SUBMIT_DIR"
fi

getState(){
    exec 10<> <(:)
    bytes=$(cat <(grep -va "^#\|^%" "$hashFile" 2> /dev/null |
            cut -d, -f1 | tee >(wc -l >&10) ) \
            <(printf 0) | paste -sd+ | bc)
    read -u 10 nfiles
    exec 10>&-
    printf '%d %d' $bytes $nfiles
}
hashStartTime=$(date +%s)
read hashStartState startFilesCount <<< $(getState)

lastFile(){
    tail -n1 "$hashFile" | cut -d, -f3
}

sleepUntil(){
    local now=$(date +%s)
    (( $1 > $now )) && sleep $(( $1 - $now ))
}

total_file=$(mktemp)
set +m
{ du -sb --apparent-size -- "${@:2}" | \
    cut -f1 | sed '/[^0-9]/d' | paste -sd+ | \
    bc > $total_file & } 2>/dev/null
duPID=$!
trap 'kill $duPID 2> /dev/null; rm $total_file' EXIT

TOTAL='?B'

displayTime(){
  local T=$1
  local D=$((T/60/60/24))
  local H=$((T/60/60%24))
  local M=$((T/60%60))
  local S=$((T%60))
  (( $D != 0 )) && printf '%d days ' $D
  (( $H != 0 )) && printf '%d h ' $H
  (( $M != 0 )) && printf '%d min ' $M
  (( $D != 0 || $H > 0 || $M > 0 )) && printf 'and '
  printf '%d sec\n' $S
}

finished(){
    printf '[%(%F %H:%M:%S)T] 100.00 %% (%s/%s)\n' \
        "$(date +%s)" "$TOTAL" "$TOTAL"
    exit 0
}
trap 'finished; exit 0' SIGUSR1

lastTime=$hashStartTime
lastState=$hashStartState
lastFileCount=$startFilesCount
STATE=' ?%'
# wait for output of the super script:
sleepUntil $(( $startTime + 1 ))
while true; do
    checkpoint=$(date +%s)
    read current filecount <<< $(getState)
    if [ -z "$total" ]; then
        total="$(cat $total_file)"
        [ -z "$total" ] || TOTAL="$(numfmt --to=iec-i $total)B"
        waitTime=$(( $checkpoint - $startTime ))
        waitTime=$(( $waitTime>$minWait?$waitTime:$minWait ))
        waitTime=$(( $waitTime<$maxWait?$waitTime:$maxWait ))
    fi
    if [ ! -z "$total" ]; then
        printf -v state "%02d" $(( 10000 * $current / $total ))
        STATE=" ${state:0:-2}.${state: -2} %"
    fi
    CURRENT=" ($(numfmt --to=iec-i $current)B/$TOTAL)"
    time=$(date "+%F %H:%M:%S")
    if [[ $current -le $hashStartState ]]; then
        hashStartState=$current
        SPEED=""
        ETA=""
    else
        now=$(date +%s)
        passed=$(( $now - $lastTime ))
        lastTime=$now
        diff=$(( $current - $lastState ))
        diffF=$(( $filecount - $lastFileCount ))
        lastState=$current
        lastFileCount=$filecount
        speed=$(( $diff / $passed ))
        speedF=$(bc <<< "scale=2; 60 * $diffF / $passed")
        SPEED=" at $(numfmt --to=iec-i $speed)B/s and $speedF files/min"
        total_passed=$(( $now - $hashStartTime ))
        total_diff=$(( $current - $hashStartState ))
        if [ ! -z "$total" ]; then
            rem=$(( $total - $current ))
            eta=$(bc <<< "$rem * $total_passed / $total_diff")
            ETA=" ~ $(displayTime $eta) remaining"
        fi
    fi
    LAST=" ($(lastFile))"
    printf '[%s]%s%s%s%s%s\n' "$time" "$STATE" "$CURRENT" "$SPEED" "$ETA" "$LAST"
    sleepUntil $(( $checkpoint + $waitTime ))
done