From 2e8c78a1ad19800fd9d4d25c72802d8b2984953c Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Fri, 21 Feb 2025 10:29:17 -0600 Subject: [PATCH] Add email-on-fail to cron job (#1241) * Add email-on-fail to cron job * Update changelog --- CHANGELOG.md | 5 ++- maintenance_scripts/README.md | 31 +++++++++++++- maintenance_scripts/example.env | 8 ++++ maintenance_scripts/run_jobs.sh | 75 +++++++++++++++++++++++++-------- 4 files changed, 99 insertions(+), 20 deletions(-) create mode 100644 maintenance_scripts/example.env diff --git a/CHANGELOG.md b/CHANGELOG.md index 7814553d0..3912e7de2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ - Ensure merge tables are declared during file insertion #1205 - Update URL for DANDI Docs #1210 -- Improve cron job documentation and script #1226 +- Improve cron job documentation and script #1226, #1241 - Only add merge parts to `source_class_dict` if present in codebase #1237 ### Pipelines @@ -18,7 +18,8 @@ - Allow population of missing `PositionIntervalMap` entries during population of `DLCPoseEstimation` #1208 - Spikesorting - - Fix compatibility bug between v1 pipeline and `SortedSpikesGroup` unit filtering #1238 + - Fix compatibility bug between v1 pipeline and `SortedSpikesGroup` unit + filtering #1238 ## [0.5.4] (December 20, 2024) diff --git a/maintenance_scripts/README.md b/maintenance_scripts/README.md index fd1dedafd..1cb68e82d 100644 --- a/maintenance_scripts/README.md +++ b/maintenance_scripts/README.md @@ -26,11 +26,40 @@ regularly as cron jobs. 1. Clone the repository to the desired location. 2. Set up a config file by copying `dj_local_conf_example.json` to `dj_local_conf.json` and filling in the necessary information. -3. Set up a cron job to run `run_jobs.sh` at the desired interval by running +3. Copy the `example.env` file to `.env` in the `maintenance_scripts` directory + and fill in the necessary information, including... + - `SPYGLASS_CONDA_ENV`: the name of the conda environment with Spyglass and + DataJoint installed. + - `SPYGLASS_REPO_PATH`: the path to the Spyglass repository. + - `SPYGLASS_LOG`: the path to the log file. + - Optional email settings. If not set, email notifications will not be sent. + - `SPYGLASS_EMAIL_SRC`: The email address from which to send notifications. + - `SPYGLASS_EMAIL_PASS`: the password for the email address. + - `SPYGLASS_EMAIL_DEST`: the email address to which to send notifications. +4. Set up a cron job to run `run_jobs.sh` at the desired interval by running `crontab -e` and adding the script. +Note that the log file will automatically be truncated to `SPYGLASS_MAX_LOG` +lines on each run. 1000 lines should be sufficient. + +### Example Cron Job + In the following example, the script is set to run every Monday at 4:00 AM. ```text 0 4 * * 1 /path/to/run_jobs.sh ``` + +### Email Service + +The script uses `curl` to send email notifications on failure. While this can +work with +[many email services](https://everything.curl.dev/usingcurl/smtp.html), Gmail is +a common choice. To use Gmail, you will need to ... + +1. Turn on [2-step verification](https://myaccount.google.com/security-checkup) +2. Turn on [less secure apps](https://myaccount.google.com/lesssecureapps) +3. Create an [app password](https://myaccount.google.com/apppasswords) + +`curl` will not work with your master Gmail password, so you will need to use +the app password instead. diff --git a/maintenance_scripts/example.env b/maintenance_scripts/example.env new file mode 100644 index 000000000..92bf12ebb --- /dev/null +++ b/maintenance_scripts/example.env @@ -0,0 +1,8 @@ +# NOTE: Use quotes for strings with spaces +export SPYGLASS_CONDA_ENV=spyglass # name of conda environment +export SPYGLASS_REPO_PATH=/home/franklab/spyglass # path to spyglass repo +export SPYGLASS_LOG=/home/franklab/spyglass.log # log file +export SPYGLASS_EMAIL_SRC=example@email.com # optional, email sender +export SPYGLASS_EMAIL_PASS="example password" # optional, email password +export SPYGLASS_EMAIL_DEST=other@email.com # optional, email recipient +export SPYGLASS_MAX_LOG=1000 # number of lines to keep in log diff --git a/maintenance_scripts/run_jobs.sh b/maintenance_scripts/run_jobs.sh index 15bbd099f..adc6048bb 100755 --- a/maintenance_scripts/run_jobs.sh +++ b/maintenance_scripts/run_jobs.sh @@ -1,14 +1,52 @@ #!/bin/bash +# AUTHOR: Chris Brozdowski +# DATE: 2025-02-20 +# +# 1. Go to SPYGLASS_REPO_PATH and pull the latest changes from the master branch +# 2. Test the SPYGLASS_CONDA_ENV conda environment +# 3. Test the connection to the database +# 4. Run the cleanup script +# +# This script is intended to be run as a cron job, weekly or more frequently. +# It will store a log of its output in SPYGLASS_LOG. +# If any of the operations fail, an email will be sent to SPYGLASS_EMAIL_DEST -# SETUP: -# 1. Create a conda environment with datajoint installed. -# 2. Edit the variables below to match your setup. -# 3. Set up the cron job (See README.md) -# Note that the log file will be truncated to the last 1000 lines. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/.env" # load environment variables from this directory -SPYGLASS_CONDA_ENV=spyglass -SPYGLASS_REPO_PATH=/home/franklab/spyglass -SPYGLASS_LOG=/home/franklab/spyglass/spyglass.log +if [[ -z "${SPYGLASS_CONDA_ENV}" \ + || -z "${SPYGLASS_REPO_PATH}" \ + || -z "${SPYGLASS_LOG}" ]]; then + echo "Error: SPYGLASS_CONDA_ENV, SPYGLASS_REPO_PATH, + and SPYGLASS_LOG must be set in .env" + exit 1 +fi + +EMAIL_TEMPLATE=$(cat <<-EOF +From: "Spyglass" <$SPYGLASS_EMAIL_SRC> +To: $SPYGLASS_EMAIL_DEST +Subject: cron fail - $(date "+%Y-%m-%d") + +%s +EOF +) + +on_fail() { # $1: error message. Echo message and send as email + echo "Error: $1" + if [ -z "$SPYGLASS_EMAIL_SRC" ]; then + return 1 # No email source, so don't send an email + fi + local error_msg="$1" + local content + content=$(printf "$EMAIL_TEMPLATE" "$error_msg") + + curl -o /dev/null --ssl-reqd \ + --url "smtps://smtp.gmail.com:465" \ + --user "${SPYGLASS_EMAIL_SRC}:${SPYGLASS_EMAIL_PASS}" \ + --mail-from "$SPYGLASS_EMAIL_SRC" \ + --mail-rcpt "$SPYGLASS_EMAIL_DEST" \ + -T <(echo "$content") +} exec >> $SPYGLASS_LOG 2>&1 @@ -17,24 +55,27 @@ echo "SPYGLASS CRON JOB START: $(date +"%Y-%m-%d %H:%M:%S")" # Run from the root of the spyglass repository cd $SPYGLASS_REPO_PATH || \ - { echo "Error: Could not change to the spyglass directory"; exit 1; } + { on_fail "Could not find repo path: $SPYGLASS_REPO_PATH"; exit 1; } + # Update the spyglass repository -git pull https://github.com/LorenFrankLab/spyglass.git master > /dev/null || \ - { echo "Error: $PWD Could not update the spyglass repository"; exit 1; } +git pull --quiet \ + https://github.com/LorenFrankLab/spyglass.git master > /dev/null || \ + { on_fail "Could not update the spyglass repo $PWD"; exit 1; } # Test conda environment if ! conda env list | grep -q $SPYGLASS_CONDA_ENV; then - echo "Error: Conda environment $SPYGLASS_CONDA_ENV not found" - exit 1 + on_fail "Conda environment $SPYGLASS_CONDA_ENV not found" + exit 1 fi # convenience function to run a command in the spyglass conda environment conda_run() { conda run --name $SPYGLASS_CONDA_ENV "$@"; } # Test connection to the database -conda_run python -c "import datajoint as dj; dj.conn()" > /dev/null || \ - { echo "Error: Could not connect to the database"; exit 1; } +CONN_TEST="import datajoint as dj; dj.logger.setLevel('ERROR'); dj.conn()" +conda_run python -c "$CONN_TEST" > /dev/null || \ + { on_fail "Could not connect to the database"; exit 1; } # Run cleanup script conda_run python maintenance_scripts/cleanup.py @@ -42,5 +83,5 @@ conda_run python maintenance_scripts/cleanup.py echo "SPYGLASS CRON JOB END" # truncate long log file -tail -n 1000 "$SPYGLASS_LOG" > "${SPYGLASS_LOG}.tmp" && \ - mv "${SPYGLASS_LOG}.tmp" "$SPYGLASS_LOG" +tail -n ${SPYGLASS_MAX_LOG:-1000} "$SPYGLASS_LOG" > "${SPYGLASS_LOG}.tmp" \ + && mv "${SPYGLASS_LOG}.tmp" "$SPYGLASS_LOG"