From cff510228927fbbd709e21e525d5e8fb35e2e646 Mon Sep 17 00:00:00 2001
From: galaxy <patrick@plealog.com>
Date: Wed, 8 Mar 2023 13:42:33 +0000
Subject: [PATCH 1/3] update script to also test BeeDeeM-Tools

---
 singularity/test_container.sh | 39 ++++++++++++++++++++++-------------
 1 file changed, 25 insertions(+), 14 deletions(-)

diff --git a/singularity/test_container.sh b/singularity/test_container.sh
index e900c7b..67d4fa7 100755
--- a/singularity/test_container.sh
+++ b/singularity/test_container.sh
@@ -9,7 +9,7 @@
 #         b. qsub test_container.sh (PBS Pro)
 #         c. srun test_container.sh (slurm, direct execution)
 #
-# P. Durand (SeBiMER, Ifremer), last update on Nov 2022
+# P. Durand (SeBiMER, Ifremer), last updated on March 2023
 
 # Sample config for Slurm; adapt partition to your cluster configuration
 #SBATCH -p fast
@@ -25,7 +25,7 @@
 #PBS -l ncpus=2
 
 # Version of BeeDeeM to test
-BDM_VERSION=4.7.4
+BDM_VERSION=4.7.5
 # Default working directory to test BeeDeeM Singularity image.
 # Il will be overriden below, given host platform
 BDM_SCRATCH_DIR=/tmp
@@ -74,7 +74,7 @@ function downloadFile(){
 # Depending on host platform, load singularity env
 if hasCommand qstat; then
   hname=$(hostname)
-  if [[ $hname == *"datarmor"* ]]; then
+  if [[ $hname == *"data"* ]]; then
     echo "running on DATARMOR using PBS Pro scheduler"
     source /etc/profile.d/modules.sh
     module purge
@@ -85,6 +85,7 @@ elif hasCommand sbatch; then
   hname=$(hostname -A)
   if [[ $hname == *"roscoff"* ]]; then
     echo "running on ABiMS using SLURM scheduler"
+    BDM_PLATFORM="abims"
     BDM_SCRATCH_DIR=$HOME
   fi
 else
@@ -118,9 +119,7 @@ BDM_SINGULITY_IMG="$BDM_SING_IMG_HOME/$BDM_SING_IMG_NAME"
 # For debugging if neeeded: dump all BDM_XXX variables
 ( set -o posix ; set ) | grep "BDM_"
 
-#rm -rf $BDM_BANKS_DIR
 mkdir -p $BDM_BANKS_DIR
-#rm -rf $BDM_WORK_DIR
 mkdir -p $BDM_WORK_DIR
 
 # Prepare env variables to be used by BeeDeeM inside the container (mandatory)
@@ -130,18 +129,30 @@ export KL_mirror__path=${BDM_BANKS_DIR}
 
 # Set the banks to install
 # These are '.dsc' files located in BeeDeeM image at path /opt/beedeem/conf/descriptors
-DESCRIPTOR="SwissProt_human,PDB_proteins,MEROPS_PepUnits"
+DESCRIPTOR="SwissProt_human,PDB_proteins"
 
 # Now, let's start a simple installation
-echo "Start simple bank installation"
-echo "Look at processing with:"
-echo "tail -f $BDM_WORK_DIR/log"
-
-singularity run \
-  ${BDM_BINDS} \
-  ${BDM_SINGULITY_IMG} \
-  install.sh -desc ${DESCRIPTOR} >& ${BDM_WORK_DIR}/log 2>&1
+echo "1/2 - Start BeeDeeM test bank installation"
+# BeeDeeM test: bank installtion
+export BDM_CONF_SCRIPTS=/opt/beedeem/conf/scripts
+# Configure PBS wrapper to submit jobs
+export BDM_SCHEDULER="pbs"
+export BDM_PLATFORM="ifremer"
+export BDM_QSTAT_CMD="ssh galaxy@datarmor qstat"
+export BDM_QSUB_CMD="ssh galaxy@datarmor qsub"
+
+singularity run ${BDM_BINDS} ${BDM_SINGULITY_IMG} install.sh -desc ${DESCRIPTOR} 
+if [ $? -eq 0 ]; then
+  echo "SUCCESS"
+else
+  echo "FAILED.   Review log file: $BDM_WORK_DIR/log"
+  exit 1
+fi
 
+# BeeDeeM-Tools full test suite
+echo "2/2 - Start BeeDeeM-Tools test suite"
+mkdir -p $BDM_WORK_DIR/bdm-tools
+singularity run ${BDM_BINDS} ${BDM_SINGULITY_IMG} /opt/beedeem-tools/test.sh -w $BDM_WORK_DIR/bdm-tools
 if [ $? -eq 0 ]; then
   echo "SUCCESS"
 else

From 52c697220ba6489f429c58029d29badd15f8010c Mon Sep 17 00:00:00 2001
From: galaxy <patrick@plealog.com>
Date: Wed, 8 Mar 2023 13:53:05 +0000
Subject: [PATCH 2/3] fix release to use

---
 singularity/test_container.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/singularity/test_container.sh b/singularity/test_container.sh
index 2cf91b3..14354af 100755
--- a/singularity/test_container.sh
+++ b/singularity/test_container.sh
@@ -2,7 +2,7 @@
 
 # Test script for BeeDeeM singularity container
 # How to?
-# Step 1: build image with: singularity build -f beedeem-4.7.5.sif Singularity
+# Step 1: build image with: singularity build -f beedeem-4.7.6.sif Singularity
 #         (update version to match BDM_VERSION variable, below)
 # Step 2: test with either
 #         a. ./test_container.sh
@@ -25,7 +25,7 @@
 #PBS -l ncpus=2
 
 # Version of BeeDeeM to test
-BDM_VERSION=4.7.5
+BDM_VERSION=4.7.6
 # Default working directory to test BeeDeeM Singularity image.
 # Il will be overriden below, given host platform
 BDM_SCRATCH_DIR=/tmp

From b2f9f57f56c1708b072400f9c64f3dd02b28f6ac Mon Sep 17 00:00:00 2001
From: galaxy <patrick@plealog.com>
Date: Thu, 9 Mar 2023 08:13:06 +0000
Subject: [PATCH 3/3] enhance test suite to test all BeeDeeM and BeeDeeM-Tools
 commands

---
 singularity/test_container.sh | 103 ++++++++++++++++++++++++++++++----
 1 file changed, 91 insertions(+), 12 deletions(-)

diff --git a/singularity/test_container.sh b/singularity/test_container.sh
index 14354af..e887069 100755
--- a/singularity/test_container.sh
+++ b/singularity/test_container.sh
@@ -1,15 +1,20 @@
 #!/usr/bin/env bash
 
-# Test script for BeeDeeM singularity container
+# #############################################################################
+# Test script for BeeDeeM singularity container.
+#
 # How to?
+#
 # Step 1: build image with: singularity build -f beedeem-4.7.6.sif Singularity
 #         (update version to match BDM_VERSION variable, below)
+#
 # Step 2: test with either
 #         a. ./test_container.sh
 #         b. qsub test_container.sh (PBS Pro)
 #         c. srun test_container.sh (slurm, direct execution)
 #
 # P. Durand (SeBiMER, Ifremer), last updated on March 2023
+# #############################################################################
 
 # Sample config for Slurm; adapt partition to your cluster configuration
 #SBATCH -p fast
@@ -24,6 +29,11 @@
 #PBS -l mem=4g
 #PBS -l ncpus=2
 
+# #############################################################################
+
+# ###
+# Section 1: prepare BeeDeeM test suite
+
 # Version of BeeDeeM to test
 BDM_VERSION=4.7.6
 # Default working directory to test BeeDeeM Singularity image.
@@ -71,11 +81,15 @@ function downloadFile(){
   eval $CMD && return 0 || return 1
 }
 
+# ###
+# Section 2: prepare environement depending on host
+#   BeeDeeM devel team used to work on two clusters, one from 
+#   Ifremer (Brest, France), one from Station Biologique (Roscoff, France)
 # Depending on host platform, load singularity env
 if hasCommand qstat; then
   hname=$(hostname)
   if [[ $hname == *"data"* ]]; then
-    echo "running on DATARMOR using PBS Pro scheduler"
+    echo "running on Ifremer using PBS Pro scheduler"
     source /etc/profile.d/modules.sh
     module purge
     module load singularity/3.4.1
@@ -84,7 +98,7 @@ if hasCommand qstat; then
 elif hasCommand sbatch; then
   hname=$(hostname -A)
   if [[ $hname == *"roscoff"* ]]; then
-    echo "running on ABiMS using SLURM scheduler"
+    echo "running on SBR using SLURM scheduler"
     BDM_PLATFORM="abims"
     BDM_SCRATCH_DIR=$HOME
   fi
@@ -93,11 +107,15 @@ else
   echo "  Execute BeeDeeM directly on THIS computer"
 fi
 
+
 # Configure BeeDeeM banks and working directories
 BDM_SCRATCH_DIR="$BDM_SCRATCH_DIR/test_beedeem"
 BDM_BANKS_DIR="$BDM_SCRATCH_DIR/banks"
 BDM_WORK_DIR="$BDM_SCRATCH_DIR/working"
 
+# ###
+# Section 3: prepare SIngularity container
+
 # Check existence of the BeeDeeM image
 if [ ! -e "$BDM_SING_IMG_HOME/$BDM_SING_IMG_NAME" ]; then
   echo "WARN: $BDM_SING_IMG_HOME/$BDM_SING_IMG_NAME not found."
@@ -116,24 +134,38 @@ fi
 # Configure Singularity runner
 BDM_BINDS="--bind ${BDM_WORK_DIR} --bind ${BDM_BANKS_DIR}"
 BDM_SINGULITY_IMG="$BDM_SING_IMG_HOME/$BDM_SING_IMG_NAME"
+
+# ###
+# Section 4: prepare tests
 # For debugging if neeeded: dump all BDM_XXX variables
 ( set -o posix ; set ) | grep "BDM_"
-
 mkdir -p $BDM_BANKS_DIR
 mkdir -p $BDM_WORK_DIR
 
-# Prepare env variables to be used by BeeDeeM inside the container (mandatory)
+
+# ###
+# Section 5: prepare env variables to be used by BeeDeeM inside the container
+# These 3 KL_ variables are mandatory to use BeeDeeM
+
+# KL_JRE_ARGS sets Java Runtime variables (memory and tmpdir) and BeeDeeM log type (none|console|file)
 export KL_JRE_ARGS="-Xms128M -Xmx2048M -Djava.io.tmpdir=${BDM_WORK_DIR} -DKL_LOG_TYPE=console"
+# KL_WORKING_DIR sets the working directory of BeeDeeM (place to store tmp files at runtime)
 export KL_WORKING_DIR=${BDM_WORK_DIR}
+# KL_mirror__path sets the directory where BeeDeeM installs banks
 export KL_mirror__path=${BDM_BANKS_DIR}
 
-# Set the banks to install
-# These are '.dsc' files located in BeeDeeM image at path /opt/beedeem/conf/descriptors
-DESCRIPTOR="SwissProt_human,PDB_proteins"
+# ###
+# Section 6: start tests
 
 # Now, let's start a simple installation
-echo "1/2 - Start BeeDeeM test bank installation"
-singularity run ${BDM_BINDS} ${BDM_SINGULITY_IMG} install.sh -desc ${DESCRIPTOR} 
+echo "###############################################################################"
+echo "# Start BeeDeeM test bank installation"
+# These are '.dsc' files located in BeeDeeM image at path /opt/beedeem/conf/descriptors
+DESCRIPTOR="SwissProt_human,PDB_proteins" 
+#DESCRIPTOR="SwissProt_human" 
+CMD="singularity run ${BDM_BINDS} ${BDM_SINGULITY_IMG} install.sh -desc ${DESCRIPTOR}"
+echo $CMD
+eval $CMD
 if [ $? -eq 0 ]; then
   echo "SUCCESS"
 else
@@ -141,10 +173,57 @@ else
   exit 1
 fi
 
+# Plast vs SwissProt DB (contains annotations)
+echo "###############################################################################"
+echo "# Run annotated PLAST using SwissProt human previously installed"
+CMD="plast.sh -p plastp -i /opt/beedeem-tools/data/query.fa -d $BDM_BANKS_DIR/p/SwissProt_human/current/SwissProt_human/SwissProt_human00 -o $KL_WORKING_DIR/query_vs_SW.xml -a 4 -maxhits 10 -maxhsps 1 -e 1e-5 -F F"
+CMD="singularity run ${BDM_BINDS} ${BDM_SINGULITY_IMG} $CMD"
+echo $CMD
+eval $CMD
+if [ $? != 0  ]; then
+  echo "FAILED"
+  exit 1
+else
+  echo "SUCCESS"
+fi
+
+# Annotate PLAST result and prepare file for BlastViewer (zml format)
+#  see https://github.com/pgdurand/BlastViewer
+echo "###############################################################################"
+echo "# Annotate results"
+CMD="annotate.sh -i $KL_WORKING_DIR/query_vs_SW.xml -o $KL_WORKING_DIR/query_vs_SW.zml -type full -writer zml"
+CMD="singularity run ${BDM_BINDS} ${BDM_SINGULITY_IMG} $CMD"
+echo $CMD
+eval $CMD
+if [ $? != 0  ]; then
+  echo "FAILED"
+  exit 1
+else
+  echo "SUCCESS"
+fi
+
+# Dump previous annotated results as CSV file
+#  you may use dumpcsh.h to get help on CSV format
+echo "###############################################################################"
+echo "# Dump annotated PLAST results as CSV file"
+CMD="dumpcsv.sh -i $KL_WORKING_DIR/query_vs_SW.zml -f zml -o $KL_WORKING_DIR/query_vs_SW.csv"
+CMD="singularity run ${BDM_BINDS} ${BDM_SINGULITY_IMG} $CMD"
+echo $CMD
+eval $CMD
+if [ $? != 0  ]; then
+  echo "FAILED"
+  exit 1
+else
+  echo "SUCCESS"
+fi
+
 # BeeDeeM-Tools full test suite
-echo "2/2 - Start BeeDeeM-Tools test suite"
+echo "###############################################################################"
+echo "# Start BeeDeeM-Tools test suite"
 mkdir -p $BDM_WORK_DIR/bdm-tools
-singularity run ${BDM_BINDS} ${BDM_SINGULITY_IMG} /opt/beedeem-tools/test.sh -w $BDM_WORK_DIR/bdm-tools
+CMD="singularity run ${BDM_BINDS} ${BDM_SINGULITY_IMG} /opt/beedeem-tools/test.sh -w $BDM_WORK_DIR/bdm-tools"
+echo $CMD
+eval $CMD
 if [ $? -eq 0 ]; then
   echo "SUCCESS"
 else