Merge pull request #16 from pgdurand/prepV5

release BeeDeeM 5.0.0

conda/build.sh

@@ -24,6 +24,7 @@ chmod +x $PREFIX/bin/bdm
 chmod +x $PREFIX/external/bin/linux/*
 chmod +x $PREFIX/external/bin/macos/*
 chmod +x $PREFIX/conf/scripts/*.sh
+chmod +x $PREFIX/conf/scripts/scheduler/*.sh
 
 # Discard Windows stuff
 cd $PREFIX/external/bin && rm -rf windows

conda/meta.yaml

@@ -1,14 +1,13 @@
 {% set name = "beedeem" %}
-{% set version = "4.7.5" %}
-{% set sha256 = "94299b66f232288e69050cf7e475ea3acaecf07fa0b3eb36f073a3b86509183d" %}
+{% set version = "5.0.0" %}
+{% set sha256 = "395b8e56919e2f20715bb636d3ccaa1a4c98302547836c768b3189f131902b58" %}
 
 package:
   name: {{ name }}
   version: {{ version }}
 
 source:
-#   path: ../distrib/beedeem-{{ version }}.tar.gz
-  url: https://github.com/pgdurand/BeeDeeM/releases/download/v{{ version }}/beedeem-{{ version }}-distrib.zip
+  path: ../distrib/beedeem-{{ version }}.tar.gz
   sha256: '{{ sha256 }}'
 
 build:

conda/run_test.sh

@@ -10,15 +10,55 @@ TMP_DIR=$(mktemp bdm.XXXXXXXXXX)
 # to override default BeeDeeM configuration
 export KL_WORKING_DIR=$SCRATCH/$TMP_DIR
 export KL_mirror__path=$KL_WORKING_DIR
+export KL_LOG_TYPE=console
 
 echo "BeeDeeM Conda test within: $KL_WORKING_DIR"
 mkdir -p $KL_WORKING_DIR
+
+# == TEST 1 =========================================== 
+echo "*** TEST 1: Start simple bank installation"
 # These are two ".dsc" files located in BeeDeem-home/conf/descriptors
 # path
 DESC_LIST="PDB_proteins,SwissProt_human"
 # Start installation
 bdm install -desc $DESC_LIST
 
+if [ $? -eq 0 ]; then
+  echo "TEST 1: SUCCESS"
+else
+  echo "TEST 1: FAILED."
+  exit 1
+fi
+
+# == TEST 2 =========================================== 
+echo "*** TEST 2: list installed bank"
+export KL_LOG_TYPE=none
+bdm info -d all -f txt
+
+if [ $? -eq 0 ]; then
+  echo "TEST 2: SUCCESS"
+else
+  echo "TEST 2: FAILED. Review log file in: $BDM_WORK_DIR"
+  exit 1
+fi
+
+
+# == TEST 3 =========================================== 
+# Change default BeeDeeM log file name to something else
+export KL_LOG_TYPE=file
+export KL_LOG_FILE=query.log
+
+SW_ENTRY="ZZZ3_HUMAN"
+echo "*** TEST 3: query bank for entry: $SW_ENTRY"
+bdm query -d p -f txt -i $SW_ENTRY
+
+if [ $? -eq 0 ]; then
+  echo "TEST 3: SUCCESS"
+else
+  echo "TEST 3: FAILED. Review log file: $BDM_WORK_DIR"
+  exit 1
+fi
+
 # Note: by design of this script, we DO NOT delete
 # $KL_WORKING_DIR... please, do it yourself.
 

conf/dbms.config

@@ -5,25 +5,40 @@
 # This file is loaded during BeeDeeM startup to define some
 # usefull resources.
 #
-# It is possible to override this default behaviour using
-# environment variables, as follows. To redefine one of the
+# It is possible to override this default behavior using either
+# -D JRE cmdline arguments or environment variables, as follows.
+# 
+# Using -D JRE args: prefix config variable name with KL_; 
+#  example:
+#   -DKL_copy.workers=4
+#   will override value defined in this configuration file by
+#   copy.workers (see below, line 57).
+#
+# Using shell variable. To redefine one of the
 # below listed variables (e.g. mirror.path), set a shell
 # variable called KL_<variable_name> where you replace dot
-# character by a double underscore (e.g. KL_mirror__path).
+# character by a double underscore (e.g. KL_mirror__path); as a
+# reminder, dot char is not allowed in shell variable names.
 #
 # Example: to force BeeDeeM using a mirror.path different than
 # the one set below, simply do this (bash shell) BEFORE starting
 # BeeDeeM:
-#  export KL_mirror__path=/a/new/path
+#    export KL_mirror__path=/a/new/path
+#    java .../...
+#  or directly:
+#    java .../... -DKL_mirror.path=/another/path
+#
+# Declaration priority is as follows:
+#  env var > -D > config file
 #
 # ==================================================================
 
 # Path where to install/manage local copies of databanks
-mirror.path = /tmp/biobanks
+mirror.path = ${HOME}/beedeem-banks
 
 # Path where to prepare local installation of databanks
 # before copying them in mirror.path during install in production task
-#mirrorprepa.path = /tmp/biobanks/tmp
+#mirrorprepa.path = ${HOME}/beedeem-banks/tmp
 
 # File storing the list of available databanks
 mirror.file = dbmirror.config
@@ -66,8 +81,7 @@ lucene.fs = default
 lucene.lock = default
 
 # Following keys are for Aspera configuration
-# Windows: do not add .exe extension to ascp binary
 # All OS: DO NOT use space in directory names !!!
-aspera.bin.path=/Users/pgdurand/Applications/Aspera-CLI/bin/ascp
-aspera.key.path=/Users/pgdurand/Applications/Aspera-CLI/etc/asperaweb_id_dsa.openssh
+aspera.bin.path=${HOME}/Applications/Aspera-CLI/bin/ascp
+aspera.key.path=${HOME}/Applications/Aspera-CLI/etc/asperaweb_id_dsa.openssh
 

conf/descriptors/PDB_proteins_task.dsc

@@ -1,9 +1,9 @@
-#PDB_proteins
-#Fri Sep 15 13:47:28 CEST 2017
-
 # Illustrates the use of various scripts (external tasks)
 # DO NOT use it for production!
 
+# Descriptor documentation, see:
+#  https://pgdurand.gitbook.io/beedeem/getting-started/descriptors-format
+
 db.name=PDB_proteins_task
 db.desc=PDB Protein databank; illustrate use of external script call
 db.type=p
@@ -12,7 +12,11 @@ db.ldir=${mirrordir}|p|PDB_proteins_task
 db.files.include=pdbaa.tar.gz
 db.files.exclude=
 
-tasks.global.pre=script(name=WaitALittle;path=wait_a_little),script(name=HelloWorld;path=hello_world)
+# This a pre-processing script; called once at the very beginning of bank precessing
+#   Illustrate a script call with additional arguments; setting an argument with '=NA" defines a no-arg parameter
+# Pre-processing script is optional.
+tasks.global.pre=script(name=WaitALittle;path=wait_a_little),script(name=HelloWorld;path=hello_world;-parse_seqid=NA;-k=19;-w=15;--verbose-mode=debug)
+
 tasks.global.post=script(name=WaitALittle;path=wait_a_little),makealias,delgz,deltar,script(name=HelloWorld;path=hello_world)
 
 tasks.unit.post=script(name=WaitALittle;path=wait_a_little),gunzip,untar,script(name=HelloWorld;path=hello_world)

conf/scripts/hello_world.sh

@@ -1,55 +1,43 @@
 #!/usr/bin/env bash
 
-# This is a BeeDeeM external task script template.
+# This is a BeeDeeM external task script example.
 #
 # This script illustrates the use of external tasks.
 # Such a task is called from a bank descriptor; e.g. see
 # for instance ../descriptors/PDB_proteins_task.dsc
 #
-# Such a BeeDeeM script is called by the task engine and
-# with these arguments: -w <path> -d <path> -f <path> -n <name> -t <type>
-#
-#  -w <path>: <path> is the working directory path.
-#             provided for both unit and global tasks.
-#  -d <path>: <path> is the bank installation path.
-#             provided for both unit and global tasks.
-#  -f <path>: <path> is the path to file under unit task processing
-#             only provided with unit task.
-#  -n <name>: <name> is the bank name.
-#  -t <type>: <path> is the bank type. One of p, n or d.
-#             p: protein
-#             n: nucleotide
-#             d: dictionary or ontology
+# Such a BeeDeeM script is called by the task engine with some
+# BeeDeeM specific arguments, see 
+#    conf/scripts/scheduler/common.sh#handleBDMArgs()
+# for more information.
 
-echo "Executing an external script"
-echo "Arguments coming from BeeDeeM are:"
-echo $@
+# If you setup a new script, simply copy this one, keep in it 
+# lines 1 and 20-31 (sections "include API" and "handle arguments"), 
+# then do whatever you have to do!
 
-echo "----"
-# Prepare arguments for processing
-WK_DIR=
-INST_DIR=
-PROCESSED_FILE=
-BANK_NAME=
-BANK_TYPE=
-while getopts w:d:f:n:t: opt
-do
-    case "$opt" in
-      w)  WK_DIR="$OPTARG";;
-      d)  INST_DIR="$OPTARG";;
-      f)  PROCESSED_FILE="$OPTARG";;
-      n)  BANK_NAME="$OPTARG";;
-      t)  BANK_TYPE="$OPTARG";;
-    esac
-done
-shift `expr $OPTIND - 1`
-# remaining arguments, if any, are stored here
-MORE_ARGS=$@
+set -eo pipefail
+
+# ========================================================================================
+# Section: include API
+S_NAME=$(realpath "$0")
+[[ -z "$BDM_CONF_SCRIPTS" ]] && script_dir=$(dirname "$S_NAME") || script_dir=$BDM_CONF_SCRIPTS
+. $script_dir/scheduler/common.sh
+
+# ========================================================================================
+# Section: handle arguments
+# Function call setting BDMC_xxx variables from cmdline arguments
+handleBDMArgs $@
+RET_CODE=$?
+[ ! $RET_CODE -eq 0 ] && errorMsg "Wrong or missing arguments" && exit $RET_CODE
+
+# ========================================================================================
+# Section: do business
 
-echo "Working dir: $WK_DIR"
-echo "Install dir: $INST_DIR"
-echo "Processed file: $PROCESSED_FILE"
-echo "Bank name: $BANK_NAME"
-echo "Bank type: $BANK_TYPE"
+echo "Working directory of BeeDeeM: $BDMC_WK_DIR"
+echo "Bank installation path: $BDMC_INST_DIR"
+echo "Current bank file processed: $BDMC_PROCESSED_FILE"
+echo "Bank name: $BDMC_BANK_NAME"
+echo "Bank type: $BDMC_BANK_TYPE"
+echo "Additional args: $BDMC_MORE_ARGS"
 echo "----"
 

conf/scripts/scheduler/common.sh

@@ -102,7 +102,7 @@ BDMC_PLATFORM=$BDM_PLATFORM
 #
 #           Such a BeeDeeM script is always  called 
 #           with these arguments: 
-#           -w <path> -d <path> -f <path> -n <name> -t <type>
+#           -w <path> -d <path> -f <path> -n <name> -t <type> -o <args>
 #
 #  -w <path>: <path> is the working directory path.
 #             provided for both unit and global tasks.
@@ -116,10 +116,13 @@ BDMC_PLATFORM=$BDM_PLATFORM
 #             n: nucleotide
 #             d: dictionary or ontology
 #  -p <name>: platform name (i.e. a specific cluster configuration)
+#  -o <args>: optional. Only set if script directive passes in some 
+#             additional arguments. 
+#             See conf/descriptors/PDB_proteins_task.dsc for an example.
 function handleBDMArgs(){
   infoMsg "Arguments coming from BeeDeeM are: [$@]"
   local OPTIND
-  while getopts w:d:f:n:t:p: opt
+  while getopts w:d:f:n:t:p:o: opt
   do
     case "$opt" in
       w)  BDMC_WK_DIR="$OPTARG";;
@@ -128,18 +131,22 @@ function handleBDMArgs(){
       n)  BDMC_BANK_NAME="$OPTARG";;
       t)  BDMC_BANK_TYPE="$OPTARG";;
       p)  BDMC_PLATFORM="$OPTARG";;
+      o)  BDMC_MORE_ARGS="$OPTARG";;
     esac
   done
   shift `expr $OPTIND - 1`
-  BDMC_MORE_ARGS=$@
+
+  # Optional arguments consists in a special encoded string made by
+  # bzh.plealog.dbmirror.task.PTaskExecScript code
+  BDMC_MORE_ARGS=$(echo $BDMC_MORE_ARGS | sed -e 's/;/ /g')
 
-  infoMsg "Working dir: $BDMC_WK_DIR"
-  infoMsg "Install dir: $BDMC_INST_DIR"
-  infoMsg "Processed file: $BDMC_PROCESSED_FILE"
+  infoMsg "Working directory of BeeDeeM: $BDMC_WK_DIR"
+  infoMsg "Bank installa path: $BDMC_INST_DIR"
+  infoMsg "Current processed bank file: $BDMC_PROCESSED_FILE"
   infoMsg "Bank name: $BDMC_BANK_NAME"
   infoMsg "Bank type: $BDMC_BANK_TYPE"
   infoMsg "Platform: $BDMC_PLATFORM"
-  infoMsg "Remaining args: $BDMC_MORE_ARGS"
+  infoMsg "Remaining task script arguments: $BDMC_MORE_ARGS"
 }
 
 # --------
@@ -193,6 +200,10 @@ function getResources(){
 # FUNCTION: figure out which Job Scheduler is available on host system
 # return: 0 if job scheduler found, 1 otherwise. Job scheduler name is echoed.
 function getScheduler(){
+  if [ ! -z "$BDM_SCHEDULER" ]; then
+    echo $BDM_SCHEDULER
+    return 0
+  fi
   local ret_value=
   if hasCommand qstat; then
     ret_value=$(qstat --version)
@@ -206,3 +217,4 @@ function getScheduler(){
   return 1
 }
 
+handleBDMArgs $@

conf/system/dbmsVersion-txt.vm

@@ -17,8 +17,15 @@ Installed banks
 #foreach( $db in $pTable )
       * $db.getName()
         Description: $db.getDescription()
+        Home dir: $db.getDbHome()
         BLAST+ use: -db $db.getDbPath().substring(0, $db.getDbPath().lastIndexOf("."))
         Annotated bank: $db.hasAnnotation()
+#if( ! ${db.getAdditionalIndex().isEmpty()} )
+        Additional index: 
+#foreach( $key in  $db.getAdditionalIndex().keySet() )
+           $key : $db.getAdditionalIndex().get($key).toString()
+#end              
+#end              
         Size (sequences): $db.getNbSequence()
         Size on disk: $db.getDiskSize()
         Release date: $db.getReleaseTimeStamp()
@@ -32,8 +39,15 @@ Installed banks
 #foreach( $db in $nTable )
       * $db.getName()
         Description: $db.getDescription()
+        Home dir: $db.getDbHome()
         BLAST+ use: -db $db.getDbPath().substring(0, $db.getDbPath().lastIndexOf("."))
         Annotated bank: $db.hasAnnotation()
+#if( ! ${db.getAdditionalIndex().isEmpty()} )
+        Additional index: 
+#foreach( $key in  $db.getAdditionalIndex().keySet() )
+           $key : $db.getAdditionalIndex().get($key).toString()
+#end              
+#end              
         Size (sequences): $db.getNbSequence()
         Size on disk: $db.getDiskSize()
         Release date: $db.getReleaseTimeStamp()
@@ -46,6 +60,7 @@ Installed banks
 #if( ${dTable.size()} != 0 )
 #foreach( $db in $dTable )
       * $db.getName()
+        Home dir: $db.getDbHome()
         Description: $db.getDescription()
         Size (terms): $db.getNbSequence()
         Size on disk: $db.getDiskSize()