diff --git a/src/ingest-pipeline/airflow/dags/cwl/sc-atac-seq-pipeline b/src/ingest-pipeline/airflow/dags/cwl/sc-atac-seq-pipeline index 812f477ca..eabf4433c 160000 --- a/src/ingest-pipeline/airflow/dags/cwl/sc-atac-seq-pipeline +++ b/src/ingest-pipeline/airflow/dags/cwl/sc-atac-seq-pipeline @@ -1 +1 @@ -Subproject commit 812f477ca73bbedd4d95295c41b7902926d1b324 +Subproject commit eabf4433c20334dfa9123da0e2900dacf680edb3 diff --git a/src/ingest-pipeline/misc/tools/auto_deploy.sh b/src/ingest-pipeline/misc/tools/auto_deploy.sh index 503a7b7ea..a25f486fc 100755 --- a/src/ingest-pipeline/misc/tools/auto_deploy.sh +++ b/src/ingest-pipeline/misc/tools/auto_deploy.sh @@ -2,15 +2,31 @@ # What variables do we need? # Define HIVE machines -#hive_machines=("l001.hive.psc.edu" "gpu002.pvt.hive.psc.edu") +# hive_machines=("l001.hive.psc.edu" "gpu002.pvt.hive.psc.edu") hive_machines=() -#b2_machines=("v004.pvt.bridges2.psc.edu") +# b2_machines=("v004.pvt.bridges2.psc.edu") b2_machines=() -repo_env="dev" -repo_dir="/opt/repositories/$(hostname -s)-$repo_env/ingest-pipeline" + +priority_list=("-prod" "-test" "-dev") +base_name="airflow" +file_dir="/etc/sysconfig" regenerate_env=false +function set_main_environment(){ + for ending in "${priority_list[@]}"; do + file="$file_dir/$base_name$ending" + if [ -f "$file" ]; then + echo "Selected file: $file" + source $file + repo_env="$HUBMAP_INSTANCE" + python_version="$HUBMAP_PYTHON_VERSION" + repo_dir="/opt/repositories/$(hostname -s)-$repo_env/ingest-pipeline" + break + fi +done +} + while getopts ":g" opt; do case $opt in g) @@ -23,23 +39,30 @@ while getopts ":g" opt; do done set -x -for machine in ${hive_machines[@]}; do + +set_main_environment + +if [ -z "$repo_env" ]; then + exit "The environment variable HUBMAP_INSTANCE is not set." +fi + +for machine in "${codcc_machines[@]}"; do # Rsync repo to machine rsync -a --exclude "src/ingest-pipeline/airflow/logs" $repo_dir/ $machine:$repo_dir # If flag set, run the conda environment regenerations if $regenerate_env ; then - ssh $machine "/usr/local/bin/update_hubmap.sh $repo_dir $repo_env" + ssh $machine "/usr/local/bin/update_hubmap.sh $repo_dir $repo_env $python_version" fi done # Separate because its easier to loop twice over a small list than insert string checking and manipulation -for machine in ${b2_machines[@]}; do +for machine in "${b2_machines[@]}"; do # Rsync repo to machine rsync -a --exclude "src/ingest-pipeline/airflow/logs" 'ssh -J bridges2.psc.edu' $repo_dir/ $machine:$repo_dir # If flag set, run the conda environment regenerations if $regenerate_env ; then - ssh -J "bridges2.psc.edu" $machine "/usr/local/bin/update_hubmap.sh $repo_dir $repo_env" + ssh -J "bridges2.psc.edu" $machine "/usr/local/bin/update_hubmap.sh $repo_dir $repo_env $python_version" fi done diff --git a/src/ingest-pipeline/submodules/ingest-validation-tools b/src/ingest-pipeline/submodules/ingest-validation-tools index 8c716ba85..77e13a2c1 160000 --- a/src/ingest-pipeline/submodules/ingest-validation-tools +++ b/src/ingest-pipeline/submodules/ingest-validation-tools @@ -1 +1 @@ -Subproject commit 8c716ba8582f72555e142b5cbe6fcbb2ca39b7f4 +Subproject commit 77e13a2c18f3533156972d34d51a44cada2cd210