copy input files to each node's TMPDIR, for multi-node PBS runs

chrisjsewell · Oct 18, 2017 · 6953603 · 6953603
1 parent 3463787
commit 6953603
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 5 deletions.
diff --git a/atomic_hpc/__init__.py b/atomic_hpc/__init__.py
@@ -1,4 +1,4 @@
 from atomic_hpc import config_yaml, deploy_runs, mockssh, context_folder, utils
 
-__version__ = "0.1.8"
+__version__ = "0.2.0"
 
diff --git a/atomic_hpc/deploy_runs.py b/atomic_hpc/deploy_runs.py
@@ -409,15 +409,34 @@ def deploy_run_normal(run, inputs, root_path, if_exists="abort", exec_errors=Fal
         echo "the TMPDIR variable does not exist"  1>&2
         exit 1
     fi
-    if [ -z "TMPDIR" ]; then
+    if [ -z "$TMPDIR" ]; then
         echo "the TMPDIR variable is empty"  1>&2
         exit 1
     fi
     echo "running in: $TMPDIR"
     cd $TMPDIR
     
     # copy required input files from $WORKDIR to $TMPDIR
-    cp -pR {wrkpath}/* $TMPDIR
+    # if running on multiple nodes, then the files need to be copied to each one
+    if [ ! -z ${{PBS_NODEFILE+x}} ]; then
+        echo '$PBS_NODEFILE' found: $PBS_NODEFILE
+
+        readarray -t PCLIST < $PBS_NODEFILE
+        # get unique items
+        IFS=$' '
+        PCLIST=($(printf "%s\n" "${{PCLIST[@]}}" | sort -u | tr '\n' ' '))
+        unset IFS
+        # echo "running on nodes: ${{PCLIST[*]}}"
+
+        for PC in "${{PCLIST[@]}}"; do
+            echo "copying input files to node $PC"
+            ssh $PC "if [ ! -d $TMPDIR ];then mkdir -p $TMPDIR;echo 'temporary directory on '$PC;fi"
+            ssh $PC cp -pR {wrkpath}/* $TMPDIR
+            # echo `ssh $PC ls $TMPDIR`
+        done
+    else
+        cp -pR {wrkpath}/* $TMPDIR
+    fi
 
 else
 

diff --git a/atomic_hpc/test_deploy_runs.py b/atomic_hpc/test_deploy_runs.py
@@ -268,15 +268,34 @@ def test_create_qsub(context):
         echo "the TMPDIR variable does not exist"  1>&2
         exit 1
     fi
-    if [ -z "TMPDIR" ]; then
+    if [ -z "$TMPDIR" ]; then
         echo "the TMPDIR variable is empty"  1>&2
         exit 1
     fi
     echo "running in: $TMPDIR"
     cd $TMPDIR
     
     # copy required input files from $WORKDIR to $TMPDIR
-    cp -pR path/to/dir/* $TMPDIR
+    # if running on multiple nodes, then the files need to be copied to each one
+    if [ ! -z ${PBS_NODEFILE+x} ]; then
+        echo '$PBS_NODEFILE' found: $PBS_NODEFILE
+
+        readarray -t PCLIST < $PBS_NODEFILE
+        # get unique items
+        IFS=$' '
+        PCLIST=($(printf "%s\n" "${PCLIST[@]}" | sort -u | tr '\n' ' '))
+        unset IFS
+        # echo "running on nodes: ${PCLIST[*]}"
+
+        for PC in "${PCLIST[@]}"; do
+            echo "copying input files to node $PC"
+            ssh $PC "if [ ! -d $TMPDIR ];then mkdir -p $TMPDIR;echo 'temporary directory on '$PC;fi"
+            ssh $PC cp -pR path/to/dir/* $TMPDIR
+            # echo `ssh $PC ls $TMPDIR`
+        done
+    else
+        cp -pR path/to/dir/* $TMPDIR
+    fi
 
 else