P3 proceesing to final cat

example/cfis/config_MsPl_psfex.ini

@@ -35,7 +35,7 @@ LOG_NAME = log_sp
 RUN_LOG_NAME = log_run_sp
 
 # Input directory, containing input files, single string or list of names
-INPUT_DIR = $SP_RUN/psf_validation_ind
+INPUT_DIR = $SP_RUN/output
 
 # Output directory
 OUTPUT_DIR = $SP_RUN/output
@@ -54,7 +54,7 @@ TIMEOUT = 96:00:00
 ## Module options
 [MERGE_STARCAT_RUNNER]
 
-INPUT_DIR = psf_validation_ind
+INPUT_DIR = last:psfex_interp_runner
 
 PSF_MODEL = psfex
 

example/cfis/config_Ms_psfex.ini

@@ -0,0 +1,68 @@
+# ShapePipe configuration file for post-processing.
+# merge star cat.
+
+
+## Default ShapePipe options
+[DEFAULT]
+
+# verbose mode (optional), default: True, print messages on terminal
+VERBOSE = True
+
+# Name of run (optional) default: shapepipe_run
+RUN_NAME = run_sp_Ms
+
+# Add date and time to RUN_NAME, optional, default: False
+RUN_DATETIME = False
+
+
+## ShapePipe execution options
+[EXECUTION]
+
+# Module name, single string or comma-separated list of valid module runner names
+MODULE = merge_starcat_runner
+
+# Parallel processing mode, SMP or MPI
+MODE = SMP
+
+
+## ShapePipe file handling options
+[FILE]
+
+# Log file master name, optional, default: shapepipe
+LOG_NAME = log_sp
+
+# Runner log file name, optional, default: shapepipe_runs
+RUN_LOG_NAME = log_run_sp
+
+# Input directory, containing input files, single string or list of names
+INPUT_DIR = $SP_RUN/output
+
+# Output directory
+OUTPUT_DIR = $SP_RUN/output
+
+
+## ShapePipe job handling options
+[JOB]
+
+# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial
+SMP_BATCH_SIZE = 4
+
+# Timeout value (optional), default is None, i.e. no timeout limit applied
+TIMEOUT = 96:00:00
+
+
+## Module options
+[MERGE_STARCAT_RUNNER]
+
+INPUT_DIR = last:psfex_interp_runner
+
+PSF_MODEL = psfex
+
+NUMBERING_SCHEME = -0000000-0
+
+# Input file pattern(s), list of strings with length matching number of expected input file types
+# Cannot contain wild cards
+FILE_PATTERN = validation_psf
+
+# FILE_EXT (optional) list of string extensions to identify input files
+FILE_EXT = .fits

scripts/sh/combine_runs.bash

@@ -98,7 +98,11 @@ run_out="run_sp_combined_$cat"
 
 if [ "$cat" == "final" ]; then
 
-  run_in="$pwd/$out_base/run_sp_Mc_*"
+  # v1
+  #run_in="$pwd/$out_base/run_sp_Mc_*"
+  # v2
+  run_in="$pwd/tile_runs/*/$out_base/run_sp_Mc_*"
+
   module="make_catalog_runner"
   pattern="final_cat-*"
 
@@ -120,7 +124,7 @@ elif [ "$cat" == "psf" ]; then
   # v1
   #run_in="$pwf/$out_base/run_sp_exp_Pi_*"
   # v2
-  run_in="$pwd/exp_runs/*/$out_base/"
+  run_in="$pwd/exp_runs/*/$out_base/run_sp_exp_Pi_*"
 
   pattern="validation_psf-*"
   if [ "$psf" == "psfex" ]; then

scripts/sh/job_sp_canfar.bash

@@ -161,10 +161,10 @@ export SP_RUN=`pwd`
 export SP_CONFIG=$SP_RUN/cfis
 export SP_CONFIG_MOD=$SP_RUN/cfis_mod
 
-## Other variables
+# To find ssl certificate
+export VM_HOME=$HOME
 
-# Input tile numbers ASCII file
-export TILE_NUMBERS_PATH=tile_numbers.txt
+## Other variables
 
 # Output
 OUTPUT=$SP_RUN/output
@@ -326,11 +326,6 @@ mkdir -p $SP_CONFIG_MOD
 
 # Processing
 
-## Check for input tile list
-#if [ ! -e $TILE_NUMBERS_PATH ]; then
-  #echo "Tile numbers file $TILE_NUMBERS_PATH not found, exiting"
-#fi
-
 ### Retrieve config files
 if [[ $config_dir == *"vos:"* ]]; then
   command_sp "$VCP $config_dir ." "Retrieve shapepipe config files"

shapepipe/utilities/summary.py

@@ -502,7 +502,6 @@ def get_names_in_dir(self, iterable, module, idx):
                 f"{self._path_left}/{subdir}/"
                 + f"{self._path_right[idx]}"
             )
-            print("MKDEBUG ", base_and_subdir)
             if os.path.isdir(base_and_subdir):
 
                 matches = []
@@ -511,10 +510,8 @@ def get_names_in_dir(self, iterable, module, idx):
                 with os.scandir(base_and_subdir) as entries:
                     for entry in entries:
 
-                        print("MKDEBUG ", entry, self._run_dir[idx])
                         # Append directory name if matches module
                         if entry.name.startswith(self._run_dir[idx]):
-                            print("MKDEBUG matches")
                             matches.append(entry)
 
                     # This entry does not match module -> next
@@ -529,7 +526,6 @@ def get_names_in_dir(self, iterable, module, idx):
                     directory = self.get_module_output_dir(
                         full_path, module
                     )
-                    print("MKDEBUG ", directory)
 
                     # Find matching file names and paths
                     self.get_matches_final(directory, idx)