651 conda activate fix mk

auxdir/CFIS/tiles_202106/tiles_P7.txt

@@ -1,3 +1,4 @@
+000.000
 052.332
 053.331
 053.332

docs/source/pipeline_canfar.md

@@ -0,0 +1,95 @@
+patch="P7"
+psf="psfex"
+N_SMP=16
+
+# Terminal title
+echo -ne "\033]0;$patch\007"
+
+# Run directory
+dir=~/cosmostat/v2/pre_v2/$psf/$patch
+cd $dir
+
+# Get tile number list
+ln -s ~/shapepipe/auxdir/CFIS/tiles_202106/tiles_$patch.txt tile_numbers.txt
+
+
+# Get images
+
+## Download and link separately
+
+### Download
+### Create and link to central image storage directory
+mkdir -p ~/cosmostat/v2/data_tiles/$patch
+ln -s ~/cosmostat/v2/data_tiles/$patch data_tiles
+
+### Download and move tiles 
+ln -s ~/shapepipe/example/cfis
+mkdir -p output
+export SP_RUN=`pwd`
+
+shapepipe_run -c cfis/config_Git_vos.ini
+mv -i output/run_sp_Git_*/get_images_runner/output/CFIS.???.???.*fits* data_tiles
+rm -rf output/run_sp_tiles_Git_*
+update_run_log_file.py
+# repeat the above block
+
+### Find exposures; this run can be stopped after Fe
+shapepipe_run -c cfis/config_GitFe_symlink.ini
+
+### Download and move exposures
+
+shapepipe_run -c cfis/config_Gie_vos.ini
+mv -i output/run_sp_Gie_*/get_images_runner/output/*.fits*fz data_exp
+rm -rf  output/run_sp_Gie_*
+update_run_log_file.py
+# repeat the above
+
+### Create links (and re-run Fe, not necessary)
+job_sp_canfar.bash -p $psf `cat tile_numbers.txt` -j 1 -r symlink
+
+# Uncompress weights,  split exposures into single HDUs
+job_sp_canfar.bash -p $psf -n $N_SMP -j 2
+
+# Mask tiles
+job_sp_canfar.bash -p $psf -n $N_SMP -j 4
+
+# If not finshed:
+combine_runs.bash -p psfex -c flag
+mv output/run_sp_combined_flag output/run_sp_exp_Ma
+
+# Mask exposures
+job_sp_canfar.bash -p $psf -n $N_SMP -j 8
+
+
+# Tile detection
+curl_canfar_local.sh -j 16 -f tile_numbers.txt -k tile -p $psf -N $N_SMP
+
+
+# Exposure detection
+## Get single-HDU single-exposure IDs
+~/shapepipe/scripts/python/summary_run.py
+
+cp summary/missing_job_32_sextractor.txt all.txt
+curl_canfar_local.sh -j 32 -f all.txt -k exp -p $psf -N $N_SMP
+
+# Tile preparation
+curl_canfar_local.sh -j 64 -f tile_numbers.txt -k tile -p $psf -N $N_SMP
+
+# Tile shape measurement
+curl_canfar_local.sh -j 128 -f tile_numbers.txt -k tile -p $psf -N 8
+
+# Merge subcatalogues, and create final cat
+job_sp_canfar.bash -p $psf -n 1 -j 256
+
+# Combine all final cats in common output dir as links
+combine_runs.bash -c final -p psfex
+
+# Merge all final cats
+# (use 192GB RAM)
+merge_final_cat -i output/run_sp_combined_final/make_catalog_runner/output -p cfis/final_cat.param -v
+
+
+# Delete jobs
+SSL=~/.ssl/cadcproxy.pem
+SESSION=https://ws-uv.canfar.net/skaha/v0/session
+for ID in `cat session_IDs.txt`; do echo $ID; curl -X DELETE -E $SSL $SESSION/$ID; done

example/cfis/config_merge_sep_cats_template.ini

@@ -35,7 +35,6 @@ RUN_LOG_NAME = log_run_sp
 
 # Input directory, containing input files, single string or list of names with length matching FILE_PATTERN
 INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output
-#, ./output/run_sp_tile_ngmix_Ng1u/galsim_shapes_v2_runner/output
 
 # Output directory
 OUTPUT_DIR = ./output
@@ -58,11 +57,9 @@ TIMEOUT = 96:00:00
 # Input file pattern(s), list of strings with length matching number of expected input file types
 # Cannot contain wild cards
 FILE_PATTERN = ngmix
-#, galsim
 
 # FILE_EXT (optional) list of string extensions to identify input files
 FILE_EXT = .fits
-#, .fits
 
 # Numbering convention, string that exemplifies a numbering pattern.
 NUMBERING_SCHEME = -000-000

example/cfis/config_tile_Sx.ini

@@ -55,7 +55,7 @@ TIMEOUT = 96:00:00
 
 [SEXTRACTOR_RUNNER]
 
-INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, run_sp_tile_Ma:mask_runner
+INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, run_sp_Ma_tile:mask_runner
 
 FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag
 

scripts/jupyter/summary_run.ipynb

@@ -143,8 +143,8 @@
      "output_type": "stream",
      "text": [
       " (Job 1)\n",
-      "get_images_runner_run_1              464       462         0         2       1.0     99.6%\n",
-      "find_exposures_runner                232       231         0         1       1.0     99.6%\n",
+      "get_images_runner_run_1              462       462         0         0       0.0    100.0%\n",
+      "find_exposures_runner                231       231         0         0       0.0    100.0%\n",
       "get_images_runner_run_2              537         0         0       537     179.0      0.0%\n"
      ]
     }

scripts/python/stats_headless_canfar.py

@@ -2,11 +2,20 @@
 
 # Name: stats_headless_canfar.py
 
+# Caution: Does not show all running or pending
+# headless jobs, for some reason.
+
 import sys
 from skaha.session import Session
 
 
 def main(argv=None):
+
+    print(
+        "# Depreciated, does not show pending jobs; use stats_jobs_canfar.sh",
+        file=sys.stderr,
+    )
+
     session = Session()
 
     n_headless = session.stats()["instances"]["headless"]

scripts/python/summary_params_pre_v2.py

@@ -74,7 +74,7 @@ def set_jobs_v2_pre_v2(patch, verbose):
     # - remove previous output dirs since only last is searched
     jobs["1"] = job_data(
         1,
-         "run_sp_Git",
+         "run_sp_GitFeGie",
        [
             "get_images_runner_run_1",
             "find_exposures_runner",
@@ -220,6 +220,16 @@ def set_jobs_v2_pre_v2(patch, verbose):
     )
 
     # Post-processing
+    jobs["512"] = job_data(
+        "512",
+        ["run_sp_combined_final"],
+        ["make_cat_runner"],
+        "tile_IDs",
+        path_main=path_main,
+        path_left="output",
+        verbose=verbose,
+    )
+
     jobs["1024"] = job_data(
         "1024",
         "run_sp_combined_psf",

scripts/sh/combine_runs.bash

@@ -17,9 +17,10 @@ usage="Usage: $(basename "$0") [OPTIONS]
 \n\nOptions:\n
    -h\tthis message\n
    -p, --psf MODEL\n
-    \tPSF model, one in ['psfex'|'mccd'|'setools'], default='$psf'\n
+    \tPSF model, allowed are 'psfex', 'mccd', 'setools', default='$psf'\n
    -c, --cat TYPE\n
-    \tCatalogue type, one in ['final'|'flag'|'psf'|'image'], default='$cat'\n
+    \tCatalogue type, allowed are 'final', 'flag_tile', 'flag_exp', \n
+    \t'psf', 'image', default='$cat'\n
 "
 
 ## Parse command line
@@ -48,10 +49,11 @@ done
 
 ## Check options
 if [ "$cat" != "final" ] \
-  && [ "$cat" != "flag" ] \
+  && [ "$cat" != "flag_tile" ] \
+  && [ "$cat" != "flag_exp" ] \
   && [ "$cat" != "psf" ] \
   && [ "$cat" != "image" ]; then
-  echo "cat (option -c) needs to be 'final', 'flag', 'psf', or 'image'"
+  echo "cat (option -c) needs to be 'final', 'flag_tile', 'flag_exp', 'psf', or 'image'"
   exit 2
 fi
 
@@ -106,11 +108,27 @@ if [ "$cat" == "final" ]; then
   module="make_catalog_runner"
   pattern="final_cat-*"
 
-elif [ "$cat" == "flag" ]; then
+elif [ "$cat" == "flag_tile" ]; then
 
-  run_in="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_1"
-  module="mask_runner_run_1"
-  pattenr="pipeline_flag-*"
+  # v1
+  #run_in="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_1"
+  # v2
+  run_in="$pwd/$out_base/run_sp_tile_Ma_*"
+  run_out="run_sp_Ma_tile"
+
+  module="mask_runner"
+  pattern="pipeline_flag-*"
+
+elif [ "$cat" == "flag_exp" ]; then
+
+  # v1
+  #run_in="$pwd/$out_base/run_sp_MaMa_*/mask_runner_run_2"
+  # v2
+  run_in="$pwd/$out_base/run_sp_exp_Ma_*"
+  run_out="run_sp_Ma_exp"
+
+  module="mask_runner"
+  pattern="pipeline_flag-*"
 
 elif [ "$cat" == "image" ]; then
 
@@ -162,20 +180,22 @@ i=0
 for dir in $run_in; do
   FILES=(`find $dir -type f -name "$pattern" -print0 | xargs -0 echo`)
 
+  echo "$dir $pattern"
+
   ## Look over source files
   for file in ${FILES[@]}; do
 
-  target=$file
-  link_name=$outdir/`basename $file`
-  link_s $target $link_name
-  ((i=i+1))
+    target=$file
+    link_name=$outdir/`basename $file`
+    link_s $target $link_name
+    ((i=i+1))
 
   done
 
 done
 
 #echo " $n_files target files, $i links created/skipped"
-echo " $i total, "n_skipped skipped, "n_created links created"
+echo " $i total, "$n_skipped skipped, "$n_created links created"
 
 # Update log file
 update_runs_log_file.py

scripts/sh/curl_canfar_local.sh

@@ -15,7 +15,7 @@ ID=-1
 file_IDs=-1
 N_SMP=1
 kind=-1
-version=1.0
+version="1.1"
 cmd_remote="shapepipe/scripts/sh/init_run_exclusive_canfar.sh"
 batch_max=200
 dry_run=0
@@ -92,7 +92,7 @@ while [ $# -gt 0 ]; do
   shift
 done
 
-# Check options                                                                 
+## Check options                                                                 
 if [ "$job" == "-1" ]; then                                                     
   echo "No job indicated, use option -j"                                        
   exit 2                                                                        
@@ -136,11 +136,13 @@ function set_arg() {
 }
 
 
+# MKDEBUG TODO
 function call_curl() {
   my_arg=$1
 
 }
 
+# Add session and image IDs to log files
 function update_session_logs() {
   echo $my_session >> session_IDs.txt
   echo "$my_session $ID" >> session_image_IDs.txt
@@ -152,7 +154,7 @@ function submit_batch() {
 
   for ID in `cat $path`; do
     my_arg=$(set_arg)
-    my_session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$my_arg"`
+    my_session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$my_arg" &> /dev/null`
     update_session_logs
   done
 
@@ -203,19 +205,20 @@ else
       echo "Split '$file_IDs' into $n_split batches of size $batch"
 
       count=1
+      n_running=`stats_jobs_canfar.sh`
       for batch in $prefix*; do
         echo "Number of running jobs = $n_running"
         echo "Submitting batch $batch ($count/$n_split)"
         echo -ne "\033]0;curl patch=$patch job=$job $count/$n_split\007"
         submit_batch $batch
         ((count=count+1))
 
-        n_running=`stats_headless_canfar.py`
+        n_running=`stats_jobs_canfar.sh`
 
         while [ "$n_running" -gt "$n_thresh" ]; do
           echo "Wait for #jobs = $n_running jobs to go < $n_thresh ..."
           sleep $sleep
-          n_running=`stats_headless_canfar.py`
+          n_running=`stats_jobs_canfar.sh`
         done
 
       done
@@ -232,7 +235,7 @@ else
 
     # Submit image
     arg=$(set_arg)
-    session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$arg"`
+    session=`curl -E $SSL $SESSION?$RESOURCES -d "image=$IMAGE:$version" -d "name=${NAME}" -d "cmd=$cmd_remote" --data-urlencode "args=$arg" &> /dev/null`
     update_session_logs
 
   fi

scripts/sh/missing_unique.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+FILES=("summary/missing_job_128_ngmix_runner_*.txt")
+temp="temp_temp.tmp"
+temp2="temp_temp2.tmp"
+out="missing_job_128_ngmix_runner_cut.txt"
+
+i=0
+for file in ${FILES[@]}; do
+
+  echo $file $i
+
+  if [ "$i" == "0" ]; then
+    cp $file $temp
+  else
+    comm -12  <(sort $file) <(sort $temp) > $temp2
+    cp $temp2 $temp
+  fi
+
+  wc $file $temp
+
+  ((i=i+1))
+
+done
+
+mv $temp $out
+rm $temp2