diff --git a/example/cfis/config_GitFeGie_cat.ini b/example/cfis/config_GitFeGie_cat.ini new file mode 100644 index 000000000..b735eef5c --- /dev/null +++ b/example/cfis/config_GitFeGie_cat.ini @@ -0,0 +1,150 @@ +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_GitFeGie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = True + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner, find_exposures_runner, get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER_RUN_1] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = vos:cfis/tiles_DR4, vos:cfis/tiles_DR4, vos:cfis/tiles_DR4 + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight, CFIS.000.000.r + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits.fz, .cat + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{3}\.\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight-, CFIS_cat- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = vos + +# Copy command options, optional +RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem + + +[FIND_EXPOSURES_RUNNER] + +INPUT_MODULE = get_images_runner_run_1 + +FILE_PATTERN = CFIS_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# column number of exposure name in HISTORY FITS header entry +COLNUM = 3 + +# Get exposures +[GET_IMAGES_RUNNER_RUN_2] + +INPUT_DIR = last:find_exposures_runner + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = vos:cfis/pitcairn, vos:cfis/weights, vos:cfis/flags +# LSB images: +#INPUT_PATH = vos:cfis/lsb_individual, vos:cfis/weights, vos:cfis/flags + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag +# LSB images +#INPUT_FILE_PATTERN = 000000s, 000000p.weight, 000000p.flag + +# Input file extensions +INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{6} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-, weight-, flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = vos + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Retrieve command options, optional +RETRIEVE_OPTIONS = --certfile=$VM_HOME/.ssl/cadcproxy.pem diff --git a/example/cfis/config_GitFeGie_symlink.ini b/example/cfis/config_GitFeGie_symlink.ini index 22e834533..f0771f6a2 100644 --- a/example/cfis/config_GitFeGie_symlink.ini +++ b/example/cfis/config_GitFeGie_symlink.ini @@ -98,6 +98,7 @@ FILE_EXT = .fits # NUMBERING_SCHEME (optional) string with numbering pattern for input files NUMBERING_SCHEME = -000-000 +# column number of exposure name in HISTORY FITS header entry COLNUM = 3 # Get exposures diff --git a/example/cfis/config_GitFeGie_vos.ini b/example/cfis/config_GitFeGie_vos.ini index fd3207bb3..83fca4335 100644 --- a/example/cfis/config_GitFeGie_vos.ini +++ b/example/cfis/config_GitFeGie_vos.ini @@ -98,6 +98,9 @@ FILE_EXT = .fits # NUMBERING_SCHEME (optional) string with numbering pattern for input files NUMBERING_SCHEME = -000-000 +# column number of exposure name in HISTORY FITS header entry +COLNUM = 1 + # Get exposures [GET_IMAGES_RUNNER_RUN_2] diff --git a/example/cfis/config_exp_SpMh.ini b/example/cfis/config_exp_SpMh.ini index 76f87ddd7..53874d4d6 100644 --- a/example/cfis/config_exp_SpMh.ini +++ b/example/cfis/config_exp_SpMh.ini @@ -45,7 +45,7 @@ OUTPUT_DIR = $SP_RUN/output [JOB] # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 16 +SMP_BATCH_SIZE = 10 # Timeout value (optional), default is None, i.e. no timeout limit applied TIMEOUT = 96:00:00 diff --git a/example/cfis/config_tile_Sx_exp_mccd.ini b/example/cfis/config_tile_Sx_exp_mccd.ini index c195a8c80..cefa1d7c2 100644 --- a/example/cfis/config_tile_Sx_exp_mccd.ini +++ b/example/cfis/config_tile_Sx_exp_mccd.ini @@ -89,9 +89,12 @@ PSF_FILE = False DETECTION_IMAGE = False # Distinct weight image for detection (SExtractor -# in dual-image mode) +# in dual-image mode) if True DETECTION_WEIGHT = False +# Association catalogue for matching detections +ASSOC_CAT = False + ZP_FROM_HEADER = False BKG_FROM_HEADER = False @@ -158,6 +161,9 @@ DETECTION_IMAGE = False # in dual-image mode) if True DETECTION_WEIGHT = False +# Association catalogue for matching detections +ASSOC_CAT = False + # Se to True if photometry zero-point is to be read from exposure image header ZP_FROM_HEADER = True diff --git a/example/cfis/config_tile_Sx_exp_psfex.ini b/example/cfis/config_tile_Sx_exp_psfex.ini index 1271eb012..ae7763c16 100644 --- a/example/cfis/config_tile_Sx_exp_psfex.ini +++ b/example/cfis/config_tile_Sx_exp_psfex.ini @@ -46,7 +46,7 @@ OUTPUT_DIR = $SP_RUN/output [JOB] # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 40 +SMP_BATCH_SIZE = 20 # Timeout value (optional), default is None, i.e. no timeout limit applied TIMEOUT = 96:00:00 @@ -92,6 +92,9 @@ DETECTION_IMAGE = False # in dual-image mode) DETECTION_WEIGHT = False +# Association catalogue for matching detections +ASSOC_CAT = False + ZP_FROM_HEADER = False BKG_FROM_HEADER = False @@ -160,6 +163,9 @@ DETECTION_IMAGE = False # in dual-image mode) DETECTION_WEIGHT = False +# Association catalogue for matching detections +ASSOC_CAT = False + # True if photometry zero-point is to be read from exposure image header ZP_FROM_HEADER = True diff --git a/example/cfis/config_tile_Sx_exp_psfex_cat.ini b/example/cfis/config_tile_Sx_exp_psfex_cat.ini new file mode 100644 index 000000000..c16778957 --- /dev/null +++ b/example/cfis/config_tile_Sx_exp_psfex_cat.ini @@ -0,0 +1,254 @@ +# ShapePipe configuration file for single-exposures. PSFex PSF model. +# Process exposures after masking, from star detection to PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Sx_exp_SxSePsf + +# Add date and time to RUN_NAME, optional, default: True +; RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner, sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 40 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER_RUN_1] + +INPUT_MODULE = get_images_runner_run_1, uncompress_fits_runner, mask_runner_run_1 + +INPUT_DIR = last:get_images_runner_run_1, last:uncompress_fits_runner, last:mask_runner_run_1 + +FILE_PATTERN = CFIS_image, CFIS_weight, pipeline_flag, CFIS_cat + +FILE_EXT = .fits, .fits, .fits, .cat + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = sex + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default_cat.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +# Association catalogue for matching detections +ASSOC_CAT = True + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# Multi-epoch mode: Path to file with single-exposure WCS header information +LOG_WCS = $SP_RUN/output/log_exp_headers.sqlite + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 + + +[SEXTRACTOR_RUNNER_RUN_2] + +# Somehow this works but not +# - omitting +# - $SP_RUN/output +#INPUT_DIR = . + +# Input from two modules +INPUT_MODULE = split_exp_runner, mask_runner + +# Read pipeline flag files created by mask module +FILE_PATTERN = image, weight, pipeline_flag + +NUMBERING_SCHEME = -0000000-0 + +# SExtractor executable path +EXEC_PATH = sex + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_exp.sex +DOT_PARAM_FILE = $SP_CONFIG//default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True. +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +# Association catalogue for matching detections +ASSOC_CAT = False + +# True if photometry zero-point is to be read from exposure image header +ZP_FROM_HEADER = True + +# If ZP_FROM_HEADER is True, zero-point key name +ZP_KEY = PHOTZP + +# Background information from image header. +# If BKG_FROM_HEADER is True, background value will be read from header. +# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. +# This is used e.g. for the LSB images. +BKG_FROM_HEADER = False +# LSB images: +# BKG_FROM_HEADER = True + +# If BKG_FROM_HEADER is True, background value key name +# LSB images: +#BKG_KEY = IMMODE + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, +# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) SUFFIX = tile +SUFFIX = sexcat + +## Post-processing + +# Not required for single exposures +MAKE_POST_PROCESS = FALSE + + +[SETOOLS_RUNNER] + +INPUT_MODULE = sextractor_runner_run_2 + +# Note: Make sure this doe not match the SExtractor background images +# (sexcat_background*) +FILE_PATTERN = sexcat + +NUMBERING_SCHEME = -0000000-0 + +# SETools config file +SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools + + +[PSFEX_RUNNER] + +# Use 80% sample for PSF model +FILE_PATTERN = star_split_ratio_80 + +NUMBERING_SCHEME = -0000000-0 + +# Path to executable for the PSF model (optional) +EXEC_PATH = psfex + +# Default psfex configuration file +DOT_PSFEX_FILE = $SP_CONFIG/default.psfex + + +[PSFEX_INTERP_RUNNER] + +# Use 20% sample for PSF validation +FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat + +FILE_EXT = .psf, .fits, .cat + +NUMBERING_SCHEME = -0000000-0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = VALIDATION + +# Column names of position parameters +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# If True, measure and store ellipticity of the PSF (using moments) +GET_SHAPES = True + +# Minimum number of stars per CCD for PSF model to be computed +STAR_THRESH = 22 + +# Maximum chi^2 for PSF model to be computed on CCD +CHI2_THRESH = 2 diff --git a/example/cfis/default.param b/example/cfis/default.param index 6521a94d5..f35822931 100644 --- a/example/cfis/default.param +++ b/example/cfis/default.param @@ -1,69 +1,56 @@ NUMBER #Running object number -EXT_NUMBER #FITS extension number -FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] +ALPHA_J2000 #Right ascension of barycenter (J2000) [deg] +DELTA_J2000 #Declination of barycenter (J2000) [deg] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +MAG_BEST #Best of MAG_AUTO and MAG_ISOCOR [mag] +MAGERR_BEST #RMS error for MAG_BEST [mag] +MAG_APER #Fixed aperture magnitude vector [mag] +MAGERR_APER #RMS error vector for fixed aperture mag. [mag] +A_WORLD #Profile RMS along major axis (world units) [deg] +ERRA_WORLD #World RMS position error along major axis [deg] +B_WORLD #Profile RMS along minor axis (world units) [deg] +ERRB_WORLD #World RMS position error along minor axis [deg] +THETA_J2000 #Position angle (east of north) (J2000) [deg] +ERRTHETA_J2000 #J2000 error ellipse pos. angle (east of north) [deg] +ISOAREA_IMAGE #Isophotal area above Analysis threshold [pixel**2] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] +FLUX_RADIUS #Fraction-of-light radii [pixel] + FLUXERR_AUTO #RMS error for AUTO flux [count] -MAG_AUTO #Kron-like elliptical aperture magnitude [mag] -MAGERR_AUTO #RMS error for AUTO magnitude [mag] FLUX_WIN #Gaussian-weighted flux [count] FLUXERR_WIN #RMS error for WIN flux [count] MAG_WIN #Gaussian-weighted magnitude [mag] MAGERR_WIN #RMS error for MAG_WIN [mag] -FLUX_APER(1) -FLUXERR_APER(1) - -FLUX_RADIUS #Fraction-of-light radii [pixel] - -SNR_WIN #Gaussian-weighted SNR - -BACKGROUND #Background at centroid position [count] -THRESHOLD #Detection threshold above background [count] - -X_IMAGE #Object position along x [pixel] -Y_IMAGE #Object position along y [pixel] - -X_WORLD #Barycenter position along world x axis [deg] -Y_WORLD #Barycenter position along world y axis [deg] - -X2_IMAGE #Variance along x [pixel**2] -Y2_IMAGE #Variance along y [pixel**2] -XY_IMAGE #Covariance between x and y [pixel**2] -ERRX2_IMAGE #Variance of position along x [pixel**2] -ERRY2_IMAGE #Variance of position along y [pixel**2] -ERRXY_IMAGE #Covariance of position between x and y [pixel**2] - +FLUX_APER(1) +FLUXERR_APER(1) + +SNR_WIN #Gaussian-weighted SNR + XWIN_IMAGE #Windowed position estimate along x [pixel] YWIN_IMAGE #Windowed position estimate along y [pixel] - + XWIN_WORLD #Windowed position along world x axis [deg] YWIN_WORLD #Windowed position along world y axis [deg] - + X2WIN_IMAGE #Windowed variance along x [pixel**2] Y2WIN_IMAGE #Windowed variance along y [pixel**2] XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] - -MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] -MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] - -FLAGS #Extraction flags -FLAGS_WIN #Flags for WINdowed parameters -#!!! REQUIRE FLAG_IMAGE !!! -IMAFLAGS_ISO #FLAG-image flags OR'ed over the iso. profile !!! REQUIRE FLAG_IMAGE !!! - -FWHM_IMAGE #FWHM assuming a gaussian core [pixel] -FWHM_WORLD #FWHM assuming a gaussian core [deg] -ELONGATION #A_IMAGE/B_IMAGE -ELLIPTICITY #1 - B_IMAGE/A_IMAGE - -VIGNET(51,51) #Pixel data around detection [count] - -#VECTOR_ASSOC #ASSOCiated parameter vector -#NUMBER_ASSOC #Number of ASSOCiated IDs +FWHM_IMAGE +FLUX_AUTO + +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS +FLAGS_WIN #Flags for WINdowed parameters + +IMAFLAGS_ISO #FLAG-image flags OR'ed over the iso. profile ``` -#SPREAD_MODEL -#SPREADERR_MODEL -#FWHMPSF_IMAGE +VIGNET(51,51) diff --git a/example/cfis/default_cat.param b/example/cfis/default_cat.param new file mode 100644 index 000000000..c511e2d57 --- /dev/null +++ b/example/cfis/default_cat.param @@ -0,0 +1,69 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +#!!! REQUIRE FLAG_IMAGE !!! +IMAFLAGS_ISO #FLAG-image flags OR'ed over the iso. profile !!! REQUIRE FLAG_IMAGE !!! + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +#VECTOR_ASSOC #ASSOCiated parameter vector +NUMBER_ASSOC #Number of ASSOCiated IDs + +#SPREAD_MODEL +#SPREADERR_MODEL +#FWHMPSF_IMAGE diff --git a/scripts/sh/job_sp.bash b/scripts/sh/job_sp.bash index 15cb1428a..bdf606f7d 100755 --- a/scripts/sh/job_sp.bash +++ b/scripts/sh/job_sp.bash @@ -355,8 +355,9 @@ mkdir -p $SP_CONFIG_MOD # Processing -## Retrieve config files and images (online if retrieve=vos) -## Retrieve and save star catalogues for masking (if star_cat_for_mask=save) +## Retrieve config files and images (requires internet access if retrieve=vos) +## Retrieve and save star catalogues for masking (if star_cat_for_mask=save, +## in which case internet access is required) (( do_job= $job & 1 )) if [[ $do_job != 0 ]]; then @@ -391,7 +392,7 @@ if [[ $do_job != 0 ]]; then fi -## Prepare images (offline) +## Prepare images (( do_job= $job & 2 )) if [[ $do_job != 0 ]]; then @@ -403,7 +404,8 @@ if [[ $do_job != 0 ]]; then fi -## Mask tiles and exposures: add star, halo, and Messier object masks (online if "star_cat_for_mask" is "onthefly") +## Mask tiles and exposures: add star, halo, and Messier object masks. +## Requires internet access if star_cat_for_mask=onthefly. (( do_job= $job & 4 )) if [[ $do_job != 0 ]]; then @@ -413,7 +415,7 @@ if [[ $do_job != 0 ]]; then fi -## Remaining exposure processing (offline) +## Tile detection and remaining exposure processing (( do_job= $job & 8 )) if [[ $do_job != 0 ]]; then @@ -425,7 +427,8 @@ if [[ $do_job != 0 ]]; then fi -## Process tiles up to shape measurement +## Tile PSF interpolation and postage stamp extraction, processing of tiles +## up to shape measurement (( do_job= $job & 16 )) if [[ $do_job != 0 ]]; then @@ -436,7 +439,7 @@ if [[ $do_job != 0 ]]; then fi -## Shape measurement (offline) +## Shape measurement (( do_job= $job & 32 )) if [[ $do_job != 0 ]]; then @@ -471,7 +474,7 @@ if [[ $do_job != 0 ]]; then fi -## Create final catalogues (offline) +## Create final catalogues (( do_job= $job & 64 )) if [[ $do_job != 0 ]]; then @@ -488,7 +491,7 @@ if [[ $do_job != 0 ]]; then fi -## Upload results (online) +## Upload results (requires internet access) (( do_job= $job & 128 )) if [[ $do_job != 0 ]]; then diff --git a/shapepipe/modules/find_exposures_package/find_exposures.py b/shapepipe/modules/find_exposures_package/find_exposures.py index a3c8f7928..9793e31d4 100644 --- a/shapepipe/modules/find_exposures_package/find_exposures.py +++ b/shapepipe/modules/find_exposures_package/find_exposures.py @@ -91,8 +91,8 @@ def get_exposure_list(self): pattern_match = re.search(pattern, temp[self._colnum]) if not pattern_match: raise IndexError( - f're match \'{pattern}\' failed for filename' - + f' \'{temp[self._colnum]}\'' + f"re match '{pattern}' in 'HISTORY' FITS header" + + " keyword column f" '{temp[self._colnum]}' failed" ) exp_name = pattern_match.group(1) diff --git a/shapepipe/modules/get_images_package/get_images.py b/shapepipe/modules/get_images_package/get_images.py index d465d0719..392dcc773 100644 --- a/shapepipe/modules/get_images_package/get_images.py +++ b/shapepipe/modules/get_images_package/get_images.py @@ -363,7 +363,7 @@ def retrieve_one(self, in_path, out_path_orig, out_path_renamed): # Create symbolic link to downloaded file with # link name in ShapePipe numbering format - os.symlink(out_path_orig, out_path_renamed) + os.symlink(os.path.abspath(out_path_orig), out_path_renamed) elif self._retrieve_method == 'symlink': src = in_path diff --git a/shapepipe/modules/sextractor_package/sextractor_script.py b/shapepipe/modules/sextractor_package/sextractor_script.py index 45f44cb49..4deabb477 100644 --- a/shapepipe/modules/sextractor_package/sextractor_script.py +++ b/shapepipe/modules/sextractor_package/sextractor_script.py @@ -77,7 +77,8 @@ def make_post_process(cat_path, f_wcs_path, pos_params, ccd_size): """ cat = file_io.FITSCatalogue( - cat_path, SEx_catalogue=True, + cat_path, + SEx_catalogue=True, open_mode=file_io.BaseCatalogue.OpenMode.ReadWrite, ) cat.open() @@ -199,6 +200,7 @@ def __init__( use_psf, use_detection_image, use_detection_weight, + use_association_cat, use_zero_point, use_background, zero_point_key=None, @@ -221,8 +223,14 @@ def __init__( self._path_dot_param = path_dot_param self._path_dot_conv = path_dot_conv - self.set_input_files(use_weight, use_flag, use_psf, - use_detection_image, use_detection_weight) + self.set_input_files( + use_weight, + use_flag, + use_psf, + use_detection_image, + use_detection_weight, + use_association_cat, + ) # Collect optional arguments for SExtractor self.get_zero_point(use_zero_point, zero_point_key) @@ -265,6 +273,7 @@ def set_input_files( use_psf, use_detect_img, use_detect_weight, + use_association_cat, ): """Set Input Files. @@ -282,6 +291,8 @@ def set_input_files( Specify if a detection image is provided use_detect_weight: bool Specify if a detection weight is provided + use_association_cat: bool + Specify an association catalogue for matching Raises ------ @@ -347,6 +358,14 @@ def set_input_files( else: self._cmd_line_extra += ' -WEIGHT_TYPE None' + # Check for association catalogue to match detections + if use_association_cat: + self._cmd_line_extra += ( + ' -ASSOC_NAME ' + + f'{self._all_input_path[extra]}' + ) + extra += 1 + if extra != len(self._all_input_path): raise ValueError( 'Incoherence between input file number and keys ' diff --git a/shapepipe/modules/sextractor_runner.py b/shapepipe/modules/sextractor_runner.py index 9e3bac7d3..93c2f3a32 100644 --- a/shapepipe/modules/sextractor_runner.py +++ b/shapepipe/modules/sextractor_runner.py @@ -45,6 +45,7 @@ def sextractor_runner( psf_file = config.getboolean(module_config_sec, 'PSF_FILE') detection_image = config.getboolean(module_config_sec, 'DETECTION_IMAGE') detection_weight = config.getboolean(module_config_sec, 'DETECTION_WEIGHT') + association_cat = config.getboolean(module_config_sec, "ASSOC_CAT") zp_from_header = config.getboolean(module_config_sec, 'ZP_FROM_HEADER') if zp_from_header: @@ -81,6 +82,7 @@ def sextractor_runner( psf_file, detection_image, detection_weight, + association_cat, zp_from_header, bkg_from_header, zero_point_key=zp_key, diff --git a/shapepipe/pipeline/file_io.py b/shapepipe/pipeline/file_io.py index a768f0c06..d45fd8e0f 100644 --- a/shapepipe/pipeline/file_io.py +++ b/shapepipe/pipeline/file_io.py @@ -758,7 +758,7 @@ def save_as_fits( names = data.dtype.names it = names else: - raise ValueError('Names not provided') + raise ValueError('Column names not provided') else: it = range(len(names)) self._save_to_fits( @@ -772,7 +772,7 @@ def save_as_fits( elif type(data) is list: if names is None: - raise ValueError('Names not provided') + raise ValueError('Column names not provided') it = range(len(names)) data = np.asarray(data) self._save_to_fits( @@ -786,7 +786,7 @@ def save_as_fits( elif type(data) is Table: if names is None: - raise ValueError('Names not provided') + raise ValueError('Column names not provided') it = names self._save_to_fits( data,