Skip to content

Commit

Permalink
Allow radiance conversion in parsing utils to work properly
Browse files Browse the repository at this point in the history
  • Loading branch information
@havardlovas
havardlovas committed May 14, 2024
1 parent bdd7094 commit 83d3c4f
Show file tree
Hide file tree
Showing 3 changed files with 123 additions and 110 deletions.
229 changes: 121 additions & 108 deletions gref4hsi/tests/test_main_uhi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,135 +26,148 @@
from gref4hsi.utils import visualize
from gref4hsi.utils.config_utils import prepend_data_dir_to_relative_paths, customize_config

# D:\HyperspectralDataAll\UHI\2020-07-01-14-40-15-ArcticSeaIce-Ben-Lange
DATA_DIR = os.path.join(base_fp, "HyperspectralDataAll/UHI/2020-07-01-14-40-15-ArcticSeaIce-Ben-Lange/")

# The configuration file stores the settings for georeferencing
config_file_mission = os.path.join(DATA_DIR, 'configuration.ini')


config_path_template = os.path.join(home, 'VsCodeProjects/gref4hsi/data/config_examples/configuration_uhi.ini')

# Copies the template to config_file_mission and sets up the necessary directories
prepend_data_dir_to_relative_paths(config_path=config_path_template, DATA_DIR=DATA_DIR)

# Non-default settings
custom_config = {'General':
{'mission_dir': DATA_DIR,
'model_export_type': 'dem_file', # Infer seafloor structure from altimeter recordings
'max_ray_length': 5,
'lab_cal_dir': os.path.join(base_fp, 'HyperspectralDataAll/UHI/Lab_Calibration_Data/NP')}, # Max distance in meters from UHI to seafloor

'Coordinate Reference Systems':
{'proj_epsg' : 3395, # The projected CRS for orthorectified data (an arctic CRS)
'geocsc_epsg_export' : 4978, # 3D cartesian system for earth consistent with GPS frame (but inconsistent with eurasian techtonic plate)
'dem_epsg' : 3395, # (Optional) If you have a DEM this can be used
'pos_epsg_orig' : 4978}, # The CRS of the positioning data we deliver to the georeferencing

'Relative Paths':
{'dem_folder': 'Input/GIS/'}, # Using altimeter, we generate one DEM per transect chunk

'Absolute Paths':
{'geoid_path': os.path.join(home, "VsCodeProjects/gref4hsi/data/world/geoids/egm08_25.gtx")}, # Using altimeter, we generate one DEM per transect chunk

'Orthorectification':
{'resample_rgb_only': True, # Good choice for speed
'resolutionhyperspectralmosaic': 0.01, # 1 cm
'raster_transform_method': 'north_east'},

'HDF.raw_nav': {'altitude': 'raw/nav/altitude',
'rotation_reference_type' : 'eul_ZYX', # The vehicles orientations are used as Yaw, Pitch, Roll
'is_global_rot' : False, # The vehicles orientations are used as Yaw, Pitch, Roll
'eul_is_degrees' : True},

'HDF.calibration': {'band2wavelength' : 'processed/radiance/calibration/spectral/band2Wavelength',
'darkframe' : 'processed/radiance/calibration/radiometric/darkFrame',
'radiometricframe' : 'processed/radiance/calibration/radiometric/radiometricFrame',
'fov' : 'processed/radiance/calibration/geometric/fieldOfView'},

# Where to find the standard data for the cube. Note that is_calibrated implies whether data is already in correct format
'HDF.hyperspectral': {'datacube' : 'processed/radiance/dataCube',
'exposuretime' : 'processed/radiance/exposureTime',
'timestamp' : 'processed/radiance/timestamp',
'is_calibrated' : True},

'HDF.rgb' :{'rgb_frames' : 'rawdata/rgb/rgbFrames',
'rgb_frames_timestamp' : 'rawdata/rgb/timestamp'}

}

# Customizes the config file
customize_config(config_path=config_file_mission, dict_custom=custom_config)

# Settings specific to the pre-processing of UHI data. At present they are hardcoded, but they could be integrated
SettingsPreprocess = namedtuple('SettingsPreprocessing', ['dtype_datacube',
'rotation_matrix_hsi_to_body',
'translation_body_to_hsi',
'rotation_matrix_alt_to_body',
'translation_alt_to_body',
'config_file_name',
'time_offset_sec',
'lon_lat_alt_origin',
'resolution_dem',
'agisoft_process'])

config_uhi_preprocess = SettingsPreprocess(dtype_datacube = np.float32,
# The fill value for empty cells (select values not occcuring in cube or ancillary data)
rotation_matrix_hsi_to_body = np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, -1]]),
# Boolean being expressing whether to rectify only composite (true) or data cube and composite (false). True is fast.
translation_body_to_hsi = np.array([0, 0, 0]),
rotation_matrix_alt_to_body = np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, -1]]),
# Boolean being expressing whether to rectify only composite (true) or data cube and composite (false). True is fast.
translation_alt_to_body = np.array([0.5, 0, 0]),
time_offset_sec = 22,
# Ben's tick s1 starts at 1593614097.992003 -> 22 s delay
# Ben's tick s2 starts at 1593614414.995001 -> 0 s delay

lon_lat_alt_origin = np.array([1, 1, 0]),
# The beast sets up a fake coordinate system at 1 deg lon/lat.
config_file_name = 'configuration.ini',
resolution_dem = 0.2, # The resolution of the Altimeter-based DEM
agisoft_process = False) # This is an option for photogrammetry-based processing in the case you have imagery





"""config = configparser.ConfigParser()
config.read(config_file)"""

def main():
def main(mission_dir, config_path_template, config_yaml):
# D:\HyperspectralDataAll\UHI\2020-07-01-14-40-15-ArcticSeaIce-Ben-Lange
DATA_DIR = mission_dir

# The configuration file stores the settings for georeferencing
config_file_mission = os.path.join(DATA_DIR, 'configuration.ini')



# Copies the template to config_file_mission and sets up the necessary directories
prepend_data_dir_to_relative_paths(config_path=config_path_template, DATA_DIR=DATA_DIR)

# Non-default settings
custom_config = {'General':
{'mission_dir': DATA_DIR,
'model_export_type': 'dem_file', # Infer seafloor structure from altimeter recordings
'max_ray_length': 20,
'lab_cal_dir': os.path.join(base_fp, 'HyperspectralDataAll/UHI/Lab_Calibration_Data/NP')}, # Max distance in meters from UHI to seafloor

'Coordinate Reference Systems':
{'proj_epsg' : 3395, # The projected CRS for orthorectified data (an arctic CRS)
'geocsc_epsg_export' : 4978, # 3D cartesian system for earth consistent with GPS frame (but inconsistent with eurasian techtonic plate)
'dem_epsg' : 3395, # (Optional) If you have a DEM this can be used
'pos_epsg_orig' : 4978}, # The CRS of the positioning data we deliver to the georeferencing

'Relative Paths':
{'dem_folder': 'Input/GIS/'}, # Using altimeter, we generate one DEM per transect chunk

'Absolute Paths':
{'geoid_path': os.path.join(home, "VsCodeProjects/gref4hsi/data/world/geoids/egm08_25.gtx")}, # Using altimeter, we generate one DEM per transect chunk

'Orthorectification':
{'resample_rgb_only': True, # Good choice for speed
'resample_ancillary': False, # Good choice for speed
'resolutionhyperspectralmosaic': 0.01, # 1 cm
'raster_transform_method': 'north_east'},

'HDF.raw_nav': {'altitude': 'raw/nav/altitude',
'rotation_reference_type' : 'eul_ZYX', # The vehicles orientations are used as Yaw, Pitch, Roll
'is_global_rot' : False, # The vehicles orientations are used as Yaw, Pitch, Roll
'eul_is_degrees' : True},

'HDF.calibration': {'band2wavelength' : 'processed/radiance/calibration/spectral/band2Wavelength',
'darkframe' : 'processed/radiance/calibration/radiometric/darkFrame',
'radiometricframe' : 'processed/radiance/calibration/radiometric/radiometricFrame',
'fov' : 'processed/radiance/calibration/geometric/fieldOfView'},

# Where to find the standard data for the cube. Note that is_calibrated implies whether data is already in correct format
'HDF.hyperspectral': {'datacube' : 'processed/radiance/dataCube',
'exposuretime' : 'processed/radiance/exposureTime',
'timestamp' : 'processed/radiance/timestamp',
'is_calibrated' : False},

'HDF.rgb' :{'rgb_frames' : 'rawdata/rgb/rgbFrames',
'rgb_frames_timestamp' : 'rawdata/rgb/timestamp'},
'Ancillary': {
'pixel_nr_grid': 'processed/georef/pixel_nr_grid', # h5 path
'unix_time_grid' : 'processed/georef/unix_time_grid' # h5 path
}

}

# Customizes the config file
customize_config(config_path=config_file_mission, dict_custom=custom_config)

# Settings specific to the pre-processing of UHI data. At present they are hardcoded, but they could be integrated
SettingsPreprocess = namedtuple('SettingsPreprocessing', ['dtype_datacube',
'rotation_matrix_hsi_to_body',
'translation_body_to_hsi',
'rotation_matrix_alt_to_body',
'translation_alt_to_body',
'config_file_name',
'time_offset_sec',
'lon_lat_alt_origin',
'resolution_dem',
'agisoft_process'])

config_uhi_preprocess = SettingsPreprocess(dtype_datacube = np.float32,
# The fill value for empty cells (select values not occcuring in cube or ancillary data)
rotation_matrix_hsi_to_body = np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, -1]]),
# Boolean being expressing whether to rectify only composite (true) or data cube and composite (false). True is fast.
translation_body_to_hsi = np.array([0, 0, 0]),
rotation_matrix_alt_to_body = np.array([[0, 1, 0],
[1, 0, 0],
[0, 0, -1]]),
# Boolean being expressing whether to rectify only composite (true) or data cube and composite (false). True is fast.
translation_alt_to_body = np.array([0.5, 0, 0]),
time_offset_sec = 22,
# Ben's tick s1 starts at 1593614097.992003 -> 22 s delay
# Ben's tick s2 starts at 1593614414.995001 -> 0 s delay

lon_lat_alt_origin = np.array([1, 1, 0]),
# The beast sets up a fake coordinate system at 1 deg lon/lat.
config_file_name = 'configuration.ini',
resolution_dem = 0.2, # The resolution of the Altimeter-based DEM
agisoft_process = False) # This is an option for photogrammetry-based processing in the case you have imagery





"""config = configparser.ConfigParser()
config.read(config_file)"""



# The config stores all relevant configurations
config = configparser.ConfigParser()
config.read(config_file_mission)

# The minimum for georeferencing is to parse 1) Mesh model and 2) The pose of the reference
uhi_parsing_utils.uhi_beast(config=config, config_uhi=config_uhi_preprocess)

config = parsing_utils.export_pose(config_file_mission)

config = configparser.ConfigParser()

config.read(config_file_mission)

# Exports model
parsing_utils.export_model(config_file_mission)

# Visualize the data 3D photo model from RGB images and the time-resolved positions/orientations
#visualize.show_mesh_camera(config)

# Georeference the line scans of the hyperspectral imager. Utilizes parsed data
georeference.main(config_file_mission, viz=False)
georeference.main(config_file_mission)

orthorectification.main(config_file_mission)
# Alternatively mode = 'calibrate'
# georeference.main(config_file, mode='calibrate', is_calibrated=True)


if __name__ == "__main__":
main()
config_path_template = os.path.join(home, 'VsCodeProjects/gref4hsi/data/config_examples/configuration_uhi.ini')

mission_dir = os.path.join(base_fp, "HyperspectralDataAll/UHI/Mosaic2020/2020-07-01/")
config_yaml = os.path.join(mission_dir, 'config.mission.yaml')
# First mission
main(mission_dir, config_path_template, config_yaml)

# Second mission
mission_dir = os.path.join(base_fp, "HyperspectralDataAll/UHI/2020-07-07/")
config_yaml = os.path.join(mission_dir, 'config.mission.yaml')
main(mission_dir, config_path_template, config_yaml)
2 changes: 1 addition & 1 deletion gref4hsi/utils/parsing_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ def digital_counts_2_radiance(self, config):
# Only calibrate if data it is not already done
is_calibrated = eval(config['HDF.hyperspectral']['is_calibrated'])

if is_calibrated == is_calibrated:
if is_calibrated:
self.dataCubeRadiance = self.dataCube.astype(np.float32)

# Add the radiance dataset
Expand Down
2 changes: 1 addition & 1 deletion gref4hsi/utils/uhi_parsing_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ def __init__(self, time = None, value = None, time_format = 'date_num'):
self.value = value
def interpolate(self, time_interp):
self.time_interp = time_interp
self.value_interp = interp1d(x = self.time, y = self.value, kind='nearest', fill_value='extrapolate')(x=self.time_interp)
self.value_interp = interp1d(x = self.time, y = self.value, kind='nearest')(x=self.time_interp)


class NAV:
Expand Down

0 comments on commit 83d3c4f

Please sign in to comment.