Merge remote-tracking branch 'refs/remotes/origin/main'

data/config_examples/configuration_specim.ini

@@ -1,28 +1,25 @@
 [General]
 mission_dir = D:/HyperspectralDataAll/HI/2020-07-01-14-34-57-NewYorkCity/
 pose_export_type = h5_embedded
-model_export_type = dem_file
+model_export_type = dem_file 
 sensor_type = Specim FX10
-tex_path = None
-modeltype = DEM
 offset_x = -1
 offset_y = -1
 offset_z = -1
-max_ray_length = 20
-blue_wave_length = 460
-green_wave_length = 530
-red_wave_length = 590
-wavelength_unit = Nanometers
-radiometric_unit = (mW/cm^2*sr*um)*1000.0000
-lever_arm_unit = m
+max_ray_length = 200 
+blue_wave_length = 460 
+green_wave_length = 530 
+red_wave_length = 590 
+wavelength_unit = Nanometers 
+radiometric_unit = (mW/cm^2*sr*um)*1000.0000 
 
 [Coordinate Reference Systems]
-proj_epsg = 4326
-geocsc_epsg_export = 4978
-dem_epsg = 4326
+proj_epsg = 25832 
+geocsc_epsg_export = 4978 
 pos_epsg_orig = 4978
 
 [Relative Paths]
+
 calib_folder = Input/Calib/
 h5_folder = Input/H5/
 rgb_point_cloud_folder = Output/3Dmodels/
@@ -98,7 +95,3 @@ ancillary_suffix = _anc
 nodata = -9999
 raster_transform_method = north_east
 
-[HDF.rgb]
-rgb_frames = rawdata/rgb/rgbFrames
-rgb_frames_timestamp = rawdata/rgb/timestamp
-

data/config_examples/configuration_template.ini

@@ -0,0 +1,110 @@
+# This file is an example of a configuration file, and entries will change during the processing depending on your specifics
+
+[General]
+mission_dir = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/
+pose_export_type = h5_embedded
+model_export_type = geoid # Change to dem_file if you have local terrain model (a.k.a 2.5D), or ply_file if you have *.ply mesh model of terrain (a.k.a 3D )
+sensor_type = Specim FX10 # Change to sensor model name
+offset_x = 2951833.126067576
+offset_y = 293161.63041865156
+offset_z = 5627566.333684149
+max_ray_length = 200 # Change to roughly 3x flying height for your platform (better too large than too small)
+blue_wave_length = 460 # Wavelength for making rgb composites
+green_wave_length = 530 # Wavelength for making rgb composites
+red_wave_length = 590 # Wavelength for making rgb composites
+wavelength_unit = Nanometers # Change to your unit
+radiometric_unit = (mW/cm^2*sr*um)*1000.0000 # Change to your unit
+
+[Coordinate Reference Systems] # Edit proj_epsg
+proj_epsg = 25832 # Change to your projected system to be used for orthorectification (this one is UTM 32, see https://epsg.io/25832)
+geocsc_epsg_export = 4978 # Geocentric system used for ray tracing with terrain model
+pos_epsg_orig = 4978 # The position CRS from navigation data
+
+[Relative Paths] # Optionally edit, Only relevant if you want to use the default generation of folders from a top folder. 
+calib_folder = Input/Calib/ # Where *.xml file with camera calibration file lies
+h5_folder = Input/H5/ # Where h5 data lies
+dem_folder = Input/GIS/ # Possible to put dem in thi
+rgb_point_cloud_folder = Output/3Dmodels/
+footprint_folder = Output/GIS/FootPrints/ 
+rgb_composite_folder = Output/GIS/RGBComposites/ # Where orthorectified composites end up
+orthorectified_cube_folder = Output/GIS/HSIDatacubes/ # Where orthorectified datacubes end up
+anc_folder = Output/GIS/AncillaryData/ # Where ancillary data ends up
+dem_path = Input/GIS/DEM_downsampled_deluxe.tif
+hsi_calib_path = Input/Calib/HSI_2b.xml
+model_path = Input/GIS/model.ply 
+pose_path = Intermediate/pose.csv
+
+[HDF.raw_nav] # Optionally edit, although easiest to not.
+rotation_reference_type = eul_ZYX
+eul_zyx = raw/nav/euler_angles # h5 path to euler angle array (n,3) ordered roll pitch yaw
+eul_is_degrees = True
+position = raw/nav/position_ecef # h5 path to ECEF position (n,3)
+timestamp = raw/nav/timestamp # h5 path to UNIX time (number of seconds since 1970 or something)
+is_global_rot = False
+
+[HDF.hyperspectral] # Optionally edit
+datacube = processed/radiance/dataCube # h5 path Where data cube should be
+exposuretime = processed/radiance/exposureTime # h5 path Where exposure time should be
+timestamp = processed/radiance/timestamp # h5 path Where data cube UNIX time should be
+is_calibrated = True # Recommended option
+
+[HDF.processed_nav] # Auto-generated
+folder = processed/nav/
+quaternion_ecef = processed/nav/quaternion_ref_ecef
+position_ecef = processed/nav/position_ref_ecef
+pos0 = processed/nav/position_offset
+timestamp = processed/nav/timestamp_hsi
+
+[HDF.calibration] # Optionally edit
+band2wavelength = processed/radiance/calibration/spectral/band2Wavelength # Where the band center wavelength array should be (n,)
+fwhm = processed/radiance/calibration/spectral/fwhm # Optionally where the band width array should be (n,)
+darkframe = processed/radiance/calibration/radiometric/darkFrame # If is_calibrated is false where the darkframe should be (m,n)
+radiometricframe = processed/radiance/calibration/radiometric/radiometricFrame # If is_calibrated is false where the radiometric frame should be (m,n)
+
+
+[Georeferencing] # No need to edit, ancillary data from georeferencing, which is orthorectified accordingly
+folder = processed/georef/
+position_ecef = processed/nav/position_hsi_ecef # Where hsi positions are stored
+quaternion_ecef = processed/nav/quaternion_hsi_ecef # Where hsi orientations are stored
+points_ecef_crs = processed/georef/points_ecef_crs # Where intersection points are stored
+points_hsi_crs = processed/georef/point_hsi_frame # Intersection points from HSI coordinate reference system (local)
+normals_hsi_crs = processed/georef/normals_hsi_frame # Surface normals expressed in HSI coordinate reference system (local)
+theta_v = processed/georef/theta_v # Nadir angles of HSI
+theta_s = processed/georef/theta_s # Nadir angles of sun
+phi_v = processed/georef/phi_v # Azimuth angles of HSI
+phi_s = processed/georef/phi_s # Azimuth angles of sun
+normals_ned_crs = processed/georef/normals_ned_crs # Surface normals in North-East-Down at intersections
+unix_time_grid = processed/georef/unix_time_grid # Time at intersections
+pixel_nr_grid = processed/georef/pixel_nr_grid # Pixel number at intersections
+frame_nr_grid = processed/georef/frame_nr_grid # Frame number at intersections
+hsi_tide_gridded = processed/georef/hsi_tide_gridded # Tide level wrt geoid (if tide file is supplied)
+hsi_alts_msl = processed/georef/hsi_alts_msl # HSI height above mean sealevel (e.g. to be used for atmos correction)
+
+[Orthorectification]
+resample_rgb_only = False # If True, only RGB composites are made. This is much faster
+resample_ancillary = True # If ancillary is needed for further analysis of data, set to True
+chunk_size_cube_gb = 1 # The working chunk size for orthorectification, set well below available RAM
+resolutionhyperspectralmosaic = 0.1 # Change to your target ground resolution
+interleave = bsq # Do not edit
+ancillary_suffix = _anc
+nodata = -9999
+raster_transform_method = north_east # Can be set to minimal_rectangle giving the memory-optimal raster transform, but these rotated rasters are unfortunaty not well supported by downstream tools
+
+
+[Absolute Paths] # Edit these according to need. For this example, the top folder is D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/
+calib_folder = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Input/Calib/
+h5_folder = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Input/H5/
+rgb_point_cloud_folder = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Output/3Dmodels/
+footprint_folder = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Output/GIS/FootPrints/
+rgb_composite_folder = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Output/GIS/RGBComposites/
+orthorectified_cube_folder = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Output/GIS/HSIDatacubes/
+anc_folder = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Output/GIS/AncillaryData/
+dem_folder = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Input/GIS/
+tide_path = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Input/tidevann_nn2000_NMA.txt
+dem_path = D:/HyperspectralDataAll/HI/2022-08-31-060000-Remoy-Specim/Input/GIS/DEM_downsampled.tif
+hsi_calib_path = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Input/Calib/HSI_2b.xml
+pose_path = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Intermediate/pose.csv
+model_path = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Input/GIS/model.ply # Change this path if model_export_type='ply_file' 
+dem_path = D:/Specim/Missions/2022-08-31-Rem�y/2022-08-31_0800_HSI/processed/Input/GIS/DEM_downsampled_deluxe.tif # Change this path if model_export_type='dem_file' 
+geoid_path = C:/Users/haavasl/VSCodeProjects/hyperspectral_toolchain/data/world/geoids/no_kv_HREF2018A_NN2000_EUREF89.tif # Change this path
+

data/config_examples/configuration_uhi.ini

@@ -94,13 +94,17 @@ hsi_alts_msl = processed/georef/hsi_alts_msl
 
 [Orthorectification]
 resample_rgb_only = True
+resample_ancillary = True
 chunk_size_cube_gb = 1
 resolutionhyperspectralmosaic = 0.01
 interleave = bsq
 ancillary_suffix = _anc
 nodata = -9999
 raster_transform_method = north_east
 
+
+
+
 [HDF.rgb]
 rgb_frames = rawdata/rgb/rgbFrames
 rgb_frames_timestamp = rawdata/rgb/timestamp

gref4hsi/scripts/georeference.py

@@ -9,10 +9,10 @@
 import pyvista as pv
 import h5py
 
-# Local resources:
-from utils.geometry_utils import CameraGeometry, CalibHSI
-from utils.parsing_utils import Hyperspectral
-from scripts import visualize
+# Lib resources:
+from gref4hsi.utils.geometry_utils import CameraGeometry, CalibHSI
+from gref4hsi.utils.parsing_utils import Hyperspectral
+from gref4hsi.scripts import visualize
 
 
 def cal_file_to_rays(filename_cal, config):
@@ -146,10 +146,17 @@ def main(iniPath, viz = False):
 
     mesh = pv.read(path_mesh)
 
-    print('Georeferencing Images')
-
-    for filename in sorted(os.listdir(dir_r)):
+
+    print("\n################ Georeferencing: ################")
+    files = sorted(os.listdir(dir_r))
+    n_files= len(sorted(os.listdir(dir_r)))
+    file_count = 0
+    for filename in files:
         if filename.endswith('h5') or filename.endswith('hdf'):
+
+            progress_perc = 100*file_count/n_files
+            print(f"Georeferencing file {file_count+1}/{n_files}, progress is {progress_perc} %")
+
             # Path to hierarchical file
             path_hdf = dir_r + filename
 
@@ -168,7 +175,7 @@ def main(iniPath, viz = False):
                                     intrinsic_geometry_dict = intrinsic_geometry_dict)
 
 
-            hsi_geometry.intersectWithMesh(mesh = mesh, max_ray_length=max_ray_length)
+            hsi_geometry.intersect_with_mesh(mesh = mesh, max_ray_length=max_ray_length)
 
             # Computes the view angles in the local NED. Computationally intensive as local NED is defined for each intersection
             hsi_geometry.compute_view_directions_local_tangent_plane()
@@ -182,15 +189,14 @@ def main(iniPath, viz = False):
 
             write_intersection_geometry_2_h5_file(hsi_geometry=hsi_geometry, config = config, h5_filename=path_hdf)
 
-            print('Writing Point Cloud')
-            hsi_geometry.writeRGBPointCloud(config = config, hyp = hyp, transect_string = filename.split('.')[0])
+            hsi_geometry.write_rgb_point_cloud(config = config, hyp = hyp, transect_string = filename.split('.')[0])
 
             if viz:
                  visualize.show_projected_hsi_points(HSICameraGeometry=hsi_geometry, 
                                                      config=config, 
                                                      transect_string = filename.split('.')[0])
 
-            print('Intersection geometry written to:\n {0}'.format(filename))
+
 
 
 
@@ -199,6 +205,7 @@ def main(iniPath, viz = False):
             #from scripts import visualize
             #visualize.show_projected_hsi_points(HSICameraGeometry=hsi_geometry, config=config, transect_string = filename.split('.')[0])
 
+        file_count+=1
 
 
 

gref4hsi/scripts/orthorectification.py

@@ -7,9 +7,8 @@
 import numpy as np
 
 # Local resources:
-from utils.geometry_utils import CameraGeometry, FeatureCalibrationObject, CalibHSI
-from utils.gis_tools import GeoSpatialAbstractionHSI
-from utils.parsing_utils import Hyperspectral
+from gref4hsi.utils.gis_tools import GeoSpatialAbstractionHSI
+from gref4hsi.utils.parsing_utils import Hyperspectral
 
 
 
@@ -119,10 +118,15 @@ def main(iniPath):
                               )
 
 
-    print('Orthorectifying Images')
-
-    for filename in sorted(os.listdir(h5_folder)):
+    print("\n################ Orthorectifying: ################")
+    files = sorted(os.listdir(h5_folder))
+    n_files= len(sorted(os.listdir(h5_folder)))
+    file_count = 0
+    for filename in files:
         if filename.endswith('h5') or filename.endswith('hdf'):
+
+            progress_perc = 100*file_count/n_files
+            print(f"Orthorectifying file {file_count+1}/{n_files}, progress is {progress_perc} %")
             # Path to hierarchical file
             h5_filename = h5_folder + filename
 
@@ -173,7 +177,8 @@ def main(iniPath):
                                         anc_dict = anc_dict, 
                                         anc_dir = anc_dir,
                                         interleave=config_ortho.interleave)
-
+
+        file_count+=1
 if __name__ == '__main__':
     args = sys.argv[1:]
     iniPath = args[0]

gref4hsi/scripts/visualize.py

@@ -6,7 +6,7 @@
 import open3d as o3d
 import pymap3d as pm
 # A simple visualization of various types of data
-from utils.geometry_utils import rotation_matrix_ecef2ned, rotation_matrix_ecef2enu
+from gref4hsi.utils.geometry_utils import rotation_matrix_ecef2ned, rotation_matrix_ecef2enu
 
 def show_mesh_camera(config, show_mesh = True, show_pose = True, ref_frame = 'ECEF'):
     """

gref4hsi/tests/test_main_hi.py

@@ -8,7 +8,7 @@
 from scripts import orthorectification
 from utils import parsing_utils
 from scripts import visualize
-from utils.config_utils import prepend_data_dir_to_relative_paths
+from gref4hsi.utils.config_utils import prepend_data_dir_to_relative_paths
 
 """
 This script is meant to be used for testing the processing pipeline of airborne HI data.