Merge branch 'develop' into main

las_trx/main.py

@@ -1,20 +1,23 @@
+import logging
 import os.path
+import queue
 import sys
 from datetime import date
 from multiprocessing import freeze_support
 from pathlib import Path
+from queue import Queue
 
-from PySide2.QtCore import QSize
-from PySide2.QtGui import QIcon
+from PySide2.QtCore import QSize, QThread, Signal
+from PySide2.QtGui import QIcon, QTextCursor
 from PySide2.QtWidgets import (
     QApplication,
     QErrorMessage,
     QFileDialog,
     QMainWindow,
     QMessageBox,
 )
-from csrspy.enums import CoordType, Reference, VerticalDatum
 
+from csrspy.enums import CoordType, Reference, VerticalDatum
 from las_trx.config import TransformConfig
 from las_trx.ui_mainwindow import Ui_MainWindow
 from las_trx.utils import (
@@ -25,6 +28,8 @@
 )
 from las_trx.worker import TransformWorker
 
+logger = logging.getLogger(__name__)
+
 
 def resource_path(relative_path: str):
     """Get absolute path to resource, works for dev and for PyInstaller"""
@@ -208,14 +213,22 @@ def output_files(self) -> list[Path]:
         out_path = self.ui.lineEdit_output_file.text()
         return [Path(out_path.format(f.stem)) for f in self.input_files]
 
+    def append_text(self, text):
+        self.ui.textBrowser_log_output.moveCursor(QTextCursor.End)
+        self.ui.textBrowser_log_output.insertPlainText(text)
+
     def on_process_success(self):
+        logger.info("Processing complete")
         self.done_msg_box.exec()
 
     def on_process_error(self, exception: BaseException):
+        logger.error(str(exception))
         self.err_msg_box.showMessage(str(exception))
         self.err_msg_box.exec()
 
     def convert(self):
+        logger.debug("Starting worker thread.")
+
         self.thread = TransformWorker(
             self.transform_config, self.input_files, self.output_files
         )
@@ -234,12 +247,50 @@ def convert(self):
         self.thread.start()
 
 
+class LogStream(object):
+    def __init__(self, queue):
+        super().__init__()
+        self.queue = queue
+
+    def write(self, text):
+        self.queue.put(text)
+
+
+class LogThread(QThread):
+    on_msg = Signal(str)
+
+    def __init__(self, queue: Queue, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.queue = queue
+
+    def run(self):
+        while not self.isInterruptionRequested():
+            try:
+                text = self.queue.get(block=False)
+                self.on_msg.emit(text)
+            except queue.Empty:
+                continue
+
+
 if __name__ == "__main__":
     freeze_support()
 
-    app = QApplication(sys.argv)
+    # Configure logging
+    log_msg_queue = Queue()
+    log_write_stream = LogStream(log_msg_queue)
+    log_handler = logging.StreamHandler(log_write_stream)
+
+    logging.basicConfig(level=logging.INFO, handlers=[log_handler])
 
+    app = QApplication(sys.argv)
     window = MainWindow()
     window.show()
 
+    # When a new message is written to the log_queue via the log_write_stream, log_thread emits a signal that causes the main
+    #   window to display that msg in the textBrowser
+    log_thread = LogThread(log_msg_queue)
+    log_thread.on_msg.connect(window.append_text)
+    app.aboutToQuit.connect(log_thread.requestInterruption)
+    log_thread.start()
+
     sys.exit(app.exec_())

las_trx/resources/mainwindow.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>576</width>
-    <height>690</height>
+    <height>789</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -273,7 +273,7 @@
               <second>59</second>
               <year>9998</year>
               <month>1</month>
-              <day>8</day>
+              <day>9</day>
              </datetime>
             </property>
             <property name="displayFormat">
@@ -680,6 +680,20 @@
       </layout>
      </widget>
     </item>
+    <item>
+     <widget class="QLabel" name="label_log_output">
+      <property name="text">
+       <string>Log Output</string>
+      </property>
+     </widget>
+    </item>
+    <item>
+     <widget class="QTextBrowser" name="textBrowser_log_output">
+      <property name="frameShadow">
+       <enum>QFrame::Sunken</enum>
+      </property>
+     </widget>
+    </item>
     <item>
      <widget class="QWidget" name="widget_actions" native="true">
       <property name="sizePolicy">

las_trx/ui_mainwindow.py

@@ -17,7 +17,7 @@ class Ui_MainWindow(object):
     def setupUi(self, MainWindow):
         if not MainWindow.objectName():
             MainWindow.setObjectName(u"MainWindow")
-        MainWindow.resize(576, 690)
+        MainWindow.resize(576, 789)
         self.centralwidget = QWidget(MainWindow)
         self.centralwidget.setObjectName(u"centralwidget")
         self.verticalLayout = QVBoxLayout(self.centralwidget)
@@ -138,7 +138,7 @@ def setupUi(self, MainWindow):
         self.dateEdit_input_epoch = QDateEdit(self.widget_input_options)
         self.dateEdit_input_epoch.setObjectName(u"dateEdit_input_epoch")
         self.dateEdit_input_epoch.setTime(QTime(8, 0, 0))
-        self.dateEdit_input_epoch.setMaximumDateTime(QDateTime(QDate(9998, 1, 8), QTime(7, 59, 59)))
+        self.dateEdit_input_epoch.setMaximumDateTime(QDateTime(QDate(9998, 1, 9), QTime(7, 59, 59)))
         self.dateEdit_input_epoch.setDisplayFormat(u"yyyy-MM-dd")
         self.dateEdit_input_epoch.setCalendarPopup(True)
         self.dateEdit_input_epoch.setTimeSpec(Qt.UTC)
@@ -329,6 +329,17 @@ def setupUi(self, MainWindow):
 
         self.verticalLayout.addWidget(self.frame_output)
 
+        self.label_log_output = QLabel(self.centralwidget)
+        self.label_log_output.setObjectName(u"label_log_output")
+
+        self.verticalLayout.addWidget(self.label_log_output)
+
+        self.textBrowser_log_output = QTextBrowser(self.centralwidget)
+        self.textBrowser_log_output.setObjectName(u"textBrowser_log_output")
+        self.textBrowser_log_output.setFrameShadow(QFrame.Sunken)
+
+        self.verticalLayout.addWidget(self.textBrowser_log_output)
+
         self.widget_actions = QWidget(self.centralwidget)
         self.widget_actions.setObjectName(u"widget_actions")
         sizePolicy.setHeightForWidth(self.widget_actions.sizePolicy().hasHeightForWidth())
@@ -456,6 +467,7 @@ def retranslateUi(self, MainWindow):
         self.comboBox_output_vertical_reference.setItemText(3, QCoreApplication.translate("MainWindow", u"CGVD28/HT2_2010v70", None))
 
         self.checkBox_epoch_trans.setText(QCoreApplication.translate("MainWindow", u"Epoch Transformation", None))
+        self.label_log_output.setText(QCoreApplication.translate("MainWindow", u"Log Output", None))
         self.pushButton_convert.setText(QCoreApplication.translate("MainWindow", u"Convert", None))
     # retranslateUi
 

las_trx/utils.py

@@ -1,11 +1,15 @@
+import logging
 from datetime import date
 from typing import TypeVar, overload
 
 import pyproj.sync
+
 from csrspy.enums import CoordType, Reference, VerticalDatum
 
 T = TypeVar("T")
 
+logger = logging.getLogger(__name__)
+
 
 @overload
 def date_to_decimal_year(d: date) -> float:
@@ -49,6 +53,9 @@ def sync_missing_grid_files():
     endpoint = pyproj.sync.get_proj_endpoint()
     grids = pyproj.sync.get_transform_grid_list(area_of_use="Canada")
 
+    if len(grids):
+        logger.info("Syncing PROJ grid files.")
+
     for grid in grids:
         filename = grid["properties"]["name"]
         pyproj.sync._download_resource_file(

las_trx/worker.py

@@ -1,22 +1,26 @@
 import copy
+import logging
 import math
 import multiprocessing
+import os
 from concurrent import futures
 from pathlib import Path
 from time import sleep
 
 import laspy
 import numpy as np
 from PySide2.QtCore import QThread, Signal
-from csrspy import CSRSTransformer
 from laspy import LasHeader
 from pyproj import CRS
 
+from csrspy import CSRSTransformer
 from las_trx.config import TransformConfig
 from las_trx.vlr import GeoAsciiParamsVlr, GeoKeyDirectoryVlr
 
 CHUNK_SIZE = 10_000
 
+logger = logging.getLogger(__name__)
+
 
 class TransformWorker(QThread):
     started = Signal()
@@ -33,16 +37,26 @@ def __init__(
         self.input_files = input_files
         self.output_files = output_files
 
+        logger.info(f"Found {len(self.input_files)} input files")
+        logger.info(f"Transform config: {self.config}")
+        logger.info(f"Output CRS\n{self.config.t_crs.to_wkt(pretty=True)}")
+        logger.debug(f"Will read points in chunk size of {CHUNK_SIZE}")
+        logger.info("Calculating total number of iterations")
+
         self.total_iters = 0
         for input_file in self.input_files:
             with laspy.open(input_file) as in_las:
                 self.total_iters += math.ceil(in_las.header.point_count / CHUNK_SIZE)
+        logger.info(f"Total iterations until complete: {self.total_iters}")
 
-        self.pool = futures.ProcessPoolExecutor()
+        num_workers = min(os.cpu_count(), 61)
+        self.pool = futures.ProcessPoolExecutor(max_workers=num_workers)
         self.manager = multiprocessing.Manager()
         self.lock = self.manager.RLock()
         self.current_iter = self.manager.Value("i", 0)
 
+        logger.info(f"CPU process pool size: {num_workers}")
+
     def check_file_names(self):
         for in_file in self.input_files:
             if in_file in self.output_files:
@@ -60,10 +74,10 @@ def check_file_names(self):
 
     def _do_transform(self):
         self.check_file_names()
-
         config = self.config.dict(exclude_none=True)
         futs = []
         for input_file, output_file in zip(self.input_files, self.output_files):
+            logger.info(f"{input_file} -> {output_file}")
             fut = self.pool.submit(
                 transform, config, input_file, output_file, self.lock, self.current_iter
             )
@@ -115,6 +129,8 @@ def transform(
         new_header = write_header_offsets(new_header, input_file, transformer)
 
         laz_backend = laspy.LazBackend.Laszip if output_file.suffix == ".laz" else None
+        logger.debug(f"{laz_backend=}")
+
         with laspy.open(
             output_file, mode="w", header=new_header, laz_backend=laz_backend
         ) as out_las:
@@ -148,6 +164,7 @@ def write_header_offsets(
 
         # Return estimated header offsets as min x,y,z of first batch
         header.offsets = np.min(data, axis=0)
+    logger.debug(f"{header.offsets=}")
     return header
 
 
@@ -164,18 +181,19 @@ def clear_header_geokeys(header: "LasHeader") -> "LasHeader":
             header.vlrs.extract(crs_vlr_name)
         except IndexError:
             pass
-
     return header
 
 
 def write_header_geokeys_from_crs(header: "LasHeader", crs: "CRS") -> "LasHeader":
     header.vlrs.append(GeoAsciiParamsVlr.from_crs(crs))
     header.vlrs.append(GeoKeyDirectoryVlr.from_crs(crs))
+    logger.debug(f"{header.vlrs=}")
     return header
 
 
 def write_header_scales(header: "LasHeader") -> "LasHeader":
     header.scales = np.array([0.01, 0.01, 0.01])
+    logger.debug(f"{header.scales=}")
     return header