From bc073f8948fe09693f5382bca62c7e21e35081fc Mon Sep 17 00:00:00 2001 From: Taylor Denouden Date: Mon, 18 Apr 2022 16:13:31 -0700 Subject: [PATCH 1/2] Add logging display --- las_trx/main.py | 59 ++++++++++++++++++++++++++++++--- las_trx/resources/mainwindow.ui | 18 ++++++++-- las_trx/ui_mainwindow.py | 16 +++++++-- las_trx/utils.py | 7 ++++ las_trx/worker.py | 26 ++++++++++++--- 5 files changed, 114 insertions(+), 12 deletions(-) diff --git a/las_trx/main.py b/las_trx/main.py index f98093a..7943e95 100644 --- a/las_trx/main.py +++ b/las_trx/main.py @@ -1,11 +1,14 @@ +import logging import os.path +import queue import sys from datetime import date from multiprocessing import freeze_support from pathlib import Path +from queue import Queue -from PySide2.QtCore import QSize -from PySide2.QtGui import QIcon +from PySide2.QtCore import QSize, QThread, Signal +from PySide2.QtGui import QIcon, QTextCursor from PySide2.QtWidgets import ( QApplication, QErrorMessage, @@ -13,8 +16,8 @@ QMainWindow, QMessageBox, ) -from csrspy.enums import CoordType, Reference, VerticalDatum +from csrspy.enums import CoordType, Reference, VerticalDatum from las_trx.config import TransformConfig from las_trx.ui_mainwindow import Ui_MainWindow from las_trx.utils import ( @@ -25,6 +28,8 @@ ) from las_trx.worker import TransformWorker +logger = logging.getLogger(__name__) + def resource_path(relative_path: str): """Get absolute path to resource, works for dev and for PyInstaller""" @@ -208,14 +213,22 @@ def output_files(self) -> list[Path]: out_path = self.ui.lineEdit_output_file.text() return [Path(out_path.format(f.stem)) for f in self.input_files] + def append_text(self, text): + self.ui.textBrowser_log_output.moveCursor(QTextCursor.End) + self.ui.textBrowser_log_output.insertPlainText(text) + def on_process_success(self): + logger.info("Processing complete") self.done_msg_box.exec() def on_process_error(self, exception: BaseException): + logger.error(str(exception)) self.err_msg_box.showMessage(str(exception)) self.err_msg_box.exec() def convert(self): + logger.debug("Starting worker thread.") + self.thread = TransformWorker( self.transform_config, self.input_files, self.output_files ) @@ -234,12 +247,50 @@ def convert(self): self.thread.start() +class LogStream(object): + def __init__(self, queue): + super().__init__() + self.queue = queue + + def write(self, text): + self.queue.put(text) + + +class LogThread(QThread): + on_msg = Signal(str) + + def __init__(self, queue: Queue, *args, **kwargs): + super().__init__(*args, **kwargs) + self.queue = queue + + def run(self): + while not self.isInterruptionRequested(): + try: + text = self.queue.get(block=False) + self.on_msg.emit(text) + except queue.Empty: + continue + + if __name__ == "__main__": freeze_support() - app = QApplication(sys.argv) + # Configure logging + log_msg_queue = Queue() + log_write_stream = LogStream(log_msg_queue) + log_handler = logging.StreamHandler(log_write_stream) + + logging.basicConfig(level=logging.INFO, handlers=[log_handler]) + app = QApplication(sys.argv) window = MainWindow() window.show() + # When a new message is written to the log_queue via the log_write_stream, log_thread emits a signal that causes the main + # window to display that msg in the textBrowser + log_thread = LogThread(log_msg_queue) + log_thread.on_msg.connect(window.append_text) + app.aboutToQuit.connect(log_thread.requestInterruption) + log_thread.start() + sys.exit(app.exec_()) diff --git a/las_trx/resources/mainwindow.ui b/las_trx/resources/mainwindow.ui index 0cb2f84..55673fc 100644 --- a/las_trx/resources/mainwindow.ui +++ b/las_trx/resources/mainwindow.ui @@ -7,7 +7,7 @@ 0 0 576 - 690 + 789 @@ -273,7 +273,7 @@ 59 9998 1 - 8 + 9 @@ -680,6 +680,20 @@ + + + + Log Output + + + + + + + QFrame::Sunken + + + diff --git a/las_trx/ui_mainwindow.py b/las_trx/ui_mainwindow.py index 53074ad..f94150d 100644 --- a/las_trx/ui_mainwindow.py +++ b/las_trx/ui_mainwindow.py @@ -17,7 +17,7 @@ class Ui_MainWindow(object): def setupUi(self, MainWindow): if not MainWindow.objectName(): MainWindow.setObjectName(u"MainWindow") - MainWindow.resize(576, 690) + MainWindow.resize(576, 789) self.centralwidget = QWidget(MainWindow) self.centralwidget.setObjectName(u"centralwidget") self.verticalLayout = QVBoxLayout(self.centralwidget) @@ -138,7 +138,7 @@ def setupUi(self, MainWindow): self.dateEdit_input_epoch = QDateEdit(self.widget_input_options) self.dateEdit_input_epoch.setObjectName(u"dateEdit_input_epoch") self.dateEdit_input_epoch.setTime(QTime(8, 0, 0)) - self.dateEdit_input_epoch.setMaximumDateTime(QDateTime(QDate(9998, 1, 8), QTime(7, 59, 59))) + self.dateEdit_input_epoch.setMaximumDateTime(QDateTime(QDate(9998, 1, 9), QTime(7, 59, 59))) self.dateEdit_input_epoch.setDisplayFormat(u"yyyy-MM-dd") self.dateEdit_input_epoch.setCalendarPopup(True) self.dateEdit_input_epoch.setTimeSpec(Qt.UTC) @@ -329,6 +329,17 @@ def setupUi(self, MainWindow): self.verticalLayout.addWidget(self.frame_output) + self.label_log_output = QLabel(self.centralwidget) + self.label_log_output.setObjectName(u"label_log_output") + + self.verticalLayout.addWidget(self.label_log_output) + + self.textBrowser_log_output = QTextBrowser(self.centralwidget) + self.textBrowser_log_output.setObjectName(u"textBrowser_log_output") + self.textBrowser_log_output.setFrameShadow(QFrame.Sunken) + + self.verticalLayout.addWidget(self.textBrowser_log_output) + self.widget_actions = QWidget(self.centralwidget) self.widget_actions.setObjectName(u"widget_actions") sizePolicy.setHeightForWidth(self.widget_actions.sizePolicy().hasHeightForWidth()) @@ -456,6 +467,7 @@ def retranslateUi(self, MainWindow): self.comboBox_output_vertical_reference.setItemText(3, QCoreApplication.translate("MainWindow", u"CGVD28/HT2_2010v70", None)) self.checkBox_epoch_trans.setText(QCoreApplication.translate("MainWindow", u"Epoch Transformation", None)) + self.label_log_output.setText(QCoreApplication.translate("MainWindow", u"Log Output", None)) self.pushButton_convert.setText(QCoreApplication.translate("MainWindow", u"Convert", None)) # retranslateUi diff --git a/las_trx/utils.py b/las_trx/utils.py index d8189d5..33abb82 100644 --- a/las_trx/utils.py +++ b/las_trx/utils.py @@ -1,11 +1,15 @@ +import logging from datetime import date from typing import TypeVar, overload import pyproj.sync + from csrspy.enums import CoordType, Reference, VerticalDatum T = TypeVar("T") +logger = logging.getLogger(__name__) + @overload def date_to_decimal_year(d: date) -> float: @@ -49,6 +53,9 @@ def sync_missing_grid_files(): endpoint = pyproj.sync.get_proj_endpoint() grids = pyproj.sync.get_transform_grid_list(area_of_use="Canada") + if len(grids): + logger.info("Syncing PROJ grid files.") + for grid in grids: filename = grid["properties"]["name"] pyproj.sync._download_resource_file( diff --git a/las_trx/worker.py b/las_trx/worker.py index d1a7a0e..d0c7175 100644 --- a/las_trx/worker.py +++ b/las_trx/worker.py @@ -1,6 +1,8 @@ import copy +import logging import math import multiprocessing +import os from concurrent import futures from pathlib import Path from time import sleep @@ -8,15 +10,17 @@ import laspy import numpy as np from PySide2.QtCore import QThread, Signal -from csrspy import CSRSTransformer from laspy import LasHeader from pyproj import CRS +from csrspy import CSRSTransformer from las_trx.config import TransformConfig from las_trx.vlr import GeoAsciiParamsVlr, GeoKeyDirectoryVlr CHUNK_SIZE = 10_000 +logger = logging.getLogger(__name__) + class TransformWorker(QThread): started = Signal() @@ -33,16 +37,26 @@ def __init__( self.input_files = input_files self.output_files = output_files + logger.info(f"Found {len(self.input_files)} input files") + logger.info(f"Transform config: {self.config}") + logger.info(f"Output CRS\n{self.config.t_crs.to_wkt(pretty=True)}") + logger.debug(f"Will read points in chunk size of {CHUNK_SIZE}") + logger.info("Calculating total number of iterations") + self.total_iters = 0 for input_file in self.input_files: with laspy.open(input_file) as in_las: self.total_iters += math.ceil(in_las.header.point_count / CHUNK_SIZE) + logger.info(f"Total iterations until complete: {self.total_iters}") - self.pool = futures.ProcessPoolExecutor() + num_workers = min(os.cpu_count(), 61) + self.pool = futures.ProcessPoolExecutor(max_workers=num_workers) self.manager = multiprocessing.Manager() self.lock = self.manager.RLock() self.current_iter = self.manager.Value("i", 0) + logger.info(f"CPU process pool size: {num_workers}") + def check_file_names(self): for in_file in self.input_files: if in_file in self.output_files: @@ -60,10 +74,10 @@ def check_file_names(self): def _do_transform(self): self.check_file_names() - config = self.config.dict(exclude_none=True) futs = [] for input_file, output_file in zip(self.input_files, self.output_files): + logger.info(f"{input_file} -> {output_file}") fut = self.pool.submit( transform, config, input_file, output_file, self.lock, self.current_iter ) @@ -115,6 +129,8 @@ def transform( new_header = write_header_offsets(new_header, input_file, transformer) laz_backend = laspy.LazBackend.Laszip if output_file.suffix == ".laz" else None + logger.debug(f"{laz_backend=}") + with laspy.open( output_file, mode="w", header=new_header, laz_backend=laz_backend ) as out_las: @@ -148,6 +164,7 @@ def write_header_offsets( # Return estimated header offsets as min x,y,z of first batch header.offsets = np.min(data, axis=0) + logger.debug(f"{header.offsets=}") return header @@ -164,18 +181,19 @@ def clear_header_geokeys(header: "LasHeader") -> "LasHeader": header.vlrs.extract(crs_vlr_name) except IndexError: pass - return header def write_header_geokeys_from_crs(header: "LasHeader", crs: "CRS") -> "LasHeader": header.vlrs.append(GeoAsciiParamsVlr.from_crs(crs)) header.vlrs.append(GeoKeyDirectoryVlr.from_crs(crs)) + logger.debug(f"{header.vlrs=}") return header def write_header_scales(header: "LasHeader") -> "LasHeader": header.scales = np.array([0.01, 0.01, 0.01]) + logger.debug(f"{header.scales=}") return header From 2908ee27600ffa988e949a6d2a2e9022588e6bf2 Mon Sep 17 00:00:00 2001 From: Taylor Denouden Date: Mon, 18 Apr 2022 16:13:41 -0700 Subject: [PATCH 2/2] update ide configs --- .idea/other.xml | 6 ++++++ .idea/watcherTasks.xml | 25 +++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 .idea/other.xml create mode 100644 .idea/watcherTasks.xml diff --git a/.idea/other.xml b/.idea/other.xml new file mode 100644 index 0000000..a708ec7 --- /dev/null +++ b/.idea/other.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/watcherTasks.xml b/.idea/watcherTasks.xml new file mode 100644 index 0000000..9cf397b --- /dev/null +++ b/.idea/watcherTasks.xml @@ -0,0 +1,25 @@ + + + + + + + + \ No newline at end of file