From 8fa6c5f7163dd2b29cf84e70ac83f7443514bd66 Mon Sep 17 00:00:00 2001 From: why-not-try-calmer Date: Mon, 28 Aug 2023 14:42:15 +0200 Subject: [PATCH] streaming parser; reporting on last visited tag; maybe fix through not re-enconding file's contents simplified, rearranged code allowing garbage collection capturing error context and providing it to user using repr to encode bytes as a string literal satisfying the formatter's strange desires passing stack to run_workflow for bubbling up to user refactoring worker utils to benefit from exceptions unwrapping clean up --- docker-qgis/process_projectfile.py | 93 ++++++++++++++++++++++-------- docker-qgis/utils.py | 76 ++++++++++++++++-------- 2 files changed, 121 insertions(+), 48 deletions(-) diff --git a/docker-qgis/process_projectfile.py b/docker-qgis/process_projectfile.py index 3a4e2a358..a1f0c0c9b 100644 --- a/docker-qgis/process_projectfile.py +++ b/docker-qgis/process_projectfile.py @@ -1,9 +1,17 @@ +import io import logging from pathlib import Path -from typing import Dict +from typing import NamedTuple, Optional from xml.etree import ElementTree -from qfieldcloud.qgis.utils import BaseException, get_layers_data, layers_data_to_string +from qfieldcloud.qgis.utils import ( + FailedThumbnailGenerationException, + InvalidFileExtensionException, + InvalidXmlFileException, + ProjectFileNotFoundException, + get_layers_data, + layers_data_to_string, +) from qgis.core import QgsMapRendererParallelJob, QgsMapSettings, QgsProject from qgis.PyQt.QtCore import QEventLoop, QSize from qgis.PyQt.QtGui import QColor @@ -11,30 +19,58 @@ logger = logging.getLogger("PROCPRJ") -class ProjectFileNotFoundException(BaseException): - message = 'Project file "%(project_filename)s" does not exist' +class XmlLocationError(NamedTuple): + line: int + column: int + @classmethod + def make(cls, invalid_token_error_msg: str) -> Optional["XmlLocationError"]: + """Get column and line numbers from the provided error message.""" + if "invalid token" not in invalid_token_error_msg.casefold(): + logger.error("Unable to find 'invalid token' details in the given message") + return None -class InvalidFileExtensionException(BaseException): - message = ( - 'Project file "%(project_filename)s" has unknown file extension "%(extension)s"' - ) + _, details = invalid_token_error_msg.split(":") + line, column = details.split(",") + _, line_number = line.strip().split(" ") + _, column_number = column.strip().split(" ") + return cls(int(line_number), int(column_number)) -class InvalidXmlFileException(BaseException): - message = "Project file is an invalid XML document:\n%(xml_error)s" +class XmlErrorReport(NamedTuple): + line_n: int + column_n: int + char: str + line: str -class InvalidQgisFileException(BaseException): - message = 'Project file "%(project_filename)s" is invalid QGIS file:\n%(error)s' + @classmethod + def make(cls, error_msg: str, fh: io.BufferedReader) -> Optional["XmlErrorReport"]: + """Get the line where the exception occurred as feedback.""" + location = XmlLocationError.make(error_msg) + if not location: + # Unable to fetch the location of the error from the exception message. Abort. + return None + fh.seek(0) + for cursor_pos, line in enumerate(fh, start=1): + if location.line == cursor_pos: + # Avoid decoding the line and faulty character to prevent + # further decoding errors in case it is not utf-8 compliant + left = location.column - 1 + right = location.column + 1 -class InvalidLayersException(BaseException): - message = 'Project file "%(project_filename)s" contains invalid layers' + return cls( + line=repr(line.strip()), + char=repr(line[left:right]), + line_n=location.line, + column_n=location.column, + ) + return None -class FailedThumbnailGenerationException(BaseException): - message = "Failed to generate project thumbnail:\n%(reason)s" + def __str__(self) -> str: + return f"Unable to parse character on line {self.line_n}, column {self.column_n}: {self.char}. Full line reads: {self.line}." def check_valid_project_file(project_filename: Path) -> None: @@ -44,14 +80,23 @@ def check_valid_project_file(project_filename: Path) -> None: raise ProjectFileNotFoundException(project_filename=project_filename) if project_filename.suffix == ".qgs": - try: - with open(project_filename) as f: - ElementTree.fromstring(f.read()) - except ElementTree.ParseError as err: - raise InvalidXmlFileException( - project_filename=project_filename, xml_error=err - ) + with open(project_filename, "rb") as fh: + try: + for event, elem in ElementTree.iterparse(fh): + continue + except ElementTree.ParseError as error: + error_msg = str(error) + report = XmlErrorReport.make(error_msg, fh) + if report: + xml_error = str(report) + else: + xml_error = error_msg + raise InvalidXmlFileException( + project_filename=project_filename, + xml_error=xml_error, + ) elif project_filename.suffix != ".qgz": + logger.error("QGIS project file is invalid.") raise InvalidFileExtensionException( project_filename=project_filename, extension=project_filename.suffix ) @@ -71,7 +116,7 @@ def load_project_file(project_filename: Path) -> QgsProject: return project -def extract_project_details(project: QgsProject) -> Dict[str, str]: +def extract_project_details(project: QgsProject) -> dict[str, str]: """Extract project details""" logger.info("Extract project details…") diff --git a/docker-qgis/utils.py b/docker-qgis/utils.py index d8e5c9aac..6644f9833 100644 --- a/docker-qgis/utils.py +++ b/docker-qgis/utils.py @@ -15,7 +15,7 @@ from contextlib import contextmanager from datetime import datetime from pathlib import Path -from typing import IO, Any, Callable, Dict, List, Optional, Union +from typing import IO, Any, Callable, Optional from libqfieldsync.layer import LayerSource from qfieldcloud_sdk import sdk @@ -39,6 +39,44 @@ qgs_msglog_logger.setLevel(logging.DEBUG) +class QfcWorkerException(Exception): + """QFieldCloud Exception""" + + message = "" + + def __init__(self, message: str = None, **kwargs): + self.message = (message or self.message) % kwargs + self.details = kwargs + + super().__init__(self.message) + + +class ProjectFileNotFoundException(QfcWorkerException): + message = 'Project file "%(project_filename)s" does not exist' + + +class InvalidFileExtensionException(QfcWorkerException): + message = ( + 'Project file "%(project_filename)s" has unknown file extension "%(extension)s"' + ) + + +class InvalidXmlFileException(QfcWorkerException): + message = "Project file is an invalid XML document:\n%(xml_error)s" + + +class InvalidQgisFileException(QfcWorkerException): + message = 'Project file "%(project_filename)s" is invalid QGIS file:\n%(error)s' + + +class InvalidLayersException(QfcWorkerException): + message = 'Project file "%(project_filename)s" contains invalid layers' + + +class FailedThumbnailGenerationException(QfcWorkerException): + message = "Failed to generate project thumbnail:\n%(reason)s" + + def _qt_message_handler(mode, context, message): log_level = logging.DEBUG if mode == QtCore.QtDebugMsg: @@ -268,7 +306,7 @@ def __init__( id: str, version: str, name: str, - steps: List["Step"], + steps: list["Step"], description: str = "", ): self.id = id @@ -331,9 +369,9 @@ def __init__( id: str, name: str, method: Callable, - arguments: Dict[str, Any] = {}, - return_names: List[str] = [], - outputs: List[str] = [], + arguments: dict[str, Any] = {}, + return_names: list[str] = [], + outputs: list[str] = [], ): self.id = id self.name = name @@ -366,18 +404,6 @@ def eval(self, root: Path) -> Path: return path -class BaseException(Exception): - """QFieldCloud Exception""" - - message = "" - - def __init__(self, message: str = None, **kwargs): - self.message = (message or self.message) % kwargs - self.details = kwargs - - super().__init__(self.message) - - @contextmanager def logger_context(step: Step): log_uuid = uuid.uuid4() @@ -437,7 +463,7 @@ def get_layer_filename(layer: QgsMapLayer) -> Optional[str]: return None -def extract_project_details(project: QgsProject) -> Dict[str, str]: +def extract_project_details(project: QgsProject) -> dict[str, str]: """Extract project details""" map_settings = QgsMapSettings() details = {} @@ -487,8 +513,8 @@ def json_default(obj): def run_workflow( workflow: Workflow, - feedback_filename: Optional[Union[IO, Path]], -) -> Dict: + feedback_filename: Optional[IO | Path], +) -> dict: """Executes the steps required to run a task and return structured feedback from the execution Each step has a method that is executed. @@ -501,7 +527,7 @@ def run_workflow( workflow (Workflow): workflow to be executed feedback_filename (Optional[Union[IO, Path]]): write feedback to an IO device, to Path filename, or don't write it """ - feedback: Dict[str, Any] = { + feedback: dict[str, Any] = { "feedback_version": "2.0", "workflow_version": workflow.version, "workflow_id": workflow.id, @@ -552,10 +578,12 @@ def run_workflow( feedback["error_type"] = "API_OTHER" elif isinstance(err, FileNotFoundError): feedback["error_type"] = "FILE_NOT_FOUND" + elif isinstance(err, InvalidXmlFileException): + feedback["error_type"] = "INVALID_PROJECT_FILE" else: feedback["error_type"] = "UNKNOWN" - (_type, _value, tb) = sys.exc_info() + _type, _value, tb = sys.exc_info() feedback["error_class"] = type(err).__name__ feedback["error_stack"] = traceback.format_tb(tb) finally: @@ -596,7 +624,7 @@ def run_workflow( return feedback -def get_layers_data(project: QgsProject) -> Dict[str, Dict]: +def get_layers_data(project: QgsProject) -> dict[str, dict]: layers_by_id = {} for layer in project.mapLayers().values(): @@ -711,7 +739,7 @@ def get_file_md5sum(filename: str) -> str: return hasher.hexdigest() -def files_list_to_string(files: List[Dict[str, Any]]) -> str: +def files_list_to_string(files: list[dict[str, Any]]) -> str: table = [ [ d["name"],