streaming parser; reporting on last visited tag; maybe fix through no…

…t re-enconding file's contents

simplified, rearranged code
allowing garbage collection
capturing error context and providing it to user
using repr to encode bytes as a string literal
satisfying the formatter's strange desires
passing stack to run_workflow for bubbling up to user
refactoring worker utils to benefit from exceptions unwrapping
clean up

docker-qgis/process_projectfile.py

@@ -1,40 +1,76 @@
+import io
 import logging
 from pathlib import Path
-from typing import Dict
+from typing import NamedTuple, Optional
 from xml.etree import ElementTree
 
-from qfieldcloud.qgis.utils import BaseException, get_layers_data, layers_data_to_string
+from qfieldcloud.qgis.utils import (
+    FailedThumbnailGenerationException,
+    InvalidFileExtensionException,
+    InvalidXmlFileException,
+    ProjectFileNotFoundException,
+    get_layers_data,
+    layers_data_to_string,
+)
 from qgis.core import QgsMapRendererParallelJob, QgsMapSettings, QgsProject
 from qgis.PyQt.QtCore import QEventLoop, QSize
 from qgis.PyQt.QtGui import QColor
 
 logger = logging.getLogger("PROCPRJ")
 
 
-class ProjectFileNotFoundException(BaseException):
-    message = 'Project file "%(project_filename)s" does not exist'
+class XmlLocationError(NamedTuple):
+    line: int
+    column: int
 
+    @classmethod
+    def make(cls, invalid_token_error_msg: str) -> Optional["XmlLocationError"]:
+        """Get column and line numbers from the provided error message."""
+        if "invalid token" not in invalid_token_error_msg.casefold():
+            logger.error("Unable to find 'invalid token' details in the given message")
+            return None
 
-class InvalidFileExtensionException(BaseException):
-    message = (
-        'Project file "%(project_filename)s" has unknown file extension "%(extension)s"'
-    )
+        _, details = invalid_token_error_msg.split(":")
+        line, column = details.split(",")
+        _, line_number = line.strip().split(" ")
+        _, column_number = column.strip().split(" ")
 
+        return cls(int(line_number), int(column_number))
 
-class InvalidXmlFileException(BaseException):
-    message = "Project file is an invalid XML document:\n%(xml_error)s"
 
+class XmlErrorReport(NamedTuple):
+    line_n: int
+    column_n: int
+    char: str
+    line: str
 
-class InvalidQgisFileException(BaseException):
-    message = 'Project file "%(project_filename)s" is invalid QGIS file:\n%(error)s'
+    @classmethod
+    def make(cls, error_msg: str, fh: io.BufferedReader) -> Optional["XmlErrorReport"]:
+        """Get the line where the exception occurred as feedback."""
+        location = XmlLocationError.make(error_msg)
+        if not location:
+            # Unable to fetch the location of the error from the exception message. Abort.
+            return None
 
+        fh.seek(0)
+        for cursor_pos, line in enumerate(fh, start=1):
+            if location.line == cursor_pos:
+                # Avoid decoding the line and faulty character to prevent
+                # further decoding errors in case it is not utf-8 compliant
+                left = location.column - 1
+                right = location.column + 1
 
-class InvalidLayersException(BaseException):
-    message = 'Project file "%(project_filename)s" contains invalid layers'
+                return cls(
+                    line=repr(line.strip()),
+                    char=repr(line[left:right]),
+                    line_n=location.line,
+                    column_n=location.column,
+                )
 
+        return None
 
-class FailedThumbnailGenerationException(BaseException):
-    message = "Failed to generate project thumbnail:\n%(reason)s"
+    def __str__(self) -> str:
+        return f"Unable to parse character on line {self.line_n}, column {self.column_n}: {self.char}. Full line reads: {self.line}."
 
 
 def check_valid_project_file(project_filename: Path) -> None:
@@ -44,14 +80,23 @@ def check_valid_project_file(project_filename: Path) -> None:
         raise ProjectFileNotFoundException(project_filename=project_filename)
 
     if project_filename.suffix == ".qgs":
-        try:
-            with open(project_filename) as f:
-                ElementTree.fromstring(f.read())
-        except ElementTree.ParseError as err:
-            raise InvalidXmlFileException(
-                project_filename=project_filename, xml_error=err
-            )
+        with open(project_filename, "rb") as fh:
+            try:
+                for event, elem in ElementTree.iterparse(fh):
+                    continue
+            except ElementTree.ParseError as error:
+                error_msg = str(error)
+                report = XmlErrorReport.make(error_msg, fh)
+                if report:
+                    xml_error = str(report)
+                else:
+                    xml_error = error_msg
+                raise InvalidXmlFileException(
+                    project_filename=project_filename,
+                    xml_error=xml_error,
+                )
     elif project_filename.suffix != ".qgz":
+        logger.error("QGIS project file is invalid.")
         raise InvalidFileExtensionException(
             project_filename=project_filename, extension=project_filename.suffix
         )
@@ -71,7 +116,7 @@ def load_project_file(project_filename: Path) -> QgsProject:
     return project
 
 
-def extract_project_details(project: QgsProject) -> Dict[str, str]:
+def extract_project_details(project: QgsProject) -> dict[str, str]:
     """Extract project details"""
     logger.info("Extract project details…")
 

docker-qgis/utils.py

@@ -15,7 +15,7 @@
 from contextlib import contextmanager
 from datetime import datetime
 from pathlib import Path
-from typing import IO, Any, Callable, Dict, List, Optional, Union
+from typing import IO, Any, Callable, Optional
 
 from libqfieldsync.layer import LayerSource
 from qfieldcloud_sdk import sdk
@@ -39,6 +39,44 @@
 qgs_msglog_logger.setLevel(logging.DEBUG)
 
 
+class QfcWorkerException(Exception):
+    """QFieldCloud Exception"""
+
+    message = ""
+
+    def __init__(self, message: str = None, **kwargs):
+        self.message = (message or self.message) % kwargs
+        self.details = kwargs
+
+        super().__init__(self.message)
+
+
+class ProjectFileNotFoundException(QfcWorkerException):
+    message = 'Project file "%(project_filename)s" does not exist'
+
+
+class InvalidFileExtensionException(QfcWorkerException):
+    message = (
+        'Project file "%(project_filename)s" has unknown file extension "%(extension)s"'
+    )
+
+
+class InvalidXmlFileException(QfcWorkerException):
+    message = "Project file is an invalid XML document:\n%(xml_error)s"
+
+
+class InvalidQgisFileException(QfcWorkerException):
+    message = 'Project file "%(project_filename)s" is invalid QGIS file:\n%(error)s'
+
+
+class InvalidLayersException(QfcWorkerException):
+    message = 'Project file "%(project_filename)s" contains invalid layers'
+
+
+class FailedThumbnailGenerationException(QfcWorkerException):
+    message = "Failed to generate project thumbnail:\n%(reason)s"
+
+
 def _qt_message_handler(mode, context, message):
     log_level = logging.DEBUG
     if mode == QtCore.QtDebugMsg:
@@ -268,7 +306,7 @@ def __init__(
         id: str,
         version: str,
         name: str,
-        steps: List["Step"],
+        steps: list["Step"],
         description: str = "",
     ):
         self.id = id
@@ -331,9 +369,9 @@ def __init__(
         id: str,
         name: str,
         method: Callable,
-        arguments: Dict[str, Any] = {},
-        return_names: List[str] = [],
-        outputs: List[str] = [],
+        arguments: dict[str, Any] = {},
+        return_names: list[str] = [],
+        outputs: list[str] = [],
     ):
         self.id = id
         self.name = name
@@ -366,18 +404,6 @@ def eval(self, root: Path) -> Path:
         return path
 
 
-class BaseException(Exception):
-    """QFieldCloud Exception"""
-
-    message = ""
-
-    def __init__(self, message: str = None, **kwargs):
-        self.message = (message or self.message) % kwargs
-        self.details = kwargs
-
-        super().__init__(self.message)
-
-
 @contextmanager
 def logger_context(step: Step):
     log_uuid = uuid.uuid4()
@@ -437,7 +463,7 @@ def get_layer_filename(layer: QgsMapLayer) -> Optional[str]:
     return None
 
 
-def extract_project_details(project: QgsProject) -> Dict[str, str]:
+def extract_project_details(project: QgsProject) -> dict[str, str]:
     """Extract project details"""
     map_settings = QgsMapSettings()
     details = {}
@@ -487,8 +513,8 @@ def json_default(obj):
 
 def run_workflow(
     workflow: Workflow,
-    feedback_filename: Optional[Union[IO, Path]],
-) -> Dict:
+    feedback_filename: Optional[IO | Path],
+) -> dict:
     """Executes the steps required to run a task and return structured feedback from the execution
 
     Each step has a method that is executed.
@@ -501,7 +527,7 @@ def run_workflow(
         workflow (Workflow): workflow to be executed
         feedback_filename (Optional[Union[IO, Path]]): write feedback to an IO device, to Path filename, or don't write it
     """
-    feedback: Dict[str, Any] = {
+    feedback: dict[str, Any] = {
         "feedback_version": "2.0",
         "workflow_version": workflow.version,
         "workflow_id": workflow.id,
@@ -552,10 +578,12 @@ def run_workflow(
                 feedback["error_type"] = "API_OTHER"
         elif isinstance(err, FileNotFoundError):
             feedback["error_type"] = "FILE_NOT_FOUND"
+        elif isinstance(err, InvalidXmlFileException):
+            feedback["error_type"] = "INVALID_PROJECT_FILE"
         else:
             feedback["error_type"] = "UNKNOWN"
 
-        (_type, _value, tb) = sys.exc_info()
+        _type, _value, tb = sys.exc_info()
         feedback["error_class"] = type(err).__name__
         feedback["error_stack"] = traceback.format_tb(tb)
     finally:
@@ -596,7 +624,7 @@ def run_workflow(
         return feedback
 
 
-def get_layers_data(project: QgsProject) -> Dict[str, Dict]:
+def get_layers_data(project: QgsProject) -> dict[str, dict]:
     layers_by_id = {}
 
     for layer in project.mapLayers().values():
@@ -711,7 +739,7 @@ def get_file_md5sum(filename: str) -> str:
     return hasher.hexdigest()
 
 
-def files_list_to_string(files: List[Dict[str, Any]]) -> str:
+def files_list_to_string(files: list[dict[str, Any]]) -> str:
     table = [
         [
             d["name"],