Handle lexing exceptions from Sybil

CHANGELOG.rst

@@ -4,8 +4,7 @@ Changelog
 Next
 ----
 
-2024.11.07
-----------
+* Skip files where we hit a lexing error.
 
 2024.11.06.1
 ------------

spelling_private_dict.txt

@@ -12,6 +12,7 @@ dulwich
 formatters
 gzip
 homebrew
+lexing
 linter
 linters
 linting

src/doccmd/__init__.py

@@ -17,6 +17,7 @@
 from pygments.lexers import get_all_lexers
 from sybil import Sybil
 from sybil.evaluators.skip import Skipper
+from sybil.exceptions import LexingException
 from sybil_extras.evaluators.shell_evaluator import ShellCommandEvaluator
 
 from ._languages import UnknownMarkupLanguageError, get_markup_language
@@ -349,6 +350,13 @@ def _run_args_against_docs(
             )
             _log_warning(message=unicode_error_message)
         return
+    except LexingException as exc:
+        lexing_error_message = (
+            f"Skipping '{document_path}' because it could not be lexed: {exc}."
+        )
+        _log_warning(message=lexing_error_message)
+        return
+
     for example in document.examples():
         if (
             verbose

tests/test_doccmd.py

@@ -2147,3 +2147,32 @@ def test_multiple_exclude_patterns(tmp_path: Path) -> None:
 
     assert result.stdout == expected_output
     assert result.stderr == ""
+
+
+def test_lexing_exception(tmp_path: Path) -> None:
+    """
+    Lexing exceptions are handled when an invalid source file is given.
+    """
+    runner = CliRunner(mix_stderr=False)
+    source_file = tmp_path / "invalid_example.md"
+    # Lexing error as there is a hyphen in the comment
+    # or... because of the word code!
+    invalid_content = """\
+    <!-- code -->
+    """
+    source_file.write_text(data=invalid_content, encoding="utf-8")
+    arguments = ["--language", "python", "--command", "cat", str(source_file)]
+    result = runner.invoke(
+        cli=main,
+        args=arguments,
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0, (result.stdout, result.stderr)
+    expected_stderr = textwrap.dedent(
+        text=(
+            f"Skipping '{source_file}' because it could not be lexed: "
+            f"Could not match '(?:(?<=\\n)    )?--+>' in {source_file}:"
+            "\n'    '"
+        ),
+    )
+    assert expected_stderr in result.stderr