From 9c4f7d865052bcbd4202f32a8f0f6d05229d1e2b Mon Sep 17 00:00:00 2001
From: deathaxe <deathaxe82@googlemail.com>
Date: Tue, 31 Oct 2023 14:54:29 +0100
Subject: [PATCH] [Python] Fix escaped quotes in raw strings (#3861)

Fixes #3860

Escaped quotation marks do not terminate raw strings, despite escaping being
widely disabled.
---
 Python/Python.sublime-syntax               |  23 ++
 Python/tests/syntax_test_python.py         |  14 +-
 Python/tests/syntax_test_python_strings.py | 262 +++++++++++++++++++++
 3 files changed, 293 insertions(+), 6 deletions(-)

diff --git a/Python/Python.sublime-syntax b/Python/Python.sublime-syntax
index b3f7668f27..d05c8f5901 100644
--- a/Python/Python.sublime-syntax
+++ b/Python/Python.sublime-syntax
@@ -2561,6 +2561,7 @@ contexts:
   double-quoted-raw-docstring-body:
     - meta_content_scope: comment.block.documentation.python
     - include: double-quoted-docstring-end
+    - include: escaped-raw-quotes
 
   single-quoted-docstrings:
     - match: ^\s*(?i)(u)?(''')
@@ -2612,6 +2613,7 @@ contexts:
   single-quoted-raw-docstring-body:
     - meta_content_scope: comment.block.documentation.python
     - include: single-quoted-docstring-end
+    - include: escaped-raw-quotes
 
 ###[ STRINGS ]################################################################
 
@@ -2660,6 +2662,7 @@ contexts:
 
   triple-double-quoted-plain-raw-b-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
 
   triple-double-quoted-raw-b-strings:
     # Triple-quoted raw string, bytes, will use regex
@@ -2698,6 +2701,7 @@ contexts:
 
   triple-double-quoted-plain-raw-f-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
     - include: triple-double-quoted-f-string-replacements
 
   triple-double-quoted-raw-f-strings:
@@ -2738,6 +2742,7 @@ contexts:
 
   triple-double-quoted-plain-raw-u-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
     - include: escaped-unicode-chars
 
   triple-double-quoted-raw-u-strings:
@@ -2786,6 +2791,7 @@ contexts:
 
   triple-double-quoted-sql-raw-u-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
     - include: escaped-unicode-chars
     - include: string-placeholders
     - include: triple-double-quoted-string-replacements
@@ -3001,6 +3007,7 @@ contexts:
 
   double-quoted-plain-raw-b-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
 
   double-quoted-raw-b-strings:
     # Single-line raw string, bytes, treated as regex
@@ -3039,6 +3046,7 @@ contexts:
 
   double-quoted-plain-raw-f-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
     - include: double-quoted-f-string-replacements
 
   double-quoted-raw-f-strings:
@@ -3079,6 +3087,7 @@ contexts:
 
   double-quoted-plain-raw-u-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
 
   double-quoted-raw-u-strings:
     - match: ([uU]?r)(")
@@ -3126,6 +3135,7 @@ contexts:
 
   double-quoted-sql-raw-u-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
     - include: string-placeholders
     - include: double-quoted-string-replacements
 
@@ -3333,6 +3343,7 @@ contexts:
 
   triple-single-quoted-plain-raw-b-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
 
   triple-single-quoted-raw-b-strings:
     # Triple-quoted raw string, bytes, will use regex
@@ -3371,6 +3382,7 @@ contexts:
 
   triple-single-quoted-plain-raw-f-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
     - include: triple-single-quoted-f-string-replacements
 
   triple-single-quoted-raw-f-strings:
@@ -3411,6 +3423,7 @@ contexts:
 
   triple-single-quoted-plain-raw-u-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
 
   triple-single-quoted-raw-u-strings:
     # Triple-quoted raw string, unicode or not, will detect SQL, otherwise regex
@@ -3458,6 +3471,7 @@ contexts:
 
   triple-single-quoted-sql-raw-u-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
     - include: escaped-unicode-chars
     - include: string-placeholders
     - include: triple-single-quoted-string-replacements
@@ -3673,6 +3687,7 @@ contexts:
 
   single-quoted-plain-raw-b-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
 
   single-quoted-raw-b-strings:
     # Single-line raw string, bytes, treated as regex
@@ -3711,6 +3726,7 @@ contexts:
 
   single-quoted-plain-raw-u-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
 
   single-quoted-raw-u-strings:
     - match: ([uU]?r)(')
@@ -3758,6 +3774,7 @@ contexts:
 
   single-quoted-sql-raw-u-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
     - include: string-placeholders
     - include: single-quoted-string-replacements
 
@@ -3777,6 +3794,7 @@ contexts:
 
   single-quoted-plain-raw-f-string-content:
     - include: string-prototype
+    - include: escaped-raw-quotes
     - include: single-quoted-f-string-replacements
 
   single-quoted-raw-f-strings:
@@ -3995,6 +4013,11 @@ contexts:
     - match: \\N\{[-a-zA-Z ]+\}
       scope: constant.character.escape.unicode.name.python
 
+  escaped-raw-quotes:
+    # consume escaped quotes in raw-strings to preven them terminating strings
+    - match: \\['"]
+      scope: constant.character.escape.python
+
   escaped-string-braces:
     - match: \{\{|\}\}
       scope: constant.character.escape.python
diff --git a/Python/tests/syntax_test_python.py b/Python/tests/syntax_test_python.py
index 9348d02a1b..61fcf5fd6c 100644
--- a/Python/tests/syntax_test_python.py
+++ b/Python/tests/syntax_test_python.py
@@ -12,11 +12,12 @@
 """
 # <- comment.block.documentation.python punctuation.definition.comment.end.python
 
-ur"""Raw docstring \"""
+ur"""Raw docstring \""""
 # <- storage.type.string.python - comment
 # ^^^ comment.block.documentation.python punctuation.definition.comment.begin.python
-#    ^^^^^^^^^^^^^^^ comment.block.documentation.summary.python
-#                   ^^^ comment.block.documentation.python punctuation.definition.comment.end.python
+#    ^^^^^^^^^^^^^^^^ comment.block.documentation.summary.python
+#                  ^^ constant.character.escape.python
+#                    ^^^ comment.block.documentation.python punctuation.definition.comment.end.python
 
 R"""
 C:\Users
@@ -87,11 +88,12 @@
 '''
 # <- comment.block.documentation.python punctuation.definition.comment.end.python
 
-ur'''Raw docstring \'''
+ur'''Raw docstring \''''
 # <- storage.type.string.python - comment
 # ^^^ comment.block.documentation.python punctuation.definition.comment.begin.python
-#    ^^^^^^^^^^^^^^^ comment.block.documentation.summary.python
-#                   ^^^ comment.block.documentation.python punctuation.definition.comment.end.python
+#    ^^^^^^^^^^^^^^^^ comment.block.documentation.summary.python
+#                  ^^ constant.character.escape.python
+#                    ^^^ comment.block.documentation.python punctuation.definition.comment.end.python
 
 R'''
 C:\Users
diff --git a/Python/tests/syntax_test_python_strings.py b/Python/tests/syntax_test_python_strings.py
index 0221ea8c9e..a07d2f2fc6 100644
--- a/Python/tests/syntax_test_python_strings.py
+++ b/Python/tests/syntax_test_python_strings.py
@@ -259,8 +259,270 @@
 #                            ^ source.sql comment.line.double-dash
 #                                  ^ punctuation.definition.string.end.python - source.sql
 
+###################################################
+# escaped quotes in raw strings prevent termination
+###################################################
+
+r'''\''''
+#^^^^^^^^ comment.block.documentation
+#   ^^ constant.character.escape.python
+
+r"""\""""
+#^^^^^^^^ comment.block.documentation
+#   ^^ constant.character.escape.python
+
+R'''\''''
+#^^^^^^^^ comment.block.documentation
+#   ^^ constant.character.escape.python
+
+R"""\""""
+#^^^^^^^^ comment.block.documentation
+#   ^^ constant.character.escape.python
+
+raw = r'foo\'' + r'foo\"'
+#      ^^^^^^^ meta.string.python string.quoted.single.python
+#      ^ punctuation.definition.string.begin.python
+#          ^^ constant.character.escape
+#            ^ punctuation.definition.string.end.python
+#                 ^^^^^^^ meta.string.python string.quoted.single.python
+#                 ^ punctuation.definition.string.begin.python
+#                     ^^ constant.character.escape
+#                       ^ punctuation.definition.string.end.python
+
+raw = r"foo\"" + r"foo\'"
+#      ^^^^^^^ meta.string.python string.quoted.double.python
+#      ^ punctuation.definition.string.begin.python
+#          ^^ constant.character.escape
+#            ^ punctuation.definition.string.end.python
+#                 ^^^^^^^ meta.string.python string.quoted.double.python
+#                 ^ punctuation.definition.string.begin.python
+#                     ^^ constant.character.escape
+#                       ^ punctuation.definition.string.end.python
+
+raw = rb'foo\'' + rb'foo\"'
+#       ^^^^^^^ meta.string.python string.quoted.single.python
+#       ^ punctuation.definition.string.begin.python
+#           ^^ constant.character.escape
+#             ^ punctuation.definition.string.end.python
+#                   ^^^^^^^ meta.string.python string.quoted.single.python
+#                   ^ punctuation.definition.string.begin.python
+#                       ^^ constant.character.escape
+#                         ^ punctuation.definition.string.end.python
+
+raw = rb"foo\"" + rb"foo\'"
+#       ^^^^^^^ meta.string.python string.quoted.double.python
+#       ^ punctuation.definition.string.begin.python
+#           ^^ constant.character.escape
+#             ^ punctuation.definition.string.end.python
+#                   ^^^^^^^ meta.string.python string.quoted.double.python
+#                   ^ punctuation.definition.string.begin.python
+#                       ^^ constant.character.escape
+#                         ^ punctuation.definition.string.end.python
+
+raw = rf'foo\'' + rf'foo\"'
+#       ^^^^^^^ meta.string.python string.quoted.single.python
+#       ^ punctuation.definition.string.begin.python
+#           ^^ constant.character.escape
+#             ^ punctuation.definition.string.end.python
+#                   ^^^^^^^ meta.string.python string.quoted.single.python
+#                   ^ punctuation.definition.string.begin.python
+#                       ^^ constant.character.escape
+#                         ^ punctuation.definition.string.end.python
+
+raw = rf"foo\"" + rf"foo\'"
+#       ^^^^^^^ meta.string.python string.quoted.double.python
+#       ^ punctuation.definition.string.begin.python
+#           ^^ constant.character.escape
+#             ^ punctuation.definition.string.end.python
+#                   ^^^^^^^ meta.string.python string.quoted.double.python
+#                   ^ punctuation.definition.string.begin.python
+#                       ^^ constant.character.escape
+#                         ^ punctuation.definition.string.end.python
+
+raw = R'foo\'' + R'foo\"'
+#      ^^^^^^^ meta.string.python string.quoted.single.python
+#      ^ punctuation.definition.string.begin.python
+#          ^^ constant.character.escape.python
+#            ^ punctuation.definition.string.end.python
+#                 ^^^^^^^ meta.string.python string.quoted.single.python
+#                 ^ punctuation.definition.string.begin.python
+#                     ^^ constant.character.escape.python
+#                       ^ punctuation.definition.string.end.python
+
+raw = R"foo\"" + R"foo\'"
+#      ^^^^^^^ meta.string.python string.quoted.double.python
+#      ^ punctuation.definition.string.begin.python
+#          ^^ constant.character.escape.python
+#            ^ punctuation.definition.string.end.python
+#                 ^^^^^^^ meta.string.python string.quoted.double.python
+#                 ^ punctuation.definition.string.begin.python
+#                     ^^ constant.character.escape.python
+#                       ^ punctuation.definition.string.end.python
+
+raw = RB'foo\'' + RB'foo\"'
+#       ^^^^^^^ meta.string.python string.quoted.single.python
+#       ^ punctuation.definition.string.begin.python
+#           ^^ constant.character.escape.python
+#             ^ punctuation.definition.string.end.python
+#                   ^^^^^^^ meta.string.python string.quoted.single.python
+#                   ^ punctuation.definition.string.begin.python
+#                       ^^ constant.character.escape.python
+#                         ^ punctuation.definition.string.end.python
+
+raw = RB"foo\"" + RB"foo\'"
+#       ^^^^^^^ meta.string.python string.quoted.double.python
+#       ^ punctuation.definition.string.begin.python
+#           ^^ constant.character.escape.python
+#             ^ punctuation.definition.string.end.python
+#                   ^^^^^^^ meta.string.python string.quoted.double.python
+#                   ^ punctuation.definition.string.begin.python
+#                       ^^ constant.character.escape.python
+#                         ^ punctuation.definition.string.end.python
+
+raw = RF'foo\'' + RF'foo\"'
+#       ^^^^^^^ meta.string.python string.quoted.single.python
+#       ^ punctuation.definition.string.begin.python
+#           ^^ constant.character.escape.python
+#             ^ punctuation.definition.string.end.python
+#                   ^^^^^^^ meta.string.python string.quoted.single.python
+#                   ^ punctuation.definition.string.begin.python
+#                       ^^ constant.character.escape.python
+#                         ^ punctuation.definition.string.end.python
+
+raw = RF"foo\"" + RF"foo\'"
+#       ^^^^^^^ meta.string.python string.quoted.double.python
+#       ^ punctuation.definition.string.begin.python
+#           ^^ constant.character.escape.python
+#             ^ punctuation.definition.string.end.python
+#                   ^^^^^^^ meta.string.python string.quoted.double.python
+#                   ^ punctuation.definition.string.begin.python
+#                       ^^ constant.character.escape.python
+#                         ^ punctuation.definition.string.end.python
+
+raw = r'''foo\'''' + r'''foo\"'''
+#      ^^^^^^^^^^^ meta.string.python string.quoted.single.block.python
+#      ^ punctuation.definition.string.begin.python
+#            ^^ constant.character.escape
+#              ^^^ punctuation.definition.string.end.python
+#                     ^^^^^^^^^^^ meta.string.python string.quoted.single.block.python
+#                     ^^^ punctuation.definition.string.begin.python
+#                           ^^ constant.character.escape
+#                             ^^^ punctuation.definition.string.end.python
+
+raw = r"""foo\"""" + r"""foo\'"""
+#      ^^^^^^^^^^^ meta.string.python string.quoted.double.block.python
+#      ^ punctuation.definition.string.begin.python
+#            ^^ constant.character.escape
+#              ^^^ punctuation.definition.string.end.python
+#                     ^^^^^^^^^^^ meta.string.python string.quoted.double.block.python
+#                     ^^^ punctuation.definition.string.begin.python
+#                           ^^ constant.character.escape
+#                             ^^^ punctuation.definition.string.end.python
+
+raw = rb'''foo\'''' + rb'''foo\"'''
+#       ^^^^^^^^^^^ meta.string.python string.quoted.single.block.python
+#       ^ punctuation.definition.string.begin.python
+#             ^^ constant.character.escape
+#               ^^^ punctuation.definition.string.end.python
+#                       ^^^^^^^^^^^ meta.string.python string.quoted.single.block.python
+#                       ^^^ punctuation.definition.string.begin.python
+#                             ^^ constant.character.escape
+#                               ^^^ punctuation.definition.string.end.python
+
+raw = rb"""foo\"""" + rb"""foo\'"""
+#       ^^^^^^^^^^^ meta.string.python string.quoted.double.block.python
+#       ^ punctuation.definition.string.begin.python
+#             ^^ constant.character.escape
+#               ^^^ punctuation.definition.string.end.python
+#                       ^^^^^^^^^^^ meta.string.python string.quoted.double.block.python
+#                       ^^^ punctuation.definition.string.begin.python
+#                             ^^ constant.character.escape
+#                               ^^^ punctuation.definition.string.end.python
+
+raw = rf'''foo\'''' + rf'''foo\"'''
+#       ^^^^^^^^^^^ meta.string.python string.quoted.single.block.python
+#       ^ punctuation.definition.string.begin.python
+#             ^^ constant.character.escape
+#               ^^^ punctuation.definition.string.end.python
+#                       ^^^^^^^^^^^ meta.string.python string.quoted.single.block.python
+#                       ^^^ punctuation.definition.string.begin.python
+#                             ^^ constant.character.escape
+#                               ^^^ punctuation.definition.string.end.python
+
+raw = rf"""foo\"""" + rf"""foo\'"""
+#       ^^^^^^^^^^^ meta.string.python string.quoted.double.block.python
+#       ^ punctuation.definition.string.begin.python
+#             ^^ constant.character.escape
+#               ^^^ punctuation.definition.string.end.python
+#                       ^^^^^^^^^^^ meta.string.python string.quoted.double.block.python
+#                       ^^^ punctuation.definition.string.begin.python
+#                             ^^ constant.character.escape
+#                               ^^^ punctuation.definition.string.end.python
+
+raw = R'''foo\'''' + R'''foo\"'''
+#      ^^^^^^^^^^^ meta.string.python string.quoted.single.block.python
+#      ^ punctuation.definition.string.begin.python
+#            ^^ constant.character.escape.python
+#              ^^^ punctuation.definition.string.end.python
+#                     ^^^^^^^^^^^ meta.string.python string.quoted.single.block.python
+#                     ^^^ punctuation.definition.string.begin.python
+#                           ^^ constant.character.escape.python
+#                             ^^^ punctuation.definition.string.end.python
+
+raw = R"""foo\"""" + R"""foo\'"""
+#      ^^^^^^^^^^^ meta.string.python string.quoted.double.block.python
+#      ^ punctuation.definition.string.begin.python
+#            ^^ constant.character.escape.python
+#              ^^^ punctuation.definition.string.end.python
+#                     ^^^^^^^^^^^ meta.string.python string.quoted.double.block.python
+#                     ^^^ punctuation.definition.string.begin.python
+#                           ^^ constant.character.escape.python
+#                             ^^^ punctuation.definition.string.end.python
+
+raw = RB'''foo\'''' + RB'''foo\"'''
+#       ^^^^^^^^^^^ meta.string.python string.quoted.single.block.python
+#       ^ punctuation.definition.string.begin.python
+#             ^^ constant.character.escape.python
+#               ^^^ punctuation.definition.string.end.python
+#                       ^^^^^^^^^^^ meta.string.python string.quoted.single.block.python
+#                       ^^^ punctuation.definition.string.begin.python
+#                             ^^ constant.character.escape.python
+#                               ^^^ punctuation.definition.string.end.python
+
+raw = RB"""foo\"""" + RB"""foo\'"""
+#       ^^^^^^^^^^^ meta.string.python string.quoted.double.block.python
+#       ^ punctuation.definition.string.begin.python
+#             ^^ constant.character.escape.python
+#               ^^^ punctuation.definition.string.end.python
+#                       ^^^^^^^^^^^ meta.string.python string.quoted.double.block.python
+#                       ^^^ punctuation.definition.string.begin.python
+#                             ^^ constant.character.escape.python
+#                               ^^^ punctuation.definition.string.end.python
+
+raw = RF'''foo\'''' + RF'''foo\"'''
+#       ^^^^^^^^^^^ meta.string.python string.quoted.single.block.python
+#       ^ punctuation.definition.string.begin.python
+#             ^^ constant.character.escape.python
+#               ^^^ punctuation.definition.string.end.python
+#                       ^^^^^^^^^^^ meta.string.python string.quoted.single.block.python
+#                       ^^^ punctuation.definition.string.begin.python
+#                             ^^ constant.character.escape.python
+#                               ^^^ punctuation.definition.string.end.python
+
+raw = RF"""foo\"""" + RF"""foo\'"""
+#       ^^^^^^^^^^^ meta.string.python string.quoted.double.block.python
+#       ^ punctuation.definition.string.begin.python
+#             ^^ constant.character.escape.python
+#               ^^^ punctuation.definition.string.end.python
+#                       ^^^^^^^^^^^ meta.string.python string.quoted.double.block.python
+#                       ^^^ punctuation.definition.string.begin.python
+#                             ^^ constant.character.escape.python
+#                               ^^^ punctuation.definition.string.end.python
 
+###################################################
 # There are many variations of making a byte string
+###################################################
+
 (b'', B'', br'', bR'', BR'', Br'', rb'', Rb'', RB'', rB'')
 #^ storage.type.string
 #     ^ storage.type.string