Deprecate unnecessary escape characters

See twigphp#4123 twigphp#2712 This allows to change the implementation in v4 so that we no longer have to escape backslashes.
ruudk · Aug 7, 2024 · 703e811 · 703e811
1 parent ea4d706
commit 703e811
Show file tree

Hide file tree

Showing 2 changed files with 142 additions and 15 deletions.
diff --git a/src/Lexer.php b/src/Lexer.php
@@ -367,7 +367,7 @@ private function lexExpression(): void
         }
         // strings
         elseif (preg_match(self::REGEX_STRING, $this->code, $match, 0, $this->cursor)) {
-            $this->pushToken(/* Token::STRING_TYPE */ 7, stripcslashes(substr($match[0], 1, -1)));
+            $this->pushToken(/* Token::STRING_TYPE */ 7, $this->stripcslashes(substr($match[0], 1, -1), substr($match[0], 0, 1)));
             $this->moveCursor($match[0]);
         }
         // opening double quoted string
@@ -382,6 +382,61 @@ private function lexExpression(): void
         }
     }
 
+    private function stripcslashes(string $str, string $quoteType): string {
+        $result = '';
+        $length = strlen($str);
+        $specialChars = [
+            'f' => "\f",
+            'n' => "\n",
+            'r' => "\r",
+            't' => "\t",
+            'v' => "\v",
+        ];
+
+        for ($i = 0; $i < $length; $i++) {
+            if ($str[$i] === '\\' && $i + 1 < $length) {
+                $nextChar = $str[$i + 1];
+                if (isset($specialChars[$nextChar])) {
+                    $result .= $specialChars[$nextChar];
+                    $i++;
+                } elseif ($nextChar === '#') {
+                    $result .= "#";
+                    $i++;
+                } elseif ($nextChar === '\\') {
+                    $result .= '\\';
+                    $i++;
+                } elseif ($nextChar === "'" || $nextChar === '"') {
+                    if ($nextChar !== $quoteType) {
+                        trigger_deprecation('twig/twig', '3.10', "Character %s at position %d does not require to be escaped.", $nextChar, $i + 2);
+                    }
+                    $result .= $nextChar;
+                    $i++;
+                } elseif ($nextChar === 'x' && $i + 2 < $length && ctype_xdigit($str[$i + 1]) && ctype_xdigit($str[$i + 2])) {
+                    $result .= chr(hexdec($str[$i + 1] . $str[$i + 2]));
+                    $i += 2;
+                } elseif (ctype_digit($nextChar) && $nextChar < '8') {
+                    $octal = $nextChar;
+                    for ($j = 1; $j <= 2; $j++) {
+                        if ($i + $j + 1 < $length && ctype_digit($str[$i + $j + 1]) && $str[$i + $j + 1] < '8') {
+                            $octal .= $str[$i + $j + 1];
+                            $i++;
+                        } else {
+                            break;
+                        }
+                    }
+                    $result .= chr(octdec($octal));
+                    $i++;
+                } else {
+                    $result .= $nextChar;
+                    $i++;
+                }
+            } else {
+                $result .= $str[$i];
+            }
+        }
+        return $result;
+    }
+
     private function lexRawData(): void
     {
         if (!preg_match($this->regexes['lex_raw_data'], $this->code, $match, \PREG_OFFSET_CAPTURE, $this->cursor)) {
@@ -423,7 +478,7 @@ private function lexString(): void
             $this->moveCursor($match[0]);
             $this->pushState(self::STATE_INTERPOLATION);
         } elseif (preg_match(self::REGEX_DQ_STRING_PART, $this->code, $match, 0, $this->cursor) && '' !== $match[0]) {
-            $this->pushToken(/* Token::STRING_TYPE */ 7, stripcslashes($match[0]));
+            $this->pushToken(/* Token::STRING_TYPE */ 7, $this->stripcslashes($match[0], '"'));
             $this->moveCursor($match[0]);
         } elseif (preg_match(self::REGEX_DQ_STRING_DELIM, $this->code, $match, 0, $this->cursor)) {
             [$expect, $lineno] = array_pop($this->brackets);

diff --git a/tests/LexerTest.php b/tests/LexerTest.php
@@ -12,6 +12,7 @@
  */
 
 use PHPUnit\Framework\TestCase;
+use Symfony\Bridge\PhpUnit\ExpectDeprecationTrait;
 use Twig\Environment;
 use Twig\Error\SyntaxError;
 use Twig\Lexer;
@@ -21,6 +22,8 @@
 
 class LexerTest extends TestCase
 {
+    use ExpectDeprecationTrait;
+
     public function testNameLabelForTag()
     {
         $template = '{% § %}';
@@ -178,23 +181,92 @@ public function testBigNumbers()
         $this->assertEquals('922337203685477580700', $node->getValue());
     }
 
-    public function testStringWithEscapedDelimiter()
+    /**
+     * @group legacy
+     * @dataProvider getStringWithEscapedDelimiter
+     */
+    public function testStringWithEscapedDelimiter(string $template, string $expected, ?string $expectedDeprecation = null)
     {
-        $tests = [
-            "{{ 'foo \' bar' }}" => 'foo \' bar',
-            '{{ "foo \" bar" }}' => 'foo " bar',
-        ];
+        if ($expectedDeprecation) {
+            $this->expectDeprecation($expectedDeprecation);
+        }
 
         $lexer = new Lexer(new Environment(new ArrayLoader()));
-        foreach ($tests as $template => $expected) {
-            $stream = $lexer->tokenize(new Source($template, 'index'));
-            $stream->expect(Token::VAR_START_TYPE);
-            $stream->expect(Token::STRING_TYPE, $expected);
+        $stream = $lexer->tokenize(new Source($template, 'index'));
+        $stream->expect(Token::VAR_START_TYPE);
+        $stream->expect(Token::STRING_TYPE, $expected);
 
-            // add a dummy assertion here to satisfy PHPUnit, the only thing we want to test is that the code above
-            // can be executed without throwing any exceptions
-            $this->addToAssertionCount(1);
-        }
+        // add a dummy assertion here to satisfy PHPUnit, the only thing we want to test is that the code above
+        // can be executed without throwing any exceptions
+        $this->addToAssertionCount(1);
+    }
+
+    public function getStringWithEscapedDelimiter()
+    {
+        yield '{{ \'App\\\\Test\' }} => App\Test' => [
+            <<<'TWIG'
+{{ 'App\\Test' }}
+TWIG,
+            <<<'EOF'
+App\Test
+EOF,
+        ];
+        yield '{{ \'App\Test\' }} => AppTest' => [
+            <<<'TWIG'
+{{ 'App\Test' }}
+TWIG,
+            <<<'EOF'
+AppTest
+EOF,
+        ];
+        yield '{{ \'foo \\\' bar\' }} => foo \' bar' => [
+            <<<'TWIG'
+{{ 'foo \' bar' }}
+TWIG,
+            <<<'EOF'
+foo ' bar
+EOF,
+        ];
+        yield '{{ "foo \\\' bar" }} => foo \' bar' => [
+            <<<'TWIG'
+{{ "foo \' bar" }}
+TWIG,
+            <<<'EOF'
+foo ' bar
+EOF,
+            "Since twig/twig 3.10: Character ' at position 6 does not require to be escaped.",
+        ];
+        yield '{{ "foo \" bar" }} => foo " bar' => [
+            <<<'TWIG'
+{{ "foo \" bar" }}
+TWIG,
+            <<<'EOF'
+foo " bar
+EOF,
+        ];
+        yield '{{ \'foo \" bar\' }} => foo " bar' => [
+            <<<'TWIG'
+{{ 'foo \" bar' }}
+TWIG,
+            <<<'EOF'
+foo " bar
+EOF,
+            'Since twig/twig 3.10: Character " at position 6 does not require to be escaped.',
+        ];
+        yield '{{ \'\f\n\r\t\v\' }} => \f\n\r\t\v' => [
+            <<<'TWIG'
+{{ '\f\n\r\t\v' }}
+TWIG,
+            "\f\n\r\t\v",
+        ];
+        yield '{{ \'\\\\f\\\\n\\\\r\\\\t\\\\v\' }} => \\f\\n\\r\\t\\v' => [
+            <<<'TWIG'
+{{ '\\f\\n\\r\\t\\v' }}
+TWIG,
+            <<<'EOF'
+\f\n\r\t\v
+EOF,
+        ];
     }
 
     public function testStringWithInterpolation()