[4.0] Support string literals

src/Lexer.php

@@ -57,12 +57,15 @@ class Lexer
     public const STATE_VAR = 2;
     public const STATE_STRING = 3;
     public const STATE_INTERPOLATION = 4;
+    public const STATE_STRING_LITERAL = 5;
 
     public const REGEX_NAME = '/[a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*/A';
     public const REGEX_NUMBER = '/[0-9]+(?:\.[0-9]+)?([Ee][\+\-][0-9]+)?/A';
     public const REGEX_STRING = '/"([^#"\\\\]*(?:\\\\.[^#"\\\\]*)*)"|\'([^\'\\\\]*(?:\\\\.[^\'\\\\]*)*)\'/As';
     public const REGEX_DQ_STRING_DELIM = '/"/A';
     public const REGEX_DQ_STRING_PART = '/[^#"\\\\]*(?:(?:\\\\.|#(?!\{))[^#"\\\\]*)*/As';
+    public const REGEX_STRING_LITERAL_DELIM = '/`/A';
+    public const REGEX_STRING_LITERAL_PART = '/[^`]+/As';
     public const PUNCTUATION = '()[]{}?:.,|';
 
     public function __construct(Environment $env, array $options = [])
@@ -218,6 +221,10 @@ public function tokenize(Source $source): TokenStream
                 case self::STATE_INTERPOLATION:
                     $this->lexInterpolation();
                     break;
+
+                case self::STATE_STRING_LITERAL:
+                    $this->lexStringLiteral();
+                    break;
             }
         }
 
@@ -390,6 +397,12 @@ private function lexExpression(): void
             $this->pushState(self::STATE_STRING);
             $this->moveCursor($match[0]);
         }
+        // opening string literal
+        elseif (preg_match(self::REGEX_STRING_LITERAL_DELIM, $this->code, $match, 0, $this->cursor)) {
+            $this->brackets[] = ['`', $this->lineno];
+            $this->pushState(self::STATE_STRING_LITERAL);
+            $this->moveCursor($match[0]);
+        }
         // unlexable
         else {
             throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
@@ -466,6 +479,25 @@ private function lexInterpolation(): void
         }
     }
 
+    private function lexStringLiteral(): void
+    {
+        if (preg_match(self::REGEX_STRING_LITERAL_PART, $this->code, $match, 0, $this->cursor) && '' !== $match[0]) {
+            $this->pushToken(/* Token::STRING_TYPE */ 7, $match[0]);
+            $this->moveCursor($match[0]);
+        } elseif (preg_match(self::REGEX_STRING_LITERAL_DELIM, $this->code, $match, 0, $this->cursor)) {
+            [$expect, $lineno] = array_pop($this->brackets);
+            if ('`' != $this->code[$this->cursor]) {
+                throw new SyntaxError(\sprintf('Unclosed "%s".', $expect), $lineno, $this->source);
+            }
+
+            $this->popState();
+            ++$this->cursor;
+        } else {
+            // unlexable
+            throw new SyntaxError(\sprintf('Unexpected character "%s".', $this->code[$this->cursor]), $this->lineno, $this->source);
+        }
+    }
+
     private function pushToken($type, $value = ''): void
     {
         // do not push empty text tokens

tests/Fixtures/tests/constant.test

@@ -4,11 +4,13 @@
 {{ 8 is constant('E_NOTICE') ? 'ok' : 'no' }}
 {{ 'bar' is constant('Twig\\Tests\\TwigTestFoo::BAR_NAME') ? 'ok' : 'no' }}
 {{ value is constant('Twig\\Tests\\TwigTestFoo::BAR_NAME') ? 'ok' : 'no' }}
+{{ value is constant(`Twig\Tests\TwigTestFoo::BAR_NAME`) ? 'ok' : 'no' }}
 {{ 2 is constant('ARRAY_AS_PROPS', object) ? 'ok' : 'no' }}
 --DATA--
 return ['value' => 'bar', 'object' => new \ArrayObject(['hi'])]
 --EXPECT--
 ok
 ok
 ok
+ok
 ok

tests/LexerTest.php

@@ -401,4 +401,19 @@ public function getTemplateForErrorsAtTheEndOfTheStream()
         yield ['{{ ='];
         yield ['{{ ..'];
     }
+
+    public function testStringLiterals()
+    {
+        $template = '{{ `My\Name\Space` }}';
+
+        $lexer = new Lexer(new Environment($this->createMock(LoaderInterface::class)));
+        $stream = $lexer->tokenize(new Source($template, 'index'));
+        $stream->expect(Token::VAR_START_TYPE);
+        $stream->expect(Token::STRING_TYPE, 'My\Name\Space');
+        $stream->expect(Token::VAR_END_TYPE);
+
+        // add a dummy assertion here to satisfy PHPUnit, the only thing we want to test is that the code above
+        // can be executed without throwing any exceptions
+        $this->addToAssertionCount(1);
+    }
 }