feature #4543 Move operator definitions to objects (fabpot)

This PR was squashed before being merged into the 3.x branch.

Discussion
----------

Move operator definitions to objects

Commits
-------

79d828b Fix version
251c0b5 Fix CS
561c662 Remove obsolete code
4b57f48 Use generics in ExpressionParsers
8ff1909 Fix precedence rules
0cb5c0e Add deprecation notices in CHANGELOG and docs
f67078b Move Operators to ExpressionParsers, deprecate ExpressionParser
9334a70 Add a script to update operator precedence documentation
42c82e1 Extract operators logic from ExpressionParser to their own classes
df877a1 Introduce operator classes to describe operators provided by extensions instead of arrays

.gitattributes

@@ -1,4 +1,5 @@
 /.github/ export-ignore
+/bin/ export-ignore
 /doc/ export-ignore
 /extra/ export-ignore
 /tests/ export-ignore

CHANGELOG

@@ -1,6 +1,11 @@
 # 3.21.0 (2025-XX-XX)
 
- * n/a
+ * Deprecate using the `|` operator in an expression with `+` or `-` without using parentheses to clarify precedence
+ * Deprecate operator precedence outside of the [0, 512] range
+ * Introduce expression parser classes to describe operators and operands provided by extensions
+   instead of arrays (it comes with many deprecations that are documented in
+   the ``deprecated`` documentation chapter)
+ * Deprecate the `Twig\ExpressionParser`, and `Twig\OperatorPrecedenceChange` classes
 
 # 3.20.0 (2025-02-13)
 

bin/generate_operators_precedence.php

@@ -0,0 +1,89 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Twig\Environment;
+use Twig\ExpressionParser\ExpressionParserDescriptionInterface;
+use Twig\ExpressionParser\ExpressionParserType;
+use Twig\ExpressionParser\InfixAssociativity;
+use Twig\ExpressionParser\InfixExpressionParserInterface;
+use Twig\Loader\ArrayLoader;
+
+require_once \dirname(__DIR__).'/vendor/autoload.php';
+
+$output = fopen(\dirname(__DIR__).'/doc/operators_precedence.rst', 'w');
+
+$twig = new Environment(new ArrayLoader([]));
+$expressionParsers = [];
+foreach ($twig->getExpressionParsers() as $expressionParser) {
+    $expressionParsers[] = $expressionParser;
+}
+
+fwrite($output, "\n=========== ================ ======= ============= ===========\n");
+fwrite($output, "Precedence  Operator         Type    Associativity Description\n");
+fwrite($output, '=========== ================ ======= ============= ===========');
+
+usort($expressionParsers, fn ($a, $b) => $b->getPrecedence() <=> $a->getPrecedence());
+
+$previous = null;
+foreach ($expressionParsers as $expressionParser) {
+    $precedence = $expressionParser->getPrecedence();
+    $previousPrecedence = $previous ? $previous->getPrecedence() : \PHP_INT_MAX;
+    $associativity = $expressionParser instanceof InfixExpressionParserInterface ? (InfixAssociativity::Left === $expressionParser->getAssociativity() ? 'Left' : 'Right') : 'n/a';
+    $previousAssociativity = $previous ? ($previous instanceof InfixExpressionParserInterface ? (InfixAssociativity::Left === $previous->getAssociativity() ? 'Left' : 'Right') : 'n/a') : 'n/a';
+    if ($previousPrecedence !== $precedence) {
+        $previous = null;
+    }
+    fwrite($output, rtrim(\sprintf("\n%-11s %-16s %-7s %-13s %s\n",
+        (!$previous || $previousPrecedence !== $precedence ? $precedence : '').($expressionParser->getPrecedenceChange() ? ' => '.$expressionParser->getPrecedenceChange()->getNewPrecedence() : ''),
+        '``'.$expressionParser->getName().'``',
+        !$previous || ExpressionParserType::getType($previous) !== ExpressionParserType::getType($expressionParser) ? ExpressionParserType::getType($expressionParser)->value : '',
+        !$previous || $previousAssociativity !== $associativity ? $associativity : '',
+        $expressionParser instanceof ExpressionParserDescriptionInterface ? $expressionParser->getDescription() : '',
+    )));
+    $previous = $expressionParser;
+}
+fwrite($output, "\n=========== ================ ======= ============= ===========\n");
+fwrite($output, "\nWhen a precedence will change in 4.0, the new precedence is indicated by the arrow ``=>``.\n");
+
+fwrite($output, "\nHere is the same table for Twig 4.0 with adjusted precedences:\n");
+
+fwrite($output, "\n=========== ================ ======= ============= ===========\n");
+fwrite($output, "Precedence  Operator         Type    Associativity Description\n");
+fwrite($output, '=========== ================ ======= ============= ===========');
+
+usort($expressionParsers, function ($a, $b) {
+    $aPrecedence = $a->getPrecedenceChange() ? $a->getPrecedenceChange()->getNewPrecedence() : $a->getPrecedence();
+    $bPrecedence = $b->getPrecedenceChange() ? $b->getPrecedenceChange()->getNewPrecedence() : $b->getPrecedence();
+
+    return $bPrecedence - $aPrecedence;
+});
+
+$previous = null;
+foreach ($expressionParsers as $expressionParser) {
+    $precedence = $expressionParser->getPrecedenceChange() ? $expressionParser->getPrecedenceChange()->getNewPrecedence() : $expressionParser->getPrecedence();
+    $previousPrecedence = $previous ? ($previous->getPrecedenceChange() ? $previous->getPrecedenceChange()->getNewPrecedence() : $previous->getPrecedence()) : \PHP_INT_MAX;
+    $associativity = $expressionParser instanceof InfixExpressionParserInterface ? (InfixAssociativity::Left === $expressionParser->getAssociativity() ? 'Left' : 'Right') : 'n/a';
+    $previousAssociativity = $previous ? ($previous instanceof InfixExpressionParserInterface ? (InfixAssociativity::Left === $previous->getAssociativity() ? 'Left' : 'Right') : 'n/a') : 'n/a';
+    if ($previousPrecedence !== $precedence) {
+        $previous = null;
+    }
+    fwrite($output, rtrim(\sprintf("\n%-11s %-16s %-7s %-13s %s\n",
+        !$previous || $previousPrecedence !== $precedence ? $precedence : '',
+        '``'.$expressionParser->getName().'``',
+        !$previous || ExpressionParserType::getType($previous) !== ExpressionParserType::getType($expressionParser) ? ExpressionParserType::getType($expressionParser)->value : '',
+        !$previous || $previousAssociativity !== $associativity ? $associativity : '',
+        $expressionParser instanceof ExpressionParserDescriptionInterface ? $expressionParser->getDescription() : '',
+    )));
+    $previous = $expressionParser;
+}
+fwrite($output, "\n=========== ================ ======= ============= ===========\n");
+
+fclose($output);

doc/advanced.rst

@@ -775,26 +775,9 @@ responsible for parsing the tag and compiling it to PHP.
 Operators
 ~~~~~~~~~
 
-The ``getOperators()`` methods lets you add new operators. Here is how to add
-the ``!``, ``||``, and ``&&`` operators::
-
-    class CustomTwigExtension extends \Twig\Extension\AbstractExtension
-    {
-        public function getOperators()
-        {
-            return [
-                [
-                    '!' => ['precedence' => 50, 'class' => \Twig\Node\Expression\Unary\NotUnary::class],
-                ],
-                [
-                    '||' => ['precedence' => 10, 'class' => \Twig\Node\Expression\Binary\OrBinary::class, 'associativity' => \Twig\ExpressionParser::OPERATOR_LEFT],
-                    '&&' => ['precedence' => 15, 'class' => \Twig\Node\Expression\Binary\AndBinary::class, 'associativity' => \Twig\ExpressionParser::OPERATOR_LEFT],
-                ],
-            ];
-        }
-
-        // ...
-    }
+The ``getOperators()`` methods lets you add new operators. To implement a new
+one, have a look at the default operators provided by
+``Twig\Extension\CoreExtension``.
 
 Tests
 ~~~~~

doc/deprecated.rst

@@ -210,27 +210,35 @@ Node Visitors
 Parser
 ------
 
-* Passing a second argument to ``ExpressionParser::parseFilterExpressionRaw()``
-  is deprecated as of Twig 3.12.
-
 * The following methods from ``Twig\Parser`` are deprecated as of Twig 3.12:
   ``getBlockStack()``, ``hasBlock()``, ``getBlock()``, ``hasMacro()``,
   ``hasTraits()``, ``getParent()``.
 
-* The ``Twig\ExpressionParser::parseHashExpression()`` method is deprecated, use
-  ``Twig\ExpressionParser::parseMappingExpression()`` instead.
-
-* The ``Twig\ExpressionParser::parseArrayExpression()`` method is deprecated, use
-  ``Twig\ExpressionParser::parseSequenceExpression()`` instead.
-
 * Passing ``null`` to ``Twig\Parser::setParent()`` is deprecated as of Twig
   3.12.
 
-* The ``Twig\ExpressionParser::parseOnlyArguments()`` and
-  ``Twig\ExpressionParser::parseArguments()`` methods are deprecated, use
-  ``Twig\ExpressionParser::parseNamedArguments()`` instead.
-
-Lexer
+* The ``Twig\Parser::getExpressionParser()`` method is deprecated as of Twig
+  3.21, use ``Twig\Parser::parseExpression()`` instead.
+
+* The ``Twig\ExpressionParser`` class is deprecated as of Twig 3.21:
+
+  * ``parseExpression()``, use ``Parser::parseExpression()``
+  * ``parsePrimaryExpression()``, use ``Parser::parseExpression()``
+  * ``parseStringExpression()``, use ``Parser::parseExpression()``
+  * ``parseHashExpression()``, use ``Parser::parseExpression()``
+  * ``parseMappingExpression()``, use ``Parser::parseExpression()``
+  * ``parseArrayExpression()``, use ``Parser::parseExpression()``
+  * ``parseSequenceExpression()``, use ``Parser::parseExpression()``
+  * ``parsePostfixExpression``
+  * ``parseSubscriptExpression``
+  * ``parseFilterExpression``
+  * ``parseFilterExpressionRaw``
+  * ``parseArguments()``, use ``Twig\ExpressionParser\Infix\ArgumentsTrait::parseNamedArguments()``
+  * ``parseAssignmentExpression``, use ``AbstractTokenParser::parseAssignmentExpression``
+  * ``parseMultitargetExpression``
+  * ``parseOnlyArguments()``, use ``Twig\ExpressionParser\Infix\ArgumentsTrait::parseNamedArguments()``
+
+Token
 -----
 
 * Not passing a ``Source`` instance to ``Twig\TokenStream`` constructor is
@@ -239,6 +247,12 @@ Lexer
 * The ``Token::getType()`` method is deprecated as of Twig 3.19, use
   ``Token::test()`` instead.
 
+* The ``Token::ARROW_TYPE`` constant is deprecated as of Twig 3.21, the arrow
+  ``=>`` is now an operator (``Token::OPERATOR_TYPE``).
+
+* The ``Token::PUNCTUATION_TYPE`` with values ``(``, ``[``, ``|``, ``.``,
+  ``?``, or ``?:`` are now of the ``Token::OPERATOR_TYPE`` type.
+
 Templates
 ---------
 
@@ -364,6 +378,8 @@ Node
 Operators
 ---------
 
+* An operator precedence must be part of the [0, 512] range as of Twig 3.21.
+
 * The ``.`` operator allows accessing class constants as of Twig 3.15.
   This can be a BC break if you don't use UPPERCASE constant names.
 
@@ -418,3 +434,45 @@ Operators
     {# or #}
 
     {{ (not 1) * 2 }} {# this is equivalent to what Twig 4.x will do without the parentheses #}
+
+* Using the ``|`` operator in an expression with ``+`` or ``-`` without explicit
+  parentheses to clarify precedence triggers a deprecation as of Twig 3.21 (in
+  Twig 4.0, ``|`` will have a higher precedence than ``+`` and ``-``).
+
+  For example, the following expression will trigger a deprecation in Twig 3.21::
+
+    {{ -1|abs }}
+
+  To avoid the deprecation, add parentheses to clarify the precedence::
+
+    {{ -(1|abs) }} {# this is equivalent to what Twig 3.x does without the parentheses #}
+
+    {# or #}
+
+    {{ (-1)|abs }} {# this is equivalent to what Twig 4.x will do without the parentheses #}
+
+* The ``Twig\Extension\ExtensionInterface::getOperators()`` method is deprecated
+  as of Twig 3.21, use ``Twig\Extension\ExtensionInterface::getExpressionParsers()``
+  instead:
+
+    Before:
+
+      public function getOperators(): array {
+          return [
+              'not' => [
+                  'precedence' => 10,
+                  'class' => NotUnary::class,
+              ],
+          ];
+      }
+
+    After:
+
+      public function getExpressionParsers(): array {
+          return [
+              new UnaryOperatorExpressionParser(NotUnary::class, 'not', 10),
+          ];
+      }
+
+* The ``Twig\OperatorPrecedenceChange`` class is deprecated as of Twig 3.21,
+  use ``Twig\ExpressionParser\PrecedenceChange`` instead.

doc/filters/number_format.rst

@@ -15,15 +15,21 @@ separator using the additional arguments:
 
     {{ 9800.333|number_format(2, '.', ',') }}
 
-To format negative numbers or math calculation, wrap the previous statement
-with parentheses (needed because of Twig's :ref:`precedence of operators
-<twig-expressions>`):
+To format negative numbers, wrap the previous statement with parentheses (note
+that as of Twig 3.21, not using parentheses is deprecated as the filter
+operator will change precedence in Twig 4.0):
 
 .. code-block:: twig
 
     {{ -9800.333|number_format(2, '.', ',') }} {# outputs : -9 #}
     {{ (-9800.333)|number_format(2, '.', ',') }} {# outputs : -9,800.33 #}
-    {{  1 + 0.2|number_format(2) }} {# outputs : 1.2 #}
+
+To format math calculation, wrap the previous statement with parentheses
+(needed because of Twig's :ref:`precedence of operators -<twig-expressions>`):
+
+.. code-block:: twig
+
+    {{ 1 + 0.2|number_format(2) }} {# outputs : 1.2 #}
     {{ (1 + 0.2)|number_format(2) }} {# outputs : 1.20 #}
 
 If no formatting options are provided then Twig will use the default formatting

doc/operators_precedence.rst

@@ -0,0 +1,104 @@
+
+=========== ================ ======= ============= ===========
+Precedence  Operator         Type    Associativity Description
+=========== ================ ======= ============= ===========
+512 => 300  ``|``            infix   Left          Twig filter call
+            ``(``                                  Twig function call
+            ``.``                                  Get an attribute on a variable
+            ``[``                                  Array access
+500         ``-``            prefix  n/a
+            ``+``
+300 => 5    ``??``           infix   Right         Null coalescing operator (a ?? b)
+250         ``=>``           infix   Left          Arrow function (x => expr)
+200         ``**``           infix   Right         Exponentiation operator
+100         ``is``           infix   Left          Twig tests
+            ``is not``                             Twig tests
+60          ``*``            infix   Left
+            ``/``
+            ``//``                                 Floor division
+            ``%``
+50 => 70    ``not``          prefix  n/a
+40 => 27    ``~``            infix   Left
+30          ``+``            infix   Left
+            ``-``
+25          ``..``           infix   Left
+20          ``==``           infix   Left
+            ``!=``
+            ``<=>``
+            ``<``
+            ``>``
+            ``>=``
+            ``<=``
+            ``not in``
+            ``in``
+            ``matches``
+            ``starts with``
+            ``ends with``
+            ``has some``
+            ``has every``
+18          ``b-and``        infix   Left
+17          ``b-xor``        infix   Left
+16          ``b-or``         infix   Left
+15          ``and``          infix   Left
+12          ``xor``          infix   Left
+10          ``or``           infix   Left
+5           ``?:``           infix   Right         Elvis operator (a ?: b)
+            ``?:``                                 Elvis operator (a ?: b)
+0           ``(``            prefix  n/a           Explicit group expression (a)
+            ``literal``                            A literal value (boolean, string, number, sequence, mapping, ...)
+            ``?``            infix   Left          Conditional operator (a ? b : c)
+=========== ================ ======= ============= ===========
+
+When a precedence will change in 4.0, the new precedence is indicated by the arrow ``=>``.
+
+Here is the same table for Twig 4.0 with adjusted precedences:
+
+=========== ============== ======= ============= ===========
+Precedence  Operator       Type    Associativity Description
+=========== ============== ======= ============= ===========
+512         `(`            infix   Left          Twig function call
+            `.`                                  Get an attribute on a variable
+            `[`                                  Array access
+500         `-`            prefix  n/a
+            `+`
+300         `|`            infix   Left          Twig filter call
+250         `=>`           infix   Left          Arrow function (x => expr)
+200         `**`           infix   Right         Exponentiation operator
+100         `is`           infix   Left          Twig tests
+            `is not`                             Twig tests
+70          `not`          prefix  n/a
+60          `*`            infix   Left
+            `/`
+            `//`                                 Floor division
+            `%`
+30          `+`            infix   Left
+            `-`
+27          `~`            infix   Left
+25          `..`           infix   Left
+20          `==`           infix   Left
+            `!=`
+            `<=>`
+            `<`
+            `>`
+            `>=`
+            `<=`
+            `not in`
+            `in`
+            `matches`
+            `starts with`
+            `ends with`
+            `has some`
+            `has every`
+18          `b-and`        infix   Left
+17          `b-xor`        infix   Left
+16          `b-or`         infix   Left
+15          `and`          infix   Left
+12          `xor`          infix   Left
+10          `or`           infix   Left
+5           `??`           infix   Right         Null coalescing operator (a ?? b)
+            `?:`                                 Elvis operator (a ?: b)
+            `?:`                                 Elvis operator (a ?: b)
+0           `(`            prefix  n/a           Explicit group expression (a)
+            `literal`                            A literal value (boolean, string, number, sequence, mapping, ...)
+            `?`            infix   Left          Conditional operator (a ? b : c)
+=========== ============== ======= ============= ===========