Remove class Tokens

It was not used anywhere in PMA project. It's very likely broken. It has serious feature envy on TokensList. The two methods had very unclear purpose and behaviour.

Signed-off-by: Kamil Tekiela <[email protected]>

phpstan-baseline.neon

@@ -820,26 +820,6 @@ parameters:
 			count: 2
 			path: src/Utils/Table.php
 
-		-
-			message: "#^Binary operation \"&\" between int\\|string and int results in an error\\.$#"
-			count: 1
-			path: src/Utils/Tokens.php
-
-		-
-			message: "#^Cannot cast mixed to string\\.$#"
-			count: 1
-			path: src/Utils/Tokens.php
-
-		-
-			message: "#^Parameter \\#1 \\$string1 of function strcasecmp expects string, int\\|string given\\.$#"
-			count: 1
-			path: src/Utils/Tokens.php
-
-		-
-			message: "#^Parameter \\#2 \\$pattern of static method PhpMyAdmin\\\\SqlParser\\\\Utils\\\\Tokens\\:\\:match\\(\\) expects array\\<string, int\\|string\\|null\\>, PhpMyAdmin\\\\SqlParser\\\\Token given\\.$#"
-			count: 1
-			path: src/Utils/Tokens.php
-
 		-
 			message: "#^Cannot call method __toString\\(\\) on PhpMyAdmin\\\\SqlParser\\\\Components\\\\Expression\\|null\\.$#"
 			count: 2
@@ -950,11 +930,6 @@ parameters:
 			count: 1
 			path: tests/Utils/TableTest.php
 
-		-
-			message: "#^Parameter \\#2 \\$find of static method PhpMyAdmin\\\\SqlParser\\\\Utils\\\\Tokens\\:\\:replaceTokens\\(\\) expects array\\<PhpMyAdmin\\\\SqlParser\\\\Token\\>, array\\<array\\<string, string\\>\\> given\\.$#"
-			count: 1
-			path: tests/Utils/TokensTest.php
-
 		-
 			message: "#^Expression \"\\$str1\\[\\$i\\]\" on a separate line does not do anything\\.$#"
 			count: 1

psalm-baseline.xml

@@ -1410,17 +1410,6 @@
       <code>has</code>
     </PossiblyNullReference>
   </file>
-  <file src="src/Utils/Tokens.php">
-    <InvalidArgument>
-      <code>$find[$k]</code>
-    </InvalidArgument>
-    <PossiblyInvalidArgument>
-      <code><![CDATA[$pattern['value_str']]]></code>
-    </PossiblyInvalidArgument>
-    <PossiblyInvalidOperand>
-      <code><![CDATA[$pattern['flags']]]></code>
-    </PossiblyInvalidOperand>
-  </file>
   <file src="tests/Builder/AlterStatementTest.php">
     <PossiblyUnusedMethod>
       <code>provideBuilderForRenameColumn</code>
@@ -1859,15 +1848,6 @@
       <code>getForeignKeysProvider</code>
     </PossiblyUnusedMethod>
   </file>
-  <file src="tests/Utils/TokensTest.php">
-    <InvalidArgument>
-      <code>$find</code>
-    </InvalidArgument>
-    <PossiblyUnusedMethod>
-      <code>matchProvider</code>
-      <code>replaceTokensProvider</code>
-    </PossiblyUnusedMethod>
-  </file>
   <file src="tests/benchmarks/UtfStringBench.php">
     <MissingConstructor>
       <code>$testContents</code>

src/Lexer.php

@@ -151,21 +151,6 @@ class Lexer extends Core
      */
     public $delimiterLen;
 
-    /**
-     * Gets the tokens list parsed by a new instance of a lexer.
-     *
-     * @param string|UtfString $str       the query to be lexed
-     * @param bool             $strict    whether strict mode should be
-     *                                    enabled or not
-     * @param string           $delimiter the delimiter to be used
-     */
-    public static function getTokens($str, $strict = false, $delimiter = null): TokensList
-    {
-        $lexer = new self($str, $strict, $delimiter);
-
-        return $lexer->list;
-    }
-
     /**
      * @param string|UtfString $str       the query to be lexed
      * @param bool             $strict    whether strict mode should be