Improve return type of tokenizer

VincentLanglet · Jul 27, 2024 · 3c6019a · 3c6019a
1 parent aa2743c
commit 3c6019a
Show file tree

Hide file tree

Showing 14 changed files with 114 additions and 55 deletions.
diff --git a/UPGRADE.md b/UPGRADE.md
@@ -54,14 +54,14 @@ If you never implemented a custom rule, nothing else changed. Otherwise, ...
 
 ```diff
 - public function lintFile(array $tokens, Report $report, array $ignoredViolations = []): void;
-+ public function lintFile(Tokens $tokens, Report $report, array $ignoredViolations = []): void;
++ public function lintFile(Tokens $tokens, Report $report): void;
 ```
 
 ### FixableRuleInterface
 
 ```diff
 - public function fixFile(array $tokens, FixerInterface $fixer, array $ignoredViolations = []): void;
-+ public function fixFile(Tokens $tokens, FixerInterface $fixer, array $ignoredViolations = []): void;
++ public function fixFile(Tokens $tokens, FixerInterface $fixer): void;
 ```
 
 ### TokenizerInterface
@@ -71,10 +71,7 @@ If you never implemented a custom rule, nothing else changed. Otherwise, ...
 -   * @return array{list<Token>, list<ViolationId>}
 -   */
 -  public function tokenize(Source $source): array;
-+ /**
-+   * @return array{Tokens, list<ViolationId>}
-+   */
-+  public function tokenize(Source $source): array;
++  public function tokenize(Source $source): Tokens;
 ```
 
 ### Token

diff --git a/src/Rules/AbstractFixableRule.php b/src/Rules/AbstractFixableRule.php
@@ -22,9 +22,9 @@ protected function init(
         $this->fixer = $fixer;
     }
 
-    public function fixFile(Tokens $tokens, FixerInterface $fixer, array $ignoredViolations = []): void
+    public function fixFile(Tokens $tokens, FixerInterface $fixer): void
     {
-        $this->init(null, $ignoredViolations, $fixer);
+        $this->init(null, $tokens->getIgnoredViolations(), $fixer);
 
         foreach (array_keys($tokens->toArray()) as $index) {
             $this->process($index, $tokens);

diff --git a/src/Rules/AbstractRule.php b/src/Rules/AbstractRule.php
@@ -14,9 +14,9 @@ abstract class AbstractRule implements RuleInterface
 {
     use RuleTrait;
 
-    public function lintFile(Tokens $tokens, Report $report, array $ignoredViolations = []): void
+    public function lintFile(Tokens $tokens, Report $report): void
     {
-        $this->init($report, $ignoredViolations);
+        $this->init($report, $tokens->getIgnoredViolations());
 
         foreach (array_keys($tokens->toArray()) as $index) {
             $this->process($index, $tokens);

diff --git a/src/Rules/FixableRuleInterface.php b/src/Rules/FixableRuleInterface.php
@@ -4,14 +4,10 @@
 
 namespace TwigCsFixer\Rules;
 
-use TwigCsFixer\Report\ViolationId;
 use TwigCsFixer\Runner\FixerInterface;
 use TwigCsFixer\Token\Tokens;
 
 interface FixableRuleInterface
 {
-    /**
-     * @param list<ViolationId> $ignoredViolations
-     */
-    public function fixFile(Tokens $tokens, FixerInterface $fixer, array $ignoredViolations = []): void;
+    public function fixFile(Tokens $tokens, FixerInterface $fixer): void;
 }
diff --git a/src/Rules/RuleInterface.php b/src/Rules/RuleInterface.php
@@ -5,15 +5,12 @@
 namespace TwigCsFixer\Rules;
 
 use TwigCsFixer\Report\Report;
-use TwigCsFixer\Report\ViolationId;
 use TwigCsFixer\Token\Tokens;
 
 interface RuleInterface
 {
     /**
      * Messages will be added to the given `$report` object.
-     *
-     * @param list<ViolationId> $ignoredViolations
      */
-    public function lintFile(Tokens $tokens, Report $report, array $ignoredViolations = []): void;
+    public function lintFile(Tokens $tokens, Report $report): void;
 }
diff --git a/src/Runner/Fixer.php b/src/Runner/Fixer.php
@@ -85,14 +85,14 @@ public function fixFile(string $content, Ruleset $ruleset): string
             $this->inConflict = false;
 
             $twigSource = new Source($content, 'TwigCsFixer');
-            [$stream, $ignoredViolations] = $this->tokenizer->tokenize($twigSource);
+            $stream = $this->tokenizer->tokenize($twigSource);
 
             $this->startFile($stream);
 
             $rules = $ruleset->getRules();
             foreach ($rules as $rule) {
                 if ($rule instanceof FixableRuleInterface) {
-                    $rule->fixFile($stream, $this, $ignoredViolations);
+                    $rule->fixFile($stream, $this);
                 }
             }
 

diff --git a/src/Runner/Linter.php b/src/Runner/Linter.php
@@ -102,7 +102,7 @@ public function run(iterable $files, Ruleset $ruleset, ?FixerInterface $fixer =
             $this->setErrorHandler($report, $filePath);
             try {
                 $twigSource = new Source($content, $filePath);
-                [$stream, $ignoredViolations] = $this->tokenizer->tokenize($twigSource);
+                $stream = $this->tokenizer->tokenize($twigSource);
             } catch (CannotTokenizeException $exception) {
                 $violation = new Violation(
                     Violation::LEVEL_FATAL,
@@ -116,7 +116,7 @@ public function run(iterable $files, Ruleset $ruleset, ?FixerInterface $fixer =
             restore_error_handler();
 
             foreach ($rules as $rule) {
-                $rule->lintFile($stream, $report, $ignoredViolations);
+                $rule->lintFile($stream, $report);
             }
 
             if ([] !== $nodeVisitorRules) {
@@ -126,7 +126,7 @@ public function run(iterable $files, Ruleset $ruleset, ?FixerInterface $fixer =
                 }
 
                 foreach ($nodeVisitorRules as $nodeVisitor) {
-                    $nodeVisitor->setReport($report, $ignoredViolations);
+                    $nodeVisitor->setReport($report, $stream->getIgnoredViolations());
                 }
 
                 $traverser->traverse($node);

diff --git a/src/Token/Tokenizer.php b/src/Token/Tokenizer.php
@@ -60,11 +60,6 @@ final class Tokenizer implements TokenizerInterface
 
     private ?Token $lastNonEmptyToken = null;
 
-    /**
-     * @var list<ViolationId>
-     */
-    private array $ignoredViolations = [];
-
     /**
      * @var list<array{fullMatch: string, position: int, match: string}>
      */
@@ -94,11 +89,9 @@ public function __construct(Environment $env)
     }
 
     /**
-     * @return array{Tokens, list<ViolationId>}
-     *
      * @throws CannotTokenizeException
      */
-    public function tokenize(Source $source): array
+    public function tokenize(Source $source): Tokens
     {
         $this->resetState($source);
         $this->pushState(self::STATE_DATA);
@@ -160,7 +153,9 @@ public function tokenize(Source $source): array
 
         $this->pushToken(Token::EOF_TYPE);
 
-        return [$this->tokens, $this->ignoredViolations];
+        $this->tokens->setReadOnly();
+
+        return $this->tokens;
     }
 
     private function resetState(Source $source): void
@@ -764,7 +759,7 @@ private function processIgnoredViolations(): void
         };
 
         if ('' === $ignoredViolations) {
-            $this->ignoredViolations[] = ViolationId::fromString($ignoredViolations, $line);
+            $this->tokens->addIgnoredViolation(ViolationId::fromString($ignoredViolations, $line));
 
             return;
         }
@@ -774,7 +769,7 @@ private function processIgnoredViolations(): void
             if ('' === $ignoredViolation) {
                 continue;
             }
-            $this->ignoredViolations[] = ViolationId::fromString($ignoredViolation, $line);
+            $this->tokens->addIgnoredViolation(ViolationId::fromString($ignoredViolation, $line));
         }
     }
 }
diff --git a/src/Token/TokenizerInterface.php b/src/Token/TokenizerInterface.php
@@ -6,14 +6,11 @@
 
 use Twig\Source;
 use TwigCsFixer\Exception\CannotTokenizeException;
-use TwigCsFixer\Report\ViolationId;
 
 interface TokenizerInterface
 {
     /**
-     * @return array{Tokens, list<ViolationId>}
-     *
      * @throws CannotTokenizeException
      */
-    public function tokenize(Source $source): array;
+    public function tokenize(Source $source): Tokens;
 }
diff --git a/src/Token/Tokens.php b/src/Token/Tokens.php
@@ -4,6 +4,8 @@
 
 namespace TwigCsFixer\Token;
 
+use TwigCsFixer\Report\ViolationId;
+
 final class Tokens
 {
     /**
@@ -21,6 +23,13 @@ final class Tokens
      */
     private array $indexes = [];
 
+    /**
+     * @var list<ViolationId>
+     */
+    private array $ignoredViolations = [];
+
+    private bool $readOnly = false;
+
     /**
      * @param array<Token> $tokens
      */
@@ -31,8 +40,24 @@ public function __construct(array $tokens = [])
         }
     }
 
+    public function setReadOnly(): self
+    {
+        $this->readOnly = true;
+
+        return $this;
+    }
+
+    public function isReadOnly(): bool
+    {
+        return $this->readOnly;
+    }
+
     public function add(Token $token): self
     {
+        if ($this->readOnly) {
+            throw new \LogicException('Cannot add token because the tokens are in read-only mode.');
+        }
+
         $this->tokens[] = $token;
         $this->indexes[spl_object_id($token)] = $this->tokenCount;
         ++$this->tokenCount;
@@ -100,4 +125,23 @@ public function findPrevious(int|string|array $type, int $start, int $end = 0, b
 
         return false;
     }
+
+    public function addIgnoredViolation(ViolationId $violationId): self
+    {
+        if ($this->readOnly) {
+            throw new \LogicException('Cannot add ignored violation because the tokens are in read-only mode.');
+        }
+
+        $this->ignoredViolations[] = $violationId;
+
+        return $this;
+    }
+
+    /**
+     * @return list<ViolationId>
+     */
+    public function getIgnoredViolations(): array
+    {
+        return $this->ignoredViolations;
+    }
 }
diff --git a/tests/Runner/FixerTest.php b/tests/Runner/FixerTest.php
@@ -36,12 +36,11 @@ public function testInvalidFile(): void
     public function testValidFile(): void
     {
         $tokenizer = $this->createMock(TokenizerInterface::class);
-        $tokenizer->expects(static::once())->method('tokenize')->willReturn([
+        $tokenizer->expects(static::once())->method('tokenize')->willReturn(
             new Tokens([
                 new Token(Token::EOF_TYPE, 0, 0, 'TwigCsFixer'),
             ]),
-            [],
-        ]);
+        );
 
         $ruleset = new Ruleset();
         $fixer = new Fixer($tokenizer);

diff --git a/tests/Runner/LinterTest.php b/tests/Runner/LinterTest.php
@@ -72,14 +72,14 @@ public function testUntokenizableFilesAreReported(): void
 
         $call = 0;
         $tokenizer->method('tokenize')->willReturnCallback(
-            static function () use (&$call): array {
+            static function () use (&$call): Tokens {
                 /** @psalm-suppress RedundantCondition https://github.com/vimeo/psalm/issues/10513 */
                 if (0 === $call) {
                     ++$call;
                     throw CannotTokenizeException::unknownError();
                 }
 
-                return [[], []];
+                return new Tokens();
             }
         );
         $ruleset = new Ruleset();
@@ -117,11 +117,11 @@ public function testUserDeprecationAreReported(): void
 
         $env = new StubbedEnvironment();
         $tokenizer = self::createStub(TokenizerInterface::class);
-        $tokenizer->method('tokenize')->willReturnCallback(static function (): array {
+        $tokenizer->method('tokenize')->willReturnCallback(static function (): Tokens {
             @trigger_error('Default');
             trigger_error('User Deprecation', \E_USER_DEPRECATED);
 
-            return [[], []];
+            return new Tokens();
         });
         $ruleset = new Ruleset();
 
@@ -330,7 +330,7 @@ public function testNodeVisitorWithInvalidFiles(): void
         $env = self::createStub(Environment::class);
         $env->method('tokenize')->willThrowException(new SyntaxError('Error.'));
         $tokenizer = self::createStub(TokenizerInterface::class);
-        $tokenizer->method('tokenize')->willReturn([new Tokens(), []]);
+        $tokenizer->method('tokenize')->willReturn(new Tokens());
 
         $linter = new Linter($env, $tokenizer);
         $report = $linter->run([new \SplFileInfo($filePath), new \SplFileInfo($filePath2)], $ruleset);
@@ -360,7 +360,7 @@ public function testNodeVisitorWithBuggyFixer(): void
 
         $env = new StubbedEnvironment();
         $tokenizer = static::createStub(TokenizerInterface::class);
-        $tokenizer->method('tokenize')->willReturn([new Tokens(), []]);
+        $tokenizer->method('tokenize')->willReturn(new Tokens());
         $linter = new Linter($env, $tokenizer);
 
         $fixer = static::createStub(FixerInterface::class);