fix bug in attributes tokenization on PHP < 8.0 (squizlabs#3294)

alekitto · Apr 12, 2021 · 6f9e5ba · 6f9e5ba
1 parent ffced0d
commit 6f9e5ba
Show file tree

Hide file tree

Showing 3 changed files with 130 additions and 7 deletions.
diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
@@ -3211,16 +3211,27 @@ private function parsePhpAttribute(array &$tokens, $stackPtr)
 
         array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);
 
+        $tmpTokens = array_slice($tokens, $stackPtr + 1);
+
         // Go looking for the close bracket.
         $bracketCloser = $this->findCloser($subTokens, 1, '[', ']');
-        if ($bracketCloser === null) {
-            $bracketCloser = $this->findCloser($tokens, $stackPtr, '[', ']');
-            if ($bracketCloser === null) {
-                return null;
+        if (PHP_VERSION_ID < 80000 && $bracketCloser === null && ! empty($tmpTokens)) {
+            while (($token = array_shift($tmpTokens))) {
+                $commentBody .= is_array($token) ? $token[1] : $token;
             }
 
-            $subTokens = array_merge($subTokens, array_slice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr)));
-            array_splice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr));
+            $subTokens = @token_get_all('<?php '.$commentBody);
+            array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);
+
+            $bracketCloser = $this->findCloser($subTokens, 1, '[', ']');
+            if ($bracketCloser !== null) {
+                array_splice($tokens, ($stackPtr + 1), count($tokens), array_slice($subTokens, $bracketCloser + 1));
+                $subTokens = array_slice($subTokens, 0, $bracketCloser + 1);
+            }
+        }
+
+        if ($bracketCloser === null) {
+            return null;
         }
 
         return $subTokens;

diff --git a/tests/Core/Tokenizer/AttributesTest.inc b/tests/Core/Tokenizer/AttributesTest.inc
@@ -75,7 +75,16 @@ function multiline_attributes_on_parameter_test(#[
     )
                                                 ] int $param) {}
 
+/* testAttributeContainingTextLookingLikeCloseTag */
+#[DeprecationReason('reason: <https://some-website/reason?>')]
+function attribute_containing_text_looking_like_close_tag() {}
+
+/* testAttributeContainingMultilineTextLookingLikeCloseTag */
+#[DeprecationReason(
+    'reason: <https://some-website/reason?>'
+)]
+function attribute_containing_mulitline_text_looking_like_close_tag() {}
+
 /* testInvalidAttribute */
 #[ThisIsNotAnAttribute
 function invalid_attribute_test() {}
-
diff --git a/tests/Core/Tokenizer/AttributesTest.php b/tests/Core/Tokenizer/AttributesTest.php
@@ -394,6 +394,109 @@ public function dataAttributeOnParameters()
 
     }//end dataAttributeOnParameters()
 
+    /**
+     * Test that an attribute containing text which looks like a PHP close tag is tokenized correctly.
+     *
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::parsePhpAttribute
+     *
+     * @dataProvider dataAttributeOnTextLookingLikeCloseTag
+     *
+     * @return void
+     */
+    public function testAttributeContainingTextLookingLikeCloseTag($testMarker, $length, array $expectedTokensAttribute, array $expectedTokensAfter)
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $attribute = $this->getTargetToken($testMarker, T_ATTRIBUTE);
+
+        $this->assertSame('T_ATTRIBUTE', $tokens[$attribute]['type']);
+        $this->assertArrayHasKey('attribute_closer', $tokens[$attribute]);
+
+        $closer = $tokens[$attribute]['attribute_closer'];
+        $this->assertSame(($attribute + $length), $closer);
+        $this->assertSame(T_ATTRIBUTE_END, $tokens[$closer]['code']);
+        $this->assertSame('T_ATTRIBUTE_END', $tokens[$closer]['type']);
+
+        $this->assertSame($tokens[$attribute]['attribute_opener'], $tokens[$closer]['attribute_opener']);
+        $this->assertSame($tokens[$attribute]['attribute_closer'], $tokens[$closer]['attribute_closer']);
+
+        $i = ($attribute + 1);
+        foreach ($expectedTokensAttribute as [$expectedType, $expectedContents]) {
+            $this->assertSame($expectedType, $tokens[$i]['type']);
+            $this->assertSame($expectedContents, $tokens[$i]['content']);
+            $this->assertArrayHasKey('attribute_opener', $tokens[$i]);
+            $this->assertArrayHasKey('attribute_closer', $tokens[$i]);
+            ++$i;
+        }
+
+        $i = ($closer + 1);
+        foreach ($expectedTokensAfter as $expectedCode) {
+            $this->assertSame($expectedCode, $tokens[$i]['code']);
+            ++$i;
+        }
+
+    }//end testAttributeContainingTextLookingLikeCloseTag()
+
+    /**
+     * Data provider.
+     *
+     * @see dataAttributeOnTextLookingLikeCloseTag()
+     *
+     * @return array
+     */
+    public function dataAttributeOnTextLookingLikeCloseTag()
+    {
+        return [
+            [
+                '/* testAttributeContainingTextLookingLikeCloseTag */',
+                5,
+                [
+                    ['T_STRING', 'DeprecationReason'],
+                    ['T_OPEN_PARENTHESIS', '('],
+                    ['T_CONSTANT_ENCAPSED_STRING', "'reason: <https://some-website/reason?>'"],
+                    ['T_CLOSE_PARENTHESIS', ')'],
+                    ['T_ATTRIBUTE_END', ']'],
+                ],
+                [
+                    T_WHITESPACE,
+                    T_FUNCTION,
+                    T_WHITESPACE,
+                    T_STRING,
+                    T_OPEN_PARENTHESIS,
+                    T_CLOSE_PARENTHESIS,
+                    T_WHITESPACE,
+                    T_OPEN_CURLY_BRACKET,
+                    T_CLOSE_CURLY_BRACKET,
+                ],
+            ],
+            [
+                '/* testAttributeContainingMultilineTextLookingLikeCloseTag */',
+                8,
+                [
+                    ['T_STRING', 'DeprecationReason'],
+                    ['T_OPEN_PARENTHESIS', '('],
+                    ['T_WHITESPACE', "\n"],
+                    ['T_WHITESPACE', "    "],
+                    ['T_CONSTANT_ENCAPSED_STRING', "'reason: <https://some-website/reason?>'"],
+                    ['T_WHITESPACE', "\n"],
+                    ['T_CLOSE_PARENTHESIS', ')'],
+                    ['T_ATTRIBUTE_END', ']'],
+                ],
+                [
+                    T_WHITESPACE,
+                    T_FUNCTION,
+                    T_WHITESPACE,
+                    T_STRING,
+                    T_OPEN_PARENTHESIS,
+                    T_CLOSE_PARENTHESIS,
+                    T_WHITESPACE,
+                    T_OPEN_CURLY_BRACKET,
+                    T_CLOSE_CURLY_BRACKET,
+                ],
+            ],
+        ];
+
+    }//end dataAttributeOnTextLookingLikeCloseTag()
 
     /**
      * Test that invalid attribute (or comment starting with #[ and without ]) are parsed correctly.