fix bug in attributes tokenization on PHP < 8.0 (squizlabs#3294)

alekitto · Apr 12, 2021 · 26a545f · 26a545f
1 parent ffced0d
commit 26a545f
Show file tree

Hide file tree

Showing 3 changed files with 180 additions and 7 deletions.
diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
@@ -3213,14 +3213,27 @@ private function parsePhpAttribute(array &$tokens, $stackPtr)
 
         // Go looking for the close bracket.
         $bracketCloser = $this->findCloser($subTokens, 1, '[', ']');
-        if ($bracketCloser === null) {
-            $bracketCloser = $this->findCloser($tokens, $stackPtr, '[', ']');
-            if ($bracketCloser === null) {
-                return null;
+        if (PHP_VERSION_ID < 80000 && $bracketCloser === null) {
+            foreach (array_slice($tokens, ($stackPtr + 1)) as $token) {
+                if (is_array($token) === true) {
+                    $commentBody .= $token[1];
+                } else {
+                    $commentBody .= $token;
+                }
+            }
+
+            $subTokens = @token_get_all('<?php '.$commentBody);
+            array_splice($subTokens, 0, 1, [[T_ATTRIBUTE, '#[']]);
+
+            $bracketCloser = $this->findCloser($subTokens, 1, '[', ']');
+            if ($bracketCloser !== null) {
+                array_splice($tokens, ($stackPtr + 1), count($tokens), array_slice($subTokens, ($bracketCloser + 1)));
+                $subTokens = array_slice($subTokens, 0, ($bracketCloser + 1));
             }
+        }
 
-            $subTokens = array_merge($subTokens, array_slice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr)));
-            array_splice($tokens, ($stackPtr + 1), ($bracketCloser - $stackPtr));
+        if ($bracketCloser === null) {
+            return null;
         }
 
         return $subTokens;

diff --git a/tests/Core/Tokenizer/AttributesTest.inc b/tests/Core/Tokenizer/AttributesTest.inc
@@ -75,7 +75,16 @@ function multiline_attributes_on_parameter_test(#[
     )
                                                 ] int $param) {}
 
+/* testAttributeContainingTextLookingLikeCloseTag */
+#[DeprecationReason('reason: <https://some-website/reason?>')]
+function attribute_containing_text_looking_like_close_tag() {}
+
+/* testAttributeContainingMultilineTextLookingLikeCloseTag */
+#[DeprecationReason(
+    'reason: <https://some-website/reason?>'
+)]
+function attribute_containing_mulitline_text_looking_like_close_tag() {}
+
 /* testInvalidAttribute */
 #[ThisIsNotAnAttribute
 function invalid_attribute_test() {}
-
diff --git a/tests/Core/Tokenizer/AttributesTest.php b/tests/Core/Tokenizer/AttributesTest.php
@@ -395,6 +395,157 @@ public function dataAttributeOnParameters()
     }//end dataAttributeOnParameters()
 
 
+    /**
+     * Test that an attribute containing text which looks like a PHP close tag is tokenized correctly.
+     *
+     * @param string $testMarker              The comment which prefaces the target token in the test file.
+     * @param int    $length                  The number of tokens between opener and closer.
+     * @param array  $expectedTokensAttribute The codes of tokens inside the attributes.
+     * @param array  $expectedTokensAfter     The codes of tokens after the attributes.
+     *
+     * @covers PHP_CodeSniffer\Tokenizers\PHP::parsePhpAttribute
+     *
+     * @dataProvider dataAttributeOnTextLookingLikeCloseTag
+     *
+     * @return void
+     */
+    public function testAttributeContainingTextLookingLikeCloseTag($testMarker, $length, array $expectedTokensAttribute, array $expectedTokensAfter)
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $attribute = $this->getTargetToken($testMarker, T_ATTRIBUTE);
+
+        $this->assertSame('T_ATTRIBUTE', $tokens[$attribute]['type']);
+        $this->assertArrayHasKey('attribute_closer', $tokens[$attribute]);
+
+        $closer = $tokens[$attribute]['attribute_closer'];
+        $this->assertSame(($attribute + $length), $closer);
+        $this->assertSame(T_ATTRIBUTE_END, $tokens[$closer]['code']);
+        $this->assertSame('T_ATTRIBUTE_END', $tokens[$closer]['type']);
+
+        $this->assertSame($tokens[$attribute]['attribute_opener'], $tokens[$closer]['attribute_opener']);
+        $this->assertSame($tokens[$attribute]['attribute_closer'], $tokens[$closer]['attribute_closer']);
+
+        $i = ($attribute + 1);
+        foreach ($expectedTokensAttribute as $item) {
+            list($expectedType, $expectedContents) = $item;
+            $this->assertSame($expectedType, $tokens[$i]['type']);
+            $this->assertSame($expectedContents, $tokens[$i]['content']);
+            $this->assertArrayHasKey('attribute_opener', $tokens[$i]);
+            $this->assertArrayHasKey('attribute_closer', $tokens[$i]);
+            ++$i;
+        }
+
+        $i = ($closer + 1);
+        foreach ($expectedTokensAfter as $expectedCode) {
+            $this->assertSame($expectedCode, $tokens[$i]['code']);
+            ++$i;
+        }
+
+    }//end testAttributeContainingTextLookingLikeCloseTag()
+
+
+    /**
+     * Data provider.
+     *
+     * @see dataAttributeOnTextLookingLikeCloseTag()
+     *
+     * @return array
+     */
+    public function dataAttributeOnTextLookingLikeCloseTag()
+    {
+        return [
+            [
+                '/* testAttributeContainingTextLookingLikeCloseTag */',
+                5,
+                [
+                    [
+                        'T_STRING',
+                        'DeprecationReason',
+                    ],
+                    [
+                        'T_OPEN_PARENTHESIS',
+                        '(',
+                    ],
+                    [
+                        'T_CONSTANT_ENCAPSED_STRING',
+                        "'reason: <https://some-website/reason?>'",
+                    ],
+                    [
+                        'T_CLOSE_PARENTHESIS',
+                        ')',
+                    ],
+                    [
+                        'T_ATTRIBUTE_END',
+                        ']',
+                    ],
+                ],
+                [
+                    T_WHITESPACE,
+                    T_FUNCTION,
+                    T_WHITESPACE,
+                    T_STRING,
+                    T_OPEN_PARENTHESIS,
+                    T_CLOSE_PARENTHESIS,
+                    T_WHITESPACE,
+                    T_OPEN_CURLY_BRACKET,
+                    T_CLOSE_CURLY_BRACKET,
+                ],
+            ],
+            [
+                '/* testAttributeContainingMultilineTextLookingLikeCloseTag */',
+                8,
+                [
+                    [
+                        'T_STRING',
+                        'DeprecationReason',
+                    ],
+                    [
+                        'T_OPEN_PARENTHESIS',
+                        '(',
+                    ],
+                    [
+                        'T_WHITESPACE',
+                        "\n",
+                    ],
+                    [
+                        'T_WHITESPACE',
+                        "    ",
+                    ],
+                    [
+                        'T_CONSTANT_ENCAPSED_STRING',
+                        "'reason: <https://some-website/reason?>'",
+                    ],
+                    [
+                        'T_WHITESPACE',
+                        "\n",
+                    ],
+                    [
+                        'T_CLOSE_PARENTHESIS',
+                        ')',
+                    ],
+                    [
+                        'T_ATTRIBUTE_END',
+                        ']',
+                    ],
+                ],
+                [
+                    T_WHITESPACE,
+                    T_FUNCTION,
+                    T_WHITESPACE,
+                    T_STRING,
+                    T_OPEN_PARENTHESIS,
+                    T_CLOSE_PARENTHESIS,
+                    T_WHITESPACE,
+                    T_OPEN_CURLY_BRACKET,
+                    T_CLOSE_CURLY_BRACKET,
+                ],
+            ],
+        ];
+
+    }//end dataAttributeOnTextLookingLikeCloseTag()
+
+
     /**
      * Test that invalid attribute (or comment starting with #[ and without ]) are parsed correctly.
      *