Tokenizer/PHP: add tests for consistent tokenization heredocs with interpolated strings

package.xml

@@ -168,6 +168,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
       <file baseinstalldir="" name="GotoLabelTest.php" role="test" />
       <file baseinstalldir="" name="HeredocNowdocCloserTest.inc" role="test" />
       <file baseinstalldir="" name="HeredocNowdocCloserTest.php" role="test" />
+      <file baseinstalldir="" name="HeredocStringTest.inc" role="test" />
+      <file baseinstalldir="" name="HeredocStringTest.php" role="test" />
       <file baseinstalldir="" name="NamedFunctionCallArgumentsTest.inc" role="test" />
       <file baseinstalldir="" name="NamedFunctionCallArgumentsTest.php" role="test" />
       <file baseinstalldir="" name="NullsafeObjectOperatorTest.inc" role="test" />
@@ -2145,6 +2147,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.inc" name="tests/Core/Tokenizer/GotoLabelTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/HeredocNowdocCloserTest.php" name="tests/Core/Tokenizer/HeredocNowdocCloserTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/HeredocNowdocCloserTest.inc" name="tests/Core/Tokenizer/HeredocNowdocCloserTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/HeredocStringTest.php" name="tests/Core/Tokenizer/HeredocStringTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/HeredocStringTest.inc" name="tests/Core/Tokenizer/HeredocStringTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />
@@ -2251,6 +2255,8 @@ http://pear.php.net/dtd/package-2.0.xsd">
    <install as="CodeSniffer/Core/Tokenizer/GotoLabelTest.inc" name="tests/Core/Tokenizer/GotoLabelTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/HeredocNowdocCloserTest.php" name="tests/Core/Tokenizer/HeredocNowdocCloserTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/HeredocNowdocCloserTest.inc" name="tests/Core/Tokenizer/HeredocNowdocCloserTest.inc" />
+   <install as="CodeSniffer/Core/Tokenizer/HeredocStringTest.php" name="tests/Core/Tokenizer/HeredocStringTest.php" />
+   <install as="CodeSniffer/Core/Tokenizer/HeredocStringTest.inc" name="tests/Core/Tokenizer/HeredocStringTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.php" />
    <install as="CodeSniffer/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" name="tests/Core/Tokenizer/NamedFunctionCallArgumentsTest.inc" />
    <install as="CodeSniffer/Core/Tokenizer/NullsafeObjectOperatorTest.php" name="tests/Core/Tokenizer/NullsafeObjectOperatorTest.php" />

tests/Core/Tokenizer/HeredocStringTest.inc

@@ -0,0 +1,193 @@
+<?php
+
+// Test source: https://gist.github.com/iluuu1994/72e2154fc4150f2258316b0255b698f2#file-test-php
+
+/* testSimple1 */
+$heredoc = <<<EOD
+$foo
+EOD;
+
+/* testSimple2 */
+$heredoc = <<<"EOD"
+{$foo}
+EOD;
+
+/* testSimple3 */
+$heredoc = <<<EOD
+${foo}
+EOD;
+
+/* testDIM1 */
+$heredoc = <<<"EOD"
+$foo[bar]
+EOD;
+
+/* testDIM2 */
+$heredoc = <<<EOD
+{$foo['bar']}
+EOD;
+
+/* testDIM3 */
+$heredoc = <<<"EOD"
+${foo['bar']}
+EOD;
+
+/* testProperty1 */
+$heredoc = <<<EOD
+$foo->bar
+EOD;
+
+/* testProperty2 */
+$heredoc = <<<"EOD"
+{$foo->bar}
+EOD;
+
+/* testMethod1 */
+$heredoc = <<<EOD
+{$foo->bar()}
+EOD;
+
+/* testClosure1 */
+$heredoc = <<<"EOD"
+{$foo()}
+EOD;
+
+/* testChain1 */
+$heredoc = <<<EOD
+{$foo['bar']->baz()()}
+EOD;
+
+/* testVariableVar1 */
+$heredoc = <<<"EOD"
+${$bar}
+EOD;
+
+/* testVariableVar2 */
+$heredoc = <<<EOD
+${(foo)}
+EOD;
+
+/* testVariableVar3 */
+$heredoc = <<<"EOD"
+${foo->bar}
+EOD;
+
+/* testNested1 */
+$heredoc = <<<EOD
+${foo["${bar}"]}
+EOD;
+
+/* testNested2 */
+$heredoc = <<<"EOD"
+${foo["${bar['baz']}"]}
+EOD;
+
+/* testNested3 */
+$heredoc = <<<EOD
+${foo->{$baz}}
+EOD;
+
+/* testNested4 */
+$heredoc = <<<"EOD"
+${foo->{${'a'}}}
+EOD;
+
+/* testNested5 */
+$heredoc = <<<EOD
+${foo->{"${'a'}"}}
+EOD;
+
+/* testSimple1Wrapped */
+$heredoc = <<<EOD
+Do $foo Something
+EOD;
+
+/* testSimple2Wrapped */
+$heredoc = <<<"EOD"
+Do {$foo} Something
+EOD;
+
+/* testSimple3Wrapped */
+$heredoc = <<<EOD
+Do ${foo} Something
+EOD;
+
+/* testDIM1Wrapped */
+$heredoc = <<<"EOD"
+Do $foo[bar] Something
+EOD;
+
+/* testDIM2Wrapped */
+$heredoc = <<<EOD
+Do {$foo['bar']} Something
+EOD;
+
+/* testDIM3Wrapped */
+$heredoc = <<<"EOD"
+Do ${foo['bar']} Something
+EOD;
+
+/* testProperty1Wrapped */
+$heredoc = <<<EOD
+Do $foo->bar Something
+EOD;
+
+/* testProperty2Wrapped */
+$heredoc = <<<"EOD"
+Do {$foo->bar} Something
+EOD;
+
+/* testMethod1Wrapped */
+$heredoc = <<<EOD
+Do {$foo->bar()} Something
+EOD;
+
+/* testClosure1Wrapped */
+$heredoc = <<<"EOD"
+Do {$foo()} Something
+EOD;
+
+/* testChain1Wrapped */
+$heredoc = <<<EOD
+Do {$foo['bar']->baz()()} Something
+EOD;
+
+/* testVariableVar1Wrapped */
+$heredoc = <<<"EOD"
+Do ${$bar} Something
+EOD;
+
+/* testVariableVar2Wrapped */
+$heredoc = <<<EOD
+Do ${(foo)} Something
+EOD;
+
+/* testVariableVar3Wrapped */
+$heredoc = <<<"EOD"
+Do ${foo->bar} Something
+EOD;
+
+/* testNested1Wrapped */
+$heredoc = <<<EOD
+Do ${foo["${bar}"]} Something
+EOD;
+
+/* testNested2Wrapped */
+$heredoc = <<<"EOD"
+Do ${foo["${bar['baz']}"]} Something
+EOD;
+
+/* testNested3Wrapped */
+$heredoc = <<<EOD
+Do ${foo->{$baz}} Something
+EOD;
+
+/* testNested4Wrapped */
+$heredoc = <<<"EOD"
+Do ${foo->{${'a'}}} Something
+EOD;
+
+/* testNested5Wrapped */
+$heredoc = <<<EOD
+Do ${foo->{"${'a'}"}} Something
+EOD;

tests/Core/Tokenizer/HeredocStringTest.php

@@ -0,0 +1,153 @@
+<?php
+/**
+ * Tests that embedded variables and expressions in heredoc strings are tokenized
+ * as one heredoc string token.
+ *
+ * @author    Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
+ * @copyright 2022 Squiz Pty Ltd (ABN 77 084 670 600)
+ * @license   https://github.com/squizlabs/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizer;
+
+use PHP_CodeSniffer\Tests\Core\AbstractMethodUnitTest;
+
+class HeredocStringTest extends AbstractMethodUnitTest
+{
+
+
+    /**
+     * Test that heredoc strings contain the complete interpolated string.
+     *
+     * @param string $testMarker      The comment which prefaces the target token in the test file.
+     * @param string $expectedContent The expected content of the heredoc string.
+     *
+     * @dataProvider dataHeredocString
+     * @covers       PHP_CodeSniffer\Tokenizers\PHP::tokenize
+     *
+     * @return void
+     */
+    public function testHeredocString($testMarker, $expectedContent)
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $target = $this->getTargetToken($testMarker, T_HEREDOC);
+        $this->assertSame($expectedContent."\n", $tokens[$target]['content']);
+
+    }//end testHeredocString()
+
+
+    /**
+     * Test that heredoc strings contain the complete interpolated string when combined with other texts.
+     *
+     * @param string $testMarker      The comment which prefaces the target token in the test file.
+     * @param string $expectedContent The expected content of the heredoc string.
+     *
+     * @dataProvider dataHeredocString
+     * @covers       PHP_CodeSniffer\Tokenizers\PHP::tokenize
+     *
+     * @return void
+     */
+    public function testHeredocStringWrapped($testMarker, $expectedContent)
+    {
+        $tokens = self::$phpcsFile->getTokens();
+
+        $testMarker = substr($testMarker, 0, -3).'Wrapped */';
+        $target     = $this->getTargetToken($testMarker, T_HEREDOC);
+        $this->assertSame('Do '.$expectedContent." Something\n", $tokens[$target]['content']);
+
+    }//end testHeredocStringWrapped()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testHeredocString()
+     *
+     * @return array
+     */
+    public function dataHeredocString()
+    {
+        return [
+            [
+                'testMarker'      => '/* testSimple1 */',
+                'expectedContent' => '$foo',
+            ],
+            [
+                'testMarker'      => '/* testSimple2 */',
+                'expectedContent' => '{$foo}',
+            ],
+            [
+                'testMarker'      => '/* testSimple3 */',
+                'expectedContent' => '${foo}',
+            ],
+            [
+                'testMarker'      => '/* testDIM1 */',
+                'expectedContent' => '$foo[bar]',
+            ],
+            [
+                'testMarker'      => '/* testDIM2 */',
+                'expectedContent' => '{$foo[\'bar\']}',
+            ],
+            [
+                'testMarker'      => '/* testDIM3 */',
+                'expectedContent' => '${foo[\'bar\']}',
+            ],
+            [
+                'testMarker'      => '/* testProperty1 */',
+                'expectedContent' => '$foo->bar',
+            ],
+            [
+                'testMarker'      => '/* testProperty2 */',
+                'expectedContent' => '{$foo->bar}',
+            ],
+            [
+                'testMarker'      => '/* testMethod1 */',
+                'expectedContent' => '{$foo->bar()}',
+            ],
+            [
+                'testMarker'      => '/* testClosure1 */',
+                'expectedContent' => '{$foo()}',
+            ],
+            [
+                'testMarker'      => '/* testChain1 */',
+                'expectedContent' => '{$foo[\'bar\']->baz()()}',
+            ],
+            [
+                'testMarker'      => '/* testVariableVar1 */',
+                'expectedContent' => '${$bar}',
+            ],
+            [
+                'testMarker'      => '/* testVariableVar2 */',
+                'expectedContent' => '${(foo)}',
+            ],
+            [
+                'testMarker'      => '/* testVariableVar3 */',
+                'expectedContent' => '${foo->bar}',
+            ],
+            [
+                'testMarker'      => '/* testNested1 */',
+                'expectedContent' => '${foo["${bar}"]}',
+            ],
+            [
+                'testMarker'      => '/* testNested2 */',
+                'expectedContent' => '${foo["${bar[\'baz\']}"]}',
+            ],
+            [
+                'testMarker'      => '/* testNested3 */',
+                'expectedContent' => '${foo->{$baz}}',
+            ],
+            [
+                'testMarker'      => '/* testNested4 */',
+                'expectedContent' => '${foo->{${\'a\'}}}',
+            ],
+            [
+                'testMarker'      => '/* testNested5 */',
+                'expectedContent' => '${foo->{"${\'a\'}"}}',
+            ],
+        ];
+
+    }//end dataHeredocString()
+
+
+}//end class