From fbac5554544eff7d8c8011657dbad8da744348aa Mon Sep 17 00:00:00 2001 From: "John Paul E. Balandan, CPA" Date: Thu, 7 Oct 2021 03:30:31 +0000 Subject: [PATCH] Fix `Tokens::insertSlices` not moving around all affected tokens --- src/Tokenizer/Tokens.php | 25 +++-- tests/Tokenizer/TokensTest.php | 184 +++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 10 deletions(-) diff --git a/src/Tokenizer/Tokens.php b/src/Tokenizer/Tokens.php index 0f347f412be..13b4aa44798 100644 --- a/src/Tokenizer/Tokens.php +++ b/src/Tokenizer/Tokens.php @@ -842,6 +842,7 @@ public function insertAt(int $index, $items): void public function insertSlices(array $slices): void { $itemsCount = 0; + foreach ($slices as $slice) { $slice = \is_array($slice) || $slice instanceof self ? $slice : [$slice]; $itemsCount += \count($slice); @@ -859,20 +860,28 @@ public function insertSlices(array $slices): void krsort($slices); + if (array_key_first($slices) > $oldSize) { + throw new \OutOfBoundsException('Cannot insert outside of collection.'); + } + $insertBound = $oldSize - 1; // since we only move already existing items around, we directly call into SplFixedArray::offset* methods. // that way we get around additional overhead this class adds with overridden offset* methods. foreach ($slices as $index => $slice) { + if (!\is_int($index) || $index < 0) { + throw new \OutOfBoundsException(sprintf('Invalid index "%s".', $index)); + } + $slice = \is_array($slice) || $slice instanceof self ? $slice : [$slice]; $sliceCount = \count($slice); for ($i = $insertBound; $i >= $index; --$i) { - $oldItem = parent::offsetExists($i) ? parent::offsetGet($i) : new Token(''); - parent::offsetSet($i + $itemsCount, $oldItem); + parent::offsetSet($i + $itemsCount, parent::offsetGet($i)); } - $insertBound = $index - $sliceCount; + // adjust $insertBound as tokens between this index and the next index in loop + $insertBound = $index - 1; $itemsCount -= $sliceCount; foreach ($slice as $indexItem => $item) { @@ -881,8 +890,8 @@ public function insertSlices(array $slices): void } $this->registerFoundToken($item); - $newOffset = $index + $itemsCount + $indexItem; - parent::offsetSet($newOffset, $item); + + parent::offsetSet($index + $itemsCount + $indexItem, $item); } } } @@ -892,11 +901,7 @@ public function insertSlices(array $slices): void */ public function isChanged(): bool { - if ($this->changed) { - return true; - } - - return false; + return $this->changed; } public function isEmptyAt(int $index): bool diff --git a/tests/Tokenizer/TokensTest.php b/tests/Tokenizer/TokensTest.php index a6f98bfe5c3..e556ec397cf 100644 --- a/tests/Tokenizer/TokensTest.php +++ b/tests/Tokenizer/TokensTest.php @@ -1375,6 +1375,190 @@ public function provideGetMeaningfulTokenSiblingCases(): \Generator yield [null, 0, -1, ' $slices + */ + public function testInsertSlicesAtMultiplePlaces(string $expected, array $slices): void + { + $input = <<<'EOF' +insertSlices([ + 16 => $slices, + 6 => $slices, + ]); + + static::assertSame($expected, $tokens->generateCode()); + } + + public function provideInsertSlicesAtMultiplePlacesCases(): \Generator + { + yield 'one slice count' => [ + <<<'EOF' + [ + <<<'EOF' + [ + <<<'EOF' +isChanged()); + static::assertFalse($tokens->isTokenKindFound(T_COMMENT)); + static::assertSame(5, $tokens->getSize()); + + $tokens->insertSlices([1 => new Token([T_COMMENT, '/* comment */'])]); + + static::assertTrue($tokens->isChanged()); + static::assertTrue($tokens->isTokenKindFound(T_COMMENT)); + static::assertSame(6, $tokens->getSize()); + } + + /** + * @dataProvider provideInsertSlicesCases + */ + public function testInsertSlices(Tokens $expected, Tokens $tokens, array $slices): void + { + $tokens->insertSlices($slices); + static::assertTokens($expected, $tokens); + } + + public function provideInsertSlicesCases(): iterable + { + // basic insert of single token at 3 different locations including appending as new token + + $template = " [ + Tokens::fromCode(sprintf($template, $commentContent, '', '')), + clone $from, + [1 => $commentToken], + ]; + + yield 'single insert @ 3' => [ + Tokens::fromCode(sprintf($template, '', $commentContent, '')), + clone $from, + [3 => Tokens::fromArray([$commentToken])], + ]; + + yield 'single insert @ 9' => [ + Tokens::fromCode(sprintf($template, '', '', $commentContent)), + clone $from, + [9 => [$commentToken]], + ]; + + // basic tests for single token, array of that token and tokens object with that token + + $openTagToken = new Token([T_OPEN_TAG, " $openTagToken], + [0 => [$openTagToken]], + [0 => Tokens::fromArray([$openTagToken])], + ]; + + foreach ($slices as $i => $slice) { + yield 'insert open tag @ 0 into empty collection '.$i => [$expected, new Tokens(), $slice]; + } + + // test insert lists of tokens, index out of order + + $setOne = [ + new Token([T_ECHO, 'echo']), + new Token([T_WHITESPACE, ' ']), + new Token([T_CONSTANT_ENCAPSED_STRING, '"new"']), + new Token(';'), + ]; + + $setTwo = [ + new Token([T_WHITESPACE, ' ']), + new Token([T_COMMENT, '/* new comment */']), + ]; + + $setThree = Tokens::fromArray([ + new Token([T_VARIABLE, '$new']), + new Token([T_WHITESPACE, ' ']), + new Token('='), + new Token([T_WHITESPACE, ' ']), + new Token([T_LNUMBER, '8899']), + new Token(';'), + new Token([T_WHITESPACE, "\n"]), + ]); + + $template = " [$expected, $from, [9 => $setThree, 1 => $setOne, 3 => $setTwo]]; + + $sets = []; + + for ($j = 0; $j < 4; ++$j) { + $set = ['tokens' => [], 'content' => '']; + + for ($i = 0; $i < 10; ++$i) { + $content = sprintf('/* new %d|%s */', $j, $i); + + $set['tokens'][] = new Token([T_COMMENT, $content]); + $set['content'] .= $content; + } + + $sets[$j] = $set; + } + + yield 'overlapping inserts of bunch of comments ' => [ + Tokens::fromCode(sprintf(" $sets[0]['tokens'], 3 => $sets[1]['tokens'], 5 => $sets[2]['tokens'], 6 => $sets[3]['tokens']], + ]; + } + private static function assertFindBlockEnd(int $expectedIndex, string $source, int $type, int $searchIndex): void { Tokens::clearCache();