Skip to content

Commit

Permalink
Drop Lexer 1
Browse files Browse the repository at this point in the history 
This is not a big deal because doctrine/lexer 1 has the same version
constraints as doctrine/lexer 2.
Note that I'm removing the BC-layer for DocLexer::peek() and DocLexer::glimpse():
in retrospect, I think it was a bad idea because there was already a
BC-break with DocLexer::$lookahead and DocLexer::$token.
  • Loading branch information
@greg0ire
greg0ire committed Dec 16, 2022
1 parent ad78521 commit eab856a
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 50 deletions.
2 changes: 1 addition & 1 deletion composer.json
View file
Expand Up @@ -34,7 +34,7 @@
"require": {
"php": "^7.1 || ^8.0",
"ext-tokenizer": "*",
"doctrine/lexer": "^1 || ^2",
"doctrine/lexer": "^2",
"psr/cache": "^1 || ^2 || ^3"
},
"require-dev": {
Expand Down
14 changes: 1 addition & 13 deletions lib/Doctrine/Common/Annotations/DocLexer.php
View file
Expand Up @@ -70,7 +70,7 @@ public function nextTokenIsAdjacent(): bool
{
return $this->token === null
|| ($this->lookahead !== null
&& ($this->lookahead['position'] - $this->token['position']) === strlen($this->token['value']));
&& ($this->lookahead->position - $this->token->position) === strlen($this->token->value));
}

/**
Expand Down Expand Up @@ -128,16 +128,4 @@ protected function getType(&$value)

return $type;
}

/** @return array{value: int|string, type:self::T_*|null, position:int} */
public function peek(): ?array
{
$token = parent::peek();

if ($token === null) {
return null;
}

return (array) $token;
}
}
40 changes: 20 additions & 20 deletions lib/Doctrine/Common/Annotations/DocParser.php
View file
Expand Up @@ -454,7 +454,7 @@ private function syntaxError(string $expected, ?array $token = null): Annotation
$message = sprintf('Expected %s, got ', $expected);
$message .= $this->lexer->lookahead === null
? 'end of string'
: sprintf("'%s' at position %s", $token['value'], $token['position']);
: sprintf("'%s' at position %s", $token->value, $token->position);

if (strlen($this->context)) {
$message .= ' in ' . $this->context;
Expand Down Expand Up @@ -684,16 +684,16 @@ private function Annotations(): array
$annotations = [];

while ($this->lexer->lookahead !== null) {
if ($this->lexer->lookahead['type'] !== DocLexer::T_AT) {
if ($this->lexer->lookahead->type !== DocLexer::T_AT) {
$this->lexer->moveNext();
continue;
}

// make sure the @ is preceded by non-catchable pattern
if (
$this->lexer->token !== null &&
$this->lexer->lookahead['position'] === $this->lexer->token['position'] + strlen(
$this->lexer->token['value']
$this->lexer->lookahead->position === $this->lexer->token->position + strlen(
$this->lexer->token->value
)
) {
$this->lexer->moveNext();
Expand All @@ -705,12 +705,12 @@ private function Annotations(): array
$peek = $this->lexer->glimpse();
if (
($peek === null)
|| ($peek['type'] !== DocLexer::T_NAMESPACE_SEPARATOR && ! in_array(
$peek['type'],
|| ($peek->type !== DocLexer::T_NAMESPACE_SEPARATOR && ! in_array(
$peek->type,
self::$classIdentifiers,
true
))
|| $peek['position'] !== $this->lexer->lookahead['position'] + 1
|| $peek->position !== $this->lexer->lookahead->position + 1
) {
$this->lexer->moveNext();
continue;
Expand Down Expand Up @@ -1186,18 +1186,18 @@ private function Identifier(): string

$this->lexer->moveNext();

$className = $this->lexer->token['value'];
$className = $this->lexer->token->value;

while (
$this->lexer->lookahead !== null &&
$this->lexer->lookahead['position'] === ($this->lexer->token['position'] +
strlen($this->lexer->token['value'])) &&
$this->lexer->lookahead->position === ($this->lexer->token->position +
strlen($this->lexer->token->value)) &&
$this->lexer->isNextToken(DocLexer::T_NAMESPACE_SEPARATOR)
) {
$this->match(DocLexer::T_NAMESPACE_SEPARATOR);
$this->matchAny(self::$classIdentifiers);

$className .= '\\' . $this->lexer->token['value'];
$className .= '\\' . $this->lexer->token->value;
}

return $className;
Expand All @@ -1215,7 +1215,7 @@ private function Value()
{
$peek = $this->lexer->glimpse();

if ($peek['type'] === DocLexer::T_EQUALS) {
if ($peek->type === DocLexer::T_EQUALS) {
return $this->FieldAssignment();
}

Expand Down Expand Up @@ -1244,21 +1244,21 @@ private function PlainValue()
return $this->Constant();
}

switch ($this->lexer->lookahead['type']) {
switch ($this->lexer->lookahead->type) {
case DocLexer::T_STRING:
$this->match(DocLexer::T_STRING);

return $this->lexer->token['value'];
return $this->lexer->token->value;

case DocLexer::T_INTEGER:
$this->match(DocLexer::T_INTEGER);

return (int) $this->lexer->token['value'];
return (int) $this->lexer->token->value;

case DocLexer::T_FLOAT:
$this->match(DocLexer::T_FLOAT);

return (float) $this->lexer->token['value'];
return (float) $this->lexer->token->value;

case DocLexer::T_TRUE:
$this->match(DocLexer::T_TRUE);
Expand Down Expand Up @@ -1290,7 +1290,7 @@ private function PlainValue()
private function FieldAssignment(): stdClass
{
$this->match(DocLexer::T_IDENTIFIER);
$fieldName = $this->lexer->token['value'];
$fieldName = $this->lexer->token->value;

$this->match(DocLexer::T_EQUALS);

Expand Down Expand Up @@ -1365,14 +1365,14 @@ private function ArrayEntry(): array
$peek = $this->lexer->glimpse();

if (
$peek['type'] === DocLexer::T_EQUALS
|| $peek['type'] === DocLexer::T_COLON
$peek->type === DocLexer::T_EQUALS
|| $peek->type === DocLexer::T_COLON
) {
if ($this->lexer->isNextToken(DocLexer::T_IDENTIFIER)) {
$key = $this->Constant();
} else {
$this->matchAny([DocLexer::T_INTEGER, DocLexer::T_STRING]);
$key = $this->lexer->token['value'];
$key = $this->lexer->token->value;
}

$this->matchAny([DocLexer::T_EQUALS, DocLexer::T_COLON]);
Expand Down
33 changes: 17 additions & 16 deletions tests/Doctrine/Tests/Common/Annotations/DocLexerTest.php
View file
Expand Up @@ -3,6 +3,7 @@
namespace Doctrine\Tests\Common\Annotations;

use Doctrine\Common\Annotations\DocLexer;
use Doctrine\Common\Lexer\Token;
use PHPUnit\Framework\TestCase;

use function str_repeat;
Expand All @@ -20,12 +21,12 @@ public function testMarkerAnnotation(): void
self::assertTrue($lexer->moveNext());
self::assertNull($lexer->token);
self::assertNotNull($lexer->lookahead);
self::assertEquals('@', $lexer->lookahead['value']);
self::assertEquals('@', $lexer->lookahead->value);

self::assertTrue($lexer->moveNext());
self::assertNotNull($lexer->token);
self::assertEquals('@', $lexer->token['value']);
self::assertEquals('Name', $lexer->lookahead['value']);
self::assertEquals('@', $lexer->token->value);
self::assertEquals('Name', $lexer->lookahead->value);

self::assertFalse($lexer->moveNext());
}
Expand Down Expand Up @@ -114,9 +115,9 @@ public function testScannerTokenizesDocBlockWhitConstants(): void
foreach ($tokens as $expected) {
$lexer->moveNext();
$lookahead = $lexer->lookahead;
self::assertEquals($expected['value'], $lookahead['value']);
self::assertEquals($expected['type'], $lookahead['type']);
self::assertEquals($expected['position'], $lookahead['position']);
self::assertEquals($expected['value'], $lookahead->value);
self::assertEquals($expected['type'], $lookahead->type);
self::assertEquals($expected['position'], $lookahead->position);
}

self::assertFalse($lexer->moveNext());
Expand Down Expand Up @@ -155,9 +156,9 @@ public function testScannerTokenizesDocBlockWhitInvalidIdentifier(): void
foreach ($tokens as $expected) {
$lexer->moveNext();
$lookahead = $lexer->lookahead;
self::assertEquals($expected['value'], $lookahead['value']);
self::assertEquals($expected['type'], $lookahead['type']);
self::assertEquals($expected['position'], $lookahead['position']);
self::assertEquals($expected['value'], $lookahead->value);
self::assertEquals($expected['type'], $lookahead->type);
self::assertEquals($expected['position'], $lookahead->position);
}

self::assertFalse($lexer->moveNext());
Expand All @@ -172,7 +173,7 @@ public function testWithinDoubleQuotesVeryVeryLongStringWillNotOverflowPregSplit

$lexer->setInput('"' . str_repeat('.', 20240) . '"');

self::assertIsArray($lexer->glimpse());
self::assertInstanceOf(Token::class, $lexer->glimpse());
}

/**
Expand Down Expand Up @@ -216,9 +217,9 @@ public function testRecognizesDoubleQuotesEscapeSequence(): void
foreach ($tokens as $expected) {
$lexer->moveNext();
$lookahead = $lexer->lookahead;
self::assertEquals($expected['value'], $lookahead['value']);
self::assertEquals($expected['type'], $lookahead['type']);
self::assertEquals($expected['position'], $lookahead['position']);
self::assertEquals($expected['value'], $lookahead->value);
self::assertEquals($expected['type'], $lookahead->type);
self::assertEquals($expected['position'], $lookahead->position);
}

self::assertFalse($lexer->moveNext());
Expand Down Expand Up @@ -296,9 +297,9 @@ private function expectDocblockTokens(string $docBlock, array $expectedTokens):
$lookahead = $lexer->lookahead;

$actualTokens[] = [
'value' => $lookahead['value'],
'type' => $lookahead['type'],
'position' => $lookahead['position'],
'value' => $lookahead->value,
'type' => $lookahead->type,
'position' => $lookahead->position,
];
}

Expand Down

0 comments on commit eab856a

Please sign in to comment.