From 07f1b8a9f2815ecb439704d99c0e2acb8313149e Mon Sep 17 00:00:00 2001 From: Jeroen Versteeg Date: Sat, 16 Mar 2024 14:26:35 +0100 Subject: [PATCH 1/2] Add CommentNode to the AST These nodes have no effect on compiled templates. Adding these to the TokenStream and AST enables extensions to add logic in node visitors, such as parsing PHPDoc-style `@var string name` comments. This in turn is invaluable for static code analysis of Twig templates as briefly discussed in [#4003](https://github.com/twigphp/Twig/issues/4003) and in more detail in [TwigStan](https://github.com/alisqi/twig-stan/)'s README. --- src/Lexer.php | 4 +++- src/Node/CommentNode.php | 35 +++++++++++++++++++++++++++++++++++ src/Parser.php | 6 ++++++ src/Token.php | 6 ++++++ tests/LexerTest.php | 21 +++++++++++++++++++++ tests/Node/CommentTest.php | 32 ++++++++++++++++++++++++++++++++ 6 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 src/Node/CommentNode.php create mode 100644 tests/Node/CommentTest.php diff --git a/src/Lexer.php b/src/Lexer.php index 9e4d6119eb7..f11d125d9e3 100644 --- a/src/Lexer.php +++ b/src/Lexer.php @@ -412,7 +412,9 @@ private function lexComment(): void throw new SyntaxError('Unclosed comment.', $this->lineno, $this->source); } - $this->moveCursor(substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0]); + $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor) . $match[0][0]; + $this->pushToken(/* Token::COMMENT_TYPE */ 14, trim(substr($text, 0, strrpos($text, '#}')))); + $this->moveCursor($text); } private function lexString(): void diff --git a/src/Node/CommentNode.php b/src/Node/CommentNode.php new file mode 100644 index 00000000000..88e4b4eabf7 --- /dev/null +++ b/src/Node/CommentNode.php @@ -0,0 +1,35 @@ + + */ +#[YieldReady] +class CommentNode extends Node +{ + public function __construct(string $data, int $lineno) + { + parent::__construct([], ['text' => $data], $lineno); + } + + public function compile(Compiler $compiler): void + { + // skip comments in compilation + } +} diff --git a/src/Parser.php b/src/Parser.php index adcaee31633..1fba72053d6 100644 --- a/src/Parser.php +++ b/src/Parser.php @@ -16,6 +16,7 @@ use Twig\Node\BlockNode; use Twig\Node\BlockReferenceNode; use Twig\Node\BodyNode; +use Twig\Node\CommentNode; use Twig\Node\Expression\AbstractExpression; use Twig\Node\MacroNode; use Twig\Node\ModuleNode; @@ -125,6 +126,11 @@ public function subparse($test, bool $dropNeedle = false): Node $rv[] = new TextNode($token->getValue(), $token->getLine()); break; + case /* Token::COMMENT_TYPE */ 14: + $token = $this->stream->next(); + $rv[] = new CommentNode($token->getValue(), $token->getLine()); + break; + case /* Token::VAR_START_TYPE */ 2: $token = $this->stream->next(); $expr = $this->expressionParser->parseExpression(); diff --git a/src/Token.php b/src/Token.php index 59279b8fe7c..12d31d4d2e8 100644 --- a/src/Token.php +++ b/src/Token.php @@ -36,6 +36,7 @@ final class Token public const INTERPOLATION_END_TYPE = 11; public const ARROW_TYPE = 12; public const SPREAD_TYPE = 13; + public const COMMENT_TYPE = 14; public function __construct(int $type, $value, int $lineno) { @@ -137,6 +138,9 @@ public static function typeToString(int $type, bool $short = false): string case self::SPREAD_TYPE: $name = 'SPREAD_TYPE'; break; + case self::COMMENT_TYPE: + $name = 'COMMENT_TYPE'; + break; default: throw new \LogicException(sprintf('Token of type "%s" does not exist.', $type)); } @@ -177,6 +181,8 @@ public static function typeToEnglish(int $type): string return 'arrow function'; case self::SPREAD_TYPE: return 'spread operator'; + case self::COMMENT_TYPE: + return 'comment'; default: throw new \LogicException(sprintf('Token of type "%s" does not exist.', $type)); } diff --git a/tests/LexerTest.php b/tests/LexerTest.php index ad62c22acfb..af7a6b1002e 100644 --- a/tests/LexerTest.php +++ b/tests/LexerTest.php @@ -353,6 +353,26 @@ public function testUnterminatedBlock() $lexer->tokenize(new Source($template, 'index')); } + public function testCommentValues() + { + $template = '{# comment #}some text{#another one#}'; + $lexer = new Lexer(new Environment($this->createMock(LoaderInterface::class))); + $stream = $lexer->tokenize(new Source($template, 'index')); + + self::assertEquals( + 'comment', // assert that whitespace is stripped + $stream->expect(Token::COMMENT_TYPE)->getValue() // implicit assertion is that expect() doesn't throw + ); + self::assertEquals( + 'some text', + $stream->expect(Token::TEXT_TYPE)->getValue() + ); + self::assertEquals( + 'another one', // assert that comment is parsed + $stream->expect(Token::COMMENT_TYPE)->getValue() + ); + } + public function testOverridingSyntax() { $template = '[# comment #]{# variable #}/# if true #/true/# endif #/'; @@ -362,6 +382,7 @@ public function testOverridingSyntax() 'tag_variable' => ['{#', '#}'], ]); $stream = $lexer->tokenize(new Source($template, 'index')); + $stream->expect(Token::COMMENT_TYPE); $stream->expect(Token::VAR_START_TYPE); $stream->expect(Token::NAME_TYPE, 'variable'); $stream->expect(Token::VAR_END_TYPE); diff --git a/tests/Node/CommentTest.php b/tests/Node/CommentTest.php new file mode 100644 index 00000000000..68da0137946 --- /dev/null +++ b/tests/Node/CommentTest.php @@ -0,0 +1,32 @@ +assertEquals('foo', $node->getAttribute('text')); + } + + public function getTests() + { + return [ + [new CommentNode('foo', 1), ""], + ]; + } +} From 4d41a2caa4e2b220785c4ecd7431ad10f5311e3a Mon Sep 17 00:00:00 2001 From: Jeroen Versteeg Date: Sat, 16 Mar 2024 14:46:12 +0100 Subject: [PATCH 2/2] Adjust code style --- src/Lexer.php | 2 +- tests/Node/CommentTest.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Lexer.php b/src/Lexer.php index f11d125d9e3..57dfadcc539 100644 --- a/src/Lexer.php +++ b/src/Lexer.php @@ -412,7 +412,7 @@ private function lexComment(): void throw new SyntaxError('Unclosed comment.', $this->lineno, $this->source); } - $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor) . $match[0][0]; + $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0]; $this->pushToken(/* Token::COMMENT_TYPE */ 14, trim(substr($text, 0, strrpos($text, '#}')))); $this->moveCursor($text); } diff --git a/tests/Node/CommentTest.php b/tests/Node/CommentTest.php index 68da0137946..0c9e83d59d8 100644 --- a/tests/Node/CommentTest.php +++ b/tests/Node/CommentTest.php @@ -26,7 +26,7 @@ public function testConstructor() public function getTests() { return [ - [new CommentNode('foo', 1), ""], + [new CommentNode('foo', 1), ''], ]; } }