Skip to content

Commit 51278fd

Browse files
jrfnljrfnl
committed
Tokenizer: apply tab replacement to heredoc/nowdoc opener
_You learn something new every day ;-)_ While probably exceedingly rare to be found in actual codebases, the PHP tokenizer apparently allows for whitespace between the `<<<` and the heredoc/nowdoc identifier. See: https://3v4l.org/NUHZd Both spaces as well as tabs are allowed. New lines are not allowed. Comments are also not allowed. See: https://3v4l.org/7PIEK The PHPCS `Tokenizer` did not execute tab replacement on these tokens leading to unexpected `'content'` and incorrect `'length'` values in the `File::$tokens` array, which in turn could lead to incorrect sniff results and incorrect fixes. This commit adds the `T_START_HEREDOC`/`T_START_NOWDOC` tokens to the array of tokens for which to do tab replacement to make them more consistent with the rest of PHPCS. Includes unit tests safeguarding this change. Ref: https://externals.io/message/124462#124518
1 parent d49ea71 commit 51278fd

File tree

3 files changed

+154
-0
lines changed

3 files changed

+154
-0
lines changed

‎src/Tokenizers/Tokenizer.php

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,8 @@ private function createPositionMap()
193193
T_DOC_COMMENT_STRING => true,
194194
T_CONSTANT_ENCAPSED_STRING => true,
195195
T_DOUBLE_QUOTED_STRING => true,
196+
T_START_HEREDOC => true,
197+
T_START_NOWDOC => true,
196198
T_HEREDOC => true,
197199
T_NOWDOC => true,
198200
T_END_HEREDOC => true,

‎tests/Core/Tokenizer/Tokenizer/HeredocNowdocOpenerTest.inc

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
<?php
2+
3+
/* testHeredocOpenerNoSpace */
4+
$heredoc = <<<EOD
5+
some text
6+
EOD;
7+
8+
/* testNowdocOpenerNoSpace */
9+
$nowdoc = <<<'EOD'
10+
some text
11+
EOD;
12+
13+
/* testHeredocOpenerHasSpace */
14+
$heredoc = <<< END
15+
some text
16+
END;
17+
18+
/* testNowdocOpenerHasSpace */
19+
$nowdoc = <<< 'END'
20+
some text
21+
END;
22+
23+
/* testHeredocOpenerHasTab */
24+
$heredoc = <<< "END"
25+
some text
26+
END;
27+
28+
/* testNowdocOpenerHasTab */
29+
$nowdoc = <<< 'END'
30+
some text
31+
END;

‎tests/Core/Tokenizer/Tokenizer/HeredocNowdocOpenerTest.php

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
<?php
2+
/**
3+
* Tests the tokenization of heredoc/nowdoc opener tokens.
4+
*
5+
* @author Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
6+
* @copyright 2024 PHPCSStandards and contributors
7+
* @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
8+
*/
9+
10+
namespace PHP_CodeSniffer\Tests\Core\Tokenizer\Tokenizer;
11+
12+
use PHP_CodeSniffer\Tests\Core\Tokenizer\AbstractTokenizerTestCase;
13+
14+
/**
15+
* Heredoc/nowdoc opener token test.
16+
*/
17+
final class HeredocNowdocOpenerTest extends AbstractTokenizerTestCase
18+
{
19+
20+
21+
/**
22+
* Verify that spaces/tabs in a heredoc/nowdoc opener token get the tab replacement treatment.
23+
*
24+
* @param string $testMarker The comment prefacing the target token.
25+
* @param array<string, int|string|null> $expected Expectations for the token array.
26+
*
27+
* @dataProvider dataHeredocNowdocOpenerTabReplacement
28+
* @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap
29+
*
30+
* @return void
31+
*/
32+
public function testHeredocNowdocOpenerTabReplacement($testMarker, $expected)
33+
{
34+
$tokens = $this->phpcsFile->getTokens();
35+
$opener = $this->getTargetToken($testMarker, [T_START_HEREDOC, T_START_NOWDOC]);
36+
37+
foreach ($expected as $key => $value) {
38+
if ($key === 'orig_content' && $value === null) {
39+
$this->assertArrayNotHasKey($key, $tokens[$opener], "Unexpected 'orig_content' key found in the token array.");
40+
continue;
41+
}
42+
43+
$this->assertArrayHasKey($key, $tokens[$opener], "Key $key not found in the token array.");
44+
$this->assertSame($value, $tokens[$opener][$key], "Value for key $key does not match expectation.");
45+
}
46+
47+
}//end testHeredocNowdocOpenerTabReplacement()
48+
49+
50+
/**
51+
* Data provider.
52+
*
53+
* @see testHeredocNowdocOpenerTabReplacement()
54+
*
55+
* @return array<string, array<string, string|array<string, int|string|null>>>
56+
*/
57+
public static function dataHeredocNowdocOpenerTabReplacement()
58+
{
59+
return [
60+
'Heredoc opener without space' => [
61+
'testMarker' => '/* testHeredocOpenerNoSpace */',
62+
'expected' => [
63+
'length' => 6,
64+
'content' => '<<<EOD
65+
',
66+
'orig_content' => null,
67+
],
68+
],
69+
'Nowdoc opener without space' => [
70+
'testMarker' => '/* testNowdocOpenerNoSpace */',
71+
'expected' => [
72+
'length' => 8,
73+
'content' => "<<<'EOD'
74+
",
75+
'orig_content' => null,
76+
],
77+
],
78+
'Heredoc opener with space(s)' => [
79+
'testMarker' => '/* testHeredocOpenerHasSpace */',
80+
'expected' => [
81+
'length' => 7,
82+
'content' => '<<< END
83+
',
84+
'orig_content' => null,
85+
],
86+
],
87+
'Nowdoc opener with space(s)' => [
88+
'testMarker' => '/* testNowdocOpenerHasSpace */',
89+
'expected' => [
90+
'length' => 21,
91+
'content' => "<<< 'END'
92+
",
93+
'orig_content' => null,
94+
],
95+
],
96+
'Heredoc opener with tab(s)' => [
97+
'testMarker' => '/* testHeredocOpenerHasTab */',
98+
'expected' => [
99+
'length' => 18,
100+
'content' => '<<< "END"
101+
',
102+
'orig_content' => '<<< "END"
103+
',
104+
],
105+
],
106+
'Nowdoc opener with tab(s)' => [
107+
'testMarker' => '/* testNowdocOpenerHasTab */',
108+
'expected' => [
109+
'length' => 11,
110+
'content' => "<<< 'END'
111+
",
112+
'orig_content' => "<<< 'END'
113+
",
114+
],
115+
],
116+
];
117+
118+
}//end dataHeredocNowdocOpenerTabReplacement()
119+
120+
121+
}//end class

0 commit comments

Comments
 (0)