diff --git a/src/Standards/Generic/Sniffs/WhiteSpace/DisallowTabIndentSniff.php b/src/Standards/Generic/Sniffs/WhiteSpace/DisallowTabIndentSniff.php index ecf19ca53f..b33a58b461 100644 --- a/src/Standards/Generic/Sniffs/WhiteSpace/DisallowTabIndentSniff.php +++ b/src/Standards/Generic/Sniffs/WhiteSpace/DisallowTabIndentSniff.php @@ -78,6 +78,7 @@ public function process(File $phpcsFile, $stackPtr) T_COMMENT => true, T_END_HEREDOC => true, T_END_NOWDOC => true, + T_YIELD_FROM => true, ]; for ($i = 0; $i < $phpcsFile->numTokens; $i++) { diff --git a/src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php b/src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php index d4e913c1ad..6cb5f92f0f 100644 --- a/src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php +++ b/src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php @@ -80,42 +80,57 @@ public function process(File $phpcsFile, $stackPtr) if ($tokens[$stackPtr]['code'] === T_YIELD_FROM && strtolower($content) !== 'yield from' ) { - if ($tokens[($stackPtr - 1)]['code'] === T_YIELD_FROM) { - // A multi-line statement that has already been processed. - return; - } + $found = $content; + $hasComment = false; + $yieldFromEnd = $stackPtr; + + // Handle potentially multi-line/multi-token "yield from" expressions. + if (preg_match('`yield\s+from`i', $content) !== 1) { + for ($i = ($stackPtr + 1); $i < $phpcsFile->numTokens; $i++) { + if (isset(Tokens::$emptyTokens[$tokens[$i]['code']]) === false + && $tokens[$i]['code'] !== T_YIELD_FROM + ) { + break; + } + + if (isset(Tokens::$commentTokens[$tokens[$i]['code']]) === true) { + $hasComment = true; + } - $found = $content; - if ($tokens[($stackPtr + 1)]['code'] === T_YIELD_FROM) { - // This yield from statement is split over multiple lines. - $i = ($stackPtr + 1); - do { $found .= $tokens[$i]['content']; - $i++; - } while ($tokens[$i]['code'] === T_YIELD_FROM); - } + + if ($tokens[$i]['code'] === T_YIELD_FROM + && strtolower(trim($tokens[$i]['content'])) === 'from' + ) { + break; + } + } + + $yieldFromEnd = $i; + }//end if $error = 'Language constructs must be followed by a single space; expected 1 space between YIELD FROM found "%s"'; $data = [Common::prepareForOutput($found)]; - $fix = $phpcsFile->addFixableError($error, $stackPtr, 'IncorrectYieldFrom', $data); - if ($fix === true) { - preg_match('/yield/i', $found, $yield); - preg_match('/from/i', $found, $from); - $phpcsFile->fixer->beginChangeset(); - $phpcsFile->fixer->replaceToken($stackPtr, $yield[0].' '.$from[0]); - - if ($tokens[($stackPtr + 1)]['code'] === T_YIELD_FROM) { - $i = ($stackPtr + 1); - do { + + if ($hasComment === true) { + $phpcsFile->addError($error, $stackPtr, 'IncorrectYieldFromWithComment', $data); + } else { + $fix = $phpcsFile->addFixableError($error, $stackPtr, 'IncorrectYieldFrom', $data); + if ($fix === true) { + preg_match('/yield/i', $found, $yield); + preg_match('/from/i', $found, $from); + $phpcsFile->fixer->beginChangeset(); + $phpcsFile->fixer->replaceToken($stackPtr, $yield[0].' '.$from[0]); + + for ($i = ($stackPtr + 1); $i <= $yieldFromEnd; $i++) { $phpcsFile->fixer->replaceToken($i, ''); - $i++; - } while ($tokens[$i]['code'] === T_YIELD_FROM); - } + } - $phpcsFile->fixer->endChangeset(); - } + $phpcsFile->fixer->endChangeset(); + } + }//end if - return; + return ($yieldFromEnd + 1); }//end if if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) { diff --git a/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc b/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc index a50ac22853..10d3ed6933 100644 --- a/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc +++ b/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc @@ -53,5 +53,12 @@ if (ISSET($a) && !Empty($a)) { UnSeT($a); } eval('foo'); eVaL('foo'); +$c = function() { + Yield /*comment*/ From fun(); + YIELD + /*comment*/ + FROM fun(); +} + __HALT_COMPILER(); // An exception due to phar support. function diff --git a/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc.fixed b/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc.fixed index 5e15765146..547f72fca9 100644 --- a/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc.fixed +++ b/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc.fixed @@ -53,5 +53,12 @@ if (isset($a) && !empty($a)) { unset($a); } eval('foo'); eval('foo'); +$c = function() { + yield /*comment*/ from fun(); + yield + /*comment*/ + from fun(); +} + __HALT_COMPILER(); // An exception due to phar support. function diff --git a/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.php b/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.php index 6d337504f4..31bfad6df6 100644 --- a/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.php +++ b/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.php @@ -52,6 +52,9 @@ public function getErrorList() 48 => 1, 52 => 3, 54 => 1, + 57 => 2, + 58 => 1, + 60 => 1, ]; }//end getErrorList() diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc index 1826b585b2..b19a58d874 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc +++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc @@ -116,3 +116,10 @@ $x = 1; Another line. */ + +// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords. +function myGenerator() { + yield + from + gen2(); +} diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc.fixed index 298b3771a5..ca037ee923 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc.fixed +++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc.fixed @@ -116,3 +116,10 @@ $x = 1; Another line. */ + +// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords. +function myGenerator() { + yield + from + gen2(); +} diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc index 1826b585b2..b19a58d874 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc +++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc @@ -116,3 +116,10 @@ $x = 1; Another line. */ + +// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords. +function myGenerator() { + yield + from + gen2(); +} diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc.fixed index 298b3771a5..ca037ee923 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc.fixed +++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc.fixed @@ -116,3 +116,10 @@ $x = 1; Another line. */ + +// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords. +function myGenerator() { + yield + from + gen2(); +} diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.php b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.php index 4a27f8839b..21e89c646f 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.php +++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.php @@ -89,6 +89,7 @@ public function getErrorList($testFile='') 115 => 1, 117 => 1, 118 => 1, + 123 => 1, ]; case 'DisallowSpaceIndentUnitTest.3.inc': diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc b/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc index cf61177e88..74fa505119 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc +++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc @@ -91,3 +91,12 @@ $var = "$hello $there"; Another line. */ + +// A `yield from` can be single-line and multiline and may contain a tab in the whitespace between the keywords. +function myGenerator() { + yield from gen1(); + + yield + from + gen2(); +} diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc.fixed index 8154179c98..3425921201 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc.fixed +++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc.fixed @@ -91,3 +91,12 @@ $var = "$hello $there"; Another line. */ + +// A `yield from` can be single-line and multiline and may contain a tab in the whitespace between the keywords. +function myGenerator() { + yield from gen1(); + + yield + from + gen2(); +} diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.php b/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.php index 7a5f15e9f5..9618e55d86 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.php +++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.php @@ -50,43 +50,45 @@ public function getErrorList($testFile='') switch ($testFile) { case 'DisallowTabIndentUnitTest.1.inc': return [ - 5 => 2, - 9 => 1, - 15 => 1, - 20 => 2, - 21 => 1, - 22 => 2, - 23 => 1, - 24 => 2, - 31 => 1, - 32 => 2, - 33 => 2, - 41 => 1, - 42 => 1, - 43 => 1, - 44 => 1, - 45 => 1, - 46 => 1, - 47 => 1, - 48 => 1, - 54 => 1, - 55 => 1, - 56 => 1, - 57 => 1, - 58 => 1, - 59 => 1, - 79 => 1, - 80 => 1, - 81 => 1, - 82 => 1, - 83 => 1, - 85 => 1, - 86 => 1, - 87 => 1, - 89 => 1, - 90 => 1, - 92 => 1, - 93 => 1, + 5 => 2, + 9 => 1, + 15 => 1, + 20 => 2, + 21 => 1, + 22 => 2, + 23 => 1, + 24 => 2, + 31 => 1, + 32 => 2, + 33 => 2, + 41 => 1, + 42 => 1, + 43 => 1, + 44 => 1, + 45 => 1, + 46 => 1, + 47 => 1, + 48 => 1, + 54 => 1, + 55 => 1, + 56 => 1, + 57 => 1, + 58 => 1, + 59 => 1, + 79 => 1, + 80 => 1, + 81 => 1, + 82 => 1, + 83 => 1, + 85 => 1, + 86 => 1, + 87 => 1, + 89 => 1, + 90 => 1, + 92 => 1, + 93 => 1, + 97 => 1, + 100 => 1, ]; case 'DisallowTabIndentUnitTest.2.inc': diff --git a/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc b/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc index 1847778d09..8d4acfe0be 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc +++ b/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc @@ -89,3 +89,12 @@ $newLine; // The following line must have a single space at the end (after return) return $spaceAndNewLine; + +// Related to issue #529. These should not be auto-fixed as we don't know what to do with the comment. +yield /*comment*/ from $test(); +yield + # comment + from $test(); +yield + // phpcs:ignore Stnd.Category.SniffName + from $test(); diff --git a/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc.fixed index 4f5d3cec2f..6f83483a4d 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc.fixed +++ b/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc.fixed @@ -83,3 +83,12 @@ return $newLine; // The following line must have a single space at the end (after return) return $spaceAndNewLine; + +// Related to issue #529. These should not be auto-fixed as we don't know what to do with the comment. +yield /*comment*/ from $test(); +yield + # comment + from $test(); +yield + // phpcs:ignore Stnd.Category.SniffName + from $test(); diff --git a/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.php b/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.php index adac2c843f..10c60b5743 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.php +++ b/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.php @@ -71,6 +71,9 @@ public function getErrorList($testFile='') 85 => 1, 86 => 1, 90 => 1, + 94 => 1, + 95 => 1, + 98 => 1, ]; default: diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc index bab866e00a..74c5c0728f 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc +++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc @@ -1614,6 +1614,12 @@ match (true) { ] }; +// Issue squizlabs/PHP_CodeSniffer#3808 +function test() { + yield + from [ 3, 4 ]; +} + /* ADD NEW TESTS ABOVE THIS LINE AND MAKE SURE THAT THE 1 (space-based) AND 2 (tab-based) FILES ARE IN SYNC! */ ?> diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc.fixed index dbbfa71c40..414ea6f7f1 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc.fixed +++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc.fixed @@ -1614,6 +1614,12 @@ match (true) { ] }; +// Issue squizlabs/PHP_CodeSniffer#3808 +function test() { + yield + from [ 3, 4 ]; +} + /* ADD NEW TESTS ABOVE THIS LINE AND MAKE SURE THAT THE 1 (space-based) AND 2 (tab-based) FILES ARE IN SYNC! */ ?> diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc index de344f9f6e..c30e5b8ddd 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc +++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc @@ -1614,6 +1614,12 @@ match (true) { ] }; +// Issue squizlabs/PHP_CodeSniffer#3808 +function test() { + yield + from [ 3, 4 ]; +} + /* ADD NEW TESTS ABOVE THIS LINE AND MAKE SURE THAT THE 1 (space-based) AND 2 (tab-based) FILES ARE IN SYNC! */ ?> diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc.fixed index 3cf7fb61a6..4660f75888 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc.fixed +++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc.fixed @@ -1614,6 +1614,12 @@ match (true) { ] }; +// Issue squizlabs/PHP_CodeSniffer#3808 +function test() { + yield + from [ 3, 4 ]; +} + /* ADD NEW TESTS ABOVE THIS LINE AND MAKE SURE THAT THE 1 (space-based) AND 2 (tab-based) FILES ARE IN SYNC! */ ?> diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc index 55d1a06ab8..dd095617c2 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc +++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc @@ -1,6 +1,6 @@ -phpcs:set Generic.WhiteSpace.ScopeIndent tabIndent false -phpcs:set Generic.WhiteSpace.ScopeIndent exact true foo() ->bar() ->baz(); + +// Issue squizlabs/PHP_CodeSniffer#3808 +function test() { + yield + from [ 3, 4 ]; +} diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc.fixed index e9ae5ff392..aaa0b1c819 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc.fixed +++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc.fixed @@ -1,6 +1,6 @@ -phpcs:set Generic.WhiteSpace.ScopeIndent tabIndent false -phpcs:set Generic.WhiteSpace.ScopeIndent exact true foo() ->bar() ->baz(); + +// Issue squizlabs/PHP_CodeSniffer#3808 +function test() { + yield + from [ 3, 4 ]; +} diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.php b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.php index bb1a5d0ed4..fc9f9a8b92 100644 --- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.php +++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.php @@ -84,6 +84,7 @@ public function getErrorList($testFile='') 6 => 1, 7 => 1, 10 => 1, + 33 => 1, ]; } @@ -192,10 +193,10 @@ public function getErrorList($testFile='') 1527 => 1, 1529 => 1, 1530 => 1, - 1625 => 1, - 1626 => 1, - 1627 => 1, - 1628 => 1, + 1631 => 1, + 1632 => 1, + 1633 => 1, + 1634 => 1, ]; }//end getErrorList() diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index 42969782b7..9c6c11e4c3 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -1514,7 +1514,7 @@ protected function tokenize($string) }//end if /* - Before PHP 7.0, the "yield from" was tokenized as + Before PHP 7.0, "yield from" was tokenized as T_YIELD, T_WHITESPACE and T_STRING. So look for and change this token in earlier versions. */ @@ -1525,12 +1525,16 @@ protected function tokenize($string) && isset($tokens[($stackPtr + 1)]) === true && isset($tokens[($stackPtr + 2)]) === true && $tokens[($stackPtr + 1)][0] === T_WHITESPACE + && strpos($tokens[($stackPtr + 1)][1], $this->eolChar) === false && $tokens[($stackPtr + 2)][0] === T_STRING && strtolower($tokens[($stackPtr + 2)][1]) === 'from' ) { - // Could be multi-line, so adjust the token stack. - $token[0] = T_YIELD_FROM; - $token[1] .= $tokens[($stackPtr + 1)][1].$tokens[($stackPtr + 2)][1]; + // Single-line "yield from" with only whitespace between. + $finalTokens[$newStackPtr] = [ + 'code' => T_YIELD_FROM, + 'type' => 'T_YIELD_FROM', + 'content' => $token[1].$tokens[($stackPtr + 1)][1].$tokens[($stackPtr + 2)][1], + ]; if (PHP_CODESNIFFER_VERBOSITY > 1) { for ($i = ($stackPtr + 1); $i <= ($stackPtr + 2); $i++) { @@ -1540,9 +1544,131 @@ protected function tokenize($string) } } - $tokens[($stackPtr + 1)] = null; - $tokens[($stackPtr + 2)] = null; - } + $newStackPtr++; + $stackPtr += 2; + + continue; + } else if (PHP_VERSION_ID < 80300 + && $tokenIsArray === true + && $token[0] === T_STRING + && strtolower($token[1]) === 'from' + && $finalTokens[$lastNotEmptyToken]['code'] === T_YIELD + ) { + /* + Before PHP 8.3, if there was a comment between the "yield" and "from" keywords, + it was tokenized as T_YIELD, T_WHITESPACE, T_COMMENT... and T_STRING. + We want to keep the tokenization of the tokens between, but need to change the + `T_YIELD` and `T_STRING` (from) keywords to `T_YIELD_FROM. + */ + + $finalTokens[$lastNotEmptyToken]['code'] = T_YIELD_FROM; + $finalTokens[$lastNotEmptyToken]['type'] = 'T_YIELD_FROM'; + + $finalTokens[$newStackPtr] = [ + 'code' => T_YIELD_FROM, + 'type' => 'T_YIELD_FROM', + 'content' => $token[1], + ]; + $newStackPtr++; + + if (PHP_CODESNIFFER_VERBOSITY > 1) { + echo "\t\t* token $lastNotEmptyToken (new stack) changed into T_YIELD_FROM; was: T_YIELD".PHP_EOL; + echo "\t\t* token $stackPtr changed into T_YIELD_FROM; was: T_STRING".PHP_EOL; + } + + continue; + } else if (PHP_VERSION_ID >= 70000 + && $tokenIsArray === true + && $token[0] === T_YIELD_FROM + && strpos($token[1], $this->eolChar) !== false + && preg_match('`^yield\s+from$`i', $token[1]) === 1 + ) { + /* + In PHP 7.0+, a multi-line "yield from" (without comment) tokenizes as a single + T_YIELD_FROM token, but we want to split it and tokenize the whitespace + separately for consistency. + */ + + $finalTokens[$newStackPtr] = [ + 'code' => T_YIELD_FROM, + 'type' => 'T_YIELD_FROM', + 'content' => substr($token[1], 0, 5), + ]; + $newStackPtr++; + + $tokenLines = explode($this->eolChar, substr($token[1], 5, -4)); + $numLines = count($tokenLines); + $newToken = [ + 'type' => 'T_WHITESPACE', + 'code' => T_WHITESPACE, + 'content' => '', + ]; + + foreach ($tokenLines as $i => $line) { + $newToken['content'] = $line; + if ($i === ($numLines - 1)) { + if ($line === '') { + break; + } + } else { + $newToken['content'] .= $this->eolChar; + } + + $finalTokens[$newStackPtr] = $newToken; + $newStackPtr++; + } + + $finalTokens[$newStackPtr] = [ + 'code' => T_YIELD_FROM, + 'type' => 'T_YIELD_FROM', + 'content' => substr($token[1], -4), + ]; + $newStackPtr++; + + if (PHP_CODESNIFFER_VERBOSITY > 1) { + echo "\t\t* token $stackPtr split into 'yield', one or more whitespace tokens and 'from'".PHP_EOL; + } + + continue; + } else if (PHP_VERSION_ID >= 80300 + && $tokenIsArray === true + && $token[0] === T_YIELD_FROM + && preg_match('`^yield[ \t]+from$`i', $token[1]) !== 1 + && stripos($token[1], 'yield') === 0 + ) { + /* + Since PHP 8.3, "yield from" allows for comments and will + swallow the comment in the `T_YIELD_FROM` token. + We need to split this up to allow for sniffs handling comments. + */ + + $finalTokens[$newStackPtr] = [ + 'code' => T_YIELD_FROM, + 'type' => 'T_YIELD_FROM', + 'content' => substr($token[1], 0, 5), + ]; + $newStackPtr++; + + $yieldFromSubtokens = @token_get_all(" T_YIELD_FROM, + 1 => substr($token[1], -4), + ]; + + // Inject the new tokens into the token stack. + array_splice($tokens, ($stackPtr + 1), 0, $yieldFromSubtokens); + $numTokens = count($tokens); + + if (PHP_CODESNIFFER_VERBOSITY > 1) { + echo "\t\t* token $stackPtr split into parts (yield from with comment)".PHP_EOL; + } + + unset($yieldFromSubtokens); + continue; + }//end if /* Before PHP 5.6, the ... operator was tokenized as three diff --git a/src/Tokenizers/Tokenizer.php b/src/Tokenizers/Tokenizer.php index aa706fbfa9..8c90cd7c19 100644 --- a/src/Tokenizers/Tokenizer.php +++ b/src/Tokenizers/Tokenizer.php @@ -200,6 +200,7 @@ private function createPositionMap() T_END_HEREDOC => true, T_END_NOWDOC => true, T_INLINE_HTML => true, + T_YIELD_FROM => true, ]; $this->numTokens = count($this->tokens); diff --git a/tests/Core/Tokenizers/PHP/YieldTest.inc b/tests/Core/Tokenizers/PHP/YieldTest.inc index fdac3c71eb..3130b84600 100644 --- a/tests/Core/Tokenizers/PHP/YieldTest.inc +++ b/tests/Core/Tokenizers/PHP/YieldTest.inc @@ -22,6 +22,31 @@ function generator() FROM gen2(); + + /* testYieldFromSplitByComment */ + yield /* comment */ from gen2(); + + /* testYieldFromWithTrailingComment */ + yield // comment + from gen2(); + + /* testYieldFromWithTrailingAnnotation */ + yield // phpcs:ignore Stnd.Cat.Sniff -- for reasons. + from gen2(); + + /* testYieldFromSplitByNewLineAndComments */ + yield + /* comment line 1 + line 2 */ + // another comment + from + gen2(); + + /* testYieldFromSplitByNewLineAndAnnotation */ + YIELD + // @phpcs:disable Stnd.Cat.Sniff -- for reasons. + From + gen2(); } /* testYieldUsedAsClassName */ diff --git a/tests/Core/Tokenizers/PHP/YieldTest.php b/tests/Core/Tokenizers/PHP/YieldTest.php index 23a01ce5a1..efb12096cf 100644 --- a/tests/Core/Tokenizers/PHP/YieldTest.php +++ b/tests/Core/Tokenizers/PHP/YieldTest.php @@ -100,7 +100,11 @@ public function testYieldFromKeywordSingleToken($testMarker, $expectedContent) $this->assertSame(T_YIELD_FROM, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD_FROM (code)'); $this->assertSame('T_YIELD_FROM', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD_FROM (type)'); - $this->assertSame($expectedContent, $tokenArray['content'], 'Token content does not match expectation'); + if (isset($tokenArray['orig_content']) === true) { + $this->assertSame($expectedContent, $tokenArray['orig_content'], 'Token (orig) content does not match expectation'); + } else { + $this->assertSame($expectedContent, $tokenArray['content'], 'Token content does not match expectation'); + } }//end testYieldFromKeywordSingleToken() @@ -181,22 +185,209 @@ public function testYieldFromKeywordMultiToken($testMarker, $expectedTokens) public static function dataYieldFromKeywordMultiToken() { return [ - 'yield from with new line' => [ + 'yield from with new line' => [ 'testMarker' => '/* testYieldFromSplitByNewLines */', 'expectedTokens' => [ [ 'type' => 'T_YIELD_FROM', - 'content' => 'yield + 'content' => 'yield', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_YIELD_FROM', + 'content' => 'FROM', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + 'yield from with comment' => [ + 'testMarker' => '/* testYieldFromSplitByComment */', + 'expectedTokens' => [ + [ + 'type' => 'T_YIELD_FROM', + 'content' => 'yield', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '/* comment */', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_YIELD_FROM', + 'content' => 'from', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + ], + ], + 'yield from with trailing comment' => [ + 'testMarker' => '/* testYieldFromWithTrailingComment */', + 'expectedTokens' => [ + [ + 'type' => 'T_YIELD_FROM', + 'content' => 'yield', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// comment +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_YIELD_FROM', + 'content' => 'from', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + ], + ], + 'yield from with trailing annotation' => [ + 'testMarker' => '/* testYieldFromWithTrailingAnnotation */', + 'expectedTokens' => [ + [ + 'type' => 'T_YIELD_FROM', + 'content' => 'yield', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_PHPCS_IGNORE', + 'content' => '// phpcs:ignore Stnd.Cat.Sniff -- for reasons. +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_YIELD_FROM', + 'content' => 'from', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + ], + ], + 'yield from with new line and comment' => [ + 'testMarker' => '/* testYieldFromSplitByNewLineAndComments */', + 'expectedTokens' => [ + [ + 'type' => 'T_YIELD_FROM', + 'content' => 'yield', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '/* comment line 1 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => ' line 2 */', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '// another comment ', ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], [ 'type' => 'T_YIELD_FROM', + 'content' => 'from', + ], + [ + 'type' => 'T_WHITESPACE', 'content' => ' ', ], + ], + ], + 'yield from with new line and annotation' => [ + 'testMarker' => '/* testYieldFromSplitByNewLineAndAnnotation */', + 'expectedTokens' => [ + [ + 'type' => 'T_YIELD_FROM', + 'content' => 'YIELD', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_PHPCS_DISABLE', + 'content' => '// @phpcs:disable Stnd.Cat.Sniff -- for reasons. +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], [ 'type' => 'T_YIELD_FROM', - 'content' => ' FROM', + 'content' => 'From', ], [ 'type' => 'T_WHITESPACE', diff --git a/tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.inc b/tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.inc new file mode 100644 index 0000000000..59365dd5ce --- /dev/null +++ b/tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.inc @@ -0,0 +1,15 @@ + + * @copyright 2024 PHPCSStandards and contributors + * @license https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence + */ + +namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer; + +use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase; + +/** + * Yield from token test. + * + * @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap + */ +final class CreatePositionMapYieldFromTest extends AbstractTokenizerTestCase +{ + + + /** + * Verify that spaces/tabs in "yield from" tokens get the tab replacement treatment. + * + * @param string $testMarker The comment prefacing the target token. + * @param array $expected Expectations for the token array. + * @param string $content Optional. The test token content to search for. + * Defaults to null. + * + * @dataProvider dataYieldFromTabReplacement + * + * @return void + */ + public function testYieldFromTabReplacement($testMarker, $expected, $content=null) + { + $tokens = $this->phpcsFile->getTokens(); + $target = $this->getTargetToken($testMarker, [T_YIELD_FROM], $content); + + foreach ($expected as $key => $value) { + if ($key === 'orig_content' && $value === null) { + $this->assertArrayNotHasKey($key, $tokens[$target], "Unexpected 'orig_content' key found in the token array."); + continue; + } + + $this->assertArrayHasKey($key, $tokens[$target], "Key $key not found in the token array."); + $this->assertSame($value, $tokens[$target][$key], "Value for key $key does not match expectation."); + } + + }//end testYieldFromTabReplacement() + + + /** + * Data provider. + * + * @see testYieldFromTabReplacement() + * + * @return array>> + */ + public static function dataYieldFromTabReplacement() + { + return [ + 'Yield from, single line, single space' => [ + 'testMarker' => '/* testYieldFromHasSingleSpace */', + 'expected' => [ + 'length' => 10, + 'content' => 'yield from', + 'orig_content' => null, + ], + ], + 'Yield from, single line, multiple spaces' => [ + 'testMarker' => '/* testYieldFromHasMultiSpace */', + 'expected' => [ + 'length' => 14, + 'content' => 'yield from', + 'orig_content' => null, + ], + ], + 'Yield from, single line, has tabs' => [ + 'testMarker' => '/* testYieldFromHasTabs */', + 'expected' => [ + 'length' => 16, + 'content' => 'yield from', + 'orig_content' => 'yield from', + ], + ], + 'Yield from, single line, mix of tabs and spaces' => [ + 'testMarker' => '/* testYieldFromMixedTabsSpaces */', + 'expected' => [ + 'length' => 20, + 'content' => 'Yield From', + 'orig_content' => 'Yield From', + ], + ], + ]; + + }//end dataYieldFromTabReplacement() + + +}//end class