Tokenizer/PHP: change tokenization of long PHP open tags

src/Sniffs/AbstractPatternSniff.php

@@ -915,12 +915,12 @@ private function createTokenPattern($str)
 
         // Don't add a space after the closing php tag as it will add a new
         // whitespace token.
-        $tokenizer = new PHP('<?php '.$str.'?>', null);
+        $tokenizer = new PHP('<?php'."\n".$str.'?>', null);
         StatusWriter::resume();
 
         // Remove the <?php tag from the front and the end php tag from the back.
         $tokens = $tokenizer->getTokens();
-        $tokens = array_slice($tokens, 1, (count($tokens) - 2));
+        $tokens = array_slice($tokens, 2, (count($tokens) - 3));
 
         $patterns = [];
         foreach ($tokens as $patternInfo) {

src/Standards/Generic/Sniffs/CodeAnalysis/EmptyPHPStatementSniff.php

@@ -113,19 +113,15 @@ private function processSemicolon(File $phpcsFile, $stackPtr)
         if ($fix === true) {
             $phpcsFile->fixer->beginChangeset();
 
-            if ($tokens[$prevNonEmpty]['code'] === T_OPEN_TAG
-                || $tokens[$prevNonEmpty]['code'] === T_OPEN_TAG_WITH_ECHO
-            ) {
-                // Check for superfluous whitespace after the semicolon which should be
-                // removed as the `<?php ` open tag token already contains whitespace,
-                // either a space or a new line.
-                if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {
-                    $replacement = str_replace(' ', '', $tokens[($stackPtr + 1)]['content']);
-                    $phpcsFile->fixer->replaceToken(($stackPtr + 1), $replacement);
-                }
+            // Make sure there always remains one space between the open tag and the next content.
+            $replacement = ' ';
+            if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {
+                $replacement = '';
             }
 
-            for ($i = $stackPtr; $i > $prevNonEmpty; $i--) {
+            $phpcsFile->fixer->replaceToken($stackPtr, $replacement);
+
+            for ($i = ($stackPtr - 1); $i > $prevNonEmpty; $i--) {
                 if ($tokens[$i]['code'] !== T_SEMICOLON
                     && $tokens[$i]['code'] !== T_WHITESPACE
                 ) {

src/Standards/Generic/Tests/CodeAnalysis/EmptyPHPStatementUnitTest.2.inc

@@ -1,5 +1,5 @@
 <!-- Tests with short open tag. -->
-
+<input name="<? ;something_else(); ?>" />
 <input name="<? ; something_else(); ?>" />
 <input name="<? something_else(); ; ?>" />
 

src/Standards/Generic/Tests/CodeAnalysis/EmptyPHPStatementUnitTest.2.inc.fixed

@@ -1,6 +1,6 @@
 <!-- Tests with short open tag. -->
-
-<input name="<?something_else(); ?>" />
+<input name="<? something_else(); ?>" />
+<input name="<? something_else(); ?>" />
 <input name="<? something_else(); ?>" />
 
 /*

src/Standards/Generic/Tests/CodeAnalysis/EmptyPHPStatementUnitTest.php

@@ -93,6 +93,7 @@ public function getWarningList($testFile='')
             ];
         case 'EmptyPHPStatementUnitTest.2.inc':
             return [
+                2  => 1,
                 3  => 1,
                 4  => 1,
                 13 => 1,

src/Standards/PSR12/Sniffs/Files/OpenTagSniff.php

@@ -56,16 +56,24 @@ public function process(File $phpcsFile, $stackPtr)
             return $phpcsFile->numTokens;
         }
 
-        $next = $phpcsFile->findNext(T_INLINE_HTML, 0);
-        if ($next !== false) {
+        $hasInlineHTML = $phpcsFile->findNext(T_INLINE_HTML, 0);
+        if ($hasInlineHTML !== false) {
             // This rule only applies to PHP-only files.
             return $phpcsFile->numTokens;
         }
 
         $error = 'Opening PHP tag must be on a line by itself';
         $fix   = $phpcsFile->addFixableError($error, $stackPtr, 'NotAlone');
         if ($fix === true) {
+            $phpcsFile->fixer->beginChangeset();
+
+            // Remove whitespace between the open tag and the next non-empty token.
+            for ($i = ($stackPtr + 1); $i < $next; $i++) {
+                $phpcsFile->fixer->replaceToken($i, '');
+            }
+
             $phpcsFile->fixer->addNewline($stackPtr);
+            $phpcsFile->fixer->endChangeset();
         }
 
         return $phpcsFile->numTokens;

src/Standards/PSR12/Tests/Files/OpenTagUnitTest.2.inc.fixed

@@ -1,2 +1,2 @@
-<?php 
+<?php
 echo 'hi';

src/Standards/Squiz/Sniffs/PHP/CommentedOutCodeSniff.php

@@ -194,11 +194,13 @@ public function process(File $phpcsFile, $stackPtr)
         */
 
         // First token is always the opening tag.
-        if ($stringTokens[0]['code'] !== T_OPEN_TAG) {
+        if ($stringTokens[0]['code'] !== T_OPEN_TAG || $stringTokens[1]['code'] !== T_WHITESPACE) {
             return ($lastCommentBlockToken + 1);
         } else {
+            // Remove the PHP open tag + the whitespace token following it.
             array_shift($stringTokens);
-            --$numTokens;
+            array_shift($stringTokens);
+            $numTokens -= 2;
         }
 
         // Last token is always the closing tag, unless something went wrong.

src/Standards/Squiz/Sniffs/PHP/EmbeddedPhpSniff.php

@@ -347,16 +347,7 @@ private function validateInlineEmbeddedPhp($phpcsFile, $stackPtr, $closeTag)
         }
 
         // Check that there is one, and only one space at the start of the statement.
-        $leadingSpace  = 0;
-        $isLongOpenTag = false;
-        if ($tokens[$stackPtr]['code'] === T_OPEN_TAG
-            && stripos($tokens[$stackPtr]['content'], '<?php') === 0
-        ) {
-            // The long open tag token in a single line tag set always contains a single space after it.
-            $leadingSpace  = 1;
-            $isLongOpenTag = true;
-        }
-
+        $leadingSpace = 0;
         if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {
             $leadingSpace += $tokens[($stackPtr + 1)]['length'];
         }
@@ -366,13 +357,11 @@ private function validateInlineEmbeddedPhp($phpcsFile, $stackPtr, $closeTag)
             $data  = [$leadingSpace];
             $fix   = $phpcsFile->addFixableError($error, $stackPtr, 'SpacingAfterOpen', $data);
             if ($fix === true) {
-                if ($isLongOpenTag === true) {
-                    $phpcsFile->fixer->replaceToken(($stackPtr + 1), '');
-                } else if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {
-                    // Short open tag with too much whitespace.
+                if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {
+                    // Open tag with too much whitespace.
                     $phpcsFile->fixer->replaceToken(($stackPtr + 1), ' ');
                 } else {
-                    // Short open tag without whitespace.
+                    // Open tag without whitespace.
                     $phpcsFile->fixer->addContent($stackPtr, ' ');
                 }
             }

src/Tokenizers/PHP.php

@@ -823,6 +823,59 @@ protected function tokenize($string)
                 continue;
             }//end if
 
+            /*
+                Split whitespace off from long PHP open tag tokens and potentially join the whitespace
+                with a subsequent whitespace token.
+            */
+
+            if ($tokenIsArray === true
+                && $token[0] === T_OPEN_TAG
+                && stripos($token[1], '<?php') === 0
+            ) {
+                $openTagAndWhiteSpace = str_split($token[1], 5);
+
+                $finalTokens[$newStackPtr] = [
+                    'code'    => T_OPEN_TAG,
+                    'type'    => 'T_OPEN_TAG',
+                    'content' => $openTagAndWhiteSpace[0],
+                ];
+                $newStackPtr++;
+
+                if (isset($openTagAndWhiteSpace[1]) === true) {
+                    // The original open tag token included whitespace.
+                    // Check whether a new whitespace token needs to be inserted or if the
+                    // whitespace needs to be joined with a pre-existing whitespace
+                    // token on the same line as the open tag.
+                    if (isset($tokens[($stackPtr + 1)]) === true
+                        && $openTagAndWhiteSpace[1] === ' '
+                        && is_array($tokens[($stackPtr + 1)]) === true
+                        && $tokens[($stackPtr + 1)][0] === T_WHITESPACE
+                    ) {
+                        // Adjusting the original token stack as the "new line may be split over two tokens"
+                        // check should still be run on this token.
+                        $tokens[($stackPtr + 1)][1] = $openTagAndWhiteSpace[1].$tokens[($stackPtr + 1)][1];
+
+                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                            StatusWriter::write("* removed whitespace from T_OPEN_TAG token $stackPtr and merged it with the next token T_WHITESPACE", 2);
+                        }
+                    } else {
+                        $finalTokens[$newStackPtr] = [
+                            'code'    => T_WHITESPACE,
+                            'type'    => 'T_WHITESPACE',
+                            'content' => $openTagAndWhiteSpace[1],
+                        ];
+
+                        if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                            StatusWriter::write("* T_OPEN_TAG token $stackPtr split into T_OPEN_TAG (without whitespace) and new T_WHITESPACE token", 2);
+                        }
+
+                        $newStackPtr++;
+                    }//end if
+                }//end if
+
+                continue;
+            }//end if
+
             /*
                 Parse doc blocks into something that can be easily iterated over.
             */

tests/Core/File/FindStartOfStatementTest.php

@@ -520,7 +520,7 @@ public function testNestedMatch()
     public function testOpenTag()
     {
         $start  = $this->getTargetToken('/* testOpenTag */', T_OPEN_TAG);
-        $start += 2;
+        $start += 3;
         $found  = self::$phpcsFile->findStartOfStatement($start);
 
         $this->assertSame(($start - 1), $found);

tests/Core/Tokenizers/PHP/PHPOpenTagEOF1Test.php

@@ -43,10 +43,18 @@ public function testLongOpenTagAtEndOfFile()
             $tokens[$stackPtr]['type'],
             'Token tokenized as '.$tokens[$stackPtr]['type'].', not T_OPEN_TAG (type)'
         );
-        $this->assertSame('<?php ', $tokens[$stackPtr]['content']);
+        $this->assertSame('<?php', $tokens[$stackPtr]['content']);
+
+        $this->assertArrayHasKey(($stackPtr + 1), $tokens, 'Missing whitespace token after open tag');
+        $this->assertSame(
+            T_WHITESPACE,
+            $tokens[($stackPtr + 1)]['code'],
+            'Missing whitespace token after open tag (code)'
+        );
+        $this->assertSame(' ', $tokens[($stackPtr + 1)]['content'], 'Missing whitespace token after open tag (content)');
 
         // Now make sure that this is the very last token in the file and there are no tokens after it.
-        $this->assertArrayNotHasKey(($stackPtr + 1), $tokens);
+        $this->assertArrayNotHasKey(($stackPtr + 2), $tokens);
 
     }//end testLongOpenTagAtEndOfFile()
 

tests/Core/Tokenizers/PHP/PHPOpenTagTest.inc

@@ -0,0 +1,115 @@
+<?php /* testLongOpenTagWithNewLine */ ?>
+<?php
+echo 'with new line';
+?>
+
+<?php /* testLongOpenTagWithOneSpaceAndNewLine */ ?>
+<?php 
+echo 'with one space and new line';
+?>
+
+<?php /* testLongOpenTagWithTrailingWhiteSpaceAndNewLine */ ?>
+<?php    
+echo 'with trailing whitespace and new line';
+?>
+
+<?php /* testLongOpenTagOneSpace */ ?>
+<?php echo 'single line, one space'; ?>
+
+<?php /* testLongOpenTagMultiSpace */ ?>
+<?php      echo 'single line, multiple spaces'; ?>
+
+<?php /* testLongOpenTagWithDoubleNewLine */ ?>
+<?php
+
+echo 'with double new line';
+?>
+
+<?php /* testLongOpenTagWithTripleNewLine */ ?>
+<?php
+
+
+echo 'with triple new line';
+?>
+
+<?php /* testLongOpenTagWithNewLineAndIndentOnNextLine */ ?>
+    <?php
+    echo 'with new line and indent on next line';
+?>
+
+<!-- ====================================== -->
+
+<?php /* testCaseLongOpenTagWithNewLine */ ?>
+<?PHP
+echo 'with new line';
+?>
+
+<?php /* testCaseLongOpenTagWithOneSpaceAndNewLine */ ?>
+<?phP 
+echo 'with one space and new line';
+?>
+
+<?php /* testCaseLongOpenTagWithTrailingWhiteSpaceAndNewLine */ ?>
+<?Php    
+echo 'with trailing whitespace and new line';
+?>
+
+<?php /* testCaseLongOpenTagOneSpace */ ?>
+<?pHp echo 'single line, one space'; ?>
+
+<?php /* testCaseLongOpenTagMultiSpace */ ?>
+<?phP      echo 'single line, multiple spaces'; ?>
+
+<!-- ====================================== -->
+
+<?php /* testShortOpenEchoTagWithNewLine */ ?>
+<?=
+'with new line';
+?>
+
+<?php /* testShortOpenEchoTagWithOneSpaceAndNewLine */ ?>
+<?= 
+'with one space and new line';
+?>
+
+<?php /* testShortOpenEchoTagWithTrailingWhiteSpaceAndNewLine */ ?>
+<?=    
+'with trailing whitespace and new line';
+?>
+
+<?php /* testShortOpenEchoTagNoSpace */ ?>
+<?='single line, no space'; ?>
+
+<?php /* testShortOpenEchoTagOneSpace */ ?>
+<?= 'single line, one space'; ?>
+
+<?php /* testShortOpenEchoTagMultiSpace */ ?>
+<?=      'single line, multiple spaces'; ?>
+
+<!-- ====================================== -->
+
+<?php /* testShortOpenTagWithNewLine */ ?>
+<?
+echo 'with new line';
+?>
+
+<?php /* testShortOpenTagWithOneSpaceAndNewLine */ ?>
+<? 
+echo 'with one space and new line';
+?>
+
+<?php /* testShortOpenTagWithTrailingWhiteSpaceAndNewLine */ ?>
+<?    
+echo 'with trailing whitespace and new line';
+?>
+
+<?php /* testShortOpenTagNoSpace */ ?>
+<?echo 'single line, no space'; ?>
+
+<?php /* testShortOpenTagOneSpace */ ?>
+<? echo 'single line, one space'; ?>
+
+<?php /* testShortOpenTagMultiSpace */ ?>
+<?      echo 'single line, multiple spaces'; ?>
+
+<!-- ====================================== -->