private function normalizeTokens(array $tokens)
    {
        $nTokens = array();
        for ($i=0,$c=count($tokens); $i<$c; $i++) {
            $token = $tokens[$i];

            if (is_string($token)) {
                $nTokens[] = new LiteralToken($token, end($nTokens)->getEndLine());
                continue;
            }

            switch ($token[0]) {
                case T_WHITESPACE:
                    $lines = explode("\n", $token[1]);
                    for ($j=0,$k=count($lines); $j<$k; $j++) {
                        $line = $lines[$j].($j+1 === $k ? '' : "\n");

                        if ($j+1 === $k && '' === $line) {
                            break;
                        }

                        $nTokens[] = new PhpToken(array(T_WHITESPACE, $line, $token[2] + $j));
                    }
                    break;

                default:
                    // remove any trailing whitespace of the token
                    if (preg_match('/^(.*?)(\s+)$/', $token[1], $match)) {
                        $nTokens[] = new PhpToken(array($token[0], $match[1], $token[2]));

                        // if the next token is whitespace, change it
                        if (isset($tokens[$i+1]) && !is_string($tokens[$i+1])
                                && T_WHITESPACE === $tokens[$i+1][0]) {
                            $tokens[$i+1][1] = $match[2].$tokens[$i+1][1];
                            $tokens[$i+1][2] = $token[2];
                        } else {
                            $nTokens[] = new PhpToken(array(T_WHITESPACE, $match[2], $token[2]));
                        }
                    } else {
                        $nTokens[] = new PhpToken($token);
                    }
            }
        }

        return $nTokens;
    }
private function normalizeTokens(array $tokens)
    {
        $nTokens = array();
        for ($i=0,$c=count($tokens); $i<$c; $i++) {
            $token = $tokens[$i];

            if (is_string($token)) {
                $nTokens[] = new LiteralToken($token, end($nTokens)->getEndLine());
                continue;
            }

            switch ($token[0]) {
                case T_WHITESPACE:
                    $lines = explode("\n", $token[1]);
                    for ($j=0,$k=count($lines); $j<$k; $j++) {
                        $line = $lines[$j].($j+1 === $k ? '' : "\n");

                        if ($j+1 === $k && '' === $line) {
                            break;
                        }

                        $nTokens[] = new PhpToken(array(T_WHITESPACE, $line, $token[2] + $j));
                    }
                    break;

                default:
                    // remove any trailing whitespace of the token
                    if (preg_match('/^(.*?)(\s+)$/', $token[1], $match)) {
                        $nTokens[] = new PhpToken(array($token[0], $match[1], $token[2]));

                        // if the next token is whitespace, change it
                        if (isset($tokens[$i+1]) && !is_string($tokens[$i+1])
                                && T_WHITESPACE === $tokens[$i+1][0]) {
                            $tokens[$i+1][1] = $match[2].$tokens[$i+1][1];
                            $tokens[$i+1][2] = $token[2];
                        } else {
                            $nTokens[] = new PhpToken(array(T_WHITESPACE, $match[2], $token[2]));
                        }
                    } else {
                        $nTokens[] = new PhpToken($token);
                    }
            }
        }

        return $nTokens;
    }
private function normalizeTokens ( array $ tokens ) { $ nTokens = array ( ) ; for ( $ i = 0 , $ c = count ( $ tokens ) ; $ i < $ c ; $ i ++ ) { $ token = $ tokens [ $ i ] ; if ( is_string ( $ token ) ) { $ nTokens [ ] = new LiteralToken ( $ token , end ( $ nTokens ) -> getEndLine ( ) ) ; continue ; } switch ( $ token [ 0 ] ) { case T_WHITESPACE : $ lines = explode ( "\n" , $ token [ 1 ] ) ; for ( $ j = 0 , $ k = count ( $ lines ) ; $ j < $ k ; $ j ++ ) { $ line = $ lines [ $ j ] . ( $ j + 1 === $ k ? '' : "\n" ) ; if ( $ j + 1 === $ k && '' === $ line ) { break ; } $ nTokens [ ] = new PhpToken ( array ( T_WHITESPACE , $ line , $ token [ 2 ] + $ j ) ) ; } break ; default : // remove any trailing whitespace of the token if ( preg_match ( '/^(.*?)(\s+)$/' , $ token [ 1 ] , $ match ) ) { $ nTokens [ ] = new PhpToken ( array ( $ token [ 0 ] , $ match [ 1 ] , $ token [ 2 ] ) ) ; // if the next token is whitespace, change it if ( isset ( $ tokens [ $ i + 1 ] ) && ! is_string ( $ tokens [ $ i + 1 ] ) && T_WHITESPACE === $ tokens [ $ i + 1 ] [ 0 ] ) { $ tokens [ $ i + 1 ] [ 1 ] = $ match [ 2 ] . $ tokens [ $ i + 1 ] [ 1 ] ; $ tokens [ $ i + 1 ] [ 2 ] = $ token [ 2 ] ; } else { $ nTokens [ ] = new PhpToken ( array ( T_WHITESPACE , $ match [ 2 ] , $ token [ 2 ] ) ) ; } } else { $ nTokens [ ] = new PhpToken ( $ token ) ; } } } return $ nTokens ; }
Normalizes the original PHP token stream into a format which is more
usable for analysis.

@param array $tokens the format returned by ``token_get_all``

@return array the normalized tokens
Normalizes the original PHP token stream into a format which is more usable for analysis .