public function tokens($source, $offset)
    {
        $tokens = array();

        $rawTokens = token_get_all('<?php ' . substr($source, $offset));
        array_shift($rawTokens);

        foreach ($rawTokens as $token) {
            $tokens = array_merge($tokens, $this->normalizeToken($token));
        }

        $tokens = $this->collapseQuotedStrings($tokens);
        $tokens = $this->collapseConsecutiveStrings($tokens);

        return $tokens;
    }
public function tokens($source, $offset)
    {
        $tokens = array();

        $rawTokens = token_get_all('<?php ' . substr($source, $offset));
        array_shift($rawTokens);

        foreach ($rawTokens as $token) {
            $tokens = array_merge($tokens, $this->normalizeToken($token));
        }

        $tokens = $this->collapseQuotedStrings($tokens);
        $tokens = $this->collapseConsecutiveStrings($tokens);

        return $tokens;
    }
public function tokens ( $ source , $ offset ) { $ tokens = array ( ) ; $ rawTokens = token_get_all ( '<?php ' . substr ( $ source , $ offset ) ) ; array_shift ( $ rawTokens ) ; foreach ( $ rawTokens as $ token ) { $ tokens = array_merge ( $ tokens , $ this -> normalizeToken ( $ token ) ) ; } $ tokens = $ this -> collapseQuotedStrings ( $ tokens ) ; $ tokens = $ this -> collapseConsecutiveStrings ( $ tokens ) ; return $ tokens ; }
Produce tokens from the provided source.

@param string  $source The source.
@param integer $offset The offset to start lexing at.

@return array<TypeToken> The tokens.
Produce tokens from the provided source .