private function _getAllTokens($source) {
		$newTokens = array();

		// Ugly trick
		// Reset the error array by calling an undefined variable
		set_error_handler('var_dump', 0);
		@$errLastResetUndefinedVar;
		restore_error_handler();

		// Get the tokens
		$tokens = @token_get_all($source);

		// Check for parsing errors
		$parsingErrors = error_get_last();
		if (!empty($parsingErrors) && $parsingErrors["type"] === 128) {
			throw new Exception($parsingErrors["message"]);
		}

		// Check each token and transform into an Object
		foreach ($tokens as $token) {
			$isTokenArray = is_array($token);

			$tokenID = $isTokenArray ? $token[0] : T_UNKNOWN;
			$tokenText = $isTokenArray ? $token[1] : $token;

			// Manage T_OPEN_TAG when php.ini setting short_open_tag is Off.
			if ($this->shortOpenTagOff && $tokenID === T_INLINE_HTML) {

				$startPos = strpos($tokenText, SHORT_OPEN_TAG);
				$endPos = strpos($tokenText, CLOSE_TAG, $startPos + strlen(SHORT_OPEN_TAG));

				// Extract the content of the short_open_tag
				while (strlen($tokenText) > 2 && $startPos !== false && $endPos !== false) {

					// Parse the beginning of the text
					$beforeText = substr($tokenText, 0, $startPos);
					$this->_identifyTokens($beforeText, $tokenID);

					// The open tag
					$openTag = new TokenInfo();
					$openTag->id = T_OPEN_TAG;
					$openTag->text = SHORT_OPEN_TAG;
					$this->tokenNumber ++;
					$openTag->position = $this->tokenNumber;
					$openTag->line = $this->lineNumber;
					$this->newTokens[] = $openTag;

					// Tokenize the content
					$inlineText = substr($tokenText, $startPos + strlen(SHORT_OPEN_TAG), $endPos - $startPos);
					$inlineText = substr($inlineText, 0, -strlen(CLOSE_TAG));

					$inline = $this->_getAllTokens(OPEN_TAG . " " . $inlineText);

					array_shift($inline); // remove <?php
					$this->_identifyTokens($newTokens, $inline);

					// Add the close tag
					$closeTag = new TokenInfo();
					$closeTag->id = T_CLOSE_TAG;
					$closeTag->text = CLOSE_TAG;
					$this->tokenNumber ++;
					$closeTag->position = $this->tokenNumber;
					$closeTag->line = $this->lineNumber;
					$this->newTokens[] = $closeTag;

					// text = the remaining text
					$tokenText = substr($tokenText, $endPos + strlen(SHORT_OPEN_TAG));

					$startPos = strpos($tokenText, SHORT_OPEN_TAG);
					$endPos = strpos($tokenText, CLOSE_TAG, $startPos + strlen(SHORT_OPEN_TAG));
				}
			}

			// Identify the tokens
			$this->_identifyTokens($tokenText, $tokenID);
		}

		return $this->newTokens;
	}
private function _getAllTokens($source) {
		$newTokens = array();

		// Ugly trick
		// Reset the error array by calling an undefined variable
		set_error_handler('var_dump', 0);
		@$errLastResetUndefinedVar;
		restore_error_handler();

		// Get the tokens
		$tokens = @token_get_all($source);

		// Check for parsing errors
		$parsingErrors = error_get_last();
		if (!empty($parsingErrors) && $parsingErrors["type"] === 128) {
			throw new Exception($parsingErrors["message"]);
		}

		// Check each token and transform into an Object
		foreach ($tokens as $token) {
			$isTokenArray = is_array($token);

			$tokenID = $isTokenArray ? $token[0] : T_UNKNOWN;
			$tokenText = $isTokenArray ? $token[1] : $token;

			// Manage T_OPEN_TAG when php.ini setting short_open_tag is Off.
			if ($this->shortOpenTagOff && $tokenID === T_INLINE_HTML) {

				$startPos = strpos($tokenText, SHORT_OPEN_TAG);
				$endPos = strpos($tokenText, CLOSE_TAG, $startPos + strlen(SHORT_OPEN_TAG));

				// Extract the content of the short_open_tag
				while (strlen($tokenText) > 2 && $startPos !== false && $endPos !== false) {

					// Parse the beginning of the text
					$beforeText = substr($tokenText, 0, $startPos);
					$this->_identifyTokens($beforeText, $tokenID);

					// The open tag
					$openTag = new TokenInfo();
					$openTag->id = T_OPEN_TAG;
					$openTag->text = SHORT_OPEN_TAG;
					$this->tokenNumber ++;
					$openTag->position = $this->tokenNumber;
					$openTag->line = $this->lineNumber;
					$this->newTokens[] = $openTag;

					// Tokenize the content
					$inlineText = substr($tokenText, $startPos + strlen(SHORT_OPEN_TAG), $endPos - $startPos);
					$inlineText = substr($inlineText, 0, -strlen(CLOSE_TAG));

					$inline = $this->_getAllTokens(OPEN_TAG . " " . $inlineText);

					array_shift($inline); // remove <?php
					$this->_identifyTokens($newTokens, $inline);

					// Add the close tag
					$closeTag = new TokenInfo();
					$closeTag->id = T_CLOSE_TAG;
					$closeTag->text = CLOSE_TAG;
					$this->tokenNumber ++;
					$closeTag->position = $this->tokenNumber;
					$closeTag->line = $this->lineNumber;
					$this->newTokens[] = $closeTag;

					// text = the remaining text
					$tokenText = substr($tokenText, $endPos + strlen(SHORT_OPEN_TAG));

					$startPos = strpos($tokenText, SHORT_OPEN_TAG);
					$endPos = strpos($tokenText, CLOSE_TAG, $startPos + strlen(SHORT_OPEN_TAG));
				}
			}

			// Identify the tokens
			$this->_identifyTokens($tokenText, $tokenID);
		}

		return $this->newTokens;
	}
private function _getAllTokens ( $ source ) { $ newTokens = array ( ) ; // Ugly trick // Reset the error array by calling an undefined variable set_error_handler ( 'var_dump' , 0 ) ; @ $ errLastResetUndefinedVar ; restore_error_handler ( ) ; // Get the tokens $ tokens = @ token_get_all ( $ source ) ; // Check for parsing errors $ parsingErrors = error_get_last ( ) ; if ( ! empty ( $ parsingErrors ) && $ parsingErrors [ "type" ] === 128 ) { throw new Exception ( $ parsingErrors [ "message" ] ) ; } // Check each token and transform into an Object foreach ( $ tokens as $ token ) { $ isTokenArray = is_array ( $ token ) ; $ tokenID = $ isTokenArray ? $ token [ 0 ] : T_UNKNOWN ; $ tokenText = $ isTokenArray ? $ token [ 1 ] : $ token ; // Manage T_OPEN_TAG when php.ini setting short_open_tag is Off. if ( $ this -> shortOpenTagOff && $ tokenID === T_INLINE_HTML ) { $ startPos = strpos ( $ tokenText , SHORT_OPEN_TAG ) ; $ endPos = strpos ( $ tokenText , CLOSE_TAG , $ startPos + strlen ( SHORT_OPEN_TAG ) ) ; // Extract the content of the short_open_tag while ( strlen ( $ tokenText ) > 2 && $ startPos !== false && $ endPos !== false ) { // Parse the beginning of the text $ beforeText = substr ( $ tokenText , 0 , $ startPos ) ; $ this -> _identifyTokens ( $ beforeText , $ tokenID ) ; // The open tag $ openTag = new TokenInfo ( ) ; $ openTag -> id = T_OPEN_TAG ; $ openTag -> text = SHORT_OPEN_TAG ; $ this -> tokenNumber ++ ; $ openTag -> position = $ this -> tokenNumber ; $ openTag -> line = $ this -> lineNumber ; $ this -> newTokens [ ] = $ openTag ; // Tokenize the content $ inlineText = substr ( $ tokenText , $ startPos + strlen ( SHORT_OPEN_TAG ) , $ endPos - $ startPos ) ; $ inlineText = substr ( $ inlineText , 0 , - strlen ( CLOSE_TAG ) ) ; $ inline = $ this -> _getAllTokens ( OPEN_TAG . " " . $ inlineText ) ; array_shift ( $ inline ) ; // remove <?php $ this -> _identifyTokens ( $ newTokens , $ inline ) ; // Add the close tag $ closeTag = new TokenInfo ( ) ; $ closeTag -> id = T_CLOSE_TAG ; $ closeTag -> text = CLOSE_TAG ; $ this -> tokenNumber ++ ; $ closeTag -> position = $ this -> tokenNumber ; $ closeTag -> line = $ this -> lineNumber ; $ this -> newTokens [ ] = $ closeTag ; // text = the remaining text $ tokenText = substr ( $ tokenText , $ endPos + strlen ( SHORT_OPEN_TAG ) ) ; $ startPos = strpos ( $ tokenText , SHORT_OPEN_TAG ) ; $ endPos = strpos ( $ tokenText , CLOSE_TAG , $ startPos + strlen ( SHORT_OPEN_TAG ) ) ; } } // Identify the tokens $ this -> _identifyTokens ( $ tokenText , $ tokenID ) ; } return $ this -> newTokens ; }
Tokenize a string and separate the newline token.

Found here : http://php.net/manual/function.token-get-all.php

@param String $source
The source code to analyse
@return array
@SuppressWarnings checkUnusedVariables
Tokenize a string and separate the newline token .