private function _identifyTokens($tokenText, $tokenID) {
		// Split the data up by newlines
		// To correctly handle T_NEW_LINE inside comments and HTML
		$splitData = preg_split('#(\r\n|\n|\r)#', $tokenText, null, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY);
		foreach ($splitData as $data) {

			$tokenInfo = new TokenInfo();
			$tokenInfo->text = $data;
			$tokenInfo->position = $this->tokenNumber;
			$tokenInfo->line = $this->lineNumber;

			if ($data === "\r\n" || $data === "\n" || $data === "\r") {
				// This is a new line token
				$tokenInfo->id = T_NEW_LINE;
				$this->lineNumber ++;
			} else if ($data === "\t") {
				// This is a tab token
				$tokenInfo->id = T_TAB;
			} else {
				// Any other token
				$tokenInfo->id = $tokenID;
			}

			$this->tokenNumber ++;

			// Added detections
			if ($tokenInfo->id === T_UNKNOWN) {
				switch ($tokenInfo->text) {
					case ";":
						$tokenInfo->id = T_SEMICOLON;
						break;
					case "{":
						$tokenInfo->id = T_BRACES_OPEN;
						break;
					case "}":
						$tokenInfo->id = T_BRACES_CLOSE;
						break;
					case "(":
						$tokenInfo->id = T_PARENTHESIS_OPEN;
						break;
					case ")":
						$tokenInfo->id = T_PARENTHESIS_CLOSE;
						break;
					case ",":
						$tokenInfo->id = T_COMMA;
						break;
					case "=":
						$tokenInfo->id = T_EQUAL;
						break;
					case ".":
						$tokenInfo->id = T_CONCAT;
						break;
					case ":":
						$tokenInfo->id = T_COLON;
						break;
					case "-":
						$tokenInfo->id = T_MINUS;
						break;
					case "+":
						$tokenInfo->id = T_PLUS;
						break;
					case ">":
						$tokenInfo->id = T_IS_GREATER;
						break;
					case "<":
						$tokenInfo->id = T_IS_SMALLER;
						break;
					case "*":
						$tokenInfo->id = T_MULTIPLY;
						break;
					case "/":
						$tokenInfo->id = T_DIVIDE;
						break;
					case "?":
						$tokenInfo->id = T_QUESTION_MARK;
						break;
					case "%":
						$tokenInfo->id = T_MODULO;
						break;
					case "!":
						$tokenInfo->id = T_EXCLAMATION_MARK;
						break;
					case "&":
						$tokenInfo->id = T_AMPERSAND;
						break;
					case "[":
						$tokenInfo->id = T_SQUARE_BRACKET_OPEN;
						break;
					case "]":
						$tokenInfo->id = T_SQUARE_BRACKET_CLOSE;
						break;
					case "@":
						$tokenInfo->id = T_AROBAS;
						break;
					case '"':
						$tokenInfo->id = T_QUOTE;
						break;
					case '$':
						$tokenInfo->id = T_DOLLAR;
						break;
					default:
				}
			}

			$this->newTokens[] = $tokenInfo;
		}

		return $this->newTokens;
	}
private function _identifyTokens($tokenText, $tokenID) {
		// Split the data up by newlines
		// To correctly handle T_NEW_LINE inside comments and HTML
		$splitData = preg_split('#(\r\n|\n|\r)#', $tokenText, null, PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY);
		foreach ($splitData as $data) {

			$tokenInfo = new TokenInfo();
			$tokenInfo->text = $data;
			$tokenInfo->position = $this->tokenNumber;
			$tokenInfo->line = $this->lineNumber;

			if ($data === "\r\n" || $data === "\n" || $data === "\r") {
				// This is a new line token
				$tokenInfo->id = T_NEW_LINE;
				$this->lineNumber ++;
			} else if ($data === "\t") {
				// This is a tab token
				$tokenInfo->id = T_TAB;
			} else {
				// Any other token
				$tokenInfo->id = $tokenID;
			}

			$this->tokenNumber ++;

			// Added detections
			if ($tokenInfo->id === T_UNKNOWN) {
				switch ($tokenInfo->text) {
					case ";":
						$tokenInfo->id = T_SEMICOLON;
						break;
					case "{":
						$tokenInfo->id = T_BRACES_OPEN;
						break;
					case "}":
						$tokenInfo->id = T_BRACES_CLOSE;
						break;
					case "(":
						$tokenInfo->id = T_PARENTHESIS_OPEN;
						break;
					case ")":
						$tokenInfo->id = T_PARENTHESIS_CLOSE;
						break;
					case ",":
						$tokenInfo->id = T_COMMA;
						break;
					case "=":
						$tokenInfo->id = T_EQUAL;
						break;
					case ".":
						$tokenInfo->id = T_CONCAT;
						break;
					case ":":
						$tokenInfo->id = T_COLON;
						break;
					case "-":
						$tokenInfo->id = T_MINUS;
						break;
					case "+":
						$tokenInfo->id = T_PLUS;
						break;
					case ">":
						$tokenInfo->id = T_IS_GREATER;
						break;
					case "<":
						$tokenInfo->id = T_IS_SMALLER;
						break;
					case "*":
						$tokenInfo->id = T_MULTIPLY;
						break;
					case "/":
						$tokenInfo->id = T_DIVIDE;
						break;
					case "?":
						$tokenInfo->id = T_QUESTION_MARK;
						break;
					case "%":
						$tokenInfo->id = T_MODULO;
						break;
					case "!":
						$tokenInfo->id = T_EXCLAMATION_MARK;
						break;
					case "&":
						$tokenInfo->id = T_AMPERSAND;
						break;
					case "[":
						$tokenInfo->id = T_SQUARE_BRACKET_OPEN;
						break;
					case "]":
						$tokenInfo->id = T_SQUARE_BRACKET_CLOSE;
						break;
					case "@":
						$tokenInfo->id = T_AROBAS;
						break;
					case '"':
						$tokenInfo->id = T_QUOTE;
						break;
					case '$':
						$tokenInfo->id = T_DOLLAR;
						break;
					default:
				}
			}

			$this->newTokens[] = $tokenInfo;
		}

		return $this->newTokens;
	}
private function _identifyTokens ( $ tokenText , $ tokenID ) { // Split the data up by newlines // To correctly handle T_NEW_LINE inside comments and HTML $ splitData = preg_split ( '#(\r\n|\n|\r)#' , $ tokenText , null , PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY ) ; foreach ( $ splitData as $ data ) { $ tokenInfo = new TokenInfo ( ) ; $ tokenInfo -> text = $ data ; $ tokenInfo -> position = $ this -> tokenNumber ; $ tokenInfo -> line = $ this -> lineNumber ; if ( $ data === "\r\n" || $ data === "\n" || $ data === "\r" ) { // This is a new line token $ tokenInfo -> id = T_NEW_LINE ; $ this -> lineNumber ++ ; } else if ( $ data === "\t" ) { // This is a tab token $ tokenInfo -> id = T_TAB ; } else { // Any other token $ tokenInfo -> id = $ tokenID ; } $ this -> tokenNumber ++ ; // Added detections if ( $ tokenInfo -> id === T_UNKNOWN ) { switch ( $ tokenInfo -> text ) { case ";" : $ tokenInfo -> id = T_SEMICOLON ; break ; case "{" : $ tokenInfo -> id = T_BRACES_OPEN ; break ; case "}" : $ tokenInfo -> id = T_BRACES_CLOSE ; break ; case "(" : $ tokenInfo -> id = T_PARENTHESIS_OPEN ; break ; case ")" : $ tokenInfo -> id = T_PARENTHESIS_CLOSE ; break ; case "," : $ tokenInfo -> id = T_COMMA ; break ; case "=" : $ tokenInfo -> id = T_EQUAL ; break ; case "." : $ tokenInfo -> id = T_CONCAT ; break ; case ":" : $ tokenInfo -> id = T_COLON ; break ; case "-" : $ tokenInfo -> id = T_MINUS ; break ; case "+" : $ tokenInfo -> id = T_PLUS ; break ; case ">" : $ tokenInfo -> id = T_IS_GREATER ; break ; case "<" : $ tokenInfo -> id = T_IS_SMALLER ; break ; case "*" : $ tokenInfo -> id = T_MULTIPLY ; break ; case "/" : $ tokenInfo -> id = T_DIVIDE ; break ; case "?" : $ tokenInfo -> id = T_QUESTION_MARK ; break ; case "%" : $ tokenInfo -> id = T_MODULO ; break ; case "!" : $ tokenInfo -> id = T_EXCLAMATION_MARK ; break ; case "&" : $ tokenInfo -> id = T_AMPERSAND ; break ; case "[" : $ tokenInfo -> id = T_SQUARE_BRACKET_OPEN ; break ; case "]" : $ tokenInfo -> id = T_SQUARE_BRACKET_CLOSE ; break ; case "@" : $ tokenInfo -> id = T_AROBAS ; break ; case '"' : $ tokenInfo -> id = T_QUOTE ; break ; case '$' : $ tokenInfo -> id = T_DOLLAR ; break ; default : } } $ this -> newTokens [ ] = $ tokenInfo ; } return $ this -> newTokens ; }
Identify the token and enventually split the new lines.

@param String $tokenText
The token text
@param Integer $tokenText
The token text
@return Array[TokenInfo]
Identify the token and enventually split the new lines .