public function tokenize($input)
	{
		$tokens = [];
		$exp = ltrim($input);
		
		for($length = strlen($exp), $i = 0; $i < $length; $i++)
		{
			if(trim($exp[$i]) == '')
			{
				continue;
			}
			
			if(NULL !== ($token = $this->getTerminalToken($exp, $i)))
			{
				$tokens[] = $token;
				
				continue;
			}
			
			if(NULL !== ($token = $this->getStringToken($exp, $length, $i)))
			{
				$tokens[] = $token;
				
				continue;
			}
			
			$sub = substr($exp, $i);
			$m = NULL;
			
			if(preg_match(self::PATTERN_DECIMAL, $sub, $m))
			{
				$tokens[] = $this->getDecimalToken($m);
				$i += strlen($m[1]) - 1;
				
				continue;
			}
			
			if(preg_match(self::PATTERN_INTEGER, $sub, $m))
			{
				$tokens[] = $this->getIntegerToken($m);	
				$i += strlen($m[1]) - 1;
				
				continue;
			}
			
			if(preg_match(self::PATTERN_IDENTIFIER, $sub, $m))
			{
				$tokens[] = $this->getIdentifierToken($m);
				$i += strlen($m[0]) - 1;
				
				continue;
			}
		}
		
		return new AnnotationTokenSequence($tokens);
	}
public function tokenize($input)
	{
		$tokens = [];
		$exp = ltrim($input);
		
		for($length = strlen($exp), $i = 0; $i < $length; $i++)
		{
			if(trim($exp[$i]) == '')
			{
				continue;
			}
			
			if(NULL !== ($token = $this->getTerminalToken($exp, $i)))
			{
				$tokens[] = $token;
				
				continue;
			}
			
			if(NULL !== ($token = $this->getStringToken($exp, $length, $i)))
			{
				$tokens[] = $token;
				
				continue;
			}
			
			$sub = substr($exp, $i);
			$m = NULL;
			
			if(preg_match(self::PATTERN_DECIMAL, $sub, $m))
			{
				$tokens[] = $this->getDecimalToken($m);
				$i += strlen($m[1]) - 1;
				
				continue;
			}
			
			if(preg_match(self::PATTERN_INTEGER, $sub, $m))
			{
				$tokens[] = $this->getIntegerToken($m);	
				$i += strlen($m[1]) - 1;
				
				continue;
			}
			
			if(preg_match(self::PATTERN_IDENTIFIER, $sub, $m))
			{
				$tokens[] = $this->getIdentifierToken($m);
				$i += strlen($m[0]) - 1;
				
				continue;
			}
		}
		
		return new AnnotationTokenSequence($tokens);
	}
public function tokenize ( $ input ) { $ tokens = [ ] ; $ exp = ltrim ( $ input ) ; for ( $ length = strlen ( $ exp ) , $ i = 0 ; $ i < $ length ; $ i ++ ) { if ( trim ( $ exp [ $ i ] ) == '' ) { continue ; } if ( NULL !== ( $ token = $ this -> getTerminalToken ( $ exp , $ i ) ) ) { $ tokens [ ] = $ token ; continue ; } if ( NULL !== ( $ token = $ this -> getStringToken ( $ exp , $ length , $ i ) ) ) { $ tokens [ ] = $ token ; continue ; } $ sub = substr ( $ exp , $ i ) ; $ m = NULL ; if ( preg_match ( self :: PATTERN_DECIMAL , $ sub , $ m ) ) { $ tokens [ ] = $ this -> getDecimalToken ( $ m ) ; $ i += strlen ( $ m [ 1 ] ) - 1 ; continue ; } if ( preg_match ( self :: PATTERN_INTEGER , $ sub , $ m ) ) { $ tokens [ ] = $ this -> getIntegerToken ( $ m ) ; $ i += strlen ( $ m [ 1 ] ) - 1 ; continue ; } if ( preg_match ( self :: PATTERN_IDENTIFIER , $ sub , $ m ) ) { $ tokens [ ] = $ this -> getIdentifierToken ( $ m ) ; $ i += strlen ( $ m [ 0 ] ) - 1 ; continue ; } } return new AnnotationTokenSequence ( $ tokens ) ; }
Tokenize the given source string into annotation tokens and returns
the created token sequence.

@param string $input
@return AnnotationTokenSequence
Tokenize the given source string into annotation tokens and returns the created token sequence .