public function tokenize($code)
    {
        $this->tokens = array();
        $lines        = preg_split('/\r?\n/', $code);
        $number       = 0;

        foreach ($lines as $number => $fragment) {
            $this->processLine($number + 1, $fragment);
            $this->tokens[] = new Token(self::TOKEN_EOL, null, $number + 1);
        }
        $this->tokens[] = new Token(self::TOKEN_EOF, null, $number + 1);

        return new TokenStream($this->tokens);
    }
public function tokenize($code)
    {
        $this->tokens = array();
        $lines        = preg_split('/\r?\n/', $code);
        $number       = 0;

        foreach ($lines as $number => $fragment) {
            $this->processLine($number + 1, $fragment);
            $this->tokens[] = new Token(self::TOKEN_EOL, null, $number + 1);
        }
        $this->tokens[] = new Token(self::TOKEN_EOF, null, $number + 1);

        return new TokenStream($this->tokens);
    }
public function tokenize ( $ code ) { $ this -> tokens = array ( ) ; $ lines = preg_split ( '/\r?\n/' , $ code ) ; $ number = 0 ; foreach ( $ lines as $ number => $ fragment ) { $ this -> processLine ( $ number + 1 , $ fragment ) ; $ this -> tokens [ ] = new Token ( self :: TOKEN_EOL , null , $ number + 1 ) ; } $ this -> tokens [ ] = new Token ( self :: TOKEN_EOF , null , $ number + 1 ) ; return new TokenStream ( $ this -> tokens ) ; }
Tokenize the code

@param string $code The content of pcd file

@return TokenStream
Tokenize the code