public function getTokens()
    {
        fseek($this->stream, 0);
        $this->buffer = new StreamBuffer($this->stream, static::BUFFER_SIZE, $this->minLength);
        $this->buffer->read();

        /** @var Token $last */
        $last = null;

        while (!$this->buffer->isEof()) {
            foreach ($this->state->match($this->buffer) as $token) {
                if ($token[0] == Token::T_BOM) {
                    $this->changeEncoding($token[1]);
                }

                $this->state = $this->state->getNextState($token[0]);

                // merge tokens together to condense T_CONTENT tokens
                if ($token[0] == Token::T_CONTENT) {
                    if (!is_null($last)) {
                        $last[1] .= $token[1];
                        $last[3] = strlen($last[1]);
                    } else {
                        $last = $token;
                    }
                } else {
                    if (!is_null($last)) {
                        yield $last;
                        $last = null;
                    }
                    yield $token;
                }

                $this->buffer->move($token[3]);
                $this->buffer->read();
            }
        }

        if (!is_null($last)) {
            yield $last;
        }

        fclose($this->stream);
    }
public function getTokens()
    {
        fseek($this->stream, 0);
        $this->buffer = new StreamBuffer($this->stream, static::BUFFER_SIZE, $this->minLength);
        $this->buffer->read();

        /** @var Token $last */
        $last = null;

        while (!$this->buffer->isEof()) {
            foreach ($this->state->match($this->buffer) as $token) {
                if ($token[0] == Token::T_BOM) {
                    $this->changeEncoding($token[1]);
                }

                $this->state = $this->state->getNextState($token[0]);

                // merge tokens together to condense T_CONTENT tokens
                if ($token[0] == Token::T_CONTENT) {
                    if (!is_null($last)) {
                        $last[1] .= $token[1];
                        $last[3] = strlen($last[1]);
                    } else {
                        $last = $token;
                    }
                } else {
                    if (!is_null($last)) {
                        yield $last;
                        $last = null;
                    }
                    yield $token;
                }

                $this->buffer->move($token[3]);
                $this->buffer->read();
            }
        }

        if (!is_null($last)) {
            yield $last;
        }

        fclose($this->stream);
    }
public function getTokens ( ) { fseek ( $ this -> stream , 0 ) ; $ this -> buffer = new StreamBuffer ( $ this -> stream , static :: BUFFER_SIZE , $ this -> minLength ) ; $ this -> buffer -> read ( ) ; /** @var Token $last */ $ last = null ; while ( ! $ this -> buffer -> isEof ( ) ) { foreach ( $ this -> state -> match ( $ this -> buffer ) as $ token ) { if ( $ token [ 0 ] == Token :: T_BOM ) { $ this -> changeEncoding ( $ token [ 1 ] ) ; } $ this -> state = $ this -> state -> getNextState ( $ token [ 0 ] ) ; // merge tokens together to condense T_CONTENT tokens if ( $ token [ 0 ] == Token :: T_CONTENT ) { if ( ! is_null ( $ last ) ) { $ last [ 1 ] .= $ token [ 1 ] ; $ last [ 3 ] = strlen ( $ last [ 1 ] ) ; } else { $ last = $ token ; } } else { if ( ! is_null ( $ last ) ) { yield $ last ; $ last = null ; } yield $ token ; } $ this -> buffer -> move ( $ token [ 3 ] ) ; $ this -> buffer -> read ( ) ; } } if ( ! is_null ( $ last ) ) { yield $ last ; } fclose ( $ this -> stream ) ; }
Loop through the stream, pulling maximum type length each time, find the largest type that matches and create a
token, then move on length characters

@return Iterator
Loop through the stream pulling maximum type length each time find the largest type that matches and create a token then move on length characters