public function lex($input, $path = null)
    {
        $stateClassName = $this->getOption('lexer_state_class_name');
        $lexEvent = new LexEvent($input, $path, $stateClassName, [
            'encoding'                 => $this->getOption('encoding'),
            'indent_style'             => $this->getOption('indent_style'),
            'indent_width'             => $this->getOption('indent_width'),
            'allow_mixed_indent'       => $this->getOption('allow_mixed_indent'),
            'multiline_markup_enabled' => $this->getOption('multiline_markup_enabled'),
            'level'                    => $this->getOption('level'),
        ]);

        $this->trigger($lexEvent);

        $input = $lexEvent->getInput();
        $path = $lexEvent->getPath();
        $stateClassName = $lexEvent->getStateClassName();
        $stateOptions = $lexEvent->getStateOptions();

        $stateOptions['path'] = $path;
        $stateOptions['keyword_names'] = array_keys($this->getOption('keywords') ?: []);

        if (!is_a($stateClassName, State::class, true)) {
            throw new \InvalidArgumentException(
                'lexer_state_class_name needs to be a valid '.State::class.' sub class, '.
                $stateClassName.' given'
            );
        }

        //Put together our initial state
        $this->state = new $stateClassName($this, $input, $stateOptions);

        $scanners = $this->getOption('scanners');

        //We always scan for text at the very end.
        $scanners['final_plain_text'] = TextScanner::class;

        //Scan for tokens
        //N> yield from $this->handleTokens($this->>state->loopScan($scanners));
        $tokens = $this->state->loopScan($scanners);
        foreach ($this->handleTokens($tokens) as $token) {
            yield $token;
        }

        $this->trigger(new EndLexEvent($lexEvent));

        //Free state
        $this->state = null;
        $this->lastToken = null;
    }
public function lex($input, $path = null)
    {
        $stateClassName = $this->getOption('lexer_state_class_name');
        $lexEvent = new LexEvent($input, $path, $stateClassName, [
            'encoding'                 => $this->getOption('encoding'),
            'indent_style'             => $this->getOption('indent_style'),
            'indent_width'             => $this->getOption('indent_width'),
            'allow_mixed_indent'       => $this->getOption('allow_mixed_indent'),
            'multiline_markup_enabled' => $this->getOption('multiline_markup_enabled'),
            'level'                    => $this->getOption('level'),
        ]);

        $this->trigger($lexEvent);

        $input = $lexEvent->getInput();
        $path = $lexEvent->getPath();
        $stateClassName = $lexEvent->getStateClassName();
        $stateOptions = $lexEvent->getStateOptions();

        $stateOptions['path'] = $path;
        $stateOptions['keyword_names'] = array_keys($this->getOption('keywords') ?: []);

        if (!is_a($stateClassName, State::class, true)) {
            throw new \InvalidArgumentException(
                'lexer_state_class_name needs to be a valid '.State::class.' sub class, '.
                $stateClassName.' given'
            );
        }

        //Put together our initial state
        $this->state = new $stateClassName($this, $input, $stateOptions);

        $scanners = $this->getOption('scanners');

        //We always scan for text at the very end.
        $scanners['final_plain_text'] = TextScanner::class;

        //Scan for tokens
        //N> yield from $this->handleTokens($this->>state->loopScan($scanners));
        $tokens = $this->state->loopScan($scanners);
        foreach ($this->handleTokens($tokens) as $token) {
            yield $token;
        }

        $this->trigger(new EndLexEvent($lexEvent));

        //Free state
        $this->state = null;
        $this->lastToken = null;
    }
public function lex ( $ input , $ path = null ) { $ stateClassName = $ this -> getOption ( 'lexer_state_class_name' ) ; $ lexEvent = new LexEvent ( $ input , $ path , $ stateClassName , [ 'encoding' => $ this -> getOption ( 'encoding' ) , 'indent_style' => $ this -> getOption ( 'indent_style' ) , 'indent_width' => $ this -> getOption ( 'indent_width' ) , 'allow_mixed_indent' => $ this -> getOption ( 'allow_mixed_indent' ) , 'multiline_markup_enabled' => $ this -> getOption ( 'multiline_markup_enabled' ) , 'level' => $ this -> getOption ( 'level' ) , ] ) ; $ this -> trigger ( $ lexEvent ) ; $ input = $ lexEvent -> getInput ( ) ; $ path = $ lexEvent -> getPath ( ) ; $ stateClassName = $ lexEvent -> getStateClassName ( ) ; $ stateOptions = $ lexEvent -> getStateOptions ( ) ; $ stateOptions [ 'path' ] = $ path ; $ stateOptions [ 'keyword_names' ] = array_keys ( $ this -> getOption ( 'keywords' ) ? : [ ] ) ; if ( ! is_a ( $ stateClassName , State :: class , true ) ) { throw new \ InvalidArgumentException ( 'lexer_state_class_name needs to be a valid ' . State :: class . ' sub class, ' . $ stateClassName . ' given' ) ; } //Put together our initial state $ this -> state = new $ stateClassName ( $ this , $ input , $ stateOptions ) ; $ scanners = $ this -> getOption ( 'scanners' ) ; //We always scan for text at the very end. $ scanners [ 'final_plain_text' ] = TextScanner :: class ; //Scan for tokens //N> yield from $this->handleTokens($this->>state->loopScan($scanners)); $ tokens = $ this -> state -> loopScan ( $ scanners ) ; foreach ( $ this -> handleTokens ( $ tokens ) as $ token ) { yield $ token ; } $ this -> trigger ( new EndLexEvent ( $ lexEvent ) ) ; //Free state $ this -> state = null ; $ this -> lastToken = null ; }
Returns a generator that will lex the passed input sequentially.

If you don't move the generator, the lexer does nothing.
Only as soon as you iterate the generator or call `next()`/`current()` on it
the lexer will start its work and spit out tokens sequentially.
This approach requires less memory during the lexing process.

The returned tokens are required to be `Phug\Lexer\TokenInterface` instances.

@param string $input the pug-string to lex into tokens.
@param null   $path

@return \Generator a generator that can be iterated sequentially
Returns a generator that will lex the passed input sequentially .