public function tokenize(string $text) : array
    {
        $pattern =
'/(\[
    \/?
    \w+
    (?:
        =
        (?:
            "[^"]+"
            |\'[^\']+\'
            |[^]]+
        )
    )?
\])/x';

        $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE;
        $tokens = preg_split($pattern, $text, -1, $flags);
        $tokens = array_map(function ($text) use ($pattern) {
            $matches = [];
            if (preg_match($pattern, $text, $matches)) {
                $token = new TagToken($text);

                // Fallback to a text token if tag is not valid.
                if (!$this->validateTagToken($token)) {
                    return new TextToken($text);
                }

                return $token;
            } else {
                return new TextToken($text);
            }
        }, $tokens);
        return $this->processRawContentTagTokens($tokens);
    }
public function tokenize(string $text) : array
    {
        $pattern =
'/(\[
    \/?
    \w+
    (?:
        =
        (?:
            "[^"]+"
            |\'[^\']+\'
            |[^]]+
        )
    )?
\])/x';

        $flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE;
        $tokens = preg_split($pattern, $text, -1, $flags);
        $tokens = array_map(function ($text) use ($pattern) {
            $matches = [];
            if (preg_match($pattern, $text, $matches)) {
                $token = new TagToken($text);

                // Fallback to a text token if tag is not valid.
                if (!$this->validateTagToken($token)) {
                    return new TextToken($text);
                }

                return $token;
            } else {
                return new TextToken($text);
            }
        }, $tokens);
        return $this->processRawContentTagTokens($tokens);
    }
public function tokenize ( string $ text ) : array { $ pattern = '/(\[
    \/?
    \w+
    (?:
        =
        (?:
            "[^"]+"
            |\'[^\']+\'
            |[^]]+
        )
    )?
\])/x' ; $ flags = PREG_SPLIT_NO_EMPTY | PREG_SPLIT_DELIM_CAPTURE ; $ tokens = preg_split ( $ pattern , $ text , - 1 , $ flags ) ; $ tokens = array_map ( function ( $ text ) use ( $ pattern ) { $ matches = [ ] ; if ( preg_match ( $ pattern , $ text , $ matches ) ) { $ token = new TagToken ( $ text ) ; // Fallback to a text token if tag is not valid. if ( ! $ this -> validateTagToken ( $ token ) ) { return new TextToken ( $ text ) ; } return $ token ; } else { return new TextToken ( $ text ) ; } } , $ tokens ) ; return $ this -> processRawContentTagTokens ( $ tokens ) ; }
Converts raw text into the array of tokens.

@param string $text

@return Token[]
Converts raw text into the array of tokens .