protected static function tokenize( array $parts ) {
		$tokens = [];
		$index  = 0;

		foreach ( $parts as $part ) {
			if ( \preg_match( self::_RE_SPACE, $part ) ) {
				$tokens[ $index ] = new Token( $part, Token::SPACE );
			} elseif ( \preg_match( self::_RE_PUNCTUATION, $part ) ) {
				$tokens[ $index ] = new Token( $part, Token::PUNCTUATION );
			} elseif ( \preg_match( self::_RE_WORD, $part ) ) {
				// Make sure that things like email addresses and URLs are not broken up
				// into words and punctuation not preceeded by an 'other'.
				self::parse_ambiguous_token( Token::WORD, $part, $tokens, $index );
			} else {
				// Make sure that things like email addresses and URLs are not broken up into words
				// and punctuation not preceeded by an 'other' or 'word'.
				self::parse_ambiguous_token( Token::OTHER, $part, $tokens, $index );
			}

			$index++;
		}

		return $tokens;
	}
protected static function tokenize( array $parts ) {
		$tokens = [];
		$index  = 0;

		foreach ( $parts as $part ) {
			if ( \preg_match( self::_RE_SPACE, $part ) ) {
				$tokens[ $index ] = new Token( $part, Token::SPACE );
			} elseif ( \preg_match( self::_RE_PUNCTUATION, $part ) ) {
				$tokens[ $index ] = new Token( $part, Token::PUNCTUATION );
			} elseif ( \preg_match( self::_RE_WORD, $part ) ) {
				// Make sure that things like email addresses and URLs are not broken up
				// into words and punctuation not preceeded by an 'other'.
				self::parse_ambiguous_token( Token::WORD, $part, $tokens, $index );
			} else {
				// Make sure that things like email addresses and URLs are not broken up into words
				// and punctuation not preceeded by an 'other' or 'word'.
				self::parse_ambiguous_token( Token::OTHER, $part, $tokens, $index );
			}

			$index++;
		}

		return $tokens;
	}
protected static function tokenize ( array $ parts ) { $ tokens = [ ] ; $ index = 0 ; foreach ( $ parts as $ part ) { if ( \ preg_match ( self :: _RE_SPACE , $ part ) ) { $ tokens [ $ index ] = new Token ( $ part , Token :: SPACE ) ; } elseif ( \ preg_match ( self :: _RE_PUNCTUATION , $ part ) ) { $ tokens [ $ index ] = new Token ( $ part , Token :: PUNCTUATION ) ; } elseif ( \ preg_match ( self :: _RE_WORD , $ part ) ) { // Make sure that things like email addresses and URLs are not broken up // into words and punctuation not preceeded by an 'other'. self :: parse_ambiguous_token ( Token :: WORD , $ part , $ tokens , $ index ) ; } else { // Make sure that things like email addresses and URLs are not broken up into words // and punctuation not preceeded by an 'other' or 'word'. self :: parse_ambiguous_token ( Token :: OTHER , $ part , $ tokens , $ index ) ; } $ index ++ ; } return $ tokens ; }
Turns the array of strings into an array of tokens.

@param string[] $parts An array of non-empty strings.

@return Token[] An array of numerically indexed tokens.
Turns the array of strings into an array of tokens .