def _generate_phrases ( self , sentences ) : phrase_list = set ( ) # Create contender phrases from sentences. for sentence in sentences : word_list = [ word . lower ( ) for word in wordpunct_tokenize ( sentence ) ] phrase_list . update ( self . _get_phrase_list_from_words ( word_list ) ) return phrase_list
how to make a list of words into corpus python
Method to generate contender phrases given the sentences of the text document .
def _generate_phrases(self, sentences):
        """Method to generate contender phrases given the sentences of the text
        document.

        :param sentences: List of strings where each string represents a
                          sentence which forms the text.
        :return: Set of string tuples where each tuple is a collection
                 of words forming a contender phrase.
        """
        phrase_list = set()
        # Create contender phrases from sentences.
        for sentence in sentences:
            word_list = [word.lower() for word in wordpunct_tokenize(sentence)]
            phrase_list.update(self._get_phrase_list_from_words(word_list))
        return phrase_list