{"action":"created","comment":{"url":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls/comments/27073044","id":27073044,"diff_hunk":"@@ -46,109 +81,374 @@\n     \"with\", \"within\", \"without\", \"would\", \"yet\", \"you\", \"your\", \"yours\", \"yourself\", \\\n     \"yourselves\" ] # Source: http://xpo6.com/list-of-english-stop-words/\n \n+\n class Categorizer:\n \n     def __init__(self):\n-        self.title_categories = Counter()\n+        self._memoized_related_articles = defaultdict(list)\n+        self._memoized_article_links = defaultdict(list)\n+        self._memoized_semantic_relatedness_scores = defaultdict(float)\n+\n \n     def categorize_item(self, title, text):\n-        categories = Counter()\n+        # Use ngram word count approach to get possible keywords\n+        keyword_candidates = self._get_keyword_candidates(title, text)\n+        \n+        # Store possible wiki articles and their links\n+        self._memoize_related_articles(keyword_candidates.keys())\n+        self._memoize_article_links(keyword_candidates.keys())\n \n-        # Extract words present in category dictonary\n-        title = filter(self.is_from_titles, self.get_valid_ngrams(title))\n-        text = filter(self.is_from_titles, self.get_valid_ngrams(text))\n+        # Assign best article for each possible keyword \n+        assigned_articles   = self._assign_closest_articles(keyword_candidates.keys())\n+        \n+        # Build a graph and identify keyword clusters\n+        semantic_graph      = self._build_semantic_graph(assigned_articles)\n+        clustering          = semantic_graph.community_multilevel(weights='weight')\n+        \n+        # Use keywords from best clusters\n+        selected_categories = self._get_best_keywords(clustering, keyword_candidates)\n \n-        # Get word counts from title, text of article\n-        categories.update(title + text)\n+        return selected_categories\n \n-        categories = self.get_weighted_categories(categories)\n \n-        return self.get_best_categories(categories)\n+    def _get_keyword_candidates(self, title, text):\n+        title_ngrams = self._get_valid_ngrams(title)\n+        text_ngrams = self._get_valid_ngrams(text)\n \n+        ngrams = title_ngrams + text_ngrams\n \n-    def reset_title_categories(self):\n-        self.title_categories.clear()\n+        # Apply weights...\n+        for ngram, count in ngrams.iteritems():\n+            \n+            # ...from title (x2)\n+            if ngram in title_ngrams:\n+                ngrams.update({ngram: count})\n \n-        one_day_ago = datetime.now() - timedelta(hours=24)\n-        titles = [item.title for item in Item.query.filter(Item.category_1==None, Item.last_updated>=one_day_ago).all()]\n-        self.parse_title_categories(titles)\n+            # ...from bigram (x2) or trigram (x3)\n+            for n in range(ngram.count(' ')):\n+                ngrams.update({ngram: count})\n \n+        return dict(ngrams.most_common(MAX_CANDIDATE_COUNT))\n \n-    def parse_title_categories(self, titles):\n-        for title in titles:\n-            self.title_categories.update(self.get_valid_ngrams(title))\n \n+    def _get_valid_ngrams(self, string):\n+        valid_ngrams = Counter()\n \n-    def get_valid_ngrams(self, string):\n-        unigrams = [word.strip(punctuation) for word in string.replace(\"'s\", '').split()]\n-        unigrams = filter(None, unigrams)   # Remove empty strings\n-        \n-        bigrams = zip(*[unigrams[i:] for i in range(2)])  # tuples\n-        bigrams = [' '.join(word) for word in bigrams]  # strings\n+        words = [self._clean_word(word) for word in string.split()]\n+\n+        for n in range(NGRAMS):\n+            ngrams = zip(*[words[i:] for i in range(n + 1)])\n+\n+            for ngram in ngrams:\n+                ngram = ' '.join(ngram)\n+                if self._is_viable_candidate(ngram):\n+                    valid_ngrams.update([ngram])\n+\n+        return valid_ngrams\n+\n+\n+    def _clean_word(self, word):","path":"worker_tasks/categorize.py","position":145,"original_position":145,"commit_id":"3ecdbffe04584db4eb2d1226e83139a3a796cc05","original_commit_id":"3ecdbffe04584db4eb2d1226e83139a3a796cc05","user":{"login":"swglad","id":5695548,"avatar_url":"https://avatars.githubusercontent.com/u/5695548?v=3","gravatar_id":"","url":"https://api.github.com/users/swglad","html_url":"https://github.com/swglad","followers_url":"https://api.github.com/users/swglad/followers","following_url":"https://api.github.com/users/swglad/following{/other_user}","gists_url":"https://api.github.com/users/swglad/gists{/gist_id}","starred_url":"https://api.github.com/users/swglad/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/swglad/subscriptions","organizations_url":"https://api.github.com/users/swglad/orgs","repos_url":"https://api.github.com/users/swglad/repos","events_url":"https://api.github.com/users/swglad/events{/privacy}","received_events_url":"https://api.github.com/users/swglad/received_events","type":"User","site_admin":false},"body":"Does wikipedia deal with capitalization? No need for .lower()? ","created_at":"2015-03-24T21:05:35Z","updated_at":"2015-03-24T21:05:35Z","html_url":"https://github.com/alexgerstein/Sourcemash/pull/160#discussion_r27073044","pull_request_url":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls/160","_links":{"self":{"href":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls/comments/27073044"},"html":{"href":"https://github.com/alexgerstein/Sourcemash/pull/160#discussion_r27073044"},"pull_request":{"href":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls/160"}}},"pull_request":{"url":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls/160","id":31586837,"html_url":"https://github.com/alexgerstein/Sourcemash/pull/160","diff_url":"https://github.com/alexgerstein/Sourcemash/pull/160.diff","patch_url":"https://github.com/alexgerstein/Sourcemash/pull/160.patch","issue_url":"https://api.github.com/repos/alexgerstein/Sourcemash/issues/160","number":160,"state":"open","locked":false,"title":"Category enhancements","user":{"login":"alexgerstein","id":1390314,"avatar_url":"https://avatars.githubusercontent.com/u/1390314?v=3","gravatar_id":"","url":"https://api.github.com/users/alexgerstein","html_url":"https://github.com/alexgerstein","followers_url":"https://api.github.com/users/alexgerstein/followers","following_url":"https://api.github.com/users/alexgerstein/following{/other_user}","gists_url":"https://api.github.com/users/alexgerstein/gists{/gist_id}","starred_url":"https://api.github.com/users/alexgerstein/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alexgerstein/subscriptions","organizations_url":"https://api.github.com/users/alexgerstein/orgs","repos_url":"https://api.github.com/users/alexgerstein/repos","events_url":"https://api.github.com/users/alexgerstein/events{/privacy}","received_events_url":"https://api.github.com/users/alexgerstein/received_events","type":"User","site_admin":false},"body":"Improve categorization to rely less on heuristics. Instead, this uses Wikipedia to normalize categories and uses clusters to identify important keywords. Resolves #106.","created_at":"2015-03-20T04:01:04Z","updated_at":"2015-03-24T21:05:35Z","closed_at":null,"merged_at":null,"merge_commit_sha":"876d41882fc6c4fbd3b77e1657195547139febe7","assignee":{"login":"swglad","id":5695548,"avatar_url":"https://avatars.githubusercontent.com/u/5695548?v=3","gravatar_id":"","url":"https://api.github.com/users/swglad","html_url":"https://github.com/swglad","followers_url":"https://api.github.com/users/swglad/followers","following_url":"https://api.github.com/users/swglad/following{/other_user}","gists_url":"https://api.github.com/users/swglad/gists{/gist_id}","starred_url":"https://api.github.com/users/swglad/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/swglad/subscriptions","organizations_url":"https://api.github.com/users/swglad/orgs","repos_url":"https://api.github.com/users/swglad/repos","events_url":"https://api.github.com/users/swglad/events{/privacy}","received_events_url":"https://api.github.com/users/swglad/received_events","type":"User","site_admin":false},"milestone":{"url":"https://api.github.com/repos/alexgerstein/Sourcemash/milestones/9","html_url":"https://github.com/alexgerstein/Sourcemash/milestones/Week%2010","labels_url":"https://api.github.com/repos/alexgerstein/Sourcemash/milestones/9/labels","id":1000866,"number":9,"title":"Week 10","description":"Final changes before end of 15W.","creator":{"login":"alexgerstein","id":1390314,"avatar_url":"https://avatars.githubusercontent.com/u/1390314?v=3","gravatar_id":"","url":"https://api.github.com/users/alexgerstein","html_url":"https://github.com/alexgerstein","followers_url":"https://api.github.com/users/alexgerstein/followers","following_url":"https://api.github.com/users/alexgerstein/following{/other_user}","gists_url":"https://api.github.com/users/alexgerstein/gists{/gist_id}","starred_url":"https://api.github.com/users/alexgerstein/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alexgerstein/subscriptions","organizations_url":"https://api.github.com/users/alexgerstein/orgs","repos_url":"https://api.github.com/users/alexgerstein/repos","events_url":"https://api.github.com/users/alexgerstein/events{/privacy}","received_events_url":"https://api.github.com/users/alexgerstein/received_events","type":"User","site_admin":false},"open_issues":2,"closed_issues":6,"state":"open","created_at":"2015-03-01T20:32:59Z","updated_at":"2015-03-20T04:01:04Z","due_on":"2015-03-08T05:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls/160/commits","review_comments_url":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls/160/comments","review_comment_url":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls/comments{/number}","comments_url":"https://api.github.com/repos/alexgerstein/Sourcemash/issues/160/comments","statuses_url":"https://api.github.com/repos/alexgerstein/Sourcemash/statuses/3ecdbffe04584db4eb2d1226e83139a3a796cc05","head":{"label":"alexgerstein:category_enhancements","ref":"category_enhancements","sha":"3ecdbffe04584db4eb2d1226e83139a3a796cc05","user":{"login":"alexgerstein","id":1390314,"avatar_url":"https://avatars.githubusercontent.com/u/1390314?v=3","gravatar_id":"","url":"https://api.github.com/users/alexgerstein","html_url":"https://github.com/alexgerstein","followers_url":"https://api.github.com/users/alexgerstein/followers","following_url":"https://api.github.com/users/alexgerstein/following{/other_user}","gists_url":"https://api.github.com/users/alexgerstein/gists{/gist_id}","starred_url":"https://api.github.com/users/alexgerstein/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alexgerstein/subscriptions","organizations_url":"https://api.github.com/users/alexgerstein/orgs","repos_url":"https://api.github.com/users/alexgerstein/repos","events_url":"https://api.github.com/users/alexgerstein/events{/privacy}","received_events_url":"https://api.github.com/users/alexgerstein/received_events","type":"User","site_admin":false},"repo":{"id":29208817,"name":"Sourcemash","full_name":"alexgerstein/Sourcemash","owner":{"login":"alexgerstein","id":1390314,"avatar_url":"https://avatars.githubusercontent.com/u/1390314?v=3","gravatar_id":"","url":"https://api.github.com/users/alexgerstein","html_url":"https://github.com/alexgerstein","followers_url":"https://api.github.com/users/alexgerstein/followers","following_url":"https://api.github.com/users/alexgerstein/following{/other_user}","gists_url":"https://api.github.com/users/alexgerstein/gists{/gist_id}","starred_url":"https://api.github.com/users/alexgerstein/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alexgerstein/subscriptions","organizations_url":"https://api.github.com/users/alexgerstein/orgs","repos_url":"https://api.github.com/users/alexgerstein/repos","events_url":"https://api.github.com/users/alexgerstein/events{/privacy}","received_events_url":"https://api.github.com/users/alexgerstein/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/alexgerstein/Sourcemash","description":"Read Your News Faster.","fork":false,"url":"https://api.github.com/repos/alexgerstein/Sourcemash","forks_url":"https://api.github.com/repos/alexgerstein/Sourcemash/forks","keys_url":"https://api.github.com/repos/alexgerstein/Sourcemash/keys{/key_id}","collaborators_url":"https://api.github.com/repos/alexgerstein/Sourcemash/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/alexgerstein/Sourcemash/teams","hooks_url":"https://api.github.com/repos/alexgerstein/Sourcemash/hooks","issue_events_url":"https://api.github.com/repos/alexgerstein/Sourcemash/issues/events{/number}","events_url":"https://api.github.com/repos/alexgerstein/Sourcemash/events","assignees_url":"https://api.github.com/repos/alexgerstein/Sourcemash/assignees{/user}","branches_url":"https://api.github.com/repos/alexgerstein/Sourcemash/branches{/branch}","tags_url":"https://api.github.com/repos/alexgerstein/Sourcemash/tags","blobs_url":"https://api.github.com/repos/alexgerstein/Sourcemash/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/alexgerstein/Sourcemash/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/alexgerstein/Sourcemash/git/refs{/sha}","trees_url":"https://api.github.com/repos/alexgerstein/Sourcemash/git/trees{/sha}","statuses_url":"https://api.github.com/repos/alexgerstein/Sourcemash/statuses/{sha}","languages_url":"https://api.github.com/repos/alexgerstein/Sourcemash/languages","stargazers_url":"https://api.github.com/repos/alexgerstein/Sourcemash/stargazers","contributors_url":"https://api.github.com/repos/alexgerstein/Sourcemash/contributors","subscribers_url":"https://api.github.com/repos/alexgerstein/Sourcemash/subscribers","subscription_url":"https://api.github.com/repos/alexgerstein/Sourcemash/subscription","commits_url":"https://api.github.com/repos/alexgerstein/Sourcemash/commits{/sha}","git_commits_url":"https://api.github.com/repos/alexgerstein/Sourcemash/git/commits{/sha}","comments_url":"https://api.github.com/repos/alexgerstein/Sourcemash/comments{/number}","issue_comment_url":"https://api.github.com/repos/alexgerstein/Sourcemash/issues/comments{/number}","contents_url":"https://api.github.com/repos/alexgerstein/Sourcemash/contents/{+path}","compare_url":"https://api.github.com/repos/alexgerstein/Sourcemash/compare/{base}...{head}","merges_url":"https://api.github.com/repos/alexgerstein/Sourcemash/merges","archive_url":"https://api.github.com/repos/alexgerstein/Sourcemash/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/alexgerstein/Sourcemash/downloads","issues_url":"https://api.github.com/repos/alexgerstein/Sourcemash/issues{/number}","pulls_url":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls{/number}","milestones_url":"https://api.github.com/repos/alexgerstein/Sourcemash/milestones{/number}","notifications_url":"https://api.github.com/repos/alexgerstein/Sourcemash/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/alexgerstein/Sourcemash/labels{/name}","releases_url":"https://api.github.com/repos/alexgerstein/Sourcemash/releases{/id}","created_at":"2015-01-13T20:12:47Z","updated_at":"2015-03-23T05:37:19Z","pushed_at":"2015-03-24T20:52:52Z","git_url":"git://github.com/alexgerstein/Sourcemash.git","ssh_url":"git@github.com:alexgerstein/Sourcemash.git","clone_url":"https://github.com/alexgerstein/Sourcemash.git","svn_url":"https://github.com/alexgerstein/Sourcemash","homepage":"http://staging.sourcemash.com","size":3344,"stargazers_count":2,"watchers_count":2,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":38,"forks":0,"open_issues":38,"watchers":2,"default_branch":"master"}},"base":{"label":"alexgerstein:master","ref":"master","sha":"6e854f0880267a2b38b842c7574f097a51a1ae9c","user":{"login":"alexgerstein","id":1390314,"avatar_url":"https://avatars.githubusercontent.com/u/1390314?v=3","gravatar_id":"","url":"https://api.github.com/users/alexgerstein","html_url":"https://github.com/alexgerstein","followers_url":"https://api.github.com/users/alexgerstein/followers","following_url":"https://api.github.com/users/alexgerstein/following{/other_user}","gists_url":"https://api.github.com/users/alexgerstein/gists{/gist_id}","starred_url":"https://api.github.com/users/alexgerstein/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alexgerstein/subscriptions","organizations_url":"https://api.github.com/users/alexgerstein/orgs","repos_url":"https://api.github.com/users/alexgerstein/repos","events_url":"https://api.github.com/users/alexgerstein/events{/privacy}","received_events_url":"https://api.github.com/users/alexgerstein/received_events","type":"User","site_admin":false},"repo":{"id":29208817,"name":"Sourcemash","full_name":"alexgerstein/Sourcemash","owner":{"login":"alexgerstein","id":1390314,"avatar_url":"https://avatars.githubusercontent.com/u/1390314?v=3","gravatar_id":"","url":"https://api.github.com/users/alexgerstein","html_url":"https://github.com/alexgerstein","followers_url":"https://api.github.com/users/alexgerstein/followers","following_url":"https://api.github.com/users/alexgerstein/following{/other_user}","gists_url":"https://api.github.com/users/alexgerstein/gists{/gist_id}","starred_url":"https://api.github.com/users/alexgerstein/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/alexgerstein/subscriptions","organizations_url":"https://api.github.com/users/alexgerstein/orgs","repos_url":"https://api.github.com/users/alexgerstein/repos","events_url":"https://api.github.com/users/alexgerstein/events{/privacy}","received_events_url":"https://api.github.com/users/alexgerstein/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/alexgerstein/Sourcemash","description":"Read Your News Faster.","fork":false,"url":"https://api.github.com/repos/alexgerstein/Sourcemash","forks_url":"https://api.github.com/repos/alexgerstein/Sourcemash/forks","keys_url":"https://api.github.com/repos/alexgerstein/Sourcemash/keys{/key_id}","collaborators_url":"https://api.github.com/repos/alexgerstein/Sourcemash/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/alexgerstein/Sourcemash/teams","hooks_url":"https://api.github.com/repos/alexgerstein/Sourcemash/hooks","issue_events_url":"https://api.github.com/repos/alexgerstein/Sourcemash/issues/events{/number}","events_url":"https://api.github.com/repos/alexgerstein/Sourcemash/events","assignees_url":"https://api.github.com/repos/alexgerstein/Sourcemash/assignees{/user}","branches_url":"https://api.github.com/repos/alexgerstein/Sourcemash/branches{/branch}","tags_url":"https://api.github.com/repos/alexgerstein/Sourcemash/tags","blobs_url":"https://api.github.com/repos/alexgerstein/Sourcemash/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/alexgerstein/Sourcemash/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/alexgerstein/Sourcemash/git/refs{/sha}","trees_url":"https://api.github.com/repos/alexgerstein/Sourcemash/git/trees{/sha}","statuses_url":"https://api.github.com/repos/alexgerstein/Sourcemash/statuses/{sha}","languages_url":"https://api.github.com/repos/alexgerstein/Sourcemash/languages","stargazers_url":"https://api.github.com/repos/alexgerstein/Sourcemash/stargazers","contributors_url":"https://api.github.com/repos/alexgerstein/Sourcemash/contributors","subscribers_url":"https://api.github.com/repos/alexgerstein/Sourcemash/subscribers","subscription_url":"https://api.github.com/repos/alexgerstein/Sourcemash/subscription","commits_url":"https://api.github.com/repos/alexgerstein/Sourcemash/commits{/sha}","git_commits_url":"https://api.github.com/repos/alexgerstein/Sourcemash/git/commits{/sha}","comments_url":"https://api.github.com/repos/alexgerstein/Sourcemash/comments{/number}","issue_comment_url":"https://api.github.com/repos/alexgerstein/Sourcemash/issues/comments{/number}","contents_url":"https://api.github.com/repos/alexgerstein/Sourcemash/contents/{+path}","compare_url":"https://api.github.com/repos/alexgerstein/Sourcemash/compare/{base}...{head}","merges_url":"https://api.github.com/repos/alexgerstein/Sourcemash/merges","archive_url":"https://api.github.com/repos/alexgerstein/Sourcemash/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/alexgerstein/Sourcemash/downloads","issues_url":"https://api.github.com/repos/alexgerstein/Sourcemash/issues{/number}","pulls_url":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls{/number}","milestones_url":"https://api.github.com/repos/alexgerstein/Sourcemash/milestones{/number}","notifications_url":"https://api.github.com/repos/alexgerstein/Sourcemash/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/alexgerstein/Sourcemash/labels{/name}","releases_url":"https://api.github.com/repos/alexgerstein/Sourcemash/releases{/id}","created_at":"2015-01-13T20:12:47Z","updated_at":"2015-03-23T05:37:19Z","pushed_at":"2015-03-24T20:52:52Z","git_url":"git://github.com/alexgerstein/Sourcemash.git","ssh_url":"git@github.com:alexgerstein/Sourcemash.git","clone_url":"https://github.com/alexgerstein/Sourcemash.git","svn_url":"https://github.com/alexgerstein/Sourcemash","homepage":"http://staging.sourcemash.com","size":3344,"stargazers_count":2,"watchers_count":2,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":38,"forks":0,"open_issues":38,"watchers":2,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls/160"},"html":{"href":"https://github.com/alexgerstein/Sourcemash/pull/160"},"issue":{"href":"https://api.github.com/repos/alexgerstein/Sourcemash/issues/160"},"comments":{"href":"https://api.github.com/repos/alexgerstein/Sourcemash/issues/160/comments"},"review_comments":{"href":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls/160/comments"},"review_comment":{"href":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/alexgerstein/Sourcemash/pulls/160/commits"},"statuses":{"href":"https://api.github.com/repos/alexgerstein/Sourcemash/statuses/3ecdbffe04584db4eb2d1226e83139a3a796cc05"}}}}