{"action":"created","comment":{"url":"https://api.github.com/repos/inveniosoftware/beard/pulls/comments/23365273","id":23365273,"diff_hunk":"@@ -97,54 +144,59 @@ def _validate(self, X, blocks):\n \n         return X, blocks\n \n-    def _fit(self, X, y, blocks):\n-        \"\"\"Fit base clustering estimators on X.\"\"\"\n-        self.labels_ = -np.ones(len(X), dtype=np.int)\n-        offset = 0\n+    def _block(self, X, y, blocks, affinity):\n+        \"\"\"Chop the training data into smaller chunks.\n \n-        for b in np.unique(blocks):\n-            # Select data from block\n-            mask = (blocks == b)\n-            X_mask = X[mask, :]\n+        A chunk is demarcated by the corresponding block. Each chunk contains\n+        only the training examples relevant to given block.\n+\n+        Returns\n+        -------\n+        :returns: array\n+            Triples in form of (block, X, y) where X and y are the training\n+            examples for given block.\n+        \"\"\"\n+        unique_blocks = np.unique(blocks)\n \n+        chunk = []\n+        for single_block in unique_blocks:\n+            mask = (blocks == single_block)\n+            X_mask = X[mask, :]\n             if y is not None:\n                 y_mask = y[mask]\n             else:\n                 y_mask = None\n-\n-            if self.affinity == \"precomputed\":\n+            if affinity == \"precomputed\":\n                 X_mask = X_mask[:, mask]\n+            chunk.append((single_block, X_mask, y_mask))\n+        return chunk\n+\n+    def _fit(self, X, y, blocks):\n+        \"\"\"Fit base clustering estimators on X.\"\"\"\n+\n+        self.labels_ = -np.ones(len(X), dtype=np.int)\n+        offset = 0\n+\n+        s = time()\n+\n+        result = (Parallel(n_jobs=self.n_jobs)\n+                  (delayed(_parallel_fit)(self.fit_, self.partial_fit_,\n+                                          self.base_estimator,\n+                                          self.clusterers_, *b) for b in","path":"beard/clustering/blocking.py","position":null,"original_position":143,"commit_id":"d4d3bccc249cfc393909f7349f33d7d01e668886","original_commit_id":"809bb972e781bd2c2f3496295e5ea12b1df534bd","user":{"login":"glouppe","id":477771,"avatar_url":"https://avatars.githubusercontent.com/u/477771?v=3","gravatar_id":"","url":"https://api.github.com/users/glouppe","html_url":"https://github.com/glouppe","followers_url":"https://api.github.com/users/glouppe/followers","following_url":"https://api.github.com/users/glouppe/following{/other_user}","gists_url":"https://api.github.com/users/glouppe/gists{/gist_id}","starred_url":"https://api.github.com/users/glouppe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/glouppe/subscriptions","organizations_url":"https://api.github.com/users/glouppe/orgs","repos_url":"https://api.github.com/users/glouppe/repos","events_url":"https://api.github.com/users/glouppe/events{/privacy}","received_events_url":"https://api.github.com/users/glouppe/received_events","type":"User","site_admin":false},"body":"Can you unpack the values yielded by `_block`? Better explicit than implicit. ","created_at":"2015-01-22T10:03:47Z","updated_at":"2015-01-22T10:03:47Z","html_url":"https://github.com/inveniosoftware/beard/pull/12#discussion_r23365273","pull_request_url":"https://api.github.com/repos/inveniosoftware/beard/pulls/12","_links":{"self":{"href":"https://api.github.com/repos/inveniosoftware/beard/pulls/comments/23365273"},"html":{"href":"https://github.com/inveniosoftware/beard/pull/12#discussion_r23365273"},"pull_request":{"href":"https://api.github.com/repos/inveniosoftware/beard/pulls/12"}}},"pull_request":{"url":"https://api.github.com/repos/inveniosoftware/beard/pulls/12","id":27684565,"html_url":"https://github.com/inveniosoftware/beard/pull/12","diff_url":"https://github.com/inveniosoftware/beard/pull/12.diff","patch_url":"https://github.com/inveniosoftware/beard/pull/12.patch","issue_url":"https://api.github.com/repos/inveniosoftware/beard/issues/12","number":12,"state":"open","locked":false,"title":"WIP clustering: parallelization of blocking","user":{"login":"MSusik","id":6649290,"avatar_url":"https://avatars.githubusercontent.com/u/6649290?v=3","gravatar_id":"","url":"https://api.github.com/users/MSusik","html_url":"https://github.com/MSusik","followers_url":"https://api.github.com/users/MSusik/followers","following_url":"https://api.github.com/users/MSusik/following{/other_user}","gists_url":"https://api.github.com/users/MSusik/gists{/gist_id}","starred_url":"https://api.github.com/users/MSusik/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/MSusik/subscriptions","organizations_url":"https://api.github.com/users/MSusik/orgs","repos_url":"https://api.github.com/users/MSusik/repos","events_url":"https://api.github.com/users/MSusik/events{/privacy}","received_events_url":"https://api.github.com/users/MSusik/received_events","type":"User","site_admin":false},"body":"* Parallelizes `_fit` function in BlockClustering class.\r\n  Each job takes care only about specific blocks.\r\n\r\n* Introduces usage of joblib library.\r\n\r\n* Adds a function to compute number of jobs from joblib's n_jobs.\r\n\r\nTime performance (used `time.time` due to the usage of `multiprocessing`. `time.clock` measures only the time consumed by the main thread):\r\nhttps://docs.google.com/spreadsheets/d/1mxxvpCxuwPhY4sVlk6wJL2copjNXsV34eMDOzBSFwPQ/edit?usp=sharing\r\n\r\nTO DO/TO CONSIDER:\r\n- [X] proper profiling. (not possible)\r\n- [X] try dividing the input data before sending it to the job.  (implemented and attached)\r\n- [ ] test for number of jobs more than 1.","created_at":"2015-01-20T15:14:05Z","updated_at":"2015-01-22T10:03:47Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d01cc6cc2d8de4b487d47503868d3141c89fb170","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/inveniosoftware/beard/pulls/12/commits","review_comments_url":"https://api.github.com/repos/inveniosoftware/beard/pulls/12/comments","review_comment_url":"https://api.github.com/repos/inveniosoftware/beard/pulls/comments/{number}","comments_url":"https://api.github.com/repos/inveniosoftware/beard/issues/12/comments","statuses_url":"https://api.github.com/repos/inveniosoftware/beard/statuses/d4d3bccc249cfc393909f7349f33d7d01e668886","head":{"label":"MSusik:joblib","ref":"joblib","sha":"d4d3bccc249cfc393909f7349f33d7d01e668886","user":{"login":"MSusik","id":6649290,"avatar_url":"https://avatars.githubusercontent.com/u/6649290?v=3","gravatar_id":"","url":"https://api.github.com/users/MSusik","html_url":"https://github.com/MSusik","followers_url":"https://api.github.com/users/MSusik/followers","following_url":"https://api.github.com/users/MSusik/following{/other_user}","gists_url":"https://api.github.com/users/MSusik/gists{/gist_id}","starred_url":"https://api.github.com/users/MSusik/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/MSusik/subscriptions","organizations_url":"https://api.github.com/users/MSusik/orgs","repos_url":"https://api.github.com/users/MSusik/repos","events_url":"https://api.github.com/users/MSusik/events{/privacy}","received_events_url":"https://api.github.com/users/MSusik/received_events","type":"User","site_admin":false},"repo":{"id":25810461,"name":"beard","full_name":"MSusik/beard","owner":{"login":"MSusik","id":6649290,"avatar_url":"https://avatars.githubusercontent.com/u/6649290?v=3","gravatar_id":"","url":"https://api.github.com/users/MSusik","html_url":"https://github.com/MSusik","followers_url":"https://api.github.com/users/MSusik/followers","following_url":"https://api.github.com/users/MSusik/following{/other_user}","gists_url":"https://api.github.com/users/MSusik/gists{/gist_id}","starred_url":"https://api.github.com/users/MSusik/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/MSusik/subscriptions","organizations_url":"https://api.github.com/users/MSusik/orgs","repos_url":"https://api.github.com/users/MSusik/repos","events_url":"https://api.github.com/users/MSusik/events{/privacy}","received_events_url":"https://api.github.com/users/MSusik/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/MSusik/beard","description":"Bibliographic Entity Automatic Recognition and Disambiguation","fork":true,"url":"https://api.github.com/repos/MSusik/beard","forks_url":"https://api.github.com/repos/MSusik/beard/forks","keys_url":"https://api.github.com/repos/MSusik/beard/keys{/key_id}","collaborators_url":"https://api.github.com/repos/MSusik/beard/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/MSusik/beard/teams","hooks_url":"https://api.github.com/repos/MSusik/beard/hooks","issue_events_url":"https://api.github.com/repos/MSusik/beard/issues/events{/number}","events_url":"https://api.github.com/repos/MSusik/beard/events","assignees_url":"https://api.github.com/repos/MSusik/beard/assignees{/user}","branches_url":"https://api.github.com/repos/MSusik/beard/branches{/branch}","tags_url":"https://api.github.com/repos/MSusik/beard/tags","blobs_url":"https://api.github.com/repos/MSusik/beard/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/MSusik/beard/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/MSusik/beard/git/refs{/sha}","trees_url":"https://api.github.com/repos/MSusik/beard/git/trees{/sha}","statuses_url":"https://api.github.com/repos/MSusik/beard/statuses/{sha}","languages_url":"https://api.github.com/repos/MSusik/beard/languages","stargazers_url":"https://api.github.com/repos/MSusik/beard/stargazers","contributors_url":"https://api.github.com/repos/MSusik/beard/contributors","subscribers_url":"https://api.github.com/repos/MSusik/beard/subscribers","subscription_url":"https://api.github.com/repos/MSusik/beard/subscription","commits_url":"https://api.github.com/repos/MSusik/beard/commits{/sha}","git_commits_url":"https://api.github.com/repos/MSusik/beard/git/commits{/sha}","comments_url":"https://api.github.com/repos/MSusik/beard/comments{/number}","issue_comment_url":"https://api.github.com/repos/MSusik/beard/issues/comments/{number}","contents_url":"https://api.github.com/repos/MSusik/beard/contents/{+path}","compare_url":"https://api.github.com/repos/MSusik/beard/compare/{base}...{head}","merges_url":"https://api.github.com/repos/MSusik/beard/merges","archive_url":"https://api.github.com/repos/MSusik/beard/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/MSusik/beard/downloads","issues_url":"https://api.github.com/repos/MSusik/beard/issues{/number}","pulls_url":"https://api.github.com/repos/MSusik/beard/pulls{/number}","milestones_url":"https://api.github.com/repos/MSusik/beard/milestones{/number}","notifications_url":"https://api.github.com/repos/MSusik/beard/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/MSusik/beard/labels{/name}","releases_url":"https://api.github.com/repos/MSusik/beard/releases{/id}","created_at":"2014-10-27T08:49:19Z","updated_at":"2014-10-27T12:04:05Z","pushed_at":"2015-01-21T16:17:22Z","git_url":"git://github.com/MSusik/beard.git","ssh_url":"git@github.com:MSusik/beard.git","clone_url":"https://github.com/MSusik/beard.git","svn_url":"https://github.com/MSusik/beard","homepage":null,"size":744,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"inveniosoftware:master","ref":"master","sha":"e1273e52cc1c08f33c75e0fad6a6139cad48cd88","user":{"login":"inveniosoftware","id":1157480,"avatar_url":"https://avatars.githubusercontent.com/u/1157480?v=3","gravatar_id":"","url":"https://api.github.com/users/inveniosoftware","html_url":"https://github.com/inveniosoftware","followers_url":"https://api.github.com/users/inveniosoftware/followers","following_url":"https://api.github.com/users/inveniosoftware/following{/other_user}","gists_url":"https://api.github.com/users/inveniosoftware/gists{/gist_id}","starred_url":"https://api.github.com/users/inveniosoftware/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/inveniosoftware/subscriptions","organizations_url":"https://api.github.com/users/inveniosoftware/orgs","repos_url":"https://api.github.com/users/inveniosoftware/repos","events_url":"https://api.github.com/users/inveniosoftware/events{/privacy}","received_events_url":"https://api.github.com/users/inveniosoftware/received_events","type":"Organization","site_admin":false},"repo":{"id":25808549,"name":"beard","full_name":"inveniosoftware/beard","owner":{"login":"inveniosoftware","id":1157480,"avatar_url":"https://avatars.githubusercontent.com/u/1157480?v=3","gravatar_id":"","url":"https://api.github.com/users/inveniosoftware","html_url":"https://github.com/inveniosoftware","followers_url":"https://api.github.com/users/inveniosoftware/followers","following_url":"https://api.github.com/users/inveniosoftware/following{/other_user}","gists_url":"https://api.github.com/users/inveniosoftware/gists{/gist_id}","starred_url":"https://api.github.com/users/inveniosoftware/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/inveniosoftware/subscriptions","organizations_url":"https://api.github.com/users/inveniosoftware/orgs","repos_url":"https://api.github.com/users/inveniosoftware/repos","events_url":"https://api.github.com/users/inveniosoftware/events{/privacy}","received_events_url":"https://api.github.com/users/inveniosoftware/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/inveniosoftware/beard","description":"Bibliographic Entity Automatic Recognition and Disambiguation","fork":false,"url":"https://api.github.com/repos/inveniosoftware/beard","forks_url":"https://api.github.com/repos/inveniosoftware/beard/forks","keys_url":"https://api.github.com/repos/inveniosoftware/beard/keys{/key_id}","collaborators_url":"https://api.github.com/repos/inveniosoftware/beard/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/inveniosoftware/beard/teams","hooks_url":"https://api.github.com/repos/inveniosoftware/beard/hooks","issue_events_url":"https://api.github.com/repos/inveniosoftware/beard/issues/events{/number}","events_url":"https://api.github.com/repos/inveniosoftware/beard/events","assignees_url":"https://api.github.com/repos/inveniosoftware/beard/assignees{/user}","branches_url":"https://api.github.com/repos/inveniosoftware/beard/branches{/branch}","tags_url":"https://api.github.com/repos/inveniosoftware/beard/tags","blobs_url":"https://api.github.com/repos/inveniosoftware/beard/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/inveniosoftware/beard/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/inveniosoftware/beard/git/refs{/sha}","trees_url":"https://api.github.com/repos/inveniosoftware/beard/git/trees{/sha}","statuses_url":"https://api.github.com/repos/inveniosoftware/beard/statuses/{sha}","languages_url":"https://api.github.com/repos/inveniosoftware/beard/languages","stargazers_url":"https://api.github.com/repos/inveniosoftware/beard/stargazers","contributors_url":"https://api.github.com/repos/inveniosoftware/beard/contributors","subscribers_url":"https://api.github.com/repos/inveniosoftware/beard/subscribers","subscription_url":"https://api.github.com/repos/inveniosoftware/beard/subscription","commits_url":"https://api.github.com/repos/inveniosoftware/beard/commits{/sha}","git_commits_url":"https://api.github.com/repos/inveniosoftware/beard/git/commits{/sha}","comments_url":"https://api.github.com/repos/inveniosoftware/beard/comments{/number}","issue_comment_url":"https://api.github.com/repos/inveniosoftware/beard/issues/comments/{number}","contents_url":"https://api.github.com/repos/inveniosoftware/beard/contents/{+path}","compare_url":"https://api.github.com/repos/inveniosoftware/beard/compare/{base}...{head}","merges_url":"https://api.github.com/repos/inveniosoftware/beard/merges","archive_url":"https://api.github.com/repos/inveniosoftware/beard/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/inveniosoftware/beard/downloads","issues_url":"https://api.github.com/repos/inveniosoftware/beard/issues{/number}","pulls_url":"https://api.github.com/repos/inveniosoftware/beard/pulls{/number}","milestones_url":"https://api.github.com/repos/inveniosoftware/beard/milestones{/number}","notifications_url":"https://api.github.com/repos/inveniosoftware/beard/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/inveniosoftware/beard/labels{/name}","releases_url":"https://api.github.com/repos/inveniosoftware/beard/releases{/id}","created_at":"2014-10-27T07:52:27Z","updated_at":"2015-01-21T08:33:42Z","pushed_at":"2015-01-21T08:33:42Z","git_url":"git://github.com/inveniosoftware/beard.git","ssh_url":"git@github.com:inveniosoftware/beard.git","clone_url":"https://github.com/inveniosoftware/beard.git","svn_url":"https://github.com/inveniosoftware/beard","homepage":null,"size":1273,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"open_issues_count":6,"forks":5,"open_issues":6,"watchers":0,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/inveniosoftware/beard/pulls/12"},"html":{"href":"https://github.com/inveniosoftware/beard/pull/12"},"issue":{"href":"https://api.github.com/repos/inveniosoftware/beard/issues/12"},"comments":{"href":"https://api.github.com/repos/inveniosoftware/beard/issues/12/comments"},"review_comments":{"href":"https://api.github.com/repos/inveniosoftware/beard/pulls/12/comments"},"review_comment":{"href":"https://api.github.com/repos/inveniosoftware/beard/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/inveniosoftware/beard/pulls/12/commits"},"statuses":{"href":"https://api.github.com/repos/inveniosoftware/beard/statuses/d4d3bccc249cfc393909f7349f33d7d01e668886"}}}}