{"action":"created","comment":{"url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments/29947604","id":29947604,"diff_hunk":"@@ -0,0 +1,668 @@\n+\"\"\"\n+\n+=============================================================\n+Online Latent Dirichlet Allocation with variational inference\n+=============================================================\n+\n+This implementation is modified from Matthew D. Hoffman's onlineldavb code\n+Link: http://www.cs.princeton.edu/~mdhoffma/code/onlineldavb.tar\n+\"\"\"\n+\n+# Author: Chyi-Kwei Yau\n+# Author: Matthew D. Hoffman (original onlineldavb implementation)\n+\n+import numpy as np\n+import scipy.sparse as sp\n+from scipy.special import gammaln\n+\n+from ..base import BaseEstimator, TransformerMixin\n+from ..utils import (check_random_state, check_array,\n+                     gen_batches, gen_even_slices)\n+from ..utils.validation import NotFittedError\n+\n+from ..externals.joblib import Parallel, delayed, cpu_count\n+from ..externals.six.moves import xrange\n+\n+from ._online_lda import (mean_change, _dirichlet_expectation_1d,\n+                          _dirichlet_expectation_2d)\n+\n+\n+def check_non_negative(X, whom):\n+    \"\"\"\n+    make sure no Negative data in X\n+    (copy from NMF directly)\n+    \"\"\"\n+    X = X.data if sp.issparse(X) else X\n+    if (X < 0).any():\n+        raise ValueError(\"Negative values in data passed to %s\" % whom)\n+\n+\n+def _dirichlet_expectation(X):\n+    \"\"\"\n+    For an array theta ~ Dir(X), computes `E[log(theta)]` given X.\n+\n+    Parameters\n+    ----------\n+    X : array-like\n+        1 or 2 dimention vector\n+\n+    Returns\n+    -------\n+    dirichlet_expect: array-like\n+        Dirichlet expectation of input array X\n+    \"\"\"\n+\n+    if len(X.shape) == 1:\n+        dirichlet_expect = _dirichlet_expectation_1d(X)\n+    else:\n+        dirichlet_expect = _dirichlet_expectation_2d(X)\n+    return dirichlet_expect\n+\n+\n+def _update_doc_distribution(X, exp_topic_word_distr, doc_topic_prior, max_iters,\n+                             mean_change_tol, cal_diff):\n+    \"\"\"\n+    E-step: update document topic distribution.\n+    (In the literature, this is called `gamma`.)\n+\n+    Parameters\n+    ----------\n+    X : sparse matrix, shape=(n_samples, n_features)\n+        Document word matrix.\n+\n+    exp_topic_word_distr : dense matrrix, shape=(n_topics, n_features)\n+        Exponential value of expection of log topic word distribution.\n+        In the literature, this is `exp(E[log(beta)])`.\n+\n+    doc_topic_prior : float\n+        Prior of document topic distribution `theta`.\n+\n+    max_iters : int\n+        Max number of iterations for updating document topic distribution in E-step.\n+\n+    mean_change_tol : float\n+        Stopping tolerance for updating document topic distribution in E-setp.\n+\n+    cal_diff : boolean\n+        Parameter that indicate to calculate differene in `component_` or not.\n+        Set `cal_diff` to `True` when we need to run M-step.\n+\n+    Returns\n+    -------\n+    (doc_topic_distr, component_diff) :\n+        `doc_topic_distr` is unnormailzed topic distribution for each document.\n+        In the literature, this is `gamma`. we can calcuate `E[log(theta)]`\n+        from it.\n+        `component_diff` is the difference of `component_` when `cal_diff` is True.\n+        Otherwise, it is None.\n+\n+    \"\"\"\n+\n+    n_samples, n_features = X.shape\n+    n_topics = exp_topic_word_distr.shape[0]\n+\n+    doc_topic_distr = np.ones((n_samples, n_topics))\n+    # In the literature, this is `exp(E[log(theta)])`\n+    exp_doc_topic = np.exp(_dirichlet_expectation(doc_topic_distr))\n+\n+    # diff on `component_` (only calculate it when `cal_diff` is True)\n+    component_diff = np.zeros(exp_topic_word_distr.shape) if cal_diff else None\n+\n+    X_data = X.data\n+    X_indices = X.indices\n+    X_indptr = X.indptr\n+\n+    for d in xrange(n_samples):\n+        ids = X_indices[X_indptr[d]:X_indptr[d + 1]]\n+        cnts = X_data[X_indptr[d]:X_indptr[d + 1]]\n+\n+        doc_topic_d = doc_topic_distr[d, :]\n+        exp_doc_topic_d = exp_doc_topic[d, :]\n+        exp_topic_word_d = exp_topic_word_distr[:, ids]\n+\n+        # The optimal phi_{dwk} is proportional to \n+        # exp(E[log(theta_{dk})]) * exp(E[log(beta_{dw})]).\n+        norm_phi = np.dot(exp_doc_topic_d, exp_topic_word_d) + 1e-100","path":"sklearn/decomposition/online_lda.py","position":125,"original_position":125,"commit_id":"100ba3bc9ddb62b929bcb00bd9ecfaf6454c2b75","original_commit_id":"100ba3bc9ddb62b929bcb00bd9ecfaf6454c2b75","user":{"login":"amueller","id":449558,"avatar_url":"https://avatars.githubusercontent.com/u/449558?v=3","gravatar_id":"","url":"https://api.github.com/users/amueller","html_url":"https://github.com/amueller","followers_url":"https://api.github.com/users/amueller/followers","following_url":"https://api.github.com/users/amueller/following{/other_user}","gists_url":"https://api.github.com/users/amueller/gists{/gist_id}","starred_url":"https://api.github.com/users/amueller/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amueller/subscriptions","organizations_url":"https://api.github.com/users/amueller/orgs","repos_url":"https://api.github.com/users/amueller/repos","events_url":"https://api.github.com/users/amueller/events{/privacy}","received_events_url":"https://api.github.com/users/amueller/received_events","type":"User","site_admin":false},"body":"why do you not compute this at the beginning of the loop? You have it once before the loop and once at the end.","created_at":"2015-05-08T15:12:04Z","updated_at":"2015-05-08T15:12:04Z","html_url":"https://github.com/scikit-learn/scikit-learn/pull/3659#discussion_r29947604","pull_request_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3659","_links":{"self":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments/29947604"},"html":{"href":"https://github.com/scikit-learn/scikit-learn/pull/3659#discussion_r29947604"},"pull_request":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3659"}}},"pull_request":{"url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3659","id":21188463,"html_url":"https://github.com/scikit-learn/scikit-learn/pull/3659","diff_url":"https://github.com/scikit-learn/scikit-learn/pull/3659.diff","patch_url":"https://github.com/scikit-learn/scikit-learn/pull/3659.patch","issue_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/3659","number":3659,"state":"open","locked":false,"title":"[MRG] implementing LDA(Latent Dirichlet Allocation) with online variational Bayes","user":{"login":"chyikwei","id":2423442,"avatar_url":"https://avatars.githubusercontent.com/u/2423442?v=3","gravatar_id":"","url":"https://api.github.com/users/chyikwei","html_url":"https://github.com/chyikwei","followers_url":"https://api.github.com/users/chyikwei/followers","following_url":"https://api.github.com/users/chyikwei/following{/other_user}","gists_url":"https://api.github.com/users/chyikwei/gists{/gist_id}","starred_url":"https://api.github.com/users/chyikwei/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chyikwei/subscriptions","organizations_url":"https://api.github.com/users/chyikwei/orgs","repos_url":"https://api.github.com/users/chyikwei/repos","events_url":"https://api.github.com/users/chyikwei/events{/privacy}","received_events_url":"https://api.github.com/users/chyikwei/received_events","type":"User","site_admin":false},"body":"This PR is an implementation of Matt Hoffman's topic modeling algorithm `LDA with online variational Bayes`.\r\n\r\nBased on previous discussion in [this email thread](http://sourceforge.net/p/scikit-learn/mailman/message/32748341/), I asked Matt if he could relicense his `onlineldavb` code to BSD. And now his code is relicensed, so I create a PR for it.\r\n\r\nI use the name `OnlineLDA` for this model and put it in `decomposition` folder. And since the model can run both online and batch update, I implemented both `fit` and `partial_fit` method.\r\nThe algorithm part and unit test is done and ready for review. Will work on an example next.\r\n\r\n###### Check List:\r\n- [X] algorithm implementation\r\n- [X] unit test \r\n- [X] profiling\r\n- [x] optimization\r\n- [X] example\r\n- [x] documentation\r\n\r\n###### Reference:\r\n[1] [\"Online Learning for Latent Dirichlet Allocation\", Matthew D. Hoffman, David M. Blei, Francis Bach](https://www.cs.princeton.edu/~blei/papers/HoffmanBleiBach2010b.pdf)\r\n[2] [original onlineldavb code (with BSD license)](https://www.dropbox.com/s/wnkro3xtqjm7bli/onlineldavb_bsd.tar)","created_at":"2014-09-13T02:28:48Z","updated_at":"2015-05-08T15:12:04Z","closed_at":null,"merged_at":null,"merge_commit_sha":"ea954c927098c186f51e62fd9f647f7d688324b2","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3659/commits","review_comments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3659/comments","review_comment_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments{/number}","comments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/3659/comments","statuses_url":"https://api.github.com/repos/scikit-learn/scikit-learn/statuses/100ba3bc9ddb62b929bcb00bd9ecfaf6454c2b75","head":{"label":"chyikwei:onlineldavb","ref":"onlineldavb","sha":"100ba3bc9ddb62b929bcb00bd9ecfaf6454c2b75","user":{"login":"chyikwei","id":2423442,"avatar_url":"https://avatars.githubusercontent.com/u/2423442?v=3","gravatar_id":"","url":"https://api.github.com/users/chyikwei","html_url":"https://github.com/chyikwei","followers_url":"https://api.github.com/users/chyikwei/followers","following_url":"https://api.github.com/users/chyikwei/following{/other_user}","gists_url":"https://api.github.com/users/chyikwei/gists{/gist_id}","starred_url":"https://api.github.com/users/chyikwei/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chyikwei/subscriptions","organizations_url":"https://api.github.com/users/chyikwei/orgs","repos_url":"https://api.github.com/users/chyikwei/repos","events_url":"https://api.github.com/users/chyikwei/events{/privacy}","received_events_url":"https://api.github.com/users/chyikwei/received_events","type":"User","site_admin":false},"repo":{"id":23800539,"name":"scikit-learn","full_name":"chyikwei/scikit-learn","owner":{"login":"chyikwei","id":2423442,"avatar_url":"https://avatars.githubusercontent.com/u/2423442?v=3","gravatar_id":"","url":"https://api.github.com/users/chyikwei","html_url":"https://github.com/chyikwei","followers_url":"https://api.github.com/users/chyikwei/followers","following_url":"https://api.github.com/users/chyikwei/following{/other_user}","gists_url":"https://api.github.com/users/chyikwei/gists{/gist_id}","starred_url":"https://api.github.com/users/chyikwei/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/chyikwei/subscriptions","organizations_url":"https://api.github.com/users/chyikwei/orgs","repos_url":"https://api.github.com/users/chyikwei/repos","events_url":"https://api.github.com/users/chyikwei/events{/privacy}","received_events_url":"https://api.github.com/users/chyikwei/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/chyikwei/scikit-learn","description":"scikit-learn: machine learning in Python","fork":true,"url":"https://api.github.com/repos/chyikwei/scikit-learn","forks_url":"https://api.github.com/repos/chyikwei/scikit-learn/forks","keys_url":"https://api.github.com/repos/chyikwei/scikit-learn/keys{/key_id}","collaborators_url":"https://api.github.com/repos/chyikwei/scikit-learn/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/chyikwei/scikit-learn/teams","hooks_url":"https://api.github.com/repos/chyikwei/scikit-learn/hooks","issue_events_url":"https://api.github.com/repos/chyikwei/scikit-learn/issues/events{/number}","events_url":"https://api.github.com/repos/chyikwei/scikit-learn/events","assignees_url":"https://api.github.com/repos/chyikwei/scikit-learn/assignees{/user}","branches_url":"https://api.github.com/repos/chyikwei/scikit-learn/branches{/branch}","tags_url":"https://api.github.com/repos/chyikwei/scikit-learn/tags","blobs_url":"https://api.github.com/repos/chyikwei/scikit-learn/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/chyikwei/scikit-learn/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/chyikwei/scikit-learn/git/refs{/sha}","trees_url":"https://api.github.com/repos/chyikwei/scikit-learn/git/trees{/sha}","statuses_url":"https://api.github.com/repos/chyikwei/scikit-learn/statuses/{sha}","languages_url":"https://api.github.com/repos/chyikwei/scikit-learn/languages","stargazers_url":"https://api.github.com/repos/chyikwei/scikit-learn/stargazers","contributors_url":"https://api.github.com/repos/chyikwei/scikit-learn/contributors","subscribers_url":"https://api.github.com/repos/chyikwei/scikit-learn/subscribers","subscription_url":"https://api.github.com/repos/chyikwei/scikit-learn/subscription","commits_url":"https://api.github.com/repos/chyikwei/scikit-learn/commits{/sha}","git_commits_url":"https://api.github.com/repos/chyikwei/scikit-learn/git/commits{/sha}","comments_url":"https://api.github.com/repos/chyikwei/scikit-learn/comments{/number}","issue_comment_url":"https://api.github.com/repos/chyikwei/scikit-learn/issues/comments{/number}","contents_url":"https://api.github.com/repos/chyikwei/scikit-learn/contents/{+path}","compare_url":"https://api.github.com/repos/chyikwei/scikit-learn/compare/{base}...{head}","merges_url":"https://api.github.com/repos/chyikwei/scikit-learn/merges","archive_url":"https://api.github.com/repos/chyikwei/scikit-learn/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/chyikwei/scikit-learn/downloads","issues_url":"https://api.github.com/repos/chyikwei/scikit-learn/issues{/number}","pulls_url":"https://api.github.com/repos/chyikwei/scikit-learn/pulls{/number}","milestones_url":"https://api.github.com/repos/chyikwei/scikit-learn/milestones{/number}","notifications_url":"https://api.github.com/repos/chyikwei/scikit-learn/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/chyikwei/scikit-learn/labels{/name}","releases_url":"https://api.github.com/repos/chyikwei/scikit-learn/releases{/id}","created_at":"2014-09-08T17:22:54Z","updated_at":"2015-04-06T16:33:09Z","pushed_at":"2015-05-01T19:31:03Z","git_url":"git://github.com/chyikwei/scikit-learn.git","ssh_url":"git@github.com:chyikwei/scikit-learn.git","clone_url":"https://github.com/chyikwei/scikit-learn.git","svn_url":"https://github.com/chyikwei/scikit-learn","homepage":"http://scikit-learn.org","size":60722,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"scikit-learn:master","ref":"master","sha":"9d5c4c68887035d30347f05e59aa49ad95cdd834","user":{"login":"scikit-learn","id":365630,"avatar_url":"https://avatars.githubusercontent.com/u/365630?v=3","gravatar_id":"","url":"https://api.github.com/users/scikit-learn","html_url":"https://github.com/scikit-learn","followers_url":"https://api.github.com/users/scikit-learn/followers","following_url":"https://api.github.com/users/scikit-learn/following{/other_user}","gists_url":"https://api.github.com/users/scikit-learn/gists{/gist_id}","starred_url":"https://api.github.com/users/scikit-learn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scikit-learn/subscriptions","organizations_url":"https://api.github.com/users/scikit-learn/orgs","repos_url":"https://api.github.com/users/scikit-learn/repos","events_url":"https://api.github.com/users/scikit-learn/events{/privacy}","received_events_url":"https://api.github.com/users/scikit-learn/received_events","type":"Organization","site_admin":false},"repo":{"id":843222,"name":"scikit-learn","full_name":"scikit-learn/scikit-learn","owner":{"login":"scikit-learn","id":365630,"avatar_url":"https://avatars.githubusercontent.com/u/365630?v=3","gravatar_id":"","url":"https://api.github.com/users/scikit-learn","html_url":"https://github.com/scikit-learn","followers_url":"https://api.github.com/users/scikit-learn/followers","following_url":"https://api.github.com/users/scikit-learn/following{/other_user}","gists_url":"https://api.github.com/users/scikit-learn/gists{/gist_id}","starred_url":"https://api.github.com/users/scikit-learn/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/scikit-learn/subscriptions","organizations_url":"https://api.github.com/users/scikit-learn/orgs","repos_url":"https://api.github.com/users/scikit-learn/repos","events_url":"https://api.github.com/users/scikit-learn/events{/privacy}","received_events_url":"https://api.github.com/users/scikit-learn/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/scikit-learn/scikit-learn","description":"scikit-learn: machine learning in Python","fork":false,"url":"https://api.github.com/repos/scikit-learn/scikit-learn","forks_url":"https://api.github.com/repos/scikit-learn/scikit-learn/forks","keys_url":"https://api.github.com/repos/scikit-learn/scikit-learn/keys{/key_id}","collaborators_url":"https://api.github.com/repos/scikit-learn/scikit-learn/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/scikit-learn/scikit-learn/teams","hooks_url":"https://api.github.com/repos/scikit-learn/scikit-learn/hooks","issue_events_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/events{/number}","events_url":"https://api.github.com/repos/scikit-learn/scikit-learn/events","assignees_url":"https://api.github.com/repos/scikit-learn/scikit-learn/assignees{/user}","branches_url":"https://api.github.com/repos/scikit-learn/scikit-learn/branches{/branch}","tags_url":"https://api.github.com/repos/scikit-learn/scikit-learn/tags","blobs_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/refs{/sha}","trees_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/trees{/sha}","statuses_url":"https://api.github.com/repos/scikit-learn/scikit-learn/statuses/{sha}","languages_url":"https://api.github.com/repos/scikit-learn/scikit-learn/languages","stargazers_url":"https://api.github.com/repos/scikit-learn/scikit-learn/stargazers","contributors_url":"https://api.github.com/repos/scikit-learn/scikit-learn/contributors","subscribers_url":"https://api.github.com/repos/scikit-learn/scikit-learn/subscribers","subscription_url":"https://api.github.com/repos/scikit-learn/scikit-learn/subscription","commits_url":"https://api.github.com/repos/scikit-learn/scikit-learn/commits{/sha}","git_commits_url":"https://api.github.com/repos/scikit-learn/scikit-learn/git/commits{/sha}","comments_url":"https://api.github.com/repos/scikit-learn/scikit-learn/comments{/number}","issue_comment_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/comments{/number}","contents_url":"https://api.github.com/repos/scikit-learn/scikit-learn/contents/{+path}","compare_url":"https://api.github.com/repos/scikit-learn/scikit-learn/compare/{base}...{head}","merges_url":"https://api.github.com/repos/scikit-learn/scikit-learn/merges","archive_url":"https://api.github.com/repos/scikit-learn/scikit-learn/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/scikit-learn/scikit-learn/downloads","issues_url":"https://api.github.com/repos/scikit-learn/scikit-learn/issues{/number}","pulls_url":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls{/number}","milestones_url":"https://api.github.com/repos/scikit-learn/scikit-learn/milestones{/number}","notifications_url":"https://api.github.com/repos/scikit-learn/scikit-learn/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/scikit-learn/scikit-learn/labels{/name}","releases_url":"https://api.github.com/repos/scikit-learn/scikit-learn/releases{/id}","created_at":"2010-08-17T09:43:38Z","updated_at":"2015-05-08T13:53:57Z","pushed_at":"2015-05-08T14:28:20Z","git_url":"git://github.com/scikit-learn/scikit-learn.git","ssh_url":"git@github.com:scikit-learn/scikit-learn.git","clone_url":"https://github.com/scikit-learn/scikit-learn.git","svn_url":"https://github.com/scikit-learn/scikit-learn","homepage":"http://scikit-learn.org","size":124204,"stargazers_count":5937,"watchers_count":5937,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":3548,"mirror_url":null,"open_issues_count":648,"forks":3548,"open_issues":648,"watchers":5937,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3659"},"html":{"href":"https://github.com/scikit-learn/scikit-learn/pull/3659"},"issue":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/3659"},"comments":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/issues/3659/comments"},"review_comments":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3659/comments"},"review_comment":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/pulls/3659/commits"},"statuses":{"href":"https://api.github.com/repos/scikit-learn/scikit-learn/statuses/100ba3bc9ddb62b929bcb00bd9ecfaf6454c2b75"}}}}