{"action":"created","comment":{"url":"https://api.github.com/repos/torch/optim/pulls/comments/38901880","id":38901880,"diff_hunk":"@@ -103,74 +112,65 @@ function optim.lbfgs(opfunc, x, config, state)\n          Hdiag = 1\n       else\n          -- do lbfgs update (update memory)\n-         local y = g:clone():add(-1, g_old)  -- g - g_old\n-         local s = d:clone():mul(t)  -- d*t\n+         local y = table.remove(state.dir_bufs)  -- pop\n+         local s = table.remove(state.stp_bufs)\n+         y:copy(g):add(-1, g_old)  -- g - g_old\n+         s:copy(d):mul(t)          -- d*t\n          local ys = y:dot(s)  -- y*s\n          if ys > 1e-10 then\n             -- updating memory\n             if #old_dirs == nCorrection then\n                -- shift history by one (limited-memory)\n-               local prev_old_dirs = old_dirs\n-               local prev_old_stps = old_stps\n-               old_dirs = {}\n-               old_stps = {}\n-               for i = 2,#prev_old_dirs do\n-                  append(old_dirs, prev_old_dirs[i])\n-                  append(old_stps, prev_old_stps[i])\n-               end\n+               local removed1 = table.remove(old_dirs, 1)\n+               local removed2 = table.remove(old_stps, 1)\n+               table.insert(state.dir_bufs, removed1)\n+               table.insert(state.stp_bufs, removed2)\n             end\n \n             -- store new direction/step\n-            append(old_dirs, s)\n-            append(old_stps, y)\n+            table.insert(old_dirs, s)\n+            table.insert(old_stps, y)\n \n             -- update scale of initial Hessian approximation\n             Hdiag = ys / y:dot(y)  -- (y*y)\n-\n-            -- cleanup\n-            collectgarbage()\n+         else\n+            -- put y and s back into the buffer pool\n+            table.insert(state.dir_bufs, y)\n+            table.insert(state.stp_bufs, s)\n          end\n \n          -- compute the approximate (L-BFGS) inverse Hessian \n          -- multiplied by the gradient\n-         local p = g:size(1)\n          local k = #old_dirs\n \n-         state.ro = state.ro or zeros(nCorrection); local ro = state.ro\n+         -- need to be accessed element-by-element, so don't re-type tensor:\n+         state.ro = state.ro or torch.Tensor(nCorrection); local ro = state.ro\n          for i = 1,k do\n             ro[i] = 1 / old_stps[i]:dot(old_dirs[i])\n          end\n \n-         state.q = state.q or zeros(nCorrection+1,p):typeAs(g) \n-         local q = state.q\n-         state.r = state.r or zeros(nCorrection+1,p):typeAs(g)\n-         local r = state.r\n-         state.al = state.al or zeros(nCorrection):typeAs(g)\n-         local al = state.al\n-         state.be = state.be or zeros(nCorrection):typeAs(g)\n-         local be = state.be\n-\n-         q[k+1] = g:clone():mul(-1)  -- -g\n+         -- iteration in L-BFGS loop collapsed to use just one buffer\n+         local q = tmp1  -- reuse tmp1 for the q buffer\n+         -- need to be accessed element-by-element, so don't re-type tensor:\n+         state.al = state.al or torch.zeros(nCorrection) local al = state.al\n \n+         q:copy(g):mul(-1)  -- -g\n          for i = k,1,-1 do\n-            al[i] = old_dirs[i]:dot(q[i+1]) * ro[i]\n-            q[i] = q[i+1]\n-            q[i]:add(-al[i], old_stps[i])\n+            al[i] = old_dirs[i]:dot(q) * ro[i]\n+            q:add(-al[i], old_stps[i])\n          end\n \n          -- multiply by initial Hessian\n-         r[1] = q[1]:clone():mul(Hdiag)  -- q[1] * Hdiag\n-\n+         r = d  -- share the same buffer, since we don't need the old d\n+         r:copy(q):mul(Hdiag)  -- q[1] * Hdiag","path":"lbfgs.lua","position":133,"original_position":133,"commit_id":"32854d2278c8e63c3fc05229e21eb0e9ee875c4d","original_commit_id":"32854d2278c8e63c3fc05229e21eb0e9ee875c4d","user":{"login":"fmassa","id":9110200,"avatar_url":"https://avatars.githubusercontent.com/u/9110200?v=3","gravatar_id":"","url":"https://api.github.com/users/fmassa","html_url":"https://github.com/fmassa","followers_url":"https://api.github.com/users/fmassa/followers","following_url":"https://api.github.com/users/fmassa/following{/other_user}","gists_url":"https://api.github.com/users/fmassa/gists{/gist_id}","starred_url":"https://api.github.com/users/fmassa/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/fmassa/subscriptions","organizations_url":"https://api.github.com/users/fmassa/orgs","repos_url":"https://api.github.com/users/fmassa/repos","events_url":"https://api.github.com/users/fmassa/events{/privacy}","received_events_url":"https://api.github.com/users/fmassa/received_events","type":"User","site_admin":false},"body":"and here","created_at":"2015-09-08T08:41:36Z","updated_at":"2015-09-08T08:41:36Z","html_url":"https://github.com/torch/optim/pull/69#discussion_r38901880","pull_request_url":"https://api.github.com/repos/torch/optim/pulls/69","_links":{"self":{"href":"https://api.github.com/repos/torch/optim/pulls/comments/38901880"},"html":{"href":"https://github.com/torch/optim/pull/69#discussion_r38901880"},"pull_request":{"href":"https://api.github.com/repos/torch/optim/pulls/69"}}},"pull_request":{"url":"https://api.github.com/repos/torch/optim/pulls/69","id":44392181,"html_url":"https://github.com/torch/optim/pull/69","diff_url":"https://github.com/torch/optim/pull/69.diff","patch_url":"https://github.com/torch/optim/pull/69.patch","issue_url":"https://api.github.com/repos/torch/optim/issues/69","number":69,"state":"open","locked":false,"title":"Memory usage improvements for L-BFGS","user":{"login":"bshillingford","id":2326749,"avatar_url":"https://avatars.githubusercontent.com/u/2326749?v=3","gravatar_id":"","url":"https://api.github.com/users/bshillingford","html_url":"https://github.com/bshillingford","followers_url":"https://api.github.com/users/bshillingford/followers","following_url":"https://api.github.com/users/bshillingford/following{/other_user}","gists_url":"https://api.github.com/users/bshillingford/gists{/gist_id}","starred_url":"https://api.github.com/users/bshillingford/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bshillingford/subscriptions","organizations_url":"https://api.github.com/users/bshillingford/orgs","repos_url":"https://api.github.com/users/bshillingford/repos","events_url":"https://api.github.com/users/bshillingford/events{/privacy}","received_events_url":"https://api.github.com/users/bshillingford/received_events","type":"User","site_admin":false},"body":"List of improvements:\r\n\r\n* a pool of buffers is preallocated for `y` and `s` and the histories, to prevent allocation/deallocation inside the loop. An earlier quick check with nvprof when using the old code with CUDA showed a lot of time spent on `cudaFree()` calls--this was the motivation for these fixes. [1]\r\n* \"matrix multiplication\" / accumulation step is done with a single temporary vector, to avoid growing on order `nCorrection`\r\n* reuse temporary buffers as much as possible\r\n\r\nLet me know if you want me to break it into a few commits, see here for an example comparing it to the original implementation, see the regression test `test.lua` here: <https://gist.github.com/bshillingford/315540c787c46e46f13d>. Output is exact for DoubleTensors. I didn't include the regression because it explicitly runs both versions of `lbfgs.lua` on the same inputs and compares them, so it'd entail keeping the old `lbfgs.lua` in `tests/`.\r\n\r\n[1] Note: it may seem weird that `y` and `s` are in this pool too, but this prevents copies in the cases where the `if` on line 108 is true, and they get inserted into the history. If not, the same tensors get recycled for `y` and `s` on the next iteration, anyway.","created_at":"2015-09-08T04:31:44Z","updated_at":"2015-09-08T08:41:36Z","closed_at":null,"merged_at":null,"merge_commit_sha":"89ec8428c5fb216d1d10ae91b2ffb61fc9122d64","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/torch/optim/pulls/69/commits","review_comments_url":"https://api.github.com/repos/torch/optim/pulls/69/comments","review_comment_url":"https://api.github.com/repos/torch/optim/pulls/comments{/number}","comments_url":"https://api.github.com/repos/torch/optim/issues/69/comments","statuses_url":"https://api.github.com/repos/torch/optim/statuses/32854d2278c8e63c3fc05229e21eb0e9ee875c4d","head":{"label":"bshillingford:master","ref":"master","sha":"32854d2278c8e63c3fc05229e21eb0e9ee875c4d","user":{"login":"bshillingford","id":2326749,"avatar_url":"https://avatars.githubusercontent.com/u/2326749?v=3","gravatar_id":"","url":"https://api.github.com/users/bshillingford","html_url":"https://github.com/bshillingford","followers_url":"https://api.github.com/users/bshillingford/followers","following_url":"https://api.github.com/users/bshillingford/following{/other_user}","gists_url":"https://api.github.com/users/bshillingford/gists{/gist_id}","starred_url":"https://api.github.com/users/bshillingford/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bshillingford/subscriptions","organizations_url":"https://api.github.com/users/bshillingford/orgs","repos_url":"https://api.github.com/users/bshillingford/repos","events_url":"https://api.github.com/users/bshillingford/events{/privacy}","received_events_url":"https://api.github.com/users/bshillingford/received_events","type":"User","site_admin":false},"repo":{"id":42088681,"name":"optim","full_name":"bshillingford/optim","owner":{"login":"bshillingford","id":2326749,"avatar_url":"https://avatars.githubusercontent.com/u/2326749?v=3","gravatar_id":"","url":"https://api.github.com/users/bshillingford","html_url":"https://github.com/bshillingford","followers_url":"https://api.github.com/users/bshillingford/followers","following_url":"https://api.github.com/users/bshillingford/following{/other_user}","gists_url":"https://api.github.com/users/bshillingford/gists{/gist_id}","starred_url":"https://api.github.com/users/bshillingford/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bshillingford/subscriptions","organizations_url":"https://api.github.com/users/bshillingford/orgs","repos_url":"https://api.github.com/users/bshillingford/repos","events_url":"https://api.github.com/users/bshillingford/events{/privacy}","received_events_url":"https://api.github.com/users/bshillingford/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/bshillingford/optim","description":"A numeric optimization package for Torch.","fork":true,"url":"https://api.github.com/repos/bshillingford/optim","forks_url":"https://api.github.com/repos/bshillingford/optim/forks","keys_url":"https://api.github.com/repos/bshillingford/optim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/bshillingford/optim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/bshillingford/optim/teams","hooks_url":"https://api.github.com/repos/bshillingford/optim/hooks","issue_events_url":"https://api.github.com/repos/bshillingford/optim/issues/events{/number}","events_url":"https://api.github.com/repos/bshillingford/optim/events","assignees_url":"https://api.github.com/repos/bshillingford/optim/assignees{/user}","branches_url":"https://api.github.com/repos/bshillingford/optim/branches{/branch}","tags_url":"https://api.github.com/repos/bshillingford/optim/tags","blobs_url":"https://api.github.com/repos/bshillingford/optim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/bshillingford/optim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/bshillingford/optim/git/refs{/sha}","trees_url":"https://api.github.com/repos/bshillingford/optim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/bshillingford/optim/statuses/{sha}","languages_url":"https://api.github.com/repos/bshillingford/optim/languages","stargazers_url":"https://api.github.com/repos/bshillingford/optim/stargazers","contributors_url":"https://api.github.com/repos/bshillingford/optim/contributors","subscribers_url":"https://api.github.com/repos/bshillingford/optim/subscribers","subscription_url":"https://api.github.com/repos/bshillingford/optim/subscription","commits_url":"https://api.github.com/repos/bshillingford/optim/commits{/sha}","git_commits_url":"https://api.github.com/repos/bshillingford/optim/git/commits{/sha}","comments_url":"https://api.github.com/repos/bshillingford/optim/comments{/number}","issue_comment_url":"https://api.github.com/repos/bshillingford/optim/issues/comments{/number}","contents_url":"https://api.github.com/repos/bshillingford/optim/contents/{+path}","compare_url":"https://api.github.com/repos/bshillingford/optim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/bshillingford/optim/merges","archive_url":"https://api.github.com/repos/bshillingford/optim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/bshillingford/optim/downloads","issues_url":"https://api.github.com/repos/bshillingford/optim/issues{/number}","pulls_url":"https://api.github.com/repos/bshillingford/optim/pulls{/number}","milestones_url":"https://api.github.com/repos/bshillingford/optim/milestones{/number}","notifications_url":"https://api.github.com/repos/bshillingford/optim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/bshillingford/optim/labels{/name}","releases_url":"https://api.github.com/repos/bshillingford/optim/releases{/id}","created_at":"2015-09-08T03:47:20Z","updated_at":"2015-09-08T03:47:23Z","pushed_at":"2015-09-08T04:19:14Z","git_url":"git://github.com/bshillingford/optim.git","ssh_url":"git@github.com:bshillingford/optim.git","clone_url":"https://github.com/bshillingford/optim.git","svn_url":"https://github.com/bshillingford/optim","homepage":null,"size":612,"stargazers_count":0,"watchers_count":0,"language":"Lua","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"torch:master","ref":"master","sha":"a078e1edf26edb5d1ec02798903da7a8d634acb6","user":{"login":"torch","id":4978924,"avatar_url":"https://avatars.githubusercontent.com/u/4978924?v=3","gravatar_id":"","url":"https://api.github.com/users/torch","html_url":"https://github.com/torch","followers_url":"https://api.github.com/users/torch/followers","following_url":"https://api.github.com/users/torch/following{/other_user}","gists_url":"https://api.github.com/users/torch/gists{/gist_id}","starred_url":"https://api.github.com/users/torch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/torch/subscriptions","organizations_url":"https://api.github.com/users/torch/orgs","repos_url":"https://api.github.com/users/torch/repos","events_url":"https://api.github.com/users/torch/events{/privacy}","received_events_url":"https://api.github.com/users/torch/received_events","type":"Organization","site_admin":false},"repo":{"id":11314007,"name":"optim","full_name":"torch/optim","owner":{"login":"torch","id":4978924,"avatar_url":"https://avatars.githubusercontent.com/u/4978924?v=3","gravatar_id":"","url":"https://api.github.com/users/torch","html_url":"https://github.com/torch","followers_url":"https://api.github.com/users/torch/followers","following_url":"https://api.github.com/users/torch/following{/other_user}","gists_url":"https://api.github.com/users/torch/gists{/gist_id}","starred_url":"https://api.github.com/users/torch/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/torch/subscriptions","organizations_url":"https://api.github.com/users/torch/orgs","repos_url":"https://api.github.com/users/torch/repos","events_url":"https://api.github.com/users/torch/events{/privacy}","received_events_url":"https://api.github.com/users/torch/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/torch/optim","description":"A numeric optimization package for Torch.","fork":false,"url":"https://api.github.com/repos/torch/optim","forks_url":"https://api.github.com/repos/torch/optim/forks","keys_url":"https://api.github.com/repos/torch/optim/keys{/key_id}","collaborators_url":"https://api.github.com/repos/torch/optim/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/torch/optim/teams","hooks_url":"https://api.github.com/repos/torch/optim/hooks","issue_events_url":"https://api.github.com/repos/torch/optim/issues/events{/number}","events_url":"https://api.github.com/repos/torch/optim/events","assignees_url":"https://api.github.com/repos/torch/optim/assignees{/user}","branches_url":"https://api.github.com/repos/torch/optim/branches{/branch}","tags_url":"https://api.github.com/repos/torch/optim/tags","blobs_url":"https://api.github.com/repos/torch/optim/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/torch/optim/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/torch/optim/git/refs{/sha}","trees_url":"https://api.github.com/repos/torch/optim/git/trees{/sha}","statuses_url":"https://api.github.com/repos/torch/optim/statuses/{sha}","languages_url":"https://api.github.com/repos/torch/optim/languages","stargazers_url":"https://api.github.com/repos/torch/optim/stargazers","contributors_url":"https://api.github.com/repos/torch/optim/contributors","subscribers_url":"https://api.github.com/repos/torch/optim/subscribers","subscription_url":"https://api.github.com/repos/torch/optim/subscription","commits_url":"https://api.github.com/repos/torch/optim/commits{/sha}","git_commits_url":"https://api.github.com/repos/torch/optim/git/commits{/sha}","comments_url":"https://api.github.com/repos/torch/optim/comments{/number}","issue_comment_url":"https://api.github.com/repos/torch/optim/issues/comments{/number}","contents_url":"https://api.github.com/repos/torch/optim/contents/{+path}","compare_url":"https://api.github.com/repos/torch/optim/compare/{base}...{head}","merges_url":"https://api.github.com/repos/torch/optim/merges","archive_url":"https://api.github.com/repos/torch/optim/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/torch/optim/downloads","issues_url":"https://api.github.com/repos/torch/optim/issues{/number}","pulls_url":"https://api.github.com/repos/torch/optim/pulls{/number}","milestones_url":"https://api.github.com/repos/torch/optim/milestones{/number}","notifications_url":"https://api.github.com/repos/torch/optim/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/torch/optim/labels{/name}","releases_url":"https://api.github.com/repos/torch/optim/releases{/id}","created_at":"2013-07-10T14:20:58Z","updated_at":"2015-08-31T02:13:03Z","pushed_at":"2015-09-08T04:31:44Z","git_url":"git://github.com/torch/optim.git","ssh_url":"git@github.com:torch/optim.git","clone_url":"https://github.com/torch/optim.git","svn_url":"https://github.com/torch/optim","homepage":null,"size":612,"stargazers_count":57,"watchers_count":57,"language":"Lua","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":52,"mirror_url":null,"open_issues_count":3,"forks":52,"open_issues":3,"watchers":57,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/torch/optim/pulls/69"},"html":{"href":"https://github.com/torch/optim/pull/69"},"issue":{"href":"https://api.github.com/repos/torch/optim/issues/69"},"comments":{"href":"https://api.github.com/repos/torch/optim/issues/69/comments"},"review_comments":{"href":"https://api.github.com/repos/torch/optim/pulls/69/comments"},"review_comment":{"href":"https://api.github.com/repos/torch/optim/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/torch/optim/pulls/69/commits"},"statuses":{"href":"https://api.github.com/repos/torch/optim/statuses/32854d2278c8e63c3fc05229e21eb0e9ee875c4d"}}}}