{"action":"created","comment":{"url":"https://api.github.com/repos/OCA/web/pulls/comments/34156045","id":34156045,"diff_hunk":"@@ -0,0 +1,201 @@\n+# -*- encoding: utf-8 -*-\n+##############################################################################\n+#\n+#    Copyright (C) 2012-2015 credativ Ltd (<http://credativ.co.uk>).\n+#\n+#    This program is free software: you can redistribute it and/or modify\n+#    it under the terms of the GNU Affero General Public License as published\n+#    by the Free Software Foundation, either version 3 of the License, or\n+#    (at your option) any later version.\n+#\n+#    This program is distributed in the hope that it will be useful,\n+#    but WITHOUT ANY WARRANTY; without even the implied warranty of\n+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the\n+#    GNU Affero General Public License for more details.\n+#\n+#    You should have received a copy of the GNU Affero General Public License\n+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.\n+#\n+##############################################################################\n+import csv\n+import operator\n+import simplejson\n+import re\n+from cStringIO import StringIO\n+try:\n+    import xlwt\n+except ImportError:\n+    xlwt = None\n+\n+import openerp\n+from openerp import sql_db\n+from openerp.addons.web.controllers.main import (ExportFormat,\n+                                                 serialize_exception)\n+from openerp import http\n+from openerp.http import request\n+\n+\n+def itter_ids(all_ids, step_size=250):\n+    \"\"\" Splits all_ids into more manageable chunks of\n+        size step_size and yields them \"\"\"\n+    offset = 0\n+    while True:\n+        id_set = all_ids[offset:offset+step_size]\n+        offset += step_size\n+        if len(id_set):\n+            yield id_set\n+        else:\n+            return\n+\n+\n+def itter_data(Model, dbname, uid, all_ids, field_names, raw_data, context):\n+    \"\"\" Reads data in chunks of ids from all_ids and\n+        yields the rows one by one \"\"\"\n+    cr = sql_db.db_connect(dbname).cursor()\n+    with cr:\n+        with openerp.api.Environment.manage():\n+            for ids in all_ids:\n+                import_data = Model.export_data(cr,\n+                                                uid,\n+                                                ids,\n+                                                field_names,\n+                                                raw_data,\n+                                                context=context\n+                                                ).get('datas', [])","path":"web_exportall/controllers/main.py","position":64,"original_position":64,"commit_id":"7e67ab29cd9a446a1dbb1687a66a5e95583cd521","original_commit_id":"7e67ab29cd9a446a1dbb1687a66a5e95583cd521","user":{"login":"bwrsandman","id":1013356,"avatar_url":"https://avatars.githubusercontent.com/u/1013356?v=3","gravatar_id":"","url":"https://api.github.com/users/bwrsandman","html_url":"https://github.com/bwrsandman","followers_url":"https://api.github.com/users/bwrsandman/followers","following_url":"https://api.github.com/users/bwrsandman/following{/other_user}","gists_url":"https://api.github.com/users/bwrsandman/gists{/gist_id}","starred_url":"https://api.github.com/users/bwrsandman/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/bwrsandman/subscriptions","organizations_url":"https://api.github.com/users/bwrsandman/orgs","repos_url":"https://api.github.com/users/bwrsandman/repos","events_url":"https://api.github.com/users/bwrsandman/events{/privacy}","received_events_url":"https://api.github.com/users/bwrsandman/received_events","type":"User","site_admin":false},"body":"> If for instance you are downloading 1M+ records the existing code will certainly crash the server as it will consume all memory (at least in version 6.1 and 7.0, it may have changed in 8.0 but not as far as I am aware). This change which streams the data will use limited memory.\r\n\r\nUnless `export_data` is a generator (I didn't check), the memory is already allocated here and `import_data` refers to it. As long as the return from this function doesn't copy this data, the memory is already used up until the generator of this function is done iterating or otherwise deallocated.","created_at":"2015-07-08T14:49:50Z","updated_at":"2015-07-08T14:49:50Z","html_url":"https://github.com/OCA/web/pull/171#discussion_r34156045","pull_request_url":"https://api.github.com/repos/OCA/web/pulls/171","_links":{"self":{"href":"https://api.github.com/repos/OCA/web/pulls/comments/34156045"},"html":{"href":"https://github.com/OCA/web/pull/171#discussion_r34156045"},"pull_request":{"href":"https://api.github.com/repos/OCA/web/pulls/171"}}},"pull_request":{"url":"https://api.github.com/repos/OCA/web/pulls/171","id":39453418,"html_url":"https://github.com/OCA/web/pull/171","diff_url":"https://github.com/OCA/web/pull/171.diff","patch_url":"https://github.com/OCA/web/pull/171.patch","issue_url":"https://api.github.com/repos/OCA/web/issues/171","number":171,"state":"open","locked":false,"title":"[ADD][8.0] New module: web_exportall","user":{"login":"craiggowing","id":3197405,"avatar_url":"https://avatars.githubusercontent.com/u/3197405?v=3","gravatar_id":"","url":"https://api.github.com/users/craiggowing","html_url":"https://github.com/craiggowing","followers_url":"https://api.github.com/users/craiggowing/followers","following_url":"https://api.github.com/users/craiggowing/following{/other_user}","gists_url":"https://api.github.com/users/craiggowing/gists{/gist_id}","starred_url":"https://api.github.com/users/craiggowing/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/craiggowing/subscriptions","organizations_url":"https://api.github.com/users/craiggowing/orgs","repos_url":"https://api.github.com/users/craiggowing/repos","events_url":"https://api.github.com/users/craiggowing/events{/privacy}","received_events_url":"https://api.github.com/users/craiggowing/received_events","type":"User","site_admin":false},"body":"This module adds a feature to export all records from the current search view including all records from all other pages. This also adds a handler to the server to send a streamed response to prevent timeouts and reduce memory usage when many records are being read.","created_at":"2015-07-08T08:40:14Z","updated_at":"2015-07-08T14:49:50Z","closed_at":null,"merged_at":null,"merge_commit_sha":"d7b2005fd50bc88f44f476d6ecd18b4c1ba2d5fb","assignee":null,"milestone":{"url":"https://api.github.com/repos/OCA/web/milestones/3","html_url":"https://github.com/OCA/web/milestones/8.0","labels_url":"https://api.github.com/repos/OCA/web/milestones/3/labels","id":819641,"number":3,"title":"8.0","description":"","creator":{"login":"legalsylvain","id":3407482,"avatar_url":"https://avatars.githubusercontent.com/u/3407482?v=3","gravatar_id":"","url":"https://api.github.com/users/legalsylvain","html_url":"https://github.com/legalsylvain","followers_url":"https://api.github.com/users/legalsylvain/followers","following_url":"https://api.github.com/users/legalsylvain/following{/other_user}","gists_url":"https://api.github.com/users/legalsylvain/gists{/gist_id}","starred_url":"https://api.github.com/users/legalsylvain/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/legalsylvain/subscriptions","organizations_url":"https://api.github.com/users/legalsylvain/orgs","repos_url":"https://api.github.com/users/legalsylvain/repos","events_url":"https://api.github.com/users/legalsylvain/events{/privacy}","received_events_url":"https://api.github.com/users/legalsylvain/received_events","type":"User","site_admin":false},"open_issues":17,"closed_issues":41,"state":"open","created_at":"2014-10-09T10:16:30Z","updated_at":"2015-07-08T14:42:22Z","due_on":null,"closed_at":null},"commits_url":"https://api.github.com/repos/OCA/web/pulls/171/commits","review_comments_url":"https://api.github.com/repos/OCA/web/pulls/171/comments","review_comment_url":"https://api.github.com/repos/OCA/web/pulls/comments{/number}","comments_url":"https://api.github.com/repos/OCA/web/issues/171/comments","statuses_url":"https://api.github.com/repos/OCA/web/statuses/7e67ab29cd9a446a1dbb1687a66a5e95583cd521","head":{"label":"credativUK:8.0-web_exportall","ref":"8.0-web_exportall","sha":"7e67ab29cd9a446a1dbb1687a66a5e95583cd521","user":{"login":"credativUK","id":5400298,"avatar_url":"https://avatars.githubusercontent.com/u/5400298?v=3","gravatar_id":"","url":"https://api.github.com/users/credativUK","html_url":"https://github.com/credativUK","followers_url":"https://api.github.com/users/credativUK/followers","following_url":"https://api.github.com/users/credativUK/following{/other_user}","gists_url":"https://api.github.com/users/credativUK/gists{/gist_id}","starred_url":"https://api.github.com/users/credativUK/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/credativUK/subscriptions","organizations_url":"https://api.github.com/users/credativUK/orgs","repos_url":"https://api.github.com/users/credativUK/repos","events_url":"https://api.github.com/users/credativUK/events{/privacy}","received_events_url":"https://api.github.com/users/credativUK/received_events","type":"Organization","site_admin":false},"repo":{"id":38690349,"name":"web","full_name":"credativUK/web","owner":{"login":"credativUK","id":5400298,"avatar_url":"https://avatars.githubusercontent.com/u/5400298?v=3","gravatar_id":"","url":"https://api.github.com/users/credativUK","html_url":"https://github.com/credativUK","followers_url":"https://api.github.com/users/credativUK/followers","following_url":"https://api.github.com/users/credativUK/following{/other_user}","gists_url":"https://api.github.com/users/credativUK/gists{/gist_id}","starred_url":"https://api.github.com/users/credativUK/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/credativUK/subscriptions","organizations_url":"https://api.github.com/users/credativUK/orgs","repos_url":"https://api.github.com/users/credativUK/repos","events_url":"https://api.github.com/users/credativUK/events{/privacy}","received_events_url":"https://api.github.com/users/credativUK/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/credativUK/web","description":"Odoo web client UI related addons","fork":true,"url":"https://api.github.com/repos/credativUK/web","forks_url":"https://api.github.com/repos/credativUK/web/forks","keys_url":"https://api.github.com/repos/credativUK/web/keys{/key_id}","collaborators_url":"https://api.github.com/repos/credativUK/web/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/credativUK/web/teams","hooks_url":"https://api.github.com/repos/credativUK/web/hooks","issue_events_url":"https://api.github.com/repos/credativUK/web/issues/events{/number}","events_url":"https://api.github.com/repos/credativUK/web/events","assignees_url":"https://api.github.com/repos/credativUK/web/assignees{/user}","branches_url":"https://api.github.com/repos/credativUK/web/branches{/branch}","tags_url":"https://api.github.com/repos/credativUK/web/tags","blobs_url":"https://api.github.com/repos/credativUK/web/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/credativUK/web/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/credativUK/web/git/refs{/sha}","trees_url":"https://api.github.com/repos/credativUK/web/git/trees{/sha}","statuses_url":"https://api.github.com/repos/credativUK/web/statuses/{sha}","languages_url":"https://api.github.com/repos/credativUK/web/languages","stargazers_url":"https://api.github.com/repos/credativUK/web/stargazers","contributors_url":"https://api.github.com/repos/credativUK/web/contributors","subscribers_url":"https://api.github.com/repos/credativUK/web/subscribers","subscription_url":"https://api.github.com/repos/credativUK/web/subscription","commits_url":"https://api.github.com/repos/credativUK/web/commits{/sha}","git_commits_url":"https://api.github.com/repos/credativUK/web/git/commits{/sha}","comments_url":"https://api.github.com/repos/credativUK/web/comments{/number}","issue_comment_url":"https://api.github.com/repos/credativUK/web/issues/comments{/number}","contents_url":"https://api.github.com/repos/credativUK/web/contents/{+path}","compare_url":"https://api.github.com/repos/credativUK/web/compare/{base}...{head}","merges_url":"https://api.github.com/repos/credativUK/web/merges","archive_url":"https://api.github.com/repos/credativUK/web/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/credativUK/web/downloads","issues_url":"https://api.github.com/repos/credativUK/web/issues{/number}","pulls_url":"https://api.github.com/repos/credativUK/web/pulls{/number}","milestones_url":"https://api.github.com/repos/credativUK/web/milestones{/number}","notifications_url":"https://api.github.com/repos/credativUK/web/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/credativUK/web/labels{/name}","releases_url":"https://api.github.com/repos/credativUK/web/releases{/id}","created_at":"2015-07-07T13:44:52Z","updated_at":"2015-07-07T13:44:53Z","pushed_at":"2015-07-08T14:35:08Z","git_url":"git://github.com/credativUK/web.git","ssh_url":"git@github.com:credativUK/web.git","clone_url":"https://github.com/credativUK/web.git","svn_url":"https://github.com/credativUK/web","homepage":"","size":7353,"stargazers_count":0,"watchers_count":0,"language":"JavaScript","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"8.0"}},"base":{"label":"OCA:8.0","ref":"8.0","sha":"c2a8edde9f533c622797171efa2410bf149b0ef1","user":{"login":"OCA","id":7600578,"avatar_url":"https://avatars.githubusercontent.com/u/7600578?v=3","gravatar_id":"","url":"https://api.github.com/users/OCA","html_url":"https://github.com/OCA","followers_url":"https://api.github.com/users/OCA/followers","following_url":"https://api.github.com/users/OCA/following{/other_user}","gists_url":"https://api.github.com/users/OCA/gists{/gist_id}","starred_url":"https://api.github.com/users/OCA/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/OCA/subscriptions","organizations_url":"https://api.github.com/users/OCA/orgs","repos_url":"https://api.github.com/users/OCA/repos","events_url":"https://api.github.com/users/OCA/events{/privacy}","received_events_url":"https://api.github.com/users/OCA/received_events","type":"Organization","site_admin":false},"repo":{"id":20895915,"name":"web","full_name":"OCA/web","owner":{"login":"OCA","id":7600578,"avatar_url":"https://avatars.githubusercontent.com/u/7600578?v=3","gravatar_id":"","url":"https://api.github.com/users/OCA","html_url":"https://github.com/OCA","followers_url":"https://api.github.com/users/OCA/followers","following_url":"https://api.github.com/users/OCA/following{/other_user}","gists_url":"https://api.github.com/users/OCA/gists{/gist_id}","starred_url":"https://api.github.com/users/OCA/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/OCA/subscriptions","organizations_url":"https://api.github.com/users/OCA/orgs","repos_url":"https://api.github.com/users/OCA/repos","events_url":"https://api.github.com/users/OCA/events{/privacy}","received_events_url":"https://api.github.com/users/OCA/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/OCA/web","description":"Odoo web client UI related addons","fork":false,"url":"https://api.github.com/repos/OCA/web","forks_url":"https://api.github.com/repos/OCA/web/forks","keys_url":"https://api.github.com/repos/OCA/web/keys{/key_id}","collaborators_url":"https://api.github.com/repos/OCA/web/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/OCA/web/teams","hooks_url":"https://api.github.com/repos/OCA/web/hooks","issue_events_url":"https://api.github.com/repos/OCA/web/issues/events{/number}","events_url":"https://api.github.com/repos/OCA/web/events","assignees_url":"https://api.github.com/repos/OCA/web/assignees{/user}","branches_url":"https://api.github.com/repos/OCA/web/branches{/branch}","tags_url":"https://api.github.com/repos/OCA/web/tags","blobs_url":"https://api.github.com/repos/OCA/web/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/OCA/web/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/OCA/web/git/refs{/sha}","trees_url":"https://api.github.com/repos/OCA/web/git/trees{/sha}","statuses_url":"https://api.github.com/repos/OCA/web/statuses/{sha}","languages_url":"https://api.github.com/repos/OCA/web/languages","stargazers_url":"https://api.github.com/repos/OCA/web/stargazers","contributors_url":"https://api.github.com/repos/OCA/web/contributors","subscribers_url":"https://api.github.com/repos/OCA/web/subscribers","subscription_url":"https://api.github.com/repos/OCA/web/subscription","commits_url":"https://api.github.com/repos/OCA/web/commits{/sha}","git_commits_url":"https://api.github.com/repos/OCA/web/git/commits{/sha}","comments_url":"https://api.github.com/repos/OCA/web/comments{/number}","issue_comment_url":"https://api.github.com/repos/OCA/web/issues/comments{/number}","contents_url":"https://api.github.com/repos/OCA/web/contents/{+path}","compare_url":"https://api.github.com/repos/OCA/web/compare/{base}...{head}","merges_url":"https://api.github.com/repos/OCA/web/merges","archive_url":"https://api.github.com/repos/OCA/web/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/OCA/web/downloads","issues_url":"https://api.github.com/repos/OCA/web/issues{/number}","pulls_url":"https://api.github.com/repos/OCA/web/pulls{/number}","milestones_url":"https://api.github.com/repos/OCA/web/milestones{/number}","notifications_url":"https://api.github.com/repos/OCA/web/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/OCA/web/labels{/name}","releases_url":"https://api.github.com/repos/OCA/web/releases{/id}","created_at":"2014-06-16T18:34:12Z","updated_at":"2015-07-01T11:13:03Z","pushed_at":"2015-07-08T14:35:09Z","git_url":"git://github.com/OCA/web.git","ssh_url":"git@github.com:OCA/web.git","clone_url":"https://github.com/OCA/web.git","svn_url":"https://github.com/OCA/web","homepage":"","size":7353,"stargazers_count":43,"watchers_count":43,"language":"JavaScript","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":125,"mirror_url":null,"open_issues_count":50,"forks":125,"open_issues":50,"watchers":43,"default_branch":"8.0"}},"_links":{"self":{"href":"https://api.github.com/repos/OCA/web/pulls/171"},"html":{"href":"https://github.com/OCA/web/pull/171"},"issue":{"href":"https://api.github.com/repos/OCA/web/issues/171"},"comments":{"href":"https://api.github.com/repos/OCA/web/issues/171/comments"},"review_comments":{"href":"https://api.github.com/repos/OCA/web/pulls/171/comments"},"review_comment":{"href":"https://api.github.com/repos/OCA/web/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/OCA/web/pulls/171/commits"},"statuses":{"href":"https://api.github.com/repos/OCA/web/statuses/7e67ab29cd9a446a1dbb1687a66a5e95583cd521"}}}}