{"action":"created","comment":{"url":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls/comments/38276645","id":38276645,"diff_hunk":"@@ -143,7 +152,83 @@ public function load($filepath) {\n \t\t\tDataObject::get($this->objectClass)->removeAll();\n \t\t}\n \n-\t\treturn $this->processAll($filepath);\n+\t\t$files = $this->splitFile($filepath);\n+\n+\t\t$result = null;\n+\n+\t\tforeach ($files as $file) {\n+\t\t\t$next = $this->processAll($file);\n+\n+\t\t\tif ($result instanceof BulkLoader_Result) {\n+\t\t\t\t$result->merge($next);\n+\t\t\t} else {\n+\t\t\t\t$result = $next;\n+\t\t\t}\n+\n+\t\t\t@unlink($file);\n+\t\t}\n+\n+\t\treturn $result;\n+\t}\n+\n+\t/**\n+\t * Splits a large file up into many smaller files.\n+\t *\n+\t * @param string $path Path to large file to split\n+\t * @param int $lines Number of lines per file\n+\t *\n+\t * @return array List of file paths\n+\t */\n+\tprotected function splitFile($path, $lines = null) {\n+\t\tini_set('auto_detect_line_endings', true);\n+\n+\t\tif (!is_int($lines)) {\n+\t\t\t$lines = $this->config()->get(\"lines\");\n+\t\t}\n+\n+\t\t$new = $this->getNewSplitFileName();","path":"dev/BulkLoader.php","position":55,"original_position":55,"commit_id":"8d21d775df48726fddbc3ea9b38871ccb8514fb0","original_commit_id":"8d21d775df48726fddbc3ea9b38871ccb8514fb0","user":{"login":"assertchris","id":200609,"avatar_url":"https://avatars.githubusercontent.com/u/200609?v=3","gravatar_id":"","url":"https://api.github.com/users/assertchris","html_url":"https://github.com/assertchris","followers_url":"https://api.github.com/users/assertchris/followers","following_url":"https://api.github.com/users/assertchris/following{/other_user}","gists_url":"https://api.github.com/users/assertchris/gists{/gist_id}","starred_url":"https://api.github.com/users/assertchris/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/assertchris/subscriptions","organizations_url":"https://api.github.com/users/assertchris/orgs","repos_url":"https://api.github.com/users/assertchris/repos","events_url":"https://api.github.com/users/assertchris/events{/privacy}","received_events_url":"https://api.github.com/users/assertchris/received_events","type":"User","site_admin":false},"body":"https://github.com/silverstripe/silverstripe-framework/pull/4563/files#diff-33224465f3edd7ea4b14e3a3a37f6423R168","created_at":"2015-08-30T22:13:36Z","updated_at":"2015-08-30T22:13:36Z","html_url":"https://github.com/silverstripe/silverstripe-framework/pull/4563#discussion_r38276645","pull_request_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls/4563","_links":{"self":{"href":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls/comments/38276645"},"html":{"href":"https://github.com/silverstripe/silverstripe-framework/pull/4563#discussion_r38276645"},"pull_request":{"href":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls/4563"}}},"pull_request":{"url":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls/4563","id":43712920,"html_url":"https://github.com/silverstripe/silverstripe-framework/pull/4563","diff_url":"https://github.com/silverstripe/silverstripe-framework/pull/4563.diff","patch_url":"https://github.com/silverstripe/silverstripe-framework/pull/4563.patch","issue_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/issues/4563","number":4563,"state":"open","locked":false,"title":"Splitting BulkLoader imports to reduce memory consumption","user":{"login":"assertchris","id":200609,"avatar_url":"https://avatars.githubusercontent.com/u/200609?v=3","gravatar_id":"","url":"https://api.github.com/users/assertchris","html_url":"https://github.com/assertchris","followers_url":"https://api.github.com/users/assertchris/followers","following_url":"https://api.github.com/users/assertchris/following{/other_user}","gists_url":"https://api.github.com/users/assertchris/gists{/gist_id}","starred_url":"https://api.github.com/users/assertchris/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/assertchris/subscriptions","organizations_url":"https://api.github.com/users/assertchris/orgs","repos_url":"https://api.github.com/users/assertchris/repos","events_url":"https://api.github.com/users/assertchris/events{/privacy}","received_events_url":"https://api.github.com/users/assertchris/received_events","type":"User","site_admin":false},"body":"This allows BulkLoaders to split large files into smaller files, so that a limited amount of CSV data is read into memory during imports.\r\n\r\nInternal issue tracker reference: https://silverstripe.atlassian.net/browse/CWP-574","created_at":"2015-08-30T13:02:21Z","updated_at":"2015-08-30T22:13:36Z","closed_at":null,"merged_at":null,"merge_commit_sha":"3afb29fc28bba2a6511eb289c17da53ada59f87f","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls/4563/commits","review_comments_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls/4563/comments","review_comment_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls/comments{/number}","comments_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/issues/4563/comments","statuses_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/statuses/8d21d775df48726fddbc3ea9b38871ccb8514fb0","head":{"label":"assertchris:split-bulk-loader-imports-to-reduce-memory-consumption","ref":"split-bulk-loader-imports-to-reduce-memory-consumption","sha":"8d21d775df48726fddbc3ea9b38871ccb8514fb0","user":{"login":"assertchris","id":200609,"avatar_url":"https://avatars.githubusercontent.com/u/200609?v=3","gravatar_id":"","url":"https://api.github.com/users/assertchris","html_url":"https://github.com/assertchris","followers_url":"https://api.github.com/users/assertchris/followers","following_url":"https://api.github.com/users/assertchris/following{/other_user}","gists_url":"https://api.github.com/users/assertchris/gists{/gist_id}","starred_url":"https://api.github.com/users/assertchris/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/assertchris/subscriptions","organizations_url":"https://api.github.com/users/assertchris/orgs","repos_url":"https://api.github.com/users/assertchris/repos","events_url":"https://api.github.com/users/assertchris/events{/privacy}","received_events_url":"https://api.github.com/users/assertchris/received_events","type":"User","site_admin":false},"repo":{"id":34084845,"name":"silverstripe-framework","full_name":"assertchris/silverstripe-framework","owner":{"login":"assertchris","id":200609,"avatar_url":"https://avatars.githubusercontent.com/u/200609?v=3","gravatar_id":"","url":"https://api.github.com/users/assertchris","html_url":"https://github.com/assertchris","followers_url":"https://api.github.com/users/assertchris/followers","following_url":"https://api.github.com/users/assertchris/following{/other_user}","gists_url":"https://api.github.com/users/assertchris/gists{/gist_id}","starred_url":"https://api.github.com/users/assertchris/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/assertchris/subscriptions","organizations_url":"https://api.github.com/users/assertchris/orgs","repos_url":"https://api.github.com/users/assertchris/repos","events_url":"https://api.github.com/users/assertchris/events{/privacy}","received_events_url":"https://api.github.com/users/assertchris/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/assertchris/silverstripe-framework","description":"SilverStripe Framework, the MVC framework that powers SilverStripe CMS","fork":true,"url":"https://api.github.com/repos/assertchris/silverstripe-framework","forks_url":"https://api.github.com/repos/assertchris/silverstripe-framework/forks","keys_url":"https://api.github.com/repos/assertchris/silverstripe-framework/keys{/key_id}","collaborators_url":"https://api.github.com/repos/assertchris/silverstripe-framework/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/assertchris/silverstripe-framework/teams","hooks_url":"https://api.github.com/repos/assertchris/silverstripe-framework/hooks","issue_events_url":"https://api.github.com/repos/assertchris/silverstripe-framework/issues/events{/number}","events_url":"https://api.github.com/repos/assertchris/silverstripe-framework/events","assignees_url":"https://api.github.com/repos/assertchris/silverstripe-framework/assignees{/user}","branches_url":"https://api.github.com/repos/assertchris/silverstripe-framework/branches{/branch}","tags_url":"https://api.github.com/repos/assertchris/silverstripe-framework/tags","blobs_url":"https://api.github.com/repos/assertchris/silverstripe-framework/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/assertchris/silverstripe-framework/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/assertchris/silverstripe-framework/git/refs{/sha}","trees_url":"https://api.github.com/repos/assertchris/silverstripe-framework/git/trees{/sha}","statuses_url":"https://api.github.com/repos/assertchris/silverstripe-framework/statuses/{sha}","languages_url":"https://api.github.com/repos/assertchris/silverstripe-framework/languages","stargazers_url":"https://api.github.com/repos/assertchris/silverstripe-framework/stargazers","contributors_url":"https://api.github.com/repos/assertchris/silverstripe-framework/contributors","subscribers_url":"https://api.github.com/repos/assertchris/silverstripe-framework/subscribers","subscription_url":"https://api.github.com/repos/assertchris/silverstripe-framework/subscription","commits_url":"https://api.github.com/repos/assertchris/silverstripe-framework/commits{/sha}","git_commits_url":"https://api.github.com/repos/assertchris/silverstripe-framework/git/commits{/sha}","comments_url":"https://api.github.com/repos/assertchris/silverstripe-framework/comments{/number}","issue_comment_url":"https://api.github.com/repos/assertchris/silverstripe-framework/issues/comments{/number}","contents_url":"https://api.github.com/repos/assertchris/silverstripe-framework/contents/{+path}","compare_url":"https://api.github.com/repos/assertchris/silverstripe-framework/compare/{base}...{head}","merges_url":"https://api.github.com/repos/assertchris/silverstripe-framework/merges","archive_url":"https://api.github.com/repos/assertchris/silverstripe-framework/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/assertchris/silverstripe-framework/downloads","issues_url":"https://api.github.com/repos/assertchris/silverstripe-framework/issues{/number}","pulls_url":"https://api.github.com/repos/assertchris/silverstripe-framework/pulls{/number}","milestones_url":"https://api.github.com/repos/assertchris/silverstripe-framework/milestones{/number}","notifications_url":"https://api.github.com/repos/assertchris/silverstripe-framework/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/assertchris/silverstripe-framework/labels{/name}","releases_url":"https://api.github.com/repos/assertchris/silverstripe-framework/releases{/id}","created_at":"2015-04-16T22:55:07Z","updated_at":"2015-05-08T01:15:53Z","pushed_at":"2015-08-30T13:00:34Z","git_url":"git://github.com/assertchris/silverstripe-framework.git","ssh_url":"git@github.com:assertchris/silverstripe-framework.git","clone_url":"https://github.com/assertchris/silverstripe-framework.git","svn_url":"https://github.com/assertchris/silverstripe-framework","homepage":"http://silverstripe.org/","size":51986,"stargazers_count":0,"watchers_count":0,"language":"PHP","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"3.1"}},"base":{"label":"silverstripe:3","ref":"3","sha":"f4b7cd3f68f78767ec02df90a30a591893f234eb","user":{"login":"silverstripe","id":379348,"avatar_url":"https://avatars.githubusercontent.com/u/379348?v=3","gravatar_id":"","url":"https://api.github.com/users/silverstripe","html_url":"https://github.com/silverstripe","followers_url":"https://api.github.com/users/silverstripe/followers","following_url":"https://api.github.com/users/silverstripe/following{/other_user}","gists_url":"https://api.github.com/users/silverstripe/gists{/gist_id}","starred_url":"https://api.github.com/users/silverstripe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/silverstripe/subscriptions","organizations_url":"https://api.github.com/users/silverstripe/orgs","repos_url":"https://api.github.com/users/silverstripe/repos","events_url":"https://api.github.com/users/silverstripe/events{/privacy}","received_events_url":"https://api.github.com/users/silverstripe/received_events","type":"Organization","site_admin":false},"repo":{"id":1318892,"name":"silverstripe-framework","full_name":"silverstripe/silverstripe-framework","owner":{"login":"silverstripe","id":379348,"avatar_url":"https://avatars.githubusercontent.com/u/379348?v=3","gravatar_id":"","url":"https://api.github.com/users/silverstripe","html_url":"https://github.com/silverstripe","followers_url":"https://api.github.com/users/silverstripe/followers","following_url":"https://api.github.com/users/silverstripe/following{/other_user}","gists_url":"https://api.github.com/users/silverstripe/gists{/gist_id}","starred_url":"https://api.github.com/users/silverstripe/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/silverstripe/subscriptions","organizations_url":"https://api.github.com/users/silverstripe/orgs","repos_url":"https://api.github.com/users/silverstripe/repos","events_url":"https://api.github.com/users/silverstripe/events{/privacy}","received_events_url":"https://api.github.com/users/silverstripe/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/silverstripe/silverstripe-framework","description":"SilverStripe Framework, the MVC framework that powers SilverStripe CMS","fork":false,"url":"https://api.github.com/repos/silverstripe/silverstripe-framework","forks_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/forks","keys_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/keys{/key_id}","collaborators_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/teams","hooks_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/hooks","issue_events_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/issues/events{/number}","events_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/events","assignees_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/assignees{/user}","branches_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/branches{/branch}","tags_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/tags","blobs_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/git/refs{/sha}","trees_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/git/trees{/sha}","statuses_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/statuses/{sha}","languages_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/languages","stargazers_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/stargazers","contributors_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/contributors","subscribers_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/subscribers","subscription_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/subscription","commits_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/commits{/sha}","git_commits_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/git/commits{/sha}","comments_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/comments{/number}","issue_comment_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/issues/comments{/number}","contents_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/contents/{+path}","compare_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/compare/{base}...{head}","merges_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/merges","archive_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/downloads","issues_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/issues{/number}","pulls_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls{/number}","milestones_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/milestones{/number}","notifications_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/labels{/name}","releases_url":"https://api.github.com/repos/silverstripe/silverstripe-framework/releases{/id}","created_at":"2011-02-02T02:04:13Z","updated_at":"2015-08-30T07:51:07Z","pushed_at":"2015-08-30T13:02:22Z","git_url":"git://github.com/silverstripe/silverstripe-framework.git","ssh_url":"git@github.com:silverstripe/silverstripe-framework.git","clone_url":"https://github.com/silverstripe/silverstripe-framework.git","svn_url":"https://github.com/silverstripe/silverstripe-framework","homepage":"http://silverstripe.org/","size":105266,"stargazers_count":460,"watchers_count":460,"language":"PHP","has_issues":true,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":516,"mirror_url":null,"open_issues_count":611,"forks":516,"open_issues":611,"watchers":460,"default_branch":"3"}},"_links":{"self":{"href":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls/4563"},"html":{"href":"https://github.com/silverstripe/silverstripe-framework/pull/4563"},"issue":{"href":"https://api.github.com/repos/silverstripe/silverstripe-framework/issues/4563"},"comments":{"href":"https://api.github.com/repos/silverstripe/silverstripe-framework/issues/4563/comments"},"review_comments":{"href":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls/4563/comments"},"review_comment":{"href":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/silverstripe/silverstripe-framework/pulls/4563/commits"},"statuses":{"href":"https://api.github.com/repos/silverstripe/silverstripe-framework/statuses/8d21d775df48726fddbc3ea9b38871ccb8514fb0"}}}}