{"action":"closed","number":11073,"pull_request":{"url":"https://api.github.com/repos/pydata/pandas/pulls/11073","id":44853669,"html_url":"https://github.com/pydata/pandas/pull/11073","diff_url":"https://github.com/pydata/pandas/pull/11073.diff","patch_url":"https://github.com/pydata/pandas/pull/11073.patch","issue_url":"https://api.github.com/repos/pydata/pandas/issues/11073","number":11073,"state":"closed","locked":false,"title":"ENH Enable streaming from S3","user":{"login":"stephen-hoover","id":8355977,"avatar_url":"https://avatars.githubusercontent.com/u/8355977?v=3","gravatar_id":"","url":"https://api.github.com/users/stephen-hoover","html_url":"https://github.com/stephen-hoover","followers_url":"https://api.github.com/users/stephen-hoover/followers","following_url":"https://api.github.com/users/stephen-hoover/following{/other_user}","gists_url":"https://api.github.com/users/stephen-hoover/gists{/gist_id}","starred_url":"https://api.github.com/users/stephen-hoover/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/stephen-hoover/subscriptions","organizations_url":"https://api.github.com/users/stephen-hoover/orgs","repos_url":"https://api.github.com/users/stephen-hoover/repos","events_url":"https://api.github.com/users/stephen-hoover/events{/privacy}","received_events_url":"https://api.github.com/users/stephen-hoover/received_events","type":"User","site_admin":false},"body":"File reading from AWS S3: Modify the `get_filepath_or_buffer` function such that it only opens the connection to S3, rather than reading the entire file at once. This allows partial reads (e.g. through the `nrows` argument) or chunked reading (e.g. through the `chunksize` argument) without needing to download the entire file first.\r\n\r\nI wasn't sure what the best place was to put the `OnceThroughKey`. (Suggestions for better names welcome.) I don't like putting an entire class inside a function like that, but this keeps the `boto` dependency contained.\r\n\r\nThe `readline` function, and modifying `next` such that it returns lines, was necessary to allow the Python engine to read uncompressed CSVs.\r\n\r\nThe Python 2 standard library's `gzip` module needs a `seek` and `tell` function on its inputs, so I reverted to the old behavior there.\r\n\r\nPartially addresses #11070 .","created_at":"2015-09-12T13:51:09Z","updated_at":"2015-09-14T22:24:49Z","closed_at":"2015-09-14T22:24:49Z","merged_at":"2015-09-14T22:24:49Z","merge_commit_sha":"26b2fbafcc8b2dd7413c746741e1576c35d27d2f","assignee":null,"milestone":{"url":"https://api.github.com/repos/pydata/pandas/milestones/34","html_url":"https://github.com/pydata/pandas/milestones/0.17.0","labels_url":"https://api.github.com/repos/pydata/pandas/milestones/34/labels","id":1033710,"number":34,"title":"0.17.0","description":"","creator":{"login":"jreback","id":953992,"avatar_url":"https://avatars.githubusercontent.com/u/953992?v=3","gravatar_id":"","url":"https://api.github.com/users/jreback","html_url":"https://github.com/jreback","followers_url":"https://api.github.com/users/jreback/followers","following_url":"https://api.github.com/users/jreback/following{/other_user}","gists_url":"https://api.github.com/users/jreback/gists{/gist_id}","starred_url":"https://api.github.com/users/jreback/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jreback/subscriptions","organizations_url":"https://api.github.com/users/jreback/orgs","repos_url":"https://api.github.com/users/jreback/repos","events_url":"https://api.github.com/users/jreback/events{/privacy}","received_events_url":"https://api.github.com/users/jreback/received_events","type":"User","site_admin":false},"open_issues":6,"closed_issues":505,"state":"open","created_at":"2015-03-23T10:47:38Z","updated_at":"2015-09-14T22:24:49Z","due_on":"2015-09-15T04:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/pydata/pandas/pulls/11073/commits","review_comments_url":"https://api.github.com/repos/pydata/pandas/pulls/11073/comments","review_comment_url":"https://api.github.com/repos/pydata/pandas/pulls/comments{/number}","comments_url":"https://api.github.com/repos/pydata/pandas/issues/11073/comments","statuses_url":"https://api.github.com/repos/pydata/pandas/statuses/67879f54b330641b3d5eff3e00344afb120265de","head":{"label":"stephen-hoover:stream-csv-from-s3","ref":"stream-csv-from-s3","sha":"67879f54b330641b3d5eff3e00344afb120265de","user":{"login":"stephen-hoover","id":8355977,"avatar_url":"https://avatars.githubusercontent.com/u/8355977?v=3","gravatar_id":"","url":"https://api.github.com/users/stephen-hoover","html_url":"https://github.com/stephen-hoover","followers_url":"https://api.github.com/users/stephen-hoover/followers","following_url":"https://api.github.com/users/stephen-hoover/following{/other_user}","gists_url":"https://api.github.com/users/stephen-hoover/gists{/gist_id}","starred_url":"https://api.github.com/users/stephen-hoover/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/stephen-hoover/subscriptions","organizations_url":"https://api.github.com/users/stephen-hoover/orgs","repos_url":"https://api.github.com/users/stephen-hoover/repos","events_url":"https://api.github.com/users/stephen-hoover/events{/privacy}","received_events_url":"https://api.github.com/users/stephen-hoover/received_events","type":"User","site_admin":false},"repo":{"id":39215160,"name":"pandas","full_name":"stephen-hoover/pandas","owner":{"login":"stephen-hoover","id":8355977,"avatar_url":"https://avatars.githubusercontent.com/u/8355977?v=3","gravatar_id":"","url":"https://api.github.com/users/stephen-hoover","html_url":"https://github.com/stephen-hoover","followers_url":"https://api.github.com/users/stephen-hoover/followers","following_url":"https://api.github.com/users/stephen-hoover/following{/other_user}","gists_url":"https://api.github.com/users/stephen-hoover/gists{/gist_id}","starred_url":"https://api.github.com/users/stephen-hoover/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/stephen-hoover/subscriptions","organizations_url":"https://api.github.com/users/stephen-hoover/orgs","repos_url":"https://api.github.com/users/stephen-hoover/repos","events_url":"https://api.github.com/users/stephen-hoover/events{/privacy}","received_events_url":"https://api.github.com/users/stephen-hoover/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/stephen-hoover/pandas","description":"Flexible and powerful data analysis / manipulation library for Python, providing labeled data structures similar to R data.frame objects, statistical functions, and much more","fork":true,"url":"https://api.github.com/repos/stephen-hoover/pandas","forks_url":"https://api.github.com/repos/stephen-hoover/pandas/forks","keys_url":"https://api.github.com/repos/stephen-hoover/pandas/keys{/key_id}","collaborators_url":"https://api.github.com/repos/stephen-hoover/pandas/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/stephen-hoover/pandas/teams","hooks_url":"https://api.github.com/repos/stephen-hoover/pandas/hooks","issue_events_url":"https://api.github.com/repos/stephen-hoover/pandas/issues/events{/number}","events_url":"https://api.github.com/repos/stephen-hoover/pandas/events","assignees_url":"https://api.github.com/repos/stephen-hoover/pandas/assignees{/user}","branches_url":"https://api.github.com/repos/stephen-hoover/pandas/branches{/branch}","tags_url":"https://api.github.com/repos/stephen-hoover/pandas/tags","blobs_url":"https://api.github.com/repos/stephen-hoover/pandas/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/stephen-hoover/pandas/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/stephen-hoover/pandas/git/refs{/sha}","trees_url":"https://api.github.com/repos/stephen-hoover/pandas/git/trees{/sha}","statuses_url":"https://api.github.com/repos/stephen-hoover/pandas/statuses/{sha}","languages_url":"https://api.github.com/repos/stephen-hoover/pandas/languages","stargazers_url":"https://api.github.com/repos/stephen-hoover/pandas/stargazers","contributors_url":"https://api.github.com/repos/stephen-hoover/pandas/contributors","subscribers_url":"https://api.github.com/repos/stephen-hoover/pandas/subscribers","subscription_url":"https://api.github.com/repos/stephen-hoover/pandas/subscription","commits_url":"https://api.github.com/repos/stephen-hoover/pandas/commits{/sha}","git_commits_url":"https://api.github.com/repos/stephen-hoover/pandas/git/commits{/sha}","comments_url":"https://api.github.com/repos/stephen-hoover/pandas/comments{/number}","issue_comment_url":"https://api.github.com/repos/stephen-hoover/pandas/issues/comments{/number}","contents_url":"https://api.github.com/repos/stephen-hoover/pandas/contents/{+path}","compare_url":"https://api.github.com/repos/stephen-hoover/pandas/compare/{base}...{head}","merges_url":"https://api.github.com/repos/stephen-hoover/pandas/merges","archive_url":"https://api.github.com/repos/stephen-hoover/pandas/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/stephen-hoover/pandas/downloads","issues_url":"https://api.github.com/repos/stephen-hoover/pandas/issues{/number}","pulls_url":"https://api.github.com/repos/stephen-hoover/pandas/pulls{/number}","milestones_url":"https://api.github.com/repos/stephen-hoover/pandas/milestones{/number}","notifications_url":"https://api.github.com/repos/stephen-hoover/pandas/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/stephen-hoover/pandas/labels{/name}","releases_url":"https://api.github.com/repos/stephen-hoover/pandas/releases{/id}","created_at":"2015-07-16T19:06:19Z","updated_at":"2015-07-16T19:06:23Z","pushed_at":"2015-09-14T20:12:22Z","git_url":"git://github.com/stephen-hoover/pandas.git","ssh_url":"git@github.com:stephen-hoover/pandas.git","clone_url":"https://github.com/stephen-hoover/pandas.git","svn_url":"https://github.com/stephen-hoover/pandas","homepage":"http://pandas.pydata.org","size":191565,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"pydata:master","ref":"master","sha":"9be218045216c207b72545a9ea5338303598d89d","user":{"login":"pydata","id":1284191,"avatar_url":"https://avatars.githubusercontent.com/u/1284191?v=3","gravatar_id":"","url":"https://api.github.com/users/pydata","html_url":"https://github.com/pydata","followers_url":"https://api.github.com/users/pydata/followers","following_url":"https://api.github.com/users/pydata/following{/other_user}","gists_url":"https://api.github.com/users/pydata/gists{/gist_id}","starred_url":"https://api.github.com/users/pydata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pydata/subscriptions","organizations_url":"https://api.github.com/users/pydata/orgs","repos_url":"https://api.github.com/users/pydata/repos","events_url":"https://api.github.com/users/pydata/events{/privacy}","received_events_url":"https://api.github.com/users/pydata/received_events","type":"Organization","site_admin":false},"repo":{"id":858127,"name":"pandas","full_name":"pydata/pandas","owner":{"login":"pydata","id":1284191,"avatar_url":"https://avatars.githubusercontent.com/u/1284191?v=3","gravatar_id":"","url":"https://api.github.com/users/pydata","html_url":"https://github.com/pydata","followers_url":"https://api.github.com/users/pydata/followers","following_url":"https://api.github.com/users/pydata/following{/other_user}","gists_url":"https://api.github.com/users/pydata/gists{/gist_id}","starred_url":"https://api.github.com/users/pydata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pydata/subscriptions","organizations_url":"https://api.github.com/users/pydata/orgs","repos_url":"https://api.github.com/users/pydata/repos","events_url":"https://api.github.com/users/pydata/events{/privacy}","received_events_url":"https://api.github.com/users/pydata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/pydata/pandas","description":"Flexible and powerful data analysis / manipulation library for Python, providing labeled data structures similar to R data.frame objects, statistical functions, and much more","fork":false,"url":"https://api.github.com/repos/pydata/pandas","forks_url":"https://api.github.com/repos/pydata/pandas/forks","keys_url":"https://api.github.com/repos/pydata/pandas/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pydata/pandas/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pydata/pandas/teams","hooks_url":"https://api.github.com/repos/pydata/pandas/hooks","issue_events_url":"https://api.github.com/repos/pydata/pandas/issues/events{/number}","events_url":"https://api.github.com/repos/pydata/pandas/events","assignees_url":"https://api.github.com/repos/pydata/pandas/assignees{/user}","branches_url":"https://api.github.com/repos/pydata/pandas/branches{/branch}","tags_url":"https://api.github.com/repos/pydata/pandas/tags","blobs_url":"https://api.github.com/repos/pydata/pandas/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pydata/pandas/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pydata/pandas/git/refs{/sha}","trees_url":"https://api.github.com/repos/pydata/pandas/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pydata/pandas/statuses/{sha}","languages_url":"https://api.github.com/repos/pydata/pandas/languages","stargazers_url":"https://api.github.com/repos/pydata/pandas/stargazers","contributors_url":"https://api.github.com/repos/pydata/pandas/contributors","subscribers_url":"https://api.github.com/repos/pydata/pandas/subscribers","subscription_url":"https://api.github.com/repos/pydata/pandas/subscription","commits_url":"https://api.github.com/repos/pydata/pandas/commits{/sha}","git_commits_url":"https://api.github.com/repos/pydata/pandas/git/commits{/sha}","comments_url":"https://api.github.com/repos/pydata/pandas/comments{/number}","issue_comment_url":"https://api.github.com/repos/pydata/pandas/issues/comments{/number}","contents_url":"https://api.github.com/repos/pydata/pandas/contents/{+path}","compare_url":"https://api.github.com/repos/pydata/pandas/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pydata/pandas/merges","archive_url":"https://api.github.com/repos/pydata/pandas/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pydata/pandas/downloads","issues_url":"https://api.github.com/repos/pydata/pandas/issues{/number}","pulls_url":"https://api.github.com/repos/pydata/pandas/pulls{/number}","milestones_url":"https://api.github.com/repos/pydata/pandas/milestones{/number}","notifications_url":"https://api.github.com/repos/pydata/pandas/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pydata/pandas/labels{/name}","releases_url":"https://api.github.com/repos/pydata/pandas/releases{/id}","created_at":"2010-08-24T01:37:33Z","updated_at":"2015-09-14T13:39:11Z","pushed_at":"2015-09-14T22:24:49Z","git_url":"git://github.com/pydata/pandas.git","ssh_url":"git@github.com:pydata/pandas.git","clone_url":"https://github.com/pydata/pandas.git","svn_url":"https://github.com/pydata/pandas","homepage":"http://pandas.pydata.org","size":157444,"stargazers_count":4934,"watchers_count":4934,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":1906,"mirror_url":null,"open_issues_count":1459,"forks":1906,"open_issues":1459,"watchers":4934,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/pydata/pandas/pulls/11073"},"html":{"href":"https://github.com/pydata/pandas/pull/11073"},"issue":{"href":"https://api.github.com/repos/pydata/pandas/issues/11073"},"comments":{"href":"https://api.github.com/repos/pydata/pandas/issues/11073/comments"},"review_comments":{"href":"https://api.github.com/repos/pydata/pandas/pulls/11073/comments"},"review_comment":{"href":"https://api.github.com/repos/pydata/pandas/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/pydata/pandas/pulls/11073/commits"},"statuses":{"href":"https://api.github.com/repos/pydata/pandas/statuses/67879f54b330641b3d5eff3e00344afb120265de"}},"merged":true,"mergeable":null,"mergeable_state":"unknown","merged_by":{"login":"jreback","id":953992,"avatar_url":"https://avatars.githubusercontent.com/u/953992?v=3","gravatar_id":"","url":"https://api.github.com/users/jreback","html_url":"https://github.com/jreback","followers_url":"https://api.github.com/users/jreback/followers","following_url":"https://api.github.com/users/jreback/following{/other_user}","gists_url":"https://api.github.com/users/jreback/gists{/gist_id}","starred_url":"https://api.github.com/users/jreback/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jreback/subscriptions","organizations_url":"https://api.github.com/users/jreback/orgs","repos_url":"https://api.github.com/users/jreback/repos","events_url":"https://api.github.com/users/jreback/events{/privacy}","received_events_url":"https://api.github.com/users/jreback/received_events","type":"User","site_admin":false},"comments":19,"review_comments":12,"commits":1,"additions":204,"deletions":13,"changed_files":4}}