{"action":"created","comment":{"url":"https://api.github.com/repos/blaze/dask/pulls/comments/38550243","id":38550243,"diff_hunk":"@@ -25,6 +25,33 @@\n \n csv_defaults = {'compression': None}\n \n+\n+def try_pd_read_csv(*args, **kwargs):\n+    msg = \"\"\"\n+    By inspecting the first 1,000 rows of your csv file we guessed that the\n+    column: %s had dtype: %s. But now that we're reading all the data we see it\n+    has some values with dtype: %s. You should probably add the following\n+    keyword arguments to your `read_csv` call:\n+\n+        dtype={'%s': %s}\n+    \"\"\"\n+\n+    try:\n+        return pd.read_csv(*args, **kwargs)\n+    except ValueError as e:\n+        try:\n+            # this is brittle\n+            e_parts = e.message.split(' ')\n+            column_number = int(e_parts[-1])\n+            column_name = kwargs.get('names')[column_number]\n+            dtype1 = e_parts[7]\n+            dtype2 = e_parts[9]\n+            msg %= (column_name, dtype1, dtype2, column_name, dtype2)\n+        except:\n+            raise e","path":"dask/dataframe/io.py","position":27,"original_position":27,"commit_id":"cba086c0635665f7feb880e3ca6511ad917bc25c","original_commit_id":"cba086c0635665f7feb880e3ca6511ad917bc25c","user":{"login":"mrocklin","id":306380,"avatar_url":"https://avatars.githubusercontent.com/u/306380?v=3","gravatar_id":"","url":"https://api.github.com/users/mrocklin","html_url":"https://github.com/mrocklin","followers_url":"https://api.github.com/users/mrocklin/followers","following_url":"https://api.github.com/users/mrocklin/following{/other_user}","gists_url":"https://api.github.com/users/mrocklin/gists{/gist_id}","starred_url":"https://api.github.com/users/mrocklin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mrocklin/subscriptions","organizations_url":"https://api.github.com/users/mrocklin/orgs","repos_url":"https://api.github.com/users/mrocklin/repos","events_url":"https://api.github.com/users/mrocklin/events{/privacy}","received_events_url":"https://api.github.com/users/mrocklin/received_events","type":"User","site_admin":false},"body":"`except: raise` seems like a no-op.","created_at":"2015-09-02T15:59:56Z","updated_at":"2015-09-02T15:59:56Z","html_url":"https://github.com/blaze/dask/pull/646#discussion_r38550243","pull_request_url":"https://api.github.com/repos/blaze/dask/pulls/646","_links":{"self":{"href":"https://api.github.com/repos/blaze/dask/pulls/comments/38550243"},"html":{"href":"https://github.com/blaze/dask/pull/646#discussion_r38550243"},"pull_request":{"href":"https://api.github.com/repos/blaze/dask/pulls/646"}}},"pull_request":{"url":"https://api.github.com/repos/blaze/dask/pulls/646","id":43833017,"html_url":"https://github.com/blaze/dask/pull/646","diff_url":"https://github.com/blaze/dask/pull/646.diff","patch_url":"https://github.com/blaze/dask/pull/646.patch","issue_url":"https://api.github.com/repos/blaze/dask/issues/646","number":646,"state":"open","locked":false,"title":"Close #551, raise a nice error message when read_csv guesses column dtype incorrectly.","user":{"login":"cowlicks","id":598099,"avatar_url":"https://avatars.githubusercontent.com/u/598099?v=3","gravatar_id":"","url":"https://api.github.com/users/cowlicks","html_url":"https://github.com/cowlicks","followers_url":"https://api.github.com/users/cowlicks/followers","following_url":"https://api.github.com/users/cowlicks/following{/other_user}","gists_url":"https://api.github.com/users/cowlicks/gists{/gist_id}","starred_url":"https://api.github.com/users/cowlicks/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cowlicks/subscriptions","organizations_url":"https://api.github.com/users/cowlicks/orgs","repos_url":"https://api.github.com/users/cowlicks/repos","events_url":"https://api.github.com/users/cowlicks/events{/privacy}","received_events_url":"https://api.github.com/users/cowlicks/received_events","type":"User","site_admin":false},"body":"Closes #551 \r\n```python\r\nIn [1]: import tempfile\r\n\r\nIn [2]: import dask.dataframe as dd\r\n\r\nIn [3]: with tempfile.NamedTemporaryFile() as f:\r\n    f.write('numbers,names\\n')\r\n    for i in range(10000):\r\n        f.write('1,foo\\n')\r\n    f.write('1.5,bar\\n')\r\n    f.seek(0)\r\n    a = dd.read_csv(f.name)\r\n    a.compute()\r\n   ...:\r\n ---------------------------------------------------------------------------\r\nValueError                                Traceback (most recent call last)\r\n<ipython-input-3-d9a65ae0259b> in <module>()\r\n      6     f.seek(0)\r\n      7     a = dd.read_csv(f.name)\r\n----> 8     a.compute()\r\n      9 \r\n\r\n/home/blake/github/dask/dask/base.pyc in compute(self, **kwargs)\r\n     27 \r\n     28     def compute(self, **kwargs):\r\n---> 29         return compute(self, **kwargs)[0]\r\n     30 \r\n     31     @classmethod\r\n\r\n/home/blake/github/dask/dask/base.pyc in compute(*args, **kwargs)\r\n     95                 for opt, val in groups.items()])\r\n     96     keys = [arg._keys() for arg in args]\r\n---> 97     results = get(dsk, keys, **kwargs)\r\n     98     return tuple(a._finalize(a, r) for a, r in zip(args, results))\r\n     99 \r\n\r\n/home/blake/github/dask/dask/threaded.pyc in get(dsk, result, cache, num_workers, **kwargs)\r\n     55     results = get_async(pool.apply_async, len(pool._pool), dsk, result,\r\n     56                         cache=cache, queue=queue, get_id=_thread_get_id,\r\n---> 57                         **kwargs)\r\n     58 \r\n     59     return results\r\n\r\n/home/blake/github/dask/dask/async.pyc in get_async(apply_async, num_workers, dsk, result, cache, queue, get_id, raise_on_exception, rerun_exceptions_locally, callbacks, **kwargs)\r\n    498                 \"    dataset.compute(rerun_exceptions_locally=True)\\n\\n\"\r\n    499                 \"The original exception and traceback follow below:\\n\\n\"\r\n--> 500                     + str(res) + \"\\n\\nTraceback:\\n\" + tb)\r\n    501         state['cache'][key] = res\r\n    502         finish_task(dsk, key, state, results, keyorder.get)\r\n\r\nValueError: Exception occurred in remote worker.\r\n\r\nSomething you've asked dask to compute raised an exception.\r\nThat exception and the traceback are copied below.\r\nTo use pdb, rerun the computation with the keyword argument\r\n    dask.set_options(rerun_exceptions_locally=True)\r\n    or\r\n    dataset.compute(rerun_exceptions_locally=True)\r\n\r\nThe original exception and traceback follow below:\r\n\r\n\r\n    By inspecting the first 1,000 rows of your csv file we guessed that the\r\n    column: numbers had dtype: <i8. But now that we're reading all the data we see it\r\n    has some values with dtype: float64. You should probably add the following\r\n    keyword arguments to your `read_csv` call:\r\n\r\n        dtype={'numbers': float64}\r\n    \r\n\r\nTraceback:\r\n  File \"/home/blake/github/dask/dask/async.py\", line 266, in execute_task\r\n    result = _execute_task(task, data)\r\n  File \"/home/blake/github/dask/dask/async.py\", line 249, in _execute_task\r\n    return func(*args2)\r\n  File \"/home/blake/github/dask/dask/dataframe/io.py\", line 49, in try_pd_read_csv\r\n    raise ValueError(msg)\r\n```","created_at":"2015-09-01T03:04:56Z","updated_at":"2015-09-02T15:59:56Z","closed_at":null,"merged_at":null,"merge_commit_sha":"205861455b31385daccfc06d58262f55e8083023","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/blaze/dask/pulls/646/commits","review_comments_url":"https://api.github.com/repos/blaze/dask/pulls/646/comments","review_comment_url":"https://api.github.com/repos/blaze/dask/pulls/comments{/number}","comments_url":"https://api.github.com/repos/blaze/dask/issues/646/comments","statuses_url":"https://api.github.com/repos/blaze/dask/statuses/cba086c0635665f7feb880e3ca6511ad917bc25c","head":{"label":"cowlicks:gh-551","ref":"gh-551","sha":"cba086c0635665f7feb880e3ca6511ad917bc25c","user":{"login":"cowlicks","id":598099,"avatar_url":"https://avatars.githubusercontent.com/u/598099?v=3","gravatar_id":"","url":"https://api.github.com/users/cowlicks","html_url":"https://github.com/cowlicks","followers_url":"https://api.github.com/users/cowlicks/followers","following_url":"https://api.github.com/users/cowlicks/following{/other_user}","gists_url":"https://api.github.com/users/cowlicks/gists{/gist_id}","starred_url":"https://api.github.com/users/cowlicks/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cowlicks/subscriptions","organizations_url":"https://api.github.com/users/cowlicks/orgs","repos_url":"https://api.github.com/users/cowlicks/repos","events_url":"https://api.github.com/users/cowlicks/events{/privacy}","received_events_url":"https://api.github.com/users/cowlicks/received_events","type":"User","site_admin":false},"repo":{"id":31992843,"name":"dask","full_name":"cowlicks/dask","owner":{"login":"cowlicks","id":598099,"avatar_url":"https://avatars.githubusercontent.com/u/598099?v=3","gravatar_id":"","url":"https://api.github.com/users/cowlicks","html_url":"https://github.com/cowlicks","followers_url":"https://api.github.com/users/cowlicks/followers","following_url":"https://api.github.com/users/cowlicks/following{/other_user}","gists_url":"https://api.github.com/users/cowlicks/gists{/gist_id}","starred_url":"https://api.github.com/users/cowlicks/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/cowlicks/subscriptions","organizations_url":"https://api.github.com/users/cowlicks/orgs","repos_url":"https://api.github.com/users/cowlicks/repos","events_url":"https://api.github.com/users/cowlicks/events{/privacy}","received_events_url":"https://api.github.com/users/cowlicks/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/cowlicks/dask","description":"Minimal task scheduling abstraction","fork":true,"url":"https://api.github.com/repos/cowlicks/dask","forks_url":"https://api.github.com/repos/cowlicks/dask/forks","keys_url":"https://api.github.com/repos/cowlicks/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/cowlicks/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/cowlicks/dask/teams","hooks_url":"https://api.github.com/repos/cowlicks/dask/hooks","issue_events_url":"https://api.github.com/repos/cowlicks/dask/issues/events{/number}","events_url":"https://api.github.com/repos/cowlicks/dask/events","assignees_url":"https://api.github.com/repos/cowlicks/dask/assignees{/user}","branches_url":"https://api.github.com/repos/cowlicks/dask/branches{/branch}","tags_url":"https://api.github.com/repos/cowlicks/dask/tags","blobs_url":"https://api.github.com/repos/cowlicks/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/cowlicks/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/cowlicks/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/cowlicks/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/cowlicks/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/cowlicks/dask/languages","stargazers_url":"https://api.github.com/repos/cowlicks/dask/stargazers","contributors_url":"https://api.github.com/repos/cowlicks/dask/contributors","subscribers_url":"https://api.github.com/repos/cowlicks/dask/subscribers","subscription_url":"https://api.github.com/repos/cowlicks/dask/subscription","commits_url":"https://api.github.com/repos/cowlicks/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/cowlicks/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/cowlicks/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/cowlicks/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/cowlicks/dask/contents/{+path}","compare_url":"https://api.github.com/repos/cowlicks/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/cowlicks/dask/merges","archive_url":"https://api.github.com/repos/cowlicks/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/cowlicks/dask/downloads","issues_url":"https://api.github.com/repos/cowlicks/dask/issues{/number}","pulls_url":"https://api.github.com/repos/cowlicks/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/cowlicks/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/cowlicks/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/cowlicks/dask/labels{/name}","releases_url":"https://api.github.com/repos/cowlicks/dask/releases{/id}","created_at":"2015-03-11T02:01:22Z","updated_at":"2015-05-07T22:35:58Z","pushed_at":"2015-09-02T15:33:52Z","git_url":"git://github.com/cowlicks/dask.git","ssh_url":"git@github.com:cowlicks/dask.git","clone_url":"https://github.com/cowlicks/dask.git","svn_url":"https://github.com/cowlicks/dask","homepage":"http://dask.pydata.org","size":8489,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"blaze:master","ref":"master","sha":"d1fa450f8858647f04f1d23a5dd096e5a1772d5c","user":{"login":"blaze","id":12833921,"avatar_url":"https://avatars.githubusercontent.com/u/12833921?v=3","gravatar_id":"","url":"https://api.github.com/users/blaze","html_url":"https://github.com/blaze","followers_url":"https://api.github.com/users/blaze/followers","following_url":"https://api.github.com/users/blaze/following{/other_user}","gists_url":"https://api.github.com/users/blaze/gists{/gist_id}","starred_url":"https://api.github.com/users/blaze/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/blaze/subscriptions","organizations_url":"https://api.github.com/users/blaze/orgs","repos_url":"https://api.github.com/users/blaze/repos","events_url":"https://api.github.com/users/blaze/events{/privacy}","received_events_url":"https://api.github.com/users/blaze/received_events","type":"Organization","site_admin":false},"repo":{"id":28782747,"name":"dask","full_name":"blaze/dask","owner":{"login":"blaze","id":12833921,"avatar_url":"https://avatars.githubusercontent.com/u/12833921?v=3","gravatar_id":"","url":"https://api.github.com/users/blaze","html_url":"https://github.com/blaze","followers_url":"https://api.github.com/users/blaze/followers","following_url":"https://api.github.com/users/blaze/following{/other_user}","gists_url":"https://api.github.com/users/blaze/gists{/gist_id}","starred_url":"https://api.github.com/users/blaze/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/blaze/subscriptions","organizations_url":"https://api.github.com/users/blaze/orgs","repos_url":"https://api.github.com/users/blaze/repos","events_url":"https://api.github.com/users/blaze/events{/privacy}","received_events_url":"https://api.github.com/users/blaze/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/blaze/dask","description":"Task scheduling and blocked algorithms for parallel processing","fork":false,"url":"https://api.github.com/repos/blaze/dask","forks_url":"https://api.github.com/repos/blaze/dask/forks","keys_url":"https://api.github.com/repos/blaze/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/blaze/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/blaze/dask/teams","hooks_url":"https://api.github.com/repos/blaze/dask/hooks","issue_events_url":"https://api.github.com/repos/blaze/dask/issues/events{/number}","events_url":"https://api.github.com/repos/blaze/dask/events","assignees_url":"https://api.github.com/repos/blaze/dask/assignees{/user}","branches_url":"https://api.github.com/repos/blaze/dask/branches{/branch}","tags_url":"https://api.github.com/repos/blaze/dask/tags","blobs_url":"https://api.github.com/repos/blaze/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/blaze/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/blaze/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/blaze/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/blaze/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/blaze/dask/languages","stargazers_url":"https://api.github.com/repos/blaze/dask/stargazers","contributors_url":"https://api.github.com/repos/blaze/dask/contributors","subscribers_url":"https://api.github.com/repos/blaze/dask/subscribers","subscription_url":"https://api.github.com/repos/blaze/dask/subscription","commits_url":"https://api.github.com/repos/blaze/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/blaze/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/blaze/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/blaze/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/blaze/dask/contents/{+path}","compare_url":"https://api.github.com/repos/blaze/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/blaze/dask/merges","archive_url":"https://api.github.com/repos/blaze/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/blaze/dask/downloads","issues_url":"https://api.github.com/repos/blaze/dask/issues{/number}","pulls_url":"https://api.github.com/repos/blaze/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/blaze/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/blaze/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/blaze/dask/labels{/name}","releases_url":"https://api.github.com/repos/blaze/dask/releases{/id}","created_at":"2015-01-04T18:50:00Z","updated_at":"2015-09-01T02:10:45Z","pushed_at":"2015-09-02T15:45:12Z","git_url":"git://github.com/blaze/dask.git","ssh_url":"git@github.com:blaze/dask.git","clone_url":"https://github.com/blaze/dask.git","svn_url":"https://github.com/blaze/dask","homepage":"http://dask.pydata.org","size":12944,"stargazers_count":311,"watchers_count":311,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":55,"mirror_url":null,"open_issues_count":88,"forks":55,"open_issues":88,"watchers":311,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/blaze/dask/pulls/646"},"html":{"href":"https://github.com/blaze/dask/pull/646"},"issue":{"href":"https://api.github.com/repos/blaze/dask/issues/646"},"comments":{"href":"https://api.github.com/repos/blaze/dask/issues/646/comments"},"review_comments":{"href":"https://api.github.com/repos/blaze/dask/pulls/646/comments"},"review_comment":{"href":"https://api.github.com/repos/blaze/dask/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/blaze/dask/pulls/646/commits"},"statuses":{"href":"https://api.github.com/repos/blaze/dask/statuses/cba086c0635665f7feb880e3ca6511ad917bc25c"}}}}