{"action":"created","comment":{"url":"https://api.github.com/repos/ContinuumIO/dask/pulls/comments/36594301","id":36594301,"diff_hunk":"@@ -402,6 +423,58 @@ def test_reductions():\n         assert eq(dds.mean(), pds.mean())\n         assert eq(dds.nunique(), pds.nunique())\n \n+def test_reductions_non_numeric_dtypes():\n+    # test non-numric blocks\n+\n+    def check_raises(d, p, func):\n+        assert raises(TypeError, lambda: getattr(d, func)().compute())\n+        assert raises(TypeError, lambda: getattr(p, func)())\n+\n+    pds = pd.Series(['a', 'b', 'c', 'd', 'e'])\n+    dds = dd.from_pandas(pds, 2)\n+    assert eq(dds.sum(), pds.sum())\n+    assert eq(dds.min(), pds.min())\n+    assert eq(dds.max(), pds.max())\n+    assert eq(dds.count(), pds.count())\n+    check_raises(dds, pds, 'std')\n+    check_raises(dds, pds, 'var')\n+    check_raises(dds, pds, 'mean')\n+    assert eq(dds.nunique(), pds.nunique())\n+\n+    for pds in [pd.Series(pd.Categorical([1, 2, 3, 4, 5], ordered=True)),\n+                pd.Series(pd.Categorical(list('abcde'), ordered=True)),\n+                pd.Series(pd.date_range('2011-01-01', freq='D', periods=5))]:\n+        dds = dd.from_pandas(pds, 2)\n+\n+        check_raises(dds, pds, 'sum')\n+        assert eq(dds.min(), pds.min())\n+        assert eq(dds.max(), pds.max())\n+        assert eq(dds.count(), pds.count())\n+        check_raises(dds, pds, 'std')\n+        check_raises(dds, pds, 'var')\n+        check_raises(dds, pds, 'mean')\n+        assert eq(dds.nunique(), pds.nunique())\n+\n+\n+    pds= pd.Series(pd.timedelta_range('1 days', freq='D', periods=5))\n+    dds = dd.from_pandas(pds, 2)\n+    assert eq(dds.sum(), pds.sum())\n+    assert eq(dds.min(), pds.min())\n+    assert eq(dds.max(), pds.max())\n+    assert eq(dds.count(), pds.count())\n+\n+    # ToDo: pandas supports timedelta std, otherwise dask raises:\n+    # incompatible type for a datetime/timedelta operation [__pow__]\n+    # assert eq(dds.std(), pds.std())","path":"dask/dataframe/tests/test_dataframe.py","position":95,"original_position":95,"commit_id":"6eb2bb4a45478b4b5f92a963d81846170b6deed3","original_commit_id":"a0989fa5346190315e39a7207e106b4c4852a3ee","user":{"login":"sinhrks","id":1696302,"avatar_url":"https://avatars.githubusercontent.com/u/1696302?v=3","gravatar_id":"","url":"https://api.github.com/users/sinhrks","html_url":"https://github.com/sinhrks","followers_url":"https://api.github.com/users/sinhrks/followers","following_url":"https://api.github.com/users/sinhrks/following{/other_user}","gists_url":"https://api.github.com/users/sinhrks/gists{/gist_id}","starred_url":"https://api.github.com/users/sinhrks/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sinhrks/subscriptions","organizations_url":"https://api.github.com/users/sinhrks/orgs","repos_url":"https://api.github.com/users/sinhrks/repos","events_url":"https://api.github.com/users/sinhrks/events{/privacy}","received_events_url":"https://api.github.com/users/sinhrks/received_events","type":"User","site_admin":false},"body":"For numerical stability, ``pandas`` is also changing to use ``two-pass`` algorithm.\r\n\r\n- https://github.com/pydata/pandas/pull/10679\r\n\r\nAnd ``pandas`` can calculate timedelta ``std``, not ``var``. Calculation looks to be performed to int representation.\r\n\r\n```\r\ns = pd.Series(pd.timedelta_range('1 day', freq='D', periods=10))\r\ns \r\n# 0    1 days\r\n# 1    2 days\r\n# 2    3 days\r\n# 3    4 days\r\n# 4    5 days\r\n# 5    6 days\r\n# 6    7 days\r\n# 7    8 days\r\n# 8    9 days\r\n# 9   10 days\r\n# dtype: timedelta64[ns]\r\n\r\ns.var()\r\n# TypeError: reduction operation 'var' not allowed for this dtype\r\n\r\ns.std()\r\n# Timedelta('3 days 00:39:48.990594')\r\n\r\n# cast to int\r\ns.astype(int).std()\r\n# 261588990594023.31\r\n\r\n# create Timedelta from the result\r\npd.Timedelta(s.astype(int).std())\r\n# Timedelta('3 days 00:39:48.990594')\r\n```","created_at":"2015-08-09T22:11:39Z","updated_at":"2015-08-09T22:11:39Z","html_url":"https://github.com/ContinuumIO/dask/pull/558#discussion_r36594301","pull_request_url":"https://api.github.com/repos/ContinuumIO/dask/pulls/558","_links":{"self":{"href":"https://api.github.com/repos/ContinuumIO/dask/pulls/comments/36594301"},"html":{"href":"https://github.com/ContinuumIO/dask/pull/558#discussion_r36594301"},"pull_request":{"href":"https://api.github.com/repos/ContinuumIO/dask/pulls/558"}}},"pull_request":{"url":"https://api.github.com/repos/ContinuumIO/dask/pulls/558","id":41970533,"html_url":"https://github.com/ContinuumIO/dask/pull/558","diff_url":"https://github.com/ContinuumIO/dask/pull/558.diff","patch_url":"https://github.com/ContinuumIO/dask/pull/558.patch","issue_url":"https://api.github.com/repos/ContinuumIO/dask/issues/558","number":558,"state":"open","locked":false,"title":"BUG: Series aggregation may output incorrect results","user":{"login":"sinhrks","id":1696302,"avatar_url":"https://avatars.githubusercontent.com/u/1696302?v=3","gravatar_id":"","url":"https://api.github.com/users/sinhrks","html_url":"https://github.com/sinhrks","followers_url":"https://api.github.com/users/sinhrks/followers","following_url":"https://api.github.com/users/sinhrks/following{/other_user}","gists_url":"https://api.github.com/users/sinhrks/gists{/gist_id}","starred_url":"https://api.github.com/users/sinhrks/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sinhrks/subscriptions","organizations_url":"https://api.github.com/users/sinhrks/orgs","repos_url":"https://api.github.com/users/sinhrks/repos","events_url":"https://api.github.com/users/sinhrks/events{/privacy}","received_events_url":"https://api.github.com/users/sinhrks/received_events","type":"User","site_admin":false},"body":"Found some cases ``dask.Series`` aggregation results are different from ``pandas``.\r\n\r\n#### 1. handiling nan\r\n\r\nIf any partition's aggregation result contains ``nan``, reduction using normal numpy function (like ``np.sum``) results in ``nan``. We can't use ``np.sum``, and also ``np.nansum`` because input can be non-numerics.\r\n\r\n```\r\nimport numpy as np\r\nimport pandas as pd\r\nimport dask.dataframe as dd\r\n\r\n# pandas\r\ns = pd.Series([1] + [np.nan] * 10)\r\ns.sum()\r\n# 1.0\r\n\r\n# dask\r\ndd.from_pandas(s, 3).sum().compute()\r\n# nan\r\n```\r\n\r\n#### 2. non-numeric dtypes\r\n\r\n``pandas`` may aggregate non-numerics, but ``dask`` doesn't.\r\n\r\n```\r\n# pandas\r\ns = pd.Series(['a', 'b', 'c', 'd', 'e'])\r\ns.sum()\r\n# 'abcde'\r\n\r\n# dask\r\ndd.from_pandas(s, 3).sum().compute()\r\n# TypeError: Exception occurred in remote worker.\r\n```\r\n\r\nThis PR fixes both issues, and adds test cases to confirm ``dask`` and ``pandas`` outputs the same result (or same error if aggregation is not supported).\r\n\r\nNOTE: ``.std`` and ``.mean`` for ``timedelta`` dtype are not supported yet.\r\n","created_at":"2015-08-09T13:38:49Z","updated_at":"2015-08-09T22:11:39Z","closed_at":null,"merged_at":null,"merge_commit_sha":"6e172d77b66ca945f0f4e762fc5942190715d5ef","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/ContinuumIO/dask/pulls/558/commits","review_comments_url":"https://api.github.com/repos/ContinuumIO/dask/pulls/558/comments","review_comment_url":"https://api.github.com/repos/ContinuumIO/dask/pulls/comments{/number}","comments_url":"https://api.github.com/repos/ContinuumIO/dask/issues/558/comments","statuses_url":"https://api.github.com/repos/ContinuumIO/dask/statuses/6eb2bb4a45478b4b5f92a963d81846170b6deed3","head":{"label":"sinhrks:nanops","ref":"nanops","sha":"6eb2bb4a45478b4b5f92a963d81846170b6deed3","user":{"login":"sinhrks","id":1696302,"avatar_url":"https://avatars.githubusercontent.com/u/1696302?v=3","gravatar_id":"","url":"https://api.github.com/users/sinhrks","html_url":"https://github.com/sinhrks","followers_url":"https://api.github.com/users/sinhrks/followers","following_url":"https://api.github.com/users/sinhrks/following{/other_user}","gists_url":"https://api.github.com/users/sinhrks/gists{/gist_id}","starred_url":"https://api.github.com/users/sinhrks/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sinhrks/subscriptions","organizations_url":"https://api.github.com/users/sinhrks/orgs","repos_url":"https://api.github.com/users/sinhrks/repos","events_url":"https://api.github.com/users/sinhrks/events{/privacy}","received_events_url":"https://api.github.com/users/sinhrks/received_events","type":"User","site_admin":false},"repo":{"id":38960014,"name":"dask","full_name":"sinhrks/dask","owner":{"login":"sinhrks","id":1696302,"avatar_url":"https://avatars.githubusercontent.com/u/1696302?v=3","gravatar_id":"","url":"https://api.github.com/users/sinhrks","html_url":"https://github.com/sinhrks","followers_url":"https://api.github.com/users/sinhrks/followers","following_url":"https://api.github.com/users/sinhrks/following{/other_user}","gists_url":"https://api.github.com/users/sinhrks/gists{/gist_id}","starred_url":"https://api.github.com/users/sinhrks/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sinhrks/subscriptions","organizations_url":"https://api.github.com/users/sinhrks/orgs","repos_url":"https://api.github.com/users/sinhrks/repos","events_url":"https://api.github.com/users/sinhrks/events{/privacy}","received_events_url":"https://api.github.com/users/sinhrks/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/sinhrks/dask","description":"Task scheduling and blocked algorithms for parallel processing","fork":true,"url":"https://api.github.com/repos/sinhrks/dask","forks_url":"https://api.github.com/repos/sinhrks/dask/forks","keys_url":"https://api.github.com/repos/sinhrks/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/sinhrks/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/sinhrks/dask/teams","hooks_url":"https://api.github.com/repos/sinhrks/dask/hooks","issue_events_url":"https://api.github.com/repos/sinhrks/dask/issues/events{/number}","events_url":"https://api.github.com/repos/sinhrks/dask/events","assignees_url":"https://api.github.com/repos/sinhrks/dask/assignees{/user}","branches_url":"https://api.github.com/repos/sinhrks/dask/branches{/branch}","tags_url":"https://api.github.com/repos/sinhrks/dask/tags","blobs_url":"https://api.github.com/repos/sinhrks/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/sinhrks/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/sinhrks/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/sinhrks/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/sinhrks/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/sinhrks/dask/languages","stargazers_url":"https://api.github.com/repos/sinhrks/dask/stargazers","contributors_url":"https://api.github.com/repos/sinhrks/dask/contributors","subscribers_url":"https://api.github.com/repos/sinhrks/dask/subscribers","subscription_url":"https://api.github.com/repos/sinhrks/dask/subscription","commits_url":"https://api.github.com/repos/sinhrks/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/sinhrks/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/sinhrks/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/sinhrks/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/sinhrks/dask/contents/{+path}","compare_url":"https://api.github.com/repos/sinhrks/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/sinhrks/dask/merges","archive_url":"https://api.github.com/repos/sinhrks/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/sinhrks/dask/downloads","issues_url":"https://api.github.com/repos/sinhrks/dask/issues{/number}","pulls_url":"https://api.github.com/repos/sinhrks/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/sinhrks/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/sinhrks/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/sinhrks/dask/labels{/name}","releases_url":"https://api.github.com/repos/sinhrks/dask/releases{/id}","created_at":"2015-07-12T11:42:08Z","updated_at":"2015-07-12T11:42:08Z","pushed_at":"2015-08-09T22:00:44Z","git_url":"git://github.com/sinhrks/dask.git","ssh_url":"git@github.com:sinhrks/dask.git","clone_url":"https://github.com/sinhrks/dask.git","svn_url":"https://github.com/sinhrks/dask","homepage":"http://dask.pydata.org","size":7673,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"ContinuumIO:master","ref":"master","sha":"576b085e2793f5b98af3ff8c43c205188df879eb","user":{"login":"ContinuumIO","id":1158637,"avatar_url":"https://avatars.githubusercontent.com/u/1158637?v=3","gravatar_id":"","url":"https://api.github.com/users/ContinuumIO","html_url":"https://github.com/ContinuumIO","followers_url":"https://api.github.com/users/ContinuumIO/followers","following_url":"https://api.github.com/users/ContinuumIO/following{/other_user}","gists_url":"https://api.github.com/users/ContinuumIO/gists{/gist_id}","starred_url":"https://api.github.com/users/ContinuumIO/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ContinuumIO/subscriptions","organizations_url":"https://api.github.com/users/ContinuumIO/orgs","repos_url":"https://api.github.com/users/ContinuumIO/repos","events_url":"https://api.github.com/users/ContinuumIO/events{/privacy}","received_events_url":"https://api.github.com/users/ContinuumIO/received_events","type":"Organization","site_admin":false},"repo":{"id":28782747,"name":"dask","full_name":"ContinuumIO/dask","owner":{"login":"ContinuumIO","id":1158637,"avatar_url":"https://avatars.githubusercontent.com/u/1158637?v=3","gravatar_id":"","url":"https://api.github.com/users/ContinuumIO","html_url":"https://github.com/ContinuumIO","followers_url":"https://api.github.com/users/ContinuumIO/followers","following_url":"https://api.github.com/users/ContinuumIO/following{/other_user}","gists_url":"https://api.github.com/users/ContinuumIO/gists{/gist_id}","starred_url":"https://api.github.com/users/ContinuumIO/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ContinuumIO/subscriptions","organizations_url":"https://api.github.com/users/ContinuumIO/orgs","repos_url":"https://api.github.com/users/ContinuumIO/repos","events_url":"https://api.github.com/users/ContinuumIO/events{/privacy}","received_events_url":"https://api.github.com/users/ContinuumIO/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/ContinuumIO/dask","description":"Task scheduling and blocked algorithms for parallel processing","fork":false,"url":"https://api.github.com/repos/ContinuumIO/dask","forks_url":"https://api.github.com/repos/ContinuumIO/dask/forks","keys_url":"https://api.github.com/repos/ContinuumIO/dask/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ContinuumIO/dask/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ContinuumIO/dask/teams","hooks_url":"https://api.github.com/repos/ContinuumIO/dask/hooks","issue_events_url":"https://api.github.com/repos/ContinuumIO/dask/issues/events{/number}","events_url":"https://api.github.com/repos/ContinuumIO/dask/events","assignees_url":"https://api.github.com/repos/ContinuumIO/dask/assignees{/user}","branches_url":"https://api.github.com/repos/ContinuumIO/dask/branches{/branch}","tags_url":"https://api.github.com/repos/ContinuumIO/dask/tags","blobs_url":"https://api.github.com/repos/ContinuumIO/dask/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ContinuumIO/dask/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ContinuumIO/dask/git/refs{/sha}","trees_url":"https://api.github.com/repos/ContinuumIO/dask/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ContinuumIO/dask/statuses/{sha}","languages_url":"https://api.github.com/repos/ContinuumIO/dask/languages","stargazers_url":"https://api.github.com/repos/ContinuumIO/dask/stargazers","contributors_url":"https://api.github.com/repos/ContinuumIO/dask/contributors","subscribers_url":"https://api.github.com/repos/ContinuumIO/dask/subscribers","subscription_url":"https://api.github.com/repos/ContinuumIO/dask/subscription","commits_url":"https://api.github.com/repos/ContinuumIO/dask/commits{/sha}","git_commits_url":"https://api.github.com/repos/ContinuumIO/dask/git/commits{/sha}","comments_url":"https://api.github.com/repos/ContinuumIO/dask/comments{/number}","issue_comment_url":"https://api.github.com/repos/ContinuumIO/dask/issues/comments{/number}","contents_url":"https://api.github.com/repos/ContinuumIO/dask/contents/{+path}","compare_url":"https://api.github.com/repos/ContinuumIO/dask/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ContinuumIO/dask/merges","archive_url":"https://api.github.com/repos/ContinuumIO/dask/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ContinuumIO/dask/downloads","issues_url":"https://api.github.com/repos/ContinuumIO/dask/issues{/number}","pulls_url":"https://api.github.com/repos/ContinuumIO/dask/pulls{/number}","milestones_url":"https://api.github.com/repos/ContinuumIO/dask/milestones{/number}","notifications_url":"https://api.github.com/repos/ContinuumIO/dask/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ContinuumIO/dask/labels{/name}","releases_url":"https://api.github.com/repos/ContinuumIO/dask/releases{/id}","created_at":"2015-01-04T18:50:00Z","updated_at":"2015-08-09T01:24:00Z","pushed_at":"2015-08-09T22:00:45Z","git_url":"git://github.com/ContinuumIO/dask.git","ssh_url":"git@github.com:ContinuumIO/dask.git","clone_url":"https://github.com/ContinuumIO/dask.git","svn_url":"https://github.com/ContinuumIO/dask","homepage":"http://dask.pydata.org","size":11415,"stargazers_count":261,"watchers_count":261,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":47,"mirror_url":null,"open_issues_count":76,"forks":47,"open_issues":76,"watchers":261,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/ContinuumIO/dask/pulls/558"},"html":{"href":"https://github.com/ContinuumIO/dask/pull/558"},"issue":{"href":"https://api.github.com/repos/ContinuumIO/dask/issues/558"},"comments":{"href":"https://api.github.com/repos/ContinuumIO/dask/issues/558/comments"},"review_comments":{"href":"https://api.github.com/repos/ContinuumIO/dask/pulls/558/comments"},"review_comment":{"href":"https://api.github.com/repos/ContinuumIO/dask/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/ContinuumIO/dask/pulls/558/commits"},"statuses":{"href":"https://api.github.com/repos/ContinuumIO/dask/statuses/6eb2bb4a45478b4b5f92a963d81846170b6deed3"}}}}