{"action":"created","comment":{"url":"https://api.github.com/repos/pydata/pandas/pulls/comments/24160820","id":24160820,"diff_hunk":"@@ -2734,30 +2734,32 @@ def duplicated(self, subset=None, take_last=False):\n         -------\n         duplicated : Series\n         \"\"\"\n-        # kludge for #1833\n-        def _m8_to_i8(x):\n-            if issubclass(x.dtype.type, np.datetime64):\n-                return x.view(np.int64)\n-            return x\n+        from pandas.core.groupby import get_group_index\n+        from pandas.hashtable import duplicated_int64, _SIZE_HINT_LIMIT\n+\n+        size_hint = min(len(self), _SIZE_HINT_LIMIT)\n+\n+        def factorize(vals):\n+            (hash_klass, vec_klass), vals = \\\n+                    algos._get_data_algo(vals, algos._hashtables)\n+\n+            uniques, table = vec_klass(), hash_klass(size_hint)","path":"pandas/core/frame.py","position":18,"original_position":18,"commit_id":"c3cf3ece9ed4ebbcae4599c51a3c944863be2edd","original_commit_id":"c3cf3ece9ed4ebbcae4599c51a3c944863be2edd","user":{"login":"jreback","id":953992,"avatar_url":"https://avatars.githubusercontent.com/u/953992?v=3","gravatar_id":"","url":"https://api.github.com/users/jreback","html_url":"https://github.com/jreback","followers_url":"https://api.github.com/users/jreback/followers","following_url":"https://api.github.com/users/jreback/following{/other_user}","gists_url":"https://api.github.com/users/jreback/gists{/gist_id}","starred_url":"https://api.github.com/users/jreback/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jreback/subscriptions","organizations_url":"https://api.github.com/users/jreback/orgs","repos_url":"https://api.github.com/users/jreback/repos","events_url":"https://api.github.com/users/jreback/events{/privacy}","received_events_url":"https://api.github.com/users/jreback/received_events","type":"User","site_admin":false},"body":"that should be refactored into a private function as well","created_at":"2015-02-05T12:47:49Z","updated_at":"2015-02-05T12:47:49Z","html_url":"https://github.com/pydata/pandas/pull/9398#discussion_r24160820","pull_request_url":"https://api.github.com/repos/pydata/pandas/pulls/9398","_links":{"self":{"href":"https://api.github.com/repos/pydata/pandas/pulls/comments/24160820"},"html":{"href":"https://github.com/pydata/pandas/pull/9398#discussion_r24160820"},"pull_request":{"href":"https://api.github.com/repos/pydata/pandas/pulls/9398"}}},"pull_request":{"url":"https://api.github.com/repos/pydata/pandas/pulls/9398","id":28515330,"html_url":"https://github.com/pydata/pandas/pull/9398","diff_url":"https://github.com/pydata/pandas/pull/9398.diff","patch_url":"https://github.com/pydata/pandas/pull/9398.patch","issue_url":"https://api.github.com/repos/pydata/pandas/issues/9398","number":9398,"state":"open","locked":false,"title":"PERF: improves performance and memory usage of DataFrame.duplicated","user":{"login":"behzadnouri","id":1288998,"avatar_url":"https://avatars.githubusercontent.com/u/1288998?v=3","gravatar_id":"","url":"https://api.github.com/users/behzadnouri","html_url":"https://github.com/behzadnouri","followers_url":"https://api.github.com/users/behzadnouri/followers","following_url":"https://api.github.com/users/behzadnouri/following{/other_user}","gists_url":"https://api.github.com/users/behzadnouri/gists{/gist_id}","starred_url":"https://api.github.com/users/behzadnouri/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/behzadnouri/subscriptions","organizations_url":"https://api.github.com/users/behzadnouri/orgs","repos_url":"https://api.github.com/users/behzadnouri/repos","events_url":"https://api.github.com/users/behzadnouri/events{/privacy}","received_events_url":"https://api.github.com/users/behzadnouri/received_events","type":"User","site_admin":false},"body":"on master:\r\n\r\n    In [1]: np.random.seed(2718281)\r\n\r\n    In [2]: n = 1 << 20\r\n\r\n    In [3]: t = pd.date_range('2015-01-01', freq='S', periods=n // 64)\r\n\r\n    In [4]: xs = np.random.randn(n // 64).round(2)\r\n\r\n    In [5]: df = DataFrame({'a':np.random.randint(- 1 << 8, 1 << 8, n),\r\n       ...:                 'b':np.random.choice(t, n),\r\n       ...:                 'c':np.random.choice(xs, n)})\r\n\r\n    In [6]: %timeit df.duplicated()\r\n    1 loops, best of 3: 8.03 s per loop\r\n\r\n    In [7]: %memit df.duplicated()\r\n    peak memory: 461.79 MiB, increment: 356.10 MiB\r\n\r\n\r\non branch:\r\n\r\n    In [6]: %timeit df.duplicated()\r\n    1 loops, best of 3: 259 ms per loop\r\n\r\n    In [7]: %memit df.duplicated()\r\n    peak memory: 154.62 MiB, increment: 49.36 MiB\r\n\r\n\r\nbenchmarks:\r\n\r\n    -------------------------------------------------------------------------------\r\n    Test name                                    | head[ms] | base[ms] |  ratio   |\r\n    -------------------------------------------------------------------------------\r\n    frame_duplicated                             | 308.4060 | 7809.8300 |   0.0395 |\r\n    frame_drop_duplicates                        |  16.1637 |  28.8030 |   0.5612 |\r\n    frame_drop_duplicates_na                     |  17.1696 |  28.3937 |   0.6047 |\r\n    multiindex_duplicated                        | 136.1593 | 141.6107 |   0.9615 |\r\n    series_drop_duplicates_int                   |   1.1793 |   1.1443 |   1.0306 |\r\n    series_drop_duplicates_string                |   0.7900 |   0.7540 |   1.0479 |\r\n    -------------------------------------------------------------------------------\r\n    Test name                                    | head[ms] | base[ms] |  ratio   |\r\n    -------------------------------------------------------------------------------\r\n\r\n    Ratio < 1.0 means the target commit is faster then the baseline.\r\n    Seed used: 1234\r\n\r\n    Target [76459d2] : performance improvement in DataFrame.duplicated\r\n    Base   [ef48c6f] : Merge pull request #9377 from cmeeren/patch-1\r\n\r\n    DOC: Clarify how date_parser is called (GH9376)\r\n","created_at":"2015-02-03T00:12:12Z","updated_at":"2015-02-05T12:47:49Z","closed_at":null,"merged_at":null,"merge_commit_sha":"cd02d09bc1cf1b3d66f5e7d66b5dba2ccaf8d575","assignee":null,"milestone":{"url":"https://api.github.com/repos/pydata/pandas/milestones/25","labels_url":"https://api.github.com/repos/pydata/pandas/milestones/25/labels","id":569113,"number":25,"title":"0.16.0","description":"after 0.15 of course!","creator":{"login":"jreback","id":953992,"avatar_url":"https://avatars.githubusercontent.com/u/953992?v=3","gravatar_id":"","url":"https://api.github.com/users/jreback","html_url":"https://github.com/jreback","followers_url":"https://api.github.com/users/jreback/followers","following_url":"https://api.github.com/users/jreback/following{/other_user}","gists_url":"https://api.github.com/users/jreback/gists{/gist_id}","starred_url":"https://api.github.com/users/jreback/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jreback/subscriptions","organizations_url":"https://api.github.com/users/jreback/orgs","repos_url":"https://api.github.com/users/jreback/repos","events_url":"https://api.github.com/users/jreback/events{/privacy}","received_events_url":"https://api.github.com/users/jreback/received_events","type":"User","site_admin":false},"open_issues":808,"closed_issues":246,"state":"open","created_at":"2014-02-14T03:31:22Z","updated_at":"2015-02-05T11:37:52Z","due_on":"2015-02-28T08:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/pydata/pandas/pulls/9398/commits","review_comments_url":"https://api.github.com/repos/pydata/pandas/pulls/9398/comments","review_comment_url":"https://api.github.com/repos/pydata/pandas/pulls/comments/{number}","comments_url":"https://api.github.com/repos/pydata/pandas/issues/9398/comments","statuses_url":"https://api.github.com/repos/pydata/pandas/statuses/c3cf3ece9ed4ebbcae4599c51a3c944863be2edd","head":{"label":"behzadnouri:dupl","ref":"dupl","sha":"c3cf3ece9ed4ebbcae4599c51a3c944863be2edd","user":{"login":"behzadnouri","id":1288998,"avatar_url":"https://avatars.githubusercontent.com/u/1288998?v=3","gravatar_id":"","url":"https://api.github.com/users/behzadnouri","html_url":"https://github.com/behzadnouri","followers_url":"https://api.github.com/users/behzadnouri/followers","following_url":"https://api.github.com/users/behzadnouri/following{/other_user}","gists_url":"https://api.github.com/users/behzadnouri/gists{/gist_id}","starred_url":"https://api.github.com/users/behzadnouri/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/behzadnouri/subscriptions","organizations_url":"https://api.github.com/users/behzadnouri/orgs","repos_url":"https://api.github.com/users/behzadnouri/repos","events_url":"https://api.github.com/users/behzadnouri/events{/privacy}","received_events_url":"https://api.github.com/users/behzadnouri/received_events","type":"User","site_admin":false},"repo":{"id":22295418,"name":"pandas","full_name":"behzadnouri/pandas","owner":{"login":"behzadnouri","id":1288998,"avatar_url":"https://avatars.githubusercontent.com/u/1288998?v=3","gravatar_id":"","url":"https://api.github.com/users/behzadnouri","html_url":"https://github.com/behzadnouri","followers_url":"https://api.github.com/users/behzadnouri/followers","following_url":"https://api.github.com/users/behzadnouri/following{/other_user}","gists_url":"https://api.github.com/users/behzadnouri/gists{/gist_id}","starred_url":"https://api.github.com/users/behzadnouri/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/behzadnouri/subscriptions","organizations_url":"https://api.github.com/users/behzadnouri/orgs","repos_url":"https://api.github.com/users/behzadnouri/repos","events_url":"https://api.github.com/users/behzadnouri/events{/privacy}","received_events_url":"https://api.github.com/users/behzadnouri/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/behzadnouri/pandas","description":"Flexible and powerful data analysis / manipulation library for Python, providing labeled data structures similar to R data.frame objects, statistical functions, and much more","fork":true,"url":"https://api.github.com/repos/behzadnouri/pandas","forks_url":"https://api.github.com/repos/behzadnouri/pandas/forks","keys_url":"https://api.github.com/repos/behzadnouri/pandas/keys{/key_id}","collaborators_url":"https://api.github.com/repos/behzadnouri/pandas/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/behzadnouri/pandas/teams","hooks_url":"https://api.github.com/repos/behzadnouri/pandas/hooks","issue_events_url":"https://api.github.com/repos/behzadnouri/pandas/issues/events{/number}","events_url":"https://api.github.com/repos/behzadnouri/pandas/events","assignees_url":"https://api.github.com/repos/behzadnouri/pandas/assignees{/user}","branches_url":"https://api.github.com/repos/behzadnouri/pandas/branches{/branch}","tags_url":"https://api.github.com/repos/behzadnouri/pandas/tags","blobs_url":"https://api.github.com/repos/behzadnouri/pandas/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/behzadnouri/pandas/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/behzadnouri/pandas/git/refs{/sha}","trees_url":"https://api.github.com/repos/behzadnouri/pandas/git/trees{/sha}","statuses_url":"https://api.github.com/repos/behzadnouri/pandas/statuses/{sha}","languages_url":"https://api.github.com/repos/behzadnouri/pandas/languages","stargazers_url":"https://api.github.com/repos/behzadnouri/pandas/stargazers","contributors_url":"https://api.github.com/repos/behzadnouri/pandas/contributors","subscribers_url":"https://api.github.com/repos/behzadnouri/pandas/subscribers","subscription_url":"https://api.github.com/repos/behzadnouri/pandas/subscription","commits_url":"https://api.github.com/repos/behzadnouri/pandas/commits{/sha}","git_commits_url":"https://api.github.com/repos/behzadnouri/pandas/git/commits{/sha}","comments_url":"https://api.github.com/repos/behzadnouri/pandas/comments{/number}","issue_comment_url":"https://api.github.com/repos/behzadnouri/pandas/issues/comments/{number}","contents_url":"https://api.github.com/repos/behzadnouri/pandas/contents/{+path}","compare_url":"https://api.github.com/repos/behzadnouri/pandas/compare/{base}...{head}","merges_url":"https://api.github.com/repos/behzadnouri/pandas/merges","archive_url":"https://api.github.com/repos/behzadnouri/pandas/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/behzadnouri/pandas/downloads","issues_url":"https://api.github.com/repos/behzadnouri/pandas/issues{/number}","pulls_url":"https://api.github.com/repos/behzadnouri/pandas/pulls{/number}","milestones_url":"https://api.github.com/repos/behzadnouri/pandas/milestones{/number}","notifications_url":"https://api.github.com/repos/behzadnouri/pandas/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/behzadnouri/pandas/labels{/name}","releases_url":"https://api.github.com/repos/behzadnouri/pandas/releases{/id}","created_at":"2014-07-26T20:09:38Z","updated_at":"2015-02-02T02:30:58Z","pushed_at":"2015-02-03T00:13:37Z","git_url":"git://github.com/behzadnouri/pandas.git","ssh_url":"git@github.com:behzadnouri/pandas.git","clone_url":"https://github.com/behzadnouri/pandas.git","svn_url":"https://github.com/behzadnouri/pandas","homepage":"http://pandas.pydata.org","size":46077,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"pydata:master","ref":"master","sha":"484f66814a32324248e3d03203ea991ed896fd7a","user":{"login":"pydata","id":1284191,"avatar_url":"https://avatars.githubusercontent.com/u/1284191?v=3","gravatar_id":"","url":"https://api.github.com/users/pydata","html_url":"https://github.com/pydata","followers_url":"https://api.github.com/users/pydata/followers","following_url":"https://api.github.com/users/pydata/following{/other_user}","gists_url":"https://api.github.com/users/pydata/gists{/gist_id}","starred_url":"https://api.github.com/users/pydata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pydata/subscriptions","organizations_url":"https://api.github.com/users/pydata/orgs","repos_url":"https://api.github.com/users/pydata/repos","events_url":"https://api.github.com/users/pydata/events{/privacy}","received_events_url":"https://api.github.com/users/pydata/received_events","type":"Organization","site_admin":false},"repo":{"id":858127,"name":"pandas","full_name":"pydata/pandas","owner":{"login":"pydata","id":1284191,"avatar_url":"https://avatars.githubusercontent.com/u/1284191?v=3","gravatar_id":"","url":"https://api.github.com/users/pydata","html_url":"https://github.com/pydata","followers_url":"https://api.github.com/users/pydata/followers","following_url":"https://api.github.com/users/pydata/following{/other_user}","gists_url":"https://api.github.com/users/pydata/gists{/gist_id}","starred_url":"https://api.github.com/users/pydata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pydata/subscriptions","organizations_url":"https://api.github.com/users/pydata/orgs","repos_url":"https://api.github.com/users/pydata/repos","events_url":"https://api.github.com/users/pydata/events{/privacy}","received_events_url":"https://api.github.com/users/pydata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/pydata/pandas","description":"Flexible and powerful data analysis / manipulation library for Python, providing labeled data structures similar to R data.frame objects, statistical functions, and much more","fork":false,"url":"https://api.github.com/repos/pydata/pandas","forks_url":"https://api.github.com/repos/pydata/pandas/forks","keys_url":"https://api.github.com/repos/pydata/pandas/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pydata/pandas/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pydata/pandas/teams","hooks_url":"https://api.github.com/repos/pydata/pandas/hooks","issue_events_url":"https://api.github.com/repos/pydata/pandas/issues/events{/number}","events_url":"https://api.github.com/repos/pydata/pandas/events","assignees_url":"https://api.github.com/repos/pydata/pandas/assignees{/user}","branches_url":"https://api.github.com/repos/pydata/pandas/branches{/branch}","tags_url":"https://api.github.com/repos/pydata/pandas/tags","blobs_url":"https://api.github.com/repos/pydata/pandas/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pydata/pandas/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pydata/pandas/git/refs{/sha}","trees_url":"https://api.github.com/repos/pydata/pandas/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pydata/pandas/statuses/{sha}","languages_url":"https://api.github.com/repos/pydata/pandas/languages","stargazers_url":"https://api.github.com/repos/pydata/pandas/stargazers","contributors_url":"https://api.github.com/repos/pydata/pandas/contributors","subscribers_url":"https://api.github.com/repos/pydata/pandas/subscribers","subscription_url":"https://api.github.com/repos/pydata/pandas/subscription","commits_url":"https://api.github.com/repos/pydata/pandas/commits{/sha}","git_commits_url":"https://api.github.com/repos/pydata/pandas/git/commits{/sha}","comments_url":"https://api.github.com/repos/pydata/pandas/comments{/number}","issue_comment_url":"https://api.github.com/repos/pydata/pandas/issues/comments/{number}","contents_url":"https://api.github.com/repos/pydata/pandas/contents/{+path}","compare_url":"https://api.github.com/repos/pydata/pandas/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pydata/pandas/merges","archive_url":"https://api.github.com/repos/pydata/pandas/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pydata/pandas/downloads","issues_url":"https://api.github.com/repos/pydata/pandas/issues{/number}","pulls_url":"https://api.github.com/repos/pydata/pandas/pulls{/number}","milestones_url":"https://api.github.com/repos/pydata/pandas/milestones{/number}","notifications_url":"https://api.github.com/repos/pydata/pandas/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pydata/pandas/labels{/name}","releases_url":"https://api.github.com/repos/pydata/pandas/releases{/id}","created_at":"2010-08-24T01:37:33Z","updated_at":"2015-02-05T11:37:26Z","pushed_at":"2015-02-05T11:37:26Z","git_url":"git://github.com/pydata/pandas.git","ssh_url":"git@github.com:pydata/pandas.git","clone_url":"https://github.com/pydata/pandas.git","svn_url":"https://github.com/pydata/pandas","homepage":"http://pandas.pydata.org","size":121765,"stargazers_count":3968,"watchers_count":3968,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":1395,"mirror_url":null,"open_issues_count":1283,"forks":1395,"open_issues":1283,"watchers":3968,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/pydata/pandas/pulls/9398"},"html":{"href":"https://github.com/pydata/pandas/pull/9398"},"issue":{"href":"https://api.github.com/repos/pydata/pandas/issues/9398"},"comments":{"href":"https://api.github.com/repos/pydata/pandas/issues/9398/comments"},"review_comments":{"href":"https://api.github.com/repos/pydata/pandas/pulls/9398/comments"},"review_comment":{"href":"https://api.github.com/repos/pydata/pandas/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/pydata/pandas/pulls/9398/commits"},"statuses":{"href":"https://api.github.com/repos/pydata/pandas/statuses/c3cf3ece9ed4ebbcae4599c51a3c944863be2edd"}}}}