{"action":"created","comment":{"url":"https://api.github.com/repos/pydata/pandas/pulls/comments/46428275","id":46428275,"diff_hunk":"@@ -486,24 +489,103 @@ def f(x):\n         else:\n             return empty_row\n \n-    if regex.groups == 1:\n-        result = np.array([f(val)[0] for val in arr], dtype=object)\n-        name = _get_single_group_name(regex)\n+    names = dict(zip(regex.groupindex.values(), regex.groupindex.keys()))\n+    columns = [names.get(1 + i, i) for i in range(regex.groups)]\n+    if len(arr) == 0:\n+        return DataFrame(columns=columns, dtype=object)\n     else:\n-        if isinstance(arr, Index):\n-            raise ValueError(\"only one regex group is supported with Index\")\n-        name = None\n-        names = dict(zip(regex.groupindex.values(), regex.groupindex.keys()))\n-        columns = [names.get(1 + i, i) for i in range(regex.groups)]\n-        if arr.empty:\n-            result = DataFrame(columns=columns, dtype=object)\n-        else:\n-            result = DataFrame([f(val) for val in arr],\n-                               columns=columns,\n-                               index=arr.index,\n-                               dtype=object)\n-    return result, name\n+        try:\n+            result_index = arr.index\n+        except AttributeError:\n+            result_index = None\n+        return DataFrame(\n+            [f(val) for val in arr],\n+            columns=columns,\n+            index=result_index,\n+            dtype=object)\n+\n+def str_extractall(arr, pat, flags=0):\n+    \"\"\"\n+    For each subject string in the Series, extract groups from all\n+    matches of regular expression pat. When each subject string in the\n+    Series has exactly one match, extractall(pat).xs(0, level='match')\n+    is the same as extract(pat).\n+\n+    .. versionadded:: 0.18.0\n+\n+    Parameters\n+    ----------\n+    pat : string\n+        Regular expression pattern with capturing groups\n+    flags : int, default 0 (no flags)\n+        re module flags, e.g. re.IGNORECASE\n+\n+    Returns\n+    -------\n+    A DataFrame with one row for each match, and one column for each\n+    group. Its rows have a MultiIndex with first levels that come\n+    from the subject Series. The last index level is named \"match\"\n+    and indicates the order in the subject.\n \n+    Examples\n+    --------\n+    A pattern with one group will return a DataFrame with one column.\n+    Indices with no matches will not appear in the result.\n+\n+    >>> s = Series([\"a1a2\", \"b1\", \"c1\"], [\"A\", \"B\", \"C\"])\n+    >>> s.str.extractall(\"[ab](?P<digit>\\d)\")\n+            digit\n+      match      \n+    A 0         1\n+      1         2\n+    B 0         1\n+    \n+    A pattern with two groups will return a DataFrame with two columns.\n+    \n+    >>> s.str.extractall(\"(?P<letter>[ab])(?P<digit>\\d)\")\n+            letter digit\n+      match             \n+    A 0          a     1\n+      1          a     2\n+    B 0          b     1\n+    \n+    Optional groups that do not match are NaN in the result.\n+    \n+    >>> s.str.extractall(\"(?P<letter>[ab])?(?P<digit>\\d)\")\n+            letter digit\n+      match             \n+    A 0          a     1\n+      1          a     2\n+    B 0          b     1\n+    C 0        NaN     1","path":"pandas/core/strings.py","position":172,"original_position":172,"commit_id":"7c35402b7d373f038a105fe3177cf5e014a5015d","original_commit_id":"a2f359c733c0822ae685f062dc2620297de56e46","user":{"login":"jreback","id":953992,"avatar_url":"https://avatars.githubusercontent.com/u/953992?v=3","gravatar_id":"","url":"https://api.github.com/users/jreback","html_url":"https://github.com/jreback","followers_url":"https://api.github.com/users/jreback/followers","following_url":"https://api.github.com/users/jreback/following{/other_user}","gists_url":"https://api.github.com/users/jreback/gists{/gist_id}","starred_url":"https://api.github.com/users/jreback/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jreback/subscriptions","organizations_url":"https://api.github.com/users/jreback/orgs","repos_url":"https://api.github.com/users/jreback/repos","events_url":"https://api.github.com/users/jreback/events{/privacy}","received_events_url":"https://api.github.com/users/jreback/received_events","type":"User","site_admin":false},"body":"show an example w/o named groups as well.","created_at":"2015-12-02T15:32:29Z","updated_at":"2015-12-02T15:32:29Z","html_url":"https://github.com/pydata/pandas/pull/11386#discussion_r46428275","pull_request_url":"https://api.github.com/repos/pydata/pandas/pulls/11386","_links":{"self":{"href":"https://api.github.com/repos/pydata/pandas/pulls/comments/46428275"},"html":{"href":"https://github.com/pydata/pandas/pull/11386#discussion_r46428275"},"pull_request":{"href":"https://api.github.com/repos/pydata/pandas/pulls/11386"}}},"pull_request":{"url":"https://api.github.com/repos/pydata/pandas/pulls/11386","id":48226562,"html_url":"https://github.com/pydata/pandas/pull/11386","diff_url":"https://github.com/pydata/pandas/pull/11386.diff","patch_url":"https://github.com/pydata/pandas/pull/11386.patch","issue_url":"https://api.github.com/repos/pydata/pandas/issues/11386","number":11386,"state":"open","locked":false,"title":"ENH: str.extractall for several matches","user":{"login":"tdhock","id":932850,"avatar_url":"https://avatars.githubusercontent.com/u/932850?v=3","gravatar_id":"","url":"https://api.github.com/users/tdhock","html_url":"https://github.com/tdhock","followers_url":"https://api.github.com/users/tdhock/followers","following_url":"https://api.github.com/users/tdhock/following{/other_user}","gists_url":"https://api.github.com/users/tdhock/gists{/gist_id}","starred_url":"https://api.github.com/users/tdhock/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tdhock/subscriptions","organizations_url":"https://api.github.com/users/tdhock/orgs","repos_url":"https://api.github.com/users/tdhock/repos","events_url":"https://api.github.com/users/tdhock/events{/privacy}","received_events_url":"https://api.github.com/users/tdhock/received_events","type":"User","site_admin":false},"body":"For a series `S`, the excellent `S.str.extract` method returns the first match in each subject of the series:\r\n```python\r\n>>> import re\r\n>>> import pandas as pd\r\n>>> import numpy as np\r\n>>> data = {\r\n...     'Dave': 'dave@google.com',\r\n...     'multiple': 'rob@gmail.com some text steve@gmail.com',\r\n...     'none': np.nan,\r\n...     }\r\n>>> pattern = r'''\r\n... (?P<user>[a-z]+)\r\n... @\r\n... (?P<domain>[a-z]+)\r\n... \\.\r\n... (?P<tld>[a-z]{2,4})\r\n... '''\r\n>>> S = pd.Series(data)\r\n>>> S.str.extract(pattern, re.VERBOSE)\r\n          user  domain  tld\r\nDave      dave  google  com\r\nmultiple   rob   gmail  com\r\nnone       NaN     NaN  NaN\r\n>>> \r\n```\r\nThat's great, but sometimes we want to extract all matches in each element of the series. You can do that with `S.str.findall` but its result does not include the names specified in the capturing groups of the regular expression:\r\n```python\r\n>>> S.str.findall(pattern, re.VERBOSE)\r\nDave                           [(dave, google, com)]\r\nmultiple    [(rob, gmail, com), (steve, gmail, com)]\r\nnone                                             NaN\r\ndtype: object\r\n>>> \r\n```\r\nI propose the `S.str.extractall` method which returns a `Series` the same length as the subject `S`. Each element of the series is a `DataFrame` with a row for each match and a column for each group:\r\n```python\r\n>>> result = S.str.extractall(pattern, re.VERBOSE)\r\n>>> result[0]\r\n   user  domain  tld\r\n0  dave  google  com\r\n>>> result[1]\r\n    user domain  tld\r\n0    rob  gmail  com\r\n1  steve  gmail  com\r\n>>> result[2]\r\nEmpty DataFrame\r\nColumns: [user, domain, tld]\r\nIndex: []\r\n>>> \r\n```\r\nBefore I write any more testing code, can we start a discussion about whether or not this is an acceptable design choice, in relation to the other functionality of pandas? @sinhrks @jorisvandenbossche @jreback @mortada since you seem to be discussing extract in #10103\r\n\r\nAlso do you have any ideas about how to get the result (a Series of DataFrames) to print more nicely? With my current fork we have\r\n```python\r\n>>> result\r\nDave                   user  domain  tld\r\n0  dave  google  com\r\nmultiple        user domain  tld\r\n0    rob  gmail  com\r\n1  s...\r\nnone        Empty DataFrame\r\nColumns: [user, domain, tld]\r\nI...\r\ndtype: object\r\n>>> \r\n```\r\nIn R the equivalent functionality is provided by the https://github.com/tdhock/namedCapture package (str_match_all_named returns a list of data.frames), and the resulting printout is readable because of the way that R prints lists:\r\n```r\r\n> library(namedCapture)\r\n> S <- c(\r\n+   Dave='dave@google.com',\r\n+   multiple='rob@gmail.com some text steve@gmail.com',\r\n+   none=NA)\r\n> pattern <- paste0(\r\n+   \"(?P<user>[a-z]+)\",\r\n+   \"@\",\r\n+   \"(?P<domain>[a-z]+)\",\r\n+   \"[.]\",\r\n+   \"(?P<tld>[a-z]{2,4})\")\r\n> str_match_all_named(S, pattern)\r\n$Dave\r\n     user   domain   tld  \r\n[1,] \"dave\" \"google\" \"com\"\r\n\r\n$multiple\r\n     user    domain  tld  \r\n[1,] \"rob\"   \"gmail\" \"com\"\r\n[2,] \"steve\" \"gmail\" \"com\"\r\n\r\n$none\r\n<0 x 0 matrix>\r\n\r\n> \r\n```","created_at":"2015-10-20T16:42:01Z","updated_at":"2015-12-02T15:32:29Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7f49769eec84291625c3e637ab0ef515fb19930d","assignee":null,"milestone":{"url":"https://api.github.com/repos/pydata/pandas/milestones/36","html_url":"https://github.com/pydata/pandas/milestones/0.18.0","labels_url":"https://api.github.com/repos/pydata/pandas/milestones/36/labels","id":1214851,"number":36,"title":"0.18.0","description":"","creator":{"login":"jreback","id":953992,"avatar_url":"https://avatars.githubusercontent.com/u/953992?v=3","gravatar_id":"","url":"https://api.github.com/users/jreback","html_url":"https://github.com/jreback","followers_url":"https://api.github.com/users/jreback/followers","following_url":"https://api.github.com/users/jreback/following{/other_user}","gists_url":"https://api.github.com/users/jreback/gists{/gist_id}","starred_url":"https://api.github.com/users/jreback/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jreback/subscriptions","organizations_url":"https://api.github.com/users/jreback/orgs","repos_url":"https://api.github.com/users/jreback/repos","events_url":"https://api.github.com/users/jreback/events{/privacy}","received_events_url":"https://api.github.com/users/jreback/received_events","type":"User","site_admin":false},"open_issues":37,"closed_issues":22,"state":"open","created_at":"2015-07-19T20:18:45Z","updated_at":"2015-12-02T15:05:09Z","due_on":"2016-01-31T05:00:00Z","closed_at":null},"commits_url":"https://api.github.com/repos/pydata/pandas/pulls/11386/commits","review_comments_url":"https://api.github.com/repos/pydata/pandas/pulls/11386/comments","review_comment_url":"https://api.github.com/repos/pydata/pandas/pulls/comments{/number}","comments_url":"https://api.github.com/repos/pydata/pandas/issues/11386/comments","statuses_url":"https://api.github.com/repos/pydata/pandas/statuses/7c35402b7d373f038a105fe3177cf5e014a5015d","head":{"label":"tdhock:extractall","ref":"extractall","sha":"7c35402b7d373f038a105fe3177cf5e014a5015d","user":{"login":"tdhock","id":932850,"avatar_url":"https://avatars.githubusercontent.com/u/932850?v=3","gravatar_id":"","url":"https://api.github.com/users/tdhock","html_url":"https://github.com/tdhock","followers_url":"https://api.github.com/users/tdhock/followers","following_url":"https://api.github.com/users/tdhock/following{/other_user}","gists_url":"https://api.github.com/users/tdhock/gists{/gist_id}","starred_url":"https://api.github.com/users/tdhock/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tdhock/subscriptions","organizations_url":"https://api.github.com/users/tdhock/orgs","repos_url":"https://api.github.com/users/tdhock/repos","events_url":"https://api.github.com/users/tdhock/events{/privacy}","received_events_url":"https://api.github.com/users/tdhock/received_events","type":"User","site_admin":false},"repo":{"id":44604581,"name":"pandas","full_name":"tdhock/pandas","owner":{"login":"tdhock","id":932850,"avatar_url":"https://avatars.githubusercontent.com/u/932850?v=3","gravatar_id":"","url":"https://api.github.com/users/tdhock","html_url":"https://github.com/tdhock","followers_url":"https://api.github.com/users/tdhock/followers","following_url":"https://api.github.com/users/tdhock/following{/other_user}","gists_url":"https://api.github.com/users/tdhock/gists{/gist_id}","starred_url":"https://api.github.com/users/tdhock/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tdhock/subscriptions","organizations_url":"https://api.github.com/users/tdhock/orgs","repos_url":"https://api.github.com/users/tdhock/repos","events_url":"https://api.github.com/users/tdhock/events{/privacy}","received_events_url":"https://api.github.com/users/tdhock/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/tdhock/pandas","description":"Flexible and powerful data analysis / manipulation library for Python, providing labeled data structures similar to R data.frame objects, statistical functions, and much more","fork":true,"url":"https://api.github.com/repos/tdhock/pandas","forks_url":"https://api.github.com/repos/tdhock/pandas/forks","keys_url":"https://api.github.com/repos/tdhock/pandas/keys{/key_id}","collaborators_url":"https://api.github.com/repos/tdhock/pandas/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/tdhock/pandas/teams","hooks_url":"https://api.github.com/repos/tdhock/pandas/hooks","issue_events_url":"https://api.github.com/repos/tdhock/pandas/issues/events{/number}","events_url":"https://api.github.com/repos/tdhock/pandas/events","assignees_url":"https://api.github.com/repos/tdhock/pandas/assignees{/user}","branches_url":"https://api.github.com/repos/tdhock/pandas/branches{/branch}","tags_url":"https://api.github.com/repos/tdhock/pandas/tags","blobs_url":"https://api.github.com/repos/tdhock/pandas/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/tdhock/pandas/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/tdhock/pandas/git/refs{/sha}","trees_url":"https://api.github.com/repos/tdhock/pandas/git/trees{/sha}","statuses_url":"https://api.github.com/repos/tdhock/pandas/statuses/{sha}","languages_url":"https://api.github.com/repos/tdhock/pandas/languages","stargazers_url":"https://api.github.com/repos/tdhock/pandas/stargazers","contributors_url":"https://api.github.com/repos/tdhock/pandas/contributors","subscribers_url":"https://api.github.com/repos/tdhock/pandas/subscribers","subscription_url":"https://api.github.com/repos/tdhock/pandas/subscription","commits_url":"https://api.github.com/repos/tdhock/pandas/commits{/sha}","git_commits_url":"https://api.github.com/repos/tdhock/pandas/git/commits{/sha}","comments_url":"https://api.github.com/repos/tdhock/pandas/comments{/number}","issue_comment_url":"https://api.github.com/repos/tdhock/pandas/issues/comments{/number}","contents_url":"https://api.github.com/repos/tdhock/pandas/contents/{+path}","compare_url":"https://api.github.com/repos/tdhock/pandas/compare/{base}...{head}","merges_url":"https://api.github.com/repos/tdhock/pandas/merges","archive_url":"https://api.github.com/repos/tdhock/pandas/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/tdhock/pandas/downloads","issues_url":"https://api.github.com/repos/tdhock/pandas/issues{/number}","pulls_url":"https://api.github.com/repos/tdhock/pandas/pulls{/number}","milestones_url":"https://api.github.com/repos/tdhock/pandas/milestones{/number}","notifications_url":"https://api.github.com/repos/tdhock/pandas/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/tdhock/pandas/labels{/name}","releases_url":"https://api.github.com/repos/tdhock/pandas/releases{/id}","created_at":"2015-10-20T12:25:51Z","updated_at":"2015-10-20T12:26:05Z","pushed_at":"2015-12-02T15:27:10Z","git_url":"git://github.com/tdhock/pandas.git","ssh_url":"git@github.com:tdhock/pandas.git","clone_url":"https://github.com/tdhock/pandas.git","svn_url":"https://github.com/tdhock/pandas","homepage":"http://pandas.pydata.org","size":68539,"stargazers_count":0,"watchers_count":0,"language":"Python","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"pydata:master","ref":"master","sha":"8ec8487a4a8362ac94630388f217f273b3ad1525","user":{"login":"pydata","id":1284191,"avatar_url":"https://avatars.githubusercontent.com/u/1284191?v=3","gravatar_id":"","url":"https://api.github.com/users/pydata","html_url":"https://github.com/pydata","followers_url":"https://api.github.com/users/pydata/followers","following_url":"https://api.github.com/users/pydata/following{/other_user}","gists_url":"https://api.github.com/users/pydata/gists{/gist_id}","starred_url":"https://api.github.com/users/pydata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pydata/subscriptions","organizations_url":"https://api.github.com/users/pydata/orgs","repos_url":"https://api.github.com/users/pydata/repos","events_url":"https://api.github.com/users/pydata/events{/privacy}","received_events_url":"https://api.github.com/users/pydata/received_events","type":"Organization","site_admin":false},"repo":{"id":858127,"name":"pandas","full_name":"pydata/pandas","owner":{"login":"pydata","id":1284191,"avatar_url":"https://avatars.githubusercontent.com/u/1284191?v=3","gravatar_id":"","url":"https://api.github.com/users/pydata","html_url":"https://github.com/pydata","followers_url":"https://api.github.com/users/pydata/followers","following_url":"https://api.github.com/users/pydata/following{/other_user}","gists_url":"https://api.github.com/users/pydata/gists{/gist_id}","starred_url":"https://api.github.com/users/pydata/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/pydata/subscriptions","organizations_url":"https://api.github.com/users/pydata/orgs","repos_url":"https://api.github.com/users/pydata/repos","events_url":"https://api.github.com/users/pydata/events{/privacy}","received_events_url":"https://api.github.com/users/pydata/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/pydata/pandas","description":"Flexible and powerful data analysis / manipulation library for Python, providing labeled data structures similar to R data.frame objects, statistical functions, and much more","fork":false,"url":"https://api.github.com/repos/pydata/pandas","forks_url":"https://api.github.com/repos/pydata/pandas/forks","keys_url":"https://api.github.com/repos/pydata/pandas/keys{/key_id}","collaborators_url":"https://api.github.com/repos/pydata/pandas/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/pydata/pandas/teams","hooks_url":"https://api.github.com/repos/pydata/pandas/hooks","issue_events_url":"https://api.github.com/repos/pydata/pandas/issues/events{/number}","events_url":"https://api.github.com/repos/pydata/pandas/events","assignees_url":"https://api.github.com/repos/pydata/pandas/assignees{/user}","branches_url":"https://api.github.com/repos/pydata/pandas/branches{/branch}","tags_url":"https://api.github.com/repos/pydata/pandas/tags","blobs_url":"https://api.github.com/repos/pydata/pandas/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/pydata/pandas/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/pydata/pandas/git/refs{/sha}","trees_url":"https://api.github.com/repos/pydata/pandas/git/trees{/sha}","statuses_url":"https://api.github.com/repos/pydata/pandas/statuses/{sha}","languages_url":"https://api.github.com/repos/pydata/pandas/languages","stargazers_url":"https://api.github.com/repos/pydata/pandas/stargazers","contributors_url":"https://api.github.com/repos/pydata/pandas/contributors","subscribers_url":"https://api.github.com/repos/pydata/pandas/subscribers","subscription_url":"https://api.github.com/repos/pydata/pandas/subscription","commits_url":"https://api.github.com/repos/pydata/pandas/commits{/sha}","git_commits_url":"https://api.github.com/repos/pydata/pandas/git/commits{/sha}","comments_url":"https://api.github.com/repos/pydata/pandas/comments{/number}","issue_comment_url":"https://api.github.com/repos/pydata/pandas/issues/comments{/number}","contents_url":"https://api.github.com/repos/pydata/pandas/contents/{+path}","compare_url":"https://api.github.com/repos/pydata/pandas/compare/{base}...{head}","merges_url":"https://api.github.com/repos/pydata/pandas/merges","archive_url":"https://api.github.com/repos/pydata/pandas/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/pydata/pandas/downloads","issues_url":"https://api.github.com/repos/pydata/pandas/issues{/number}","pulls_url":"https://api.github.com/repos/pydata/pandas/pulls{/number}","milestones_url":"https://api.github.com/repos/pydata/pandas/milestones{/number}","notifications_url":"https://api.github.com/repos/pydata/pandas/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/pydata/pandas/labels{/name}","releases_url":"https://api.github.com/repos/pydata/pandas/releases{/id}","created_at":"2010-08-24T01:37:33Z","updated_at":"2015-12-02T14:18:58Z","pushed_at":"2015-12-02T15:27:12Z","git_url":"git://github.com/pydata/pandas.git","ssh_url":"git@github.com:pydata/pandas.git","clone_url":"https://github.com/pydata/pandas.git","svn_url":"https://github.com/pydata/pandas","homepage":"http://pandas.pydata.org","size":128816,"stargazers_count":5375,"watchers_count":5375,"language":"Python","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":2120,"mirror_url":null,"open_issues_count":1516,"forks":2120,"open_issues":1516,"watchers":5375,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/pydata/pandas/pulls/11386"},"html":{"href":"https://github.com/pydata/pandas/pull/11386"},"issue":{"href":"https://api.github.com/repos/pydata/pandas/issues/11386"},"comments":{"href":"https://api.github.com/repos/pydata/pandas/issues/11386/comments"},"review_comments":{"href":"https://api.github.com/repos/pydata/pandas/pulls/11386/comments"},"review_comment":{"href":"https://api.github.com/repos/pydata/pandas/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/pydata/pandas/pulls/11386/commits"},"statuses":{"href":"https://api.github.com/repos/pydata/pandas/statuses/7c35402b7d373f038a105fe3177cf5e014a5015d"}}}}