{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/24253601","id":24253601,"diff_hunk":"@@ -1469,6 +1470,44 @@ def applySchema(self, rdd, schema):\n         df = self._ssql_ctx.applySchemaToPythonRDD(jrdd.rdd(), schema.json())\n         return DataFrame(df, self)\n \n+    def applyNames(self, nameString, plainRdd):\n+        \"\"\"\n+        Builds a DataFrame from an RDD based on column names.\n+\n+        Assumes RDD contains iterables of equal length.\n+        >>> unparsedStrings = sc.parallelize([\"1, A1, true\", \"2, B2, false\", \"3, C3, true\", \"4, D4, false\"])\n+        >>> input = unparsedStrings.map(lambda x: x.split(\",\")).map(lambda x: [int(x[0]), x[1], bool(x[2])])\n+        >>> df1 = sqlCtx.applyNames(\"a b c\", input)\n+        >>> df1.registerTempTable(\"df1\")\n+        >>> sqlCtx.sql(\"select a from df1\").collect()\n+        [Row(a=1), Row(a=2), Row(a=3), Row(a=4)]\n+        >>> input2 = unparsedStrings.map(lambda x: x.split(\",\")).map(lambda x: [int(x[0]), x[1], bool(x[2]), {\"k\":int(x[0]), \"v\":2*int(x[0])}, x])\n+        >>> df2 = sqlCtx.applyNames(\"a b c d e\", input2)\n+        >>> df2.registerTempTable(\"df2\")\n+        >>> sqlCtx.sql(\"select d['k']+d['v'] from df2\").collect()\n+        [Row(c0=3), Row(c0=6), Row(c0=9), Row(c0=12)]\n+        >>> sqlCtx.sql(\"select b, e[1] from df2\").collect()\n+        [Row(b=u' A1', c1=u' A1'), Row(b=u' B2', c1=u' B2'), Row(b=u' C3', c1=u' C3'), Row(b=u' D4', c1=u' D4')]\n+        \"\"\"\n+        fieldNames = [f for f in re.split(\"( |\\\\\\\".*?\\\\\\\"|'.*?')\", nameString) if f.strip()]\n+        reservedWords = set(map(string.lower,[\"ABS\",\"ALL\",\"AND\", \"APPROXIMATE\", \"AS\", \"ASC\", \"AVG\", \"BETWEEN\", \"BY\", \\","path":"python/pyspark/sql.py","position":32,"original_position":32,"commit_id":"bc57c0e31bd048db4d489ca3ca965fb0ade5c834","original_commit_id":"bc57c0e31bd048db4d489ca3ca965fb0ade5c834","user":{"login":"dwmclary","id":622500,"avatar_url":"https://avatars.githubusercontent.com/u/622500?v=3","gravatar_id":"","url":"https://api.github.com/users/dwmclary","html_url":"https://github.com/dwmclary","followers_url":"https://api.github.com/users/dwmclary/followers","following_url":"https://api.github.com/users/dwmclary/following{/other_user}","gists_url":"https://api.github.com/users/dwmclary/gists{/gist_id}","starred_url":"https://api.github.com/users/dwmclary/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwmclary/subscriptions","organizations_url":"https://api.github.com/users/dwmclary/orgs","repos_url":"https://api.github.com/users/dwmclary/repos","events_url":"https://api.github.com/users/dwmclary/events{/privacy}","received_events_url":"https://api.github.com/users/dwmclary/received_events","type":"User","site_admin":false},"body":"Seems like a reasonable request to me.  I couldn't decide if it was better\nto have to pickle and ship a list of words or just to have it instantiated\nin both places.\n\nOn Fri, Feb 6, 2015 at 7:31 AM, Josh Rosen <notifications@github.com> wrote:\n\n> In python/pyspark/sql.py\n> <https://github.com/apache/spark/pull/4421#discussion_r24247234>:\n>\n> > +        >>> unparsedStrings = sc.parallelize([\"1, A1, true\", \"2, B2, false\", \"3, C3, true\", \"4, D4, false\"])\n> > +        >>> input = unparsedStrings.map(lambda x: x.split(\",\")).map(lambda x: [int(x[0]), x[1], bool(x[2])])\n> > +        >>> df1 = sqlCtx.applyNames(\"a b c\", input)\n> > +        >>> df1.registerTempTable(\"df1\")\n> > +        >>> sqlCtx.sql(\"select a from df1\").collect()\n> > +        [Row(a=1), Row(a=2), Row(a=3), Row(a=4)]\n> > +        >>> input2 = unparsedStrings.map(lambda x: x.split(\",\")).map(lambda x: [int(x[0]), x[1], bool(x[2]), {\"k\":int(x[0]), \"v\":2*int(x[0])}, x])\n> > +        >>> df2 = sqlCtx.applyNames(\"a b c d e\", input2)\n> > +        >>> df2.registerTempTable(\"df2\")\n> > +        >>> sqlCtx.sql(\"select d['k']+d['v'] from df2\").collect()\n> > +        [Row(c0=3), Row(c0=6), Row(c0=9), Row(c0=12)]\n> > +        >>> sqlCtx.sql(\"select b, e[1] from df2\").collect()\n> > +        [Row(b=u' A1', c1=u' A1'), Row(b=u' B2', c1=u' B2'), Row(b=u' C3', c1=u' C3'), Row(b=u' D4', c1=u' D4')]\n> > +        \"\"\"\n> > +        fieldNames = [f for f in re.split(\"( |\\\\\\\".*?\\\\\\\"|'.*?')\", nameString) if f.strip()]\n> > +        reservedWords = set(map(string.lower,[\"ABS\",\"ALL\",\"AND\", \"APPROXIMATE\", \"AS\", \"ASC\", \"AVG\", \"BETWEEN\", \"BY\", \\\n>\n> I can't really speak to this patch in general, since I don't know much\n> about this part of Spark SQL, but to avoid duplication it probably makes\n> sense to keep the list of reserved words in the JVM and fetch it into\n> Python from there.\n>\n> —\n> Reply to this email directly or view it on GitHub\n> <https://github.com/apache/spark/pull/4421/files#r24247234>.\n>\n","created_at":"2015-02-06T16:50:20Z","updated_at":"2015-02-06T16:50:20Z","html_url":"https://github.com/apache/spark/pull/4421#discussion_r24253601","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/4421","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/24253601"},"html":{"href":"https://github.com/apache/spark/pull/4421#discussion_r24253601"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/4421"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/4421","id":28791521,"html_url":"https://github.com/apache/spark/pull/4421","diff_url":"https://github.com/apache/spark/pull/4421.diff","patch_url":"https://github.com/apache/spark/pull/4421.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/4421","number":4421,"state":"open","locked":false,"title":"Spark-2789: Apply names to RDD to create DataFrame","user":{"login":"dwmclary","id":622500,"avatar_url":"https://avatars.githubusercontent.com/u/622500?v=3","gravatar_id":"","url":"https://api.github.com/users/dwmclary","html_url":"https://github.com/dwmclary","followers_url":"https://api.github.com/users/dwmclary/followers","following_url":"https://api.github.com/users/dwmclary/following{/other_user}","gists_url":"https://api.github.com/users/dwmclary/gists{/gist_id}","starred_url":"https://api.github.com/users/dwmclary/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwmclary/subscriptions","organizations_url":"https://api.github.com/users/dwmclary/orgs","repos_url":"https://api.github.com/users/dwmclary/repos","events_url":"https://api.github.com/users/dwmclary/events{/privacy}","received_events_url":"https://api.github.com/users/dwmclary/received_events","type":"User","site_admin":false},"body":"This seemed like a reasonably useful function to add to SparkSQL.  However, unlike the [JIRA](https://issues.apache.org/jira/browse/SPARK-2789), this implementation does not parse type characters (e.g. brackets and braces).  This method creates a DataFrame with column names that map to the existing types in the RDD.  In general, this seems far more useful, as users likely wish to quickly apply names to existing collections.\r\n\r\n","created_at":"2015-02-06T08:12:26Z","updated_at":"2015-02-06T16:50:20Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4ccac61941728ec6fd39a2b00c8fa6efef733301","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/4421/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/4421/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments/{number}","comments_url":"https://api.github.com/repos/apache/spark/issues/4421/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/bc57c0e31bd048db4d489ca3ca965fb0ade5c834","head":{"label":"dwmclary:SPARK-2789","ref":"SPARK-2789","sha":"bc57c0e31bd048db4d489ca3ca965fb0ade5c834","user":{"login":"dwmclary","id":622500,"avatar_url":"https://avatars.githubusercontent.com/u/622500?v=3","gravatar_id":"","url":"https://api.github.com/users/dwmclary","html_url":"https://github.com/dwmclary","followers_url":"https://api.github.com/users/dwmclary/followers","following_url":"https://api.github.com/users/dwmclary/following{/other_user}","gists_url":"https://api.github.com/users/dwmclary/gists{/gist_id}","starred_url":"https://api.github.com/users/dwmclary/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwmclary/subscriptions","organizations_url":"https://api.github.com/users/dwmclary/orgs","repos_url":"https://api.github.com/users/dwmclary/repos","events_url":"https://api.github.com/users/dwmclary/events{/privacy}","received_events_url":"https://api.github.com/users/dwmclary/received_events","type":"User","site_admin":false},"repo":{"id":17487301,"name":"spark","full_name":"dwmclary/spark","owner":{"login":"dwmclary","id":622500,"avatar_url":"https://avatars.githubusercontent.com/u/622500?v=3","gravatar_id":"","url":"https://api.github.com/users/dwmclary","html_url":"https://github.com/dwmclary","followers_url":"https://api.github.com/users/dwmclary/followers","following_url":"https://api.github.com/users/dwmclary/following{/other_user}","gists_url":"https://api.github.com/users/dwmclary/gists{/gist_id}","starred_url":"https://api.github.com/users/dwmclary/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/dwmclary/subscriptions","organizations_url":"https://api.github.com/users/dwmclary/orgs","repos_url":"https://api.github.com/users/dwmclary/repos","events_url":"https://api.github.com/users/dwmclary/events{/privacy}","received_events_url":"https://api.github.com/users/dwmclary/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/dwmclary/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/dwmclary/spark","forks_url":"https://api.github.com/repos/dwmclary/spark/forks","keys_url":"https://api.github.com/repos/dwmclary/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/dwmclary/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/dwmclary/spark/teams","hooks_url":"https://api.github.com/repos/dwmclary/spark/hooks","issue_events_url":"https://api.github.com/repos/dwmclary/spark/issues/events{/number}","events_url":"https://api.github.com/repos/dwmclary/spark/events","assignees_url":"https://api.github.com/repos/dwmclary/spark/assignees{/user}","branches_url":"https://api.github.com/repos/dwmclary/spark/branches{/branch}","tags_url":"https://api.github.com/repos/dwmclary/spark/tags","blobs_url":"https://api.github.com/repos/dwmclary/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/dwmclary/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/dwmclary/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/dwmclary/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/dwmclary/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/dwmclary/spark/languages","stargazers_url":"https://api.github.com/repos/dwmclary/spark/stargazers","contributors_url":"https://api.github.com/repos/dwmclary/spark/contributors","subscribers_url":"https://api.github.com/repos/dwmclary/spark/subscribers","subscription_url":"https://api.github.com/repos/dwmclary/spark/subscription","commits_url":"https://api.github.com/repos/dwmclary/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/dwmclary/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/dwmclary/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/dwmclary/spark/issues/comments/{number}","contents_url":"https://api.github.com/repos/dwmclary/spark/contents/{+path}","compare_url":"https://api.github.com/repos/dwmclary/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/dwmclary/spark/merges","archive_url":"https://api.github.com/repos/dwmclary/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/dwmclary/spark/downloads","issues_url":"https://api.github.com/repos/dwmclary/spark/issues{/number}","pulls_url":"https://api.github.com/repos/dwmclary/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/dwmclary/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/dwmclary/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/dwmclary/spark/labels{/name}","releases_url":"https://api.github.com/repos/dwmclary/spark/releases{/id}","created_at":"2014-03-06T18:05:10Z","updated_at":"2014-07-22T16:38:14Z","pushed_at":"2015-02-06T08:13:41Z","git_url":"git://github.com/dwmclary/spark.git","ssh_url":"git@github.com:dwmclary/spark.git","clone_url":"https://github.com/dwmclary/spark.git","svn_url":"https://github.com/dwmclary/spark","homepage":null,"size":86328,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"dc0c4490a12ecedd8ca5a1bb256c7ccbdf0be04f","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments/{number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-02-06T15:15:12Z","pushed_at":"2015-02-06T12:22:42Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1233200,"stargazers_count":2801,"watchers_count":2801,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":2451,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":304,"forks":2451,"open_issues":304,"watchers":2801,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/4421"},"html":{"href":"https://github.com/apache/spark/pull/4421"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/4421"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/4421/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/4421/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/4421/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/bc57c0e31bd048db4d489ca3ca965fb0ade5c834"}}}}