{"action":"created","comment":{"url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/comments/26367008","id":26367008,"diff_hunk":"@@ -1,5 +1,179 @@\n # SQLcontext.R: SQLContext-driven functions\n \n+#' infer the SQL type\n+infer_type <- function(x) {\n+  if (is.null(x)) {\n+    stop(\"can not infer type from NULL\")\n+  }\n+\n+  # class of POSIXlt is c(\"POSIXlt\" \"POSIXt\")\n+  type <- switch(class(x)[[1]],\n+                 integer = \"integer\",\n+                 character = \"string\",\n+                 logical = \"boolean\",\n+                 double = \"double\",\n+                 numeric = \"double\",\n+                 raw = \"binary\",\n+                 list = \"array\",\n+                 environment = \"map\",\n+                 Date = \"date\",\n+                 POSIXlt = \"timestamp\",\n+                 POSIXct = \"timestamp\",\n+                 stop(paste(\"Unsupported type for DataFrame:\", class(x))))\n+\n+  if (type == \"map\") {\n+    stopifnot(length(x) > 0)\n+    key <- ls(x)[[1]]\n+    list(type = \"map\",\n+         keyType = \"string\",\n+         valueType = infer_type(get(key, x)),\n+         valueContainsNull = TRUE)\n+  } else if (type == \"array\") {\n+    stopifnot(length(x) > 0)\n+    names <- names(x)\n+    if (is.null(names)) {\n+      list(type = \"array\", elementType = infer_type(x[[1]]), containsNull = TRUE)\n+    } else {\n+      # StructType\n+      types <- lapply(x, infer_type)\n+      fields <- lapply(1:length(x), function(i) {\n+        list(name = names[[i]], type = types[[i]], nullable = TRUE)\n+      })\n+      list(type = \"struct\", fields = fields)\n+    }\n+  } else if (length(x) > 1) {\n+    list(type = \"array\", elementType = type, containsNull = TRUE)\n+  } else {\n+    type\n+  }\n+}\n+\n+#' dump the schema into JSON string\n+tojson <- function(x) {\n+  if (is.list(x)) {\n+    names <- names(x)\n+    if (!is.null(names)) {\n+      items <- lapply(names, function(n) {\n+        safe_n <- gsub('\"', '\\\\\"', n)\n+        paste(tojson(safe_n), ':', tojson(x[[n]]), sep = '')\n+      })\n+      d <- paste(items, collapse = ', ')\n+      paste('{', d, '}', sep = '')\n+    } else {\n+      l <- paste(lapply(x, tojson), collapse = ', ')\n+      paste('[', l, ']', sep = '')\n+    }\n+  } else if (is.character(x)) {\n+    paste('\"', x, '\"', sep = '')\n+  } else if (is.logical(x)) {\n+    if (x) \"true\" else \"false\"\n+  }\n+}\n+\n+#' Create a DataFrame from an RDD\n+#'\n+#' Converts an RDD to a DataFrame by infer the types.\n+#'\n+#' @param sqlCtx A SQLContext\n+#' @param data An RDD or list or data.frame\n+#' @param schema a list of column names or named list (StructType), optional\n+#' @return an DataFrame\n+#' @export\n+#' @examples\n+#'\\dontrun{\n+#' sc <- sparkR.init()\n+#' sqlCtx <- sparkRSQL.init(sc)\n+#' rdd <- lapply(parallelize(sc, 1:10), function(x) list(a=x, b=as.character(x)))\n+#' df <- createDataFrame(sqlCtx, rdd)\n+#' }\n+\n+# TODO(davies): support sampling and infer type from NA\n+createDataFrame <- function(sqlCtx, data, schema = NULL, samplingRatio = 1.0) {\n+  if (is.data.frame(data)) {\n+      schema <- names(data)","path":"pkg/R/SQLContext.R","position":93,"original_position":93,"commit_id":"e87bb9876804af46c64137db934cfb89f4a14936","original_commit_id":"e87bb9876804af46c64137db934cfb89f4a14936","user":{"login":"davies","id":40902,"avatar_url":"https://avatars.githubusercontent.com/u/40902?v=3","gravatar_id":"","url":"https://api.github.com/users/davies","html_url":"https://github.com/davies","followers_url":"https://api.github.com/users/davies/followers","following_url":"https://api.github.com/users/davies/following{/other_user}","gists_url":"https://api.github.com/users/davies/gists{/gist_id}","starred_url":"https://api.github.com/users/davies/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/davies/subscriptions","organizations_url":"https://api.github.com/users/davies/orgs","repos_url":"https://api.github.com/users/davies/repos","events_url":"https://api.github.com/users/davies/events{/privacy}","received_events_url":"https://api.github.com/users/davies/received_events","type":"User","site_admin":false},"body":"It will be used later. We did not put the names of column into RDD.","created_at":"2015-03-13T06:19:31Z","updated_at":"2015-03-13T06:19:31Z","html_url":"https://github.com/amplab-extras/SparkR-pkg/pull/212#discussion_r26367008","pull_request_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/212","_links":{"self":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/comments/26367008"},"html":{"href":"https://github.com/amplab-extras/SparkR-pkg/pull/212#discussion_r26367008"},"pull_request":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/212"}}},"pull_request":{"url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/212","id":30908430,"html_url":"https://github.com/amplab-extras/SparkR-pkg/pull/212","diff_url":"https://github.com/amplab-extras/SparkR-pkg/pull/212.diff","patch_url":"https://github.com/amplab-extras/SparkR-pkg/pull/212.patch","issue_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues/212","number":212,"state":"open","locked":false,"title":"[SPARKR-192]  createDataFrame from rdd","user":{"login":"davies","id":40902,"avatar_url":"https://avatars.githubusercontent.com/u/40902?v=3","gravatar_id":"","url":"https://api.github.com/users/davies","html_url":"https://github.com/davies","followers_url":"https://api.github.com/users/davies/followers","following_url":"https://api.github.com/users/davies/following{/other_user}","gists_url":"https://api.github.com/users/davies/gists{/gist_id}","starred_url":"https://api.github.com/users/davies/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/davies/subscriptions","organizations_url":"https://api.github.com/users/davies/orgs","repos_url":"https://api.github.com/users/davies/repos","events_url":"https://api.github.com/users/davies/events{/privacy}","received_events_url":"https://api.github.com/users/davies/received_events","type":"User","site_admin":false},"body":"This PR introduce createDataFrame() and toDF() to create DataFrames from existing RDDs.","created_at":"2015-03-11T01:19:59Z","updated_at":"2015-03-13T06:19:31Z","closed_at":null,"merged_at":null,"merge_commit_sha":"587ce201188712225664cfae9841d7705c62efe1","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/212/commits","review_comments_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/212/comments","review_comment_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/comments{/number}","comments_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues/212/comments","statuses_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/statuses/e87bb9876804af46c64137db934cfb89f4a14936","head":{"label":"davies:toDF","ref":"toDF","sha":"e87bb9876804af46c64137db934cfb89f4a14936","user":{"login":"davies","id":40902,"avatar_url":"https://avatars.githubusercontent.com/u/40902?v=3","gravatar_id":"","url":"https://api.github.com/users/davies","html_url":"https://github.com/davies","followers_url":"https://api.github.com/users/davies/followers","following_url":"https://api.github.com/users/davies/following{/other_user}","gists_url":"https://api.github.com/users/davies/gists{/gist_id}","starred_url":"https://api.github.com/users/davies/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/davies/subscriptions","organizations_url":"https://api.github.com/users/davies/orgs","repos_url":"https://api.github.com/users/davies/repos","events_url":"https://api.github.com/users/davies/events{/privacy}","received_events_url":"https://api.github.com/users/davies/received_events","type":"User","site_admin":false},"repo":{"id":31027661,"name":"SparkR-pkg","full_name":"davies/SparkR-pkg","owner":{"login":"davies","id":40902,"avatar_url":"https://avatars.githubusercontent.com/u/40902?v=3","gravatar_id":"","url":"https://api.github.com/users/davies","html_url":"https://github.com/davies","followers_url":"https://api.github.com/users/davies/followers","following_url":"https://api.github.com/users/davies/following{/other_user}","gists_url":"https://api.github.com/users/davies/gists{/gist_id}","starred_url":"https://api.github.com/users/davies/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/davies/subscriptions","organizations_url":"https://api.github.com/users/davies/orgs","repos_url":"https://api.github.com/users/davies/repos","events_url":"https://api.github.com/users/davies/events{/privacy}","received_events_url":"https://api.github.com/users/davies/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/davies/SparkR-pkg","description":"R frontend for Spark","fork":true,"url":"https://api.github.com/repos/davies/SparkR-pkg","forks_url":"https://api.github.com/repos/davies/SparkR-pkg/forks","keys_url":"https://api.github.com/repos/davies/SparkR-pkg/keys{/key_id}","collaborators_url":"https://api.github.com/repos/davies/SparkR-pkg/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/davies/SparkR-pkg/teams","hooks_url":"https://api.github.com/repos/davies/SparkR-pkg/hooks","issue_events_url":"https://api.github.com/repos/davies/SparkR-pkg/issues/events{/number}","events_url":"https://api.github.com/repos/davies/SparkR-pkg/events","assignees_url":"https://api.github.com/repos/davies/SparkR-pkg/assignees{/user}","branches_url":"https://api.github.com/repos/davies/SparkR-pkg/branches{/branch}","tags_url":"https://api.github.com/repos/davies/SparkR-pkg/tags","blobs_url":"https://api.github.com/repos/davies/SparkR-pkg/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/davies/SparkR-pkg/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/davies/SparkR-pkg/git/refs{/sha}","trees_url":"https://api.github.com/repos/davies/SparkR-pkg/git/trees{/sha}","statuses_url":"https://api.github.com/repos/davies/SparkR-pkg/statuses/{sha}","languages_url":"https://api.github.com/repos/davies/SparkR-pkg/languages","stargazers_url":"https://api.github.com/repos/davies/SparkR-pkg/stargazers","contributors_url":"https://api.github.com/repos/davies/SparkR-pkg/contributors","subscribers_url":"https://api.github.com/repos/davies/SparkR-pkg/subscribers","subscription_url":"https://api.github.com/repos/davies/SparkR-pkg/subscription","commits_url":"https://api.github.com/repos/davies/SparkR-pkg/commits{/sha}","git_commits_url":"https://api.github.com/repos/davies/SparkR-pkg/git/commits{/sha}","comments_url":"https://api.github.com/repos/davies/SparkR-pkg/comments{/number}","issue_comment_url":"https://api.github.com/repos/davies/SparkR-pkg/issues/comments{/number}","contents_url":"https://api.github.com/repos/davies/SparkR-pkg/contents/{+path}","compare_url":"https://api.github.com/repos/davies/SparkR-pkg/compare/{base}...{head}","merges_url":"https://api.github.com/repos/davies/SparkR-pkg/merges","archive_url":"https://api.github.com/repos/davies/SparkR-pkg/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/davies/SparkR-pkg/downloads","issues_url":"https://api.github.com/repos/davies/SparkR-pkg/issues{/number}","pulls_url":"https://api.github.com/repos/davies/SparkR-pkg/pulls{/number}","milestones_url":"https://api.github.com/repos/davies/SparkR-pkg/milestones{/number}","notifications_url":"https://api.github.com/repos/davies/SparkR-pkg/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/davies/SparkR-pkg/labels{/name}","releases_url":"https://api.github.com/repos/davies/SparkR-pkg/releases{/id}","created_at":"2015-02-19T17:37:52Z","updated_at":"2015-02-25T08:33:46Z","pushed_at":"2015-03-12T20:57:21Z","git_url":"git://github.com/davies/SparkR-pkg.git","ssh_url":"git@github.com:davies/SparkR-pkg.git","clone_url":"https://github.com/davies/SparkR-pkg.git","svn_url":"https://github.com/davies/SparkR-pkg","homepage":"http://amplab-extras.github.io/SparkR-pkg","size":4023,"stargazers_count":0,"watchers_count":0,"language":"R","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"amplab-extras:sparkr-sql","ref":"sparkr-sql","sha":"09ff163cd8cfa05f81fa38188ef1db5aea41acbb","user":{"login":"amplab-extras","id":6403540,"avatar_url":"https://avatars.githubusercontent.com/u/6403540?v=3","gravatar_id":"","url":"https://api.github.com/users/amplab-extras","html_url":"https://github.com/amplab-extras","followers_url":"https://api.github.com/users/amplab-extras/followers","following_url":"https://api.github.com/users/amplab-extras/following{/other_user}","gists_url":"https://api.github.com/users/amplab-extras/gists{/gist_id}","starred_url":"https://api.github.com/users/amplab-extras/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplab-extras/subscriptions","organizations_url":"https://api.github.com/users/amplab-extras/orgs","repos_url":"https://api.github.com/users/amplab-extras/repos","events_url":"https://api.github.com/users/amplab-extras/events{/privacy}","received_events_url":"https://api.github.com/users/amplab-extras/received_events","type":"Organization","site_admin":false},"repo":{"id":15365789,"name":"SparkR-pkg","full_name":"amplab-extras/SparkR-pkg","owner":{"login":"amplab-extras","id":6403540,"avatar_url":"https://avatars.githubusercontent.com/u/6403540?v=3","gravatar_id":"","url":"https://api.github.com/users/amplab-extras","html_url":"https://github.com/amplab-extras","followers_url":"https://api.github.com/users/amplab-extras/followers","following_url":"https://api.github.com/users/amplab-extras/following{/other_user}","gists_url":"https://api.github.com/users/amplab-extras/gists{/gist_id}","starred_url":"https://api.github.com/users/amplab-extras/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplab-extras/subscriptions","organizations_url":"https://api.github.com/users/amplab-extras/orgs","repos_url":"https://api.github.com/users/amplab-extras/repos","events_url":"https://api.github.com/users/amplab-extras/events{/privacy}","received_events_url":"https://api.github.com/users/amplab-extras/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/amplab-extras/SparkR-pkg","description":"R frontend for Spark","fork":false,"url":"https://api.github.com/repos/amplab-extras/SparkR-pkg","forks_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/forks","keys_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/keys{/key_id}","collaborators_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/teams","hooks_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/hooks","issue_events_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues/events{/number}","events_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/events","assignees_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/assignees{/user}","branches_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/branches{/branch}","tags_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/tags","blobs_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/git/refs{/sha}","trees_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/git/trees{/sha}","statuses_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/statuses/{sha}","languages_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/languages","stargazers_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/stargazers","contributors_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/contributors","subscribers_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/subscribers","subscription_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/subscription","commits_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/commits{/sha}","git_commits_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/git/commits{/sha}","comments_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/comments{/number}","issue_comment_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues/comments{/number}","contents_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/contents/{+path}","compare_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/compare/{base}...{head}","merges_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/merges","archive_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/downloads","issues_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues{/number}","pulls_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls{/number}","milestones_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/milestones{/number}","notifications_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/labels{/name}","releases_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/releases{/id}","created_at":"2013-12-21T21:16:54Z","updated_at":"2015-03-13T02:22:18Z","pushed_at":"2015-03-13T02:22:17Z","git_url":"git://github.com/amplab-extras/SparkR-pkg.git","ssh_url":"git@github.com:amplab-extras/SparkR-pkg.git","clone_url":"https://github.com/amplab-extras/SparkR-pkg.git","svn_url":"https://github.com/amplab-extras/SparkR-pkg","homepage":"http://amplab-extras.github.io/SparkR-pkg","size":6147,"stargazers_count":405,"watchers_count":405,"language":"R","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":145,"mirror_url":null,"open_issues_count":28,"forks":145,"open_issues":28,"watchers":405,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/212"},"html":{"href":"https://github.com/amplab-extras/SparkR-pkg/pull/212"},"issue":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues/212"},"comments":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues/212/comments"},"review_comments":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/212/comments"},"review_comment":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/212/commits"},"statuses":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/statuses/e87bb9876804af46c64137db934cfb89f4a14936"}}}}