{"action":"created","comment":{"url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/comments/23508123","id":23508123,"diff_hunk":"@@ -0,0 +1,297 @@\n+package edu.berkeley.cs.amplab.sparkr\n+\n+import scala.collection.mutable.HashMap\n+import scala.collection.JavaConversions._\n+\n+import java.io.DataInputStream\n+import java.io.DataOutputStream\n+\n+/**\n+ * Utility functions to serialize, deserialize objects to / from R\n+ */\n+object SerializeJavaR {\n+\n+  // Type mapping from R to Java\n+  //\n+  // integer -> Int\n+  // character -> String\n+  // logical -> Boolean\n+  // double, numeric -> Double\n+  // raw -> Array[Byte]\n+  //\n+  // list[T] -> Array[T], where T is one of above mentioned types\n+  // environment -> Map[String, T], where T is a native type\n+  // jobj -> Object, where jobj is an object created in the backend\n+\n+  def readObjectType(dis: DataInputStream) = {\n+    dis.readByte().toChar\n+  }\n+\n+  def readObject(dis: DataInputStream): Object = {\n+    val dataType = readObjectType(dis)\n+    readTypedObject(dis, dataType)\n+  }\n+\n+  def readTypedObject(\n+      dis: DataInputStream,\n+      dataType: Char): Object = {\n+    dataType match {\n+      case 'i' => new java.lang.Integer(readInt(dis))\n+      case 'd' => new java.lang.Double(readDouble(dis))\n+      case 'b' => new java.lang.Boolean(readBoolean(dis))\n+      case 'c' => readString(dis)\n+      case 'e' => readMap(dis)\n+      case 'r' => readBytes(dis)\n+      case 'l' => readList(dis)\n+      case 'j' => JavaObjectTracker.getObject(readString(dis))\n+      case _ => throw new IllegalArgumentException(s\"Invalid type $dataType\")\n+    }\n+  }\n+\n+  def readBytes(in: DataInputStream) = {\n+    val len = readInt(in)\n+    val out = new Array[Byte](len)\n+    val bytesRead = in.read(out, 0, len)\n+    assert(len == bytesRead)\n+    out\n+  }\n+\n+  def readInt(in: DataInputStream) = {\n+    in.readInt()\n+  }\n+\n+  def readDouble(in: DataInputStream) = {\n+    in.readDouble()\n+  }\n+\n+  def readString(in: DataInputStream) = {\n+    val len = in.readInt()\n+    val asciiBytes = new Array[Byte](len)\n+    in.read(asciiBytes, 0, len)\n+    assert(asciiBytes(len - 1) == 0)\n+    val str = new String(asciiBytes.dropRight(1).map(_.toChar))\n+    str\n+  }\n+\n+  def readBoolean(in: DataInputStream) = {\n+    val intVal = in.readInt()\n+    if (intVal == 0) false else true\n+  }\n+\n+  def readBytesArr(in: DataInputStream) = {\n+    val len = readInt(in)\n+    (0 until len).map(_ => readBytes(in)).toArray\n+  }\n+\n+  def readIntArr(in: DataInputStream) = {\n+    val len = readInt(in)\n+    (0 until len).map(_ => readInt(in)).toArray\n+  }\n+\n+  def readDoubleArr(in: DataInputStream) = {\n+    val len = readInt(in)\n+    (0 until len).map(_ => readDouble(in)).toArray\n+  }\n+\n+  def readBooleanArr(in: DataInputStream) = {\n+    val len = readInt(in)\n+    (0 until len).map(_ => readBoolean(in)).toArray\n+  }\n+\n+  def readStringArr(in: DataInputStream) = {\n+    val len = readInt(in)\n+    (0 until len).map(_ => readString(in)).toArray\n+  }\n+\n+  def readList(dis: DataInputStream) = {\n+    val arrType = readObjectType(dis)\n+    arrType match {\n+      case 'i' => readIntArr(dis)\n+      case 'c' => readStringArr(dis)\n+      case 'd' => readDoubleArr(dis)\n+      case 'l' => readBooleanArr(dis)\n+      case 'j' => readStringArr(dis).map(x => JavaObjectTracker.getObject(x))\n+      case 'r' => readBytesArr(dis)\n+      case _ => throw new IllegalArgumentException(s\"Invalid array type $arrType\")\n+    }\n+  }\n+\n+  def readMap(in: DataInputStream): java.util.Map[Object, Object] = {\n+    val len = readInt(in)\n+    if (len > 0) {\n+      val keysType = readObjectType(in)\n+      val keysLen = readInt(in)\n+      val keys = (0 until keysLen).map(_ => readTypedObject(in, keysType))\n+\n+      val valuesType = readObjectType(in)\n+      val valuesLen = readInt(in)\n+      val values = (0 until len).map(_ => readTypedObject(in, valuesType))\n+      mapAsJavaMap(keys.zip(values).toMap)\n+    } else {\n+      new java.util.HashMap[Object, Object]()\n+    }\n+  }\n+\n+  // Methods to write out data from Java to R\n+  //\n+  // Type mapping from Java to R\n+  //\n+  // void -> NULL\n+  // Int -> integer\n+  // String -> character\n+  // Boolean -> logical\n+  // Double -> numeric / double","path":"pkg/src/src/main/scala/edu/berkeley/cs/amplab/sparkr/SerializeJavaR.scala","position":null,"original_position":143,"commit_id":"f00b531d32da434f5755da79547147f98baac140","original_commit_id":"c09ba05c99a01ee03545b324e2f36b2af0323dbb","user":{"login":"shivaram","id":143893,"avatar_url":"https://avatars.githubusercontent.com/u/143893?v=3","gravatar_id":"","url":"https://api.github.com/users/shivaram","html_url":"https://github.com/shivaram","followers_url":"https://api.github.com/users/shivaram/followers","following_url":"https://api.github.com/users/shivaram/following{/other_user}","gists_url":"https://api.github.com/users/shivaram/gists{/gist_id}","starred_url":"https://api.github.com/users/shivaram/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shivaram/subscriptions","organizations_url":"https://api.github.com/users/shivaram/orgs","repos_url":"https://api.github.com/users/shivaram/repos","events_url":"https://api.github.com/users/shivaram/events{/privacy}","received_events_url":"https://api.github.com/users/shivaram/received_events","type":"User","site_admin":false},"body":"Fixed now","created_at":"2015-01-25T22:02:31Z","updated_at":"2015-01-25T22:02:31Z","html_url":"https://github.com/amplab-extras/SparkR-pkg/pull/131#discussion_r23508123","pull_request_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/131","_links":{"self":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/comments/23508123"},"html":{"href":"https://github.com/amplab-extras/SparkR-pkg/pull/131#discussion_r23508123"},"pull_request":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/131"}}},"pull_request":{"url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/131","id":27804688,"html_url":"https://github.com/amplab-extras/SparkR-pkg/pull/131","diff_url":"https://github.com/amplab-extras/SparkR-pkg/pull/131.diff","patch_url":"https://github.com/amplab-extras/SparkR-pkg/pull/131.patch","issue_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues/131","number":131,"state":"open","locked":false,"title":"[SPARKR-177] New R to Java bridge","user":{"login":"shivaram","id":143893,"avatar_url":"https://avatars.githubusercontent.com/u/143893?v=3","gravatar_id":"","url":"https://api.github.com/users/shivaram","html_url":"https://github.com/shivaram","followers_url":"https://api.github.com/users/shivaram/followers","following_url":"https://api.github.com/users/shivaram/following{/other_user}","gists_url":"https://api.github.com/users/shivaram/gists{/gist_id}","starred_url":"https://api.github.com/users/shivaram/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shivaram/subscriptions","organizations_url":"https://api.github.com/users/shivaram/orgs","repos_url":"https://api.github.com/users/shivaram/repos","events_url":"https://api.github.com/users/shivaram/events{/privacy}","received_events_url":"https://api.github.com/users/shivaram/received_events","type":"User","site_admin":false},"body":"This PR introduces a new R to Java bridge to be used on the driver side. The newly introduced SparkRBackend is a netty-based server that opens a socket for communicating with R. We use Java reflection to support creating objects and invoking methods on these objects. More details can be found in https://docs.google.com/document/d/1hscJX3VAp3GW2h24GvUIP1UXfcq8gmxTqml1dcDuL6I/edit\r\n\r\nThis is joint work with @sun-rui \r\n\r\n@concretevitamin @sun-rui -- It will be great if you can review this change for code structure, understandability and make sure I haven't missed anything that needs to be updated.","created_at":"2015-01-21T23:19:41Z","updated_at":"2015-01-25T22:02:31Z","closed_at":null,"merged_at":null,"merge_commit_sha":"98dd64eb67f03ae11fe3f69d7d9d7faecf158ae6","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/131/commits","review_comments_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/131/comments","review_comment_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/comments/{number}","comments_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues/131/comments","statuses_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/statuses/f00b531d32da434f5755da79547147f98baac140","head":{"label":"shivaram:rJavaExpt","ref":"rJavaExpt","sha":"f00b531d32da434f5755da79547147f98baac140","user":{"login":"shivaram","id":143893,"avatar_url":"https://avatars.githubusercontent.com/u/143893?v=3","gravatar_id":"","url":"https://api.github.com/users/shivaram","html_url":"https://github.com/shivaram","followers_url":"https://api.github.com/users/shivaram/followers","following_url":"https://api.github.com/users/shivaram/following{/other_user}","gists_url":"https://api.github.com/users/shivaram/gists{/gist_id}","starred_url":"https://api.github.com/users/shivaram/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shivaram/subscriptions","organizations_url":"https://api.github.com/users/shivaram/orgs","repos_url":"https://api.github.com/users/shivaram/repos","events_url":"https://api.github.com/users/shivaram/events{/privacy}","received_events_url":"https://api.github.com/users/shivaram/received_events","type":"User","site_admin":false},"repo":{"id":16079623,"name":"SparkR-pkg","full_name":"shivaram/SparkR-pkg","owner":{"login":"shivaram","id":143893,"avatar_url":"https://avatars.githubusercontent.com/u/143893?v=3","gravatar_id":"","url":"https://api.github.com/users/shivaram","html_url":"https://github.com/shivaram","followers_url":"https://api.github.com/users/shivaram/followers","following_url":"https://api.github.com/users/shivaram/following{/other_user}","gists_url":"https://api.github.com/users/shivaram/gists{/gist_id}","starred_url":"https://api.github.com/users/shivaram/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/shivaram/subscriptions","organizations_url":"https://api.github.com/users/shivaram/orgs","repos_url":"https://api.github.com/users/shivaram/repos","events_url":"https://api.github.com/users/shivaram/events{/privacy}","received_events_url":"https://api.github.com/users/shivaram/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/shivaram/SparkR-pkg","description":"R frontend for Spark","fork":true,"url":"https://api.github.com/repos/shivaram/SparkR-pkg","forks_url":"https://api.github.com/repos/shivaram/SparkR-pkg/forks","keys_url":"https://api.github.com/repos/shivaram/SparkR-pkg/keys{/key_id}","collaborators_url":"https://api.github.com/repos/shivaram/SparkR-pkg/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/shivaram/SparkR-pkg/teams","hooks_url":"https://api.github.com/repos/shivaram/SparkR-pkg/hooks","issue_events_url":"https://api.github.com/repos/shivaram/SparkR-pkg/issues/events{/number}","events_url":"https://api.github.com/repos/shivaram/SparkR-pkg/events","assignees_url":"https://api.github.com/repos/shivaram/SparkR-pkg/assignees{/user}","branches_url":"https://api.github.com/repos/shivaram/SparkR-pkg/branches{/branch}","tags_url":"https://api.github.com/repos/shivaram/SparkR-pkg/tags","blobs_url":"https://api.github.com/repos/shivaram/SparkR-pkg/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/shivaram/SparkR-pkg/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/shivaram/SparkR-pkg/git/refs{/sha}","trees_url":"https://api.github.com/repos/shivaram/SparkR-pkg/git/trees{/sha}","statuses_url":"https://api.github.com/repos/shivaram/SparkR-pkg/statuses/{sha}","languages_url":"https://api.github.com/repos/shivaram/SparkR-pkg/languages","stargazers_url":"https://api.github.com/repos/shivaram/SparkR-pkg/stargazers","contributors_url":"https://api.github.com/repos/shivaram/SparkR-pkg/contributors","subscribers_url":"https://api.github.com/repos/shivaram/SparkR-pkg/subscribers","subscription_url":"https://api.github.com/repos/shivaram/SparkR-pkg/subscription","commits_url":"https://api.github.com/repos/shivaram/SparkR-pkg/commits{/sha}","git_commits_url":"https://api.github.com/repos/shivaram/SparkR-pkg/git/commits{/sha}","comments_url":"https://api.github.com/repos/shivaram/SparkR-pkg/comments{/number}","issue_comment_url":"https://api.github.com/repos/shivaram/SparkR-pkg/issues/comments/{number}","contents_url":"https://api.github.com/repos/shivaram/SparkR-pkg/contents/{+path}","compare_url":"https://api.github.com/repos/shivaram/SparkR-pkg/compare/{base}...{head}","merges_url":"https://api.github.com/repos/shivaram/SparkR-pkg/merges","archive_url":"https://api.github.com/repos/shivaram/SparkR-pkg/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/shivaram/SparkR-pkg/downloads","issues_url":"https://api.github.com/repos/shivaram/SparkR-pkg/issues{/number}","pulls_url":"https://api.github.com/repos/shivaram/SparkR-pkg/pulls{/number}","milestones_url":"https://api.github.com/repos/shivaram/SparkR-pkg/milestones{/number}","notifications_url":"https://api.github.com/repos/shivaram/SparkR-pkg/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/shivaram/SparkR-pkg/labels{/name}","releases_url":"https://api.github.com/repos/shivaram/SparkR-pkg/releases{/id}","created_at":"2014-01-20T18:46:29Z","updated_at":"2015-01-02T20:32:53Z","pushed_at":"2015-01-25T21:56:26Z","git_url":"git://github.com/shivaram/SparkR-pkg.git","ssh_url":"git@github.com:shivaram/SparkR-pkg.git","clone_url":"https://github.com/shivaram/SparkR-pkg.git","svn_url":"https://github.com/shivaram/SparkR-pkg","homepage":"http://amplab-extras.github.io/SparkR-pkg","size":3698,"stargazers_count":1,"watchers_count":1,"language":"R","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"amplab-extras:master","ref":"master","sha":"bcd4258888473c5dac5c2a6e8f1d61e3ba847857","user":{"login":"amplab-extras","id":6403540,"avatar_url":"https://avatars.githubusercontent.com/u/6403540?v=3","gravatar_id":"","url":"https://api.github.com/users/amplab-extras","html_url":"https://github.com/amplab-extras","followers_url":"https://api.github.com/users/amplab-extras/followers","following_url":"https://api.github.com/users/amplab-extras/following{/other_user}","gists_url":"https://api.github.com/users/amplab-extras/gists{/gist_id}","starred_url":"https://api.github.com/users/amplab-extras/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplab-extras/subscriptions","organizations_url":"https://api.github.com/users/amplab-extras/orgs","repos_url":"https://api.github.com/users/amplab-extras/repos","events_url":"https://api.github.com/users/amplab-extras/events{/privacy}","received_events_url":"https://api.github.com/users/amplab-extras/received_events","type":"Organization","site_admin":false},"repo":{"id":15365789,"name":"SparkR-pkg","full_name":"amplab-extras/SparkR-pkg","owner":{"login":"amplab-extras","id":6403540,"avatar_url":"https://avatars.githubusercontent.com/u/6403540?v=3","gravatar_id":"","url":"https://api.github.com/users/amplab-extras","html_url":"https://github.com/amplab-extras","followers_url":"https://api.github.com/users/amplab-extras/followers","following_url":"https://api.github.com/users/amplab-extras/following{/other_user}","gists_url":"https://api.github.com/users/amplab-extras/gists{/gist_id}","starred_url":"https://api.github.com/users/amplab-extras/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/amplab-extras/subscriptions","organizations_url":"https://api.github.com/users/amplab-extras/orgs","repos_url":"https://api.github.com/users/amplab-extras/repos","events_url":"https://api.github.com/users/amplab-extras/events{/privacy}","received_events_url":"https://api.github.com/users/amplab-extras/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/amplab-extras/SparkR-pkg","description":"R frontend for Spark","fork":false,"url":"https://api.github.com/repos/amplab-extras/SparkR-pkg","forks_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/forks","keys_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/keys{/key_id}","collaborators_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/teams","hooks_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/hooks","issue_events_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues/events{/number}","events_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/events","assignees_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/assignees{/user}","branches_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/branches{/branch}","tags_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/tags","blobs_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/git/refs{/sha}","trees_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/git/trees{/sha}","statuses_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/statuses/{sha}","languages_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/languages","stargazers_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/stargazers","contributors_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/contributors","subscribers_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/subscribers","subscription_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/subscription","commits_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/commits{/sha}","git_commits_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/git/commits{/sha}","comments_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/comments{/number}","issue_comment_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues/comments/{number}","contents_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/contents/{+path}","compare_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/compare/{base}...{head}","merges_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/merges","archive_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/downloads","issues_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues{/number}","pulls_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls{/number}","milestones_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/milestones{/number}","notifications_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/labels{/name}","releases_url":"https://api.github.com/repos/amplab-extras/SparkR-pkg/releases{/id}","created_at":"2013-12-21T21:16:54Z","updated_at":"2015-01-25T03:38:20Z","pushed_at":"2015-01-21T06:49:35Z","git_url":"git://github.com/amplab-extras/SparkR-pkg.git","ssh_url":"git@github.com:amplab-extras/SparkR-pkg.git","clone_url":"https://github.com/amplab-extras/SparkR-pkg.git","svn_url":"https://github.com/amplab-extras/SparkR-pkg","homepage":"http://amplab-extras.github.io/SparkR-pkg","size":4391,"stargazers_count":374,"watchers_count":374,"language":"R","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":true,"forks_count":122,"mirror_url":null,"open_issues_count":21,"forks":122,"open_issues":21,"watchers":374,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/131"},"html":{"href":"https://github.com/amplab-extras/SparkR-pkg/pull/131"},"issue":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues/131"},"comments":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/issues/131/comments"},"review_comments":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/131/comments"},"review_comment":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/pulls/131/commits"},"statuses":{"href":"https://api.github.com/repos/amplab-extras/SparkR-pkg/statuses/f00b531d32da434f5755da79547147f98baac140"}}}}