{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/34970544","id":34970544,"diff_hunk":"@@ -0,0 +1,142 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.sql.execution\n+\n+import java.io.{DataInputStream, DataOutputStream, OutputStream, InputStream}\n+import java.nio.ByteBuffer\n+\n+import scala.reflect.ClassTag\n+\n+import com.google.common.io.ByteStreams\n+\n+import org.apache.spark.serializer.{SerializationStream, DeserializationStream, SerializerInstance, Serializer}\n+import org.apache.spark.sql.catalyst.expressions.UnsafeRow\n+import org.apache.spark.unsafe.PlatformDependent\n+\n+/**\n+ * Serializer for serializing [[UnsafeRow]]s during shuffle. Since UnsafeRows are already stored as\n+ * bytes, this serializer simply copies those bytes to the underlying output stream. When\n+ * deserializing a stream of rows, instances of this serializer mutate and return a single UnsafeRow\n+ * instance that is backed by an on-heap byte array.\n+ *\n+ * Note that this serializer implements only the [[Serializer]] methods that are used during\n+ * shuffle, so certain [[SerializerInstance]] methods will throw UnsupportedOperationException.\n+ *\n+ * This serializer does not support UnsafeRows that use\n+ * [[org.apache.spark.sql.catalyst.util.ObjectPool]].\n+ *\n+ * @param numFields the number of fields in the row being serialized.\n+ */\n+private[sql] class UnsafeRowSerializer(numFields: Int) extends Serializer with Serializable {\n+  override def newInstance(): SerializerInstance = new UnsafeRowSerializerInstance(numFields)\n+  override private[spark] def supportsRelocationOfSerializedObjects: Boolean = true\n+}\n+\n+private class UnsafeRowSerializerInstance(numFields: Int) extends SerializerInstance {\n+\n+  private[this] val EOF: Int = -1\n+\n+  override def serializeStream(out: OutputStream): SerializationStream = new SerializationStream {\n+    private[this] var writeBuffer: Array[Byte] = new Array[Byte](4096)\n+    private[this] val dOut: DataOutputStream = new DataOutputStream(out)\n+\n+    override def writeValue[T: ClassTag](value: T): SerializationStream = {\n+      val row = value.asInstanceOf[UnsafeRow]\n+      assert(row.getPool == null, \"UnsafeRowSerializer does not support ObjectPool\")\n+      dOut.writeInt(row.getSizeInBytes)\n+      var dataRemaining: Int = row.getSizeInBytes\n+      val baseObject = row.getBaseObject\n+      var rowReadPosition: Long = row.getBaseOffset\n+      while (dataRemaining > 0) {","path":"sql/core/src/main/scala/org/apache/spark/sql/execution/UnsafeRowSerializer.scala","position":65,"original_position":65,"commit_id":"7e7525995d383af03a8b4179e8f3139cd97e2e12","original_commit_id":"7e7525995d383af03a8b4179e8f3139cd97e2e12","user":{"login":"rxin","id":323388,"avatar_url":"https://avatars.githubusercontent.com/u/323388?v=3","gravatar_id":"","url":"https://api.github.com/users/rxin","html_url":"https://github.com/rxin","followers_url":"https://api.github.com/users/rxin/followers","following_url":"https://api.github.com/users/rxin/following{/other_user}","gists_url":"https://api.github.com/users/rxin/gists{/gist_id}","starred_url":"https://api.github.com/users/rxin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/rxin/subscriptions","organizations_url":"https://api.github.com/users/rxin/orgs","repos_url":"https://api.github.com/users/rxin/repos","events_url":"https://api.github.com/users/rxin/events{/privacy}","received_events_url":"https://api.github.com/users/rxin/received_events","type":"User","site_admin":false},"body":"probably doesn't matter in the MVP, but if we know the UnsafeRow is backed by a byte array, we don't need to do this copying, do we?\r\n","created_at":"2015-07-20T06:38:43Z","updated_at":"2015-07-20T06:38:43Z","html_url":"https://github.com/apache/spark/pull/7456#discussion_r34970544","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/7456","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/34970544"},"html":{"href":"https://github.com/apache/spark/pull/7456#discussion_r34970544"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/7456"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/7456","id":40193432,"html_url":"https://github.com/apache/spark/pull/7456","diff_url":"https://github.com/apache/spark/pull/7456.diff","patch_url":"https://github.com/apache/spark/pull/7456.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/7456","number":7456,"state":"open","locked":false,"title":"[SPARK-9023] [SQL] Efficiency improvements for UnsafeRows in Exchange","user":{"login":"JoshRosen","id":50748,"avatar_url":"https://avatars.githubusercontent.com/u/50748?v=3","gravatar_id":"","url":"https://api.github.com/users/JoshRosen","html_url":"https://github.com/JoshRosen","followers_url":"https://api.github.com/users/JoshRosen/followers","following_url":"https://api.github.com/users/JoshRosen/following{/other_user}","gists_url":"https://api.github.com/users/JoshRosen/gists{/gist_id}","starred_url":"https://api.github.com/users/JoshRosen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JoshRosen/subscriptions","organizations_url":"https://api.github.com/users/JoshRosen/orgs","repos_url":"https://api.github.com/users/JoshRosen/repos","events_url":"https://api.github.com/users/JoshRosen/events{/privacy}","received_events_url":"https://api.github.com/users/JoshRosen/received_events","type":"User","site_admin":false},"body":"This pull request aims to improve the performance of SQL's Exchange operator when shuffling UnsafeRows.  It also makes several general efficiency improvements to Exchange.\r\n\r\nKey changes:\r\n\r\n- When performing hash partitioning, the old Exchange projected the partitioning columns into a new row then passed a `(partitioningColumRow: InternalRow, row: InternalRow)` pair into the shuffle. This is very inefficient because it ends up redundantly serializing the partitioning columns only to immediately discard them after the shuffle.  After this patch's changes, Exchange now shuffles `(partitionId: Int, row: InternalRow)` pairs.  This still isn't optimal, since we're still shuffling extra data that we don't need, but it's significantly more efficient than the old implementation; in the future, we may be able to further optimize this once we implement a new shuffle write interface that accepts non-key-value-pair inputs.\r\n- Exchange's `compute()` method has been significantly simplified; the new code has less duplication and thus is easier to understand.\r\n- When the Exchange's input operator produces UnsafeRows, Exchange will use a specialized `UnsafeRowSerializer` to serialize these rows.  This serializer is significantly more efficient since it simply copies the UnsafeRow's underlying bytes.  Note that this approach does not work for UnsafeRows that use the ObjectPool mechanism; I did not add support for this because we are planning to remove ObjectPool in the next few weeks.","created_at":"2015-07-17T01:37:59Z","updated_at":"2015-07-20T06:38:43Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7d3b1493179dd31cf99bbe94e8a8d82ec7ea3412","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/7456/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/7456/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/7456/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/7e7525995d383af03a8b4179e8f3139cd97e2e12","head":{"label":"JoshRosen:unsafe-exchange","ref":"unsafe-exchange","sha":"7e7525995d383af03a8b4179e8f3139cd97e2e12","user":{"login":"JoshRosen","id":50748,"avatar_url":"https://avatars.githubusercontent.com/u/50748?v=3","gravatar_id":"","url":"https://api.github.com/users/JoshRosen","html_url":"https://github.com/JoshRosen","followers_url":"https://api.github.com/users/JoshRosen/followers","following_url":"https://api.github.com/users/JoshRosen/following{/other_user}","gists_url":"https://api.github.com/users/JoshRosen/gists{/gist_id}","starred_url":"https://api.github.com/users/JoshRosen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JoshRosen/subscriptions","organizations_url":"https://api.github.com/users/JoshRosen/orgs","repos_url":"https://api.github.com/users/JoshRosen/repos","events_url":"https://api.github.com/users/JoshRosen/events{/privacy}","received_events_url":"https://api.github.com/users/JoshRosen/received_events","type":"User","site_admin":false},"repo":{"id":22238710,"name":"spark","full_name":"JoshRosen/spark","owner":{"login":"JoshRosen","id":50748,"avatar_url":"https://avatars.githubusercontent.com/u/50748?v=3","gravatar_id":"","url":"https://api.github.com/users/JoshRosen","html_url":"https://github.com/JoshRosen","followers_url":"https://api.github.com/users/JoshRosen/followers","following_url":"https://api.github.com/users/JoshRosen/following{/other_user}","gists_url":"https://api.github.com/users/JoshRosen/gists{/gist_id}","starred_url":"https://api.github.com/users/JoshRosen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JoshRosen/subscriptions","organizations_url":"https://api.github.com/users/JoshRosen/orgs","repos_url":"https://api.github.com/users/JoshRosen/repos","events_url":"https://api.github.com/users/JoshRosen/events{/privacy}","received_events_url":"https://api.github.com/users/JoshRosen/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/JoshRosen/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/JoshRosen/spark","forks_url":"https://api.github.com/repos/JoshRosen/spark/forks","keys_url":"https://api.github.com/repos/JoshRosen/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/JoshRosen/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/JoshRosen/spark/teams","hooks_url":"https://api.github.com/repos/JoshRosen/spark/hooks","issue_events_url":"https://api.github.com/repos/JoshRosen/spark/issues/events{/number}","events_url":"https://api.github.com/repos/JoshRosen/spark/events","assignees_url":"https://api.github.com/repos/JoshRosen/spark/assignees{/user}","branches_url":"https://api.github.com/repos/JoshRosen/spark/branches{/branch}","tags_url":"https://api.github.com/repos/JoshRosen/spark/tags","blobs_url":"https://api.github.com/repos/JoshRosen/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/JoshRosen/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/JoshRosen/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/JoshRosen/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/JoshRosen/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/JoshRosen/spark/languages","stargazers_url":"https://api.github.com/repos/JoshRosen/spark/stargazers","contributors_url":"https://api.github.com/repos/JoshRosen/spark/contributors","subscribers_url":"https://api.github.com/repos/JoshRosen/spark/subscribers","subscription_url":"https://api.github.com/repos/JoshRosen/spark/subscription","commits_url":"https://api.github.com/repos/JoshRosen/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/JoshRosen/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/JoshRosen/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/JoshRosen/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/JoshRosen/spark/contents/{+path}","compare_url":"https://api.github.com/repos/JoshRosen/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/JoshRosen/spark/merges","archive_url":"https://api.github.com/repos/JoshRosen/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/JoshRosen/spark/downloads","issues_url":"https://api.github.com/repos/JoshRosen/spark/issues{/number}","pulls_url":"https://api.github.com/repos/JoshRosen/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/JoshRosen/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/JoshRosen/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/JoshRosen/spark/labels{/name}","releases_url":"https://api.github.com/repos/JoshRosen/spark/releases{/id}","created_at":"2014-07-25T00:47:52Z","updated_at":"2015-05-01T22:45:32Z","pushed_at":"2015-07-19T05:01:56Z","git_url":"git://github.com/JoshRosen/spark.git","ssh_url":"git@github.com:JoshRosen/spark.git","clone_url":"https://github.com/JoshRosen/spark.git","svn_url":"https://github.com/JoshRosen/spark","homepage":null,"size":116207,"stargazers_count":1,"watchers_count":1,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":0,"forks":3,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"6e1e2eba696e89ba57bf5450b9c72c4386e43dc8","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-07-20T04:23:31Z","pushed_at":"2015-07-20T06:38:14Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1560849,"stargazers_count":4796,"watchers_count":4796,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":4294,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":336,"forks":4294,"open_issues":336,"watchers":4796,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/7456"},"html":{"href":"https://github.com/apache/spark/pull/7456"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/7456"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/7456/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/7456/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/7456/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/7e7525995d383af03a8b4179e8f3139cd97e2e12"}}}}