{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/34850864","id":34850864,"diff_hunk":"@@ -0,0 +1,146 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.util.collection.unsafe.sort;\n+\n+import java.io.File;\n+import java.io.IOException;\n+\n+import scala.Tuple2;\n+\n+import org.apache.spark.executor.ShuffleWriteMetrics;\n+import org.apache.spark.serializer.DummySerializerInstance;\n+import org.apache.spark.storage.BlockId;\n+import org.apache.spark.storage.BlockManager;\n+import org.apache.spark.storage.BlockObjectWriter;\n+import org.apache.spark.storage.TempLocalBlockId;\n+import org.apache.spark.unsafe.PlatformDependent;\n+\n+/**\n+ * Spills a list of sorted records to disk. Spill files have the following format:\n+ *\n+ *   [# of records (int)] [[len (int)][prefix (long)][data (bytes)]...]\n+ */\n+final class UnsafeSorterSpillWriter {\n+\n+  static final int DISK_WRITE_BUFFER_SIZE = 1024 * 1024;\n+\n+  // Small writes to DiskBlockObjectWriter will be fairly inefficient. Since there doesn't seem to\n+  // be an API to directly transfer bytes from managed memory to the disk writer, we buffer\n+  // data through a byte array.\n+  private byte[] writeBuffer = new byte[DISK_WRITE_BUFFER_SIZE];\n+\n+  private final File file;\n+  private final BlockId blockId;\n+  private final int numRecordsToWrite;\n+  private BlockObjectWriter writer;\n+  private int numRecordsSpilled = 0;\n+\n+  public UnsafeSorterSpillWriter(\n+      BlockManager blockManager,\n+      int fileBufferSize,\n+      ShuffleWriteMetrics writeMetrics,\n+      int numRecordsToWrite) throws IOException {\n+    final Tuple2<TempLocalBlockId, File> spilledFileInfo =\n+      blockManager.diskBlockManager().createTempLocalBlock();\n+    this.file = spilledFileInfo._2();\n+    this.blockId = spilledFileInfo._1();\n+    this.numRecordsToWrite = numRecordsToWrite;\n+    // Unfortunately, we need a serializer instance in order to construct a DiskBlockObjectWriter.\n+    // Our write path doesn't actually use this serializer (since we end up calling the `write()`\n+    // OutputStream methods), but DiskBlockObjectWriter still calls some methods on it. To work\n+    // around this, we pass a dummy no-op serializer.\n+    writer = blockManager.getDiskWriter(\n+      blockId, file, DummySerializerInstance.INSTANCE, fileBufferSize, writeMetrics);\n+    // Write the number of records\n+    writeIntToBuffer(numRecordsToWrite, 0);\n+    writer.write(writeBuffer, 0, 4);\n+  }\n+\n+  // Based on DataOutputStream.writeLong.\n+  private void writeLongToBuffer(long v, int offset) throws IOException {\n+    writeBuffer[offset + 0] = (byte)(v >>> 56);\n+    writeBuffer[offset + 1] = (byte)(v >>> 48);\n+    writeBuffer[offset + 2] = (byte)(v >>> 40);\n+    writeBuffer[offset + 3] = (byte)(v >>> 32);\n+    writeBuffer[offset + 4] = (byte)(v >>> 24);\n+    writeBuffer[offset + 5] = (byte)(v >>> 16);\n+    writeBuffer[offset + 6] = (byte)(v >>>  8);\n+    writeBuffer[offset + 7] = (byte)(v >>>  0);\n+  }\n+\n+  // Based on DataOutputStream.writeInt.\n+  private void writeIntToBuffer(int v, int offset) throws IOException {\n+    writeBuffer[offset + 0] = (byte)(v >>> 24);\n+    writeBuffer[offset + 1] = (byte)(v >>> 16);\n+    writeBuffer[offset + 2] = (byte)(v >>>  8);\n+    writeBuffer[offset + 3] = (byte)(v >>>  0);\n+  }\n+\n+  /**\n+   * Write a record to a spill file.\n+   *\n+   * @param baseObject the base object / memory page containing the record\n+   * @param baseOffset the base offset which points directly to the record data.\n+   * @param recordLength the length of the record.\n+   * @param keyPrefix a sort key prefix\n+   */\n+  public void write(\n+      Object baseObject,\n+      long baseOffset,\n+      int recordLength,\n+      long keyPrefix) throws IOException {\n+    if (numRecordsSpilled == numRecordsToWrite) {\n+      throw new IllegalStateException(\n+        \"Number of records written exceeded numRecordsToWrite = \" + numRecordsToWrite);\n+    } else {\n+      numRecordsSpilled++;\n+    }\n+    writeIntToBuffer(recordLength, 0);\n+    writeLongToBuffer(keyPrefix, 4);\n+    int dataRemaining = recordLength;\n+    int freeSpaceInWriteBuffer = DISK_WRITE_BUFFER_SIZE - 4 - 8; // space used by prefix + len\n+    long recordReadPosition = baseOffset;\n+    while (dataRemaining > 0) {\n+      final int toTransfer = Math.min(freeSpaceInWriteBuffer, dataRemaining);\n+      PlatformDependent.copyMemory(\n+        baseObject,\n+        recordReadPosition,\n+        writeBuffer,\n+        PlatformDependent.BYTE_ARRAY_OFFSET + (DISK_WRITE_BUFFER_SIZE - freeSpaceInWriteBuffer),\n+        toTransfer);\n+      writer.write(writeBuffer, 0, (DISK_WRITE_BUFFER_SIZE - freeSpaceInWriteBuffer) + toTransfer);\n+      recordReadPosition += toTransfer;\n+      dataRemaining -= toTransfer;\n+      freeSpaceInWriteBuffer = DISK_WRITE_BUFFER_SIZE;\n+    }\n+    if (freeSpaceInWriteBuffer < DISK_WRITE_BUFFER_SIZE) {","path":"core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeSorterSpillWriter.java","position":131,"original_position":131,"commit_id":"6beb4674999820126861fdac99fb84f8ea5d57ff","original_commit_id":"6beb4674999820126861fdac99fb84f8ea5d57ff","user":{"login":"tedyu","id":235188,"avatar_url":"https://avatars.githubusercontent.com/u/235188?v=3","gravatar_id":"","url":"https://api.github.com/users/tedyu","html_url":"https://github.com/tedyu","followers_url":"https://api.github.com/users/tedyu/followers","following_url":"https://api.github.com/users/tedyu/following{/other_user}","gists_url":"https://api.github.com/users/tedyu/gists{/gist_id}","starred_url":"https://api.github.com/users/tedyu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tedyu/subscriptions","organizations_url":"https://api.github.com/users/tedyu/orgs","repos_url":"https://api.github.com/users/tedyu/repos","events_url":"https://api.github.com/users/tedyu/events{/privacy}","received_events_url":"https://api.github.com/users/tedyu/received_events","type":"User","site_admin":false},"body":"If I understand correctly, this would be true if the above loop is skipped.\r\nIn that case we write 12 bytes. Is this intended ?","created_at":"2015-07-16T23:49:46Z","updated_at":"2015-07-16T23:49:46Z","html_url":"https://github.com/apache/spark/pull/6444#discussion_r34850864","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/6444","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/34850864"},"html":{"href":"https://github.com/apache/spark/pull/6444#discussion_r34850864"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/6444"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/6444","id":36347686,"html_url":"https://github.com/apache/spark/pull/6444","diff_url":"https://github.com/apache/spark/pull/6444.diff","patch_url":"https://github.com/apache/spark/pull/6444.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/6444","number":6444,"state":"closed","locked":false,"title":"[SPARK-7078] [SPARK-7079] Binary processing sort for Spark SQL","user":{"login":"JoshRosen","id":50748,"avatar_url":"https://avatars.githubusercontent.com/u/50748?v=3","gravatar_id":"","url":"https://api.github.com/users/JoshRosen","html_url":"https://github.com/JoshRosen","followers_url":"https://api.github.com/users/JoshRosen/followers","following_url":"https://api.github.com/users/JoshRosen/following{/other_user}","gists_url":"https://api.github.com/users/JoshRosen/gists{/gist_id}","starred_url":"https://api.github.com/users/JoshRosen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JoshRosen/subscriptions","organizations_url":"https://api.github.com/users/JoshRosen/orgs","repos_url":"https://api.github.com/users/JoshRosen/repos","events_url":"https://api.github.com/users/JoshRosen/events{/privacy}","received_events_url":"https://api.github.com/users/JoshRosen/received_events","type":"User","site_admin":false},"body":"This patch adds a cache-friendly external sorter which operates on serialized bytes and uses this sorter to implement a new sort operator for Spark SQL and DataFrames.\r\n\r\n### Overview of the new sorter\r\n\r\nThe new sorter design is inspired by [Alphasort](http://research.microsoft.com/pubs/68249/alphasort.doc) and implements a key-prefix optimization in order to improve the cache friendliness of the sort.  In naive sort implementations, the sorting algorithm operates on an array of record pointers.  To compare two records for ordering, the sorter must dereference these pointers, which likely involves random memory access, then compare the objects themselves.\r\n\r\n![image](https://cloud.githubusercontent.com/assets/50748/8611390/3b1402ae-2675-11e5-8308-1a10bf347e6e.png)\r\n\r\nIn a key-prefix sort, the sort operates on an array which stores the record pointer alongside a prefix of the record's key. When comparing two records for ordering, the sorter first compares the the stored key prefixes. If the ordering can be determined from the key prefixes (i.e. the prefixes are unequal), then the sort can avoid directly comparing the records, avoiding random memory accesses and full record comparisons. For example, if we're sorting a list of strings then we can store the first 8 bytes of the UTF-8 encoded string as the key-prefix and can perform unsigned byte-at-a-time comparisons to determine the ordering of strings based on their prefixes, only resorting to full comparisons for strings that share a common prefix.  In cases where the sort key can fit entirely in the space allotted for the key prefix (e.g. the sorting key is an integer), we completely avoid direct record comparison.\r\n\r\nIn this patch's implementation of key-prefix sorting, our sorter's internal array stores a 64-bit long and 64-bit pointer for each record being sorted. The key prefixes are generated by the user when inserting records into the sorter, which uses a user-defined comparison function for comparing them.  The `PrefixComparators` object implements a set of comparators for many common types, including primitive numeric types and UTF-8 strings.\r\n\r\nThe actual sorting is implemented by `UnsafeInMemorySorter`.  Most consumers will not use this directly, but instead will use `UnsafeExternalSorter`, a class which implements a sort that can spill to disk in response to memory pressure.  Internally, `UnsafeExternalSorter` creates `UnsafeInMemorySorters` to perform sorting and uses `UnsafeSortSpillReader/Writer` to spill and read back runs of sorted records and `UnsafeSortSpillMerger` to merge multiple sorted spills into a single sorted iterator.  This external sorter integrates with Spark's existing ShuffleMemoryManager for controlling spilling.\r\n\r\nMany parts of this sorter's design are based on / copied from the more specialized external sort implementation that I designed for the new UnsafeShuffleManager write path; see #5868 for more details on that patch.\r\n\r\n### Sorting rows in Spark SQL\r\n\r\nFor now, `UnsafeExternalSorter` is only used by Spark SQL, which uses it to implement a new sort operator, `UnsafeExternalSort`.  This sort operator uses a SQL-specific class called `UnsafeExternalRowSorter` that configures an `UnsafeExternalSorter` to use prefix generators and comparators that operate on rows encoded in the UnsafeRow format that was designed for Project Tungsten.\r\n\r\nI used some interesting unit-testing techniques to test this patch's SQL-specific components.  `UnsafeExternalSortSuite` uses the SQL random data generators introduced in #7176 to test the UnsafeSort operator with all atomic types both with and without nullability and in both ascending and descending sort orders.  `PrefixComparatorsSuite` contains a cool use of ScalaCheck + ScalaTest's `GeneratorDrivenPropertyChecks` in order to test UTF8String prefix comparison.\r\n\r\n### Misc. additional improvements made in this patch\r\n\r\nThis patch made several miscellaneous improvements to related code in Spark SQL:\r\n\r\n- The logic for selecting physical sort operator implementations, which was partially duplicated in both `Exchange` and `SparkStrategies, has now been consolidated into a `getSortOperator()` helper function in `SparkStrategies`.\r\n- The `SparkPlanTest` unit testing helper trait has been extended with new methods for comparing the output produced by two different physical plans. This makes it easy to write tests which assert that two physical operator implementations should produce the same output.  I also added a method for disabling the implicit sorting of outputs prior to comparing them, a change which is necessary in order to be able to write proper SparkPlan tests for sort operators.\r\n\r\n### Tasks deferred to followup patches\r\n\r\nWhile most of this patch's features are reasonably well-tested and complete, there are a number of tasks that are intentionally being deferred to followup patches:\r\n\r\n- Add tests which mock the ShuffleMemoryManager to check that memory pressure properly triggers spilling (there are examples of this type of test in #5868).\r\n- Add tests to ensure that spill files are properly cleaned up after errors.  I'd like to do this in the context of a patch which introduces more general metrics for ensuring proper cleanup of tasks' temporary files; see https://issues.apache.org/jira/browse/SPARK-8966 for more details.\r\n- Metrics integration: there are some open questions regarding how to track / report spill metrics for non-shuffle operations, so I've deferred most of the IO / shuffle metrics integration for now.\r\n- Performance profiling.\r\n\r\n<!-- Reviewable:start -->\r\n[<img src=\"https://reviewable.io/review_button.png\" height=40 alt=\"Review on Reviewable\"/>](https://reviewable.io/reviews/apache/spark/6444)\r\n<!-- Reviewable:end -->\r\n","created_at":"2015-05-27T20:55:55Z","updated_at":"2015-07-16T23:49:46Z","closed_at":"2015-07-11T22:36:31Z","merged_at":null,"merge_commit_sha":"21a9fa3dc37fb22ba912078611e9db29e201ddb2","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/6444/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/6444/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/6444/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/6beb4674999820126861fdac99fb84f8ea5d57ff","head":{"label":"JoshRosen:sql-external-sort","ref":"sql-external-sort","sha":"6beb4674999820126861fdac99fb84f8ea5d57ff","user":{"login":"JoshRosen","id":50748,"avatar_url":"https://avatars.githubusercontent.com/u/50748?v=3","gravatar_id":"","url":"https://api.github.com/users/JoshRosen","html_url":"https://github.com/JoshRosen","followers_url":"https://api.github.com/users/JoshRosen/followers","following_url":"https://api.github.com/users/JoshRosen/following{/other_user}","gists_url":"https://api.github.com/users/JoshRosen/gists{/gist_id}","starred_url":"https://api.github.com/users/JoshRosen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JoshRosen/subscriptions","organizations_url":"https://api.github.com/users/JoshRosen/orgs","repos_url":"https://api.github.com/users/JoshRosen/repos","events_url":"https://api.github.com/users/JoshRosen/events{/privacy}","received_events_url":"https://api.github.com/users/JoshRosen/received_events","type":"User","site_admin":false},"repo":{"id":22238710,"name":"spark","full_name":"JoshRosen/spark","owner":{"login":"JoshRosen","id":50748,"avatar_url":"https://avatars.githubusercontent.com/u/50748?v=3","gravatar_id":"","url":"https://api.github.com/users/JoshRosen","html_url":"https://github.com/JoshRosen","followers_url":"https://api.github.com/users/JoshRosen/followers","following_url":"https://api.github.com/users/JoshRosen/following{/other_user}","gists_url":"https://api.github.com/users/JoshRosen/gists{/gist_id}","starred_url":"https://api.github.com/users/JoshRosen/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/JoshRosen/subscriptions","organizations_url":"https://api.github.com/users/JoshRosen/orgs","repos_url":"https://api.github.com/users/JoshRosen/repos","events_url":"https://api.github.com/users/JoshRosen/events{/privacy}","received_events_url":"https://api.github.com/users/JoshRosen/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/JoshRosen/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/JoshRosen/spark","forks_url":"https://api.github.com/repos/JoshRosen/spark/forks","keys_url":"https://api.github.com/repos/JoshRosen/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/JoshRosen/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/JoshRosen/spark/teams","hooks_url":"https://api.github.com/repos/JoshRosen/spark/hooks","issue_events_url":"https://api.github.com/repos/JoshRosen/spark/issues/events{/number}","events_url":"https://api.github.com/repos/JoshRosen/spark/events","assignees_url":"https://api.github.com/repos/JoshRosen/spark/assignees{/user}","branches_url":"https://api.github.com/repos/JoshRosen/spark/branches{/branch}","tags_url":"https://api.github.com/repos/JoshRosen/spark/tags","blobs_url":"https://api.github.com/repos/JoshRosen/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/JoshRosen/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/JoshRosen/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/JoshRosen/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/JoshRosen/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/JoshRosen/spark/languages","stargazers_url":"https://api.github.com/repos/JoshRosen/spark/stargazers","contributors_url":"https://api.github.com/repos/JoshRosen/spark/contributors","subscribers_url":"https://api.github.com/repos/JoshRosen/spark/subscribers","subscription_url":"https://api.github.com/repos/JoshRosen/spark/subscription","commits_url":"https://api.github.com/repos/JoshRosen/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/JoshRosen/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/JoshRosen/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/JoshRosen/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/JoshRosen/spark/contents/{+path}","compare_url":"https://api.github.com/repos/JoshRosen/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/JoshRosen/spark/merges","archive_url":"https://api.github.com/repos/JoshRosen/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/JoshRosen/spark/downloads","issues_url":"https://api.github.com/repos/JoshRosen/spark/issues{/number}","pulls_url":"https://api.github.com/repos/JoshRosen/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/JoshRosen/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/JoshRosen/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/JoshRosen/spark/labels{/name}","releases_url":"https://api.github.com/repos/JoshRosen/spark/releases{/id}","created_at":"2014-07-25T00:47:52Z","updated_at":"2015-05-01T22:45:32Z","pushed_at":"2015-07-16T20:47:56Z","git_url":"git://github.com/JoshRosen/spark.git","ssh_url":"git@github.com:JoshRosen/spark.git","clone_url":"https://github.com/JoshRosen/spark.git","svn_url":"https://github.com/JoshRosen/spark","homepage":null,"size":115381,"stargazers_count":1,"watchers_count":1,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3,"mirror_url":null,"open_issues_count":0,"forks":3,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"11e22b74a080ea58fb9410b5cc6fa4c03f9198f2","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-07-16T23:09:49Z","pushed_at":"2015-07-16T23:45:06Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1544645,"stargazers_count":4772,"watchers_count":4772,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":4254,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":336,"forks":4254,"open_issues":336,"watchers":4772,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/6444"},"html":{"href":"https://github.com/apache/spark/pull/6444"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/6444"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/6444/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/6444/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/6444/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/6beb4674999820126861fdac99fb84f8ea5d57ff"}}}}