{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/43437967","id":43437967,"diff_hunk":"@@ -0,0 +1,336 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.deploy\n+\n+import java.net.URI\n+import java.util\n+\n+import scala.collection.JavaConverters._\n+import scala.collection.mutable.ArrayBuffer\n+\n+import com.amazonaws.{AmazonClientException, AmazonServiceException, ClientConfiguration, Protocol}\n+import com.amazonaws.auth.{AWSCredentialsProvider, BasicAWSCredentials, InstanceProfileCredentialsProvider, STSAssumeRoleSessionCredentialsProvider}\n+import com.amazonaws.internal.StaticCredentialsProvider\n+import com.amazonaws.services.s3.AmazonS3Client\n+import com.amazonaws.services.s3.model.{ListObjectsRequest, ObjectListing, S3ObjectSummary}\n+\n+import com.google.common.annotations.VisibleForTesting\n+import com.google.common.base.{Preconditions, Strings}\n+import com.google.common.cache.{Cache, CacheBuilder}\n+import com.google.common.collect.AbstractSequentialIterator\n+\n+import org.apache.hadoop.conf.Configuration\n+import org.apache.hadoop.fs.{FileStatus, GlobPattern, Path, PathFilter}\n+import org.apache.hadoop.fs.s3.S3Credentials\n+import org.apache.hadoop.io.compress.{CompressionCodecFactory, SplittableCompressionCodec}\n+import org.apache.hadoop.mapred.{FileInputFormat, FileSplit, InputSplit, JobConf}\n+\n+import org.apache.spark.{SparkConf, Logging}\n+import org.apache.spark.annotation.DeveloperApi\n+import org.apache.spark.util.Utils\n+\n+/**\n+ * :: DeveloperApi ::\n+ * Contains util methods to interact with S3 from Spark.\n+ */\n+@DeveloperApi\n+object SparkS3Util extends Logging {\n+  val sparkConf = new SparkConf()\n+  val conf: Configuration = SparkHadoopUtil.get.newConfiguration(sparkConf)\n+\n+  private val s3ClientCache: Cache[String, AmazonS3Client] = CacheBuilder\n+    .newBuilder\n+    .concurrencyLevel(Runtime.getRuntime.availableProcessors)\n+    .build[String, AmazonS3Client]\n+\n+  // Flag to enable S3 bulk listing. It is true by default.\n+  private val S3_BULK_LISTING_ENABLED: String = \"spark.s3.bulk.listing.enabled\"\n+\n+  // Properties for AmazonS3Client. Default values should just work most of time.\n+  private val S3_CONNECT_TIMEOUT: String = \"spark.s3.connect.timeout\"\n+  private val S3_MAX_CONNECTIONS: String = \"spark.s3.max.connections\"\n+  private val S3_MAX_ERROR_RETRIES: String = \"spark.s3.max.error.retries\"\n+  private val S3_SOCKET_TIMEOUT: String = \"spark.s3.socket.timeout\"\n+  private val S3_SSL_ENABLED: String = \"spark.s3.ssl.enabled\"\n+  private val S3_USE_INSTANCE_CREDENTIALS: String = \"spark.s3.use.instance.credentials\"\n+\n+  // Ignore hidden files whose name starts with \"_\" and \".\", or ends with \"$folder$\".\n+  private val hiddenFileFilter = new PathFilter() {\n+    override def accept(p: Path): Boolean = {\n+      val name: String = p.getName()\n+      !name.startsWith(\"_\") && !name.startsWith(\".\") && !name.endsWith(\"$folder$\")\n+    }\n+  }\n+\n+  /**\n+   * Initialize AmazonS3Client per bucket. Since access permissions might be different from bucket\n+   * to bucket, it is necessary to initialize AmazonS3Client on a per bucket basis.\n+   */\n+  private def getS3Client(bucket: String): AmazonS3Client = {\n+    val sslEnabled: Boolean = sparkConf.getBoolean(S3_SSL_ENABLED, true)\n+    val maxErrorRetries: Int = sparkConf.getInt(S3_MAX_ERROR_RETRIES, 10)\n+    val connectTimeout: Int = sparkConf.getInt(S3_CONNECT_TIMEOUT, 5000)\n+    val socketTimeout: Int = sparkConf.getInt(S3_SOCKET_TIMEOUT, 5000)\n+    val maxConnections: Int = sparkConf.getInt(S3_MAX_CONNECTIONS, 5)\n+    val useInstanceCredentials: Boolean = sparkConf.getBoolean(S3_USE_INSTANCE_CREDENTIALS, false)\n+\n+    val clientConf: ClientConfiguration = new ClientConfiguration\n+    clientConf.setMaxErrorRetry(maxErrorRetries)\n+    clientConf.setProtocol(if (sslEnabled) Protocol.HTTPS else Protocol.HTTP)\n+    clientConf.setConnectionTimeout(connectTimeout)\n+    clientConf.setSocketTimeout(socketTimeout)\n+    clientConf.setMaxConnections(maxConnections)\n+\n+    // There are different ways of obtaining S3 bucket access. Try them in the following order:\n+    //   1) Check if user specified an IAM role. If so, use it.\n+    //   2) Check if instance is associated with an IAM role. If so, use it.\n+    //   3) Check if default IAM role is set in Hadoop conf. If so, use it.\n+    //   4) If no IAM role is found, search for an AWS key pair.\n+    // If no credentials are found, throw an exception.\n+    val s3RoleArn = Option(conf.get(\"aws.iam.role.arn\"))\n+    val s3RoleArnDefault = Option(conf.get(\"aws.iam.role.arn.default\"))\n+    val credentialsProvider: AWSCredentialsProvider =\n+      s3RoleArn match {\n+        case Some(role) =>\n+          logDebug(\"Use user-specified IAM role: \" + role)\n+          new STSAssumeRoleSessionCredentialsProvider(\n+            role, \"RoleSessionName-\" + Utils.random.nextInt)\n+        case None if useInstanceCredentials =>\n+          logDebug(\"Use IAM role associated with the instance\")\n+          new InstanceProfileCredentialsProvider\n+        case _ =>\n+          s3RoleArnDefault match {\n+            case Some(role) =>\n+              logDebug(\"Use default IAM role configured in Hadoop config: \" + role)\n+              new STSAssumeRoleSessionCredentialsProvider(\n+                role, \"RoleSessionName-\" + Utils.random.nextInt)\n+            case _ =>\n+              try {\n+                logDebug(\"Use AWS key pair\")\n+                val credentials: S3Credentials = new S3Credentials\n+                credentials.initialize(URI.create(bucket), conf)\n+                new StaticCredentialsProvider(\n+                  new BasicAWSCredentials(credentials.getAccessKey, credentials.getSecretAccessKey))\n+              } catch {\n+                case e: Exception => throw new RuntimeException(\"S3 credentials not configured\", e)\n+              }\n+        }\n+      }\n+\n+    val providerName: String = credentialsProvider.getClass.getSimpleName\n+    val s3ClientKey: String =\n+      if (s3RoleArn == null) providerName\n+      else providerName + \"_\" + s3RoleArn\n+\n+    Option(s3ClientCache.getIfPresent(s3ClientKey)).getOrElse {\n+      val newClient = new AmazonS3Client(credentialsProvider, clientConf)\n+      s3ClientCache.put(s3ClientKey, newClient)\n+      newClient\n+    }\n+  }\n+\n+  /**\n+   * Helper function to extract S3 object key name from the given path.\n+   */\n+  private def keyFromPath(path: Path): String = {\n+    Preconditions.checkArgument(path.isAbsolute, \"Path is not absolute: %s\", path)\n+    var key: String = Strings.nullToEmpty(path.toUri.getPath)\n+    if (key.startsWith(\"/\")) {\n+      key = key.substring(1)\n+    }\n+    if (key.endsWith(\"/\")) {\n+      key = key.substring(0, key.length - 1)\n+    }\n+    key\n+  }\n+\n+  /**\n+   * Helper function to convert S3ObjectSummary into FileStatus.\n+   */\n+  private def statusFromObjects(\n+      bucket: String,\n+      objects: util.List[S3ObjectSummary]): Iterator[FileStatus] = {\n+    val blockSize: Long = getS3BlockSize()\n+    val list: ArrayBuffer[FileStatus] = ArrayBuffer()\n+    for (obj: S3ObjectSummary <- objects.asScala) {\n+      if (!obj.getKey.endsWith(\"/\")) {\n+        val path = new Path(bucket + \"/\" + obj.getKey)\n+        if (hiddenFileFilter.accept(path)) {\n+          val status: FileStatus = new FileStatus(\n+            obj.getSize,\n+            false,\n+            1,\n+            blockSize,\n+            obj.getLastModified.getTime,\n+            path)\n+          list += status\n+        }\n+      }\n+    }\n+    list.iterator\n+  }\n+\n+  /**\n+   * For the given path, list S3 objects and return an iterator of returned FileStatuses.\n+   */\n+  @throws(classOf[AmazonClientException])\n+  @throws(classOf[AmazonServiceException])\n+  private def listPrefix(s3: AmazonS3Client, path: Path): Iterator[FileStatus] = {\n+    val uri: URI = path.toUri\n+    val key: String = keyFromPath(path)\n+    val request: ListObjectsRequest = new ListObjectsRequest()\n+      .withBucketName(uri.getAuthority)\n+      .withPrefix(key)\n+\n+    val listings = new AbstractSequentialIterator[ObjectListing](s3.listObjects(request)) {\n+      protected def computeNext(previous: ObjectListing): ObjectListing = {\n+        if (!previous.isTruncated) {\n+          return null\n+        }\n+        s3.listNextBatchOfObjects(previous)\n+      }\n+    }.asScala\n+\n+    val bucket: String = uri.getScheme + \"://\" + uri.getAuthority\n+    listings\n+      .map(listing => statusFromObjects(bucket, listing.getObjectSummaries))\n+      .reduceLeft(_ ++ _)\n+  }\n+\n+  /**\n+   * For the given list of paths, sequentially list S3 objects per path and combine returned\n+   * FileStatuses into a single array.\n+   */\n+  private def listStatus(s3: AmazonS3Client, paths: List[Path]): Array[FileStatus] = {\n+    val list: ArrayBuffer[FileStatus] = ArrayBuffer()\n+    paths.foreach { path =>\n+      val iterator = listPrefix(s3, path)\n+      while (iterator.hasNext) {\n+        list += iterator.next\n+      }\n+    }\n+    list.toArray\n+  }\n+\n+  /**\n+   * Find S3 block size from Hadoop conf. Try both s3 and s3n names.\n+   */\n+  private def getS3BlockSize(): Long = {\n+    val value = Option(conf.get(\"fs.s3.block.size\"))\n+      .getOrElse(conf.get(\"fs.s3n.block.size\", \"67108864\"))\n+    value.toLong\n+  }\n+\n+  /**\n+   * Find min split size from Hadoop conf. Try both Hadoop 1 and 2 names.\n+   */\n+  private def getMinSplitSize(): Long = {\n+    val value = Option(conf.get(\"mapred.min.split.size\"))\n+      .getOrElse(conf.get(\"mapreduce.input.fileinputformat.split.minsize\", \"134217728\"))\n+    value.toLong\n+  }\n+\n+  /**\n+   * Return whether the given path is an S3 path or not.\n+   */\n+  private def isS3Path(path: Path): Boolean = {\n+    Option(path.toUri.getScheme).exists(_.toUpperCase.startsWith(\"S3\"))\n+  }\n+\n+  /**\n+   * Return whether S3 bulk listing can be enabled or not.\n+   */\n+  def s3BulkListingEnabled(jobConf: JobConf): Boolean = {\n+    val enabledByUser = sparkConf.getBoolean(S3_BULK_LISTING_ENABLED, true)","path":"core/src/main/scala/org/apache/spark/deploy/SparkS3Util.scala","position":259,"original_position":259,"commit_id":"c1fa9ce06b5bace1be7ff702e2ee0bc223311076","original_commit_id":"c1fa9ce06b5bace1be7ff702e2ee0bc223311076","user":{"login":"yhuai","id":2072857,"avatar_url":"https://avatars.githubusercontent.com/u/2072857?v=3","gravatar_id":"","url":"https://api.github.com/users/yhuai","html_url":"https://github.com/yhuai","followers_url":"https://api.github.com/users/yhuai/followers","following_url":"https://api.github.com/users/yhuai/following{/other_user}","gists_url":"https://api.github.com/users/yhuai/gists{/gist_id}","starred_url":"https://api.github.com/users/yhuai/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yhuai/subscriptions","organizations_url":"https://api.github.com/users/yhuai/orgs","repos_url":"https://api.github.com/users/yhuai/repos","events_url":"https://api.github.com/users/yhuai/events{/privacy}","received_events_url":"https://api.github.com/users/yhuai/received_events","type":"User","site_admin":false},"body":"How about we set the default value to `false`?","created_at":"2015-10-29T20:12:47Z","updated_at":"2015-10-29T20:12:47Z","html_url":"https://github.com/apache/spark/pull/8512#discussion_r43437967","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/8512","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/43437967"},"html":{"href":"https://github.com/apache/spark/pull/8512#discussion_r43437967"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/8512"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/8512","id":43672915,"html_url":"https://github.com/apache/spark/pull/8512","diff_url":"https://github.com/apache/spark/pull/8512.diff","patch_url":"https://github.com/apache/spark/pull/8512.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/8512","number":8512,"state":"open","locked":false,"title":"[SPARK-9926] [SPARK-10340] [SQL] Use S3 bulk listing for S3-backed Hive tables","user":{"login":"piaozhexiu","id":179618,"avatar_url":"https://avatars.githubusercontent.com/u/179618?v=3","gravatar_id":"","url":"https://api.github.com/users/piaozhexiu","html_url":"https://github.com/piaozhexiu","followers_url":"https://api.github.com/users/piaozhexiu/followers","following_url":"https://api.github.com/users/piaozhexiu/following{/other_user}","gists_url":"https://api.github.com/users/piaozhexiu/gists{/gist_id}","starred_url":"https://api.github.com/users/piaozhexiu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/piaozhexiu/subscriptions","organizations_url":"https://api.github.com/users/piaozhexiu/orgs","repos_url":"https://api.github.com/users/piaozhexiu/repos","events_url":"https://api.github.com/users/piaozhexiu/events{/privacy}","received_events_url":"https://api.github.com/users/piaozhexiu/received_events","type":"User","site_admin":false},"body":"This PR addresses both [SPARK-9926](https://issues.apache.org/jira/browse/SPARK-9926) and [SPARK-10340](https://issues.apache.org/jira/browse/SPARK-10340).\r\n\r\n[SPARK-9926](https://issues.apache.org/jira/browse/SPARK-9926) makes it possible to calculate input splits for multiple partitions all together in `TableReader` and cache them in `HadoopRDD`. This improves performance because file listing can be done in parallel by taking advantage of `mapreduce.input.fileinputformat.list-status.num-threads`. This optimization can be enabled by a new property `spark.sql.hive.parallelFileListing`.\r\n\r\n[SPARK-10340](https://issues.apache.org/jira/browse/SPARK-10340) makes it possible to list S3 files using `AmazonS3Client` instead of using `S3N`. This improves performance particularly when listing a large number of S3 files. Note that since S3 bulk listing may list more objects than what meets predicates, they must be additionally filtered after listed. This is done by comparing prefixes of S3 objects against original input paths in `JobConf`. This optimization can be enabled by a new property ` spark.sql.hive.s3BulkListing`.\r\n\r\nHere is my benchmark results for the following queries-\r\n```\r\n1 partition: select * from nccp_log where dateint=20150601 and hour=0 limit 10;\r\n24 partitions: select * from nccp_log where dateint=20150601 limit 10;\r\n240 partitions: select * from nccp_log where dateint>=20150601 and dateint<=20150610 limit 10;\r\n720 partitions: select * from nccp_log where dateint>=20150601 and dateint<=20150630 limit 10;\r\n```\r\n\r\n\\# of files|\\# of partitions|1.5 rc2 (a) |SPARK-9926 w/ 10 threads (b) |SPARK-10340 (c)|\r\n------------|-------------------|---------|------|-----\r\n972 | 1 | 38s | 36s | 27s\r\n13646 | 24 | 354s | 41s | 30s\r\n136222 | 240 | 1h | 321s | 195s\r\n445377 | 720 | 3h | 1194s | 274s\r\n\r\n![image](https://cloud.githubusercontent.com/assets/179618/9556501/fda27662-4d89-11e5-8ce1-0b5d7bab02d1.png)\r\n\r\nThe performance gains come from two factors-\r\n* S3 bulk listing.\r\n* Skipping data locality hints computation for S3 objects. This is possible because S3 objects have no data locality to begin with.\r\n\r\nThe code for S3 bulk listing is fully  implemented in `SparkS3Util`. My code is heavily based on Presto [S3FileSystem](https://github.com/facebook/presto/blob/master/presto-hive/src/main/java/com/facebook/presto/hive/PrestoS3FileSystem.java) and Hadoop [FileInputFormat](https://hadoop.apache.org/docs/current/api/org/apache/hadoop/mapred/FileInputFormat.html).","created_at":"2015-08-28T21:15:19Z","updated_at":"2015-10-29T20:12:47Z","closed_at":null,"merged_at":null,"merge_commit_sha":"49d8544183a46570de46098ff272107ca66105e5","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/8512/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/8512/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/8512/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/c1fa9ce06b5bace1be7ff702e2ee0bc223311076","head":{"label":"piaozhexiu:SPARK-10340","ref":"SPARK-10340","sha":"c1fa9ce06b5bace1be7ff702e2ee0bc223311076","user":{"login":"piaozhexiu","id":179618,"avatar_url":"https://avatars.githubusercontent.com/u/179618?v=3","gravatar_id":"","url":"https://api.github.com/users/piaozhexiu","html_url":"https://github.com/piaozhexiu","followers_url":"https://api.github.com/users/piaozhexiu/followers","following_url":"https://api.github.com/users/piaozhexiu/following{/other_user}","gists_url":"https://api.github.com/users/piaozhexiu/gists{/gist_id}","starred_url":"https://api.github.com/users/piaozhexiu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/piaozhexiu/subscriptions","organizations_url":"https://api.github.com/users/piaozhexiu/orgs","repos_url":"https://api.github.com/users/piaozhexiu/repos","events_url":"https://api.github.com/users/piaozhexiu/events{/privacy}","received_events_url":"https://api.github.com/users/piaozhexiu/received_events","type":"User","site_admin":false},"repo":{"id":31330425,"name":"spark","full_name":"piaozhexiu/spark","owner":{"login":"piaozhexiu","id":179618,"avatar_url":"https://avatars.githubusercontent.com/u/179618?v=3","gravatar_id":"","url":"https://api.github.com/users/piaozhexiu","html_url":"https://github.com/piaozhexiu","followers_url":"https://api.github.com/users/piaozhexiu/followers","following_url":"https://api.github.com/users/piaozhexiu/following{/other_user}","gists_url":"https://api.github.com/users/piaozhexiu/gists{/gist_id}","starred_url":"https://api.github.com/users/piaozhexiu/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/piaozhexiu/subscriptions","organizations_url":"https://api.github.com/users/piaozhexiu/orgs","repos_url":"https://api.github.com/users/piaozhexiu/repos","events_url":"https://api.github.com/users/piaozhexiu/events{/privacy}","received_events_url":"https://api.github.com/users/piaozhexiu/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/piaozhexiu/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/piaozhexiu/spark","forks_url":"https://api.github.com/repos/piaozhexiu/spark/forks","keys_url":"https://api.github.com/repos/piaozhexiu/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/piaozhexiu/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/piaozhexiu/spark/teams","hooks_url":"https://api.github.com/repos/piaozhexiu/spark/hooks","issue_events_url":"https://api.github.com/repos/piaozhexiu/spark/issues/events{/number}","events_url":"https://api.github.com/repos/piaozhexiu/spark/events","assignees_url":"https://api.github.com/repos/piaozhexiu/spark/assignees{/user}","branches_url":"https://api.github.com/repos/piaozhexiu/spark/branches{/branch}","tags_url":"https://api.github.com/repos/piaozhexiu/spark/tags","blobs_url":"https://api.github.com/repos/piaozhexiu/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/piaozhexiu/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/piaozhexiu/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/piaozhexiu/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/piaozhexiu/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/piaozhexiu/spark/languages","stargazers_url":"https://api.github.com/repos/piaozhexiu/spark/stargazers","contributors_url":"https://api.github.com/repos/piaozhexiu/spark/contributors","subscribers_url":"https://api.github.com/repos/piaozhexiu/spark/subscribers","subscription_url":"https://api.github.com/repos/piaozhexiu/spark/subscription","commits_url":"https://api.github.com/repos/piaozhexiu/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/piaozhexiu/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/piaozhexiu/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/piaozhexiu/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/piaozhexiu/spark/contents/{+path}","compare_url":"https://api.github.com/repos/piaozhexiu/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/piaozhexiu/spark/merges","archive_url":"https://api.github.com/repos/piaozhexiu/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/piaozhexiu/spark/downloads","issues_url":"https://api.github.com/repos/piaozhexiu/spark/issues{/number}","pulls_url":"https://api.github.com/repos/piaozhexiu/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/piaozhexiu/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/piaozhexiu/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/piaozhexiu/spark/labels{/name}","releases_url":"https://api.github.com/repos/piaozhexiu/spark/releases{/id}","created_at":"2015-02-25T18:57:29Z","updated_at":"2015-04-02T00:01:52Z","pushed_at":"2015-10-29T16:53:59Z","git_url":"git://github.com/piaozhexiu/spark.git","ssh_url":"git@github.com:piaozhexiu/spark.git","clone_url":"https://github.com/piaozhexiu/spark.git","svn_url":"https://github.com/piaozhexiu/spark","homepage":null,"size":122958,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"6a2359ff1f7ad2233af2c530313d6ec2ecf70d19","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-10-29T20:10:57Z","pushed_at":"2015-10-29T20:09:38Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1647197,"stargazers_count":5943,"watchers_count":5943,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":5533,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":362,"forks":5533,"open_issues":362,"watchers":5943,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/8512"},"html":{"href":"https://github.com/apache/spark/pull/8512"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/8512"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/8512/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/8512/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/8512/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/c1fa9ce06b5bace1be7ff702e2ee0bc223311076"}}}}