{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/28468204","id":28468204,"diff_hunk":"@@ -0,0 +1,630 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.deploy.history.yarn\n+\n+import java.net.{ConnectException, URL}\n+import java.util.concurrent.LinkedBlockingQueue\n+import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}\n+\n+import scala.collection.mutable.LinkedList\n+\n+import org.apache.hadoop.conf.Configuration\n+import org.apache.hadoop.security.UserGroupInformation\n+import org.apache.hadoop.service.{AbstractService, Service}\n+import org.apache.hadoop.yarn.api.records.ApplicationId\n+import org.apache.hadoop.yarn.api.records.timeline.{TimelineDomain, TimelineEntity, TimelinePutResponse}\n+import org.apache.hadoop.yarn.client.api.TimelineClient\n+import org.apache.hadoop.yarn.conf.YarnConfiguration\n+import org.apache.hadoop.yarn.exceptions.YarnException\n+\n+import org.apache.spark.deploy.history.yarn.YarnTimelineUtils._\n+import org.apache.spark.scheduler._\n+import org.apache.spark.scheduler.cluster.YarnService\n+import org.apache.spark.{Logging, SparkContext}\n+\n+/**\n+ * Implements a Hadoop service with the init/start logic replaced by that\n+ * of the YarnService.\n+ * <p>\n+ * As <code>AbstractService</code> implements <code>close()</code>, routing\n+ * to its <code>stop</code> method, calling <code>close()</code> is sufficient\n+ * to stop the service instance.\n+ * <p>\n+ * However, when registered to receive spark events, the service will continue to\n+ * receive them until the spark context is stopped. Events received when this service\n+ * is in a <code>STOPPED</code> state will be discarded.\n+ */\n+private[spark] class YarnHistoryService  extends AbstractService(\"History Service\")\n+  with YarnService with Logging {\n+\n+  private var sc: SparkContext = _\n+  private var appId: ApplicationId = _\n+  private var timelineClient: Option[TimelineClient] = None\n+  private var listener: YarnEventListener = _\n+  private var appName: String = null\n+  private var userName: String = null\n+  private var startTime: Long = _\n+\n+  private var batchSize: Int = YarnHistoryService.DEFAULT_BATCH_SIZE\n+\n+  // enqueue event to avoid blocking on main thread.\n+  private val eventQueue = new LinkedBlockingQueue[TimestampEvent]\n+  // cache layer to handle timeline client failure.\n+  private var entityList = new LinkedList[TimelineEntity]\n+  private var curEntity: Option[TimelineEntity] = None\n+  // Do we have enough information filled for the entity\n+  private var bAppStart = false\n+  private var bAppEnd = false\n+  // How many event we saved\n+  private var curEventNum = 0\n+  private val eventsProcessed: AtomicInteger = new AtomicInteger(0)\n+  private val eventsQueued: AtomicInteger = new AtomicInteger(0)\n+  private val eventPostFailures: AtomicInteger = new AtomicInteger(0)\n+  private val flushCount = new AtomicInteger(0)\n+  private var eventHandlingThread: Thread = null\n+  private val stopped: AtomicBoolean = new AtomicBoolean(true)\n+  private final val lock: AnyRef = new AnyRef\n+  private var maxTimeToWaitOnShutdown: Long = YarnHistoryService.SHUTDOWN_WAIT_TIME\n+  private var clientFailure = 0\n+  private var domainId: String = null\n+  private var timelineWebappAddress: URL = _\n+\n+\n+  /**\n+   * Create a timeline client\n+   * @return the timeline client\n+   */\n+  def createTimelineClient = {\n+    clientFailure += 1\n+    logInfo(\"Creating timelineClient \" + clientFailure)\n+    val client = TimelineClient.createTimelineClient()\n+    client.init(sc.hadoopConfiguration)\n+    client.start\n+    timelineClient = Some(client)\n+    client\n+  }\n+\n+  /**\n+   * Get the timeline client; this will create it if needed\n+   * @return the client\n+   */\n+  def getTimelineClient: TimelineClient = {\n+    timelineClient.getOrElse(createTimelineClient)\n+  }\n+\n+  def getEventsProcessed: Int = {\n+    eventsProcessed.get()\n+  }\n+\n+  def getEventsQueued: Int = {\n+    eventsQueued.get()\n+  }\n+\n+  def getQueueSize: Int = {\n+    eventQueue.size()\n+  }\n+\n+  def getBatchSize: Int = {\n+    batchSize\n+  }\n+\n+  def getEventPostFailures: Int = {\n+    eventPostFailures.get()\n+  }\n+\n+  /**\n+   * Reset the timeline client\n+   * <p>\n+   * 1. Stop the timeline client service if running.\n+   * 2. set the <code>timelineClient</code> field to <code>None</code>\n+   */\n+  def resetTimelineClient(): Unit = {\n+    stopOptionalService(timelineClient)\n+    timelineClient = None\n+  }\n+\n+  /**\n+   * Split a comma separated String, filter out any empty items, and return a Set of strings\n+   */\n+  private def stringToSet(list: String): Set[String] = {\n+    list.split(',').map(_.trim).filter(!_.isEmpty).toSet\n+  }\n+\n+  private def createTimelineDomain(): String = {\n+    val sparkConf = sc.getConf\n+    val aclsOn = sparkConf.getOption(\"spark.acls.enable\").getOrElse(\n+      sparkConf.get(\"spark.ui.acls.enable\", \"false\")).toBoolean\n+    if (!aclsOn) {\n+      return null\n+    }\n+    val predefDomain = sparkConf.getOption(\"spark.ui.domain\")\n+    if (predefDomain.isDefined) {\n+      domainId = predefDomain.get\n+      return null\n+    }\n+    val current = UserGroupInformation.getCurrentUser.getShortUserName\n+    val adminAcls  = stringToSet(sparkConf.get(\"spark.admin.acls\", \"\"))\n+    val viewAcls = stringToSet(sparkConf.get(\"spark.ui.view.acls\", \"\"))\n+    val modifyAcls = stringToSet(sparkConf.get(\"spark.modify.acls\", \"\"))\n+\n+    val readers = (adminAcls ++ modifyAcls ++ viewAcls).foldLeft(current)(_ + \" \" + _)\n+    val writers = (adminAcls ++ modifyAcls).foldLeft(current)(_ + \" \" + _)\n+    var tmpId = YarnHistoryService.DOMAIN_ID_PREFIX + appId\n+    logInfo(\"Creating domain \" + tmpId + \" with  readers: \"\n+      + readers + \" and writers:\" + writers)\n+    val timelineDomain = new TimelineDomain()\n+    timelineDomain.setId(tmpId)\n+\n+    timelineDomain.setReaders(readers)\n+    timelineDomain.setWriters(writers)\n+    try {\n+      getTimelineClient.putDomain(timelineDomain)\n+    } catch {\n+      case e: YarnException => {\n+        logError(\"cannot create the domain\")\n+        // fallback to default\n+        tmpId = null\n+      }\n+    }\n+    tmpId\n+  }\n+\n+  def start(context: SparkContext, id: ApplicationId): Boolean = {\n+    sc = context\n+    appId = id\n+    val yarnConf = new YarnConfiguration(context.hadoopConfiguration)\n+    // the init() operation checks the state machine & prevents invocation out of sequence\n+    init(yarnConf)\n+    batchSize = sc.conf.getInt(YarnHistoryService.BATCH_SIZE, batchSize)\n+\n+    start()\n+    if (yarnConf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,\n+      YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {\n+      true\n+    } else {\n+      logInfo(\"Yarn timeline service not available, disabling client.\")\n+      false\n+    }\n+  }\n+\n+  override protected def serviceStart {\n+    super.serviceStart()\n+    val conf: Configuration = getConfig\n+    if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,\n+                             YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {\n+      timelineWebappAddress = timelineWebappUri(conf).toURL\n+      createTimelineClient\n+      domainId = createTimelineDomain\n+      eventHandlingThread = new Thread(new Dequeue(), \"HistoryEventHandlingThread\")\n+      eventHandlingThread.start\n+    } else {\n+      logInfo(\"Yarn timeline service not available\")\n+    }\n+    // irrespective of state, hook up to the listener\n+    registerListener\n+    logInfo(s\"$this\")\n+  }\n+\n+  /**\n+   * Return a summary of the service state to help diagnose problems\n+   * during test runs, possibly even production\n+   * @return a summary of the current service state\n+   */\n+  override def toString: String = {\n+    super.toString +\n+        s\" endpoint=$timelineWebappAddress\" +\n+        s\" bonded to ATS=$bondedToATS\" +\n+        s\" listening=$listening\" +\n+        s\" batchSize=$batchSize\" +\n+        s\" flush count=$getFlushCount()\" +\n+        s\" queue size=$getEventsQueued processed=$getEventsProcessed\" +\n+        s\" post failures = $getEventPostFailures\"\n+  }\n+\n+  def listening: Boolean = {\n+    listener != null;\n+  }\n+\n+  /**\n+   * Is the service hooked up to an ATS server. This does not\n+   * check the validity of the link, only whether or not the service\n+   * has been set up to talk to ATS.\n+   * @return true if the service has a timeline client\n+   */\n+  def bondedToATS: Boolean = {\n+    timelineClient != None;\n+  }\n+\n+  /**\n+   * Add the listener if it is not disabled.\n+   * This is accessible in the same package purely for testing\n+   * @return true if the register was enabled\n+   */\n+  private [yarn] def registerListener: Boolean = {\n+    assert(sc != null, \"Null context\")\n+    if (sc.conf.getBoolean(YarnHistoryService.REGISTER_LISTENER, true)) {\n+      log.debug(\"Registering listener to spark context\")","path":"yarn/history/src/main/scala/org/apache/spark/deploy/history/yarn/YarnHistoryService.scala","position":261,"original_position":261,"commit_id":"8446042b62369382b312dedcad047b27af3ccd54","original_commit_id":"8446042b62369382b312dedcad047b27af3ccd54","user":{"login":"zhzhan","id":4451616,"avatar_url":"https://avatars.githubusercontent.com/u/4451616?v=3","gravatar_id":"","url":"https://api.github.com/users/zhzhan","html_url":"https://github.com/zhzhan","followers_url":"https://api.github.com/users/zhzhan/followers","following_url":"https://api.github.com/users/zhzhan/following{/other_user}","gists_url":"https://api.github.com/users/zhzhan/gists{/gist_id}","starred_url":"https://api.github.com/users/zhzhan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zhzhan/subscriptions","organizations_url":"https://api.github.com/users/zhzhan/orgs","repos_url":"https://api.github.com/users/zhzhan/repos","events_url":"https://api.github.com/users/zhzhan/events{/privacy}","received_events_url":"https://api.github.com/users/zhzhan/received_events","type":"User","site_admin":false},"body":"use logDebug","created_at":"2015-04-15T22:09:50Z","updated_at":"2015-04-15T22:09:50Z","html_url":"https://github.com/apache/spark/pull/5423#discussion_r28468204","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/5423","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/28468204"},"html":{"href":"https://github.com/apache/spark/pull/5423#discussion_r28468204"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/5423"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/5423","id":32861761,"html_url":"https://github.com/apache/spark/pull/5423","diff_url":"https://github.com/apache/spark/pull/5423.diff","patch_url":"https://github.com/apache/spark/pull/5423.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/5423","number":5423,"state":"open","locked":false,"title":"SPARK-1537 [WiP] Application Timeline Server integration","user":{"login":"steveloughran","id":162090,"avatar_url":"https://avatars.githubusercontent.com/u/162090?v=3","gravatar_id":"","url":"https://api.github.com/users/steveloughran","html_url":"https://github.com/steveloughran","followers_url":"https://api.github.com/users/steveloughran/followers","following_url":"https://api.github.com/users/steveloughran/following{/other_user}","gists_url":"https://api.github.com/users/steveloughran/gists{/gist_id}","starred_url":"https://api.github.com/users/steveloughran/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/steveloughran/subscriptions","organizations_url":"https://api.github.com/users/steveloughran/orgs","repos_url":"https://api.github.com/users/steveloughran/repos","events_url":"https://api.github.com/users/steveloughran/events{/privacy}","received_events_url":"https://api.github.com/users/steveloughran/received_events","type":"User","site_admin":false},"body":"This a snapshot of the work in progress. It's a superset of zhzhan's work, compiling against the master branch and with a lot more tests.\r\n\r\n1. there's now a hadoop-2.6 profile in the root pom; this is used to include the yarn/history source tree containing the new sources and tests\r\n1. As well as Marcelo's mock tests (which are all working happily), there's now some functional tests which use the timeline client to marshall a spark event, push it to an in-VM timeline server, then verify it's there/unmarshallable.\r\n1. There's some improved robustness in the production-side code.\r\n\r\nThis is not complete; i've pushed it up to show the current status. \r\n\r\nWhat's needed now is the REST operations to retrieve data, tests that everything wired up works.\r\n\r\n## Production Code\r\n\r\n1. I'm going to add a Jersey client to talk to ATS, avoiding any use of private hadoop classes, and resulting in code that works with Hadoop 2.6+\r\n1. I think the code to do PUT operations needs to be made async, so transient ATS outages don't block things. It's already queued & running in its own thread, but its posting in a synchronized section.\r\n\r\n## Tests\r\n\r\n* listening to spark context -> posted events in the ATS\r\n* retrieving events from the ATS via GET\r\n* joined up history provider\r\n\r\n## Hadoop side\r\n\r\nA statement of the compatibility policy of the current ATS read and write REST API","created_at":"2015-04-08T13:52:10Z","updated_at":"2015-04-15T22:09:50Z","closed_at":null,"merged_at":null,"merge_commit_sha":"20e46b9a8e30a8076b18e92db67089b8e5bdc189","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/5423/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/5423/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/5423/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/8446042b62369382b312dedcad047b27af3ccd54","head":{"label":"steveloughran:stevel/feature/SPARK-1537-ATS","ref":"stevel/feature/SPARK-1537-ATS","sha":"8446042b62369382b312dedcad047b27af3ccd54","user":{"login":"steveloughran","id":162090,"avatar_url":"https://avatars.githubusercontent.com/u/162090?v=3","gravatar_id":"","url":"https://api.github.com/users/steveloughran","html_url":"https://github.com/steveloughran","followers_url":"https://api.github.com/users/steveloughran/followers","following_url":"https://api.github.com/users/steveloughran/following{/other_user}","gists_url":"https://api.github.com/users/steveloughran/gists{/gist_id}","starred_url":"https://api.github.com/users/steveloughran/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/steveloughran/subscriptions","organizations_url":"https://api.github.com/users/steveloughran/orgs","repos_url":"https://api.github.com/users/steveloughran/repos","events_url":"https://api.github.com/users/steveloughran/events{/privacy}","received_events_url":"https://api.github.com/users/steveloughran/received_events","type":"User","site_admin":false},"repo":{"id":32387210,"name":"spark","full_name":"steveloughran/spark","owner":{"login":"steveloughran","id":162090,"avatar_url":"https://avatars.githubusercontent.com/u/162090?v=3","gravatar_id":"","url":"https://api.github.com/users/steveloughran","html_url":"https://github.com/steveloughran","followers_url":"https://api.github.com/users/steveloughran/followers","following_url":"https://api.github.com/users/steveloughran/following{/other_user}","gists_url":"https://api.github.com/users/steveloughran/gists{/gist_id}","starred_url":"https://api.github.com/users/steveloughran/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/steveloughran/subscriptions","organizations_url":"https://api.github.com/users/steveloughran/orgs","repos_url":"https://api.github.com/users/steveloughran/repos","events_url":"https://api.github.com/users/steveloughran/events{/privacy}","received_events_url":"https://api.github.com/users/steveloughran/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/steveloughran/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/steveloughran/spark","forks_url":"https://api.github.com/repos/steveloughran/spark/forks","keys_url":"https://api.github.com/repos/steveloughran/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/steveloughran/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/steveloughran/spark/teams","hooks_url":"https://api.github.com/repos/steveloughran/spark/hooks","issue_events_url":"https://api.github.com/repos/steveloughran/spark/issues/events{/number}","events_url":"https://api.github.com/repos/steveloughran/spark/events","assignees_url":"https://api.github.com/repos/steveloughran/spark/assignees{/user}","branches_url":"https://api.github.com/repos/steveloughran/spark/branches{/branch}","tags_url":"https://api.github.com/repos/steveloughran/spark/tags","blobs_url":"https://api.github.com/repos/steveloughran/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/steveloughran/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/steveloughran/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/steveloughran/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/steveloughran/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/steveloughran/spark/languages","stargazers_url":"https://api.github.com/repos/steveloughran/spark/stargazers","contributors_url":"https://api.github.com/repos/steveloughran/spark/contributors","subscribers_url":"https://api.github.com/repos/steveloughran/spark/subscribers","subscription_url":"https://api.github.com/repos/steveloughran/spark/subscription","commits_url":"https://api.github.com/repos/steveloughran/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/steveloughran/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/steveloughran/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/steveloughran/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/steveloughran/spark/contents/{+path}","compare_url":"https://api.github.com/repos/steveloughran/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/steveloughran/spark/merges","archive_url":"https://api.github.com/repos/steveloughran/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/steveloughran/spark/downloads","issues_url":"https://api.github.com/repos/steveloughran/spark/issues{/number}","pulls_url":"https://api.github.com/repos/steveloughran/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/steveloughran/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/steveloughran/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/steveloughran/spark/labels{/name}","releases_url":"https://api.github.com/repos/steveloughran/spark/releases{/id}","created_at":"2015-03-17T10:36:33Z","updated_at":"2015-03-17T10:36:52Z","pushed_at":"2015-04-15T21:52:18Z","git_url":"git://github.com/steveloughran/spark.git","ssh_url":"git@github.com:steveloughran/spark.git","clone_url":"https://github.com/steveloughran/spark.git","svn_url":"https://github.com/steveloughran/spark","homepage":null,"size":99092,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"15e0d2bd1304db62fad286c1bb687e87c361e16c","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-04-15T22:08:50Z","pushed_at":"2015-04-15T21:52:21Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1499677,"stargazers_count":3622,"watchers_count":3622,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3126,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":287,"forks":3126,"open_issues":287,"watchers":3622,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/5423"},"html":{"href":"https://github.com/apache/spark/pull/5423"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/5423"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/5423/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/5423/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/5423/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/8446042b62369382b312dedcad047b27af3ccd54"}}}}