{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/40606368","id":40606368,"diff_hunk":"@@ -0,0 +1,1048 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.deploy.history.yarn\n+\n+import java.net.{ConnectException, URI}\n+import java.util.concurrent.atomic.{AtomicLong, AtomicBoolean, AtomicInteger}\n+import java.util.concurrent.{TimeUnit, LinkedBlockingDeque}\n+\n+import scala.collection.JavaConverters._\n+import scala.collection.mutable\n+\n+import org.apache.hadoop.security.UserGroupInformation\n+import org.apache.hadoop.yarn.api.records.timeline.{TimelineDomain, TimelineEntity, TimelineEvent, TimelinePutResponse}\n+import org.apache.hadoop.yarn.api.records.{ApplicationAttemptId, ApplicationId}\n+import org.apache.hadoop.yarn.client.api.TimelineClient\n+import org.apache.hadoop.yarn.conf.YarnConfiguration\n+\n+import org.apache.spark.deploy.history.yarn.YarnTimelineUtils._\n+import org.apache.spark.scheduler._\n+import org.apache.spark.scheduler.cluster.{YarnExtensionService, YarnExtensionServiceBinding}\n+import org.apache.spark.util.{SystemClock, Utils}\n+import org.apache.spark.{Logging, SparkContext}\n+\n+/**\n+ * A Yarn Extension Service to post lifecycle events to a registered\n+ * YARN Timeline Server.\n+ */\n+private[spark] class YarnHistoryService extends YarnExtensionService with Logging {\n+\n+  import org.apache.spark.deploy.history.yarn.YarnHistoryService._\n+\n+  /** Simple state model implemented in an atomic integer */\n+  private val _serviceState = new AtomicInteger(CreatedState)\n+\n+  def serviceState: Int = {\n+    _serviceState.get()\n+  }\n+  def enterState(state: Int): Int = {\n+    logDebug(s\"Entering state $state from $serviceState\")\n+    _serviceState.getAndSet(state)\n+  }\n+\n+  /**\n+   * Spark context; valid once started\n+   */\n+  private var sparkContext: SparkContext = _\n+\n+  /** YARN configuration from the spark context */\n+  private var config: YarnConfiguration = _\n+\n+  /** application ID. */\n+  private var _applicationId: ApplicationId = _\n+\n+  /** attempt ID this will be null if the service is started in yarn-client mode */\n+  private var _attemptId: Option[ApplicationAttemptId] = None\n+\n+  /** YARN timeline client */\n+  private var _timelineClient: Option[TimelineClient] = None\n+\n+  /** registered event listener */\n+  private var listener: Option[YarnEventListener] = None\n+\n+  /** Application name  from the spark start event */\n+  private var applicationName: String = _\n+\n+  /** Application ID*/\n+  private var sparkApplicationId: Option[String] = None\n+\n+  /** Optional Attempt ID from the spark start event */\n+  private var sparkApplicationAttemptId: Option[String] = None\n+\n+  /** user name as derived from `SPARK_USER` env var or `UGI` */\n+  private var userName = Utils.getCurrentUserName()\n+\n+  /** Clock for recording time */\n+  private val clock = new SystemClock()\n+\n+  /**\n+   * Start time of the application, as received in the start event.\n+   */\n+  private var startTime: Long = _\n+\n+  /**\n+   * Start time of the application, as received in the end event.\n+   */\n+  private var endTime: Long = _\n+\n+  /** number of events to batch up before posting*/\n+  private var _batchSize = DEFAULT_BATCH_SIZE\n+\n+  /** queue of entities to asynchronously post, plus the number of events in each entry */\n+  private var _entityQueue = new LinkedBlockingDeque[TimelineEntity]()\n+\n+  /** limit on the total number of events permitted */\n+  private var _postQueueLimit = DEFAULT_POST_QUEUE_LIMIT\n+\n+  /**\n+   * List of events which will be pulled into a timeline\n+   * entity when created\n+   */\n+  private var pendingEvents = new mutable.LinkedList[TimelineEvent]()\n+\n+  private var applicationStartEvent: Option[SparkListenerApplicationStart] = None\n+  private var applicationEndEvent: Option[SparkListenerApplicationEnd] = None\n+\n+  /** Has a start event been processed? */\n+  private val appStartEventProcessed = new AtomicBoolean(false)\n+\n+  /* has the application event event been processed */\n+  private val appEndEventProcessed = new AtomicBoolean(false)\n+\n+  /** counter of events processed -that is have been through handleEvent()*/\n+  private val _eventsProcessed = new AtomicLong(0)\n+\n+  /** counter of events queued. */\n+  private val _eventsQueued = new AtomicLong(0)\n+\n+  private val _entityPostAttempts = new AtomicLong(0)\n+  private val _entityPostSuccesses = new AtomicLong(0)\n+  /** how many entity postings failed? */\n+  private val _entityPostFailures = new AtomicLong(0)\n+  private val _eventsDropped = new AtomicLong(0)\n+\n+  /** how many flushes have taken place? */\n+  private val flushCount = new AtomicLong(0)\n+\n+  /** Event handler */\n+  private var eventHandlingThread: Option[Thread] = None\n+\n+  /**\n+   * Flag to indicate the thread is stopped; events aren't being\n+   * processed.\n+   */\n+  private val stopped = new AtomicBoolean(true)\n+\n+  /**\n+   * Boolean to track whether a thread is active or not, for tests to\n+   * monitor and see if the thread has completed.\n+   */\n+  private val postThreadActive = new AtomicBoolean(false)\n+\n+  /** How long to wait for shutdown before giving up */\n+  private var shutdownWaitTime = 0L\n+\n+  /**\n+   * What is the initial and incrementing interval for POST retries?\n+   */\n+  private var retryInterval = 0L\n+\n+  /** Domain ID for entities: may be null */\n+  private var domainId: Option[String] = None\n+\n+  /** URI to timeline web application -valid after `serviceStart()` */\n+  private var _timelineWebappAddress: URI = _\n+\n+  /**\n+   * Create a timeline client and start it. This does not update the\n+   * `timelineClient` field, though it does verify that the field\n+   * is unset.\n+   *\n+   * The method is private to the package so that tests can access it, which\n+   * some of the mock tests do to override the timeline client creation.\n+   * @return the timeline client\n+   */\n+  private[yarn] def createTimelineClient(): TimelineClient = {\n+    require(_timelineClient.isEmpty, \"timeline client already set\")\n+    YarnTimelineUtils.createTimelineClient(sparkContext)\n+  }\n+\n+  /**\n+   * Get the timeline client.\n+   * @return the client\n+   * @throws Exception if the timeline client is not currently running\n+   */\n+  def timelineClient: TimelineClient = {\n+    require(_timelineClient.isDefined)\n+    _timelineClient.get\n+  }\n+\n+  /**\n+   * Get the total number of events dropped due to the queue of\n+   * outstanding posts being too long\n+   * @return counter of events processed\n+   */\n+\n+  def eventsDropped: Long = {\n+    _eventsDropped.get()\n+  }\n+\n+  /**\n+   * Get the total number of processed events, those handled in the back-end thread without\n+   * being rejected\n+   * @return counter of events processed\n+   */\n+  def eventsProcessed: Long = {\n+    _eventsProcessed.get\n+  }\n+\n+  /**\n+   * Get the total number of events queued\n+   * @return the total event count\n+   */\n+  def eventsQueued: Long = {\n+    _eventsQueued.get\n+  }\n+\n+  /**\n+   * Get the current size of the queue\n+   * @return the current queue length\n+   */\n+  def getQueueSize: Int = {\n+    _entityQueue.size()\n+  }\n+\n+  /**\n+   * Get the current batch size\n+   * @return the batch size\n+   */\n+  def batchSize: Int = {\n+    _batchSize\n+  }\n+\n+  /**\n+   * Query the counter of attempts to post events\n+   * @return\n+   */\n+  def postAttempts: Long = _entityPostAttempts.get()\n+\n+  /**\n+   * Get the total number of failed post operations\n+   * @return counter of timeline post operations which failed\n+   */\n+  def postFailures: Long = {\n+    _entityPostFailures.get\n+  }\n+\n+  /**\n+   * Query the counter of post successes\n+   * @return the number of successful posts\n+   */\n+  def postSuccesses: Long = _entityPostSuccesses.get()\n+\n+  /**\n+   * is the asynchronous posting thread active?\n+   * @return true if the post thread has started; false if it has not yet/ever started, or\n+   *         if it has finished.\n+   */\n+  def isPostThreadActive: Boolean = {\n+    postThreadActive.get\n+  }\n+\n+  /**\n+   * The YARN application ID of this history service\n+   * @return the application ID provided when the service started\n+   */\n+  def applicationId: ApplicationId =  _applicationId\n+\n+  /**\n+   * The YARN attempt ID of this history service\n+   * @return the attempt ID provided when the service started\n+   */\n+  def attemptId: Option[ApplicationAttemptId] =  _attemptId\n+\n+  /**\n+   * Reset the timeline client\n+   * <p>\n+   * 1. Stop the timeline client service if running.\n+   * 2. set the `timelineClient` field to `None`\n+   */\n+  def stopTimelineClient(): Unit = {\n+    _timelineClient.foreach(_.stop())\n+    _timelineClient = None\n+  }\n+\n+  /**\n+   * Create the timeline domain.\n+   *\n+   * A Timeline Domain is a uniquely identified 'namespace' for accessing parts of the timeline.\n+   * Security levels are are managed at the domain level, so one is created if the\n+   * spark acls are enabled. Full access is then granted to the current user,\n+   * all users in the configuration options `\"spark.modify.acls\"` and `\"spark.admin.acls\"`;\n+   * read access to those users and those listed in `\"spark.ui.view.acls\"`\n+   *\n+   * @return an optional domain string. If `None`, then no domain was created.\n+   */\n+  private def createTimelineDomain(): Option[String] = {\n+    val sparkConf = sparkContext.getConf\n+    val aclsOn = sparkConf.getBoolean(\"spark.ui.acls.enable\",\n+        sparkConf.getBoolean(\"spark.acls.enable\", false))\n+    if (!aclsOn) {\n+      logDebug(\"ACLs are disabled; not creating the timeline domain\")\n+      return None\n+    }\n+    val predefDomain = sparkConf.getOption(TIMELINE_DOMAIN)\n+    if (predefDomain.isDefined) {\n+      logDebug(s\"Using predefined domain $predefDomain\")\n+      return predefDomain\n+    }\n+    val current = UserGroupInformation.getCurrentUser.getShortUserName\n+    val adminAcls  = stringToSet(sparkConf.get(\"spark.admin.acls\", \"\"))\n+    val viewAcls = stringToSet(sparkConf.get(\"spark.ui.view.acls\", \"\"))\n+    val modifyAcls = stringToSet(sparkConf.get(\"spark.modify.acls\", \"\"))\n+\n+    val readers = (Seq(current) ++ adminAcls ++ modifyAcls ++ viewAcls).mkString(\" \")\n+    val writers = (Seq(current) ++ adminAcls ++ modifyAcls).mkString(\" \")\n+    var domain = DOMAIN_ID_PREFIX + _applicationId\n+    logInfo(s\"Creating domain $domain with readers: $readers and writers: $writers\")\n+\n+    // create the timeline domain with the reader and writer permissions\n+    val timelineDomain = new TimelineDomain()\n+    timelineDomain.setId(domain)\n+    timelineDomain.setReaders(readers)\n+    timelineDomain.setWriters(writers)\n+    try {\n+      timelineClient.putDomain(timelineDomain)\n+      Some(domain)\n+    } catch {\n+      case e: Exception => {\n+        logError(s\"cannot create the domain $domain\", e)\n+        // fallback to default\n+        None\n+      }\n+    }\n+  }\n+\n+  /**\n+   * Start the service, calling the service's `init()` and `start()` actions in the\n+   * correct order\n+   * @param binding binding to the spark application and YARN\n+   */\n+  override def start(binding: YarnExtensionServiceBinding): Unit = {\n+    val oldstate = enterState(StartedState)\n+    if (oldstate != CreatedState) {\n+      // state model violation\n+      _serviceState.set(oldstate)\n+      throw new IllegalArgumentException(s\"Cannot start the service from state $oldstate\")\n+    }\n+    val context = binding.sparkContext\n+    val appId = binding.applicationId\n+    val attemptId = binding.attemptId\n+    require(context != null, \"Null context parameter\")\n+    bindToYarnApplication(appId, attemptId)\n+    this.sparkContext = context\n+\n+    this.config = new YarnConfiguration(context.hadoopConfiguration)\n+\n+    val sparkConf = sparkContext.conf\n+\n+    // work out the attempt ID from the YARN attempt ID. No attempt, assume \"1\".\n+    // this is assumed by the AM, which uses it when creating a path to an attempt\n+    val attempt1 = attemptId match {\n+      case Some(attempt) => attempt.getAttemptId.toString\n+      case None => CLIENT_BACKEND_ATTEMPT_ID\n+    }\n+    setContextAppAndAttemptInfo(Some(appId.toString), Some(attempt1))\n+    _batchSize = sparkConf.getInt(BATCH_SIZE, _batchSize)\n+    _postQueueLimit = sparkConf.getInt(POST_QUEUE_LIMIT, _postQueueLimit)\n+    retryInterval = 1000 * sparkConf.getTimeAsSeconds(POST_RETRY_INTERVAL,\n+      DEFAULT_POST_RETRY_INTERVAL)\n+    shutdownWaitTime = 1000 * sparkConf.getTimeAsSeconds(SHUTDOWN_WAIT_TIME,\n+      DEFAULT_SHUTDOWN_WAIT_TIME)\n+\n+\n+    if (timelineServiceEnabled) {\n+      _timelineWebappAddress = getTimelineEndpoint(config)\n+\n+      logInfo(s\"Starting $this\")\n+      logInfo(s\"Spark events will be published to the Timeline service at ${_timelineWebappAddress}\")\n+      _timelineClient = Some(createTimelineClient())\n+      domainId = createTimelineDomain()\n+      // declare that the processing is started\n+      stopped.set(false)\n+      eventHandlingThread = Some(new Thread(new Dequeue(), \"HistoryEventHandlingThread\"))\n+      eventHandlingThread.get.start()\n+    } else {\n+      logInfo(\"Timeline service is disabled\")\n+    }\n+    if (registerListener()) {\n+      logInfo(s\"History Service listening for events: $this\")\n+    } else {\n+      logInfo(s\"History Service is not listening for events: $this\")\n+    }\n+  }\n+\n+  /**\n+   * Check the service configuration to see if the timeline service is enabled\n+   * @return true if `YarnConfiguration.TIMELINE_SERVICE_ENABLED` is set.\n+   */\n+  def timelineServiceEnabled: Boolean = {\n+    YarnTimelineUtils.timelineServiceEnabled(config)\n+  }\n+\n+  /**\n+   * Return a summary of the service state to help diagnose problems\n+   * during test runs, possibly even production\n+   * @return a summary of the current service state\n+   */\n+  override def toString(): String = {\n+    s\"YarnHistoryService for application $applicationId attempt $attemptId;\" +\n+    s\" state=$serviceState;\" +\n+    s\" endpoint=${_timelineWebappAddress};\" +\n+    s\" bonded to ATS=$bondedToATS;\" +\n+    s\" listening=$listening;\" +\n+    s\" batchSize=$batchSize;\" +\n+    s\" flush count=$getFlushCount;\" +\n+    s\" total number queued=$eventsQueued, processed=$eventsProcessed;\" +\n+    s\" attempted entity posts=$postAttempts\" +\n+    s\" successful entity posts=$postSuccesses\" +\n+    s\" failed entity posts=$postFailures;\" +\n+    s\" events dropped=$eventsDropped;\" +\n+    s\" app start event received=$appStartEventProcessed;\" +\n+    s\" app end event received=$appEndEventProcessed;\"\n+  }\n+\n+  /**\n+   * Is the service listening to events from the spark context?\n+   * @return true if it has registered as a listener\n+   */\n+  def listening: Boolean = {\n+    listener.isDefined\n+  }\n+\n+  /**\n+   * Is the service hooked up to an ATS server. This does not\n+   * check the validity of the link, only whether or not the service\n+   * has been set up to talk to ATS.\n+   * @return true if the service has a timeline client\n+   */\n+  def bondedToATS: Boolean = {\n+    _timelineClient.isDefined\n+  }\n+\n+  /**\n+   * Set the YARN binding information. This is called during startup. It is private\n+   * to the package so that tests may update this data\n+   * @param appId YARN application ID\n+   * @param attemptId optional attempt ID\n+   */\n+  private[yarn] def bindToYarnApplication(appId: ApplicationId,\n+      attemptId: Option[ApplicationAttemptId]): Unit = {\n+    require(appId != null, \"Null appId parameter\")\n+    _applicationId = appId\n+    _attemptId = attemptId\n+  }\n+\n+  /**\n+   * Set the \"spark\" application and attempt information -the information\n+   * provided in the start event. The attempt ID here may be `None`; even\n+   * if set it may only be unique amongst the attempts of this application.\n+   * That is: not unique enough to be used as the entity ID\n+   * @param appId application ID\n+   * @param attemptId attempt ID\n+   */\n+  private def setContextAppAndAttemptInfo(appId: Option[String],","path":"yarn/history/src/main/scala/org/apache/spark/deploy/history/yarn/YarnHistoryService.scala","position":469,"original_position":469,"commit_id":"231301a943f449db1ccf46573f2e98a47034c797","original_commit_id":"231301a943f449db1ccf46573f2e98a47034c797","user":{"login":"vanzin","id":1694083,"avatar_url":"https://avatars.githubusercontent.com/u/1694083?v=3","gravatar_id":"","url":"https://api.github.com/users/vanzin","html_url":"https://github.com/vanzin","followers_url":"https://api.github.com/users/vanzin/followers","following_url":"https://api.github.com/users/vanzin/following{/other_user}","gists_url":"https://api.github.com/users/vanzin/gists{/gist_id}","starred_url":"https://api.github.com/users/vanzin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vanzin/subscriptions","organizations_url":"https://api.github.com/users/vanzin/orgs","repos_url":"https://api.github.com/users/vanzin/repos","events_url":"https://api.github.com/users/vanzin/events{/privacy}","received_events_url":"https://api.github.com/users/vanzin/received_events","type":"User","site_admin":false},"body":"I'm confused as to why you need both `bindToYarnApplication` and this. In what situation would `_applicationId` be different than `sparkApplicationId`? If that situation exists, it sounds like a bug to me.","created_at":"2015-09-28T21:13:26Z","updated_at":"2015-09-28T21:13:26Z","html_url":"https://github.com/apache/spark/pull/8744#discussion_r40606368","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/8744","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/40606368"},"html":{"href":"https://github.com/apache/spark/pull/8744#discussion_r40606368"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/8744"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/8744","id":44922174,"html_url":"https://github.com/apache/spark/pull/8744","diff_url":"https://github.com/apache/spark/pull/8744.diff","patch_url":"https://github.com/apache/spark/pull/8744.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/8744","number":8744,"state":"open","locked":false,"title":"SPARK-1537 publisher-side code and tests","user":{"login":"steveloughran","id":162090,"avatar_url":"https://avatars.githubusercontent.com/u/162090?v=3","gravatar_id":"","url":"https://api.github.com/users/steveloughran","html_url":"https://github.com/steveloughran","followers_url":"https://api.github.com/users/steveloughran/followers","following_url":"https://api.github.com/users/steveloughran/following{/other_user}","gists_url":"https://api.github.com/users/steveloughran/gists{/gist_id}","starred_url":"https://api.github.com/users/steveloughran/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/steveloughran/subscriptions","organizations_url":"https://api.github.com/users/steveloughran/orgs","repos_url":"https://api.github.com/users/steveloughran/repos","events_url":"https://api.github.com/users/steveloughran/events{/privacy}","received_events_url":"https://api.github.com/users/steveloughran/received_events","type":"User","site_admin":false},"body":"This is the publisher side of the SPARK-1537 Yarn timeline server integration: the event listener to asynchronously publish events to ATS; the basic unit and mock tests to verify functionality.","created_at":"2015-09-14T12:07:58Z","updated_at":"2015-09-28T21:13:26Z","closed_at":null,"merged_at":null,"merge_commit_sha":"bcbee74b14a9f0f694d8ad9618b0e3eda21aad4f","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/8744/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/8744/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/8744/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/231301a943f449db1ccf46573f2e98a47034c797","head":{"label":"steveloughran:stevel/feature/SPARK-1537-publisher","ref":"stevel/feature/SPARK-1537-publisher","sha":"231301a943f449db1ccf46573f2e98a47034c797","user":{"login":"steveloughran","id":162090,"avatar_url":"https://avatars.githubusercontent.com/u/162090?v=3","gravatar_id":"","url":"https://api.github.com/users/steveloughran","html_url":"https://github.com/steveloughran","followers_url":"https://api.github.com/users/steveloughran/followers","following_url":"https://api.github.com/users/steveloughran/following{/other_user}","gists_url":"https://api.github.com/users/steveloughran/gists{/gist_id}","starred_url":"https://api.github.com/users/steveloughran/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/steveloughran/subscriptions","organizations_url":"https://api.github.com/users/steveloughran/orgs","repos_url":"https://api.github.com/users/steveloughran/repos","events_url":"https://api.github.com/users/steveloughran/events{/privacy}","received_events_url":"https://api.github.com/users/steveloughran/received_events","type":"User","site_admin":false},"repo":{"id":32387210,"name":"spark","full_name":"steveloughran/spark","owner":{"login":"steveloughran","id":162090,"avatar_url":"https://avatars.githubusercontent.com/u/162090?v=3","gravatar_id":"","url":"https://api.github.com/users/steveloughran","html_url":"https://github.com/steveloughran","followers_url":"https://api.github.com/users/steveloughran/followers","following_url":"https://api.github.com/users/steveloughran/following{/other_user}","gists_url":"https://api.github.com/users/steveloughran/gists{/gist_id}","starred_url":"https://api.github.com/users/steveloughran/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/steveloughran/subscriptions","organizations_url":"https://api.github.com/users/steveloughran/orgs","repos_url":"https://api.github.com/users/steveloughran/repos","events_url":"https://api.github.com/users/steveloughran/events{/privacy}","received_events_url":"https://api.github.com/users/steveloughran/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/steveloughran/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/steveloughran/spark","forks_url":"https://api.github.com/repos/steveloughran/spark/forks","keys_url":"https://api.github.com/repos/steveloughran/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/steveloughran/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/steveloughran/spark/teams","hooks_url":"https://api.github.com/repos/steveloughran/spark/hooks","issue_events_url":"https://api.github.com/repos/steveloughran/spark/issues/events{/number}","events_url":"https://api.github.com/repos/steveloughran/spark/events","assignees_url":"https://api.github.com/repos/steveloughran/spark/assignees{/user}","branches_url":"https://api.github.com/repos/steveloughran/spark/branches{/branch}","tags_url":"https://api.github.com/repos/steveloughran/spark/tags","blobs_url":"https://api.github.com/repos/steveloughran/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/steveloughran/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/steveloughran/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/steveloughran/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/steveloughran/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/steveloughran/spark/languages","stargazers_url":"https://api.github.com/repos/steveloughran/spark/stargazers","contributors_url":"https://api.github.com/repos/steveloughran/spark/contributors","subscribers_url":"https://api.github.com/repos/steveloughran/spark/subscribers","subscription_url":"https://api.github.com/repos/steveloughran/spark/subscription","commits_url":"https://api.github.com/repos/steveloughran/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/steveloughran/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/steveloughran/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/steveloughran/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/steveloughran/spark/contents/{+path}","compare_url":"https://api.github.com/repos/steveloughran/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/steveloughran/spark/merges","archive_url":"https://api.github.com/repos/steveloughran/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/steveloughran/spark/downloads","issues_url":"https://api.github.com/repos/steveloughran/spark/issues{/number}","pulls_url":"https://api.github.com/repos/steveloughran/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/steveloughran/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/steveloughran/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/steveloughran/spark/labels{/name}","releases_url":"https://api.github.com/repos/steveloughran/spark/releases{/id}","created_at":"2015-03-17T10:36:33Z","updated_at":"2015-03-17T10:36:52Z","pushed_at":"2015-09-24T19:06:51Z","git_url":"git://github.com/steveloughran/spark.git","ssh_url":"git@github.com:steveloughran/spark.git","clone_url":"https://github.com/steveloughran/spark.git","svn_url":"https://github.com/steveloughran/spark","homepage":null,"size":122809,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"d81565465cc6d4f38b4ed78036cded630c700388","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-09-28T20:48:45Z","pushed_at":"2015-09-28T21:00:43Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1633374,"stargazers_count":5578,"watchers_count":5578,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":5148,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":341,"forks":5148,"open_issues":341,"watchers":5578,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/8744"},"html":{"href":"https://github.com/apache/spark/pull/8744"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/8744"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/8744/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/8744/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/8744/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/231301a943f449db1ccf46573f2e98a47034c797"}}}}