{"action":"created","comment":{"url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/comments/48415716","id":48415716,"diff_hunk":"@@ -6,115 +6,191 @@ import java.util.concurrent.atomic.AtomicReference\n import org.apache.spark.rdd.RDD\n import org.apache.spark.storage.StorageLevel\n import scala.concurrent.duration.Duration\n+import org.apache.spark.SparkContext\n \n /**\n- * NamedRdds - a trait that gives you safe, concurrent creation and access to named RDDs\n- * (the native SparkContext interface only has access to RDDs by numbers).\n- * It facilitates easy sharing of RDDs amongst jobs sharing the same SparkContext.\n- * If two jobs simultaneously tries to create an RDD with the same name,\n- * only one will win and the other will retrieve the same one.\n- *\n- * Note that to take advantage of NamedRddSupport, a job must mix this in and use the APIs here instead of\n- * the native RDD `cache()`, otherwise we will not know about the names.\n+ * @note please use NamedObjectSupport instead !\n  */\n-trait NamedRdds {\n-  // Default timeout is 60 seconds. Hopefully that is enough to let most RDD generator functions finish.\n-  val defaultTimeout = akka.util.Timeout(Duration(60, java.util.concurrent.TimeUnit.SECONDS))\n-\n-  // Default level to cache RDDs at.\n-  val defaultStorageLevel = StorageLevel.MEMORY_ONLY\n+trait NamedRddSupport extends NamedObjectSupport { self: SparkJob =>\n \n   /**\n-   * Gets an RDD with the given name, or creates it if one doesn't already exist.\n-   *\n-   * If the given RDD has already been computed by another job and cached in memory, this method will return\n-   * a reference to the cached RDD. If the RDD has never been computed, then the generator will be called\n-   * to compute it, in the caller's thread, and the result will be cached and returned to the caller.\n+   * NamedRdds - a trait that gives you safe, concurrent creation and access to named RDDs\n+   * (the native SparkContext interface only has access to RDDs by numbers).\n+   * It facilitates easy sharing of RDDs amongst jobs sharing the same SparkContext.\n+   * If two jobs simultaneously tries to create an RDD with the same name,\n+   * only one will win and the other will retrieve the same one.\n    *\n-   * If an RDD is requested by thread B while thread A is generating the RDD, thread B will block up to\n-   * the duration specified by @timeout. If thread A finishes generating the RDD within that time, then\n-   * thread B will get a reference to the newly-created RDD. If thread A does not finish generating the\n-   * RDD within that time, then thread B will throw a timeout exception.\n-   *\n-   * @param name the unique name of the RDD. The uniqueness is scoped to the current SparkContext.\n-   * @param rddGen a 0-ary function which will generate the RDD if it doesn't already exist.\n-   * @param forceComputation if true, forces the RDD to be computed by calling count().\n-   * @param storageLevel the storage level to persist the RDD with. Default: StorageLevel.MEMORY_ONLY.\n-   * @param timeout if the RddManager doesn't respond within this timeout, an error will be thrown.\n-   * @tparam T the generic type of the RDD.\n-   * @return the RDD with the given name.\n-   * @throws java.util.concurrent.TimeoutException if the request to the RddManager times out.\n-   * @throws java.lang.RuntimeException wrapping any error that occurs within the generator function.\n+   * Note that to take advantage of NamedRddSupport, a job must mix this in and use the APIs here instead of\n+   * the native RDD `cache()`, otherwise we will not know about the names.\n    */\n-  def getOrElseCreate[T](name: String,\n-                         rddGen: => RDD[T],\n-                         forceComputation: Boolean = true,\n-                         storageLevel: StorageLevel = defaultStorageLevel)\n-                        (implicit timeout: Timeout = defaultTimeout): RDD[T]\n+  trait _NamedRdds {\n \n-  /**\n-   * Gets an RDD with the given name if it already exists and is cached by the RddManager.\n-   * If the RDD does not exist, None is returned.\n-   *\n-   * Note that a previously-known RDD could 'disappear' if it hasn't been used for a while, because the\n-   * SparkContext garbage-collects old cached RDDs.\n-   *\n-   * @param name the unique name of the RDD. The uniqueness is scoped to the current SparkContext.\n-   * @param timeout if the RddManager doesn't respond within this timeout, an error will be thrown.\n-   * @tparam T the generic type of the RDD.\n-   * @return the RDD with the given name.\n-   * @throws java.util.concurrent.TimeoutException if the request to the RddManager times out.\n-   */\n-  def get[T](name: String)(implicit timeout: Timeout = defaultTimeout): Option[RDD[T]]\n+    // Default timeout is 60 seconds. Hopefully that is enough\n+    // to let most RDD/DataFrame generator functions finish.\n+    val defaultTimeout = akka.util.Timeout(Duration(60, java.util.concurrent.TimeUnit.SECONDS))\n \n-  /**\n-   * Replaces an existing RDD with a given name with a new RDD. If an old RDD for the given name existed,\n-   * it is un-persisted (non-blocking) and destroyed. It is safe to call this method when there is no\n-   * existing RDD with the given name. If multiple threads call this around the same time, the end result\n-   * is undefined - one of the generated RDDs will win and will be returned from future calls to get().\n-   *\n-   * The rdd generator function will be called from the caller's thread. Note that if this is called at the\n-   * same time as getOrElseCreate() for the same name, and completes before the getOrElseCreate() call,\n-   * then threads waiting for the result of getOrElseCreate() will unblock with the result of this\n-   * update() call. When the getOrElseCreate() succeeds, it will replace the result of this update() call.\n-   *\n-   * @param name the unique name of the RDD. The uniqueness is scoped to the current SparkContext.\n-   * @param rddGen a 0-ary function which will be called to generate the RDD in the caller's thread.\n-   * @param forceComputation if true, forces the RDD to be computed by calling count().\n-   * @param storageLevel the storage level to persist the RDD with. Default: StorageLevel.MEMORY_ONLY.\n-   * @tparam T the generic type of the RDD.\n-   * @return the RDD with the given name.\n-   */\n-  def update[T](name: String,\n-                rddGen: => RDD[T],\n-                forceComputation: Boolean = true,\n-                storageLevel: StorageLevel = defaultStorageLevel): RDD[T]\n+    // Default level to cache RDDs at.\n+    val defaultStorageLevel = StorageLevel.MEMORY_ONLY\n \n-  /**\n-   * Destroys an RDD with the given name, if one existed. Has no effect if no RDD with this name exists.\n-   *\n-   * @param name the unique name of the RDD. The uniqueness is scoped to the current SparkContext.\n-   */\n-  def destroy(name: String): Unit\n+    /**\n+     * Gets an RDD with the given name, or creates it if one doesn't already exist.\n+     *\n+     * If the given RDD has already been computed by another job and cached in memory, this method will return\n+     * a reference to the cached RDD. If the RDD has never been computed, then the generator will be called\n+     * to compute it, in the caller's thread, and the result will be cached and returned to the caller.\n+     *\n+     * If an RDD is requested by thread B while thread A is generating the RDD, thread B will block up to\n+     * the duration specified by @timeout. If thread A finishes generating the RDD within that time, then\n+     * thread B will get a reference to the newly-created RDD. If thread A does not finish generating the\n+     * RDD within that time, then thread B will throw a timeout exception.\n+     *\n+     * @param name the unique name of the RDD. The uniqueness is scoped to the current SparkContext.\n+     * @param rddGen a 0-ary function which will generate the RDD if it doesn't already exist.\n+     * @param forceComputation if true, forces the RDD to be computed by calling count().\n+     * @param storageLevel the storage level to persist the RDD with. Default: StorageLevel.MEMORY_ONLY.\n+     * @param timeout if the RddManager doesn't respond within this timeout, an error will be thrown.\n+     * @tparam T the generic type of the RDD.\n+     * @return the RDD with the given name.\n+     * @throws java.util.concurrent.TimeoutException if the request to the RddManager times out.\n+     * @throws java.lang.RuntimeException wrapping any error that occurs within the generator function.\n+     */\n+    def getOrElseCreate[T](name: String,\n+                           rddGen: => RDD[T],\n+                           forceComputation: Boolean = true,\n+                           storageLevel: StorageLevel = defaultStorageLevel)\n+                           (implicit timeout: Timeout = defaultTimeout): RDD[T]\n \n-  /**\n-   * Returns the names of all named RDDs that are managed by the RddManager.\n-   *\n-   * Note: this returns a snapshot of RDD names at one point in time. The caller should always expect\n-   * that the data returned from this method may be stale and incorrect.\n+    /**\n+     * Gets an RDD with the given name if it already exists and is cached by the RddManager.\n+     * If the RDD does not exist, None is returned.\n+     *\n+     * Note that a previously-known RDD could 'disappear' if it hasn't been used for a while, because the\n+     * SparkContext garbage-collects old cached RDDs.\n+     *\n+     * @param name the unique name of the RDD. The uniqueness is scoped to the current SparkContext.\n+     * @param timeout if the RddManager doesn't respond within this timeout, an error will be thrown.\n+     * @tparam T the generic type of the RDD.\n+     * @return the RDD with the given name.\n+     * @throws java.util.concurrent.TimeoutException if the request to the RddManager times out.\n+     */\n+    def get[T](name: String)(implicit timeout: Timeout = defaultTimeout): Option[RDD[T]]\n \n-   * @param timeout if the RddManager doesn't respond within this timeout, an error will be thrown.\n-   * @return a collection of RDD names representing RDDs managed by the RddManager.\n-   */\n-  def getNames()(implicit timeout: Timeout = defaultTimeout): Iterable[String]\n-}\n+    /**\n+     * Replaces an existing RDD with a given name with a new RDD. If an old RDD for the given name existed,\n+     * it is un-persisted (non-blocking) and destroyed. It is safe to call this method when there is no\n+     * existing RDD with the given name. If multiple threads call this around the same time, the end result\n+     * is undefined - one of the generated RDDs will win and will be returned from future calls to get().\n+     *\n+     * The rdd generator function will be called from the caller's thread. Note that if this is called at the\n+     * same time as getOrElseCreate() for the same name, and completes before the getOrElseCreate() call,\n+     * then threads waiting for the result of getOrElseCreate() will unblock with the result of this\n+     * update() call. When the getOrElseCreate() succeeds, it will replace the result of this update() call.\n+     *\n+     * @param name the unique name of the RDD. The uniqueness is scoped to the current SparkContext.\n+     * @param rddGen a 0-ary function which will be called to generate the RDD in the caller's thread.\n+     * @param forceComputation if true, forces the RDD to be computed by calling count().\n+     * @param storageLevel the storage level to persist the RDD with. Default: StorageLevel.MEMORY_ONLY.\n+     * @tparam T the generic type of the RDD.\n+     * @return the RDD with the given name.\n+     */\n+    def update[T](name: String,\n+                  rddGen: => RDD[T],\n+                  forceComputation: Boolean = true,\n+                  storageLevel: StorageLevel = defaultStorageLevel): RDD[T]\n+\n+    /**\n+     * Destroys an RDD with the given name, if one existed. Has no effect if no RDD with this name exists.\n+     *\n+     * @param name the unique name of the RDD. The uniqueness is scoped to the current SparkContext.\n+     */\n+    def destroy(name: String): Unit\n+\n+    /**\n+     * Returns the names of all named RDDs that are managed by the RddManager.\n+     *\n+     * Note: this returns a snapshot of RDD names at one point in time. The caller should always expect\n+     * that the data returned from this method may be stale and incorrect.\n+     *\n+     * @param timeout if the RddManager doesn't respond within this timeout, an error will be thrown.\n+     * @return a collection of RDD names representing RDDs managed by the RddManager.\n+     */\n+    def getNames()(implicit timeout: Timeout = defaultTimeout): Iterable[String]\n+  }\n+\n+  private def rddPersister[T](forceComputation: Boolean) = new NamedObjectPersister[NamedRDD[T]] {\n+    override def saveToContext(namedObj: NamedRDD[T], name: String) {\n+      namedObj match {\n+        case NamedRDD(rdd, storageLevel) =>\n+          require(!forceComputation || storageLevel != StorageLevel.NONE,\n+            \"forceComputation implies storageLevel != NONE\")\n+          rdd.setName(name)\n+          rdd.getStorageLevel match {\n+            case StorageLevel.NONE => rdd.persist(storageLevel)\n+            case currentLevel      => rdd.persist(currentLevel)\n+          }\n+          // TODO: figure out if there is a better way to force the RDD to be computed\n+          if (forceComputation) rdd.count()\n+      }\n+    }\n+\n+//    override def getFromContext(c: ContextLike, name: String): NamedRDD[T] = {\n+//      c.sparkContext.getPersistentRDDs.filter(_._2.name.equals(name)).map(_._2).headOption match {\n+//        case Some(rdd: RDD[T]) => NamedRDD(rdd, rdd.getStorageLevel)\n+//      }\n+//    }\n+\n+    /**\n+     * Calls rdd.persist(), which updates the RDD's cached timestamp, meaning it won't get","path":"job-server-api/src/spark.jobserver/NamedRddSupport.scala","position":228,"original_position":228,"commit_id":"484fe0c86ea2d1b94c761fa7d4766c70e08699d3","original_commit_id":"484fe0c86ea2d1b94c761fa7d4766c70e08699d3","user":{"login":"zeitos","id":1010275,"avatar_url":"https://avatars.githubusercontent.com/u/1010275?v=3","gravatar_id":"","url":"https://api.github.com/users/zeitos","html_url":"https://github.com/zeitos","followers_url":"https://api.github.com/users/zeitos/followers","following_url":"https://api.github.com/users/zeitos/following{/other_user}","gists_url":"https://api.github.com/users/zeitos/gists{/gist_id}","starred_url":"https://api.github.com/users/zeitos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zeitos/subscriptions","organizations_url":"https://api.github.com/users/zeitos/orgs","repos_url":"https://api.github.com/users/zeitos/repos","events_url":"https://api.github.com/users/zeitos/events{/privacy}","received_events_url":"https://api.github.com/users/zeitos/received_events","type":"User","site_admin":false},"body":"I *think* this behaviour changed in spark, unless isLocallyCheckpointed is true persist() would not have any effect","created_at":"2015-12-24T14:03:27Z","updated_at":"2015-12-24T14:03:27Z","html_url":"https://github.com/spark-jobserver/spark-jobserver/pull/350#discussion_r48415716","pull_request_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350","_links":{"self":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/comments/48415716"},"html":{"href":"https://github.com/spark-jobserver/spark-jobserver/pull/350#discussion_r48415716"},"pull_request":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350"}}},"pull_request":{"url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350","id":54545187,"html_url":"https://github.com/spark-jobserver/spark-jobserver/pull/350","diff_url":"https://github.com/spark-jobserver/spark-jobserver/pull/350.diff","patch_url":"https://github.com/spark-jobserver/spark-jobserver/pull/350.patch","issue_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues/350","number":350,"state":"open","locked":false,"title":"Named Objects","user":{"login":"TimMaltGermany","id":7852234,"avatar_url":"https://avatars.githubusercontent.com/u/7852234?v=3","gravatar_id":"","url":"https://api.github.com/users/TimMaltGermany","html_url":"https://github.com/TimMaltGermany","followers_url":"https://api.github.com/users/TimMaltGermany/followers","following_url":"https://api.github.com/users/TimMaltGermany/following{/other_user}","gists_url":"https://api.github.com/users/TimMaltGermany/gists{/gist_id}","starred_url":"https://api.github.com/users/TimMaltGermany/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TimMaltGermany/subscriptions","organizations_url":"https://api.github.com/users/TimMaltGermany/orgs","repos_url":"https://api.github.com/users/TimMaltGermany/repos","events_url":"https://api.github.com/users/TimMaltGermany/events{/privacy}","received_events_url":"https://api.github.com/users/TimMaltGermany/received_events","type":"User","site_admin":false},"body":"Pull Request: Named Object \r\n\r\nThe SJS currently supports NamedRDDs via the NamedRddSupport trait. Only objects of type RDD[T] can be stored and retrieved.\r\nWe would like to use the same functionality for DataFrames and possibly future similar implementations and object in general.\r\n\r\nProposed changes:\r\n- introduce new traits NamedObjectSupport and NamedObjects that contain the generalized functionality of NamedRDDSupport and NamedRDDs\r\n\r\nThis is the initial pull request that is based on some eMail discussions between Tobias, Evan and myself. There are still quite a few TODOs and open questions, but I thought that this pull request might make it easier to discuss things.\r\n\r\nThe current version only has an implementation for RDDs. In fact, the old trait NamedRddSupport still exsists (but now extends NamedObjectSupport) and ensures backwards compatibility of job implementations.\r\nAn implementation that support DataFrames should follow as soon as we finalize the actual API. \r\nI wil add some inline comments to highlight the open issues that I am aware of.\r\n\r\nThanks,\r\nTim\r\n","created_at":"2015-12-24T11:17:52Z","updated_at":"2015-12-24T14:03:27Z","closed_at":null,"merged_at":null,"merge_commit_sha":"be09f3ff22eb7254bbbbcc72b532c97b2acfb1df","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350/commits","review_comments_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350/comments","review_comment_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/comments{/number}","comments_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues/350/comments","statuses_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/statuses/484fe0c86ea2d1b94c761fa7d4766c70e08699d3","head":{"label":"koettert:NamedDS","ref":"NamedDS","sha":"484fe0c86ea2d1b94c761fa7d4766c70e08699d3","user":{"login":"koettert","id":11409922,"avatar_url":"https://avatars.githubusercontent.com/u/11409922?v=3","gravatar_id":"","url":"https://api.github.com/users/koettert","html_url":"https://github.com/koettert","followers_url":"https://api.github.com/users/koettert/followers","following_url":"https://api.github.com/users/koettert/following{/other_user}","gists_url":"https://api.github.com/users/koettert/gists{/gist_id}","starred_url":"https://api.github.com/users/koettert/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/koettert/subscriptions","organizations_url":"https://api.github.com/users/koettert/orgs","repos_url":"https://api.github.com/users/koettert/repos","events_url":"https://api.github.com/users/koettert/events{/privacy}","received_events_url":"https://api.github.com/users/koettert/received_events","type":"User","site_admin":false},"repo":{"id":41287905,"name":"spark-jobserver","full_name":"koettert/spark-jobserver","owner":{"login":"koettert","id":11409922,"avatar_url":"https://avatars.githubusercontent.com/u/11409922?v=3","gravatar_id":"","url":"https://api.github.com/users/koettert","html_url":"https://github.com/koettert","followers_url":"https://api.github.com/users/koettert/followers","following_url":"https://api.github.com/users/koettert/following{/other_user}","gists_url":"https://api.github.com/users/koettert/gists{/gist_id}","starred_url":"https://api.github.com/users/koettert/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/koettert/subscriptions","organizations_url":"https://api.github.com/users/koettert/orgs","repos_url":"https://api.github.com/users/koettert/repos","events_url":"https://api.github.com/users/koettert/events{/privacy}","received_events_url":"https://api.github.com/users/koettert/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/koettert/spark-jobserver","description":"REST job server for Apache Spark","fork":true,"url":"https://api.github.com/repos/koettert/spark-jobserver","forks_url":"https://api.github.com/repos/koettert/spark-jobserver/forks","keys_url":"https://api.github.com/repos/koettert/spark-jobserver/keys{/key_id}","collaborators_url":"https://api.github.com/repos/koettert/spark-jobserver/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/koettert/spark-jobserver/teams","hooks_url":"https://api.github.com/repos/koettert/spark-jobserver/hooks","issue_events_url":"https://api.github.com/repos/koettert/spark-jobserver/issues/events{/number}","events_url":"https://api.github.com/repos/koettert/spark-jobserver/events","assignees_url":"https://api.github.com/repos/koettert/spark-jobserver/assignees{/user}","branches_url":"https://api.github.com/repos/koettert/spark-jobserver/branches{/branch}","tags_url":"https://api.github.com/repos/koettert/spark-jobserver/tags","blobs_url":"https://api.github.com/repos/koettert/spark-jobserver/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/koettert/spark-jobserver/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/koettert/spark-jobserver/git/refs{/sha}","trees_url":"https://api.github.com/repos/koettert/spark-jobserver/git/trees{/sha}","statuses_url":"https://api.github.com/repos/koettert/spark-jobserver/statuses/{sha}","languages_url":"https://api.github.com/repos/koettert/spark-jobserver/languages","stargazers_url":"https://api.github.com/repos/koettert/spark-jobserver/stargazers","contributors_url":"https://api.github.com/repos/koettert/spark-jobserver/contributors","subscribers_url":"https://api.github.com/repos/koettert/spark-jobserver/subscribers","subscription_url":"https://api.github.com/repos/koettert/spark-jobserver/subscription","commits_url":"https://api.github.com/repos/koettert/spark-jobserver/commits{/sha}","git_commits_url":"https://api.github.com/repos/koettert/spark-jobserver/git/commits{/sha}","comments_url":"https://api.github.com/repos/koettert/spark-jobserver/comments{/number}","issue_comment_url":"https://api.github.com/repos/koettert/spark-jobserver/issues/comments{/number}","contents_url":"https://api.github.com/repos/koettert/spark-jobserver/contents/{+path}","compare_url":"https://api.github.com/repos/koettert/spark-jobserver/compare/{base}...{head}","merges_url":"https://api.github.com/repos/koettert/spark-jobserver/merges","archive_url":"https://api.github.com/repos/koettert/spark-jobserver/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/koettert/spark-jobserver/downloads","issues_url":"https://api.github.com/repos/koettert/spark-jobserver/issues{/number}","pulls_url":"https://api.github.com/repos/koettert/spark-jobserver/pulls{/number}","milestones_url":"https://api.github.com/repos/koettert/spark-jobserver/milestones{/number}","notifications_url":"https://api.github.com/repos/koettert/spark-jobserver/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/koettert/spark-jobserver/labels{/name}","releases_url":"https://api.github.com/repos/koettert/spark-jobserver/releases{/id}","created_at":"2015-08-24T07:08:17Z","updated_at":"2015-08-24T07:08:18Z","pushed_at":"2015-12-24T11:06:40Z","git_url":"git://github.com/koettert/spark-jobserver.git","ssh_url":"git@github.com:koettert/spark-jobserver.git","clone_url":"https://github.com/koettert/spark-jobserver.git","svn_url":"https://github.com/koettert/spark-jobserver","homepage":"","size":1073,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"spark-jobserver:master","ref":"master","sha":"4c1eabededfe84dbd4835c4156a323e7e57f8862","user":{"login":"spark-jobserver","id":8494302,"avatar_url":"https://avatars.githubusercontent.com/u/8494302?v=3","gravatar_id":"","url":"https://api.github.com/users/spark-jobserver","html_url":"https://github.com/spark-jobserver","followers_url":"https://api.github.com/users/spark-jobserver/followers","following_url":"https://api.github.com/users/spark-jobserver/following{/other_user}","gists_url":"https://api.github.com/users/spark-jobserver/gists{/gist_id}","starred_url":"https://api.github.com/users/spark-jobserver/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spark-jobserver/subscriptions","organizations_url":"https://api.github.com/users/spark-jobserver/orgs","repos_url":"https://api.github.com/users/spark-jobserver/repos","events_url":"https://api.github.com/users/spark-jobserver/events{/privacy}","received_events_url":"https://api.github.com/users/spark-jobserver/received_events","type":"Organization","site_admin":false},"repo":{"id":23205911,"name":"spark-jobserver","full_name":"spark-jobserver/spark-jobserver","owner":{"login":"spark-jobserver","id":8494302,"avatar_url":"https://avatars.githubusercontent.com/u/8494302?v=3","gravatar_id":"","url":"https://api.github.com/users/spark-jobserver","html_url":"https://github.com/spark-jobserver","followers_url":"https://api.github.com/users/spark-jobserver/followers","following_url":"https://api.github.com/users/spark-jobserver/following{/other_user}","gists_url":"https://api.github.com/users/spark-jobserver/gists{/gist_id}","starred_url":"https://api.github.com/users/spark-jobserver/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spark-jobserver/subscriptions","organizations_url":"https://api.github.com/users/spark-jobserver/orgs","repos_url":"https://api.github.com/users/spark-jobserver/repos","events_url":"https://api.github.com/users/spark-jobserver/events{/privacy}","received_events_url":"https://api.github.com/users/spark-jobserver/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/spark-jobserver/spark-jobserver","description":"REST job server for Apache Spark","fork":false,"url":"https://api.github.com/repos/spark-jobserver/spark-jobserver","forks_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/forks","keys_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/keys{/key_id}","collaborators_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/teams","hooks_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/hooks","issue_events_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues/events{/number}","events_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/events","assignees_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/assignees{/user}","branches_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/branches{/branch}","tags_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/tags","blobs_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/git/refs{/sha}","trees_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/git/trees{/sha}","statuses_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/statuses/{sha}","languages_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/languages","stargazers_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/stargazers","contributors_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/contributors","subscribers_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/subscribers","subscription_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/subscription","commits_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/commits{/sha}","git_commits_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/git/commits{/sha}","comments_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/comments{/number}","issue_comment_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues/comments{/number}","contents_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/contents/{+path}","compare_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/compare/{base}...{head}","merges_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/merges","archive_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/downloads","issues_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues{/number}","pulls_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls{/number}","milestones_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/milestones{/number}","notifications_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/labels{/name}","releases_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/releases{/id}","created_at":"2014-08-21T23:07:47Z","updated_at":"2015-12-23T21:36:18Z","pushed_at":"2015-12-24T11:17:52Z","git_url":"git://github.com/spark-jobserver/spark-jobserver.git","ssh_url":"git@github.com:spark-jobserver/spark-jobserver.git","clone_url":"https://github.com/spark-jobserver/spark-jobserver.git","svn_url":"https://github.com/spark-jobserver/spark-jobserver","homepage":"","size":2319,"stargazers_count":760,"watchers_count":760,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":310,"mirror_url":null,"open_issues_count":59,"forks":310,"open_issues":59,"watchers":760,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350"},"html":{"href":"https://github.com/spark-jobserver/spark-jobserver/pull/350"},"issue":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues/350"},"comments":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues/350/comments"},"review_comments":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350/comments"},"review_comment":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350/commits"},"statuses":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/statuses/484fe0c86ea2d1b94c761fa7d4766c70e08699d3"}}}}