{"action":"created","comment":{"url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/comments/48414638","id":48414638,"diff_hunk":"@@ -0,0 +1,134 @@\n+package spark.jobserver\n+\n+import akka.util.Timeout\n+import java.lang.ThreadLocal\n+import java.util.concurrent.atomic.AtomicReference\n+import org.apache.spark.rdd.RDD\n+import org.apache.spark.SparkContext\n+\n+import org.apache.spark.sql.DataFrame\n+import org.apache.spark.storage.StorageLevel\n+import scala.concurrent.duration.Duration\n+\n+sealed trait NamedObject\n+case class NamedRDD[T](rdd: RDD[T], storageLevel: StorageLevel) extends NamedObject\n+case class NamedDataFrame(df: DataFrame) extends NamedObject\n+\n+abstract class NamedObjectPersister[O <: NamedObject] {\n+  //TODO what is the context needed for? c: ContextLike,\n+  def saveToContext(namedObj: O, name: String): Unit\n+  //TODO - what is this for?\n+  //def getFromContext(c: ContextLike, name: String): O\n+  /**\n+   * update reference to named object so that it does not get GCed\n+   */\n+  def refresh(namedObject: O): O\n+  //TODO - what about 'unpersist' ?\n+}\n+\n+/**\n+ * NamedObjects - a trait that gives you safe, concurrent creation and access to named objects\n+ * such as RDDs or DataFrames\n+ * (the native SparkContext interface only has access to RDDs by numbers).\n+ * It facilitates easy sharing of data objects amongst jobs sharing the same SparkContext.\n+ * If two jobs simultaneously tries to create a data object with the same name and in the same namespace,\n+ * only one will win and the other will retrieve the same one.\n+ *\n+ * Note that to take advantage of NamedObjectSupport, a job must mix this in and use the APIs here instead of\n+ * the native DataFrame/RDD `cache()`, otherwise we will not know about the names.\n+ */\n+trait NamedObjects {\n+  // Default timeout is 60 seconds. Hopefully that is enough\n+  // to let most RDD/DataFrame generator functions finish.\n+  val defaultTimeout = akka.util.Timeout(Duration(60, java.util.concurrent.TimeUnit.SECONDS))\n+\n+  /**\n+   * Gets a named object (NObj) with the given name, or creates it if one doesn't already exist.\n+   *\n+   * If the given NObj has already been computed by another job and cached in memory, this method will return\n+   * a reference to the cached NObj. If the NObj has never been computed, then the generator will be called\n+   * to compute it, in the caller's thread, and the result will be cached and returned to the caller.\n+   *\n+   * If an NObj is requested by thread B while thread A is generating the NObj, thread B will block up to\n+   * the duration specified by @timeout. If thread A finishes generating the NObj within that time, then\n+   * thread B will get a reference to the newly-created RDD. If thread A does not finish generating the\n+   * NObj within that time, then thread B will throw a timeout exception.\n+   *\n+   * @param name the unique name of the NObj. The uniqueness is scoped to the current SparkContext.\n+   * @param rddGen a 0-ary function which will generate the NObj if it doesn't already exist.\n+   * @param timeout if the named object isn't created within this timeout, an error will be thrown.","path":"job-server-api/src/spark.jobserver/NamedObjectSupport.scala","position":59,"original_position":59,"commit_id":"484fe0c86ea2d1b94c761fa7d4766c70e08699d3","original_commit_id":"484fe0c86ea2d1b94c761fa7d4766c70e08699d3","user":{"login":"zeitos","id":1010275,"avatar_url":"https://avatars.githubusercontent.com/u/1010275?v=3","gravatar_id":"","url":"https://api.github.com/users/zeitos","html_url":"https://github.com/zeitos","followers_url":"https://api.github.com/users/zeitos/followers","following_url":"https://api.github.com/users/zeitos/following{/other_user}","gists_url":"https://api.github.com/users/zeitos/gists{/gist_id}","starred_url":"https://api.github.com/users/zeitos/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/zeitos/subscriptions","organizations_url":"https://api.github.com/users/zeitos/orgs","repos_url":"https://api.github.com/users/zeitos/repos","events_url":"https://api.github.com/users/zeitos/events{/privacy}","received_events_url":"https://api.github.com/users/zeitos/received_events","type":"User","site_admin":false},"body":"we need doc for persister","created_at":"2015-12-24T13:30:54Z","updated_at":"2015-12-24T13:30:54Z","html_url":"https://github.com/spark-jobserver/spark-jobserver/pull/350#discussion_r48414638","pull_request_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350","_links":{"self":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/comments/48414638"},"html":{"href":"https://github.com/spark-jobserver/spark-jobserver/pull/350#discussion_r48414638"},"pull_request":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350"}}},"pull_request":{"url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350","id":54545187,"html_url":"https://github.com/spark-jobserver/spark-jobserver/pull/350","diff_url":"https://github.com/spark-jobserver/spark-jobserver/pull/350.diff","patch_url":"https://github.com/spark-jobserver/spark-jobserver/pull/350.patch","issue_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues/350","number":350,"state":"open","locked":false,"title":"Named Objects","user":{"login":"TimMaltGermany","id":7852234,"avatar_url":"https://avatars.githubusercontent.com/u/7852234?v=3","gravatar_id":"","url":"https://api.github.com/users/TimMaltGermany","html_url":"https://github.com/TimMaltGermany","followers_url":"https://api.github.com/users/TimMaltGermany/followers","following_url":"https://api.github.com/users/TimMaltGermany/following{/other_user}","gists_url":"https://api.github.com/users/TimMaltGermany/gists{/gist_id}","starred_url":"https://api.github.com/users/TimMaltGermany/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/TimMaltGermany/subscriptions","organizations_url":"https://api.github.com/users/TimMaltGermany/orgs","repos_url":"https://api.github.com/users/TimMaltGermany/repos","events_url":"https://api.github.com/users/TimMaltGermany/events{/privacy}","received_events_url":"https://api.github.com/users/TimMaltGermany/received_events","type":"User","site_admin":false},"body":"Pull Request: Named Object \r\n\r\nThe SJS currently supports NamedRDDs via the NamedRddSupport trait. Only objects of type RDD[T] can be stored and retrieved.\r\nWe would like to use the same functionality for DataFrames and possibly future similar implementations and object in general.\r\n\r\nProposed changes:\r\n- introduce new traits NamedObjectSupport and NamedObjects that contain the generalized functionality of NamedRDDSupport and NamedRDDs\r\n\r\nThis is the initial pull request that is based on some eMail discussions between Tobias, Evan and myself. There are still quite a few TODOs and open questions, but I thought that this pull request might make it easier to discuss things.\r\n\r\nThe current version only has an implementation for RDDs. In fact, the old trait NamedRddSupport still exsists (but now extends NamedObjectSupport) and ensures backwards compatibility of job implementations.\r\nAn implementation that support DataFrames should follow as soon as we finalize the actual API. \r\nI wil add some inline comments to highlight the open issues that I am aware of.\r\n\r\nThanks,\r\nTim\r\n","created_at":"2015-12-24T11:17:52Z","updated_at":"2015-12-24T13:30:54Z","closed_at":null,"merged_at":null,"merge_commit_sha":"be09f3ff22eb7254bbbbcc72b532c97b2acfb1df","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350/commits","review_comments_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350/comments","review_comment_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/comments{/number}","comments_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues/350/comments","statuses_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/statuses/484fe0c86ea2d1b94c761fa7d4766c70e08699d3","head":{"label":"koettert:NamedDS","ref":"NamedDS","sha":"484fe0c86ea2d1b94c761fa7d4766c70e08699d3","user":{"login":"koettert","id":11409922,"avatar_url":"https://avatars.githubusercontent.com/u/11409922?v=3","gravatar_id":"","url":"https://api.github.com/users/koettert","html_url":"https://github.com/koettert","followers_url":"https://api.github.com/users/koettert/followers","following_url":"https://api.github.com/users/koettert/following{/other_user}","gists_url":"https://api.github.com/users/koettert/gists{/gist_id}","starred_url":"https://api.github.com/users/koettert/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/koettert/subscriptions","organizations_url":"https://api.github.com/users/koettert/orgs","repos_url":"https://api.github.com/users/koettert/repos","events_url":"https://api.github.com/users/koettert/events{/privacy}","received_events_url":"https://api.github.com/users/koettert/received_events","type":"User","site_admin":false},"repo":{"id":41287905,"name":"spark-jobserver","full_name":"koettert/spark-jobserver","owner":{"login":"koettert","id":11409922,"avatar_url":"https://avatars.githubusercontent.com/u/11409922?v=3","gravatar_id":"","url":"https://api.github.com/users/koettert","html_url":"https://github.com/koettert","followers_url":"https://api.github.com/users/koettert/followers","following_url":"https://api.github.com/users/koettert/following{/other_user}","gists_url":"https://api.github.com/users/koettert/gists{/gist_id}","starred_url":"https://api.github.com/users/koettert/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/koettert/subscriptions","organizations_url":"https://api.github.com/users/koettert/orgs","repos_url":"https://api.github.com/users/koettert/repos","events_url":"https://api.github.com/users/koettert/events{/privacy}","received_events_url":"https://api.github.com/users/koettert/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/koettert/spark-jobserver","description":"REST job server for Apache Spark","fork":true,"url":"https://api.github.com/repos/koettert/spark-jobserver","forks_url":"https://api.github.com/repos/koettert/spark-jobserver/forks","keys_url":"https://api.github.com/repos/koettert/spark-jobserver/keys{/key_id}","collaborators_url":"https://api.github.com/repos/koettert/spark-jobserver/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/koettert/spark-jobserver/teams","hooks_url":"https://api.github.com/repos/koettert/spark-jobserver/hooks","issue_events_url":"https://api.github.com/repos/koettert/spark-jobserver/issues/events{/number}","events_url":"https://api.github.com/repos/koettert/spark-jobserver/events","assignees_url":"https://api.github.com/repos/koettert/spark-jobserver/assignees{/user}","branches_url":"https://api.github.com/repos/koettert/spark-jobserver/branches{/branch}","tags_url":"https://api.github.com/repos/koettert/spark-jobserver/tags","blobs_url":"https://api.github.com/repos/koettert/spark-jobserver/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/koettert/spark-jobserver/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/koettert/spark-jobserver/git/refs{/sha}","trees_url":"https://api.github.com/repos/koettert/spark-jobserver/git/trees{/sha}","statuses_url":"https://api.github.com/repos/koettert/spark-jobserver/statuses/{sha}","languages_url":"https://api.github.com/repos/koettert/spark-jobserver/languages","stargazers_url":"https://api.github.com/repos/koettert/spark-jobserver/stargazers","contributors_url":"https://api.github.com/repos/koettert/spark-jobserver/contributors","subscribers_url":"https://api.github.com/repos/koettert/spark-jobserver/subscribers","subscription_url":"https://api.github.com/repos/koettert/spark-jobserver/subscription","commits_url":"https://api.github.com/repos/koettert/spark-jobserver/commits{/sha}","git_commits_url":"https://api.github.com/repos/koettert/spark-jobserver/git/commits{/sha}","comments_url":"https://api.github.com/repos/koettert/spark-jobserver/comments{/number}","issue_comment_url":"https://api.github.com/repos/koettert/spark-jobserver/issues/comments{/number}","contents_url":"https://api.github.com/repos/koettert/spark-jobserver/contents/{+path}","compare_url":"https://api.github.com/repos/koettert/spark-jobserver/compare/{base}...{head}","merges_url":"https://api.github.com/repos/koettert/spark-jobserver/merges","archive_url":"https://api.github.com/repos/koettert/spark-jobserver/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/koettert/spark-jobserver/downloads","issues_url":"https://api.github.com/repos/koettert/spark-jobserver/issues{/number}","pulls_url":"https://api.github.com/repos/koettert/spark-jobserver/pulls{/number}","milestones_url":"https://api.github.com/repos/koettert/spark-jobserver/milestones{/number}","notifications_url":"https://api.github.com/repos/koettert/spark-jobserver/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/koettert/spark-jobserver/labels{/name}","releases_url":"https://api.github.com/repos/koettert/spark-jobserver/releases{/id}","created_at":"2015-08-24T07:08:17Z","updated_at":"2015-08-24T07:08:18Z","pushed_at":"2015-12-24T11:06:40Z","git_url":"git://github.com/koettert/spark-jobserver.git","ssh_url":"git@github.com:koettert/spark-jobserver.git","clone_url":"https://github.com/koettert/spark-jobserver.git","svn_url":"https://github.com/koettert/spark-jobserver","homepage":"","size":1073,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"spark-jobserver:master","ref":"master","sha":"4c1eabededfe84dbd4835c4156a323e7e57f8862","user":{"login":"spark-jobserver","id":8494302,"avatar_url":"https://avatars.githubusercontent.com/u/8494302?v=3","gravatar_id":"","url":"https://api.github.com/users/spark-jobserver","html_url":"https://github.com/spark-jobserver","followers_url":"https://api.github.com/users/spark-jobserver/followers","following_url":"https://api.github.com/users/spark-jobserver/following{/other_user}","gists_url":"https://api.github.com/users/spark-jobserver/gists{/gist_id}","starred_url":"https://api.github.com/users/spark-jobserver/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spark-jobserver/subscriptions","organizations_url":"https://api.github.com/users/spark-jobserver/orgs","repos_url":"https://api.github.com/users/spark-jobserver/repos","events_url":"https://api.github.com/users/spark-jobserver/events{/privacy}","received_events_url":"https://api.github.com/users/spark-jobserver/received_events","type":"Organization","site_admin":false},"repo":{"id":23205911,"name":"spark-jobserver","full_name":"spark-jobserver/spark-jobserver","owner":{"login":"spark-jobserver","id":8494302,"avatar_url":"https://avatars.githubusercontent.com/u/8494302?v=3","gravatar_id":"","url":"https://api.github.com/users/spark-jobserver","html_url":"https://github.com/spark-jobserver","followers_url":"https://api.github.com/users/spark-jobserver/followers","following_url":"https://api.github.com/users/spark-jobserver/following{/other_user}","gists_url":"https://api.github.com/users/spark-jobserver/gists{/gist_id}","starred_url":"https://api.github.com/users/spark-jobserver/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/spark-jobserver/subscriptions","organizations_url":"https://api.github.com/users/spark-jobserver/orgs","repos_url":"https://api.github.com/users/spark-jobserver/repos","events_url":"https://api.github.com/users/spark-jobserver/events{/privacy}","received_events_url":"https://api.github.com/users/spark-jobserver/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/spark-jobserver/spark-jobserver","description":"REST job server for Apache Spark","fork":false,"url":"https://api.github.com/repos/spark-jobserver/spark-jobserver","forks_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/forks","keys_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/keys{/key_id}","collaborators_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/teams","hooks_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/hooks","issue_events_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues/events{/number}","events_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/events","assignees_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/assignees{/user}","branches_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/branches{/branch}","tags_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/tags","blobs_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/git/refs{/sha}","trees_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/git/trees{/sha}","statuses_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/statuses/{sha}","languages_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/languages","stargazers_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/stargazers","contributors_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/contributors","subscribers_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/subscribers","subscription_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/subscription","commits_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/commits{/sha}","git_commits_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/git/commits{/sha}","comments_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/comments{/number}","issue_comment_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues/comments{/number}","contents_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/contents/{+path}","compare_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/compare/{base}...{head}","merges_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/merges","archive_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/downloads","issues_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues{/number}","pulls_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls{/number}","milestones_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/milestones{/number}","notifications_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/labels{/name}","releases_url":"https://api.github.com/repos/spark-jobserver/spark-jobserver/releases{/id}","created_at":"2014-08-21T23:07:47Z","updated_at":"2015-12-23T21:36:18Z","pushed_at":"2015-12-24T11:17:52Z","git_url":"git://github.com/spark-jobserver/spark-jobserver.git","ssh_url":"git@github.com:spark-jobserver/spark-jobserver.git","clone_url":"https://github.com/spark-jobserver/spark-jobserver.git","svn_url":"https://github.com/spark-jobserver/spark-jobserver","homepage":"","size":2319,"stargazers_count":760,"watchers_count":760,"language":"Scala","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":310,"mirror_url":null,"open_issues_count":59,"forks":310,"open_issues":59,"watchers":760,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350"},"html":{"href":"https://github.com/spark-jobserver/spark-jobserver/pull/350"},"issue":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues/350"},"comments":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/issues/350/comments"},"review_comments":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350/comments"},"review_comment":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/pulls/350/commits"},"statuses":{"href":"https://api.github.com/repos/spark-jobserver/spark-jobserver/statuses/484fe0c86ea2d1b94c761fa7d4766c70e08699d3"}}}}