{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/28285532","id":28285532,"diff_hunk":"@@ -0,0 +1,106 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+package org.apache.spark.deploy\n+\n+import java.util.concurrent.{Executors, TimeUnit}\n+import java.util.{Comparator, Arrays}\n+\n+import com.google.common.primitives.Longs\n+import org.apache.hadoop.conf.Configuration\n+import org.apache.hadoop.fs.{PathFilter, FileStatus, Path, FileSystem}\n+import org.apache.hadoop.security.{Credentials, UserGroupInformation}\n+\n+import org.apache.spark.util.Utils\n+import org.apache.spark.{Logging, SparkConf}\n+\n+private[spark] class ExecutorDelegationTokenUpdater(\n+    sparkConf: SparkConf,\n+    hadoopConf: Configuration) extends Logging {\n+\n+  @volatile private var lastCredentialsFileSuffix = 0\n+\n+  private lazy val delegationTokenRenewer =\n+    Executors.newSingleThreadScheduledExecutor(\n+      Utils.namedThreadFactory(\"Delegation Token Refresh Thread\"))\n+\n+  // On the executor, this thread wakes up and picks up new tokens from HDFS, if any.\n+  private lazy val executorUpdaterRunnable =\n+    new Runnable {\n+      override def run(): Unit = Utils.logUncaughtExceptions(updateCredentialsIfRequired())\n+    }\n+\n+  def updateCredentialsIfRequired(): Unit = {\n+    try {\n+      sparkConf.getOption(\"spark.yarn.credentials.file\").foreach { credentialsFile =>","path":"core/src/main/scala/org/apache/spark/deploy/ExecutorDelegationTokenUpdater.scala","position":48,"original_position":48,"commit_id":"e800c8b0929d2696aa421f98f89256ef9f498959","original_commit_id":"e800c8b0929d2696aa421f98f89256ef9f498959","user":{"login":"vanzin","id":1694083,"avatar_url":"https://avatars.githubusercontent.com/u/1694083?v=3","gravatar_id":"","url":"https://api.github.com/users/vanzin","html_url":"https://github.com/vanzin","followers_url":"https://api.github.com/users/vanzin/followers","following_url":"https://api.github.com/users/vanzin/following{/other_user}","gists_url":"https://api.github.com/users/vanzin/gists{/gist_id}","starred_url":"https://api.github.com/users/vanzin/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/vanzin/subscriptions","organizations_url":"https://api.github.com/users/vanzin/orgs","repos_url":"https://api.github.com/users/vanzin/repos","events_url":"https://api.github.com/users/vanzin/events{/privacy}","received_events_url":"https://api.github.com/users/vanzin/received_events","type":"User","site_admin":false},"body":"(Actually, it's a lazy val. I hate lazy vals, so confusing.)\r\n\r\nAnyway, I still think it would be better to have this as an `Option` on the caller site and avoid these weird \"no-op by accident\" semantics.","created_at":"2015-04-13T22:25:44Z","updated_at":"2015-04-13T22:25:44Z","html_url":"https://github.com/apache/spark/pull/4688#discussion_r28285532","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/4688","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/28285532"},"html":{"href":"https://github.com/apache/spark/pull/4688#discussion_r28285532"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/4688"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/4688","id":29590556,"html_url":"https://github.com/apache/spark/pull/4688","diff_url":"https://github.com/apache/spark/pull/4688.diff","patch_url":"https://github.com/apache/spark/pull/4688.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/4688","number":4688,"state":"open","locked":false,"title":"[SPARK-5342][YARN] Allow long running Spark apps to run on secure YARN/HDFS","user":{"login":"harishreedharan","id":1481194,"avatar_url":"https://avatars.githubusercontent.com/u/1481194?v=3","gravatar_id":"","url":"https://api.github.com/users/harishreedharan","html_url":"https://github.com/harishreedharan","followers_url":"https://api.github.com/users/harishreedharan/followers","following_url":"https://api.github.com/users/harishreedharan/following{/other_user}","gists_url":"https://api.github.com/users/harishreedharan/gists{/gist_id}","starred_url":"https://api.github.com/users/harishreedharan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/harishreedharan/subscriptions","organizations_url":"https://api.github.com/users/harishreedharan/orgs","repos_url":"https://api.github.com/users/harishreedharan/repos","events_url":"https://api.github.com/users/harishreedharan/events{/privacy}","received_events_url":"https://api.github.com/users/harishreedharan/received_events","type":"User","site_admin":false},"body":"Current Spark apps running on Secure YARN/HDFS would not be able to write data\r\nto HDFS after 7 days, since delegation tokens cannot be renewed beyond that. This\r\nmeans Spark Streaming apps will not be able to run on Secure YARN.\r\n\r\nThis commit adds basic functionality to fix this issue. In this patch:\r\n- new parameters are added - principal and keytab, which can be used to login to a KDC\r\n- the client logs in, and then get tokens to start the AM\r\n- the keytab is copied to the staging directory\r\n- the AM waits for 60% of the time till expiry of the tokens and then logs in using the keytab\r\n- each time after 60% of the time, new tokens are created and sent to the executors\r\n\r\nCurrently, to avoid complicating the architecture, we set the keytab and principal in the\r\nSparkHadoopUtil singleton, and schedule a login. Once the login is completed, a callback is scheduled.\r\n\r\nThis is being posted for feedback, so I can gather feedback on the general implementation.\r\n\r\nThere are currently a bunch of things to do:\r\n- [x] logging\r\n- [ ] testing - I plan to manually test this soon. If you have ideas of how to add unit tests, comment.\r\n- [x] add code to ensure that if these params are set in non-YARN cluster mode, we complain\r\n- [x] documentation \r\n- [x] Have the executors request for credentials from the AM, so that retries are possible.","created_at":"2015-02-19T06:50:49Z","updated_at":"2015-04-13T22:25:44Z","closed_at":null,"merged_at":null,"merge_commit_sha":"47770e94d61fa1c37284dca637b264a56bfb7851","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/4688/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/4688/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/4688/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/e800c8b0929d2696aa421f98f89256ef9f498959","head":{"label":"harishreedharan:kerberos-longrunning","ref":"kerberos-longrunning","sha":"e800c8b0929d2696aa421f98f89256ef9f498959","user":{"login":"harishreedharan","id":1481194,"avatar_url":"https://avatars.githubusercontent.com/u/1481194?v=3","gravatar_id":"","url":"https://api.github.com/users/harishreedharan","html_url":"https://github.com/harishreedharan","followers_url":"https://api.github.com/users/harishreedharan/followers","following_url":"https://api.github.com/users/harishreedharan/following{/other_user}","gists_url":"https://api.github.com/users/harishreedharan/gists{/gist_id}","starred_url":"https://api.github.com/users/harishreedharan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/harishreedharan/subscriptions","organizations_url":"https://api.github.com/users/harishreedharan/orgs","repos_url":"https://api.github.com/users/harishreedharan/repos","events_url":"https://api.github.com/users/harishreedharan/events{/privacy}","received_events_url":"https://api.github.com/users/harishreedharan/received_events","type":"User","site_admin":false},"repo":{"id":18419952,"name":"spark","full_name":"harishreedharan/spark","owner":{"login":"harishreedharan","id":1481194,"avatar_url":"https://avatars.githubusercontent.com/u/1481194?v=3","gravatar_id":"","url":"https://api.github.com/users/harishreedharan","html_url":"https://github.com/harishreedharan","followers_url":"https://api.github.com/users/harishreedharan/followers","following_url":"https://api.github.com/users/harishreedharan/following{/other_user}","gists_url":"https://api.github.com/users/harishreedharan/gists{/gist_id}","starred_url":"https://api.github.com/users/harishreedharan/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/harishreedharan/subscriptions","organizations_url":"https://api.github.com/users/harishreedharan/orgs","repos_url":"https://api.github.com/users/harishreedharan/repos","events_url":"https://api.github.com/users/harishreedharan/events{/privacy}","received_events_url":"https://api.github.com/users/harishreedharan/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/harishreedharan/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/harishreedharan/spark","forks_url":"https://api.github.com/repos/harishreedharan/spark/forks","keys_url":"https://api.github.com/repos/harishreedharan/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/harishreedharan/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/harishreedharan/spark/teams","hooks_url":"https://api.github.com/repos/harishreedharan/spark/hooks","issue_events_url":"https://api.github.com/repos/harishreedharan/spark/issues/events{/number}","events_url":"https://api.github.com/repos/harishreedharan/spark/events","assignees_url":"https://api.github.com/repos/harishreedharan/spark/assignees{/user}","branches_url":"https://api.github.com/repos/harishreedharan/spark/branches{/branch}","tags_url":"https://api.github.com/repos/harishreedharan/spark/tags","blobs_url":"https://api.github.com/repos/harishreedharan/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/harishreedharan/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/harishreedharan/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/harishreedharan/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/harishreedharan/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/harishreedharan/spark/languages","stargazers_url":"https://api.github.com/repos/harishreedharan/spark/stargazers","contributors_url":"https://api.github.com/repos/harishreedharan/spark/contributors","subscribers_url":"https://api.github.com/repos/harishreedharan/spark/subscribers","subscription_url":"https://api.github.com/repos/harishreedharan/spark/subscription","commits_url":"https://api.github.com/repos/harishreedharan/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/harishreedharan/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/harishreedharan/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/harishreedharan/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/harishreedharan/spark/contents/{+path}","compare_url":"https://api.github.com/repos/harishreedharan/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/harishreedharan/spark/merges","archive_url":"https://api.github.com/repos/harishreedharan/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/harishreedharan/spark/downloads","issues_url":"https://api.github.com/repos/harishreedharan/spark/issues{/number}","pulls_url":"https://api.github.com/repos/harishreedharan/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/harishreedharan/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/harishreedharan/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/harishreedharan/spark/labels{/name}","releases_url":"https://api.github.com/repos/harishreedharan/spark/releases{/id}","created_at":"2014-04-03T22:17:22Z","updated_at":"2014-07-10T22:11:34Z","pushed_at":"2015-04-10T06:08:07Z","git_url":"git://github.com/harishreedharan/spark.git","ssh_url":"git@github.com:harishreedharan/spark.git","clone_url":"https://github.com/harishreedharan/spark.git","svn_url":"https://github.com/harishreedharan/spark","homepage":null,"size":94882,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"470d7453a56c56a41b2851551fe1830065f88b2c","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-04-13T21:45:07Z","pushed_at":"2015-04-13T22:16:56Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1490601,"stargazers_count":3588,"watchers_count":3588,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3102,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":304,"forks":3102,"open_issues":304,"watchers":3588,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/4688"},"html":{"href":"https://github.com/apache/spark/pull/4688"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/4688"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/4688/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/4688/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/4688/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/e800c8b0929d2696aa421f98f89256ef9f498959"}}}}