{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/27754138","id":27754138,"diff_hunk":"@@ -609,24 +593,58 @@ private[spark] class Client(\n     throw new SparkException(\"While loop is depleted! This should never happen...\")\n   }\n \n+  private def formatReportDetails(report: ApplicationReport): String = {\n+    val details = Seq[(String, String)](\n+      (\"client token\", getClientToken(report)),\n+      (\"diagnostics\", report.getDiagnostics),\n+      (\"ApplicationMaster host\", report.getHost),\n+      (\"ApplicationMaster RPC port\", report.getRpcPort.toString),\n+      (\"queue\", report.getQueue),\n+      (\"start time\", report.getStartTime.toString),\n+      (\"final status\", report.getFinalApplicationStatus.toString),\n+      (\"tracking URL\", report.getTrackingUrl),\n+      (\"user\", report.getUser)\n+    )\n+\n+    // Use more loggable format if value is null or empty\n+    details.map { case (k, v) =>\n+      val newValue = Option(v).filter(_.nonEmpty).getOrElse(\"N/A\")\n+      s\"\\n\\t $k: $newValue\"\n+    }.mkString(\"\")\n+  }\n+\n   /**\n-   * Submit an application to the ResourceManager and monitor its state.\n-   * This continues until the application has exited for any reason.\n+   * Submit an application to the ResourceManager.\n+   * If set spark.yarn.submit.waitAppCompletion to true, it will stay alive\n+   * reporting the application's status until the application has exited for any reason.\n+   * Otherwise, the client process will exit after submission.\n    * If the application finishes with a failed, killed, or undefined status,\n    * throw an appropriate SparkException.\n    */\n   def run(): Unit = {\n-    val (yarnApplicationState, finalApplicationStatus) = monitorApplication(submitApplication())\n-    if (yarnApplicationState == YarnApplicationState.FAILED ||\n-      finalApplicationStatus == FinalApplicationStatus.FAILED) {\n-      throw new SparkException(\"Application finished with failed status\")\n-    }\n-    if (yarnApplicationState == YarnApplicationState.KILLED ||\n-      finalApplicationStatus == FinalApplicationStatus.KILLED) {\n-      throw new SparkException(\"Application is killed\")\n-    }\n-    if (finalApplicationStatus == FinalApplicationStatus.UNDEFINED) {\n-      throw new SparkException(\"The final status of application is undefined\")\n+    val appId = submitApplication()\n+    if (fireAndForget) {\n+      val report = getApplicationReport(appId)\n+      val state = report.getYarnApplicationState\n+      if (state == YarnApplicationState.FAILED || state == YarnApplicationState.KILLED) {\n+        logInfo(formatReportDetails(report))","path":"yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala","position":94,"original_position":94,"commit_id":"fea390dcd855e8060447c73254f19643f2b94bfd","original_commit_id":"fea390dcd855e8060447c73254f19643f2b94bfd","user":{"login":"tgravescs","id":4563792,"avatar_url":"https://avatars.githubusercontent.com/u/4563792?v=3","gravatar_id":"","url":"https://api.github.com/users/tgravescs","html_url":"https://github.com/tgravescs","followers_url":"https://api.github.com/users/tgravescs/followers","following_url":"https://api.github.com/users/tgravescs/following{/other_user}","gists_url":"https://api.github.com/users/tgravescs/gists{/gist_id}","starred_url":"https://api.github.com/users/tgravescs/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/tgravescs/subscriptions","organizations_url":"https://api.github.com/users/tgravescs/orgs","repos_url":"https://api.github.com/users/tgravescs/repos","events_url":"https://api.github.com/users/tgravescs/events{/privacy}","received_events_url":"https://api.github.com/users/tgravescs/received_events","type":"User","site_admin":false},"body":"this is kind of a nit but we could just move line 633-634 up before this if statement and we wouldn't need this second  logInfo(formatReportDetails(report))","created_at":"2015-04-03T20:37:54Z","updated_at":"2015-04-03T20:37:54Z","html_url":"https://github.com/apache/spark/pull/5297#discussion_r27754138","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/5297","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/27754138"},"html":{"href":"https://github.com/apache/spark/pull/5297#discussion_r27754138"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/5297"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/5297","id":32320919,"html_url":"https://github.com/apache/spark/pull/5297","diff_url":"https://github.com/apache/spark/pull/5297.diff","patch_url":"https://github.com/apache/spark/pull/5297.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/5297","number":5297,"state":"open","locked":false,"title":"[SPARK-3591][YARN]fire and forget for YARN cluster mode","user":{"login":"WangTaoTheTonic","id":5276001,"avatar_url":"https://avatars.githubusercontent.com/u/5276001?v=3","gravatar_id":"","url":"https://api.github.com/users/WangTaoTheTonic","html_url":"https://github.com/WangTaoTheTonic","followers_url":"https://api.github.com/users/WangTaoTheTonic/followers","following_url":"https://api.github.com/users/WangTaoTheTonic/following{/other_user}","gists_url":"https://api.github.com/users/WangTaoTheTonic/gists{/gist_id}","starred_url":"https://api.github.com/users/WangTaoTheTonic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/WangTaoTheTonic/subscriptions","organizations_url":"https://api.github.com/users/WangTaoTheTonic/orgs","repos_url":"https://api.github.com/users/WangTaoTheTonic/repos","events_url":"https://api.github.com/users/WangTaoTheTonic/events{/privacy}","received_events_url":"https://api.github.com/users/WangTaoTheTonic/received_events","type":"User","site_admin":false},"body":"https://issues.apache.org/jira/browse/SPARK-3591\r\n\r\nThe output after this patch:\r\n>doggie153:/opt/oss/spark-1.3.0-bin-hadoop2.4/bin # ./spark-submit  --class org.apache.spark.examples.SparkPi --master yarn-cluster ../lib/spark-examples*.jar\r\n15/03/31 21:15:25 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\r\n15/03/31 21:15:25 INFO RMProxy: Connecting to ResourceManager at doggie153/10.177.112.153:8032\r\n15/03/31 21:15:25 INFO Client: Requesting a new application from cluster with 4 NodeManagers\r\n15/03/31 21:15:25 INFO Client: Verifying our application has not requested more than the maximum memory capability of the cluster (8192 MB per container)\r\n15/03/31 21:15:25 INFO Client: Will allocate AM container, with 896 MB memory including 384 MB overhead\r\n15/03/31 21:15:25 INFO Client: Setting up container launch context for our AM\r\n15/03/31 21:15:25 INFO Client: Preparing resources for our AM container\r\n15/03/31 21:15:26 INFO Client: Uploading resource file:/opt/oss/spark-1.3.0-bin-hadoop2.4/lib/spark-assembly-1.4.0-SNAPSHOT-hadoop2.4.1.jar -> hdfs://doggie153:9000/user/root/.sparkStaging/application_1427257505534_0016/spark-assembly-1.4.0-SNAPSHOT-hadoop2.4.1.jar\r\n15/03/31 21:15:27 INFO Client: Uploading resource file:/opt/oss/spark-1.3.0-bin-hadoop2.4/lib/spark-examples-1.3.0-hadoop2.4.0.jar -> hdfs://doggie153:9000/user/root/.sparkStaging/application_1427257505534_0016/spark-examples-1.3.0-hadoop2.4.0.jar\r\n15/03/31 21:15:28 INFO Client: Setting up the launch environment for our AM container\r\n15/03/31 21:15:28 INFO SecurityManager: Changing view acls to: root\r\n15/03/31 21:15:28 INFO SecurityManager: Changing modify acls to: root\r\n15/03/31 21:15:28 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); users with modify permissions: Set(root)\r\n15/03/31 21:15:28 INFO Client: Submitting application 16 to ResourceManager\r\n15/03/31 21:15:28 INFO YarnClientImpl: Submitted application application_1427257505534_0016\r\n15/03/31 21:15:28 INFO Client: ... waiting before polling ResourceManager for application state\r\n15/03/31 21:15:33 INFO Client: ... polling ResourceManager for application state\r\n15/03/31 21:15:33 INFO Client: Application report for application_1427257505534_0016 (state: RUNNING)\r\n15/03/31 21:15:33 INFO Client: \r\n         client token: N/A\r\n         diagnostics: N/A\r\n         ApplicationMaster host: doggie157\r\n         ApplicationMaster RPC port: 0\r\n         queue: default\r\n         start time: 1427807728307\r\n         final status: UNDEFINED\r\n         tracking URL: http://doggie153:8088/proxy/application_1427257505534_0016/\r\n         user: root\r\n\r\n/cc @andrewor14","created_at":"2015-03-31T13:17:51Z","updated_at":"2015-04-03T20:37:54Z","closed_at":null,"merged_at":null,"merge_commit_sha":"8b731a07ffe6bcf76607fe3bb6e725bbe8c98770","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/5297/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/5297/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/5297/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/fea390dcd855e8060447c73254f19643f2b94bfd","head":{"label":"WangTaoTheTonic:SPARK-3591","ref":"SPARK-3591","sha":"fea390dcd855e8060447c73254f19643f2b94bfd","user":{"login":"WangTaoTheTonic","id":5276001,"avatar_url":"https://avatars.githubusercontent.com/u/5276001?v=3","gravatar_id":"","url":"https://api.github.com/users/WangTaoTheTonic","html_url":"https://github.com/WangTaoTheTonic","followers_url":"https://api.github.com/users/WangTaoTheTonic/followers","following_url":"https://api.github.com/users/WangTaoTheTonic/following{/other_user}","gists_url":"https://api.github.com/users/WangTaoTheTonic/gists{/gist_id}","starred_url":"https://api.github.com/users/WangTaoTheTonic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/WangTaoTheTonic/subscriptions","organizations_url":"https://api.github.com/users/WangTaoTheTonic/orgs","repos_url":"https://api.github.com/users/WangTaoTheTonic/repos","events_url":"https://api.github.com/users/WangTaoTheTonic/events{/privacy}","received_events_url":"https://api.github.com/users/WangTaoTheTonic/received_events","type":"User","site_admin":false},"repo":{"id":18954614,"name":"spark","full_name":"WangTaoTheTonic/spark","owner":{"login":"WangTaoTheTonic","id":5276001,"avatar_url":"https://avatars.githubusercontent.com/u/5276001?v=3","gravatar_id":"","url":"https://api.github.com/users/WangTaoTheTonic","html_url":"https://github.com/WangTaoTheTonic","followers_url":"https://api.github.com/users/WangTaoTheTonic/followers","following_url":"https://api.github.com/users/WangTaoTheTonic/following{/other_user}","gists_url":"https://api.github.com/users/WangTaoTheTonic/gists{/gist_id}","starred_url":"https://api.github.com/users/WangTaoTheTonic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/WangTaoTheTonic/subscriptions","organizations_url":"https://api.github.com/users/WangTaoTheTonic/orgs","repos_url":"https://api.github.com/users/WangTaoTheTonic/repos","events_url":"https://api.github.com/users/WangTaoTheTonic/events{/privacy}","received_events_url":"https://api.github.com/users/WangTaoTheTonic/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/WangTaoTheTonic/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/WangTaoTheTonic/spark","forks_url":"https://api.github.com/repos/WangTaoTheTonic/spark/forks","keys_url":"https://api.github.com/repos/WangTaoTheTonic/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/WangTaoTheTonic/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/WangTaoTheTonic/spark/teams","hooks_url":"https://api.github.com/repos/WangTaoTheTonic/spark/hooks","issue_events_url":"https://api.github.com/repos/WangTaoTheTonic/spark/issues/events{/number}","events_url":"https://api.github.com/repos/WangTaoTheTonic/spark/events","assignees_url":"https://api.github.com/repos/WangTaoTheTonic/spark/assignees{/user}","branches_url":"https://api.github.com/repos/WangTaoTheTonic/spark/branches{/branch}","tags_url":"https://api.github.com/repos/WangTaoTheTonic/spark/tags","blobs_url":"https://api.github.com/repos/WangTaoTheTonic/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/WangTaoTheTonic/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/WangTaoTheTonic/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/WangTaoTheTonic/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/WangTaoTheTonic/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/WangTaoTheTonic/spark/languages","stargazers_url":"https://api.github.com/repos/WangTaoTheTonic/spark/stargazers","contributors_url":"https://api.github.com/repos/WangTaoTheTonic/spark/contributors","subscribers_url":"https://api.github.com/repos/WangTaoTheTonic/spark/subscribers","subscription_url":"https://api.github.com/repos/WangTaoTheTonic/spark/subscription","commits_url":"https://api.github.com/repos/WangTaoTheTonic/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/WangTaoTheTonic/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/WangTaoTheTonic/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/WangTaoTheTonic/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/WangTaoTheTonic/spark/contents/{+path}","compare_url":"https://api.github.com/repos/WangTaoTheTonic/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/WangTaoTheTonic/spark/merges","archive_url":"https://api.github.com/repos/WangTaoTheTonic/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/WangTaoTheTonic/spark/downloads","issues_url":"https://api.github.com/repos/WangTaoTheTonic/spark/issues{/number}","pulls_url":"https://api.github.com/repos/WangTaoTheTonic/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/WangTaoTheTonic/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/WangTaoTheTonic/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/WangTaoTheTonic/spark/labels{/name}","releases_url":"https://api.github.com/repos/WangTaoTheTonic/spark/releases{/id}","created_at":"2014-04-20T01:10:21Z","updated_at":"2014-11-25T14:23:14Z","pushed_at":"2015-04-03T09:27:59Z","git_url":"git://github.com/WangTaoTheTonic/spark.git","ssh_url":"git@github.com:WangTaoTheTonic/spark.git","clone_url":"https://github.com/WangTaoTheTonic/spark.git","svn_url":"https://github.com/WangTaoTheTonic/spark","homepage":null,"size":108986,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"5db89127e72630aec7c5552f2c84018ae18d03fe","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-04-03T19:35:43Z","pushed_at":"2015-04-03T20:36:02Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1469650,"stargazers_count":3483,"watchers_count":3483,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3001,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":309,"forks":3001,"open_issues":309,"watchers":3483,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/5297"},"html":{"href":"https://github.com/apache/spark/pull/5297"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/5297"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/5297/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/5297/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/5297/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/fea390dcd855e8060447c73254f19643f2b94bfd"}}}}