{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/28473266","id":28473266,"diff_hunk":"@@ -63,45 +71,109 @@ private[deploy] class StandaloneRestClient extends Logging {\n    * it to the user. Otherwise, report the error message provided by the server.\n    */\n   private[rest] def createSubmission(\n-      master: String,\n       request: CreateSubmissionRequest): SubmitRestProtocolResponse = {\n     logInfo(s\"Submitting a request to launch an application in $master.\")\n-    validateMaster(master)\n-    val url = getSubmitUrl(master)\n-    val response = postJson(url, request.toJson)\n-    response match {\n-      case s: CreateSubmissionResponse =>\n-        reportSubmissionStatus(master, s)\n-        handleRestResponse(s)\n-      case unexpected =>\n-        handleUnexpectedRestResponse(unexpected)\n+    var suc: Boolean = false\n+    var response: SubmitRestProtocolResponse = null\n+    for (m <- masters if !suc) {\n+      validateMaster(m)\n+      val url = getSubmitUrl(m)\n+      try {\n+        response = postJson(url, request.toJson)\n+        response match {\n+          case s: CreateSubmissionResponse =>\n+            if (s.success) {\n+              reportSubmissionStatus(s)\n+              handleRestResponse(s)\n+              suc = true\n+            }\n+          case unexpected =>\n+            handleUnexpectedRestResponse(unexpected)\n+        }\n+      } catch {\n+        case unreachable @ (_: FileNotFoundException | _: SocketException | _: ConnectException) =>\n+          if(handleConnectionException(m)) {\n+            throw new SubmitRestConnectionException(\n+              s\"Unable to connect to server\", unreachable)\n+          }\n+        case malformed @ (_: JsonProcessingException | _: SubmitRestProtocolException) =>\n+          if(handleConnectionException(m)) {\n+            throw new SubmitRestProtocolException(\n+              \"Malformed response received from server\", malformed)\n+          }\n+      }\n     }\n     response\n   }\n \n   /** Request that the server kill the specified submission. */\n-  def killSubmission(master: String, submissionId: String): SubmitRestProtocolResponse = {\n+  def killSubmission(submissionId: String): SubmitRestProtocolResponse = {\n     logInfo(s\"Submitting a request to kill submission $submissionId in $master.\")\n-    validateMaster(master)\n-    val response = post(getKillUrl(master, submissionId))\n-    response match {\n-      case k: KillSubmissionResponse => handleRestResponse(k)\n-      case unexpected => handleUnexpectedRestResponse(unexpected)\n+    var suc: Boolean = false\n+    var response: SubmitRestProtocolResponse = null\n+    for (m <- masters if !suc) {\n+      validateMaster(m)\n+      val url = getKillUrl(m, submissionId)\n+      try {\n+        response = post(url)\n+        response match {\n+          case k: KillSubmissionResponse =>\n+            if (!k.message.startsWith(Utils.MASTER_NOT_ALIVE_STRING)) {\n+              handleRestResponse(k)\n+              suc = true\n+            }\n+          case unexpected =>\n+            handleUnexpectedRestResponse(unexpected)\n+        }\n+      } catch {\n+        case unreachable @ (_: FileNotFoundException | _: SocketException | _: ConnectException) =>\n+          if(handleConnectionException(m)) {\n+            throw new SubmitRestConnectionException(\n+              s\"Unable to connect to server\", unreachable)\n+          }\n+        case malformed @ (_: JsonProcessingException | _: SubmitRestProtocolException) =>\n+          if(handleConnectionException(m)) {\n+            throw new SubmitRestProtocolException(\n+              \"Malformed response received from server\", malformed)\n+          }\n+      }\n     }\n     response\n   }\n \n   /** Request the status of a submission from the server. */\n   def requestSubmissionStatus(\n-      master: String,\n       submissionId: String,\n       quiet: Boolean = false): SubmitRestProtocolResponse = {\n     logInfo(s\"Submitting a request for the status of submission $submissionId in $master.\")\n-    validateMaster(master)\n-    val response = get(getStatusUrl(master, submissionId))\n-    response match {\n-      case s: SubmissionStatusResponse => if (!quiet) { handleRestResponse(s) }\n-      case unexpected => handleUnexpectedRestResponse(unexpected)\n+    var suc: Boolean = false\n+    var response: SubmitRestProtocolResponse = null\n+    for (m <- masters) {\n+      validateMaster(m)\n+      val url = getStatusUrl(m, submissionId)\n+      try {\n+        response = get(url)\n+        response match {\n+          case s: SubmissionStatusResponse if s.success =>\n+            if (!quiet) {\n+              handleRestResponse(s)\n+            }\n+            suc = true\n+          case unexpected =>\n+            handleUnexpectedRestResponse(unexpected)\n+        }\n+      } catch {\n+        case unreachable @ (_: FileNotFoundException | _: SocketException | _: ConnectException) =>\n+          if(handleConnectionException(m)) {\n+            throw new SubmitRestConnectionException(\n+              s\"Unable to connect to server\", unreachable)\n+          }\n+        case malformed @ (_: JsonProcessingException | _: SubmitRestProtocolException) =>\n+          if(handleConnectionException(m)) {","path":"core/src/main/scala/org/apache/spark/deploy/rest/StandaloneRestClient.scala","position":155,"original_position":155,"commit_id":"979760c36338345e4e52152517eb83c5a27b63e9","original_commit_id":"979760c36338345e4e52152517eb83c5a27b63e9","user":{"login":"andrewor14","id":2133137,"avatar_url":"https://avatars.githubusercontent.com/u/2133137?v=3","gravatar_id":"","url":"https://api.github.com/users/andrewor14","html_url":"https://github.com/andrewor14","followers_url":"https://api.github.com/users/andrewor14/followers","following_url":"https://api.github.com/users/andrewor14/following{/other_user}","gists_url":"https://api.github.com/users/andrewor14/gists{/gist_id}","starred_url":"https://api.github.com/users/andrewor14/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/andrewor14/subscriptions","organizations_url":"https://api.github.com/users/andrewor14/orgs","repos_url":"https://api.github.com/users/andrewor14/repos","events_url":"https://api.github.com/users/andrewor14/events{/privacy}","received_events_url":"https://api.github.com/users/andrewor14/received_events","type":"User","site_admin":false},"body":"This is not a connection exception. We should just throw the exception here","created_at":"2015-04-15T23:30:48Z","updated_at":"2015-04-15T23:30:48Z","html_url":"https://github.com/apache/spark/pull/5116#discussion_r28473266","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/5116","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/28473266"},"html":{"href":"https://github.com/apache/spark/pull/5116#discussion_r28473266"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/5116"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/5116","id":31668448,"html_url":"https://github.com/apache/spark/pull/5116","diff_url":"https://github.com/apache/spark/pull/5116.diff","patch_url":"https://github.com/apache/spark/pull/5116.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/5116","number":5116,"state":"open","locked":false,"title":"[SPARK-6443][Spark Submit]Could not submit app in standalone cluster mode when HA is enabled","user":{"login":"WangTaoTheTonic","id":5276001,"avatar_url":"https://avatars.githubusercontent.com/u/5276001?v=3","gravatar_id":"","url":"https://api.github.com/users/WangTaoTheTonic","html_url":"https://github.com/WangTaoTheTonic","followers_url":"https://api.github.com/users/WangTaoTheTonic/followers","following_url":"https://api.github.com/users/WangTaoTheTonic/following{/other_user}","gists_url":"https://api.github.com/users/WangTaoTheTonic/gists{/gist_id}","starred_url":"https://api.github.com/users/WangTaoTheTonic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/WangTaoTheTonic/subscriptions","organizations_url":"https://api.github.com/users/WangTaoTheTonic/orgs","repos_url":"https://api.github.com/users/WangTaoTheTonic/repos","events_url":"https://api.github.com/users/WangTaoTheTonic/events{/privacy}","received_events_url":"https://api.github.com/users/WangTaoTheTonic/received_events","type":"User","site_admin":false},"body":"**3/26 update:**\r\n* Akka-based:\r\n  Use an array of `ActorSelection` to represent multiple master. Add an `activeMasterActor` for query status of driver. And will add lost masters( including the standby one) to `lostMasters`.\r\n  When size of `lostMasters` equals or greater than # of all masters, we should give an error that all masters are not avalible.\r\n\r\n* Rest-based:\r\n  When all masters are not available(throw an exception), we use akka gateway to submit apps.\r\n\r\nI have tested simply on standalone HA cluster(with two masters alive and one alive/one dead), it worked.\r\n\r\nThere might remains some issues on style or message print, but we can check the solution then fix them together.\r\n\r\n/cc @srowen @andrewor14 \r\n","created_at":"2015-03-21T08:46:55Z","updated_at":"2015-04-15T23:30:48Z","closed_at":null,"merged_at":null,"merge_commit_sha":"2dcb67d5b4d95c9617d3105c588039e65fce6ef4","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/5116/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/5116/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/5116/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/979760c36338345e4e52152517eb83c5a27b63e9","head":{"label":"WangTaoTheTonic:SPARK-6443","ref":"SPARK-6443","sha":"979760c36338345e4e52152517eb83c5a27b63e9","user":{"login":"WangTaoTheTonic","id":5276001,"avatar_url":"https://avatars.githubusercontent.com/u/5276001?v=3","gravatar_id":"","url":"https://api.github.com/users/WangTaoTheTonic","html_url":"https://github.com/WangTaoTheTonic","followers_url":"https://api.github.com/users/WangTaoTheTonic/followers","following_url":"https://api.github.com/users/WangTaoTheTonic/following{/other_user}","gists_url":"https://api.github.com/users/WangTaoTheTonic/gists{/gist_id}","starred_url":"https://api.github.com/users/WangTaoTheTonic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/WangTaoTheTonic/subscriptions","organizations_url":"https://api.github.com/users/WangTaoTheTonic/orgs","repos_url":"https://api.github.com/users/WangTaoTheTonic/repos","events_url":"https://api.github.com/users/WangTaoTheTonic/events{/privacy}","received_events_url":"https://api.github.com/users/WangTaoTheTonic/received_events","type":"User","site_admin":false},"repo":{"id":18954614,"name":"spark","full_name":"WangTaoTheTonic/spark","owner":{"login":"WangTaoTheTonic","id":5276001,"avatar_url":"https://avatars.githubusercontent.com/u/5276001?v=3","gravatar_id":"","url":"https://api.github.com/users/WangTaoTheTonic","html_url":"https://github.com/WangTaoTheTonic","followers_url":"https://api.github.com/users/WangTaoTheTonic/followers","following_url":"https://api.github.com/users/WangTaoTheTonic/following{/other_user}","gists_url":"https://api.github.com/users/WangTaoTheTonic/gists{/gist_id}","starred_url":"https://api.github.com/users/WangTaoTheTonic/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/WangTaoTheTonic/subscriptions","organizations_url":"https://api.github.com/users/WangTaoTheTonic/orgs","repos_url":"https://api.github.com/users/WangTaoTheTonic/repos","events_url":"https://api.github.com/users/WangTaoTheTonic/events{/privacy}","received_events_url":"https://api.github.com/users/WangTaoTheTonic/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/WangTaoTheTonic/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/WangTaoTheTonic/spark","forks_url":"https://api.github.com/repos/WangTaoTheTonic/spark/forks","keys_url":"https://api.github.com/repos/WangTaoTheTonic/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/WangTaoTheTonic/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/WangTaoTheTonic/spark/teams","hooks_url":"https://api.github.com/repos/WangTaoTheTonic/spark/hooks","issue_events_url":"https://api.github.com/repos/WangTaoTheTonic/spark/issues/events{/number}","events_url":"https://api.github.com/repos/WangTaoTheTonic/spark/events","assignees_url":"https://api.github.com/repos/WangTaoTheTonic/spark/assignees{/user}","branches_url":"https://api.github.com/repos/WangTaoTheTonic/spark/branches{/branch}","tags_url":"https://api.github.com/repos/WangTaoTheTonic/spark/tags","blobs_url":"https://api.github.com/repos/WangTaoTheTonic/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/WangTaoTheTonic/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/WangTaoTheTonic/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/WangTaoTheTonic/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/WangTaoTheTonic/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/WangTaoTheTonic/spark/languages","stargazers_url":"https://api.github.com/repos/WangTaoTheTonic/spark/stargazers","contributors_url":"https://api.github.com/repos/WangTaoTheTonic/spark/contributors","subscribers_url":"https://api.github.com/repos/WangTaoTheTonic/spark/subscribers","subscription_url":"https://api.github.com/repos/WangTaoTheTonic/spark/subscription","commits_url":"https://api.github.com/repos/WangTaoTheTonic/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/WangTaoTheTonic/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/WangTaoTheTonic/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/WangTaoTheTonic/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/WangTaoTheTonic/spark/contents/{+path}","compare_url":"https://api.github.com/repos/WangTaoTheTonic/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/WangTaoTheTonic/spark/merges","archive_url":"https://api.github.com/repos/WangTaoTheTonic/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/WangTaoTheTonic/spark/downloads","issues_url":"https://api.github.com/repos/WangTaoTheTonic/spark/issues{/number}","pulls_url":"https://api.github.com/repos/WangTaoTheTonic/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/WangTaoTheTonic/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/WangTaoTheTonic/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/WangTaoTheTonic/spark/labels{/name}","releases_url":"https://api.github.com/repos/WangTaoTheTonic/spark/releases{/id}","created_at":"2014-04-20T01:10:21Z","updated_at":"2014-11-25T14:23:14Z","pushed_at":"2015-04-15T06:09:47Z","git_url":"git://github.com/WangTaoTheTonic/spark.git","ssh_url":"git@github.com:WangTaoTheTonic/spark.git","clone_url":"https://github.com/WangTaoTheTonic/spark.git","svn_url":"https://github.com/WangTaoTheTonic/spark","homepage":null,"size":111046,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"b65bad65c3500475b974ca0219f218eef296db2c","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-04-15T23:15:29Z","pushed_at":"2015-04-15T23:22:54Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1521363,"stargazers_count":3623,"watchers_count":3623,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":3125,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":286,"forks":3125,"open_issues":286,"watchers":3623,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/5116"},"html":{"href":"https://github.com/apache/spark/pull/5116"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/5116"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/5116/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/5116/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/5116/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/979760c36338345e4e52152517eb83c5a27b63e9"}}}}