{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/45790885","id":45790885,"diff_hunk":"@@ -0,0 +1,247 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.scheduler\n+\n+import java.util.concurrent.TimeUnit\n+import scala.collection.mutable\n+import org.apache.spark.SparkConf\n+import org.apache.spark.Success\n+import org.apache.spark.TaskEndReason\n+import org.apache.spark.annotation.DeveloperApi\n+import org.apache.spark.util.SystemClock\n+import org.apache.spark.util.ThreadUtils\n+import org.apache.spark.util.Utils\n+import org.apache.spark.util.Clock\n+\n+import com.google.common.annotations.VisibleForTesting\n+\n+/**\n+ * BlacklistTracker is design to track problematic executors and node on application level.\n+ * It is shared by all TaskSet, so that once a new TaskSet coming, it could be benefit from\n+ * previous experience of other TaskSet.\n+ *\n+ * Once task finished, the callback method in TaskSetManager should update\n+ * executorIdToFailureStatus Map.\n+ */\n+private[spark] class BlacklistTracker(\n+    sparkConf: SparkConf,\n+    clock: Clock = new SystemClock()) extends BlacklistCache{\n+  // maintain a ExecutorId --> FailureStatus HashMap\n+  private val executorIdToFailureStatus: mutable.HashMap[String, FailureStatus] = mutable.HashMap()\n+\n+  // Apply Strategy pattern here to change different blacklist detection logic\n+  private val strategy = BlacklistStrategy(sparkConf)\n+\n+  // A daemon thread to expire blacklist executor periodically\n+  private val scheduler = ThreadUtils.newDaemonSingleThreadScheduledExecutor(\n+      \"spark-scheduler-blacklist-expire-timer\")\n+\n+  private val recoverPeriod = sparkConf.getTimeAsSeconds(\n+    \"spark.scheduler.blacklist.recoverPeriod\", \"60s\")\n+\n+  def start(): Unit = {\n+    val scheduleTask = new Runnable() {\n+      override def run(): Unit = {\n+        Utils.logUncaughtExceptions(expireExecutorsInBlackList())\n+      }\n+    }\n+    scheduler.scheduleAtFixedRate(scheduleTask, 0L, recoverPeriod, TimeUnit.SECONDS)\n+  }\n+\n+  def stop(): Unit = {\n+    scheduler.shutdown()\n+    scheduler.awaitTermination(10, TimeUnit.SECONDS)\n+  }\n+\n+  // The actual implementation is delegated to strategy\n+  /** VisibleForTesting */\n+  private[scheduler] def expireExecutorsInBlackList(): Unit = synchronized {\n+    val updated = strategy.expireExecutorsInBlackList(executorIdToFailureStatus, clock)\n+    if (updated) {\n+      invalidAllCache()\n+    }\n+  }\n+\n+  def updateFailureExecutors(\n+      stageId: Int, info: TaskInfo, reason: TaskEndReason) : Unit = synchronized {\n+    val index = info.index\n+    val atomTask = BlacklistAtomTask(stageId, index)\n+    reason match {\n+      // If task succeeding, remove related record from executorIdToFailureStatus\n+      case Success =>\n+        removeFailureExecutorsForTaskId(info.executorId, stageId, index)\n+\n+      // If task failing, update latest failure time and failedTaskIds\n+      case _ =>\n+        val executorId = info.executorId\n+        executorIdToFailureStatus.get(executorId) match {\n+          case Some(failureStatus) =>\n+            failureStatus.updatedTime = clock.getTimeMillis()\n+            val failedTimes = failureStatus.numFailuresPerTask.getOrElse(atomTask, 0) + 1\n+            failureStatus.numFailuresPerTask.update(atomTask, failedTimes)","path":"core/src/main/scala/org/apache/spark/scheduler/BlacklistTracker.scala","position":96,"original_position":96,"commit_id":"1df57fb69aed1148afe0415e2eb5426c8c09f5f3","original_commit_id":"1df57fb69aed1148afe0415e2eb5426c8c09f5f3","user":{"login":"squito","id":71240,"avatar_url":"https://avatars.githubusercontent.com/u/71240?v=3","gravatar_id":"","url":"https://api.github.com/users/squito","html_url":"https://github.com/squito","followers_url":"https://api.github.com/users/squito/followers","following_url":"https://api.github.com/users/squito/following{/other_user}","gists_url":"https://api.github.com/users/squito/gists{/gist_id}","starred_url":"https://api.github.com/users/squito/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/squito/subscriptions","organizations_url":"https://api.github.com/users/squito/orgs","repos_url":"https://api.github.com/users/squito/repos","events_url":"https://api.github.com/users/squito/events{/privacy}","received_events_url":"https://api.github.com/users/squito/received_events","type":"User","site_admin":false},"body":"nit: rather than calling `update`, its more idiomatic to write this as `failureStatus.numFailuresPerTask(atomTask) = failedTimes` (scala syntax-sugar for the `update` method)","created_at":"2015-11-24T20:35:02Z","updated_at":"2015-11-24T20:35:02Z","html_url":"https://github.com/apache/spark/pull/8760#discussion_r45790885","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/8760","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/45790885"},"html":{"href":"https://github.com/apache/spark/pull/8760#discussion_r45790885"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/8760"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/8760","id":45013815,"html_url":"https://github.com/apache/spark/pull/8760","diff_url":"https://github.com/apache/spark/pull/8760.diff","patch_url":"https://github.com/apache/spark/pull/8760.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/8760","number":8760,"state":"open","locked":false,"title":"[Spark-8426] [scheduler] enhance blacklist mechanism","user":{"login":"mwws","id":13216322,"avatar_url":"https://avatars.githubusercontent.com/u/13216322?v=3","gravatar_id":"","url":"https://api.github.com/users/mwws","html_url":"https://github.com/mwws","followers_url":"https://api.github.com/users/mwws/followers","following_url":"https://api.github.com/users/mwws/following{/other_user}","gists_url":"https://api.github.com/users/mwws/gists{/gist_id}","starred_url":"https://api.github.com/users/mwws/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mwws/subscriptions","organizations_url":"https://api.github.com/users/mwws/orgs","repos_url":"https://api.github.com/users/mwws/repos","events_url":"https://api.github.com/users/mwws/events{/privacy}","received_events_url":"https://api.github.com/users/mwws/received_events","type":"User","site_admin":false},"body":"1. add a global BlacklistTracker which is able to share blacklisted executors information across TaskSetManager. Different BlacklistStrategy could be applied to determine which executors should be put in blacklist. And it's convenient to add new strategy later according to current design.\r\n2. remove duplicate \"failedExecutors\" list in TaskSetManager and make BlacklistTracker being the only interface for blacklist mechanism in codebase which is logically cleaner.\r\n3. for backward compatibility, implement DefaultStrategy to keep the same semantics as current by default \r\n4. apply BlacklistTracker in YARN to avoid allocating new executors on blacklisted node, the blacklisted node information will be passed to YARN ResourceManager. it affects in dynamic allocation mode.\r\n","created_at":"2015-09-15T06:40:58Z","updated_at":"2015-11-24T20:35:02Z","closed_at":null,"merged_at":null,"merge_commit_sha":"7ce221daf7d6ca2f50903ffff0edc0e405ec7a00","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/8760/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/8760/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/8760/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/1df57fb69aed1148afe0415e2eb5426c8c09f5f3","head":{"label":"mwws:SPARK-8426","ref":"SPARK-8426","sha":"1df57fb69aed1148afe0415e2eb5426c8c09f5f3","user":{"login":"mwws","id":13216322,"avatar_url":"https://avatars.githubusercontent.com/u/13216322?v=3","gravatar_id":"","url":"https://api.github.com/users/mwws","html_url":"https://github.com/mwws","followers_url":"https://api.github.com/users/mwws/followers","following_url":"https://api.github.com/users/mwws/following{/other_user}","gists_url":"https://api.github.com/users/mwws/gists{/gist_id}","starred_url":"https://api.github.com/users/mwws/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mwws/subscriptions","organizations_url":"https://api.github.com/users/mwws/orgs","repos_url":"https://api.github.com/users/mwws/repos","events_url":"https://api.github.com/users/mwws/events{/privacy}","received_events_url":"https://api.github.com/users/mwws/received_events","type":"User","site_admin":false},"repo":{"id":38867540,"name":"spark","full_name":"mwws/spark","owner":{"login":"mwws","id":13216322,"avatar_url":"https://avatars.githubusercontent.com/u/13216322?v=3","gravatar_id":"","url":"https://api.github.com/users/mwws","html_url":"https://github.com/mwws","followers_url":"https://api.github.com/users/mwws/followers","following_url":"https://api.github.com/users/mwws/following{/other_user}","gists_url":"https://api.github.com/users/mwws/gists{/gist_id}","starred_url":"https://api.github.com/users/mwws/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mwws/subscriptions","organizations_url":"https://api.github.com/users/mwws/orgs","repos_url":"https://api.github.com/users/mwws/repos","events_url":"https://api.github.com/users/mwws/events{/privacy}","received_events_url":"https://api.github.com/users/mwws/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/mwws/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/mwws/spark","forks_url":"https://api.github.com/repos/mwws/spark/forks","keys_url":"https://api.github.com/repos/mwws/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/mwws/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/mwws/spark/teams","hooks_url":"https://api.github.com/repos/mwws/spark/hooks","issue_events_url":"https://api.github.com/repos/mwws/spark/issues/events{/number}","events_url":"https://api.github.com/repos/mwws/spark/events","assignees_url":"https://api.github.com/repos/mwws/spark/assignees{/user}","branches_url":"https://api.github.com/repos/mwws/spark/branches{/branch}","tags_url":"https://api.github.com/repos/mwws/spark/tags","blobs_url":"https://api.github.com/repos/mwws/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/mwws/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/mwws/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/mwws/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/mwws/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/mwws/spark/languages","stargazers_url":"https://api.github.com/repos/mwws/spark/stargazers","contributors_url":"https://api.github.com/repos/mwws/spark/contributors","subscribers_url":"https://api.github.com/repos/mwws/spark/subscribers","subscription_url":"https://api.github.com/repos/mwws/spark/subscription","commits_url":"https://api.github.com/repos/mwws/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/mwws/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/mwws/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/mwws/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/mwws/spark/contents/{+path}","compare_url":"https://api.github.com/repos/mwws/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/mwws/spark/merges","archive_url":"https://api.github.com/repos/mwws/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/mwws/spark/downloads","issues_url":"https://api.github.com/repos/mwws/spark/issues{/number}","pulls_url":"https://api.github.com/repos/mwws/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/mwws/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/mwws/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/mwws/spark/labels{/name}","releases_url":"https://api.github.com/repos/mwws/spark/releases{/id}","created_at":"2015-07-10T07:35:43Z","updated_at":"2015-09-02T07:35:19Z","pushed_at":"2015-11-23T07:50:08Z","git_url":"git://github.com/mwws/spark.git","ssh_url":"git@github.com:mwws/spark.git","clone_url":"https://github.com/mwws/spark.git","svn_url":"https://github.com/mwws/spark","homepage":null,"size":135342,"stargazers_count":1,"watchers_count":1,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":1,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"bc5d6c03893a9bd340d6b94d3550e25648412241","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-11-24T18:08:42Z","pushed_at":"2015-11-24T20:24:13Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":386254,"stargazers_count":6330,"watchers_count":6330,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":5823,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":366,"forks":5823,"open_issues":366,"watchers":6330,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/8760"},"html":{"href":"https://github.com/apache/spark/pull/8760"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/8760"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/8760/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/8760/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/8760/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/1df57fb69aed1148afe0415e2eb5426c8c09f5f3"}}}}