{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/26179915","id":26179915,"diff_hunk":"@@ -108,6 +110,53 @@ class GradientBoostedTreesModel(\n   }\n \n   override protected def formatVersion: String = TreeEnsembleModel.SaveLoadV1_0.thisFormatVersion\n+\n+  /**\n+   * Method to compute error or loss for every iteration of gradient boosting.\n+   * @param data: RDD of [[org.apache.spark.mllib.regression.LabeledPoint]]\n+   * @param loss: evaluation metric.\n+   * @return an array with index i having the losses or errors for the ensemble\n+   *         containing trees 1 to i + 1\n+   */\n+  def evaluateEachIteration(\n+      data: RDD[LabeledPoint],\n+      loss: Loss) : Array[Double] = {\n+\n+    val sc = data.sparkContext\n+    val remappedData = algo match {\n+      case Classification => data.map(x => new LabeledPoint((x.label * 2) - 1, x.features))\n+      case _ => data\n+    }\n+    val initialTree = trees(0)\n+    val numIterations = trees.length\n+    val evaluationArray = Array.fill(numIterations)(0.0)\n+\n+    // Initial weight is 1.0\n+    var predictionErrorModel = remappedData.map {i =>\n+      val pred = initialTree.predict(i.features)\n+      val error = loss.computeError(i, pred)\n+      (pred, error)\n+    }\n+    evaluationArray(0) = predictionErrorModel.values.mean()\n+\n+    // Avoid the model being copied across numIterations.\n+    val broadcastTrees = sc.broadcast(trees)\n+    val broadcastWeights = sc.broadcast(treeWeights)\n+\n+    (1 until numIterations).map {nTree =>\n+      predictionErrorModel = (remappedData zip predictionErrorModel) map {\n+        case (point, (pred, error)) => {\n+          val newPred = pred + (","path":"mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala","position":52,"original_position":52,"commit_id":"bc99ac68fdd81e5f0569a9cfd2d1eb97498885b6","original_commit_id":"bc99ac68fdd81e5f0569a9cfd2d1eb97498885b6","user":{"login":"jkbradley","id":5084283,"avatar_url":"https://avatars.githubusercontent.com/u/5084283?v=3","gravatar_id":"","url":"https://api.github.com/users/jkbradley","html_url":"https://github.com/jkbradley","followers_url":"https://api.github.com/users/jkbradley/followers","following_url":"https://api.github.com/users/jkbradley/following{/other_user}","gists_url":"https://api.github.com/users/jkbradley/gists{/gist_id}","starred_url":"https://api.github.com/users/jkbradley/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jkbradley/subscriptions","organizations_url":"https://api.github.com/users/jkbradley/orgs","repos_url":"https://api.github.com/users/jkbradley/repos","events_url":"https://api.github.com/users/jkbradley/events{/privacy}","received_events_url":"https://api.github.com/users/jkbradley/received_events","type":"User","site_admin":false},"body":"style: parenthesis on next line","created_at":"2015-03-11T00:21:03Z","updated_at":"2015-03-11T00:21:03Z","html_url":"https://github.com/apache/spark/pull/4906#discussion_r26179915","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/4906","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/26179915"},"html":{"href":"https://github.com/apache/spark/pull/4906#discussion_r26179915"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/4906"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/4906","id":30539865,"html_url":"https://github.com/apache/spark/pull/4906","diff_url":"https://github.com/apache/spark/pull/4906.diff","patch_url":"https://github.com/apache/spark/pull/4906.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/4906","number":4906,"state":"open","locked":false,"title":"[SPARK-6025] [MLlib] Add helper method evaluateEachIteration to extract learning curve","user":{"login":"MechCoder","id":1867024,"avatar_url":"https://avatars.githubusercontent.com/u/1867024?v=3","gravatar_id":"","url":"https://api.github.com/users/MechCoder","html_url":"https://github.com/MechCoder","followers_url":"https://api.github.com/users/MechCoder/followers","following_url":"https://api.github.com/users/MechCoder/following{/other_user}","gists_url":"https://api.github.com/users/MechCoder/gists{/gist_id}","starred_url":"https://api.github.com/users/MechCoder/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/MechCoder/subscriptions","organizations_url":"https://api.github.com/users/MechCoder/orgs","repos_url":"https://api.github.com/users/MechCoder/repos","events_url":"https://api.github.com/users/MechCoder/events{/privacy}","received_events_url":"https://api.github.com/users/MechCoder/received_events","type":"User","site_admin":false},"body":"Added evaluateEachIteration to allow the user to manually extract the error for each iteration of GradientBoosting. The internal optimisation can be dealt with later.","created_at":"2015-03-05T09:01:27Z","updated_at":"2015-03-11T00:21:04Z","closed_at":null,"merged_at":null,"merge_commit_sha":"aba797e50070d3178542f0eaefec8d4f24f18ca2","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/4906/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/4906/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments{/number}","comments_url":"https://api.github.com/repos/apache/spark/issues/4906/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/bc99ac68fdd81e5f0569a9cfd2d1eb97498885b6","head":{"label":"MechCoder:spark-6025","ref":"spark-6025","sha":"bc99ac68fdd81e5f0569a9cfd2d1eb97498885b6","user":{"login":"MechCoder","id":1867024,"avatar_url":"https://avatars.githubusercontent.com/u/1867024?v=3","gravatar_id":"","url":"https://api.github.com/users/MechCoder","html_url":"https://github.com/MechCoder","followers_url":"https://api.github.com/users/MechCoder/followers","following_url":"https://api.github.com/users/MechCoder/following{/other_user}","gists_url":"https://api.github.com/users/MechCoder/gists{/gist_id}","starred_url":"https://api.github.com/users/MechCoder/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/MechCoder/subscriptions","organizations_url":"https://api.github.com/users/MechCoder/orgs","repos_url":"https://api.github.com/users/MechCoder/repos","events_url":"https://api.github.com/users/MechCoder/events{/privacy}","received_events_url":"https://api.github.com/users/MechCoder/received_events","type":"User","site_admin":false},"repo":{"id":23950071,"name":"spark","full_name":"MechCoder/spark","owner":{"login":"MechCoder","id":1867024,"avatar_url":"https://avatars.githubusercontent.com/u/1867024?v=3","gravatar_id":"","url":"https://api.github.com/users/MechCoder","html_url":"https://github.com/MechCoder","followers_url":"https://api.github.com/users/MechCoder/followers","following_url":"https://api.github.com/users/MechCoder/following{/other_user}","gists_url":"https://api.github.com/users/MechCoder/gists{/gist_id}","starred_url":"https://api.github.com/users/MechCoder/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/MechCoder/subscriptions","organizations_url":"https://api.github.com/users/MechCoder/orgs","repos_url":"https://api.github.com/users/MechCoder/repos","events_url":"https://api.github.com/users/MechCoder/events{/privacy}","received_events_url":"https://api.github.com/users/MechCoder/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/MechCoder/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/MechCoder/spark","forks_url":"https://api.github.com/repos/MechCoder/spark/forks","keys_url":"https://api.github.com/repos/MechCoder/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/MechCoder/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/MechCoder/spark/teams","hooks_url":"https://api.github.com/repos/MechCoder/spark/hooks","issue_events_url":"https://api.github.com/repos/MechCoder/spark/issues/events{/number}","events_url":"https://api.github.com/repos/MechCoder/spark/events","assignees_url":"https://api.github.com/repos/MechCoder/spark/assignees{/user}","branches_url":"https://api.github.com/repos/MechCoder/spark/branches{/branch}","tags_url":"https://api.github.com/repos/MechCoder/spark/tags","blobs_url":"https://api.github.com/repos/MechCoder/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/MechCoder/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/MechCoder/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/MechCoder/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/MechCoder/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/MechCoder/spark/languages","stargazers_url":"https://api.github.com/repos/MechCoder/spark/stargazers","contributors_url":"https://api.github.com/repos/MechCoder/spark/contributors","subscribers_url":"https://api.github.com/repos/MechCoder/spark/subscribers","subscription_url":"https://api.github.com/repos/MechCoder/spark/subscription","commits_url":"https://api.github.com/repos/MechCoder/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/MechCoder/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/MechCoder/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/MechCoder/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/MechCoder/spark/contents/{+path}","compare_url":"https://api.github.com/repos/MechCoder/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/MechCoder/spark/merges","archive_url":"https://api.github.com/repos/MechCoder/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/MechCoder/spark/downloads","issues_url":"https://api.github.com/repos/MechCoder/spark/issues{/number}","pulls_url":"https://api.github.com/repos/MechCoder/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/MechCoder/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/MechCoder/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/MechCoder/spark/labels{/name}","releases_url":"https://api.github.com/repos/MechCoder/spark/releases{/id}","created_at":"2014-09-12T06:52:44Z","updated_at":"2015-03-09T16:40:50Z","pushed_at":"2015-03-10T19:42:10Z","git_url":"git://github.com/MechCoder/spark.git","ssh_url":"git@github.com:MechCoder/spark.git","clone_url":"https://github.com/MechCoder/spark.git","svn_url":"https://github.com/MechCoder/spark","homepage":null,"size":91413,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"7c7d2d5e093b0623edc75dd166ec1179b4e62062","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments{/number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-03-10T20:45:16Z","pushed_at":"2015-03-10T11:03:38Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1381410,"stargazers_count":3135,"watchers_count":3135,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":2722,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":286,"forks":2722,"open_issues":286,"watchers":3135,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/4906"},"html":{"href":"https://github.com/apache/spark/pull/4906"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/4906"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/4906/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/4906/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments{/number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/4906/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/bc99ac68fdd81e5f0569a9cfd2d1eb97498885b6"}}}}