{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/24392639","id":24392639,"diff_hunk":"@@ -0,0 +1,176 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.examples.mllib\n+\n+import org.apache.log4j.{Level, Logger}\n+import org.apache.spark.mllib.clustering.PowerIterationClustering\n+import org.apache.spark.rdd.RDD\n+import org.apache.spark.{SparkConf, SparkContext}\n+import scopt.OptionParser\n+\n+/**\n+ * An example Power Iteration Clustering app.  Takes an input of K concentric circles\n+ * with a total of \"n\" sampled points (total here means \"across ALL of the circles\").\n+ * The output should be K clusters - each cluster containing precisely the points associated\n+ * with each of the input circles.\n+ *\n+ * Run with\n+ * {{{\n+ * ./bin/run-example org.apache.spark.examples.mllib.PowerIterationClusteringExample [options]\n+ *\n+ * Where options include:\n+ *   k:  Number of circles/ clusters\n+ *   n:  Total number of sampled points. There are proportionally more points within the\n+ *      outer/larger circles\n+ *   numIterations:   Number of Power Iterations\n+ *   outerRadius:  radius of the outermost of the concentric circles\n+ * }}}\n+ *\n+ * Here is a sample run and output:\n+ *\n+ * ./bin/run-example org.apache.spark.examples.mllib.PowerIterationClusteringExample -k 3 --n 30 --numIterations 15\n+ *\n+ * Cluster assignments: 1 -> [0,1,2,3,4],2 -> [5,6,7,8,9,10,11,12,13,14],\n+ *    0 -> [15,16,17,18,19,20,21,22,23,24,25,26,27,28,29]\n+ *\n+ *\n+ * If you use it as a template to create your own app, please use `spark-submit` to submit your app.\n+ */\n+object PowerIterationClusteringExample {\n+\n+  case class Params(\n+      input: String = null,\n+      k: Int = 3,\n+      numPoints: Int = 30,\n+      numIterations: Int = 10,\n+      outerRadius: Double = 3.0\n+      ) extends AbstractParams[Params]\n+\n+  def main(args: Array[String]) {\n+    val defaultParams = Params()\n+\n+    val parser = new OptionParser[Params](\"PIC Circles\") {\n+      head(\"PowerIterationClusteringExample: an example PIC app using concentric circles.\")\n+      opt[Int]('k', \"k\")\n+        .text(s\"number of circles (/clusters), default: ${defaultParams.k}\")\n+        .action((x, c) => c.copy(k = x))\n+      opt[Int]('n', \"n\")\n+        .text(s\"number of points, default: ${defaultParams.numPoints}\")\n+        .action((x, c) => c.copy(numPoints = x))\n+      opt[Int](\"numIterations\")\n+        .text(s\"number of iterations, default: ${defaultParams.numIterations}\")\n+        .action((x, c) => c.copy(numIterations = x))\n+    }\n+\n+    parser.parse(args, defaultParams).map { params =>\n+      run(params)\n+    }.getOrElse {\n+      sys.exit(1)\n+    }\n+  }\n+\n+  def generateCircle(n: Int, r: Double): Array[(Double, Double)] = {\n+    val pi2 = 2 * math.Pi\n+    (0.0 until pi2 by pi2 / n).map { x =>\n+      (r * math.cos(x), r * math.sin(x))\n+    }.toArray\n+  }\n+\n+  def generateCirclesRdd(sc: SparkContext, nCircles: Int = 3, nTotalPoints: Int = 30,\n+                         outerRadius: Double):\n+  RDD[(Long, Long, Double)] = {\n+    // The circles are generated as follows:\n+    // The Radii are equal to the largestRadius/(C - circleIndex)\n+    //   where  C=Number of circles\n+    //      and the circleIndex is 0 for the innermost and (nCircles-1) for the outermost circle\n+    // The number of points in each circle (and thus in each final cluster) is:\n+    //     x, 2x, .., nCircles*x\n+    //     Where x is found from  x = N * C(C+1)/2\n+    //  The # points in the LAST circle is adjusted downwards so that the total sum is equal\n+    //  to the nTotalPoints\n+\n+    val smallestRad = math.ceil(nTotalPoints / (nCircles * (nCircles + 1) / 2.0))\n+    var groupSizes = (1 to nCircles).map(gs => (gs * smallestRad).toInt)\n+    groupSizes.zipWithIndex.map { case (gs, ix) =>\n+      ix match {\n+        case _ if ix == groupSizes.length => gs - (groupSizes.sum - nTotalPoints)\n+        case _ => gs\n+      }\n+    }\n+\n+    val radii = for (cx <- 0 until nCircles) yield {\n+      cx match {\n+        case 0 => 0.1 * outerRadius / nCircles\n+        case _ if cx == nCircles - 1 => outerRadius\n+        case _ => outerRadius * cx / (nCircles - 1)\n+      }\n+    }\n+    var ix = 0\n+    val points = for (cx <- 0 until nCircles;\n+                      px <- 0 until groupSizes(cx)) yield {\n+      val theta = 2.0 * math.Pi * px / groupSizes(cx)\n+      val out = (ix, (radii(cx) * math.cos(theta), radii(cx) * math.sin(theta)))\n+      ix += 1\n+      out\n+    }\n+    val rdd = sc.parallelize(points)\n+    val distancesRdd = rdd.cartesian(rdd).flatMap { case ((i0, (x0, y0)), (i1, (x1, y1))) =>\n+      if (i0 < i1) {\n+        Some((i0.toLong, i1.toLong, similarity((x0, y0), (x1, y1))))","path":"examples/src/main/scala/org/apache/spark/examples/mllib/PowerIterationClusteringExample.scala","position":135,"original_position":134,"commit_id":"f7ff43d94d3a3ac02f2aa3c9521990a96e28d252","original_commit_id":"efeec458cb08b39bee4f026a2e5233f3bfc2d0c2","user":{"login":"mengxr","id":829644,"avatar_url":"https://avatars.githubusercontent.com/u/829644?v=3","gravatar_id":"","url":"https://api.github.com/users/mengxr","html_url":"https://github.com/mengxr","followers_url":"https://api.github.com/users/mengxr/followers","following_url":"https://api.github.com/users/mengxr/following{/other_user}","gists_url":"https://api.github.com/users/mengxr/gists{/gist_id}","starred_url":"https://api.github.com/users/mengxr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/mengxr/subscriptions","organizations_url":"https://api.github.com/users/mengxr/orgs","repos_url":"https://api.github.com/users/mengxr/repos","events_url":"https://api.github.com/users/mengxr/events{/privacy}","received_events_url":"https://api.github.com/users/mengxr/received_events","type":"User","site_admin":false},"body":"call `gaussianSimilarity` directly and remove `def similarity` below.","created_at":"2015-02-10T07:02:35Z","updated_at":"2015-02-10T07:02:35Z","html_url":"https://github.com/apache/spark/pull/4495#discussion_r24392639","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/4495","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/24392639"},"html":{"href":"https://github.com/apache/spark/pull/4495#discussion_r24392639"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/4495"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/4495","id":28984254,"html_url":"https://github.com/apache/spark/pull/4495","diff_url":"https://github.com/apache/spark/pull/4495.diff","patch_url":"https://github.com/apache/spark/pull/4495.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/4495","number":4495,"state":"open","locked":false,"title":"[SPARK-5503][MLLIB] Example code for Power Iteration Clustering","user":{"login":"javadba","id":410769,"avatar_url":"https://avatars.githubusercontent.com/u/410769?v=3","gravatar_id":"","url":"https://api.github.com/users/javadba","html_url":"https://github.com/javadba","followers_url":"https://api.github.com/users/javadba/followers","following_url":"https://api.github.com/users/javadba/following{/other_user}","gists_url":"https://api.github.com/users/javadba/gists{/gist_id}","starred_url":"https://api.github.com/users/javadba/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/javadba/subscriptions","organizations_url":"https://api.github.com/users/javadba/orgs","repos_url":"https://api.github.com/users/javadba/repos","events_url":"https://api.github.com/users/javadba/events{/privacy}","received_events_url":"https://api.github.com/users/javadba/received_events","type":"User","site_admin":false},"body":"","created_at":"2015-02-10T06:34:14Z","updated_at":"2015-02-10T07:02:35Z","closed_at":null,"merged_at":null,"merge_commit_sha":"c34e43f6e6e7d04da57208aeffc64ffafd60f3e3","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/4495/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/4495/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments/{number}","comments_url":"https://api.github.com/repos/apache/spark/issues/4495/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/f7ff43d94d3a3ac02f2aa3c9521990a96e28d252","head":{"label":"Huawei-Spark:picexamples","ref":"picexamples","sha":"f7ff43d94d3a3ac02f2aa3c9521990a96e28d252","user":{"login":"Huawei-Spark","id":8421689,"avatar_url":"https://avatars.githubusercontent.com/u/8421689?v=3","gravatar_id":"","url":"https://api.github.com/users/Huawei-Spark","html_url":"https://github.com/Huawei-Spark","followers_url":"https://api.github.com/users/Huawei-Spark/followers","following_url":"https://api.github.com/users/Huawei-Spark/following{/other_user}","gists_url":"https://api.github.com/users/Huawei-Spark/gists{/gist_id}","starred_url":"https://api.github.com/users/Huawei-Spark/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Huawei-Spark/subscriptions","organizations_url":"https://api.github.com/users/Huawei-Spark/orgs","repos_url":"https://api.github.com/users/Huawei-Spark/repos","events_url":"https://api.github.com/users/Huawei-Spark/events{/privacy}","received_events_url":"https://api.github.com/users/Huawei-Spark/received_events","type":"Organization","site_admin":false},"repo":{"id":22938010,"name":"spark","full_name":"Huawei-Spark/spark","owner":{"login":"Huawei-Spark","id":8421689,"avatar_url":"https://avatars.githubusercontent.com/u/8421689?v=3","gravatar_id":"","url":"https://api.github.com/users/Huawei-Spark","html_url":"https://github.com/Huawei-Spark","followers_url":"https://api.github.com/users/Huawei-Spark/followers","following_url":"https://api.github.com/users/Huawei-Spark/following{/other_user}","gists_url":"https://api.github.com/users/Huawei-Spark/gists{/gist_id}","starred_url":"https://api.github.com/users/Huawei-Spark/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/Huawei-Spark/subscriptions","organizations_url":"https://api.github.com/users/Huawei-Spark/orgs","repos_url":"https://api.github.com/users/Huawei-Spark/repos","events_url":"https://api.github.com/users/Huawei-Spark/events{/privacy}","received_events_url":"https://api.github.com/users/Huawei-Spark/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/Huawei-Spark/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/Huawei-Spark/spark","forks_url":"https://api.github.com/repos/Huawei-Spark/spark/forks","keys_url":"https://api.github.com/repos/Huawei-Spark/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/Huawei-Spark/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/Huawei-Spark/spark/teams","hooks_url":"https://api.github.com/repos/Huawei-Spark/spark/hooks","issue_events_url":"https://api.github.com/repos/Huawei-Spark/spark/issues/events{/number}","events_url":"https://api.github.com/repos/Huawei-Spark/spark/events","assignees_url":"https://api.github.com/repos/Huawei-Spark/spark/assignees{/user}","branches_url":"https://api.github.com/repos/Huawei-Spark/spark/branches{/branch}","tags_url":"https://api.github.com/repos/Huawei-Spark/spark/tags","blobs_url":"https://api.github.com/repos/Huawei-Spark/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/Huawei-Spark/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/Huawei-Spark/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/Huawei-Spark/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/Huawei-Spark/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/Huawei-Spark/spark/languages","stargazers_url":"https://api.github.com/repos/Huawei-Spark/spark/stargazers","contributors_url":"https://api.github.com/repos/Huawei-Spark/spark/contributors","subscribers_url":"https://api.github.com/repos/Huawei-Spark/spark/subscribers","subscription_url":"https://api.github.com/repos/Huawei-Spark/spark/subscription","commits_url":"https://api.github.com/repos/Huawei-Spark/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/Huawei-Spark/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/Huawei-Spark/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/Huawei-Spark/spark/issues/comments/{number}","contents_url":"https://api.github.com/repos/Huawei-Spark/spark/contents/{+path}","compare_url":"https://api.github.com/repos/Huawei-Spark/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/Huawei-Spark/spark/merges","archive_url":"https://api.github.com/repos/Huawei-Spark/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/Huawei-Spark/spark/downloads","issues_url":"https://api.github.com/repos/Huawei-Spark/spark/issues{/number}","pulls_url":"https://api.github.com/repos/Huawei-Spark/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/Huawei-Spark/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/Huawei-Spark/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/Huawei-Spark/spark/labels{/name}","releases_url":"https://api.github.com/repos/Huawei-Spark/spark/releases{/id}","created_at":"2014-08-14T02:06:44Z","updated_at":"2015-02-09T23:42:41Z","pushed_at":"2015-02-10T06:45:43Z","git_url":"git://github.com/Huawei-Spark/spark.git","ssh_url":"git@github.com:Huawei-Spark/spark.git","clone_url":"https://github.com/Huawei-Spark/spark.git","svn_url":"https://github.com/Huawei-Spark/spark","homepage":null,"size":94610,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":2,"mirror_url":null,"open_issues_count":1,"forks":2,"open_issues":1,"watchers":0,"default_branch":"hbase"}},"base":{"label":"apache:master","ref":"master","sha":"ef2f55b97f58fa06acb30e9e0172fb66fba383bc","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments/{number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-02-10T06:46:09Z","pushed_at":"2015-02-10T06:47:25Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1231259,"stargazers_count":2830,"watchers_count":2830,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":2481,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":274,"forks":2481,"open_issues":274,"watchers":2830,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/4495"},"html":{"href":"https://github.com/apache/spark/pull/4495"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/4495"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/4495/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/4495/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/4495/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/f7ff43d94d3a3ac02f2aa3c9521990a96e28d252"}}}}