{"action":"created","comment":{"url":"https://api.github.com/repos/apache/spark/pulls/comments/23106795","id":23106795,"diff_hunk":"@@ -0,0 +1,217 @@\n+/*\n+ * Licensed to the Apache Software Foundation (ASF) under one or more\n+ * contributor license agreements.  See the NOTICE file distributed with\n+ * this work for additional information regarding copyright ownership.\n+ * The ASF licenses this file to You under the Apache License, Version 2.0\n+ * (the \"License\"); you may not use this file except in compliance with\n+ * the License.  You may obtain a copy of the License at\n+ *\n+ *    http://www.apache.org/licenses/LICENSE-2.0\n+ *\n+ * Unless required by applicable law or agreed to in writing, software\n+ * distributed under the License is distributed on an \"AS IS\" BASIS,\n+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n+ * See the License for the specific language governing permissions and\n+ * limitations under the License.\n+ */\n+\n+package org.apache.spark.mllib.linalg.distributed\n+\n+import breeze.linalg.{DenseMatrix => BDM}\n+\n+import org.apache.spark._\n+import org.apache.spark.mllib.linalg._\n+import org.apache.spark.mllib.rdd.RDDFunctions._\n+import org.apache.spark.rdd.RDD\n+import org.apache.spark.storage.StorageLevel\n+import org.apache.spark.util.Utils\n+\n+/**\n+ * A grid partitioner, which stores every block in a separate partition.\n+ *\n+ * @param numRowBlocks Number of blocks that form the rows of the matrix.\n+ * @param numColBlocks Number of blocks that form the columns of the matrix.\n+ * @param rowPerBlock Number of rows that make up each block.\n+ * @param colPerBlock Number of columns that make up each block.\n+ */\n+private[mllib] class GridPartitioner(\n+    val numRowBlocks: Int,\n+    val numColBlocks: Int,\n+    val rowPerBlock: Int,\n+    val colPerBlock: Int,\n+    override val numPartitions: Int) extends Partitioner {\n+\n+  /**\n+   * Returns the index of the partition the SubMatrix belongs to.\n+   *\n+   * @param key The key for the SubMatrix. Can be its position in the grid (its column major index)\n+   *            or a tuple of three integers that are the final row index after the multiplication,\n+   *            the index of the block to multiply with, and the final column index after the\n+   *            multiplication.\n+   * @return The index of the partition, which the SubMatrix belongs to.\n+   */\n+  override def getPartition(key: Any): Int = {\n+    key match {\n+      case ind: (Int, Int) =>\n+        Utils.nonNegativeMod(ind._1 + ind._2 * numRowBlocks, numPartitions)\n+      case indices: (Int, Int, Int) =>\n+        Utils.nonNegativeMod(indices._1 + indices._3 * numRowBlocks, numPartitions)\n+      case _ =>\n+        throw new IllegalArgumentException(\"Unrecognized key\")\n+    }\n+  }\n+\n+  /** Checks whether the partitioners have the same characteristics */\n+  override def equals(obj: Any): Boolean = {\n+    obj match {\n+      case r: GridPartitioner =>\n+        (this.numPartitions == r.numPartitions) && (this.rowPerBlock == r.rowPerBlock) &&\n+          (this.colPerBlock == r.colPerBlock)\n+      case _ =>\n+        false\n+    }\n+  }\n+}\n+\n+/**\n+ * Represents a distributed matrix in blocks of local matrices.\n+ *\n+ * @param numRowBlocks Number of blocks that form the rows of this matrix\n+ * @param numColBlocks Number of blocks that form the columns of this matrix\n+ * @param rdd The RDD of SubMatrices (local matrices) that form this matrix\n+ */\n+class BlockMatrix(\n+    val numRowBlocks: Int,\n+    val numColBlocks: Int,\n+    val rdd: RDD[((Int, Int), Matrix)]) extends DistributedMatrix with Logging {\n+\n+  type SubMatrix = ((Int, Int), Matrix) // ((blockRowIndex, blockColIndex), matrix)\n+\n+  /**\n+   * Alternate constructor for BlockMatrix without the input of a partitioner. Will use a Grid\n+   * Partitioner by default.\n+   *\n+   * @param numRowBlocks Number of blocks that form the rows of this matrix\n+   * @param numColBlocks Number of blocks that form the columns of this matrix\n+   * @param rdd The RDD of SubMatrices (local matrices) that form this matrix\n+   * @param rowPerBlock Number of rows that make up each block.\n+   * @param colPerBlock Number of columns that make up each block.\n+   */\n+  def this(\n+      numRowBlocks: Int,\n+      numColBlocks: Int,\n+      rdd: RDD[((Int, Int), Matrix)],\n+      rowPerBlock: Int,\n+      colPerBlock: Int) = {\n+    this(numRowBlocks, numColBlocks, rdd)\n+    val part = new GridPartitioner(numRowBlocks, numColBlocks, rowPerBlock,\n+      colPerBlock, rdd.partitions.length)\n+    setPartitioner(part)\n+  }\n+\n+  private[mllib] var partitioner: GridPartitioner = {\n+    val firstSubMatrix = rdd.first()._2\n+    new GridPartitioner(numRowBlocks, numColBlocks,\n+      firstSubMatrix.numRows, firstSubMatrix.numCols, rdd.partitions.length)\n+  }\n+\n+  /**\n+   * Set the partitioner for the matrix. For internal use only. Users should use `repartition`.\n+   * @param part A partitioner that specifies how SubMatrices are stored in the cluster\n+   */\n+  private def setPartitioner(part: GridPartitioner): Unit = {\n+    partitioner = part\n+  }\n+\n+  private lazy val dims: (Long, Long) = getDim\n+\n+  override def numRows(): Long = dims._1\n+  override def numCols(): Long = dims._2\n+\n+  /** Returns the dimensions of the matrix. */\n+  def getDim: (Long, Long) = {\n+    // picks the sizes of the matrix with the maximum indices\n+    def pickSizeByGreaterIndex(\n+        example: (Int, Int, Int, Int),\n+        base: (Int, Int, Int, Int)): (Int, Int, Int, Int) = {\n+      if (example._1 > base._1 && example._2 > base._2) {\n+        (example._1, example._2, example._3, example._4)\n+      } else if (example._1 > base._1) {\n+        (example._1, base._2, example._3, base._4)\n+      } else if (example._2 > base._2) {\n+        (base._1, example._2, base._3, example._4)\n+      } else {\n+        (base._1, base._2, base._3, base._4)\n+      }\n+    }\n+\n+    val lastRowCol = rdd.treeAggregate((0, 0, 0, 0))(\n+      seqOp = (c, v) => (c, v) match { case (base, ((blockXInd, blockYInd), mat)) =>\n+        pickSizeByGreaterIndex((blockXInd, blockYInd, mat.numRows, mat.numCols), base)\n+      },\n+      combOp = (c1, c2) => (c1, c2) match {\n+        case (res1, res2) =>\n+          pickSizeByGreaterIndex(res1, res2)\n+      })\n+\n+    (lastRowCol._1.toLong * partitioner.rowPerBlock + lastRowCol._3,\n+      lastRowCol._2.toLong * partitioner.colPerBlock + lastRowCol._4)\n+  }\n+\n+  /** Returns the Frobenius Norm of the matrix */\n+  def normFro(): Double = {\n+    math.sqrt(rdd.map { mat => mat._2 match {\n+      case sparse: SparseMatrix =>\n+        sparse.values.map(x => math.pow(x, 2)).sum\n+      case dense: DenseMatrix =>\n+        dense.values.map(x => math.pow(x, 2)).sum\n+    }\n+    }.reduce(_ + _))\n+  }\n+\n+  /** Cache the underlying RDD. */\n+  def cache(): DistributedMatrix = {\n+    rdd.cache()\n+    this\n+  }\n+\n+  /** Set the storage level for the underlying RDD. */\n+  def persist(storageLevel: StorageLevel): DistributedMatrix = {\n+    rdd.persist(storageLevel)\n+    this\n+  }\n+\n+  /** Collect the distributed matrix on the driver as a local matrix. */\n+  def toLocalMatrix(): Matrix = {\n+    val parts = rdd.collect().sortBy(x => (x._1._2, x._1._1))\n+    val nRows = numRows().toInt\n+    val nCols = numCols().toInt\n+    val values = new Array[Double](nRows * nCols)\n+\n+    parts.foreach { part =>","path":"mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/BlockMatrix.scala","position":191,"original_position":191,"commit_id":"ba414d2c6a16de987c9aa456dafa193d191014d5","original_commit_id":"ba414d2c6a16de987c9aa456dafa193d191014d5","user":{"login":"jkbradley","id":5084283,"avatar_url":"https://avatars.githubusercontent.com/u/5084283?v=3","gravatar_id":"","url":"https://api.github.com/users/jkbradley","html_url":"https://github.com/jkbradley","followers_url":"https://api.github.com/users/jkbradley/followers","following_url":"https://api.github.com/users/jkbradley/following{/other_user}","gists_url":"https://api.github.com/users/jkbradley/gists{/gist_id}","starred_url":"https://api.github.com/users/jkbradley/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jkbradley/subscriptions","organizations_url":"https://api.github.com/users/jkbradley/orgs","repos_url":"https://api.github.com/users/jkbradley/repos","events_url":"https://api.github.com/users/jkbradley/events{/privacy}","received_events_url":"https://api.github.com/users/jkbradley/received_events","type":"User","site_admin":false},"body":"Use a match statement so you don't have to use the ._1._1 style of indexing","created_at":"2015-01-16T20:23:10Z","updated_at":"2015-01-16T20:23:10Z","html_url":"https://github.com/apache/spark/pull/3200#discussion_r23106795","pull_request_url":"https://api.github.com/repos/apache/spark/pulls/3200","_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/23106795"},"html":{"href":"https://github.com/apache/spark/pull/3200#discussion_r23106795"},"pull_request":{"href":"https://api.github.com/repos/apache/spark/pulls/3200"}}},"pull_request":{"url":"https://api.github.com/repos/apache/spark/pulls/3200","id":24199657,"html_url":"https://github.com/apache/spark/pull/3200","diff_url":"https://github.com/apache/spark/pull/3200.diff","patch_url":"https://github.com/apache/spark/pull/3200.patch","issue_url":"https://api.github.com/repos/apache/spark/issues/3200","number":3200,"state":"open","locked":false,"title":"[SPARK-3974][MLlib] Distributed Block Matrix Abstractions","user":{"login":"brkyvz","id":5243515,"avatar_url":"https://avatars.githubusercontent.com/u/5243515?v=3","gravatar_id":"","url":"https://api.github.com/users/brkyvz","html_url":"https://github.com/brkyvz","followers_url":"https://api.github.com/users/brkyvz/followers","following_url":"https://api.github.com/users/brkyvz/following{/other_user}","gists_url":"https://api.github.com/users/brkyvz/gists{/gist_id}","starred_url":"https://api.github.com/users/brkyvz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brkyvz/subscriptions","organizations_url":"https://api.github.com/users/brkyvz/orgs","repos_url":"https://api.github.com/users/brkyvz/repos","events_url":"https://api.github.com/users/brkyvz/events{/privacy}","received_events_url":"https://api.github.com/users/brkyvz/received_events","type":"User","site_admin":false},"body":"This pull request includes the abstractions for the distributed BlockMatrix representation. \r\n`BlockMatrix` will allow users to store very large matrices in small blocks of local matrices. Specific partitioners, such as `RowBasedPartitioner` and `ColumnBasedPartitioner`, are implemented in order to optimize addition and multiplication operations that will be added in a following PR.\r\n\r\nThis work is based on the ml-matrix repo developed at the AMPLab at UC Berkeley, CA.\r\nhttps://github.com/amplab/ml-matrix\r\n\r\nAdditional thanks to @rezazadeh, @shivaram, and @mengxr for guidance on the design.","created_at":"2014-11-11T05:38:57Z","updated_at":"2015-01-16T20:23:10Z","closed_at":null,"merged_at":null,"merge_commit_sha":"4a05923c19c9a9e1416e379d273b8ec855ef7179","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/apache/spark/pulls/3200/commits","review_comments_url":"https://api.github.com/repos/apache/spark/pulls/3200/comments","review_comment_url":"https://api.github.com/repos/apache/spark/pulls/comments/{number}","comments_url":"https://api.github.com/repos/apache/spark/issues/3200/comments","statuses_url":"https://api.github.com/repos/apache/spark/statuses/ba414d2c6a16de987c9aa456dafa193d191014d5","head":{"label":"brkyvz:SPARK-3974","ref":"SPARK-3974","sha":"ba414d2c6a16de987c9aa456dafa193d191014d5","user":{"login":"brkyvz","id":5243515,"avatar_url":"https://avatars.githubusercontent.com/u/5243515?v=3","gravatar_id":"","url":"https://api.github.com/users/brkyvz","html_url":"https://github.com/brkyvz","followers_url":"https://api.github.com/users/brkyvz/followers","following_url":"https://api.github.com/users/brkyvz/following{/other_user}","gists_url":"https://api.github.com/users/brkyvz/gists{/gist_id}","starred_url":"https://api.github.com/users/brkyvz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brkyvz/subscriptions","organizations_url":"https://api.github.com/users/brkyvz/orgs","repos_url":"https://api.github.com/users/brkyvz/repos","events_url":"https://api.github.com/users/brkyvz/events{/privacy}","received_events_url":"https://api.github.com/users/brkyvz/received_events","type":"User","site_admin":false},"repo":{"id":21919521,"name":"spark","full_name":"brkyvz/spark","owner":{"login":"brkyvz","id":5243515,"avatar_url":"https://avatars.githubusercontent.com/u/5243515?v=3","gravatar_id":"","url":"https://api.github.com/users/brkyvz","html_url":"https://github.com/brkyvz","followers_url":"https://api.github.com/users/brkyvz/followers","following_url":"https://api.github.com/users/brkyvz/following{/other_user}","gists_url":"https://api.github.com/users/brkyvz/gists{/gist_id}","starred_url":"https://api.github.com/users/brkyvz/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/brkyvz/subscriptions","organizations_url":"https://api.github.com/users/brkyvz/orgs","repos_url":"https://api.github.com/users/brkyvz/repos","events_url":"https://api.github.com/users/brkyvz/events{/privacy}","received_events_url":"https://api.github.com/users/brkyvz/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/brkyvz/spark","description":"Mirror of Apache Spark","fork":true,"url":"https://api.github.com/repos/brkyvz/spark","forks_url":"https://api.github.com/repos/brkyvz/spark/forks","keys_url":"https://api.github.com/repos/brkyvz/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/brkyvz/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/brkyvz/spark/teams","hooks_url":"https://api.github.com/repos/brkyvz/spark/hooks","issue_events_url":"https://api.github.com/repos/brkyvz/spark/issues/events{/number}","events_url":"https://api.github.com/repos/brkyvz/spark/events","assignees_url":"https://api.github.com/repos/brkyvz/spark/assignees{/user}","branches_url":"https://api.github.com/repos/brkyvz/spark/branches{/branch}","tags_url":"https://api.github.com/repos/brkyvz/spark/tags","blobs_url":"https://api.github.com/repos/brkyvz/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/brkyvz/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/brkyvz/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/brkyvz/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/brkyvz/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/brkyvz/spark/languages","stargazers_url":"https://api.github.com/repos/brkyvz/spark/stargazers","contributors_url":"https://api.github.com/repos/brkyvz/spark/contributors","subscribers_url":"https://api.github.com/repos/brkyvz/spark/subscribers","subscription_url":"https://api.github.com/repos/brkyvz/spark/subscription","commits_url":"https://api.github.com/repos/brkyvz/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/brkyvz/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/brkyvz/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/brkyvz/spark/issues/comments/{number}","contents_url":"https://api.github.com/repos/brkyvz/spark/contents/{+path}","compare_url":"https://api.github.com/repos/brkyvz/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/brkyvz/spark/merges","archive_url":"https://api.github.com/repos/brkyvz/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/brkyvz/spark/downloads","issues_url":"https://api.github.com/repos/brkyvz/spark/issues{/number}","pulls_url":"https://api.github.com/repos/brkyvz/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/brkyvz/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/brkyvz/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/brkyvz/spark/labels{/name}","releases_url":"https://api.github.com/repos/brkyvz/spark/releases{/id}","created_at":"2014-07-16T21:51:06Z","updated_at":"2014-08-12T23:17:42Z","pushed_at":"2015-01-15T15:34:06Z","git_url":"git://github.com/brkyvz/spark.git","ssh_url":"git@github.com:brkyvz/spark.git","clone_url":"https://github.com/brkyvz/spark.git","svn_url":"https://github.com/brkyvz/spark","homepage":null,"size":85922,"stargazers_count":0,"watchers_count":0,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"apache:master","ref":"master","sha":"e421072da0ea87e7056cc3f2130ddaafc731530f","user":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"repo":{"id":17165658,"name":"spark","full_name":"apache/spark","owner":{"login":"apache","id":47359,"avatar_url":"https://avatars.githubusercontent.com/u/47359?v=3","gravatar_id":"","url":"https://api.github.com/users/apache","html_url":"https://github.com/apache","followers_url":"https://api.github.com/users/apache/followers","following_url":"https://api.github.com/users/apache/following{/other_user}","gists_url":"https://api.github.com/users/apache/gists{/gist_id}","starred_url":"https://api.github.com/users/apache/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/apache/subscriptions","organizations_url":"https://api.github.com/users/apache/orgs","repos_url":"https://api.github.com/users/apache/repos","events_url":"https://api.github.com/users/apache/events{/privacy}","received_events_url":"https://api.github.com/users/apache/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/apache/spark","description":"Mirror of Apache Spark","fork":false,"url":"https://api.github.com/repos/apache/spark","forks_url":"https://api.github.com/repos/apache/spark/forks","keys_url":"https://api.github.com/repos/apache/spark/keys{/key_id}","collaborators_url":"https://api.github.com/repos/apache/spark/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/apache/spark/teams","hooks_url":"https://api.github.com/repos/apache/spark/hooks","issue_events_url":"https://api.github.com/repos/apache/spark/issues/events{/number}","events_url":"https://api.github.com/repos/apache/spark/events","assignees_url":"https://api.github.com/repos/apache/spark/assignees{/user}","branches_url":"https://api.github.com/repos/apache/spark/branches{/branch}","tags_url":"https://api.github.com/repos/apache/spark/tags","blobs_url":"https://api.github.com/repos/apache/spark/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/apache/spark/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/apache/spark/git/refs{/sha}","trees_url":"https://api.github.com/repos/apache/spark/git/trees{/sha}","statuses_url":"https://api.github.com/repos/apache/spark/statuses/{sha}","languages_url":"https://api.github.com/repos/apache/spark/languages","stargazers_url":"https://api.github.com/repos/apache/spark/stargazers","contributors_url":"https://api.github.com/repos/apache/spark/contributors","subscribers_url":"https://api.github.com/repos/apache/spark/subscribers","subscription_url":"https://api.github.com/repos/apache/spark/subscription","commits_url":"https://api.github.com/repos/apache/spark/commits{/sha}","git_commits_url":"https://api.github.com/repos/apache/spark/git/commits{/sha}","comments_url":"https://api.github.com/repos/apache/spark/comments{/number}","issue_comment_url":"https://api.github.com/repos/apache/spark/issues/comments/{number}","contents_url":"https://api.github.com/repos/apache/spark/contents/{+path}","compare_url":"https://api.github.com/repos/apache/spark/compare/{base}...{head}","merges_url":"https://api.github.com/repos/apache/spark/merges","archive_url":"https://api.github.com/repos/apache/spark/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/apache/spark/downloads","issues_url":"https://api.github.com/repos/apache/spark/issues{/number}","pulls_url":"https://api.github.com/repos/apache/spark/pulls{/number}","milestones_url":"https://api.github.com/repos/apache/spark/milestones{/number}","notifications_url":"https://api.github.com/repos/apache/spark/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/apache/spark/labels{/name}","releases_url":"https://api.github.com/repos/apache/spark/releases{/id}","created_at":"2014-02-25T08:00:08Z","updated_at":"2015-01-16T20:19:23Z","pushed_at":"2015-01-16T20:19:23Z","git_url":"git://github.com/apache/spark.git","ssh_url":"git@github.com:apache/spark.git","clone_url":"https://github.com/apache/spark.git","svn_url":"https://github.com/apache/spark","homepage":null,"size":1146863,"stargazers_count":2575,"watchers_count":2575,"language":"Scala","has_issues":false,"has_downloads":true,"has_wiki":false,"has_pages":false,"forks_count":2277,"mirror_url":"git://git.apache.org/spark.git","open_issues_count":311,"forks":2277,"open_issues":311,"watchers":2575,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/apache/spark/pulls/3200"},"html":{"href":"https://github.com/apache/spark/pull/3200"},"issue":{"href":"https://api.github.com/repos/apache/spark/issues/3200"},"comments":{"href":"https://api.github.com/repos/apache/spark/issues/3200/comments"},"review_comments":{"href":"https://api.github.com/repos/apache/spark/pulls/3200/comments"},"review_comment":{"href":"https://api.github.com/repos/apache/spark/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/apache/spark/pulls/3200/commits"},"statuses":{"href":"https://api.github.com/repos/apache/spark/statuses/ba414d2c6a16de987c9aa456dafa193d191014d5"}}}}